519 	u64 address;  in iommu_print_event()  local
526 	address = (u64)(((u64)event[3]) << 32) | event[2];  in iommu_print_event()
545 		       address, flags);  in iommu_print_event()
552 		       domid, address, flags);  in iommu_print_event()
558 		       address, flags);  in iommu_print_event()
564 		       domid, address, flags);  in iommu_print_event()
567 		printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);  in iommu_print_event()
568 		dump_command(address);  in iommu_print_event()
572 		       "flags=0x%04x]\n", address, flags);  in iommu_print_event()
578 		       address);  in iommu_print_event()
584 		       address, flags);  in iommu_print_event()
619 	fault.address   = raw[1];  in iommu_handle_ppr_entry()
761 static void build_completion_wait(struct iommu_cmd *cmd, u64 address)  in build_completion_wait()  argument
763 	WARN_ON(address & 0x7ULL);  in build_completion_wait()
766 	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;  in build_completion_wait()
767 	cmd->data[1] = upper_32_bits(__pa(address));  in build_completion_wait()
779 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,  in build_inv_iommu_pages()  argument
785 	pages = iommu_num_pages(address, size, PAGE_SIZE);  in build_inv_iommu_pages()
793 		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;  in build_inv_iommu_pages()
797 	address &= PAGE_MASK;  in build_inv_iommu_pages()
801 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iommu_pages()
802 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iommu_pages()
811 				  u64 address, size_t size)  in build_inv_iotlb_pages()  argument
816 	pages = iommu_num_pages(address, size, PAGE_SIZE);  in build_inv_iotlb_pages()
824 		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;  in build_inv_iotlb_pages()
828 	address &= PAGE_MASK;  in build_inv_iotlb_pages()
834 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iotlb_pages()
835 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iotlb_pages()
842 				  u64 address, bool size)  in build_inv_iommu_pasid()  argument
846 	address &= ~(0xfffULL);  in build_inv_iommu_pasid()
850 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iommu_pasid()
851 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iommu_pasid()
860 				  int qdep, u64 address, bool size)  in build_inv_iotlb_pasid()  argument
864 	address &= ~(0xfffULL);  in build_inv_iotlb_pasid()
871 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iotlb_pasid()
873 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iotlb_pasid()
1061 			      u64 address, size_t size)  in device_flush_iotlb()  argument
1070 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);  in device_flush_iotlb()
1105 				 u64 address, size_t size, int pde)  in __domain_flush_pages()  argument
1111 	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);  in __domain_flush_pages()
1129 		ret |= device_flush_iotlb(dev_data, address, size);  in __domain_flush_pages()
1136 			       u64 address, size_t size)  in domain_flush_pages()  argument
1138 	__domain_flush_pages(domain, address, size, 0);  in domain_flush_pages()
1216 		      unsigned long address,  in alloc_pte()  argument
1226 	while (address > PM_LEVEL_SIZE(domain->mode))  in alloc_pte()
1230 	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1231 	address = PAGE_SIZE_ALIGN(address, page_size);  in alloc_pte()
1253 		pte = &pte[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1264 		      unsigned long address,  in fetch_pte()  argument
1270 	if (address > PM_LEVEL_SIZE(domain->mode))  in fetch_pte()
1274 	pte	   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1296 		pte	   = &pte[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1465 		unsigned long address = dma_dom->aperture_size;  in alloc_new_range()  local
1470 			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,  in alloc_new_range()
1477 			address += APERTURE_RANGE_SIZE / 64;  in alloc_new_range()
1553 	unsigned long address = -1;  in dma_ops_area_alloc()  local
1572 		address = iommu_area_alloc(dom->aperture[i]->bitmap,  in dma_ops_area_alloc()
1575 		if (address != -1) {  in dma_ops_area_alloc()
1576 			address = dom->aperture[i]->offset +  in dma_ops_area_alloc()
1577 				  (address << PAGE_SHIFT);  in dma_ops_area_alloc()
1578 			dom->next_address = address + (pages << PAGE_SHIFT);  in dma_ops_area_alloc()
1585 	return address;  in dma_ops_area_alloc()
1594 	unsigned long address;  in dma_ops_alloc_addresses()  local
1601 	address = dma_ops_area_alloc(dev, dom, pages, align_mask,  in dma_ops_alloc_addresses()
1604 	if (address == -1) {  in dma_ops_alloc_addresses()
1606 		address = dma_ops_area_alloc(dev, dom, pages, align_mask,  in dma_ops_alloc_addresses()
1611 	if (unlikely(address == -1))  in dma_ops_alloc_addresses()
1612 		address = DMA_ERROR_CODE;  in dma_ops_alloc_addresses()
1614 	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);  in dma_ops_alloc_addresses()
1616 	return address;  in dma_ops_alloc_addresses()
1625 				   unsigned long address,  in dma_ops_free_addresses()  argument
1628 	unsigned i = address >> APERTURE_RANGE_SHIFT;  in dma_ops_free_addresses()
1638 	if (address >= dom->next_address)  in dma_ops_free_addresses()
1641 	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;  in dma_ops_free_addresses()
1643 	bitmap_clear(range->bitmap, address, pages);  in dma_ops_free_addresses()
2346 			    unsigned long address)  in dma_ops_get_pte()  argument
2351 	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];  in dma_ops_get_pte()
2355 	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_get_pte()
2357 		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,  in dma_ops_get_pte()
2359 		aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;  in dma_ops_get_pte()
2361 		pte += PM_LEVEL_INDEX(0, address);  in dma_ops_get_pte()
2373 				     unsigned long address,  in dma_ops_domain_map()  argument
2379 	WARN_ON(address > dom->aperture_size);  in dma_ops_domain_map()
2383 	pte  = dma_ops_get_pte(dom, address);  in dma_ops_domain_map()
2400 	return (dma_addr_t)address;  in dma_ops_domain_map()
2407 				 unsigned long address)  in dma_ops_domain_unmap()  argument
2412 	if (address >= dom->aperture_size)  in dma_ops_domain_unmap()
2415 	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];  in dma_ops_domain_unmap()
2419 	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_domain_unmap()
2423 	pte += PM_LEVEL_INDEX(0, address);  in dma_ops_domain_unmap()
2445 	dma_addr_t address, start, ret;  in __map_single()  local
2462 	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,  in __map_single()
2464 	if (unlikely(address == DMA_ERROR_CODE)) {  in __map_single()
2482 	start = address;  in __map_single()
2491 	address += offset;  in __map_single()
2499 		domain_flush_pages(&dma_dom->domain, address, size);  in __map_single()
2502 	return address;  in __map_single()
2511 	dma_ops_free_addresses(dma_dom, address, pages);  in __map_single()
3268 			 u64 address, bool size)  in __flush_pasid()  argument
3277 	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);  in __flush_pasid()
3311 				      qdep, address, size);  in __flush_pasid()
3329 				  u64 address)  in __amd_iommu_flush_page()  argument
3333 	return __flush_pasid(domain, pasid, address, false);  in __amd_iommu_flush_page()
3337 			 u64 address)  in amd_iommu_flush_page()  argument
3344 	ret = __amd_iommu_flush_page(domain, pasid, address);  in amd_iommu_flush_page()