588 	u64 address;  in iommu_print_event()  local
595 	address = (u64)(((u64)event[3]) << 32) | event[2];  in iommu_print_event()
614 		       address, flags);  in iommu_print_event()
621 		       domid, address, flags);  in iommu_print_event()
627 		       address, flags);  in iommu_print_event()
633 		       domid, address, flags);  in iommu_print_event()
636 		printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);  in iommu_print_event()
637 		dump_command(address);  in iommu_print_event()
641 		       "flags=0x%04x]\n", address, flags);  in iommu_print_event()
647 		       address);  in iommu_print_event()
653 		       address, flags);  in iommu_print_event()
688 	fault.address   = raw[1];  in iommu_handle_ppr_entry()
830 static void build_completion_wait(struct iommu_cmd *cmd, u64 address)  in build_completion_wait()  argument
832 	WARN_ON(address & 0x7ULL);  in build_completion_wait()
835 	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;  in build_completion_wait()
836 	cmd->data[1] = upper_32_bits(__pa(address));  in build_completion_wait()
848 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,  in build_inv_iommu_pages()  argument
854 	pages = iommu_num_pages(address, size, PAGE_SIZE);  in build_inv_iommu_pages()
862 		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;  in build_inv_iommu_pages()
866 	address &= PAGE_MASK;  in build_inv_iommu_pages()
870 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iommu_pages()
871 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iommu_pages()
880 				  u64 address, size_t size)  in build_inv_iotlb_pages()  argument
885 	pages = iommu_num_pages(address, size, PAGE_SIZE);  in build_inv_iotlb_pages()
893 		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;  in build_inv_iotlb_pages()
897 	address &= PAGE_MASK;  in build_inv_iotlb_pages()
903 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iotlb_pages()
904 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iotlb_pages()
911 				  u64 address, bool size)  in build_inv_iommu_pasid()  argument
915 	address &= ~(0xfffULL);  in build_inv_iommu_pasid()
919 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iommu_pasid()
920 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iommu_pasid()
929 				  int qdep, u64 address, bool size)  in build_inv_iotlb_pasid()  argument
933 	address &= ~(0xfffULL);  in build_inv_iotlb_pasid()
940 	cmd->data[2]  = lower_32_bits(address);  in build_inv_iotlb_pasid()
942 	cmd->data[3]  = upper_32_bits(address);  in build_inv_iotlb_pasid()
1132 			      u64 address, size_t size)  in device_flush_iotlb()  argument
1141 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);  in device_flush_iotlb()
1172 				 u64 address, size_t size, int pde)  in __domain_flush_pages()  argument
1178 	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);  in __domain_flush_pages()
1196 		ret |= device_flush_iotlb(dev_data, address, size);  in __domain_flush_pages()
1203 			       u64 address, size_t size)  in domain_flush_pages()  argument
1205 	__domain_flush_pages(domain, address, size, 0);  in domain_flush_pages()
1283 		      unsigned long address,  in alloc_pte()  argument
1293 	while (address > PM_LEVEL_SIZE(domain->mode))  in alloc_pte()
1297 	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1298 	address = PAGE_SIZE_ALIGN(address, page_size);  in alloc_pte()
1320 		pte = &pte[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1331 		      unsigned long address,  in fetch_pte()  argument
1337 	if (address > PM_LEVEL_SIZE(domain->mode))  in fetch_pte()
1341 	pte	   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1363 		pte	   = &pte[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1620 		unsigned long address = dma_dom->aperture_size;  in alloc_new_range()  local
1625 			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,  in alloc_new_range()
1632 			address += APERTURE_RANGE_SIZE / 64;  in alloc_new_range()
1708 	unsigned long address = -1;  in dma_ops_area_alloc()  local
1725 		address = iommu_area_alloc(dom->aperture[i]->bitmap,  in dma_ops_area_alloc()
1728 		if (address != -1) {  in dma_ops_area_alloc()
1729 			address = dom->aperture[i]->offset +  in dma_ops_area_alloc()
1730 				  (address << PAGE_SHIFT);  in dma_ops_area_alloc()
1731 			dom->next_address = address + (pages << PAGE_SHIFT);  in dma_ops_area_alloc()
1738 	return address;  in dma_ops_area_alloc()
1747 	unsigned long address;  in dma_ops_alloc_addresses()  local
1754 	address = dma_ops_area_alloc(dev, dom, pages, align_mask,  in dma_ops_alloc_addresses()
1757 	if (address == -1) {  in dma_ops_alloc_addresses()
1759 		address = dma_ops_area_alloc(dev, dom, pages, align_mask,  in dma_ops_alloc_addresses()
1764 	if (unlikely(address == -1))  in dma_ops_alloc_addresses()
1765 		address = DMA_ERROR_CODE;  in dma_ops_alloc_addresses()
1767 	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);  in dma_ops_alloc_addresses()
1769 	return address;  in dma_ops_alloc_addresses()
1778 				   unsigned long address,  in dma_ops_free_addresses()  argument
1781 	unsigned i = address >> APERTURE_RANGE_SHIFT;  in dma_ops_free_addresses()
1791 	if (address >= dom->next_address)  in dma_ops_free_addresses()
1794 	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;  in dma_ops_free_addresses()
1796 	bitmap_clear(range->bitmap, address, pages);  in dma_ops_free_addresses()
2553 			    unsigned long address)  in dma_ops_get_pte()  argument
2558 	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];  in dma_ops_get_pte()
2562 	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_get_pte()
2564 		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,  in dma_ops_get_pte()
2566 		aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;  in dma_ops_get_pte()
2568 		pte += PM_LEVEL_INDEX(0, address);  in dma_ops_get_pte()
2580 				     unsigned long address,  in dma_ops_domain_map()  argument
2586 	WARN_ON(address > dom->aperture_size);  in dma_ops_domain_map()
2590 	pte  = dma_ops_get_pte(dom, address);  in dma_ops_domain_map()
2607 	return (dma_addr_t)address;  in dma_ops_domain_map()
2614 				 unsigned long address)  in dma_ops_domain_unmap()  argument
2619 	if (address >= dom->aperture_size)  in dma_ops_domain_unmap()
2622 	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];  in dma_ops_domain_unmap()
2626 	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_domain_unmap()
2630 	pte += PM_LEVEL_INDEX(0, address);  in dma_ops_domain_unmap()
2652 	dma_addr_t address, start, ret;  in __map_single()  local
2669 	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,  in __map_single()
2671 	if (unlikely(address == DMA_ERROR_CODE)) {  in __map_single()
2689 	start = address;  in __map_single()
2698 	address += offset;  in __map_single()
2706 		domain_flush_pages(&dma_dom->domain, address, size);  in __map_single()
2709 	return address;  in __map_single()
2718 	dma_ops_free_addresses(dma_dom, address, pages);  in __map_single()
3554 			 u64 address, bool size)  in __flush_pasid()  argument
3563 	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);  in __flush_pasid()
3592 				      qdep, address, size);  in __flush_pasid()
3610 				  u64 address)  in __amd_iommu_flush_page()  argument
3614 	return __flush_pasid(domain, pasid, address, false);  in __amd_iommu_flush_page()
3618 			 u64 address)  in amd_iommu_flush_page()  argument
3625 	ret = __amd_iommu_flush_page(domain, pasid, address);  in amd_iommu_flush_page()