264 			    unsigned long iova, size_t size, int lvl,
268 			     unsigned long iova, phys_addr_t paddr,  in arm_lpae_init_pte()  argument
287 		tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
288 		if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz))  in arm_lpae_init_pte()
307 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,  in __arm_lpae_map()  argument
316 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_map()
320 		return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);  in __arm_lpae_map()
343 	return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep);  in __arm_lpae_map()
379 static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_map()  argument
392 	ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);  in arm_lpae_map()
442 				    unsigned long iova, size_t size,  in arm_lpae_split_blk_unmap()  argument
451 	blk_start = iova & ~(blk_size - 1);  in arm_lpae_split_blk_unmap()
459 		if (blk_start == iova)  in arm_lpae_split_blk_unmap()
476 	iova &= ~(blk_size - 1);  in arm_lpae_split_blk_unmap()
477 	cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);  in arm_lpae_split_blk_unmap()
482 			    unsigned long iova, size_t size, int lvl,  in __arm_lpae_unmap()  argument
490 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_unmap()
503 			tlb->tlb_add_flush(iova, size, false, cookie);  in __arm_lpae_unmap()
508 			tlb->tlb_add_flush(iova, size, true, cookie);  in __arm_lpae_unmap()
517 		return arm_lpae_split_blk_unmap(data, iova, size,  in __arm_lpae_unmap()
524 	return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);  in __arm_lpae_unmap()
527 static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_unmap()  argument
536 	unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep);  in arm_lpae_unmap()
544 					 unsigned long iova)  in arm_lpae_iova_to_phys()  argument
556 		pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data));  in arm_lpae_iova_to_phys()
574 	iova &= ((1 << data->pg_shift) - 1);  in arm_lpae_iova_to_phys()
575 	return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;  in arm_lpae_iova_to_phys()
893 static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf,  in dummy_tlb_add_flush()  argument
938 	unsigned long iova;  in arm_lpae_run_tests()  local
968 		iova = 0;  in arm_lpae_run_tests()
973 			if (ops->map(ops, iova, iova, size, IOMMU_READ |  in arm_lpae_run_tests()
980 			if (!ops->map(ops, iova, iova + size, size,  in arm_lpae_run_tests()
984 			if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_lpae_run_tests()
987 			iova += SZ_1G;  in arm_lpae_run_tests()
1005 		iova = 0;  in arm_lpae_run_tests()
1010 			if (ops->unmap(ops, iova, size) != size)  in arm_lpae_run_tests()
1013 			if (ops->iova_to_phys(ops, iova + 42))  in arm_lpae_run_tests()
1017 			if (ops->map(ops, iova, iova, size, IOMMU_WRITE))  in arm_lpae_run_tests()
1020 			if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_lpae_run_tests()
1023 			iova += SZ_1G;  in arm_lpae_run_tests()