211 				    struct io_pgtable_cfg *cfg)  in __arm_lpae_alloc_pages()  argument
213 	struct device *dev = cfg->iommu_dev;  in __arm_lpae_alloc_pages()
244 				  struct io_pgtable_cfg *cfg)  in __arm_lpae_free_pages()  argument
247 		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),  in __arm_lpae_free_pages()
253 			       struct io_pgtable_cfg *cfg)  in __arm_lpae_set_pte()  argument
258 		dma_sync_single_for_device(cfg->iommu_dev,  in __arm_lpae_set_pte()
273 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_init_pte()  local
292 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_init_pte()
303 	__arm_lpae_set_pte(ptep, pte, cfg);  in arm_lpae_init_pte()
313 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_map()  local
319 	if (size == block_size && (size & cfg->pgsize_bitmap))  in __arm_lpae_map()
330 					       GFP_ATOMIC, cfg);  in __arm_lpae_map()
335 		if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in __arm_lpae_map()
337 		__arm_lpae_set_pte(ptep, pte, cfg);  in __arm_lpae_map()
430 	__arm_lpae_free_pages(start, table_size, &data->iop.cfg);  in __arm_lpae_free_pgtable()
449 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_split_blk_unmap()  local
475 	__arm_lpae_set_pte(ptep, table, cfg);  in arm_lpae_split_blk_unmap()
477 	cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);  in arm_lpae_split_blk_unmap()
486 	const struct iommu_gather_ops *tlb = data->iop.cfg.tlb;  in __arm_lpae_unmap()
499 		__arm_lpae_set_pte(ptep, 0, &data->iop.cfg);  in __arm_lpae_unmap()
538 		iop->cfg.tlb->tlb_sync(iop->cookie);  in arm_lpae_unmap()
578 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)  in arm_lpae_restrict_pgsizes()  argument
589 	if (cfg->pgsize_bitmap & PAGE_SIZE)  in arm_lpae_restrict_pgsizes()
591 	else if (cfg->pgsize_bitmap & ~PAGE_MASK)  in arm_lpae_restrict_pgsizes()
592 		granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);  in arm_lpae_restrict_pgsizes()
593 	else if (cfg->pgsize_bitmap & PAGE_MASK)  in arm_lpae_restrict_pgsizes()
594 		granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);  in arm_lpae_restrict_pgsizes()
600 		cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_lpae_restrict_pgsizes()
603 		cfg->pgsize_bitmap &= (SZ_16K | SZ_32M);  in arm_lpae_restrict_pgsizes()
606 		cfg->pgsize_bitmap &= (SZ_64K | SZ_512M);  in arm_lpae_restrict_pgsizes()
609 		cfg->pgsize_bitmap = 0;  in arm_lpae_restrict_pgsizes()
614 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)  in arm_lpae_alloc_pgtable()  argument
619 	arm_lpae_restrict_pgsizes(cfg);  in arm_lpae_alloc_pgtable()
621 	if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))  in arm_lpae_alloc_pgtable()
624 	if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
627 	if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
630 	if (!selftest_running && cfg->iommu_dev->dma_pfn_offset) {  in arm_lpae_alloc_pgtable()
631 		dev_err(cfg->iommu_dev, "Cannot accommodate DMA offset for IOMMU page tables\n");  in arm_lpae_alloc_pgtable()
639 	data->pg_shift = __ffs(cfg->pgsize_bitmap);  in arm_lpae_alloc_pgtable()
642 	va_bits = cfg->ias - data->pg_shift;  in arm_lpae_alloc_pgtable()
659 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s1()  argument
662 	struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s1()
684 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s1()
707 	reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;  in arm_64_lpae_alloc_pgtable_s1()
711 	cfg->arm_lpae_s1_cfg.tcr = reg;  in arm_64_lpae_alloc_pgtable_s1()
721 	cfg->arm_lpae_s1_cfg.mair[0] = reg;  in arm_64_lpae_alloc_pgtable_s1()
722 	cfg->arm_lpae_s1_cfg.mair[1] = 0;  in arm_64_lpae_alloc_pgtable_s1()
725 	data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);  in arm_64_lpae_alloc_pgtable_s1()
733 	cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s1()
734 	cfg->arm_lpae_s1_cfg.ttbr[1] = 0;  in arm_64_lpae_alloc_pgtable_s1()
743 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s2()  argument
746 	struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s2()
786 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s2()
809 	reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;  in arm_64_lpae_alloc_pgtable_s2()
811 	cfg->arm_lpae_s2_cfg.vtcr = reg;  in arm_64_lpae_alloc_pgtable_s2()
814 	data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);  in arm_64_lpae_alloc_pgtable_s2()
822 	cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s2()
831 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s1()  argument
835 	if (cfg->ias > 32 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s1()
838 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s1()
839 	iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s1()
841 		cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE;  in arm_32_lpae_alloc_pgtable_s1()
842 		cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff;  in arm_32_lpae_alloc_pgtable_s1()
849 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s2()  argument
853 	if (cfg->ias > 40 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s2()
856 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s2()
857 	iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s2()
859 		cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff;  in arm_32_lpae_alloc_pgtable_s2()
914 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()  local
917 		cfg->pgsize_bitmap, cfg->ias);  in arm_lpae_dump_ops()
930 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)  in arm_lpae_run_tests()  argument
945 		cfg_cookie = cfg;  in arm_lpae_run_tests()
946 		ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);  in arm_lpae_run_tests()
969 		j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);  in arm_lpae_run_tests()
989 			j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);  in arm_lpae_run_tests()
993 		size = 1UL << __ffs(cfg->pgsize_bitmap);  in arm_lpae_run_tests()
1006 		j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);  in arm_lpae_run_tests()
1025 			j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);  in arm_lpae_run_tests()
1048 	struct io_pgtable_cfg cfg = {  in arm_lpae_do_selftests()  local
1055 			cfg.pgsize_bitmap = pgsize[i];  in arm_lpae_do_selftests()
1056 			cfg.ias = ias[j];  in arm_lpae_do_selftests()
1059 			if (arm_lpae_run_tests(&cfg))  in arm_lpae_do_selftests()