231 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_init_pte()
243 	data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), data->iop.cookie);  in arm_lpae_init_pte()
259 	if (size == block_size && (size & data->iop.cfg.pgsize_bitmap))  in __arm_lpae_map()
274 		data->iop.cfg.tlb->flush_pgtable(cptep, 1UL << data->pg_shift,  in __arm_lpae_map()
277 		if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in __arm_lpae_map()
280 		data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);  in __arm_lpae_map()
386 	const struct iommu_gather_ops *tlb = data->iop.cfg.tlb;  in arm_lpae_split_blk_unmap()
424 	const struct iommu_gather_ops *tlb = data->iop.cfg.tlb;  in __arm_lpae_unmap()
477 		iop->cfg.tlb->tlb_sync(iop->cookie);  in arm_lpae_unmap()
517 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)  in arm_lpae_restrict_pgsizes()  argument
528 	if (cfg->pgsize_bitmap & PAGE_SIZE)  in arm_lpae_restrict_pgsizes()
530 	else if (cfg->pgsize_bitmap & ~PAGE_MASK)  in arm_lpae_restrict_pgsizes()
531 		granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);  in arm_lpae_restrict_pgsizes()
532 	else if (cfg->pgsize_bitmap & PAGE_MASK)  in arm_lpae_restrict_pgsizes()
533 		granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);  in arm_lpae_restrict_pgsizes()
539 		cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_lpae_restrict_pgsizes()
542 		cfg->pgsize_bitmap &= (SZ_16K | SZ_32M);  in arm_lpae_restrict_pgsizes()
545 		cfg->pgsize_bitmap &= (SZ_64K | SZ_512M);  in arm_lpae_restrict_pgsizes()
548 		cfg->pgsize_bitmap = 0;  in arm_lpae_restrict_pgsizes()
553 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)  in arm_lpae_alloc_pgtable()  argument
558 	arm_lpae_restrict_pgsizes(cfg);  in arm_lpae_alloc_pgtable()
560 	if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))  in arm_lpae_alloc_pgtable()
563 	if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
566 	if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
573 	data->pg_shift = __ffs(cfg->pgsize_bitmap);  in arm_lpae_alloc_pgtable()
576 	va_bits = cfg->ias - data->pg_shift;  in arm_lpae_alloc_pgtable()
593 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s1()  argument
596 	struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s1()
618 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s1()
641 	reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;  in arm_64_lpae_alloc_pgtable_s1()
645 	cfg->arm_lpae_s1_cfg.tcr = reg;  in arm_64_lpae_alloc_pgtable_s1()
655 	cfg->arm_lpae_s1_cfg.mair[0] = reg;  in arm_64_lpae_alloc_pgtable_s1()
656 	cfg->arm_lpae_s1_cfg.mair[1] = 0;  in arm_64_lpae_alloc_pgtable_s1()
663 	cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);  in arm_64_lpae_alloc_pgtable_s1()
666 	cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s1()
667 	cfg->arm_lpae_s1_cfg.ttbr[1] = 0;  in arm_64_lpae_alloc_pgtable_s1()
676 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s2()  argument
679 	struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s2()
719 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s2()
742 	reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;  in arm_64_lpae_alloc_pgtable_s2()
744 	cfg->arm_lpae_s2_cfg.vtcr = reg;  in arm_64_lpae_alloc_pgtable_s2()
751 	cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);  in arm_64_lpae_alloc_pgtable_s2()
754 	cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s2()
763 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s1()  argument
767 	if (cfg->ias > 32 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s1()
770 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s1()
771 	iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s1()
773 		cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE;  in arm_32_lpae_alloc_pgtable_s1()
774 		cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff;  in arm_32_lpae_alloc_pgtable_s1()
781 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s2()  argument
785 	if (cfg->ias > 40 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s2()
788 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s2()
789 	iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s2()
791 		cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff;  in arm_32_lpae_alloc_pgtable_s2()
852 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()  local
855 		cfg->pgsize_bitmap, cfg->ias);  in arm_lpae_dump_ops()
868 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)  in arm_lpae_run_tests()  argument
883 		cfg_cookie = cfg;  in arm_lpae_run_tests()
884 		ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);  in arm_lpae_run_tests()
907 		j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);  in arm_lpae_run_tests()
927 			j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);  in arm_lpae_run_tests()
931 		size = 1UL << __ffs(cfg->pgsize_bitmap);  in arm_lpae_run_tests()
944 		j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);  in arm_lpae_run_tests()
963 			j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);  in arm_lpae_run_tests()
986 	struct io_pgtable_cfg cfg = {  in arm_lpae_do_selftests()  local
993 			cfg.pgsize_bitmap = pgsize[i];  in arm_lpae_do_selftests()
994 			cfg.ias = ias[j];  in arm_lpae_do_selftests()
997 			if (arm_lpae_run_tests(&cfg))  in arm_lpae_do_selftests()