Lines Matching refs:data
203 static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
207 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, in arm_lpae_init_pte() argument
224 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); in arm_lpae_init_pte()
226 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); in arm_lpae_init_pte()
227 if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz)) in arm_lpae_init_pte()
231 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_init_pte()
240 pte |= pfn_to_iopte(paddr >> data->pg_shift, data); in arm_lpae_init_pte()
243 data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), data->iop.cookie); in arm_lpae_init_pte()
247 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, in __arm_lpae_map() argument
252 void *cookie = data->iop.cookie; in __arm_lpae_map()
253 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); in __arm_lpae_map()
256 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); in __arm_lpae_map()
259 if (size == block_size && (size & data->iop.cfg.pgsize_bitmap)) in __arm_lpae_map()
260 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); in __arm_lpae_map()
269 cptep = alloc_pages_exact(1UL << data->pg_shift, in __arm_lpae_map()
274 data->iop.cfg.tlb->flush_pgtable(cptep, 1UL << data->pg_shift, in __arm_lpae_map()
277 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) in __arm_lpae_map()
280 data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); in __arm_lpae_map()
282 cptep = iopte_deref(pte, data); in __arm_lpae_map()
286 return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); in __arm_lpae_map()
289 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, in arm_lpae_prot_to_pte() argument
294 if (data->iop.fmt == ARM_64_LPAE_S1 || in arm_lpae_prot_to_pte()
295 data->iop.fmt == ARM_32_LPAE_S1) { in arm_lpae_prot_to_pte()
325 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_map() local
326 arm_lpae_iopte *ptep = data->pgd; in arm_lpae_map()
327 int lvl = ARM_LPAE_START_LVL(data); in arm_lpae_map()
334 prot = arm_lpae_prot_to_pte(data, iommu_prot); in arm_lpae_map()
335 return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); in arm_lpae_map()
338 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, in __arm_lpae_free_pgtable() argument
344 if (lvl == ARM_LPAE_START_LVL(data)) in __arm_lpae_free_pgtable()
345 table_size = data->pgd_size; in __arm_lpae_free_pgtable()
347 table_size = 1UL << data->pg_shift; in __arm_lpae_free_pgtable()
363 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); in __arm_lpae_free_pgtable()
371 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); in arm_lpae_free_pgtable() local
373 __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd); in arm_lpae_free_pgtable()
374 kfree(data); in arm_lpae_free_pgtable()
377 static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, in arm_lpae_split_blk_unmap() argument
385 void *cookie = data->iop.cookie; in arm_lpae_split_blk_unmap()
386 const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; in arm_lpae_split_blk_unmap()
390 blk_paddr = iopte_to_pfn(*ptep, data) << data->pg_shift; in arm_lpae_split_blk_unmap()
400 tablep = &table - ARM_LPAE_LVL_IDX(blk_start, lvl, data); in arm_lpae_split_blk_unmap()
401 if (__arm_lpae_map(data, blk_start, blk_paddr, size, prot, lvl, in arm_lpae_split_blk_unmap()
405 tablep = iopte_deref(table, data); in arm_lpae_split_blk_unmap()
406 __arm_lpae_free_pgtable(data, lvl + 1, tablep); in arm_lpae_split_blk_unmap()
419 static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, in __arm_lpae_unmap() argument
424 const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; in __arm_lpae_unmap()
425 void *cookie = data->iop.cookie; in __arm_lpae_unmap()
426 size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data); in __arm_lpae_unmap()
428 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); in __arm_lpae_unmap()
443 tlb->tlb_sync(data->iop.cookie); in __arm_lpae_unmap()
444 ptep = iopte_deref(pte, data); in __arm_lpae_unmap()
445 __arm_lpae_free_pgtable(data, lvl + 1, ptep); in __arm_lpae_unmap()
456 return arm_lpae_split_blk_unmap(data, iova, size, in __arm_lpae_unmap()
462 ptep = iopte_deref(pte, data); in __arm_lpae_unmap()
463 return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep); in __arm_lpae_unmap()
470 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_unmap() local
471 struct io_pgtable *iop = &data->iop; in arm_lpae_unmap()
472 arm_lpae_iopte *ptep = data->pgd; in arm_lpae_unmap()
473 int lvl = ARM_LPAE_START_LVL(data); in arm_lpae_unmap()
475 unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep); in arm_lpae_unmap()
485 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_iova_to_phys() local
486 arm_lpae_iopte pte, *ptep = data->pgd; in arm_lpae_iova_to_phys()
487 int lvl = ARM_LPAE_START_LVL(data); in arm_lpae_iova_to_phys()
495 pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data)); in arm_lpae_iova_to_phys()
506 ptep = iopte_deref(pte, data); in arm_lpae_iova_to_phys()
513 iova &= ((1 << data->pg_shift) - 1); in arm_lpae_iova_to_phys()
514 return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; in arm_lpae_iova_to_phys()
556 struct arm_lpae_io_pgtable *data; in arm_lpae_alloc_pgtable() local
569 data = kmalloc(sizeof(*data), GFP_KERNEL); in arm_lpae_alloc_pgtable()
570 if (!data) in arm_lpae_alloc_pgtable()
573 data->pg_shift = __ffs(cfg->pgsize_bitmap); in arm_lpae_alloc_pgtable()
574 data->bits_per_level = data->pg_shift - ilog2(sizeof(arm_lpae_iopte)); in arm_lpae_alloc_pgtable()
576 va_bits = cfg->ias - data->pg_shift; in arm_lpae_alloc_pgtable()
577 data->levels = DIV_ROUND_UP(va_bits, data->bits_per_level); in arm_lpae_alloc_pgtable()
580 pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1)); in arm_lpae_alloc_pgtable()
581 data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); in arm_lpae_alloc_pgtable()
583 data->iop.ops = (struct io_pgtable_ops) { in arm_lpae_alloc_pgtable()
589 return data; in arm_lpae_alloc_pgtable()
596 struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s1() local
598 if (!data) in arm_64_lpae_alloc_pgtable_s1()
606 switch (1 << data->pg_shift) { in arm_64_lpae_alloc_pgtable_s1()
659 data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); in arm_64_lpae_alloc_pgtable_s1()
660 if (!data->pgd) in arm_64_lpae_alloc_pgtable_s1()
663 cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); in arm_64_lpae_alloc_pgtable_s1()
666 cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s1()
668 return &data->iop; in arm_64_lpae_alloc_pgtable_s1()
671 kfree(data); in arm_64_lpae_alloc_pgtable_s1()
679 struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s2() local
681 if (!data) in arm_64_lpae_alloc_pgtable_s2()
688 if (data->levels == ARM_LPAE_MAX_LEVELS) { in arm_64_lpae_alloc_pgtable_s2()
691 pgd_pages = data->pgd_size >> ilog2(sizeof(arm_lpae_iopte)); in arm_64_lpae_alloc_pgtable_s2()
693 data->pgd_size = pgd_pages << data->pg_shift; in arm_64_lpae_alloc_pgtable_s2()
694 data->levels--; in arm_64_lpae_alloc_pgtable_s2()
704 sl = ARM_LPAE_START_LVL(data); in arm_64_lpae_alloc_pgtable_s2()
706 switch (1 << data->pg_shift) { in arm_64_lpae_alloc_pgtable_s2()
747 data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); in arm_64_lpae_alloc_pgtable_s2()
748 if (!data->pgd) in arm_64_lpae_alloc_pgtable_s2()
751 cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); in arm_64_lpae_alloc_pgtable_s2()
754 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s2()
755 return &data->iop; in arm_64_lpae_alloc_pgtable_s2()
758 kfree(data); in arm_64_lpae_alloc_pgtable_s2()
851 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_dump_ops() local
852 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_dump_ops()
857 data->levels, data->pgd_size, data->pg_shift, in arm_lpae_dump_ops()
858 data->bits_per_level, data->pgd); in arm_lpae_dump_ops()