arch/arm/mm/dma-mapping.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
arm_dma_buffer_find
arm_dma_map_page
arm_coherent_dma_map_page
arm_dma_unmap_page
arm_dma_sync_single_for_cpu
arm_dma_sync_single_for_device
__dma_supported
get_coherent_dma_mask
__dma_clear_buffer
__dma_alloc_buffer
__dma_free_buffer
early_coherent_pool
atomic_pool_init
dma_contiguous_early_fixup
dma_contiguous_remap
__dma_update_pte
__dma_remap
__alloc_remap_buffer
__alloc_from_pool
__in_atomic_pool
__free_from_pool
__alloc_from_contiguous
__free_from_contiguous
__get_dma_pgprot
__alloc_simple_buffer
simple_allocator_alloc
simple_allocator_free
cma_allocator_alloc
cma_allocator_free
pool_allocator_alloc
pool_allocator_free
remap_allocator_alloc
remap_allocator_free
__dma_alloc
arm_dma_alloc
arm_coherent_dma_alloc
__arm_dma_mmap
arm_coherent_dma_mmap
arm_dma_mmap
__arm_dma_free
arm_dma_free
arm_coherent_dma_free
arm_dma_get_sgtable
dma_cache_maint_page
__dma_page_cpu_to_dev
__dma_page_dev_to_cpu
arm_dma_map_sg
arm_dma_unmap_sg
arm_dma_sync_sg_for_cpu
arm_dma_sync_sg_for_device
arm_dma_supported
arm_get_dma_map_ops
__dma_info_to_prot
__alloc_iova
__free_iova
__iommu_alloc_buffer
__iommu_free_buffer
__iommu_create_mapping
__iommu_remove_mapping
__atomic_get_pages
__iommu_get_pages
__iommu_alloc_simple
__iommu_free_atomic
__arm_iommu_alloc_attrs
arm_iommu_alloc_attrs
arm_coherent_iommu_alloc_attrs
__arm_iommu_mmap_attrs
arm_iommu_mmap_attrs
arm_coherent_iommu_mmap_attrs
__arm_iommu_free_attrs
arm_iommu_free_attrs
arm_coherent_iommu_free_attrs
arm_iommu_get_sgtable
__map_sg_chunk
__iommu_map_sg
arm_coherent_iommu_map_sg
arm_iommu_map_sg
__iommu_unmap_sg
arm_coherent_iommu_unmap_sg
arm_iommu_unmap_sg
arm_iommu_sync_sg_for_cpu
arm_iommu_sync_sg_for_device
arm_coherent_iommu_map_page
arm_iommu_map_page
arm_coherent_iommu_unmap_page
arm_iommu_unmap_page
arm_iommu_map_resource
arm_iommu_unmap_resource
arm_iommu_sync_single_for_cpu
arm_iommu_sync_single_for_device
arm_iommu_create_mapping
release_iommu_mapping
extend_iommu_mapping
arm_iommu_release_mapping
__arm_iommu_attach_device
arm_iommu_attach_device
arm_iommu_detach_device
arm_get_iommu_dma_map_ops
arm_setup_iommu_dma_ops
arm_teardown_iommu_dma_ops
arm_setup_iommu_dma_ops
arm_teardown_iommu_dma_ops
arch_setup_dma_ops
arch_teardown_dma_ops
arch_sync_dma_for_device
arch_sync_dma_for_cpu
arch_dma_coherent_to_pfn
arch_dma_alloc
arch_dma_free
   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/arch/arm/mm/dma-mapping.c
   4  *
   5  *  Copyright (C) 2000-2004 Russell King
   6  *
   7  *  DMA uncached mapping support.
   8  */
   9 #include <linux/module.h>
  10 #include <linux/mm.h>
  11 #include <linux/genalloc.h>
  12 #include <linux/gfp.h>
  13 #include <linux/errno.h>
  14 #include <linux/list.h>
  15 #include <linux/init.h>
  16 #include <linux/device.h>
  17 #include <linux/dma-direct.h>
  18 #include <linux/dma-mapping.h>
  19 #include <linux/dma-noncoherent.h>
  20 #include <linux/dma-contiguous.h>
  21 #include <linux/highmem.h>
  22 #include <linux/memblock.h>
  23 #include <linux/slab.h>
  24 #include <linux/iommu.h>
  25 #include <linux/io.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/sizes.h>
  28 #include <linux/cma.h>
  29 
  30 #include <asm/memory.h>
  31 #include <asm/highmem.h>
  32 #include <asm/cacheflush.h>
  33 #include <asm/tlbflush.h>
  34 #include <asm/mach/arch.h>
  35 #include <asm/dma-iommu.h>
  36 #include <asm/mach/map.h>
  37 #include <asm/system_info.h>
  38 #include <asm/dma-contiguous.h>
  39 #include <xen/swiotlb-xen.h>
  40 
  41 #include "dma.h"
  42 #include "mm.h"
  43 
  44 struct arm_dma_alloc_args {
  45         struct device *dev;
  46         size_t size;
  47         gfp_t gfp;
  48         pgprot_t prot;
  49         const void *caller;
  50         bool want_vaddr;
  51         int coherent_flag;
  52 };
  53 
  54 struct arm_dma_free_args {
  55         struct device *dev;
  56         size_t size;
  57         void *cpu_addr;
  58         struct page *page;
  59         bool want_vaddr;
  60 };
  61 
  62 #define NORMAL      0
  63 #define COHERENT    1
  64 
  65 struct arm_dma_allocator {
  66         void *(*alloc)(struct arm_dma_alloc_args *args,
  67                        struct page **ret_page);
  68         void (*free)(struct arm_dma_free_args *args);
  69 };
  70 
  71 struct arm_dma_buffer {
  72         struct list_head list;
  73         void *virt;
  74         struct arm_dma_allocator *allocator;
  75 };
  76 
  77 static LIST_HEAD(arm_dma_bufs);
  78 static DEFINE_SPINLOCK(arm_dma_bufs_lock);
  79 
  80 static struct arm_dma_buffer *arm_dma_buffer_find(void *virt)
  81 {
  82         struct arm_dma_buffer *buf, *found = NULL;
  83         unsigned long flags;
  84 
  85         spin_lock_irqsave(&arm_dma_bufs_lock, flags);
  86         list_for_each_entry(buf, &arm_dma_bufs, list) {
  87                 if (buf->virt == virt) {
  88                         list_del(&buf->list);
  89                         found = buf;
  90                         break;
  91                 }
  92         }
  93         spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
  94         return found;
  95 }
  96 
  97 /*
  98  * The DMA API is built upon the notion of "buffer ownership".  A buffer
  99  * is either exclusively owned by the CPU (and therefore may be accessed
 100  * by it) or exclusively owned by the DMA device.  These helper functions
 101  * represent the transitions between these two ownership states.
 102  *
 103  * Note, however, that on later ARMs, this notion does not work due to
 104  * speculative prefetches.  We model our approach on the assumption that
 105  * the CPU does do speculative prefetches, which means we clean caches
 106  * before transfers and delay cache invalidation until transfer completion.
 107  *
 108  */
 109 static void __dma_page_cpu_to_dev(struct page *, unsigned long,
 110                 size_t, enum dma_data_direction);
 111 static void __dma_page_dev_to_cpu(struct page *, unsigned long,
 112                 size_t, enum dma_data_direction);
 113 
 114 /**
 115  * arm_dma_map_page - map a portion of a page for streaming DMA
 116  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 117  * @page: page that buffer resides in
 118  * @offset: offset into page for start of buffer
 119  * @size: size of buffer to map
 120  * @dir: DMA transfer direction
 121  *
 122  * Ensure that any data held in the cache is appropriately discarded
 123  * or written back.
 124  *
 125  * The device owns this memory once this call has completed.  The CPU
 126  * can regain ownership by calling dma_unmap_page().
 127  */
 128 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
 129              unsigned long offset, size_t size, enum dma_data_direction dir,
 130              unsigned long attrs)
 131 {
 132         if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
 133                 __dma_page_cpu_to_dev(page, offset, size, dir);
 134         return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 135 }
 136 
 137 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
 138              unsigned long offset, size_t size, enum dma_data_direction dir,
 139              unsigned long attrs)
 140 {
 141         return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 142 }
 143 
 144 /**
 145  * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
 146  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 147  * @handle: DMA address of buffer
 148  * @size: size of buffer (same as passed to dma_map_page)
 149  * @dir: DMA transfer direction (same as passed to dma_map_page)
 150  *
 151  * Unmap a page streaming mode DMA translation.  The handle and size
 152  * must match what was provided in the previous dma_map_page() call.
 153  * All other usages are undefined.
 154  *
 155  * After this call, reads by the CPU to the buffer are guaranteed to see
 156  * whatever the device wrote there.
 157  */
 158 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
 159                 size_t size, enum dma_data_direction dir, unsigned long attrs)
 160 {
 161         if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
 162                 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
 163                                       handle & ~PAGE_MASK, size, dir);
 164 }
 165 
 166 static void arm_dma_sync_single_for_cpu(struct device *dev,
 167                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
 168 {
 169         unsigned int offset = handle & (PAGE_SIZE - 1);
 170         struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 171         __dma_page_dev_to_cpu(page, offset, size, dir);
 172 }
 173 
 174 static void arm_dma_sync_single_for_device(struct device *dev,
 175                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
 176 {
 177         unsigned int offset = handle & (PAGE_SIZE - 1);
 178         struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
 179         __dma_page_cpu_to_dev(page, offset, size, dir);
 180 }
 181 
 182 const struct dma_map_ops arm_dma_ops = {
 183         .alloc                  = arm_dma_alloc,
 184         .free                   = arm_dma_free,
 185         .mmap                   = arm_dma_mmap,
 186         .get_sgtable            = arm_dma_get_sgtable,
 187         .map_page               = arm_dma_map_page,
 188         .unmap_page             = arm_dma_unmap_page,
 189         .map_sg                 = arm_dma_map_sg,
 190         .unmap_sg               = arm_dma_unmap_sg,
 191         .map_resource           = dma_direct_map_resource,
 192         .sync_single_for_cpu    = arm_dma_sync_single_for_cpu,
 193         .sync_single_for_device = arm_dma_sync_single_for_device,
 194         .sync_sg_for_cpu        = arm_dma_sync_sg_for_cpu,
 195         .sync_sg_for_device     = arm_dma_sync_sg_for_device,
 196         .dma_supported          = arm_dma_supported,
 197         .get_required_mask      = dma_direct_get_required_mask,
 198 };
 199 EXPORT_SYMBOL(arm_dma_ops);
 200 
 201 static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 202         dma_addr_t *handle, gfp_t gfp, unsigned long attrs);
 203 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 204                                   dma_addr_t handle, unsigned long attrs);
 205 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 206                  void *cpu_addr, dma_addr_t dma_addr, size_t size,
 207                  unsigned long attrs);
 208 
 209 const struct dma_map_ops arm_coherent_dma_ops = {
 210         .alloc                  = arm_coherent_dma_alloc,
 211         .free                   = arm_coherent_dma_free,
 212         .mmap                   = arm_coherent_dma_mmap,
 213         .get_sgtable            = arm_dma_get_sgtable,
 214         .map_page               = arm_coherent_dma_map_page,
 215         .map_sg                 = arm_dma_map_sg,
 216         .map_resource           = dma_direct_map_resource,
 217         .dma_supported          = arm_dma_supported,
 218         .get_required_mask      = dma_direct_get_required_mask,
 219 };
 220 EXPORT_SYMBOL(arm_coherent_dma_ops);
 221 
 222 static int __dma_supported(struct device *dev, u64 mask, bool warn)
 223 {
 224         unsigned long max_dma_pfn = min(max_pfn - 1, arm_dma_pfn_limit);
 225 
 226         /*
 227          * Translate the device's DMA mask to a PFN limit.  This
 228          * PFN number includes the page which we can DMA to.
 229          */
 230         if (dma_to_pfn(dev, mask) < max_dma_pfn) {
 231                 if (warn)
 232                         dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
 233                                  mask,
 234                                  dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
 235                                  max_dma_pfn + 1);
 236                 return 0;
 237         }
 238 
 239         return 1;
 240 }
 241 
 242 static u64 get_coherent_dma_mask(struct device *dev)
 243 {
 244         u64 mask = (u64)DMA_BIT_MASK(32);
 245 
 246         if (dev) {
 247                 mask = dev->coherent_dma_mask;
 248 
 249                 /*
 250                  * Sanity check the DMA mask - it must be non-zero, and
 251                  * must be able to be satisfied by a DMA allocation.
 252                  */
 253                 if (mask == 0) {
 254                         dev_warn(dev, "coherent DMA mask is unset\n");
 255                         return 0;
 256                 }
 257 
 258                 if (!__dma_supported(dev, mask, true))
 259                         return 0;
 260         }
 261 
 262         return mask;
 263 }
 264 
 265 static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag)
 266 {
 267         /*
 268          * Ensure that the allocated pages are zeroed, and that any data
 269          * lurking in the kernel direct-mapped region is invalidated.
 270          */
 271         if (PageHighMem(page)) {
 272                 phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
 273                 phys_addr_t end = base + size;
 274                 while (size > 0) {
 275                         void *ptr = kmap_atomic(page);
 276                         memset(ptr, 0, PAGE_SIZE);
 277                         if (coherent_flag != COHERENT)
 278                                 dmac_flush_range(ptr, ptr + PAGE_SIZE);
 279                         kunmap_atomic(ptr);
 280                         page++;
 281                         size -= PAGE_SIZE;
 282                 }
 283                 if (coherent_flag != COHERENT)
 284                         outer_flush_range(base, end);
 285         } else {
 286                 void *ptr = page_address(page);
 287                 memset(ptr, 0, size);
 288                 if (coherent_flag != COHERENT) {
 289                         dmac_flush_range(ptr, ptr + size);
 290                         outer_flush_range(__pa(ptr), __pa(ptr) + size);
 291                 }
 292         }
 293 }
 294 
 295 /*
 296  * Allocate a DMA buffer for 'dev' of size 'size' using the
 297  * specified gfp mask.  Note that 'size' must be page aligned.
 298  */
 299 static struct page *__dma_alloc_buffer(struct device *dev, size_t size,
 300                                        gfp_t gfp, int coherent_flag)
 301 {
 302         unsigned long order = get_order(size);
 303         struct page *page, *p, *e;
 304 
 305         page = alloc_pages(gfp, order);
 306         if (!page)
 307                 return NULL;
 308 
 309         /*
 310          * Now split the huge page and free the excess pages
 311          */
 312         split_page(page, order);
 313         for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
 314                 __free_page(p);
 315 
 316         __dma_clear_buffer(page, size, coherent_flag);
 317 
 318         return page;
 319 }
 320 
 321 /*
 322  * Free a DMA buffer.  'size' must be page aligned.
 323  */
 324 static void __dma_free_buffer(struct page *page, size_t size)
 325 {
 326         struct page *e = page + (size >> PAGE_SHIFT);
 327 
 328         while (page < e) {
 329                 __free_page(page);
 330                 page++;
 331         }
 332 }
 333 
 334 static void *__alloc_from_contiguous(struct device *dev, size_t size,
 335                                      pgprot_t prot, struct page **ret_page,
 336                                      const void *caller, bool want_vaddr,
 337                                      int coherent_flag, gfp_t gfp);
 338 
 339 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 340                                  pgprot_t prot, struct page **ret_page,
 341                                  const void *caller, bool want_vaddr);
 342 
 343 #define DEFAULT_DMA_COHERENT_POOL_SIZE  SZ_256K
 344 static struct gen_pool *atomic_pool __ro_after_init;
 345 
 346 static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
 347 
 348 static int __init early_coherent_pool(char *p)
 349 {
 350         atomic_pool_size = memparse(p, &p);
 351         return 0;
 352 }
 353 early_param("coherent_pool", early_coherent_pool);
 354 
 355 /*
 356  * Initialise the coherent pool for atomic allocations.
 357  */
 358 static int __init atomic_pool_init(void)
 359 {
 360         pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
 361         gfp_t gfp = GFP_KERNEL | GFP_DMA;
 362         struct page *page;
 363         void *ptr;
 364 
 365         atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
 366         if (!atomic_pool)
 367                 goto out;
 368         /*
 369          * The atomic pool is only used for non-coherent allocations
 370          * so we must pass NORMAL for coherent_flag.
 371          */
 372         if (dev_get_cma_area(NULL))
 373                 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
 374                                       &page, atomic_pool_init, true, NORMAL,
 375                                       GFP_KERNEL);
 376         else
 377                 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
 378                                            &page, atomic_pool_init, true);
 379         if (ptr) {
 380                 int ret;
 381 
 382                 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
 383                                         page_to_phys(page),
 384                                         atomic_pool_size, -1);
 385                 if (ret)
 386                         goto destroy_genpool;
 387 
 388                 gen_pool_set_algo(atomic_pool,
 389                                 gen_pool_first_fit_order_align,
 390                                 NULL);
 391                 pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n",
 392                        atomic_pool_size / 1024);
 393                 return 0;
 394         }
 395 
 396 destroy_genpool:
 397         gen_pool_destroy(atomic_pool);
 398         atomic_pool = NULL;
 399 out:
 400         pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
 401                atomic_pool_size / 1024);
 402         return -ENOMEM;
 403 }
 404 /*
 405  * CMA is activated by core_initcall, so we must be called after it.
 406  */
 407 postcore_initcall(atomic_pool_init);
 408 
 409 struct dma_contig_early_reserve {
 410         phys_addr_t base;
 411         unsigned long size;
 412 };
 413 
 414 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
 415 
 416 static int dma_mmu_remap_num __initdata;
 417 
 418 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
 419 {
 420         dma_mmu_remap[dma_mmu_remap_num].base = base;
 421         dma_mmu_remap[dma_mmu_remap_num].size = size;
 422         dma_mmu_remap_num++;
 423 }
 424 
 425 void __init dma_contiguous_remap(void)
 426 {
 427         int i;
 428         for (i = 0; i < dma_mmu_remap_num; i++) {
 429                 phys_addr_t start = dma_mmu_remap[i].base;
 430                 phys_addr_t end = start + dma_mmu_remap[i].size;
 431                 struct map_desc map;
 432                 unsigned long addr;
 433 
 434                 if (end > arm_lowmem_limit)
 435                         end = arm_lowmem_limit;
 436                 if (start >= end)
 437                         continue;
 438 
 439                 map.pfn = __phys_to_pfn(start);
 440                 map.virtual = __phys_to_virt(start);
 441                 map.length = end - start;
 442                 map.type = MT_MEMORY_DMA_READY;
 443 
 444                 /*
 445                  * Clear previous low-memory mapping to ensure that the
 446                  * TLB does not see any conflicting entries, then flush
 447                  * the TLB of the old entries before creating new mappings.
 448                  *
 449                  * This ensures that any speculatively loaded TLB entries
 450                  * (even though they may be rare) can not cause any problems,
 451                  * and ensures that this code is architecturally compliant.
 452                  */
 453                 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
 454                      addr += PMD_SIZE)
 455                         pmd_clear(pmd_off_k(addr));
 456 
 457                 flush_tlb_kernel_range(__phys_to_virt(start),
 458                                        __phys_to_virt(end));
 459 
 460                 iotable_init(&map, 1);
 461         }
 462 }
 463 
 464 static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data)
 465 {
 466         struct page *page = virt_to_page(addr);
 467         pgprot_t prot = *(pgprot_t *)data;
 468 
 469         set_pte_ext(pte, mk_pte(page, prot), 0);
 470         return 0;
 471 }
 472 
 473 static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
 474 {
 475         unsigned long start = (unsigned long) page_address(page);
 476         unsigned end = start + size;
 477 
 478         apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
 479         flush_tlb_kernel_range(start, end);
 480 }
 481 
 482 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 483                                  pgprot_t prot, struct page **ret_page,
 484                                  const void *caller, bool want_vaddr)
 485 {
 486         struct page *page;
 487         void *ptr = NULL;
 488         /*
 489          * __alloc_remap_buffer is only called when the device is
 490          * non-coherent
 491          */
 492         page = __dma_alloc_buffer(dev, size, gfp, NORMAL);
 493         if (!page)
 494                 return NULL;
 495         if (!want_vaddr)
 496                 goto out;
 497 
 498         ptr = dma_common_contiguous_remap(page, size, prot, caller);
 499         if (!ptr) {
 500                 __dma_free_buffer(page, size);
 501                 return NULL;
 502         }
 503 
 504  out:
 505         *ret_page = page;
 506         return ptr;
 507 }
 508 
 509 static void *__alloc_from_pool(size_t size, struct page **ret_page)
 510 {
 511         unsigned long val;
 512         void *ptr = NULL;
 513 
 514         if (!atomic_pool) {
 515                 WARN(1, "coherent pool not initialised!\n");
 516                 return NULL;
 517         }
 518 
 519         val = gen_pool_alloc(atomic_pool, size);
 520         if (val) {
 521                 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
 522 
 523                 *ret_page = phys_to_page(phys);
 524                 ptr = (void *)val;
 525         }
 526 
 527         return ptr;
 528 }
 529 
 530 static bool __in_atomic_pool(void *start, size_t size)
 531 {
 532         return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
 533 }
 534 
 535 static int __free_from_pool(void *start, size_t size)
 536 {
 537         if (!__in_atomic_pool(start, size))
 538                 return 0;
 539 
 540         gen_pool_free(atomic_pool, (unsigned long)start, size);
 541 
 542         return 1;
 543 }
 544 
 545 static void *__alloc_from_contiguous(struct device *dev, size_t size,
 546                                      pgprot_t prot, struct page **ret_page,
 547                                      const void *caller, bool want_vaddr,
 548                                      int coherent_flag, gfp_t gfp)
 549 {
 550         unsigned long order = get_order(size);
 551         size_t count = size >> PAGE_SHIFT;
 552         struct page *page;
 553         void *ptr = NULL;
 554 
 555         page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN);
 556         if (!page)
 557                 return NULL;
 558 
 559         __dma_clear_buffer(page, size, coherent_flag);
 560 
 561         if (!want_vaddr)
 562                 goto out;
 563 
 564         if (PageHighMem(page)) {
 565                 ptr = dma_common_contiguous_remap(page, size, prot, caller);
 566                 if (!ptr) {
 567                         dma_release_from_contiguous(dev, page, count);
 568                         return NULL;
 569                 }
 570         } else {
 571                 __dma_remap(page, size, prot);
 572                 ptr = page_address(page);
 573         }
 574 
 575  out:
 576         *ret_page = page;
 577         return ptr;
 578 }
 579 
 580 static void __free_from_contiguous(struct device *dev, struct page *page,
 581                                    void *cpu_addr, size_t size, bool want_vaddr)
 582 {
 583         if (want_vaddr) {
 584                 if (PageHighMem(page))
 585                         dma_common_free_remap(cpu_addr, size);
 586                 else
 587                         __dma_remap(page, size, PAGE_KERNEL);
 588         }
 589         dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
 590 }
 591 
 592 static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot)
 593 {
 594         prot = (attrs & DMA_ATTR_WRITE_COMBINE) ?
 595                         pgprot_writecombine(prot) :
 596                         pgprot_dmacoherent(prot);
 597         return prot;
 598 }
 599 
 600 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
 601                                    struct page **ret_page)
 602 {
 603         struct page *page;
 604         /* __alloc_simple_buffer is only called when the device is coherent */
 605         page = __dma_alloc_buffer(dev, size, gfp, COHERENT);
 606         if (!page)
 607                 return NULL;
 608 
 609         *ret_page = page;
 610         return page_address(page);
 611 }
 612 
 613 static void *simple_allocator_alloc(struct arm_dma_alloc_args *args,
 614                                     struct page **ret_page)
 615 {
 616         return __alloc_simple_buffer(args->dev, args->size, args->gfp,
 617                                      ret_page);
 618 }
 619 
 620 static void simple_allocator_free(struct arm_dma_free_args *args)
 621 {
 622         __dma_free_buffer(args->page, args->size);
 623 }
 624 
 625 static struct arm_dma_allocator simple_allocator = {
 626         .alloc = simple_allocator_alloc,
 627         .free = simple_allocator_free,
 628 };
 629 
 630 static void *cma_allocator_alloc(struct arm_dma_alloc_args *args,
 631                                  struct page **ret_page)
 632 {
 633         return __alloc_from_contiguous(args->dev, args->size, args->prot,
 634                                        ret_page, args->caller,
 635                                        args->want_vaddr, args->coherent_flag,
 636                                        args->gfp);
 637 }
 638 
 639 static void cma_allocator_free(struct arm_dma_free_args *args)
 640 {
 641         __free_from_contiguous(args->dev, args->page, args->cpu_addr,
 642                                args->size, args->want_vaddr);
 643 }
 644 
 645 static struct arm_dma_allocator cma_allocator = {
 646         .alloc = cma_allocator_alloc,
 647         .free = cma_allocator_free,
 648 };
 649 
 650 static void *pool_allocator_alloc(struct arm_dma_alloc_args *args,
 651                                   struct page **ret_page)
 652 {
 653         return __alloc_from_pool(args->size, ret_page);
 654 }
 655 
 656 static void pool_allocator_free(struct arm_dma_free_args *args)
 657 {
 658         __free_from_pool(args->cpu_addr, args->size);
 659 }
 660 
 661 static struct arm_dma_allocator pool_allocator = {
 662         .alloc = pool_allocator_alloc,
 663         .free = pool_allocator_free,
 664 };
 665 
 666 static void *remap_allocator_alloc(struct arm_dma_alloc_args *args,
 667                                    struct page **ret_page)
 668 {
 669         return __alloc_remap_buffer(args->dev, args->size, args->gfp,
 670                                     args->prot, ret_page, args->caller,
 671                                     args->want_vaddr);
 672 }
 673 
 674 static void remap_allocator_free(struct arm_dma_free_args *args)
 675 {
 676         if (args->want_vaddr)
 677                 dma_common_free_remap(args->cpu_addr, args->size);
 678 
 679         __dma_free_buffer(args->page, args->size);
 680 }
 681 
 682 static struct arm_dma_allocator remap_allocator = {
 683         .alloc = remap_allocator_alloc,
 684         .free = remap_allocator_free,
 685 };
 686 
 687 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 688                          gfp_t gfp, pgprot_t prot, bool is_coherent,
 689                          unsigned long attrs, const void *caller)
 690 {
 691         u64 mask = get_coherent_dma_mask(dev);
 692         struct page *page = NULL;
 693         void *addr;
 694         bool allowblock, cma;
 695         struct arm_dma_buffer *buf;
 696         struct arm_dma_alloc_args args = {
 697                 .dev = dev,
 698                 .size = PAGE_ALIGN(size),
 699                 .gfp = gfp,
 700                 .prot = prot,
 701                 .caller = caller,
 702                 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
 703                 .coherent_flag = is_coherent ? COHERENT : NORMAL,
 704         };
 705 
 706 #ifdef CONFIG_DMA_API_DEBUG
 707         u64 limit = (mask + 1) & ~mask;
 708         if (limit && size >= limit) {
 709                 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
 710                         size, mask);
 711                 return NULL;
 712         }
 713 #endif
 714 
 715         if (!mask)
 716                 return NULL;
 717 
 718         buf = kzalloc(sizeof(*buf),
 719                       gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM));
 720         if (!buf)
 721                 return NULL;
 722 
 723         if (mask < 0xffffffffULL)
 724                 gfp |= GFP_DMA;
 725 
 726         /*
 727          * Following is a work-around (a.k.a. hack) to prevent pages
 728          * with __GFP_COMP being passed to split_page() which cannot
 729          * handle them.  The real problem is that this flag probably
 730          * should be 0 on ARM as it is not supported on this
 731          * platform; see CONFIG_HUGETLBFS.
 732          */
 733         gfp &= ~(__GFP_COMP);
 734         args.gfp = gfp;
 735 
 736         *handle = DMA_MAPPING_ERROR;
 737         allowblock = gfpflags_allow_blocking(gfp);
 738         cma = allowblock ? dev_get_cma_area(dev) : false;
 739 
 740         if (cma)
 741                 buf->allocator = &cma_allocator;
 742         else if (is_coherent)
 743                 buf->allocator = &simple_allocator;
 744         else if (allowblock)
 745                 buf->allocator = &remap_allocator;
 746         else
 747                 buf->allocator = &pool_allocator;
 748 
 749         addr = buf->allocator->alloc(&args, &page);
 750 
 751         if (page) {
 752                 unsigned long flags;
 753 
 754                 *handle = pfn_to_dma(dev, page_to_pfn(page));
 755                 buf->virt = args.want_vaddr ? addr : page;
 756 
 757                 spin_lock_irqsave(&arm_dma_bufs_lock, flags);
 758                 list_add(&buf->list, &arm_dma_bufs);
 759                 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
 760         } else {
 761                 kfree(buf);
 762         }
 763 
 764         return args.want_vaddr ? addr : page;
 765 }
 766 
 767 /*
 768  * Allocate DMA-coherent memory space and return both the kernel remapped
 769  * virtual and bus address for that space.
 770  */
 771 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 772                     gfp_t gfp, unsigned long attrs)
 773 {
 774         pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
 775 
 776         return __dma_alloc(dev, size, handle, gfp, prot, false,
 777                            attrs, __builtin_return_address(0));
 778 }
 779 
 780 static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
 781         dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
 782 {
 783         return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true,
 784                            attrs, __builtin_return_address(0));
 785 }
 786 
 787 static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 788                  void *cpu_addr, dma_addr_t dma_addr, size_t size,
 789                  unsigned long attrs)
 790 {
 791         int ret = -ENXIO;
 792         unsigned long nr_vma_pages = vma_pages(vma);
 793         unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 794         unsigned long pfn = dma_to_pfn(dev, dma_addr);
 795         unsigned long off = vma->vm_pgoff;
 796 
 797         if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
 798                 return ret;
 799 
 800         if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
 801                 ret = remap_pfn_range(vma, vma->vm_start,
 802                                       pfn + off,
 803                                       vma->vm_end - vma->vm_start,
 804                                       vma->vm_page_prot);
 805         }
 806 
 807         return ret;
 808 }
 809 
 810 /*
 811  * Create userspace mapping for the DMA-coherent memory.
 812  */
 813 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 814                  void *cpu_addr, dma_addr_t dma_addr, size_t size,
 815                  unsigned long attrs)
 816 {
 817         return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 818 }
 819 
 820 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 821                  void *cpu_addr, dma_addr_t dma_addr, size_t size,
 822                  unsigned long attrs)
 823 {
 824         vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
 825         return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 826 }
 827 
 828 /*
 829  * Free a buffer as defined by the above mapping.
 830  */
 831 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 832                            dma_addr_t handle, unsigned long attrs,
 833                            bool is_coherent)
 834 {
 835         struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 836         struct arm_dma_buffer *buf;
 837         struct arm_dma_free_args args = {
 838                 .dev = dev,
 839                 .size = PAGE_ALIGN(size),
 840                 .cpu_addr = cpu_addr,
 841                 .page = page,
 842                 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
 843         };
 844 
 845         buf = arm_dma_buffer_find(cpu_addr);
 846         if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr))
 847                 return;
 848 
 849         buf->allocator->free(&args);
 850         kfree(buf);
 851 }
 852 
 853 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 854                   dma_addr_t handle, unsigned long attrs)
 855 {
 856         __arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
 857 }
 858 
 859 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
 860                                   dma_addr_t handle, unsigned long attrs)
 861 {
 862         __arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
 863 }
 864 
 865 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
 866                  void *cpu_addr, dma_addr_t handle, size_t size,
 867                  unsigned long attrs)
 868 {
 869         unsigned long pfn = dma_to_pfn(dev, handle);
 870         struct page *page;
 871         int ret;
 872 
 873         /* If the PFN is not valid, we do not have a struct page */
 874         if (!pfn_valid(pfn))
 875                 return -ENXIO;
 876 
 877         page = pfn_to_page(pfn);
 878 
 879         ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
 880         if (unlikely(ret))
 881                 return ret;
 882 
 883         sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
 884         return 0;
 885 }
 886 
 887 static void dma_cache_maint_page(struct page *page, unsigned long offset,
 888         size_t size, enum dma_data_direction dir,
 889         void (*op)(const void *, size_t, int))
 890 {
 891         unsigned long pfn;
 892         size_t left = size;
 893 
 894         pfn = page_to_pfn(page) + offset / PAGE_SIZE;
 895         offset %= PAGE_SIZE;
 896 
 897         /*
 898          * A single sg entry may refer to multiple physically contiguous
 899          * pages.  But we still need to process highmem pages individually.
 900          * If highmem is not configured then the bulk of this loop gets
 901          * optimized out.
 902          */
 903         do {
 904                 size_t len = left;
 905                 void *vaddr;
 906 
 907                 page = pfn_to_page(pfn);
 908 
 909                 if (PageHighMem(page)) {
 910                         if (len + offset > PAGE_SIZE)
 911                                 len = PAGE_SIZE - offset;
 912 
 913                         if (cache_is_vipt_nonaliasing()) {
 914                                 vaddr = kmap_atomic(page);
 915                                 op(vaddr + offset, len, dir);
 916                                 kunmap_atomic(vaddr);
 917                         } else {
 918                                 vaddr = kmap_high_get(page);
 919                                 if (vaddr) {
 920                                         op(vaddr + offset, len, dir);
 921                                         kunmap_high(page);
 922                                 }
 923                         }
 924                 } else {
 925                         vaddr = page_address(page) + offset;
 926                         op(vaddr, len, dir);
 927                 }
 928                 offset = 0;
 929                 pfn++;
 930                 left -= len;
 931         } while (left);
 932 }
 933 
 934 /*
 935  * Make an area consistent for devices.
 936  * Note: Drivers should NOT use this function directly, as it will break
 937  * platforms with CONFIG_DMABOUNCE.
 938  * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
 939  */
 940 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
 941         size_t size, enum dma_data_direction dir)
 942 {
 943         phys_addr_t paddr;
 944 
 945         dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 946 
 947         paddr = page_to_phys(page) + off;
 948         if (dir == DMA_FROM_DEVICE) {
 949                 outer_inv_range(paddr, paddr + size);
 950         } else {
 951                 outer_clean_range(paddr, paddr + size);
 952         }
 953         /* FIXME: non-speculating: flush on bidirectional mappings? */
 954 }
 955 
 956 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
 957         size_t size, enum dma_data_direction dir)
 958 {
 959         phys_addr_t paddr = page_to_phys(page) + off;
 960 
 961         /* FIXME: non-speculating: not required */
 962         /* in any case, don't bother invalidating if DMA to device */
 963         if (dir != DMA_TO_DEVICE) {
 964                 outer_inv_range(paddr, paddr + size);
 965 
 966                 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
 967         }
 968 
 969         /*
 970          * Mark the D-cache clean for these pages to avoid extra flushing.
 971          */
 972         if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
 973                 unsigned long pfn;
 974                 size_t left = size;
 975 
 976                 pfn = page_to_pfn(page) + off / PAGE_SIZE;
 977                 off %= PAGE_SIZE;
 978                 if (off) {
 979                         pfn++;
 980                         left -= PAGE_SIZE - off;
 981                 }
 982                 while (left >= PAGE_SIZE) {
 983                         page = pfn_to_page(pfn++);
 984                         set_bit(PG_dcache_clean, &page->flags);
 985                         left -= PAGE_SIZE;
 986                 }
 987         }
 988 }
 989 
 990 /**
 991  * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
 992  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 993  * @sg: list of buffers
 994  * @nents: number of buffers to map
 995  * @dir: DMA transfer direction
 996  *
 997  * Map a set of buffers described by scatterlist in streaming mode for DMA.
 998  * This is the scatter-gather version of the dma_map_single interface.
 999  * Here the scatter gather list elements are each tagged with the
1000  * appropriate dma address and length.  They are obtained via
1001  * sg_dma_{address,length}.
1002  *
1003  * Device ownership issues as mentioned for dma_map_single are the same
1004  * here.
1005  */
1006 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1007                 enum dma_data_direction dir, unsigned long attrs)
1008 {
1009         const struct dma_map_ops *ops = get_dma_ops(dev);
1010         struct scatterlist *s;
1011         int i, j;
1012 
1013         for_each_sg(sg, s, nents, i) {
1014 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1015                 s->dma_length = s->length;
1016 #endif
1017                 s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
1018                                                 s->length, dir, attrs);
1019                 if (dma_mapping_error(dev, s->dma_address))
1020                         goto bad_mapping;
1021         }
1022         return nents;
1023 
1024  bad_mapping:
1025         for_each_sg(sg, s, i, j)
1026                 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1027         return 0;
1028 }
1029 
1030 /**
1031  * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1032  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1033  * @sg: list of buffers
1034  * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1035  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1036  *
1037  * Unmap a set of streaming mode DMA translations.  Again, CPU access
1038  * rules concerning calls here are the same as for dma_unmap_single().
1039  */
1040 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1041                 enum dma_data_direction dir, unsigned long attrs)
1042 {
1043         const struct dma_map_ops *ops = get_dma_ops(dev);
1044         struct scatterlist *s;
1045 
1046         int i;
1047 
1048         for_each_sg(sg, s, nents, i)
1049                 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1050 }
1051 
1052 /**
1053  * arm_dma_sync_sg_for_cpu
1054  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1055  * @sg: list of buffers
1056  * @nents: number of buffers to map (returned from dma_map_sg)
1057  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1058  */
1059 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1060                         int nents, enum dma_data_direction dir)
1061 {
1062         const struct dma_map_ops *ops = get_dma_ops(dev);
1063         struct scatterlist *s;
1064         int i;
1065 
1066         for_each_sg(sg, s, nents, i)
1067                 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
1068                                          dir);
1069 }
1070 
1071 /**
1072  * arm_dma_sync_sg_for_device
1073  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1074  * @sg: list of buffers
1075  * @nents: number of buffers to map (returned from dma_map_sg)
1076  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1077  */
1078 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1079                         int nents, enum dma_data_direction dir)
1080 {
1081         const struct dma_map_ops *ops = get_dma_ops(dev);
1082         struct scatterlist *s;
1083         int i;
1084 
1085         for_each_sg(sg, s, nents, i)
1086                 ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1087                                             dir);
1088 }
1089 
1090 /*
1091  * Return whether the given device DMA address mask can be supported
1092  * properly.  For example, if your device can only drive the low 24-bits
1093  * during bus mastering, then you would pass 0x00ffffff as the mask
1094  * to this function.
1095  */
1096 int arm_dma_supported(struct device *dev, u64 mask)
1097 {
1098         return __dma_supported(dev, mask, false);
1099 }
1100 
1101 static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
1102 {
1103         /*
1104          * When CONFIG_ARM_LPAE is set, physical address can extend above
1105          * 32-bits, which then can't be addressed by devices that only support
1106          * 32-bit DMA.
1107          * Use the generic dma-direct / swiotlb ops code in that case, as that
1108          * handles bounce buffering for us.
1109          */
1110         if (IS_ENABLED(CONFIG_ARM_LPAE))
1111                 return NULL;
1112         return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;
1113 }
1114 
1115 #ifdef CONFIG_ARM_DMA_USE_IOMMU
1116 
1117 static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs)
1118 {
1119         int prot = 0;
1120 
1121         if (attrs & DMA_ATTR_PRIVILEGED)
1122                 prot |= IOMMU_PRIV;
1123 
1124         switch (dir) {
1125         case DMA_BIDIRECTIONAL:
1126                 return prot | IOMMU_READ | IOMMU_WRITE;
1127         case DMA_TO_DEVICE:
1128                 return prot | IOMMU_READ;
1129         case DMA_FROM_DEVICE:
1130                 return prot | IOMMU_WRITE;
1131         default:
1132                 return prot;
1133         }
1134 }
1135 
1136 /* IOMMU */
1137 
1138 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
1139 
1140 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1141                                       size_t size)
1142 {
1143         unsigned int order = get_order(size);
1144         unsigned int align = 0;
1145         unsigned int count, start;
1146         size_t mapping_size = mapping->bits << PAGE_SHIFT;
1147         unsigned long flags;
1148         dma_addr_t iova;
1149         int i;
1150 
1151         if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1152                 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1153 
1154         count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1155         align = (1 << order) - 1;
1156 
1157         spin_lock_irqsave(&mapping->lock, flags);
1158         for (i = 0; i < mapping->nr_bitmaps; i++) {
1159                 start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1160                                 mapping->bits, 0, count, align);
1161 
1162                 if (start > mapping->bits)
1163                         continue;
1164 
1165                 bitmap_set(mapping->bitmaps[i], start, count);
1166                 break;
1167         }
1168 
1169         /*
1170          * No unused range found. Try to extend the existing mapping
1171          * and perform a second attempt to reserve an IO virtual
1172          * address range of size bytes.
1173          */
1174         if (i == mapping->nr_bitmaps) {
1175                 if (extend_iommu_mapping(mapping)) {
1176                         spin_unlock_irqrestore(&mapping->lock, flags);
1177                         return DMA_MAPPING_ERROR;
1178                 }
1179 
1180                 start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1181                                 mapping->bits, 0, count, align);
1182 
1183                 if (start > mapping->bits) {
1184                         spin_unlock_irqrestore(&mapping->lock, flags);
1185                         return DMA_MAPPING_ERROR;
1186                 }
1187 
1188                 bitmap_set(mapping->bitmaps[i], start, count);
1189         }
1190         spin_unlock_irqrestore(&mapping->lock, flags);
1191 
1192         iova = mapping->base + (mapping_size * i);
1193         iova += start << PAGE_SHIFT;
1194 
1195         return iova;
1196 }
1197 
1198 static inline void __free_iova(struct dma_iommu_mapping *mapping,
1199                                dma_addr_t addr, size_t size)
1200 {
1201         unsigned int start, count;
1202         size_t mapping_size = mapping->bits << PAGE_SHIFT;
1203         unsigned long flags;
1204         dma_addr_t bitmap_base;
1205         u32 bitmap_index;
1206 
1207         if (!size)
1208                 return;
1209 
1210         bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;
1211         BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);
1212 
1213         bitmap_base = mapping->base + mapping_size * bitmap_index;
1214 
1215         start = (addr - bitmap_base) >> PAGE_SHIFT;
1216 
1217         if (addr + size > bitmap_base + mapping_size) {
1218                 /*
1219                  * The address range to be freed reaches into the iova
1220                  * range of the next bitmap. This should not happen as
1221                  * we don't allow this in __alloc_iova (at the
1222                  * moment).
1223                  */
1224                 BUG();
1225         } else
1226                 count = size >> PAGE_SHIFT;
1227 
1228         spin_lock_irqsave(&mapping->lock, flags);
1229         bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
1230         spin_unlock_irqrestore(&mapping->lock, flags);
1231 }
1232 
1233 /* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */
1234 static const int iommu_order_array[] = { 9, 8, 4, 0 };
1235 
1236 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1237                                           gfp_t gfp, unsigned long attrs,
1238                                           int coherent_flag)
1239 {
1240         struct page **pages;
1241         int count = size >> PAGE_SHIFT;
1242         int array_size = count * sizeof(struct page *);
1243         int i = 0;
1244         int order_idx = 0;
1245 
1246         if (array_size <= PAGE_SIZE)
1247                 pages = kzalloc(array_size, GFP_KERNEL);
1248         else
1249                 pages = vzalloc(array_size);
1250         if (!pages)
1251                 return NULL;
1252 
1253         if (attrs & DMA_ATTR_FORCE_CONTIGUOUS)
1254         {
1255                 unsigned long order = get_order(size);
1256                 struct page *page;
1257 
1258                 page = dma_alloc_from_contiguous(dev, count, order,
1259                                                  gfp & __GFP_NOWARN);
1260                 if (!page)
1261                         goto error;
1262 
1263                 __dma_clear_buffer(page, size, coherent_flag);
1264 
1265                 for (i = 0; i < count; i++)
1266                         pages[i] = page + i;
1267 
1268                 return pages;
1269         }
1270 
1271         /* Go straight to 4K chunks if caller says it's OK. */
1272         if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES)
1273                 order_idx = ARRAY_SIZE(iommu_order_array) - 1;
1274 
1275         /*
1276          * IOMMU can map any pages, so himem can also be used here
1277          */
1278         gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1279 
1280         while (count) {
1281                 int j, order;
1282 
1283                 order = iommu_order_array[order_idx];
1284 
1285                 /* Drop down when we get small */
1286                 if (__fls(count) < order) {
1287                         order_idx++;
1288                         continue;
1289                 }
1290 
1291                 if (order) {
1292                         /* See if it's easy to allocate a high-order chunk */
1293                         pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
1294 
1295                         /* Go down a notch at first sign of pressure */
1296                         if (!pages[i]) {
1297                                 order_idx++;
1298                                 continue;
1299                         }
1300                 } else {
1301                         pages[i] = alloc_pages(gfp, 0);
1302                         if (!pages[i])
1303                                 goto error;
1304                 }
1305 
1306                 if (order) {
1307                         split_page(pages[i], order);
1308                         j = 1 << order;
1309                         while (--j)
1310                                 pages[i + j] = pages[i] + j;
1311                 }
1312 
1313                 __dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent_flag);
1314                 i += 1 << order;
1315                 count -= 1 << order;
1316         }
1317 
1318         return pages;
1319 error:
1320         while (i--)
1321                 if (pages[i])
1322                         __free_pages(pages[i], 0);
1323         kvfree(pages);
1324         return NULL;
1325 }
1326 
1327 static int __iommu_free_buffer(struct device *dev, struct page **pages,
1328                                size_t size, unsigned long attrs)
1329 {
1330         int count = size >> PAGE_SHIFT;
1331         int i;
1332 
1333         if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
1334                 dma_release_from_contiguous(dev, pages[0], count);
1335         } else {
1336                 for (i = 0; i < count; i++)
1337                         if (pages[i])
1338                                 __free_pages(pages[i], 0);
1339         }
1340 
1341         kvfree(pages);
1342         return 0;
1343 }
1344 
1345 /*
1346  * Create a mapping in device IO address space for specified pages
1347  */
1348 static dma_addr_t
1349 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size,
1350                        unsigned long attrs)
1351 {
1352         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1353         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1354         dma_addr_t dma_addr, iova;
1355         int i;
1356 
1357         dma_addr = __alloc_iova(mapping, size);
1358         if (dma_addr == DMA_MAPPING_ERROR)
1359                 return dma_addr;
1360 
1361         iova = dma_addr;
1362         for (i = 0; i < count; ) {
1363                 int ret;
1364 
1365                 unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1366                 phys_addr_t phys = page_to_phys(pages[i]);
1367                 unsigned int len, j;
1368 
1369                 for (j = i + 1; j < count; j++, next_pfn++)
1370                         if (page_to_pfn(pages[j]) != next_pfn)
1371                                 break;
1372 
1373                 len = (j - i) << PAGE_SHIFT;
1374                 ret = iommu_map(mapping->domain, iova, phys, len,
1375                                 __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs));
1376                 if (ret < 0)
1377                         goto fail;
1378                 iova += len;
1379                 i = j;
1380         }
1381         return dma_addr;
1382 fail:
1383         iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1384         __free_iova(mapping, dma_addr, size);
1385         return DMA_MAPPING_ERROR;
1386 }
1387 
1388 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1389 {
1390         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1391 
1392         /*
1393          * add optional in-page offset from iova to size and align
1394          * result to page size
1395          */
1396         size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1397         iova &= PAGE_MASK;
1398 
1399         iommu_unmap(mapping->domain, iova, size);
1400         __free_iova(mapping, iova, size);
1401         return 0;
1402 }
1403 
1404 static struct page **__atomic_get_pages(void *addr)
1405 {
1406         struct page *page;
1407         phys_addr_t phys;
1408 
1409         phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
1410         page = phys_to_page(phys);
1411 
1412         return (struct page **)page;
1413 }
1414 
1415 static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs)
1416 {
1417         if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1418                 return __atomic_get_pages(cpu_addr);
1419 
1420         if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
1421                 return cpu_addr;
1422 
1423         return dma_common_find_pages(cpu_addr);
1424 }
1425 
1426 static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp,
1427                                   dma_addr_t *handle, int coherent_flag,
1428                                   unsigned long attrs)
1429 {
1430         struct page *page;
1431         void *addr;
1432 
1433         if (coherent_flag  == COHERENT)
1434                 addr = __alloc_simple_buffer(dev, size, gfp, &page);
1435         else
1436                 addr = __alloc_from_pool(size, &page);
1437         if (!addr)
1438                 return NULL;
1439 
1440         *handle = __iommu_create_mapping(dev, &page, size, attrs);
1441         if (*handle == DMA_MAPPING_ERROR)
1442                 goto err_mapping;
1443 
1444         return addr;
1445 
1446 err_mapping:
1447         __free_from_pool(addr, size);
1448         return NULL;
1449 }
1450 
1451 static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1452                         dma_addr_t handle, size_t size, int coherent_flag)
1453 {
1454         __iommu_remove_mapping(dev, handle, size);
1455         if (coherent_flag == COHERENT)
1456                 __dma_free_buffer(virt_to_page(cpu_addr), size);
1457         else
1458                 __free_from_pool(cpu_addr, size);
1459 }
1460 
1461 static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size,
1462             dma_addr_t *handle, gfp_t gfp, unsigned long attrs,
1463             int coherent_flag)
1464 {
1465         pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1466         struct page **pages;
1467         void *addr = NULL;
1468 
1469         *handle = DMA_MAPPING_ERROR;
1470         size = PAGE_ALIGN(size);
1471 
1472         if (coherent_flag  == COHERENT || !gfpflags_allow_blocking(gfp))
1473                 return __iommu_alloc_simple(dev, size, gfp, handle,
1474                                             coherent_flag, attrs);
1475 
1476         /*
1477          * Following is a work-around (a.k.a. hack) to prevent pages
1478          * with __GFP_COMP being passed to split_page() which cannot
1479          * handle them.  The real problem is that this flag probably
1480          * should be 0 on ARM as it is not supported on this
1481          * platform; see CONFIG_HUGETLBFS.
1482          */
1483         gfp &= ~(__GFP_COMP);
1484 
1485         pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag);
1486         if (!pages)
1487                 return NULL;
1488 
1489         *handle = __iommu_create_mapping(dev, pages, size, attrs);
1490         if (*handle == DMA_MAPPING_ERROR)
1491                 goto err_buffer;
1492 
1493         if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
1494                 return pages;
1495 
1496         addr = dma_common_pages_remap(pages, size, prot,
1497                                    __builtin_return_address(0));
1498         if (!addr)
1499                 goto err_mapping;
1500 
1501         return addr;
1502 
1503 err_mapping:
1504         __iommu_remove_mapping(dev, *handle, size);
1505 err_buffer:
1506         __iommu_free_buffer(dev, pages, size, attrs);
1507         return NULL;
1508 }
1509 
1510 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1511             dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1512 {
1513         return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, NORMAL);
1514 }
1515 
1516 static void *arm_coherent_iommu_alloc_attrs(struct device *dev, size_t size,
1517                     dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1518 {
1519         return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, COHERENT);
1520 }
1521 
1522 static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1523                     void *cpu_addr, dma_addr_t dma_addr, size_t size,
1524                     unsigned long attrs)
1525 {
1526         struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1527         unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1528         int err;
1529 
1530         if (!pages)
1531                 return -ENXIO;
1532 
1533         if (vma->vm_pgoff >= nr_pages)
1534                 return -ENXIO;
1535 
1536         err = vm_map_pages(vma, pages, nr_pages);
1537         if (err)
1538                 pr_err("Remapping memory failed: %d\n", err);
1539 
1540         return err;
1541 }
1542 static int arm_iommu_mmap_attrs(struct device *dev,
1543                 struct vm_area_struct *vma, void *cpu_addr,
1544                 dma_addr_t dma_addr, size_t size, unsigned long attrs)
1545 {
1546         vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1547 
1548         return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs);
1549 }
1550 
1551 static int arm_coherent_iommu_mmap_attrs(struct device *dev,
1552                 struct vm_area_struct *vma, void *cpu_addr,
1553                 dma_addr_t dma_addr, size_t size, unsigned long attrs)
1554 {
1555         return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs);
1556 }
1557 
1558 /*
1559  * free a page as defined by the above mapping.
1560  * Must not be called with IRQs disabled.
1561  */
1562 void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1563         dma_addr_t handle, unsigned long attrs, int coherent_flag)
1564 {
1565         struct page **pages;
1566         size = PAGE_ALIGN(size);
1567 
1568         if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) {
1569                 __iommu_free_atomic(dev, cpu_addr, handle, size, coherent_flag);
1570                 return;
1571         }
1572 
1573         pages = __iommu_get_pages(cpu_addr, attrs);
1574         if (!pages) {
1575                 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1576                 return;
1577         }
1578 
1579         if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0)
1580                 dma_common_free_remap(cpu_addr, size);
1581 
1582         __iommu_remove_mapping(dev, handle, size);
1583         __iommu_free_buffer(dev, pages, size, attrs);
1584 }
1585 
1586 void arm_iommu_free_attrs(struct device *dev, size_t size,
1587                     void *cpu_addr, dma_addr_t handle, unsigned long attrs)
1588 {
1589         __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, NORMAL);
1590 }
1591 
1592 void arm_coherent_iommu_free_attrs(struct device *dev, size_t size,
1593                     void *cpu_addr, dma_addr_t handle, unsigned long attrs)
1594 {
1595         __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, COHERENT);
1596 }
1597 
1598 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1599                                  void *cpu_addr, dma_addr_t dma_addr,
1600                                  size_t size, unsigned long attrs)
1601 {
1602         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1603         struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1604 
1605         if (!pages)
1606                 return -ENXIO;
1607 
1608         return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1609                                          GFP_KERNEL);
1610 }
1611 
1612 /*
1613  * Map a part of the scatter-gather list into contiguous io address space
1614  */
1615 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1616                           size_t size, dma_addr_t *handle,
1617                           enum dma_data_direction dir, unsigned long attrs,
1618                           bool is_coherent)
1619 {
1620         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1621         dma_addr_t iova, iova_base;
1622         int ret = 0;
1623         unsigned int count;
1624         struct scatterlist *s;
1625         int prot;
1626 
1627         size = PAGE_ALIGN(size);
1628         *handle = DMA_MAPPING_ERROR;
1629 
1630         iova_base = iova = __alloc_iova(mapping, size);
1631         if (iova == DMA_MAPPING_ERROR)
1632                 return -ENOMEM;
1633 
1634         for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1635                 phys_addr_t phys = page_to_phys(sg_page(s));
1636                 unsigned int len = PAGE_ALIGN(s->offset + s->length);
1637 
1638                 if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1639                         __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1640 
1641                 prot = __dma_info_to_prot(dir, attrs);
1642 
1643                 ret = iommu_map(mapping->domain, iova, phys, len, prot);
1644                 if (ret < 0)
1645                         goto fail;
1646                 count += len >> PAGE_SHIFT;
1647                 iova += len;
1648         }
1649         *handle = iova_base;
1650 
1651         return 0;
1652 fail:
1653         iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1654         __free_iova(mapping, iova_base, size);
1655         return ret;
1656 }
1657 
1658 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1659                      enum dma_data_direction dir, unsigned long attrs,
1660                      bool is_coherent)
1661 {
1662         struct scatterlist *s = sg, *dma = sg, *start = sg;
1663         int i, count = 0;
1664         unsigned int offset = s->offset;
1665         unsigned int size = s->offset + s->length;
1666         unsigned int max = dma_get_max_seg_size(dev);
1667 
1668         for (i = 1; i < nents; i++) {
1669                 s = sg_next(s);
1670 
1671                 s->dma_address = DMA_MAPPING_ERROR;
1672                 s->dma_length = 0;
1673 
1674                 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1675                         if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1676                             dir, attrs, is_coherent) < 0)
1677                                 goto bad_mapping;
1678 
1679                         dma->dma_address += offset;
1680                         dma->dma_length = size - offset;
1681 
1682                         size = offset = s->offset;
1683                         start = s;
1684                         dma = sg_next(dma);
1685                         count += 1;
1686                 }
1687                 size += s->length;
1688         }
1689         if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1690                 is_coherent) < 0)
1691                 goto bad_mapping;
1692 
1693         dma->dma_address += offset;
1694         dma->dma_length = size - offset;
1695 
1696         return count+1;
1697 
1698 bad_mapping:
1699         for_each_sg(sg, s, count, i)
1700                 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1701         return 0;
1702 }
1703 
1704 /**
1705  * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1706  * @dev: valid struct device pointer
1707  * @sg: list of buffers
1708  * @nents: number of buffers to map
1709  * @dir: DMA transfer direction
1710  *
1711  * Map a set of i/o coherent buffers described by scatterlist in streaming
1712  * mode for DMA. The scatter gather list elements are merged together (if
1713  * possible) and tagged with the appropriate dma address and length. They are
1714  * obtained via sg_dma_{address,length}.
1715  */
1716 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1717                 int nents, enum dma_data_direction dir, unsigned long attrs)
1718 {
1719         return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1720 }
1721 
1722 /**
1723  * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1724  * @dev: valid struct device pointer
1725  * @sg: list of buffers
1726  * @nents: number of buffers to map
1727  * @dir: DMA transfer direction
1728  *
1729  * Map a set of buffers described by scatterlist in streaming mode for DMA.
1730  * The scatter gather list elements are merged together (if possible) and
1731  * tagged with the appropriate dma address and length. They are obtained via
1732  * sg_dma_{address,length}.
1733  */
1734 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1735                 int nents, enum dma_data_direction dir, unsigned long attrs)
1736 {
1737         return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1738 }
1739 
1740 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1741                 int nents, enum dma_data_direction dir,
1742                 unsigned long attrs, bool is_coherent)
1743 {
1744         struct scatterlist *s;
1745         int i;
1746 
1747         for_each_sg(sg, s, nents, i) {
1748                 if (sg_dma_len(s))
1749                         __iommu_remove_mapping(dev, sg_dma_address(s),
1750                                                sg_dma_len(s));
1751                 if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1752                         __dma_page_dev_to_cpu(sg_page(s), s->offset,
1753                                               s->length, dir);
1754         }
1755 }
1756 
1757 /**
1758  * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1759  * @dev: valid struct device pointer
1760  * @sg: list of buffers
1761  * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1762  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1763  *
1764  * Unmap a set of streaming mode DMA translations.  Again, CPU access
1765  * rules concerning calls here are the same as for dma_unmap_single().
1766  */
1767 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1768                 int nents, enum dma_data_direction dir,
1769                 unsigned long attrs)
1770 {
1771         __iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1772 }
1773 
1774 /**
1775  * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1776  * @dev: valid struct device pointer
1777  * @sg: list of buffers
1778  * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1779  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1780  *
1781  * Unmap a set of streaming mode DMA translations.  Again, CPU access
1782  * rules concerning calls here are the same as for dma_unmap_single().
1783  */
1784 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1785                         enum dma_data_direction dir,
1786                         unsigned long attrs)
1787 {
1788         __iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1789 }
1790 
1791 /**
1792  * arm_iommu_sync_sg_for_cpu
1793  * @dev: valid struct device pointer
1794  * @sg: list of buffers
1795  * @nents: number of buffers to map (returned from dma_map_sg)
1796  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1797  */
1798 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1799                         int nents, enum dma_data_direction dir)
1800 {
1801         struct scatterlist *s;
1802         int i;
1803 
1804         for_each_sg(sg, s, nents, i)
1805                 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1806 
1807 }
1808 
1809 /**
1810  * arm_iommu_sync_sg_for_device
1811  * @dev: valid struct device pointer
1812  * @sg: list of buffers
1813  * @nents: number of buffers to map (returned from dma_map_sg)
1814  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1815  */
1816 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1817                         int nents, enum dma_data_direction dir)
1818 {
1819         struct scatterlist *s;
1820         int i;
1821 
1822         for_each_sg(sg, s, nents, i)
1823                 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1824 }
1825 
1826 
1827 /**
1828  * arm_coherent_iommu_map_page
1829  * @dev: valid struct device pointer
1830  * @page: page that buffer resides in
1831  * @offset: offset into page for start of buffer
1832  * @size: size of buffer to map
1833  * @dir: DMA transfer direction
1834  *
1835  * Coherent IOMMU aware version of arm_dma_map_page()
1836  */
1837 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1838              unsigned long offset, size_t size, enum dma_data_direction dir,
1839              unsigned long attrs)
1840 {
1841         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1842         dma_addr_t dma_addr;
1843         int ret, prot, len = PAGE_ALIGN(size + offset);
1844 
1845         dma_addr = __alloc_iova(mapping, len);
1846         if (dma_addr == DMA_MAPPING_ERROR)
1847                 return dma_addr;
1848 
1849         prot = __dma_info_to_prot(dir, attrs);
1850 
1851         ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
1852         if (ret < 0)
1853                 goto fail;
1854 
1855         return dma_addr + offset;
1856 fail:
1857         __free_iova(mapping, dma_addr, len);
1858         return DMA_MAPPING_ERROR;
1859 }
1860 
1861 /**
1862  * arm_iommu_map_page
1863  * @dev: valid struct device pointer
1864  * @page: page that buffer resides in
1865  * @offset: offset into page for start of buffer
1866  * @size: size of buffer to map
1867  * @dir: DMA transfer direction
1868  *
1869  * IOMMU aware version of arm_dma_map_page()
1870  */
1871 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1872              unsigned long offset, size_t size, enum dma_data_direction dir,
1873              unsigned long attrs)
1874 {
1875         if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1876                 __dma_page_cpu_to_dev(page, offset, size, dir);
1877 
1878         return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1879 }
1880 
1881 /**
1882  * arm_coherent_iommu_unmap_page
1883  * @dev: valid struct device pointer
1884  * @handle: DMA address of buffer
1885  * @size: size of buffer (same as passed to dma_map_page)
1886  * @dir: DMA transfer direction (same as passed to dma_map_page)
1887  *
1888  * Coherent IOMMU aware version of arm_dma_unmap_page()
1889  */
1890 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1891                 size_t size, enum dma_data_direction dir, unsigned long attrs)
1892 {
1893         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1894         dma_addr_t iova = handle & PAGE_MASK;
1895         int offset = handle & ~PAGE_MASK;
1896         int len = PAGE_ALIGN(size + offset);
1897 
1898         if (!iova)
1899                 return;
1900 
1901         iommu_unmap(mapping->domain, iova, len);
1902         __free_iova(mapping, iova, len);
1903 }
1904 
1905 /**
1906  * arm_iommu_unmap_page
1907  * @dev: valid struct device pointer
1908  * @handle: DMA address of buffer
1909  * @size: size of buffer (same as passed to dma_map_page)
1910  * @dir: DMA transfer direction (same as passed to dma_map_page)
1911  *
1912  * IOMMU aware version of arm_dma_unmap_page()
1913  */
1914 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1915                 size_t size, enum dma_data_direction dir, unsigned long attrs)
1916 {
1917         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1918         dma_addr_t iova = handle & PAGE_MASK;
1919         struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1920         int offset = handle & ~PAGE_MASK;
1921         int len = PAGE_ALIGN(size + offset);
1922 
1923         if (!iova)
1924                 return;
1925 
1926         if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1927                 __dma_page_dev_to_cpu(page, offset, size, dir);
1928 
1929         iommu_unmap(mapping->domain, iova, len);
1930         __free_iova(mapping, iova, len);
1931 }
1932 
1933 /**
1934  * arm_iommu_map_resource - map a device resource for DMA
1935  * @dev: valid struct device pointer
1936  * @phys_addr: physical address of resource
1937  * @size: size of resource to map
1938  * @dir: DMA transfer direction
1939  */
1940 static dma_addr_t arm_iommu_map_resource(struct device *dev,
1941                 phys_addr_t phys_addr, size_t size,
1942                 enum dma_data_direction dir, unsigned long attrs)
1943 {
1944         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1945         dma_addr_t dma_addr;
1946         int ret, prot;
1947         phys_addr_t addr = phys_addr & PAGE_MASK;
1948         unsigned int offset = phys_addr & ~PAGE_MASK;
1949         size_t len = PAGE_ALIGN(size + offset);
1950 
1951         dma_addr = __alloc_iova(mapping, len);
1952         if (dma_addr == DMA_MAPPING_ERROR)
1953                 return dma_addr;
1954 
1955         prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO;
1956 
1957         ret = iommu_map(mapping->domain, dma_addr, addr, len, prot);
1958         if (ret < 0)
1959                 goto fail;
1960 
1961         return dma_addr + offset;
1962 fail:
1963         __free_iova(mapping, dma_addr, len);
1964         return DMA_MAPPING_ERROR;
1965 }
1966 
1967 /**
1968  * arm_iommu_unmap_resource - unmap a device DMA resource
1969  * @dev: valid struct device pointer
1970  * @dma_handle: DMA address to resource
1971  * @size: size of resource to map
1972  * @dir: DMA transfer direction
1973  */
1974 static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle,
1975                 size_t size, enum dma_data_direction dir,
1976                 unsigned long attrs)
1977 {
1978         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1979         dma_addr_t iova = dma_handle & PAGE_MASK;
1980         unsigned int offset = dma_handle & ~PAGE_MASK;
1981         size_t len = PAGE_ALIGN(size + offset);
1982 
1983         if (!iova)
1984                 return;
1985 
1986         iommu_unmap(mapping->domain, iova, len);
1987         __free_iova(mapping, iova, len);
1988 }
1989 
1990 static void arm_iommu_sync_single_for_cpu(struct device *dev,
1991                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1992 {
1993         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1994         dma_addr_t iova = handle & PAGE_MASK;
1995         struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1996         unsigned int offset = handle & ~PAGE_MASK;
1997 
1998         if (!iova)
1999                 return;
2000 
2001         __dma_page_dev_to_cpu(page, offset, size, dir);
2002 }
2003 
2004 static void arm_iommu_sync_single_for_device(struct device *dev,
2005                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
2006 {
2007         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2008         dma_addr_t iova = handle & PAGE_MASK;
2009         struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
2010         unsigned int offset = handle & ~PAGE_MASK;
2011 
2012         if (!iova)
2013                 return;
2014 
2015         __dma_page_cpu_to_dev(page, offset, size, dir);
2016 }
2017 
2018 const struct dma_map_ops iommu_ops = {
2019         .alloc          = arm_iommu_alloc_attrs,
2020         .free           = arm_iommu_free_attrs,
2021         .mmap           = arm_iommu_mmap_attrs,
2022         .get_sgtable    = arm_iommu_get_sgtable,
2023 
2024         .map_page               = arm_iommu_map_page,
2025         .unmap_page             = arm_iommu_unmap_page,
2026         .sync_single_for_cpu    = arm_iommu_sync_single_for_cpu,
2027         .sync_single_for_device = arm_iommu_sync_single_for_device,
2028 
2029         .map_sg                 = arm_iommu_map_sg,
2030         .unmap_sg               = arm_iommu_unmap_sg,
2031         .sync_sg_for_cpu        = arm_iommu_sync_sg_for_cpu,
2032         .sync_sg_for_device     = arm_iommu_sync_sg_for_device,
2033 
2034         .map_resource           = arm_iommu_map_resource,
2035         .unmap_resource         = arm_iommu_unmap_resource,
2036 
2037         .dma_supported          = arm_dma_supported,
2038 };
2039 
2040 const struct dma_map_ops iommu_coherent_ops = {
2041         .alloc          = arm_coherent_iommu_alloc_attrs,
2042         .free           = arm_coherent_iommu_free_attrs,
2043         .mmap           = arm_coherent_iommu_mmap_attrs,
2044         .get_sgtable    = arm_iommu_get_sgtable,
2045 
2046         .map_page       = arm_coherent_iommu_map_page,
2047         .unmap_page     = arm_coherent_iommu_unmap_page,
2048 
2049         .map_sg         = arm_coherent_iommu_map_sg,
2050         .unmap_sg       = arm_coherent_iommu_unmap_sg,
2051 
2052         .map_resource   = arm_iommu_map_resource,
2053         .unmap_resource = arm_iommu_unmap_resource,
2054 
2055         .dma_supported          = arm_dma_supported,
2056 };
2057 
2058 /**
2059  * arm_iommu_create_mapping
2060  * @bus: pointer to the bus holding the client device (for IOMMU calls)
2061  * @base: start address of the valid IO address space
2062  * @size: maximum size of the valid IO address space
2063  *
2064  * Creates a mapping structure which holds information about used/unused
2065  * IO address ranges, which is required to perform memory allocation and
2066  * mapping with IOMMU aware functions.
2067  *
2068  * The client device need to be attached to the mapping with
2069  * arm_iommu_attach_device function.
2070  */
2071 struct dma_iommu_mapping *
2072 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size)
2073 {
2074         unsigned int bits = size >> PAGE_SHIFT;
2075         unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
2076         struct dma_iommu_mapping *mapping;
2077         int extensions = 1;
2078         int err = -ENOMEM;
2079 
2080         /* currently only 32-bit DMA address space is supported */
2081         if (size > DMA_BIT_MASK(32) + 1)
2082                 return ERR_PTR(-ERANGE);
2083 
2084         if (!bitmap_size)
2085                 return ERR_PTR(-EINVAL);
2086 
2087         if (bitmap_size > PAGE_SIZE) {
2088                 extensions = bitmap_size / PAGE_SIZE;
2089                 bitmap_size = PAGE_SIZE;
2090         }
2091 
2092         mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
2093         if (!mapping)
2094                 goto err;
2095 
2096         mapping->bitmap_size = bitmap_size;
2097         mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *),
2098                                    GFP_KERNEL);
2099         if (!mapping->bitmaps)
2100                 goto err2;
2101 
2102         mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
2103         if (!mapping->bitmaps[0])
2104                 goto err3;
2105 
2106         mapping->nr_bitmaps = 1;
2107         mapping->extensions = extensions;
2108         mapping->base = base;
2109         mapping->bits = BITS_PER_BYTE * bitmap_size;
2110 
2111         spin_lock_init(&mapping->lock);
2112 
2113         mapping->domain = iommu_domain_alloc(bus);
2114         if (!mapping->domain)
2115                 goto err4;
2116 
2117         kref_init(&mapping->kref);
2118         return mapping;
2119 err4:
2120         kfree(mapping->bitmaps[0]);
2121 err3:
2122         kfree(mapping->bitmaps);
2123 err2:
2124         kfree(mapping);
2125 err:
2126         return ERR_PTR(err);
2127 }
2128 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
2129 
2130 static void release_iommu_mapping(struct kref *kref)
2131 {
2132         int i;
2133         struct dma_iommu_mapping *mapping =
2134                 container_of(kref, struct dma_iommu_mapping, kref);
2135 
2136         iommu_domain_free(mapping->domain);
2137         for (i = 0; i < mapping->nr_bitmaps; i++)
2138                 kfree(mapping->bitmaps[i]);
2139         kfree(mapping->bitmaps);
2140         kfree(mapping);
2141 }
2142 
2143 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
2144 {
2145         int next_bitmap;
2146 
2147         if (mapping->nr_bitmaps >= mapping->extensions)
2148                 return -EINVAL;
2149 
2150         next_bitmap = mapping->nr_bitmaps;
2151         mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
2152                                                 GFP_ATOMIC);
2153         if (!mapping->bitmaps[next_bitmap])
2154                 return -ENOMEM;
2155 
2156         mapping->nr_bitmaps++;
2157 
2158         return 0;
2159 }
2160 
2161 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
2162 {
2163         if (mapping)
2164                 kref_put(&mapping->kref, release_iommu_mapping);
2165 }
2166 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
2167 
2168 static int __arm_iommu_attach_device(struct device *dev,
2169                                      struct dma_iommu_mapping *mapping)
2170 {
2171         int err;
2172 
2173         err = iommu_attach_device(mapping->domain, dev);
2174         if (err)
2175                 return err;
2176 
2177         kref_get(&mapping->kref);
2178         to_dma_iommu_mapping(dev) = mapping;
2179 
2180         pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
2181         return 0;
2182 }
2183 
2184 /**
2185  * arm_iommu_attach_device
2186  * @dev: valid struct device pointer
2187  * @mapping: io address space mapping structure (returned from
2188  *      arm_iommu_create_mapping)
2189  *
2190  * Attaches specified io address space mapping to the provided device.
2191  * This replaces the dma operations (dma_map_ops pointer) with the
2192  * IOMMU aware version.
2193  *
2194  * More than one client might be attached to the same io address space
2195  * mapping.
2196  */
2197 int arm_iommu_attach_device(struct device *dev,
2198                             struct dma_iommu_mapping *mapping)
2199 {
2200         int err;
2201 
2202         err = __arm_iommu_attach_device(dev, mapping);
2203         if (err)
2204                 return err;
2205 
2206         set_dma_ops(dev, &iommu_ops);
2207         return 0;
2208 }
2209 EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
2210 
2211 /**
2212  * arm_iommu_detach_device
2213  * @dev: valid struct device pointer
2214  *
2215  * Detaches the provided device from a previously attached map.
2216  * This overwrites the dma_ops pointer with appropriate non-IOMMU ops.
2217  */
2218 void arm_iommu_detach_device(struct device *dev)
2219 {
2220         struct dma_iommu_mapping *mapping;
2221 
2222         mapping = to_dma_iommu_mapping(dev);
2223         if (!mapping) {
2224                 dev_warn(dev, "Not attached\n");
2225                 return;
2226         }
2227 
2228         iommu_detach_device(mapping->domain, dev);
2229         kref_put(&mapping->kref, release_iommu_mapping);
2230         to_dma_iommu_mapping(dev) = NULL;
2231         set_dma_ops(dev, arm_get_dma_map_ops(dev->archdata.dma_coherent));
2232 
2233         pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2234 }
2235 EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2236 
2237 static const struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
2238 {
2239         return coherent ? &iommu_coherent_ops : &iommu_ops;
2240 }
2241 
2242 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2243                                     const struct iommu_ops *iommu)
2244 {
2245         struct dma_iommu_mapping *mapping;
2246 
2247         if (!iommu)
2248                 return false;
2249 
2250         mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
2251         if (IS_ERR(mapping)) {
2252                 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
2253                                 size, dev_name(dev));
2254                 return false;
2255         }
2256 
2257         if (__arm_iommu_attach_device(dev, mapping)) {
2258                 pr_warn("Failed to attached device %s to IOMMU_mapping\n",
2259                                 dev_name(dev));
2260                 arm_iommu_release_mapping(mapping);
2261                 return false;
2262         }
2263 
2264         return true;
2265 }
2266 
2267 static void arm_teardown_iommu_dma_ops(struct device *dev)
2268 {
2269         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2270 
2271         if (!mapping)
2272                 return;
2273 
2274         arm_iommu_detach_device(dev);
2275         arm_iommu_release_mapping(mapping);
2276 }
2277 
2278 #else
2279 
2280 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2281                                     const struct iommu_ops *iommu)
2282 {
2283         return false;
2284 }
2285 
2286 static void arm_teardown_iommu_dma_ops(struct device *dev) { }
2287 
2288 #define arm_get_iommu_dma_map_ops arm_get_dma_map_ops
2289 
2290 #endif  /* CONFIG_ARM_DMA_USE_IOMMU */
2291 
2292 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
2293                         const struct iommu_ops *iommu, bool coherent)
2294 {
2295         const struct dma_map_ops *dma_ops;
2296 
2297         dev->archdata.dma_coherent = coherent;
2298 #ifdef CONFIG_SWIOTLB
2299         dev->dma_coherent = coherent;
2300 #endif
2301 
2302         /*
2303          * Don't override the dma_ops if they have already been set. Ideally
2304          * this should be the only location where dma_ops are set, remove this
2305          * check when all other callers of set_dma_ops will have disappeared.
2306          */
2307         if (dev->dma_ops)
2308                 return;
2309 
2310         if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
2311                 dma_ops = arm_get_iommu_dma_map_ops(coherent);
2312         else
2313                 dma_ops = arm_get_dma_map_ops(coherent);
2314 
2315         set_dma_ops(dev, dma_ops);
2316 
2317 #ifdef CONFIG_XEN
2318         if (xen_initial_domain())
2319                 dev->dma_ops = &xen_swiotlb_dma_ops;
2320 #endif
2321         dev->archdata.dma_ops_setup = true;
2322 }
2323 
2324 void arch_teardown_dma_ops(struct device *dev)
2325 {
2326         if (!dev->archdata.dma_ops_setup)
2327                 return;
2328 
2329         arm_teardown_iommu_dma_ops(dev);
2330         /* Let arch_setup_dma_ops() start again from scratch upon re-probe */
2331         set_dma_ops(dev, NULL);
2332 }
2333 
2334 #ifdef CONFIG_SWIOTLB
2335 void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
2336                 size_t size, enum dma_data_direction dir)
2337 {
2338         __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
2339                               size, dir);
2340 }
2341 
2342 void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
2343                 size_t size, enum dma_data_direction dir)
2344 {
2345         __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
2346                               size, dir);
2347 }
2348 
2349 long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
2350                 dma_addr_t dma_addr)
2351 {
2352         return dma_to_pfn(dev, dma_addr);
2353 }
2354 
2355 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
2356                 gfp_t gfp, unsigned long attrs)
2357 {
2358         return __dma_alloc(dev, size, dma_handle, gfp,
2359                            __get_dma_pgprot(attrs, PAGE_KERNEL), false,
2360                            attrs, __builtin_return_address(0));
2361 }
2362 
2363 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
2364                 dma_addr_t dma_handle, unsigned long attrs)
2365 {
2366         __arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false);
2367 }
2368 #endif /* CONFIG_SWIOTLB */
/* [<][>][^][v][top][bottom][index][help] */
root/arch/arm/mm/dma-mapping.c

DEFINITIONS
