46 	struct iova_domain *iovad;  in iommu_get_dma_cookie()  local
51 	iovad = kzalloc(sizeof(*iovad), GFP_KERNEL);  in iommu_get_dma_cookie()
52 	domain->iova_cookie = iovad;  in iommu_get_dma_cookie()
54 	return iovad ? 0 : -ENOMEM;  in iommu_get_dma_cookie()
66 	struct iova_domain *iovad = domain->iova_cookie;  in iommu_put_dma_cookie()  local
68 	if (!iovad)  in iommu_put_dma_cookie()
71 	put_iova_domain(iovad);  in iommu_put_dma_cookie()
72 	kfree(iovad);  in iommu_put_dma_cookie()
90 	struct iova_domain *iovad = domain->iova_cookie;  in iommu_dma_init_domain()  local
93 	if (!iovad)  in iommu_dma_init_domain()
116 	if (iovad->start_pfn) {  in iommu_dma_init_domain()
117 		if (1UL << order != iovad->granule ||  in iommu_dma_init_domain()
118 		    base_pfn != iovad->start_pfn ||  in iommu_dma_init_domain()
119 		    end_pfn < iovad->dma_32bit_pfn) {  in iommu_dma_init_domain()
123 		iovad->dma_32bit_pfn = end_pfn;  in iommu_dma_init_domain()
125 		init_iova_domain(iovad, 1UL << order, base_pfn, end_pfn);  in iommu_dma_init_domain()
154 static struct iova *__alloc_iova(struct iova_domain *iovad, size_t size,  in __alloc_iova()  argument
157 	unsigned long shift = iova_shift(iovad);  in __alloc_iova()
158 	unsigned long length = iova_align(iovad, size) >> shift;  in __alloc_iova()
164 	return alloc_iova(iovad, length, dma_limit >> shift, true);  in __alloc_iova()
170 	struct iova_domain *iovad = domain->iova_cookie;  in __iommu_dma_unmap()  local
171 	unsigned long shift = iova_shift(iovad);  in __iommu_dma_unmap()
173 	struct iova *iova = find_iova(iovad, pfn);  in __iommu_dma_unmap()
183 	__free_iova(iovad, iova);  in __iommu_dma_unmap()
286 	struct iova_domain *iovad = domain->iova_cookie;  in iommu_dma_alloc()  local
299 	iova = __alloc_iova(iovad, size, dev->coherent_dma_mask);  in iommu_dma_alloc()
303 	size = iova_align(iovad, size);  in iommu_dma_alloc()
319 	dma_addr = iova_dma_addr(iovad, iova);  in iommu_dma_alloc()
331 	__free_iova(iovad, iova);  in iommu_dma_alloc()
367 	struct iova_domain *iovad = domain->iova_cookie;  in iommu_dma_map_page()  local
369 	size_t iova_off = iova_offset(iovad, phys);  in iommu_dma_map_page()
370 	size_t len = iova_align(iovad, size + iova_off);  in iommu_dma_map_page()
371 	struct iova *iova = __alloc_iova(iovad, len, dma_get_mask(dev));  in iommu_dma_map_page()
376 	dma_addr = iova_dma_addr(iovad, iova);  in iommu_dma_map_page()
378 		__free_iova(iovad, iova);  in iommu_dma_map_page()
444 	struct iova_domain *iovad = domain->iova_cookie;  in iommu_dma_map_sg()  local
458 		size_t s_offset = iova_offset(iovad, s->offset);  in iommu_dma_map_sg()
464 		s_length = iova_align(iovad, s_length + s_offset);  in iommu_dma_map_sg()
485 	iova = __alloc_iova(iovad, iova_len, dma_get_mask(dev));  in iommu_dma_map_sg()
493 	dma_addr = iova_dma_addr(iovad, iova);  in iommu_dma_map_sg()
500 	__free_iova(iovad, iova);  in iommu_dma_map_sg()