1030 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
1032 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,  in __alloc_iova()  argument
1038 	size_t mapping_size = mapping->bits << PAGE_SHIFT;  in __alloc_iova()
1049 	spin_lock_irqsave(&mapping->lock, flags);  in __alloc_iova()
1050 	for (i = 0; i < mapping->nr_bitmaps; i++) {  in __alloc_iova()
1051 		start = bitmap_find_next_zero_area(mapping->bitmaps[i],  in __alloc_iova()
1052 				mapping->bits, 0, count, align);  in __alloc_iova()
1054 		if (start > mapping->bits)  in __alloc_iova()
1057 		bitmap_set(mapping->bitmaps[i], start, count);  in __alloc_iova()
1066 	if (i == mapping->nr_bitmaps) {  in __alloc_iova()
1067 		if (extend_iommu_mapping(mapping)) {  in __alloc_iova()
1068 			spin_unlock_irqrestore(&mapping->lock, flags);  in __alloc_iova()
1072 		start = bitmap_find_next_zero_area(mapping->bitmaps[i],  in __alloc_iova()
1073 				mapping->bits, 0, count, align);  in __alloc_iova()
1075 		if (start > mapping->bits) {  in __alloc_iova()
1076 			spin_unlock_irqrestore(&mapping->lock, flags);  in __alloc_iova()
1080 		bitmap_set(mapping->bitmaps[i], start, count);  in __alloc_iova()
1082 	spin_unlock_irqrestore(&mapping->lock, flags);  in __alloc_iova()
1084 	iova = mapping->base + (mapping_size * i);  in __alloc_iova()
1090 static inline void __free_iova(struct dma_iommu_mapping *mapping,  in __free_iova()  argument
1094 	size_t mapping_size = mapping->bits << PAGE_SHIFT;  in __free_iova()
1102 	bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;  in __free_iova()
1103 	BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);  in __free_iova()
1105 	bitmap_base = mapping->base + mapping_size * bitmap_index;  in __free_iova()
1120 	spin_lock_irqsave(&mapping->lock, flags);  in __free_iova()
1121 	bitmap_clear(mapping->bitmaps[bitmap_index], start, count);  in __free_iova()
1122 	spin_unlock_irqrestore(&mapping->lock, flags);  in __free_iova()
1249 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in __iommu_create_mapping()  local
1254 	dma_addr = __alloc_iova(mapping, size);  in __iommu_create_mapping()
1271 		ret = iommu_map(mapping->domain, iova, phys, len,  in __iommu_create_mapping()
1280 	iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);  in __iommu_create_mapping()
1281 	__free_iova(mapping, dma_addr, size);  in __iommu_create_mapping()
1287 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in __iommu_remove_mapping()  local
1296 	iommu_unmap(mapping->domain, iova, size);  in __iommu_remove_mapping()
1297 	__free_iova(mapping, iova, size);  in __iommu_remove_mapping()
1509 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in __map_sg_chunk()  local
1519 	iova_base = iova = __alloc_iova(mapping, size);  in __map_sg_chunk()
1533 		ret = iommu_map(mapping->domain, iova, phys, len, prot);  in __map_sg_chunk()
1543 	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);  in __map_sg_chunk()
1544 	__free_iova(mapping, iova_base, size);  in __map_sg_chunk()
1730 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_coherent_iommu_map_page()  local
1734 	dma_addr = __alloc_iova(mapping, len);  in arm_coherent_iommu_map_page()
1740 	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);  in arm_coherent_iommu_map_page()
1746 	__free_iova(mapping, dma_addr, len);  in arm_coherent_iommu_map_page()
1783 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_coherent_iommu_unmap_page()  local
1791 	iommu_unmap(mapping->domain, iova, len);  in arm_coherent_iommu_unmap_page()
1792 	__free_iova(mapping, iova, len);  in arm_coherent_iommu_unmap_page()
1808 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_unmap_page()  local
1810 	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));  in arm_iommu_unmap_page()
1820 	iommu_unmap(mapping->domain, iova, len);  in arm_iommu_unmap_page()
1821 	__free_iova(mapping, iova, len);  in arm_iommu_unmap_page()
1827 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_sync_single_for_cpu()  local
1829 	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));  in arm_iommu_sync_single_for_cpu()
1841 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_sync_single_for_device()  local
1843 	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));  in arm_iommu_sync_single_for_device()
1904 	struct dma_iommu_mapping *mapping;  in arm_iommu_create_mapping()  local
1920 	mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);  in arm_iommu_create_mapping()
1921 	if (!mapping)  in arm_iommu_create_mapping()
1924 	mapping->bitmap_size = bitmap_size;  in arm_iommu_create_mapping()
1925 	mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *),  in arm_iommu_create_mapping()
1927 	if (!mapping->bitmaps)  in arm_iommu_create_mapping()
1930 	mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);  in arm_iommu_create_mapping()
1931 	if (!mapping->bitmaps[0])  in arm_iommu_create_mapping()
1934 	mapping->nr_bitmaps = 1;  in arm_iommu_create_mapping()
1935 	mapping->extensions = extensions;  in arm_iommu_create_mapping()
1936 	mapping->base = base;  in arm_iommu_create_mapping()
1937 	mapping->bits = BITS_PER_BYTE * bitmap_size;  in arm_iommu_create_mapping()
1939 	spin_lock_init(&mapping->lock);  in arm_iommu_create_mapping()
1941 	mapping->domain = iommu_domain_alloc(bus);  in arm_iommu_create_mapping()
1942 	if (!mapping->domain)  in arm_iommu_create_mapping()
1945 	kref_init(&mapping->kref);  in arm_iommu_create_mapping()
1946 	return mapping;  in arm_iommu_create_mapping()
1948 	kfree(mapping->bitmaps[0]);  in arm_iommu_create_mapping()
1950 	kfree(mapping->bitmaps);  in arm_iommu_create_mapping()
1952 	kfree(mapping);  in arm_iommu_create_mapping()
1961 	struct dma_iommu_mapping *mapping =  in release_iommu_mapping()  local
1964 	iommu_domain_free(mapping->domain);  in release_iommu_mapping()
1965 	for (i = 0; i < mapping->nr_bitmaps; i++)  in release_iommu_mapping()
1966 		kfree(mapping->bitmaps[i]);  in release_iommu_mapping()
1967 	kfree(mapping->bitmaps);  in release_iommu_mapping()
1968 	kfree(mapping);  in release_iommu_mapping()
1971 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)  in extend_iommu_mapping()  argument
1975 	if (mapping->nr_bitmaps >= mapping->extensions)  in extend_iommu_mapping()
1978 	next_bitmap = mapping->nr_bitmaps;  in extend_iommu_mapping()
1979 	mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,  in extend_iommu_mapping()
1981 	if (!mapping->bitmaps[next_bitmap])  in extend_iommu_mapping()
1984 	mapping->nr_bitmaps++;  in extend_iommu_mapping()
1989 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)  in arm_iommu_release_mapping()  argument
1991 	if (mapping)  in arm_iommu_release_mapping()
1992 		kref_put(&mapping->kref, release_iommu_mapping);  in arm_iommu_release_mapping()
1997 				     struct dma_iommu_mapping *mapping)  in __arm_iommu_attach_device()  argument
2001 	err = iommu_attach_device(mapping->domain, dev);  in __arm_iommu_attach_device()
2005 	kref_get(&mapping->kref);  in __arm_iommu_attach_device()
2006 	to_dma_iommu_mapping(dev) = mapping;  in __arm_iommu_attach_device()
2026 			    struct dma_iommu_mapping *mapping)  in arm_iommu_attach_device()  argument
2030 	err = __arm_iommu_attach_device(dev, mapping);  in arm_iommu_attach_device()
2041 	struct dma_iommu_mapping *mapping;  in __arm_iommu_detach_device()  local
2043 	mapping = to_dma_iommu_mapping(dev);  in __arm_iommu_detach_device()
2044 	if (!mapping) {  in __arm_iommu_detach_device()
2049 	iommu_detach_device(mapping->domain, dev);  in __arm_iommu_detach_device()
2050 	kref_put(&mapping->kref, release_iommu_mapping);  in __arm_iommu_detach_device()
2078 	struct dma_iommu_mapping *mapping;  in arm_setup_iommu_dma_ops()  local
2083 	mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);  in arm_setup_iommu_dma_ops()
2084 	if (IS_ERR(mapping)) {  in arm_setup_iommu_dma_ops()
2090 	if (__arm_iommu_attach_device(dev, mapping)) {  in arm_setup_iommu_dma_ops()
2093 		arm_iommu_release_mapping(mapping);  in arm_setup_iommu_dma_ops()
2102 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_teardown_iommu_dma_ops()  local
2104 	if (!mapping)  in arm_teardown_iommu_dma_ops()
2108 	arm_iommu_release_mapping(mapping);  in arm_teardown_iommu_dma_ops()