30 					 sizeof(struct iova),  in iommu_iova_cache_init()
47 struct iova *alloc_iova_mem(void)  in alloc_iova_mem()
52 void free_iova_mem(struct iova *iova)  in free_iova_mem()  argument
54 	kmem_cache_free(iommu_iova_cache, iova);  in free_iova_mem()
84 		struct iova *curr_iova =  in __get_cached_rbnode()
85 			container_of(iovad->cached32_node, struct iova, node);  in __get_cached_rbnode()
93 	unsigned long limit_pfn, struct iova *new)  in __cached_rbnode_insert_update()
101 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)  in __cached_rbnode_delete_update()
103 	struct iova *cached_iova;  in __cached_rbnode_delete_update()
109 	cached_iova = container_of(curr, struct iova, node);  in __cached_rbnode_delete_update()
113 		struct iova *iova = container_of(node, struct iova, node);  in __cached_rbnode_delete_update()  local
116 		if (node && iova->pfn_lo < iovad->dma_32bit_pfn)  in __cached_rbnode_delete_update()
140 			struct iova *new, bool size_aligned)  in __alloc_and_insert_iova_range()
153 		struct iova *curr_iova = container_of(curr, struct iova, node);  in __alloc_and_insert_iova_range()
199 			struct iova *this = container_of(*entry,  in __alloc_and_insert_iova_range()
200 							struct iova, node);  in __alloc_and_insert_iova_range()
224 iova_insert_rbtree(struct rb_root *root, struct iova *iova)  in iova_insert_rbtree()  argument
229 		struct iova *this = container_of(*new, struct iova, node);  in iova_insert_rbtree()
232 		if (iova->pfn_lo < this->pfn_lo)  in iova_insert_rbtree()
234 		else if (iova->pfn_lo > this->pfn_lo)  in iova_insert_rbtree()
240 	rb_link_node(&iova->node, parent, new);  in iova_insert_rbtree()
241 	rb_insert_color(&iova->node, root);  in iova_insert_rbtree()
255 struct iova *
260 	struct iova *new_iova;  in alloc_iova()
291 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)  in find_iova()
300 		struct iova *iova = container_of(node, struct iova, node);  in find_iova()  local
303 		if ((pfn >= iova->pfn_lo) && (pfn <= iova->pfn_hi)) {  in find_iova()
311 			return iova;  in find_iova()
314 		if (pfn < iova->pfn_lo)  in find_iova()
316 		else if (pfn > iova->pfn_lo)  in find_iova()
331 __free_iova(struct iova_domain *iovad, struct iova *iova)  in __free_iova()  argument
336 	__cached_rbnode_delete_update(iovad, iova);  in __free_iova()
337 	rb_erase(&iova->node, &iovad->rbroot);  in __free_iova()
339 	free_iova_mem(iova);  in __free_iova()
352 	struct iova *iova = find_iova(iovad, pfn);  in free_iova()  local
353 	if (iova)  in free_iova()
354 		__free_iova(iovad, iova);  in free_iova()
371 		struct iova *iova = container_of(node, struct iova, node);  in put_iova_domain()  local
373 		free_iova_mem(iova);  in put_iova_domain()
383 	struct iova *iova = container_of(node, struct iova, node);  in __is_range_overlap()  local
385 	if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo))  in __is_range_overlap()
390 static inline struct iova *
393 	struct iova *iova;  in alloc_and_init_iova()  local
395 	iova = alloc_iova_mem();  in alloc_and_init_iova()
396 	if (iova) {  in alloc_and_init_iova()
397 		iova->pfn_lo = pfn_lo;  in alloc_and_init_iova()
398 		iova->pfn_hi = pfn_hi;  in alloc_and_init_iova()
401 	return iova;  in alloc_and_init_iova()
404 static struct iova *
408 	struct iova *iova;  in __insert_new_range()  local
410 	iova = alloc_and_init_iova(pfn_lo, pfn_hi);  in __insert_new_range()
411 	if (iova)  in __insert_new_range()
412 		iova_insert_rbtree(&iovad->rbroot, iova);  in __insert_new_range()
414 	return iova;  in __insert_new_range()
418 __adjust_overlap_range(struct iova *iova,  in __adjust_overlap_range()  argument
421 	if (*pfn_lo < iova->pfn_lo)  in __adjust_overlap_range()
422 		iova->pfn_lo = *pfn_lo;  in __adjust_overlap_range()
423 	if (*pfn_hi > iova->pfn_hi)  in __adjust_overlap_range()
424 		*pfn_lo = iova->pfn_hi + 1;  in __adjust_overlap_range()
435 struct iova *
441 	struct iova *iova;  in reserve_iova()  local
447 			iova = container_of(node, struct iova, node);  in reserve_iova()
448 			__adjust_overlap_range(iova, &pfn_lo, &pfn_hi);  in reserve_iova()
449 			if ((pfn_lo >= iova->pfn_lo) &&  in reserve_iova()
450 				(pfn_hi <= iova->pfn_hi))  in reserve_iova()
461 	iova = __insert_new_range(iovad, pfn_lo, pfn_hi);  in reserve_iova()
465 	return iova;  in reserve_iova()
483 		struct iova *iova = container_of(node, struct iova, node);  in copy_reserved_iova()  local
484 		struct iova *new_iova;  in copy_reserved_iova()
485 		new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi);  in copy_reserved_iova()
488 				iova->pfn_lo, iova->pfn_lo);  in copy_reserved_iova()
493 struct iova *
494 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,  in split_and_remove_iova()  argument
498 	struct iova *prev = NULL, *next = NULL;  in split_and_remove_iova()
501 	if (iova->pfn_lo < pfn_lo) {  in split_and_remove_iova()
502 		prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1);  in split_and_remove_iova()
506 	if (iova->pfn_hi > pfn_hi) {  in split_and_remove_iova()
507 		next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi);  in split_and_remove_iova()
512 	__cached_rbnode_delete_update(iovad, iova);  in split_and_remove_iova()
513 	rb_erase(&iova->node, &iovad->rbroot);  in split_and_remove_iova()
517 		iova->pfn_lo = pfn_lo;  in split_and_remove_iova()
521 		iova->pfn_hi = pfn_hi;  in split_and_remove_iova()
525 	return iova;  in split_and_remove_iova()