52 		struct iova *curr_iova =  in __get_cached_rbnode()
53 			container_of(iovad->cached32_node, struct iova, node);  in __get_cached_rbnode()
61 	unsigned long limit_pfn, struct iova *new)  in __cached_rbnode_insert_update()
69 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)  in __cached_rbnode_delete_update()
71 	struct iova *cached_iova;  in __cached_rbnode_delete_update()
77 	cached_iova = container_of(curr, struct iova, node);  in __cached_rbnode_delete_update()
81 		struct iova *iova = container_of(node, struct iova, node);  in __cached_rbnode_delete_update()  local
84 		if (node && iova->pfn_lo < iovad->dma_32bit_pfn)  in __cached_rbnode_delete_update()
103 			struct iova *new, bool size_aligned)  in __alloc_and_insert_iova_range()
116 		struct iova *curr_iova = container_of(curr, struct iova, node);  in __alloc_and_insert_iova_range()
162 			struct iova *this = container_of(*entry,  in __alloc_and_insert_iova_range()
163 							struct iova, node);  in __alloc_and_insert_iova_range()
187 iova_insert_rbtree(struct rb_root *root, struct iova *iova)  in iova_insert_rbtree()  argument
192 		struct iova *this = container_of(*new, struct iova, node);  in iova_insert_rbtree()
196 		if (iova->pfn_lo < this->pfn_lo)  in iova_insert_rbtree()
198 		else if (iova->pfn_lo > this->pfn_lo)  in iova_insert_rbtree()
204 	rb_link_node(&iova->node, parent, new);  in iova_insert_rbtree()
205 	rb_insert_color(&iova->node, root);  in iova_insert_rbtree()
212 struct iova *alloc_iova_mem(void)  in alloc_iova_mem()
218 void free_iova_mem(struct iova *iova)  in free_iova_mem()  argument
220 	kmem_cache_free(iova_cache, iova);  in free_iova_mem()
229 			"iommu_iova", sizeof(struct iova), 0,  in iova_cache_get()
270 struct iova *
275 	struct iova *new_iova;  in alloc_iova()
301 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)  in find_iova()
310 		struct iova *iova = container_of(node, struct iova, node);  in find_iova()  local
313 		if ((pfn >= iova->pfn_lo) && (pfn <= iova->pfn_hi)) {  in find_iova()
321 			return iova;  in find_iova()
324 		if (pfn < iova->pfn_lo)  in find_iova()
326 		else if (pfn > iova->pfn_lo)  in find_iova()
342 __free_iova(struct iova_domain *iovad, struct iova *iova)  in __free_iova()  argument
347 	__cached_rbnode_delete_update(iovad, iova);  in __free_iova()
348 	rb_erase(&iova->node, &iovad->rbroot);  in __free_iova()
350 	free_iova_mem(iova);  in __free_iova()
364 	struct iova *iova = find_iova(iovad, pfn);  in free_iova()  local
366 	if (iova)  in free_iova()
367 		__free_iova(iovad, iova);  in free_iova()
385 		struct iova *iova = container_of(node, struct iova, node);  in put_iova_domain()  local
388 		free_iova_mem(iova);  in put_iova_domain()
399 	struct iova *iova = container_of(node, struct iova, node);  in __is_range_overlap()  local
401 	if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo))  in __is_range_overlap()
406 static inline struct iova *
409 	struct iova *iova;  in alloc_and_init_iova()  local
411 	iova = alloc_iova_mem();  in alloc_and_init_iova()
412 	if (iova) {  in alloc_and_init_iova()
413 		iova->pfn_lo = pfn_lo;  in alloc_and_init_iova()
414 		iova->pfn_hi = pfn_hi;  in alloc_and_init_iova()
417 	return iova;  in alloc_and_init_iova()
420 static struct iova *
424 	struct iova *iova;  in __insert_new_range()  local
426 	iova = alloc_and_init_iova(pfn_lo, pfn_hi);  in __insert_new_range()
427 	if (iova)  in __insert_new_range()
428 		iova_insert_rbtree(&iovad->rbroot, iova);  in __insert_new_range()
430 	return iova;  in __insert_new_range()
434 __adjust_overlap_range(struct iova *iova,  in __adjust_overlap_range()  argument
437 	if (*pfn_lo < iova->pfn_lo)  in __adjust_overlap_range()
438 		iova->pfn_lo = *pfn_lo;  in __adjust_overlap_range()
439 	if (*pfn_hi > iova->pfn_hi)  in __adjust_overlap_range()
440 		*pfn_lo = iova->pfn_hi + 1;  in __adjust_overlap_range()
451 struct iova *
457 	struct iova *iova;  in reserve_iova()  local
463 			iova = container_of(node, struct iova, node);  in reserve_iova()
464 			__adjust_overlap_range(iova, &pfn_lo, &pfn_hi);  in reserve_iova()
465 			if ((pfn_lo >= iova->pfn_lo) &&  in reserve_iova()
466 				(pfn_hi <= iova->pfn_hi))  in reserve_iova()
477 	iova = __insert_new_range(iovad, pfn_lo, pfn_hi);  in reserve_iova()
481 	return iova;  in reserve_iova()
500 		struct iova *iova = container_of(node, struct iova, node);  in copy_reserved_iova()  local
501 		struct iova *new_iova;  in copy_reserved_iova()
503 		new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi);  in copy_reserved_iova()
506 				iova->pfn_lo, iova->pfn_lo);  in copy_reserved_iova()
512 struct iova *
513 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,  in split_and_remove_iova()  argument
517 	struct iova *prev = NULL, *next = NULL;  in split_and_remove_iova()
520 	if (iova->pfn_lo < pfn_lo) {  in split_and_remove_iova()
521 		prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1);  in split_and_remove_iova()
525 	if (iova->pfn_hi > pfn_hi) {  in split_and_remove_iova()
526 		next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi);  in split_and_remove_iova()
531 	__cached_rbnode_delete_update(iovad, iova);  in split_and_remove_iova()
532 	rb_erase(&iova->node, &iovad->rbroot);  in split_and_remove_iova()
536 		iova->pfn_lo = pfn_lo;  in split_and_remove_iova()
540 		iova->pfn_hi = pfn_hi;  in split_and_remove_iova()
544 	return iova;  in split_and_remove_iova()