140 static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl)  in tce_iommu_userspace_view_alloc()  argument
142 	unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) *  in tce_iommu_userspace_view_alloc()
143 			tbl->it_size, PAGE_SIZE);  in tce_iommu_userspace_view_alloc()
147 	BUG_ON(tbl->it_userspace);  in tce_iommu_userspace_view_alloc()
158 	tbl->it_userspace = uas;  in tce_iommu_userspace_view_alloc()
163 static void tce_iommu_userspace_view_free(struct iommu_table *tbl)  in tce_iommu_userspace_view_free()  argument
165 	unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) *  in tce_iommu_userspace_view_free()
166 			tbl->it_size, PAGE_SIZE);  in tce_iommu_userspace_view_free()
168 	if (!tbl->it_userspace)  in tce_iommu_userspace_view_free()
171 	vfree(tbl->it_userspace);  in tce_iommu_userspace_view_free()
172 	tbl->it_userspace = NULL;  in tce_iommu_userspace_view_free()
197 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_find_table()  local
199 		if (tbl) {  in tce_iommu_find_table()
200 			unsigned long entry = ioba >> tbl->it_page_shift;  in tce_iommu_find_table()
201 			unsigned long start = tbl->it_offset;  in tce_iommu_find_table()
202 			unsigned long end = start + tbl->it_size;  in tce_iommu_find_table()
205 				*ptbl = tbl;  in tce_iommu_find_table()
327 		struct iommu_table *tbl,
329 static void tce_iommu_free_table(struct iommu_table *tbl);
350 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_release()  local
352 		if (!tbl)  in tce_iommu_release()
355 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_release()
356 		tce_iommu_free_table(tbl);  in tce_iommu_release()
393 static void tce_iommu_unuse_page_v2(struct iommu_table *tbl,  in tce_iommu_unuse_page_v2()  argument
399 	unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);  in tce_iommu_unuse_page_v2()
404 	ret = tce_iommu_prereg_ua_to_hpa(*pua, IOMMU_PAGE_SIZE(tbl),  in tce_iommu_unuse_page_v2()
416 		struct iommu_table *tbl,  in tce_iommu_clear()  argument
426 		ret = iommu_tce_xchg(tbl, entry, &oldhpa, &direction);  in tce_iommu_clear()
434 			tce_iommu_unuse_page_v2(tbl, entry);  in tce_iommu_clear()
459 		struct iommu_table *tbl,  in tce_iommu_build()  argument
469 		unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;  in tce_iommu_build()
476 		if (!tce_page_is_contained(page, tbl->it_page_shift)) {  in tce_iommu_build()
483 		ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp);  in tce_iommu_build()
487 					__func__, entry << tbl->it_page_shift,  in tce_iommu_build()
495 		tce += IOMMU_PAGE_SIZE(tbl);  in tce_iommu_build()
499 		tce_iommu_clear(container, tbl, entry, i);  in tce_iommu_build()
505 		struct iommu_table *tbl,  in tce_iommu_build_v2()  argument
516 		unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl,  in tce_iommu_build_v2()
519 		ret = tce_iommu_prereg_ua_to_hpa(tce, IOMMU_PAGE_SIZE(tbl),  in tce_iommu_build_v2()
525 		if (!tce_page_is_contained(page, tbl->it_page_shift)) {  in tce_iommu_build_v2()
531 		hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;  in tce_iommu_build_v2()
538 		ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp);  in tce_iommu_build_v2()
541 			tce_iommu_unuse_page_v2(tbl, entry + i);  in tce_iommu_build_v2()
543 					__func__, entry << tbl->it_page_shift,  in tce_iommu_build_v2()
549 			tce_iommu_unuse_page_v2(tbl, entry + i);  in tce_iommu_build_v2()
553 		tce += IOMMU_PAGE_SIZE(tbl);  in tce_iommu_build_v2()
557 		tce_iommu_clear(container, tbl, entry, i);  in tce_iommu_build_v2()
599 static void tce_iommu_free_table(struct iommu_table *tbl)  in tce_iommu_free_table()  argument
601 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;  in tce_iommu_free_table()
603 	tce_iommu_userspace_view_free(tbl);  in tce_iommu_free_table()
604 	tbl->it_ops->free(tbl);  in tce_iommu_free_table()
614 	struct iommu_table *tbl = NULL;  in tce_iommu_create_window()  local
638 			page_shift, window_size, levels, &tbl);  in tce_iommu_create_window()
642 	BUG_ON(!tbl->it_ops->free);  in tce_iommu_create_window()
651 		ret = table_group->ops->set_window(table_group, num, tbl);  in tce_iommu_create_window()
656 	container->tables[num] = tbl;  in tce_iommu_create_window()
659 	*start_addr = tbl->it_offset << tbl->it_page_shift;  in tce_iommu_create_window()
668 	tce_iommu_free_table(tbl);  in tce_iommu_create_window()
677 	struct iommu_table *tbl;  in tce_iommu_remove_window()  local
681 	num = tce_iommu_find_table(container, start_addr, &tbl);  in tce_iommu_remove_window()
685 	BUG_ON(!tbl->it_size);  in tce_iommu_remove_window()
705 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_remove_window()
706 	tce_iommu_free_table(tbl);  in tce_iommu_remove_window()
783 		struct iommu_table *tbl = NULL;  in tce_iommu_ioctl()  local
802 		num = tce_iommu_find_table(container, param.iova, &tbl);  in tce_iommu_ioctl()
806 		if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||  in tce_iommu_ioctl()
807 				(param.vaddr & ~IOMMU_PAGE_MASK(tbl)))  in tce_iommu_ioctl()
823 		ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);  in tce_iommu_ioctl()
828 			ret = tce_iommu_build_v2(container, tbl,  in tce_iommu_ioctl()
829 					param.iova >> tbl->it_page_shift,  in tce_iommu_ioctl()
831 					param.size >> tbl->it_page_shift,  in tce_iommu_ioctl()
834 			ret = tce_iommu_build(container, tbl,  in tce_iommu_ioctl()
835 					param.iova >> tbl->it_page_shift,  in tce_iommu_ioctl()
837 					param.size >> tbl->it_page_shift,  in tce_iommu_ioctl()
840 		iommu_flush_tce(tbl);  in tce_iommu_ioctl()
846 		struct iommu_table *tbl = NULL;  in tce_iommu_ioctl()  local
865 		num = tce_iommu_find_table(container, param.iova, &tbl);  in tce_iommu_ioctl()
869 		if (param.size & ~IOMMU_PAGE_MASK(tbl))  in tce_iommu_ioctl()
872 		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,  in tce_iommu_ioctl()
873 				param.size >> tbl->it_page_shift);  in tce_iommu_ioctl()
877 		ret = tce_iommu_clear(container, tbl,  in tce_iommu_ioctl()
878 				param.iova >> tbl->it_page_shift,  in tce_iommu_ioctl()
879 				param.size >> tbl->it_page_shift);  in tce_iommu_ioctl()
880 		iommu_flush_tce(tbl);  in tce_iommu_ioctl()
1042 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_release_ownership()  local
1044 		if (!tbl)  in tce_iommu_release_ownership()
1047 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_release_ownership()
1048 		tce_iommu_userspace_view_free(tbl);  in tce_iommu_release_ownership()
1049 		if (tbl->it_map)  in tce_iommu_release_ownership()
1050 			iommu_release_ownership(tbl);  in tce_iommu_release_ownership()
1062 		struct iommu_table *tbl = table_group->tables[i];  in tce_iommu_take_ownership()  local
1064 		if (!tbl || !tbl->it_map)  in tce_iommu_take_ownership()
1067 		rc = tce_iommu_userspace_view_alloc(tbl);  in tce_iommu_take_ownership()
1069 			rc = iommu_take_ownership(tbl);  in tce_iommu_take_ownership()
1106 	struct iommu_table *tbl = NULL;  in tce_iommu_take_ownership_ddw()  local
1128 				&tbl);  in tce_iommu_take_ownership_ddw()
1132 			container->tables[0] = tbl;  in tce_iommu_take_ownership_ddw()
1137 		tbl = container->tables[i];  in tce_iommu_take_ownership_ddw()
1139 		if (!tbl)  in tce_iommu_take_ownership_ddw()
1143 		ret = table_group->ops->set_window(table_group, i, tbl);  in tce_iommu_take_ownership_ddw()