105 static long tce_iommu_unregister_pages(struct tce_container *container,  in tce_iommu_unregister_pages()  argument
120 static long tce_iommu_register_pages(struct tce_container *container,  in tce_iommu_register_pages()  argument
135 	container->enabled = true;  in tce_iommu_register_pages()
186 static inline bool tce_groups_attached(struct tce_container *container)  in tce_groups_attached()  argument
188 	return !list_empty(&container->group_list);  in tce_groups_attached()
191 static long tce_iommu_find_table(struct tce_container *container,  in tce_iommu_find_table()  argument
197 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_find_table()
214 static int tce_iommu_find_free_table(struct tce_container *container)  in tce_iommu_find_free_table()  argument
219 		if (!container->tables[i])  in tce_iommu_find_free_table()
226 static int tce_iommu_enable(struct tce_container *container)  in tce_iommu_enable()  argument
236 	if (container->enabled)  in tce_iommu_enable()
268 	if (!tce_groups_attached(container))  in tce_iommu_enable()
271 	tcegrp = list_first_entry(&container->group_list,  in tce_iommu_enable()
285 	container->locked_pages = locked;  in tce_iommu_enable()
287 	container->enabled = true;  in tce_iommu_enable()
292 static void tce_iommu_disable(struct tce_container *container)  in tce_iommu_disable()  argument
294 	if (!container->enabled)  in tce_iommu_disable()
297 	container->enabled = false;  in tce_iommu_disable()
302 	decrement_locked_vm(container->locked_pages);  in tce_iommu_disable()
307 	struct tce_container *container;  in tce_iommu_open()  local
314 	container = kzalloc(sizeof(*container), GFP_KERNEL);  in tce_iommu_open()
315 	if (!container)  in tce_iommu_open()
318 	mutex_init(&container->lock);  in tce_iommu_open()
319 	INIT_LIST_HEAD_RCU(&container->group_list);  in tce_iommu_open()
321 	container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;  in tce_iommu_open()
323 	return container;  in tce_iommu_open()
326 static int tce_iommu_clear(struct tce_container *container,
333 	struct tce_container *container = iommu_data;  in tce_iommu_release()  local
338 	while (tce_groups_attached(container)) {  in tce_iommu_release()
339 		tcegrp = list_first_entry(&container->group_list,  in tce_iommu_release()
350 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_release()
355 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_release()
359 	tce_iommu_disable(container);  in tce_iommu_release()
360 	mutex_destroy(&container->lock);  in tce_iommu_release()
362 	kfree(container);  in tce_iommu_release()
365 static void tce_iommu_unuse_page(struct tce_container *container,  in tce_iommu_unuse_page()  argument
415 static int tce_iommu_clear(struct tce_container *container,  in tce_iommu_clear()  argument
433 		if (container->v2) {  in tce_iommu_clear()
438 		tce_iommu_unuse_page(container, oldhpa);  in tce_iommu_clear()
458 static long tce_iommu_build(struct tce_container *container,  in tce_iommu_build()  argument
485 			tce_iommu_unuse_page(container, hpa);  in tce_iommu_build()
493 			tce_iommu_unuse_page(container, hpa);  in tce_iommu_build()
499 		tce_iommu_clear(container, tbl, entry, i);  in tce_iommu_build()
504 static long tce_iommu_build_v2(struct tce_container *container,  in tce_iommu_build_v2()  argument
557 		tce_iommu_clear(container, tbl, entry, i);  in tce_iommu_build_v2()
562 static long tce_iommu_create_table(struct tce_container *container,  in tce_iommu_create_table()  argument
587 	if (!ret && container->v2) {  in tce_iommu_create_table()
608 static long tce_iommu_create_window(struct tce_container *container,  in tce_iommu_create_window()  argument
617 	num = tce_iommu_find_free_table(container);  in tce_iommu_create_window()
622 	tcegrp = list_first_entry(&container->group_list,  in tce_iommu_create_window()
637 	ret = tce_iommu_create_table(container, table_group, num,  in tce_iommu_create_window()
648 	list_for_each_entry(tcegrp, &container->group_list, next) {  in tce_iommu_create_window()
656 	container->tables[num] = tbl;  in tce_iommu_create_window()
664 	list_for_each_entry(tcegrp, &container->group_list, next) {  in tce_iommu_create_window()
673 static long tce_iommu_remove_window(struct tce_container *container,  in tce_iommu_remove_window()  argument
681 	num = tce_iommu_find_table(container, start_addr, &tbl);  in tce_iommu_remove_window()
688 	list_for_each_entry(tcegrp, &container->group_list, next) {  in tce_iommu_remove_window()
705 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_remove_window()
707 	container->tables[num] = NULL;  in tce_iommu_remove_window()
715 	struct tce_container *container = iommu_data;  in tce_iommu_ioctl()  local
738 		if (!tce_groups_attached(container))  in tce_iommu_ioctl()
741 		tcegrp = list_first_entry(&container->group_list,  in tce_iommu_ioctl()
763 				container->v2) {  in tce_iommu_ioctl()
787 		if (!container->enabled)  in tce_iommu_ioctl()
802 		num = tce_iommu_find_table(container, param.iova, &tbl);  in tce_iommu_ioctl()
827 		if (container->v2)  in tce_iommu_ioctl()
828 			ret = tce_iommu_build_v2(container, tbl,  in tce_iommu_ioctl()
834 			ret = tce_iommu_build(container, tbl,  in tce_iommu_ioctl()
849 		if (!container->enabled)  in tce_iommu_ioctl()
865 		num = tce_iommu_find_table(container, param.iova, &tbl);  in tce_iommu_ioctl()
877 		ret = tce_iommu_clear(container, tbl,  in tce_iommu_ioctl()
887 		if (!container->v2)  in tce_iommu_ioctl()
903 		mutex_lock(&container->lock);  in tce_iommu_ioctl()
904 		ret = tce_iommu_register_pages(container, param.vaddr,  in tce_iommu_ioctl()
906 		mutex_unlock(&container->lock);  in tce_iommu_ioctl()
913 		if (!container->v2)  in tce_iommu_ioctl()
929 		mutex_lock(&container->lock);  in tce_iommu_ioctl()
930 		ret = tce_iommu_unregister_pages(container, param.vaddr,  in tce_iommu_ioctl()
932 		mutex_unlock(&container->lock);  in tce_iommu_ioctl()
937 		if (container->v2)  in tce_iommu_ioctl()
940 		mutex_lock(&container->lock);  in tce_iommu_ioctl()
941 		ret = tce_iommu_enable(container);  in tce_iommu_ioctl()
942 		mutex_unlock(&container->lock);  in tce_iommu_ioctl()
947 		if (container->v2)  in tce_iommu_ioctl()
950 		mutex_lock(&container->lock);  in tce_iommu_ioctl()
951 		tce_iommu_disable(container);  in tce_iommu_ioctl()
952 		mutex_unlock(&container->lock);  in tce_iommu_ioctl()
959 		list_for_each_entry(tcegrp, &container->group_list, next) {  in tce_iommu_ioctl()
971 		if (!container->v2)  in tce_iommu_ioctl()
974 		if (!tce_groups_attached(container))  in tce_iommu_ioctl()
989 		mutex_lock(&container->lock);  in tce_iommu_ioctl()
991 		ret = tce_iommu_create_window(container, create.page_shift,  in tce_iommu_ioctl()
995 		mutex_unlock(&container->lock);  in tce_iommu_ioctl()
1005 		if (!container->v2)  in tce_iommu_ioctl()
1008 		if (!tce_groups_attached(container))  in tce_iommu_ioctl()
1023 		mutex_lock(&container->lock);  in tce_iommu_ioctl()
1025 		ret = tce_iommu_remove_window(container, remove.start_addr);  in tce_iommu_ioctl()
1027 		mutex_unlock(&container->lock);  in tce_iommu_ioctl()
1036 static void tce_iommu_release_ownership(struct tce_container *container,  in tce_iommu_release_ownership()  argument
1042 		struct iommu_table *tbl = container->tables[i];  in tce_iommu_release_ownership()
1047 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);  in tce_iommu_release_ownership()
1052 		container->tables[i] = NULL;  in tce_iommu_release_ownership()
1056 static int tce_iommu_take_ownership(struct tce_container *container,  in tce_iommu_take_ownership()  argument
1081 		container->tables[i] = table_group->tables[i];  in tce_iommu_take_ownership()
1086 static void tce_iommu_release_ownership_ddw(struct tce_container *container,  in tce_iommu_release_ownership_ddw()  argument
1102 static long tce_iommu_take_ownership_ddw(struct tce_container *container,  in tce_iommu_take_ownership_ddw()  argument
1121 	if (!tce_groups_attached(container) && !container->tables[0]) {  in tce_iommu_take_ownership_ddw()
1122 		ret = tce_iommu_create_table(container,  in tce_iommu_take_ownership_ddw()
1132 			container->tables[0] = tbl;  in tce_iommu_take_ownership_ddw()
1137 		tbl = container->tables[i];  in tce_iommu_take_ownership_ddw()
1163 	struct tce_container *container = iommu_data;  in tce_iommu_attach_group()  local
1167 	mutex_lock(&container->lock);  in tce_iommu_attach_group()
1173 	if (tce_groups_attached(container) && (!table_group->ops ||  in tce_iommu_attach_group()
1181 	list_for_each_entry(tcegrp, &container->group_list, next) {  in tce_iommu_attach_group()
1208 		ret = tce_iommu_take_ownership(container, table_group);  in tce_iommu_attach_group()
1210 		ret = tce_iommu_take_ownership_ddw(container, table_group);  in tce_iommu_attach_group()
1214 		list_add(&tcegrp->next, &container->group_list);  in tce_iommu_attach_group()
1221 	mutex_unlock(&container->lock);  in tce_iommu_attach_group()
1229 	struct tce_container *container = iommu_data;  in tce_iommu_detach_group()  local
1234 	mutex_lock(&container->lock);  in tce_iommu_detach_group()
1236 	list_for_each_entry(tcegrp, &container->group_list, next) {  in tce_iommu_detach_group()
1256 		tce_iommu_release_ownership(container, table_group);  in tce_iommu_detach_group()
1258 		tce_iommu_release_ownership_ddw(container, table_group);  in tce_iommu_detach_group()
1261 	mutex_unlock(&container->lock);  in tce_iommu_detach_group()