93 	struct vfio_group		*group;  member
150 static int vfio_alloc_group_minor(struct vfio_group *group)  in vfio_alloc_group_minor()  argument
152 	return idr_alloc(&vfio.group_idr, group, 0, MINORMASK + 1, GFP_KERNEL);  in vfio_alloc_group_minor()
162 static void vfio_group_get(struct vfio_group *group);
188 static void vfio_group_unlock_and_free(struct vfio_group *group)  in vfio_group_unlock_and_free()  argument
195 	iommu_group_unregister_notifier(group->iommu_group, &group->nb);  in vfio_group_unlock_and_free()
196 	kfree(group);  in vfio_group_unlock_and_free()
204 	struct vfio_group *group, *tmp;  in vfio_create_group()  local
208 	group = kzalloc(sizeof(*group), GFP_KERNEL);  in vfio_create_group()
209 	if (!group)  in vfio_create_group()
212 	kref_init(&group->kref);  in vfio_create_group()
213 	INIT_LIST_HEAD(&group->device_list);  in vfio_create_group()
214 	mutex_init(&group->device_lock);  in vfio_create_group()
215 	INIT_LIST_HEAD(&group->unbound_list);  in vfio_create_group()
216 	mutex_init(&group->unbound_lock);  in vfio_create_group()
217 	atomic_set(&group->container_users, 0);  in vfio_create_group()
218 	atomic_set(&group->opened, 0);  in vfio_create_group()
219 	group->iommu_group = iommu_group;  in vfio_create_group()
221 	group->nb.notifier_call = vfio_iommu_group_notifier;  in vfio_create_group()
230 	ret = iommu_group_register_notifier(iommu_group, &group->nb);  in vfio_create_group()
232 		kfree(group);  in vfio_create_group()
242 			vfio_group_unlock_and_free(group);  in vfio_create_group()
247 	minor = vfio_alloc_group_minor(group);  in vfio_create_group()
249 		vfio_group_unlock_and_free(group);  in vfio_create_group()
255 			    group, "%d", iommu_group_id(iommu_group));  in vfio_create_group()
258 		vfio_group_unlock_and_free(group);  in vfio_create_group()
262 	group->minor = minor;  in vfio_create_group()
263 	group->dev = dev;  in vfio_create_group()
265 	list_add(&group->vfio_next, &vfio.group_list);  in vfio_create_group()
269 	return group;  in vfio_create_group()
275 	struct vfio_group *group = container_of(kref, struct vfio_group, kref);  in vfio_group_release()  local
277 	struct iommu_group *iommu_group = group->iommu_group;  in vfio_group_release()
279 	WARN_ON(!list_empty(&group->device_list));  in vfio_group_release()
282 				 &group->unbound_list, unbound_next) {  in vfio_group_release()
287 	device_destroy(vfio.class, MKDEV(MAJOR(vfio.group_devt), group->minor));  in vfio_group_release()
288 	list_del(&group->vfio_next);  in vfio_group_release()
289 	vfio_free_group_minor(group->minor);  in vfio_group_release()
290 	vfio_group_unlock_and_free(group);  in vfio_group_release()
294 static void vfio_group_put(struct vfio_group *group)  in vfio_group_put()  argument
296 	kref_put_mutex(&group->kref, vfio_group_release, &vfio.group_lock);  in vfio_group_put()
300 static void vfio_group_get(struct vfio_group *group)  in vfio_group_get()  argument
302 	kref_get(&group->kref);  in vfio_group_get()
309 static struct vfio_group *vfio_group_try_get(struct vfio_group *group)  in vfio_group_try_get()  argument
311 	struct vfio_group *target = group;  in vfio_group_try_get()
314 	list_for_each_entry(group, &vfio.group_list, vfio_next) {  in vfio_group_try_get()
315 		if (group == target) {  in vfio_group_try_get()
316 			vfio_group_get(group);  in vfio_group_try_get()
318 			return group;  in vfio_group_try_get()
329 	struct vfio_group *group;  in vfio_group_get_from_iommu()  local
332 	list_for_each_entry(group, &vfio.group_list, vfio_next) {  in vfio_group_get_from_iommu()
333 		if (group->iommu_group == iommu_group) {  in vfio_group_get_from_iommu()
334 			vfio_group_get(group);  in vfio_group_get_from_iommu()
336 			return group;  in vfio_group_get_from_iommu()
346 	struct vfio_group *group;  in vfio_group_get_from_minor()  local
349 	group = idr_find(&vfio.group_idr, minor);  in vfio_group_get_from_minor()
350 	if (!group) {  in vfio_group_get_from_minor()
354 	vfio_group_get(group);  in vfio_group_get_from_minor()
357 	return group;  in vfio_group_get_from_minor()
364 struct vfio_device *vfio_group_create_device(struct vfio_group *group,  in vfio_group_create_device()  argument
377 	device->group = group;  in vfio_group_create_device()
383 	vfio_group_get(group);  in vfio_group_create_device()
385 	mutex_lock(&group->device_lock);  in vfio_group_create_device()
386 	list_add(&device->group_next, &group->device_list);  in vfio_group_create_device()
387 	mutex_unlock(&group->device_lock);  in vfio_group_create_device()
396 	struct vfio_group *group = device->group;  in vfio_device_release()  local
399 	mutex_unlock(&group->device_lock);  in vfio_device_release()
412 	struct vfio_group *group = device->group;  in vfio_device_put()  local
413 	kref_put_mutex(&device->kref, vfio_device_release, &group->device_lock);  in vfio_device_put()
414 	vfio_group_put(group);  in vfio_device_put()
420 	vfio_group_get(device->group);  in vfio_device_get()
424 static struct vfio_device *vfio_group_get_device(struct vfio_group *group,  in vfio_group_get_device()  argument
429 	mutex_lock(&group->device_lock);  in vfio_group_get_device()
430 	list_for_each_entry(device, &group->device_list, group_next) {  in vfio_group_get_device()
433 			mutex_unlock(&group->device_lock);  in vfio_group_get_device()
437 	mutex_unlock(&group->device_lock);  in vfio_group_get_device()
493 	struct vfio_group *group = data;  in vfio_dev_viable()  local
499 	mutex_lock(&group->unbound_lock);  in vfio_dev_viable()
500 	list_for_each_entry(unbound, &group->unbound_list, unbound_next) {  in vfio_dev_viable()
506 	mutex_unlock(&group->unbound_lock);  in vfio_dev_viable()
511 	device = vfio_group_get_device(group, dev);  in vfio_dev_viable()
523 static int vfio_group_nb_add_dev(struct vfio_group *group, struct device *dev)  in vfio_group_nb_add_dev()  argument
528 	device = vfio_group_get_device(group, dev);  in vfio_group_nb_add_dev()
535 	if (!atomic_read(&group->container_users))  in vfio_group_nb_add_dev()
540 	     iommu_group_id(group->iommu_group));  in vfio_group_nb_add_dev()
545 static int vfio_group_nb_verify(struct vfio_group *group, struct device *dev)  in vfio_group_nb_verify()  argument
548 	if (!atomic_read(&group->container_users))  in vfio_group_nb_verify()
551 	return vfio_dev_viable(dev, group);  in vfio_group_nb_verify()
557 	struct vfio_group *group = container_of(nb, struct vfio_group, nb);  in vfio_iommu_group_notifier()  local
565 	group = vfio_group_try_get(group);  in vfio_iommu_group_notifier()
566 	if (!group)  in vfio_iommu_group_notifier()
571 		vfio_group_nb_add_dev(group, dev);  in vfio_iommu_group_notifier()
585 			 iommu_group_id(group->iommu_group));  in vfio_iommu_group_notifier()
590 			 iommu_group_id(group->iommu_group), dev->driver->name);  in vfio_iommu_group_notifier()
591 		BUG_ON(vfio_group_nb_verify(group, dev));  in vfio_iommu_group_notifier()
596 			 iommu_group_id(group->iommu_group), dev->driver->name);  in vfio_iommu_group_notifier()
601 			 iommu_group_id(group->iommu_group));  in vfio_iommu_group_notifier()
610 		mutex_lock(&group->unbound_lock);  in vfio_iommu_group_notifier()
612 				    &group->unbound_list, unbound_next) {  in vfio_iommu_group_notifier()
619 		mutex_unlock(&group->unbound_lock);  in vfio_iommu_group_notifier()
623 	vfio_group_put(group);  in vfio_iommu_group_notifier()
634 	struct vfio_group *group;  in vfio_add_group_dev()  local
641 	group = vfio_group_get_from_iommu(iommu_group);  in vfio_add_group_dev()
642 	if (!group) {  in vfio_add_group_dev()
643 		group = vfio_create_group(iommu_group);  in vfio_add_group_dev()
644 		if (IS_ERR(group)) {  in vfio_add_group_dev()
646 			return PTR_ERR(group);  in vfio_add_group_dev()
656 	device = vfio_group_get_device(group, dev);  in vfio_add_group_dev()
661 		vfio_group_put(group);  in vfio_add_group_dev()
665 	device = vfio_group_create_device(group, dev, ops, device_data);  in vfio_add_group_dev()
667 		vfio_group_put(group);  in vfio_add_group_dev()
676 	vfio_group_put(group);  in vfio_add_group_dev()
692 	struct vfio_group *group;  in vfio_device_get_from_dev()  local
699 	group = vfio_group_get_from_iommu(iommu_group);  in vfio_device_get_from_dev()
701 	if (!group)  in vfio_device_get_from_dev()
704 	device = vfio_group_get_device(group, dev);  in vfio_device_get_from_dev()
705 	vfio_group_put(group);  in vfio_device_get_from_dev()
711 static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group,  in vfio_device_get_from_name()  argument
716 	mutex_lock(&group->device_lock);  in vfio_device_get_from_name()
717 	list_for_each_entry(it, &group->device_list, group_next) {  in vfio_device_get_from_name()
724 	mutex_unlock(&group->device_lock);  in vfio_device_get_from_name()
739 static bool vfio_dev_present(struct vfio_group *group, struct device *dev)  in vfio_dev_present()  argument
743 	device = vfio_group_get_device(group, dev);  in vfio_dev_present()
757 	struct vfio_group *group = device->group;  in vfio_del_group_dev()  local
768 	vfio_group_get(group);  in vfio_del_group_dev()
782 		mutex_lock(&group->unbound_lock);  in vfio_del_group_dev()
783 		list_add(&unbound->unbound_next, &group->unbound_list);  in vfio_del_group_dev()
784 		mutex_unlock(&group->unbound_lock);  in vfio_del_group_dev()
799 		device = vfio_group_get_device(group, dev);  in vfio_del_group_dev()
810 					!vfio_dev_present(group, dev), HZ * 10);  in vfio_del_group_dev()
813 					!vfio_dev_present(group, dev), HZ * 10);  in vfio_del_group_dev()
825 	vfio_group_put(group);  in vfio_del_group_dev()
883 	struct vfio_group *group;  in __vfio_container_attach_groups()  local
886 	list_for_each_entry(group, &container->group_list, container_next) {  in __vfio_container_attach_groups()
887 		ret = driver->ops->attach_group(data, group->iommu_group);  in __vfio_container_attach_groups()
895 	list_for_each_entry_continue_reverse(group, &container->group_list,  in __vfio_container_attach_groups()
897 		driver->ops->detach_group(data, group->iommu_group);  in __vfio_container_attach_groups()
1120 static void __vfio_group_unset_container(struct vfio_group *group)  in __vfio_group_unset_container()  argument
1122 	struct vfio_container *container = group->container;  in __vfio_group_unset_container()
1130 					  group->iommu_group);  in __vfio_group_unset_container()
1132 	group->container = NULL;  in __vfio_group_unset_container()
1133 	list_del(&group->container_next);  in __vfio_group_unset_container()
1154 static int vfio_group_unset_container(struct vfio_group *group)  in vfio_group_unset_container()  argument
1156 	int users = atomic_cmpxchg(&group->container_users, 1, 0);  in vfio_group_unset_container()
1163 	__vfio_group_unset_container(group);  in vfio_group_unset_container()
1174 static void vfio_group_try_dissolve_container(struct vfio_group *group)  in vfio_group_try_dissolve_container()  argument
1176 	if (0 == atomic_dec_if_positive(&group->container_users))  in vfio_group_try_dissolve_container()
1177 		__vfio_group_unset_container(group);  in vfio_group_try_dissolve_container()
1180 static int vfio_group_set_container(struct vfio_group *group, int container_fd)  in vfio_group_set_container()  argument
1187 	if (atomic_read(&group->container_users))  in vfio_group_set_container()
1208 						group->iommu_group);  in vfio_group_set_container()
1213 	group->container = container;  in vfio_group_set_container()
1214 	list_add(&group->container_next, &container->group_list);  in vfio_group_set_container()
1218 	atomic_inc(&group->container_users);  in vfio_group_set_container()
1226 static bool vfio_group_viable(struct vfio_group *group)  in vfio_group_viable()  argument
1228 	return (iommu_group_for_each_dev(group->iommu_group,  in vfio_group_viable()
1229 					 group, vfio_dev_viable) == 0);  in vfio_group_viable()
1234 static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)  in vfio_group_get_device_fd()  argument
1240 	if (0 == atomic_read(&group->container_users) ||  in vfio_group_get_device_fd()
1241 	    !group->container->iommu_driver || !vfio_group_viable(group))  in vfio_group_get_device_fd()
1244 	device = vfio_device_get_from_name(group, buf);  in vfio_group_get_device_fd()
1282 	atomic_inc(&group->container_users);  in vfio_group_get_device_fd()
1292 	struct vfio_group *group = filep->private_data;  in vfio_group_fops_unl_ioctl()  local
1311 		if (vfio_group_viable(group))  in vfio_group_fops_unl_ioctl()
1314 		if (group->container)  in vfio_group_fops_unl_ioctl()
1333 		ret = vfio_group_set_container(group, fd);  in vfio_group_fops_unl_ioctl()
1337 		ret = vfio_group_unset_container(group);  in vfio_group_fops_unl_ioctl()
1347 		ret = vfio_group_get_device_fd(group, buf);  in vfio_group_fops_unl_ioctl()
1367 	struct vfio_group *group;  in vfio_group_fops_open()  local
1370 	group = vfio_group_get_from_minor(iminor(inode));  in vfio_group_fops_open()
1371 	if (!group)  in vfio_group_fops_open()
1375 	opened = atomic_cmpxchg(&group->opened, 0, 1);  in vfio_group_fops_open()
1377 		vfio_group_put(group);  in vfio_group_fops_open()
1382 	if (group->container) {  in vfio_group_fops_open()
1383 		atomic_dec(&group->opened);  in vfio_group_fops_open()
1384 		vfio_group_put(group);  in vfio_group_fops_open()
1388 	filep->private_data = group;  in vfio_group_fops_open()
1395 	struct vfio_group *group = filep->private_data;  in vfio_group_fops_release()  local
1399 	vfio_group_try_dissolve_container(group);  in vfio_group_fops_release()
1401 	atomic_dec(&group->opened);  in vfio_group_fops_release()
1403 	vfio_group_put(group);  in vfio_group_fops_release()
1427 	vfio_group_try_dissolve_container(device->group);  in vfio_device_fops_release()
1528 	struct vfio_group *group = filep->private_data;  in vfio_group_get_external_user()  local
1533 	if (!atomic_inc_not_zero(&group->container_users))  in vfio_group_get_external_user()
1536 	if (!group->container->iommu_driver ||  in vfio_group_get_external_user()
1537 			!vfio_group_viable(group)) {  in vfio_group_get_external_user()
1538 		atomic_dec(&group->container_users);  in vfio_group_get_external_user()
1542 	vfio_group_get(group);  in vfio_group_get_external_user()
1544 	return group;  in vfio_group_get_external_user()
1548 void vfio_group_put_external_user(struct vfio_group *group)  in vfio_group_put_external_user()  argument
1550 	vfio_group_put(group);  in vfio_group_put_external_user()
1551 	vfio_group_try_dissolve_container(group);  in vfio_group_put_external_user()
1555 int vfio_external_user_iommu_id(struct vfio_group *group)  in vfio_external_user_iommu_id()  argument
1557 	return iommu_group_id(group->iommu_group);  in vfio_external_user_iommu_id()
1561 long vfio_external_check_extension(struct vfio_group *group, unsigned long arg)  in vfio_external_check_extension()  argument
1563 	return vfio_ioctl_check_extension(group->container, arg);  in vfio_external_check_extension()