50 	struct kvm_kernel_irqfd *irqfd =  in irqfd_inject()  local
52 	struct kvm *kvm = irqfd->kvm;  in irqfd_inject()
54 	if (!irqfd->resampler) {  in irqfd_inject()
55 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,  in irqfd_inject()
57 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,  in irqfd_inject()
61 			    irqfd->gsi, 1, false);  in irqfd_inject()
74 	struct kvm_kernel_irqfd *irqfd;  in irqfd_resampler_ack()  local
86 	list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link)  in irqfd_resampler_ack()
87 		eventfd_signal(irqfd->resamplefd, 1);  in irqfd_resampler_ack()
93 irqfd_resampler_shutdown(struct kvm_kernel_irqfd *irqfd)  in irqfd_resampler_shutdown()  argument
95 	struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;  in irqfd_resampler_shutdown()
100 	list_del_rcu(&irqfd->resampler_link);  in irqfd_resampler_shutdown()
120 	struct kvm_kernel_irqfd *irqfd =  in irqfd_shutdown()  local
128 	eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);  in irqfd_shutdown()
134 	flush_work(&irqfd->inject);  in irqfd_shutdown()
136 	if (irqfd->resampler) {  in irqfd_shutdown()
137 		irqfd_resampler_shutdown(irqfd);  in irqfd_shutdown()
138 		eventfd_ctx_put(irqfd->resamplefd);  in irqfd_shutdown()
145 	irq_bypass_unregister_consumer(&irqfd->consumer);  in irqfd_shutdown()
147 	eventfd_ctx_put(irqfd->eventfd);  in irqfd_shutdown()
148 	kfree(irqfd);  in irqfd_shutdown()
154 irqfd_is_active(struct kvm_kernel_irqfd *irqfd)  in irqfd_is_active()  argument
156 	return list_empty(&irqfd->list) ? false : true;  in irqfd_is_active()
165 irqfd_deactivate(struct kvm_kernel_irqfd *irqfd)  in irqfd_deactivate()  argument
167 	BUG_ON(!irqfd_is_active(irqfd));  in irqfd_deactivate()
169 	list_del_init(&irqfd->list);  in irqfd_deactivate()
171 	queue_work(irqfd_cleanup_wq, &irqfd->shutdown);  in irqfd_deactivate()
189 	struct kvm_kernel_irqfd *irqfd =  in irqfd_wakeup()  local
193 	struct kvm *kvm = irqfd->kvm;  in irqfd_wakeup()
200 			seq = read_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_wakeup()
201 			irq = irqfd->irq_entry;  in irqfd_wakeup()
202 		} while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));  in irqfd_wakeup()
207 			schedule_work(&irqfd->inject);  in irqfd_wakeup()
226 		if (irqfd_is_active(irqfd))  in irqfd_wakeup()
227 			irqfd_deactivate(irqfd);  in irqfd_wakeup()
239 	struct kvm_kernel_irqfd *irqfd =  in irqfd_ptable_queue_proc()  local
241 	add_wait_queue(wqh, &irqfd->wait);  in irqfd_ptable_queue_proc()
245 static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)  in irqfd_update()  argument
251 	n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);  in irqfd_update()
253 	write_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_update()
257 		irqfd->irq_entry = *e;  in irqfd_update()
259 		irqfd->irq_entry.type = 0;  in irqfd_update()
261 	write_seqcount_end(&irqfd->irq_entry_sc);  in irqfd_update()
286 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_assign()  local
296 	irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL);  in kvm_irqfd_assign()
297 	if (!irqfd)  in kvm_irqfd_assign()
300 	irqfd->kvm = kvm;  in kvm_irqfd_assign()
301 	irqfd->gsi = args->gsi;  in kvm_irqfd_assign()
302 	INIT_LIST_HEAD(&irqfd->list);  in kvm_irqfd_assign()
303 	INIT_WORK(&irqfd->inject, irqfd_inject);  in kvm_irqfd_assign()
304 	INIT_WORK(&irqfd->shutdown, irqfd_shutdown);  in kvm_irqfd_assign()
305 	seqcount_init(&irqfd->irq_entry_sc);  in kvm_irqfd_assign()
319 	irqfd->eventfd = eventfd;  in kvm_irqfd_assign()
330 		irqfd->resamplefd = resamplefd;  in kvm_irqfd_assign()
331 		INIT_LIST_HEAD(&irqfd->resampler_link);  in kvm_irqfd_assign()
337 			if (resampler->notifier.gsi == irqfd->gsi) {  in kvm_irqfd_assign()
338 				irqfd->resampler = resampler;  in kvm_irqfd_assign()
343 		if (!irqfd->resampler) {  in kvm_irqfd_assign()
353 			resampler->notifier.gsi = irqfd->gsi;  in kvm_irqfd_assign()
360 			irqfd->resampler = resampler;  in kvm_irqfd_assign()
363 		list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);  in kvm_irqfd_assign()
373 	init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);  in kvm_irqfd_assign()
374 	init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);  in kvm_irqfd_assign()
380 		if (irqfd->eventfd != tmp->eventfd)  in kvm_irqfd_assign()
389 	irqfd_update(kvm, irqfd);  in kvm_irqfd_assign()
392 	list_add_tail(&irqfd->list, &kvm->irqfds.items);  in kvm_irqfd_assign()
400 	events = f.file->f_op->poll(f.file, &irqfd->pt);  in kvm_irqfd_assign()
403 		schedule_work(&irqfd->inject);  in kvm_irqfd_assign()
411 	irqfd->consumer.token = (void *)irqfd->eventfd;  in kvm_irqfd_assign()
412 	irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;  in kvm_irqfd_assign()
413 	irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;  in kvm_irqfd_assign()
414 	irqfd->consumer.stop = kvm_arch_irq_bypass_stop;  in kvm_irqfd_assign()
415 	irqfd->consumer.start = kvm_arch_irq_bypass_start;  in kvm_irqfd_assign()
416 	ret = irq_bypass_register_consumer(&irqfd->consumer);  in kvm_irqfd_assign()
419 				irqfd->consumer.token, ret);  in kvm_irqfd_assign()
425 	if (irqfd->resampler)  in kvm_irqfd_assign()
426 		irqfd_resampler_shutdown(irqfd);  in kvm_irqfd_assign()
437 	kfree(irqfd);  in kvm_irqfd_assign()
524 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_deassign()  local
533 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_deassign()
534 		if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {  in kvm_irqfd_deassign()
541 			write_seqcount_begin(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
542 			irqfd->irq_entry.type = 0;  in kvm_irqfd_deassign()
543 			write_seqcount_end(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
544 			irqfd_deactivate(irqfd);  in kvm_irqfd_deassign()
580 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_release()  local
584 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)  in kvm_irqfd_release()
585 		irqfd_deactivate(irqfd);  in kvm_irqfd_release()
603 	struct kvm_kernel_irqfd *irqfd;  in kvm_irq_routing_update()  local
607 	list_for_each_entry(irqfd, &kvm->irqfds.items, list) {  in kvm_irq_routing_update()
608 		irqfd_update(kvm, irqfd);  in kvm_irq_routing_update()
611 		if (irqfd->producer) {  in kvm_irq_routing_update()
613 					irqfd->kvm, irqfd->producer->irq,  in kvm_irq_routing_update()
614 					irqfd->gsi, 1);  in kvm_irq_routing_update()