129 static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)  in allocate_signal_page()  argument
151 	if (list_empty(&p->signal_event_pages))  in allocate_signal_page()
154 		page->page_index = list_tail_entry(&p->signal_event_pages,  in allocate_signal_page()
159 			page, p);  in allocate_signal_page()
162 	list_add(&page->event_pages, &p->signal_event_pages);  in allocate_signal_page()
173 					struct kfd_process *p,  in allocate_event_notification_slot()  argument
179 	ret = allocate_free_slot(p, page, signal_slot_index);  in allocate_event_notification_slot()
181 		ret = allocate_signal_page(devkfd, p);  in allocate_event_notification_slot()
183 			ret = allocate_free_slot(p, page, signal_slot_index);  in allocate_event_notification_slot()
200 static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p,  in lookup_signal_page_by_index()  argument
209 	list_for_each_entry(page, &p->signal_event_pages, event_pages)  in lookup_signal_page_by_index()
220 static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)  in lookup_event_by_id()  argument
224 	hash_for_each_possible(p->events, ev, events, id)  in lookup_event_by_id()
243 static u32 make_nonsignal_event_id(struct kfd_process *p)  in make_nonsignal_event_id()  argument
247 	for (id = p->next_nonsignal_event_id;  in make_nonsignal_event_id()
249 		lookup_event_by_id(p, id) != NULL;  in make_nonsignal_event_id()
261 		p->next_nonsignal_event_id = id + 1;  in make_nonsignal_event_id()
268 		lookup_event_by_id(p, id) != NULL;  in make_nonsignal_event_id()
274 		p->next_nonsignal_event_id = id + 1;  in make_nonsignal_event_id()
278 	p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID;  in make_nonsignal_event_id()
282 static struct kfd_event *lookup_event_by_page_slot(struct kfd_process *p,  in lookup_event_by_page_slot()  argument
286 	return lookup_event_by_id(p, make_signal_event_id(page, signal_slot));  in lookup_event_by_page_slot()
290 				struct kfd_process *p,  in create_signal_event()  argument
293 	if (p->signal_event_count == KFD_SIGNAL_EVENT_LIMIT) {  in create_signal_event()
298 	if (!allocate_event_notification_slot(devkfd, p, &ev->signal_page,  in create_signal_event()
304 	p->signal_event_count++;  in create_signal_event()
313 			p->signal_event_count, ev->event_id,  in create_signal_event()
317 			p->signal_event_count, ev->event_id,  in create_signal_event()
328 static int create_other_event(struct kfd_process *p, struct kfd_event *ev)  in create_other_event()  argument
330 	ev->event_id = make_nonsignal_event_id(p);  in create_other_event()
337 void kfd_event_init_process(struct kfd_process *p)  in kfd_event_init_process()  argument
339 	mutex_init(&p->event_mutex);  in kfd_event_init_process()
340 	hash_init(p->events);  in kfd_event_init_process()
341 	INIT_LIST_HEAD(&p->signal_event_pages);  in kfd_event_init_process()
342 	p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID;  in kfd_event_init_process()
343 	p->signal_event_count = 0;  in kfd_event_init_process()
346 static void destroy_event(struct kfd_process *p, struct kfd_event *ev)  in destroy_event()  argument
351 		p->signal_event_count--;  in destroy_event()
364 static void destroy_events(struct kfd_process *p)  in destroy_events()  argument
370 	hash_for_each_safe(p->events, hash_bkt, tmp, ev, events)  in destroy_events()
371 		destroy_event(p, ev);  in destroy_events()
378 static void shutdown_signal_pages(struct kfd_process *p)  in shutdown_signal_pages()  argument
382 	list_for_each_entry_safe(page, tmp, &p->signal_event_pages,  in shutdown_signal_pages()
390 void kfd_event_free_process(struct kfd_process *p)  in kfd_event_free_process()  argument
392 	destroy_events(p);  in kfd_event_free_process()
393 	shutdown_signal_pages(p);  in kfd_event_free_process()
407 int kfd_event_create(struct file *devkfd, struct kfd_process *p,  in kfd_event_create()  argument
426 	mutex_lock(&p->event_mutex);  in kfd_event_create()
431 		ret = create_signal_event(devkfd, p, ev);  in kfd_event_create()
440 		ret = create_other_event(p, ev);  in kfd_event_create()
445 		hash_add(p->events, &ev->events, ev->event_id);  in kfd_event_create()
453 	mutex_unlock(&p->event_mutex);  in kfd_event_create()
459 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)  in kfd_event_destroy()  argument
464 	mutex_lock(&p->event_mutex);  in kfd_event_destroy()
466 	ev = lookup_event_by_id(p, event_id);  in kfd_event_destroy()
469 		destroy_event(p, ev);  in kfd_event_destroy()
473 	mutex_unlock(&p->event_mutex);  in kfd_event_destroy()
496 int kfd_set_event(struct kfd_process *p, uint32_t event_id)  in kfd_set_event()  argument
501 	mutex_lock(&p->event_mutex);  in kfd_set_event()
503 	ev = lookup_event_by_id(p, event_id);  in kfd_set_event()
510 	mutex_unlock(&p->event_mutex);  in kfd_set_event()
520 int kfd_reset_event(struct kfd_process *p, uint32_t event_id)  in kfd_reset_event()  argument
525 	mutex_lock(&p->event_mutex);  in kfd_reset_event()
527 	ev = lookup_event_by_id(p, event_id);  in kfd_reset_event()
534 	mutex_unlock(&p->event_mutex);  in kfd_reset_event()
539 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)  in acknowledge_signal()  argument
550 static void set_event_from_interrupt(struct kfd_process *p,  in set_event_from_interrupt()  argument
554 		acknowledge_signal(p, ev);  in set_event_from_interrupt()
569 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);  in kfd_signal_event_interrupt()  local
571 	if (!p)  in kfd_signal_event_interrupt()
574 	mutex_lock(&p->event_mutex);  in kfd_signal_event_interrupt()
578 		ev = lookup_event_by_id(p, partial_id);  in kfd_signal_event_interrupt()
579 		set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
589 		list_for_each_entry(page, &p->signal_event_pages, event_pages)  in kfd_signal_event_interrupt()
592 					ev = lookup_event_by_page_slot(p,  in kfd_signal_event_interrupt()
594 					set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
598 	mutex_unlock(&p->event_mutex);  in kfd_signal_event_interrupt()
599 	mutex_unlock(&p->mutex);  in kfd_signal_event_interrupt()
620 static int init_event_waiter(struct kfd_process *p,  in init_event_waiter()  argument
625 	struct kfd_event *ev = lookup_event_by_id(p, event_id);  in init_event_waiter()
718 int kfd_wait_on_events(struct kfd_process *p,  in kfd_wait_on_events()  argument
730 	mutex_lock(&p->event_mutex);  in kfd_wait_on_events()
745 		ret = init_event_waiter(p, &event_waiters[i],  in kfd_wait_on_events()
751 	mutex_unlock(&p->event_mutex);  in kfd_wait_on_events()
790 	mutex_lock(&p->event_mutex);  in kfd_wait_on_events()
792 	mutex_unlock(&p->event_mutex);  in kfd_wait_on_events()
800 	mutex_unlock(&p->event_mutex);  in kfd_wait_on_events()
807 int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)  in kfd_event_mmap()  argument
823 	page = lookup_signal_page_by_index(p, page_index);  in kfd_event_mmap()
856 static void lookup_events_by_type_and_signal(struct kfd_process *p,  in lookup_events_by_type_and_signal()  argument
866 	hash_for_each(p->events, bkt, ev, events)  in lookup_events_by_type_and_signal()
882 					p->lead_thread->pid);  in lookup_events_by_type_and_signal()
883 			send_sig(SIGTERM, p->lead_thread, 0);  in lookup_events_by_type_and_signal()
887 				p->lead_thread->pid);  in lookup_events_by_type_and_signal()
904 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);  in kfd_signal_iommu_event()  local
906 	if (!p)  in kfd_signal_iommu_event()
911 	down_read(&p->mm->mmap_sem);  in kfd_signal_iommu_event()
912 	vma = find_vma(p->mm, address);  in kfd_signal_iommu_event()
938 	up_read(&p->mm->mmap_sem);  in kfd_signal_iommu_event()
940 	mutex_lock(&p->event_mutex);  in kfd_signal_iommu_event()
943 	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,  in kfd_signal_iommu_event()
946 	mutex_unlock(&p->event_mutex);  in kfd_signal_iommu_event()
947 	mutex_unlock(&p->mutex);  in kfd_signal_iommu_event()
957 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);  in kfd_signal_hw_exception_event()  local
959 	if (!p)  in kfd_signal_hw_exception_event()
962 	mutex_lock(&p->event_mutex);  in kfd_signal_hw_exception_event()
965 	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);  in kfd_signal_hw_exception_event()
967 	mutex_unlock(&p->event_mutex);  in kfd_signal_hw_exception_event()
968 	mutex_unlock(&p->mutex);  in kfd_signal_hw_exception_event()