| H A D | eventpoll.c | 245 struct epitem *epi; member in struct:ep_pqueue
353 return container_of(p, struct ep_pqueue, pt)->epi; ep_item_from_epqueue()
533 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) ep_unregister_pollwait() argument
535 struct list_head *lsthead = &epi->pwqlist; ep_unregister_pollwait()
548 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi) ep_wakeup_source() argument
550 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); ep_wakeup_source()
554 static inline void ep_pm_stay_awake(struct epitem *epi) ep_pm_stay_awake() argument
556 struct wakeup_source *ws = ep_wakeup_source(epi); ep_pm_stay_awake()
562 static inline bool ep_has_wakeup_source(struct epitem *epi) ep_has_wakeup_source() argument
564 return rcu_access_pointer(epi->ws) ? true : false; ep_has_wakeup_source()
568 static inline void ep_pm_stay_awake_rcu(struct epitem *epi) ep_pm_stay_awake_rcu() argument
573 ws = rcu_dereference(epi->ws); ep_pm_stay_awake_rcu()
599 struct epitem *epi, *nepi; ep_scan_ready_list() local
634 for (nepi = ep->ovflist; (epi = nepi) != NULL; ep_scan_ready_list()
635 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { ep_scan_ready_list()
642 if (!ep_is_linked(&epi->rdllink)) { ep_scan_ready_list()
643 list_add_tail(&epi->rdllink, &ep->rdllist); ep_scan_ready_list()
644 ep_pm_stay_awake(epi); ep_scan_ready_list()
684 struct epitem *epi = container_of(head, struct epitem, rcu); epi_rcu_free() local
685 kmem_cache_free(epi_cache, epi); epi_rcu_free()
692 static int ep_remove(struct eventpoll *ep, struct epitem *epi) ep_remove() argument
695 struct file *file = epi->ffd.file; ep_remove()
705 ep_unregister_pollwait(ep, epi); ep_remove()
709 list_del_rcu(&epi->fllink); ep_remove()
712 rb_erase(&epi->rbn, &ep->rbr); ep_remove()
715 if (ep_is_linked(&epi->rdllink)) ep_remove()
716 list_del_init(&epi->rdllink); ep_remove()
719 wakeup_source_unregister(ep_wakeup_source(epi)); ep_remove()
722 * field epi->rcu, since we are trying to minimize the size of ep_remove()
727 call_rcu(&epi->rcu, epi_rcu_free); ep_remove()
737 struct epitem *epi; ep_free() local
757 epi = rb_entry(rbp, struct epitem, rbn); ep_free()
759 ep_unregister_pollwait(ep, epi); ep_free()
773 epi = rb_entry(rbp, struct epitem, rbn); ep_free()
774 ep_remove(ep, epi); ep_free()
796 static inline unsigned int ep_item_poll(struct epitem *epi, poll_table *pt) ep_item_poll() argument
798 pt->_key = epi->event.events; ep_item_poll()
800 return epi->ffd.file->f_op->poll(epi->ffd.file, pt) & epi->event.events; ep_item_poll()
806 struct epitem *epi, *tmp; ep_read_events_proc() local
811 list_for_each_entry_safe(epi, tmp, head, rdllink) { list_for_each_entry_safe()
812 if (ep_item_poll(epi, &pt)) list_for_each_entry_safe()
820 __pm_relax(ep_wakeup_source(epi)); list_for_each_entry_safe()
821 list_del_init(&epi->rdllink); list_for_each_entry_safe()
880 struct epitem *epi = rb_entry(rbp, struct epitem, rbn); ep_show_fdinfo() local
883 epi->ffd.fd, epi->event.events, ep_show_fdinfo()
884 (long long)epi->event.data); ep_show_fdinfo()
910 struct epitem *epi, *next; eventpoll_release_file() local
926 list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) { eventpoll_release_file()
927 ep = epi->ep; eventpoll_release_file()
929 ep_remove(ep, epi); eventpoll_release_file()
974 struct epitem *epi, *epir = NULL; ep_find() local
979 epi = rb_entry(rbp, struct epitem, rbn); ep_find()
980 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); ep_find()
986 epir = epi; ep_find()
1003 struct epitem *epi = ep_item_from_wait(wait); ep_poll_callback() local
1004 struct eventpoll *ep = epi->ep; ep_poll_callback()
1025 if (!(epi->event.events & ~EP_PRIVATE_BITS)) ep_poll_callback()
1034 if (key && !((unsigned long) key & epi->event.events)) ep_poll_callback()
1044 if (epi->next == EP_UNACTIVE_PTR) { ep_poll_callback()
1045 epi->next = ep->ovflist; ep_poll_callback()
1046 ep->ovflist = epi; ep_poll_callback()
1047 if (epi->ws) { ep_poll_callback()
1049 * Activate ep->ws since epi->ws may get ep_poll_callback()
1060 if (!ep_is_linked(&epi->rdllink)) { ep_poll_callback()
1061 list_add_tail(&epi->rdllink, &ep->rdllist); ep_poll_callback()
1062 ep_pm_stay_awake_rcu(epi); ep_poll_callback()
1091 struct epitem *epi = ep_item_from_epqueue(pt); ep_ptable_queue_proc() local
1094 if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) { ep_ptable_queue_proc()
1097 pwq->base = epi; ep_ptable_queue_proc()
1099 list_add_tail(&pwq->llink, &epi->pwqlist); ep_ptable_queue_proc()
1100 epi->nwait++; ep_ptable_queue_proc()
1103 epi->nwait = -1; ep_ptable_queue_proc()
1107 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) ep_rbtree_insert() argument
1116 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); ep_rbtree_insert()
1122 rb_link_node(&epi->rbn, parent, p); ep_rbtree_insert()
1123 rb_insert_color(&epi->rbn, &ep->rbr); ep_rbtree_insert()
1167 struct epitem *epi; reverse_path_check_proc() local
1171 list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) { reverse_path_check_proc()
1172 child_file = epi->ep->file; reverse_path_check_proc()
1224 static int ep_create_wakeup_source(struct epitem *epi) ep_create_wakeup_source() argument
1229 if (!epi->ep->ws) { ep_create_wakeup_source()
1230 epi->ep->ws = wakeup_source_register("eventpoll"); ep_create_wakeup_source()
1231 if (!epi->ep->ws) ep_create_wakeup_source()
1235 name = epi->ffd.file->f_path.dentry->d_name.name; ep_create_wakeup_source()
1240 rcu_assign_pointer(epi->ws, ws); ep_create_wakeup_source()
1246 static noinline void ep_destroy_wakeup_source(struct epitem *epi) ep_destroy_wakeup_source() argument
1248 struct wakeup_source *ws = ep_wakeup_source(epi); ep_destroy_wakeup_source()
1250 RCU_INIT_POINTER(epi->ws, NULL); ep_destroy_wakeup_source()
1270 struct epitem *epi; ep_insert() local
1276 if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL))) ep_insert()
1280 INIT_LIST_HEAD(&epi->rdllink); ep_insert()
1281 INIT_LIST_HEAD(&epi->fllink); ep_insert()
1282 INIT_LIST_HEAD(&epi->pwqlist); ep_insert()
1283 epi->ep = ep; ep_insert()
1284 ep_set_ffd(&epi->ffd, tfile, fd); ep_insert()
1285 epi->event = *event; ep_insert()
1286 epi->nwait = 0; ep_insert()
1287 epi->next = EP_UNACTIVE_PTR; ep_insert()
1288 if (epi->event.events & EPOLLWAKEUP) { ep_insert()
1289 error = ep_create_wakeup_source(epi); ep_insert()
1293 RCU_INIT_POINTER(epi->ws, NULL); ep_insert()
1297 epq.epi = epi; ep_insert()
1307 revents = ep_item_poll(epi, &epq.pt); ep_insert()
1315 if (epi->nwait < 0) ep_insert()
1320 list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links); ep_insert()
1327 ep_rbtree_insert(ep, epi); ep_insert()
1338 if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) { ep_insert()
1339 list_add_tail(&epi->rdllink, &ep->rdllist); ep_insert()
1340 ep_pm_stay_awake(epi); ep_insert()
1361 list_del_rcu(&epi->fllink); ep_insert()
1364 rb_erase(&epi->rbn, &ep->rbr); ep_insert()
1367 ep_unregister_pollwait(ep, epi); ep_insert()
1376 if (ep_is_linked(&epi->rdllink)) ep_insert()
1377 list_del_init(&epi->rdllink); ep_insert()
1380 wakeup_source_unregister(ep_wakeup_source(epi)); ep_insert()
1383 kmem_cache_free(epi_cache, epi); ep_insert()
1392 static int ep_modify(struct eventpoll *ep, struct epitem *epi, struct epoll_event *event) ep_modify() argument
1405 epi->event.events = event->events; /* need barrier below */ ep_modify()
1406 epi->event.data = event->data; /* protected by mtx */ ep_modify()
1407 if (epi->event.events & EPOLLWAKEUP) { ep_modify()
1408 if (!ep_has_wakeup_source(epi)) ep_modify()
1409 ep_create_wakeup_source(epi); ep_modify()
1410 } else if (ep_has_wakeup_source(epi)) { ep_modify()
1411 ep_destroy_wakeup_source(epi); ep_modify()
1417 * 1) Flush epi changes above to other CPUs. This ensures ep_modify()
1421 * changing epi above (but ep_poll_callback does take ep_modify()
1438 revents = ep_item_poll(epi, &pt); ep_modify()
1446 if (!ep_is_linked(&epi->rdllink)) { ep_modify()
1447 list_add_tail(&epi->rdllink, &ep->rdllist); ep_modify()
1448 ep_pm_stay_awake(epi); ep_modify()
1472 struct epitem *epi; ep_send_events_proc() local
1486 epi = list_first_entry(head, struct epitem, rdllink); ep_send_events_proc()
1489 * Activate ep->ws before deactivating epi->ws to prevent ep_send_events_proc()
1490 * triggering auto-suspend here (in case we reactive epi->ws ep_send_events_proc()
1493 * This could be rearranged to delay the deactivation of epi->ws ep_send_events_proc()
1494 * instead, but then epi->ws would temporarily be out of sync ep_send_events_proc()
1497 ws = ep_wakeup_source(epi); ep_send_events_proc()
1504 list_del_init(&epi->rdllink); ep_send_events_proc()
1506 revents = ep_item_poll(epi, &pt); ep_send_events_proc()
1516 __put_user(epi->event.data, &uevent->data)) { ep_send_events_proc()
1517 list_add(&epi->rdllink, head); ep_send_events_proc()
1518 ep_pm_stay_awake(epi); ep_send_events_proc()
1523 if (epi->event.events & EPOLLONESHOT) ep_send_events_proc()
1524 epi->event.events &= EP_PRIVATE_BITS; ep_send_events_proc()
1525 else if (!(epi->event.events & EPOLLET)) { ep_send_events_proc()
1537 list_add_tail(&epi->rdllink, &ep->rdllist); ep_send_events_proc()
1538 ep_pm_stay_awake(epi); ep_send_events_proc()
1686 struct epitem *epi; ep_loop_check_proc() local
1692 epi = rb_entry(rbp, struct epitem, rbn); ep_loop_check_proc()
1693 if (unlikely(is_file_epoll(epi->ffd.file))) { ep_loop_check_proc()
1694 ep_tovisit = epi->ffd.file->private_data; ep_loop_check_proc()
1698 ep_loop_check_proc, epi->ffd.file, ep_loop_check_proc()
1711 if (list_empty(&epi->ffd.file->f_tfile_llink)) ep_loop_check_proc()
1712 list_add(&epi->ffd.file->f_tfile_llink, ep_loop_check_proc()
1827 struct epitem *epi; SYSCALL_DEFINE4() local
1914 epi = ep_find(ep, tf.file, fd); SYSCALL_DEFINE4()
1919 if (!epi) { SYSCALL_DEFINE4()
1928 if (epi) SYSCALL_DEFINE4()
1929 error = ep_remove(ep, epi); SYSCALL_DEFINE4()
1934 if (epi) { SYSCALL_DEFINE4()
1936 error = ep_modify(ep, epi, &epds); SYSCALL_DEFINE4()
|