Lines Matching refs:epi
245 struct epitem *epi; member
353 return container_of(p, struct ep_pqueue, pt)->epi; in ep_item_from_epqueue()
533 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) in ep_unregister_pollwait() argument
535 struct list_head *lsthead = &epi->pwqlist; in ep_unregister_pollwait()
548 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi) in ep_wakeup_source() argument
550 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
554 static inline void ep_pm_stay_awake(struct epitem *epi) in ep_pm_stay_awake() argument
556 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_pm_stay_awake()
562 static inline bool ep_has_wakeup_source(struct epitem *epi) in ep_has_wakeup_source() argument
564 return rcu_access_pointer(epi->ws) ? true : false; in ep_has_wakeup_source()
568 static inline void ep_pm_stay_awake_rcu(struct epitem *epi) in ep_pm_stay_awake_rcu() argument
573 ws = rcu_dereference(epi->ws); in ep_pm_stay_awake_rcu()
599 struct epitem *epi, *nepi; in ep_scan_ready_list() local
634 for (nepi = ep->ovflist; (epi = nepi) != NULL; in ep_scan_ready_list()
635 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { in ep_scan_ready_list()
642 if (!ep_is_linked(&epi->rdllink)) { in ep_scan_ready_list()
643 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_scan_ready_list()
644 ep_pm_stay_awake(epi); in ep_scan_ready_list()
684 struct epitem *epi = container_of(head, struct epitem, rcu); in epi_rcu_free() local
685 kmem_cache_free(epi_cache, epi); in epi_rcu_free()
692 static int ep_remove(struct eventpoll *ep, struct epitem *epi) in ep_remove() argument
695 struct file *file = epi->ffd.file; in ep_remove()
705 ep_unregister_pollwait(ep, epi); in ep_remove()
709 list_del_rcu(&epi->fllink); in ep_remove()
712 rb_erase(&epi->rbn, &ep->rbr); in ep_remove()
715 if (ep_is_linked(&epi->rdllink)) in ep_remove()
716 list_del_init(&epi->rdllink); in ep_remove()
719 wakeup_source_unregister(ep_wakeup_source(epi)); in ep_remove()
727 call_rcu(&epi->rcu, epi_rcu_free); in ep_remove()
737 struct epitem *epi; in ep_free() local
757 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
759 ep_unregister_pollwait(ep, epi); in ep_free()
773 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
774 ep_remove(ep, epi); in ep_free()
796 static inline unsigned int ep_item_poll(struct epitem *epi, poll_table *pt) in ep_item_poll() argument
798 pt->_key = epi->event.events; in ep_item_poll()
800 return epi->ffd.file->f_op->poll(epi->ffd.file, pt) & epi->event.events; in ep_item_poll()
806 struct epitem *epi, *tmp; in ep_read_events_proc() local
811 list_for_each_entry_safe(epi, tmp, head, rdllink) { in ep_read_events_proc()
812 if (ep_item_poll(epi, &pt)) in ep_read_events_proc()
820 __pm_relax(ep_wakeup_source(epi)); in ep_read_events_proc()
821 list_del_init(&epi->rdllink); in ep_read_events_proc()
880 struct epitem *epi = rb_entry(rbp, struct epitem, rbn); in ep_show_fdinfo() local
883 epi->ffd.fd, epi->event.events, in ep_show_fdinfo()
884 (long long)epi->event.data); in ep_show_fdinfo()
910 struct epitem *epi, *next; in eventpoll_release_file() local
926 list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) { in eventpoll_release_file()
927 ep = epi->ep; in eventpoll_release_file()
929 ep_remove(ep, epi); in eventpoll_release_file()
974 struct epitem *epi, *epir = NULL; in ep_find() local
979 epi = rb_entry(rbp, struct epitem, rbn); in ep_find()
980 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); in ep_find()
986 epir = epi; in ep_find()
1003 struct epitem *epi = ep_item_from_wait(wait); in ep_poll_callback() local
1004 struct eventpoll *ep = epi->ep; in ep_poll_callback()
1025 if (!(epi->event.events & ~EP_PRIVATE_BITS)) in ep_poll_callback()
1034 if (key && !((unsigned long) key & epi->event.events)) in ep_poll_callback()
1044 if (epi->next == EP_UNACTIVE_PTR) { in ep_poll_callback()
1045 epi->next = ep->ovflist; in ep_poll_callback()
1046 ep->ovflist = epi; in ep_poll_callback()
1047 if (epi->ws) { in ep_poll_callback()
1060 if (!ep_is_linked(&epi->rdllink)) { in ep_poll_callback()
1061 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_poll_callback()
1062 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1091 struct epitem *epi = ep_item_from_epqueue(pt); in ep_ptable_queue_proc() local
1094 if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) { in ep_ptable_queue_proc()
1097 pwq->base = epi; in ep_ptable_queue_proc()
1099 list_add_tail(&pwq->llink, &epi->pwqlist); in ep_ptable_queue_proc()
1100 epi->nwait++; in ep_ptable_queue_proc()
1103 epi->nwait = -1; in ep_ptable_queue_proc()
1107 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) in ep_rbtree_insert() argument
1116 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); in ep_rbtree_insert()
1122 rb_link_node(&epi->rbn, parent, p); in ep_rbtree_insert()
1123 rb_insert_color(&epi->rbn, &ep->rbr); in ep_rbtree_insert()
1167 struct epitem *epi; in reverse_path_check_proc() local
1171 list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) { in reverse_path_check_proc()
1172 child_file = epi->ep->file; in reverse_path_check_proc()
1224 static int ep_create_wakeup_source(struct epitem *epi) in ep_create_wakeup_source() argument
1229 if (!epi->ep->ws) { in ep_create_wakeup_source()
1230 epi->ep->ws = wakeup_source_register("eventpoll"); in ep_create_wakeup_source()
1231 if (!epi->ep->ws) in ep_create_wakeup_source()
1235 name = epi->ffd.file->f_path.dentry->d_name.name; in ep_create_wakeup_source()
1240 rcu_assign_pointer(epi->ws, ws); in ep_create_wakeup_source()
1246 static noinline void ep_destroy_wakeup_source(struct epitem *epi) in ep_destroy_wakeup_source() argument
1248 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_destroy_wakeup_source()
1250 RCU_INIT_POINTER(epi->ws, NULL); in ep_destroy_wakeup_source()
1270 struct epitem *epi; in ep_insert() local
1276 if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL))) in ep_insert()
1280 INIT_LIST_HEAD(&epi->rdllink); in ep_insert()
1281 INIT_LIST_HEAD(&epi->fllink); in ep_insert()
1282 INIT_LIST_HEAD(&epi->pwqlist); in ep_insert()
1283 epi->ep = ep; in ep_insert()
1284 ep_set_ffd(&epi->ffd, tfile, fd); in ep_insert()
1285 epi->event = *event; in ep_insert()
1286 epi->nwait = 0; in ep_insert()
1287 epi->next = EP_UNACTIVE_PTR; in ep_insert()
1288 if (epi->event.events & EPOLLWAKEUP) { in ep_insert()
1289 error = ep_create_wakeup_source(epi); in ep_insert()
1293 RCU_INIT_POINTER(epi->ws, NULL); in ep_insert()
1297 epq.epi = epi; in ep_insert()
1307 revents = ep_item_poll(epi, &epq.pt); in ep_insert()
1315 if (epi->nwait < 0) in ep_insert()
1320 list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links); in ep_insert()
1327 ep_rbtree_insert(ep, epi); in ep_insert()
1338 if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) { in ep_insert()
1339 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1340 ep_pm_stay_awake(epi); in ep_insert()
1361 list_del_rcu(&epi->fllink); in ep_insert()
1364 rb_erase(&epi->rbn, &ep->rbr); in ep_insert()
1367 ep_unregister_pollwait(ep, epi); in ep_insert()
1376 if (ep_is_linked(&epi->rdllink)) in ep_insert()
1377 list_del_init(&epi->rdllink); in ep_insert()
1380 wakeup_source_unregister(ep_wakeup_source(epi)); in ep_insert()
1383 kmem_cache_free(epi_cache, epi); in ep_insert()
1392 static int ep_modify(struct eventpoll *ep, struct epitem *epi, struct epoll_event *event) in ep_modify() argument
1405 epi->event.events = event->events; /* need barrier below */ in ep_modify()
1406 epi->event.data = event->data; /* protected by mtx */ in ep_modify()
1407 if (epi->event.events & EPOLLWAKEUP) { in ep_modify()
1408 if (!ep_has_wakeup_source(epi)) in ep_modify()
1409 ep_create_wakeup_source(epi); in ep_modify()
1410 } else if (ep_has_wakeup_source(epi)) { in ep_modify()
1411 ep_destroy_wakeup_source(epi); in ep_modify()
1438 revents = ep_item_poll(epi, &pt); in ep_modify()
1446 if (!ep_is_linked(&epi->rdllink)) { in ep_modify()
1447 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1448 ep_pm_stay_awake(epi); in ep_modify()
1472 struct epitem *epi; in ep_send_events_proc() local
1486 epi = list_first_entry(head, struct epitem, rdllink); in ep_send_events_proc()
1497 ws = ep_wakeup_source(epi); in ep_send_events_proc()
1504 list_del_init(&epi->rdllink); in ep_send_events_proc()
1506 revents = ep_item_poll(epi, &pt); in ep_send_events_proc()
1516 __put_user(epi->event.data, &uevent->data)) { in ep_send_events_proc()
1517 list_add(&epi->rdllink, head); in ep_send_events_proc()
1518 ep_pm_stay_awake(epi); in ep_send_events_proc()
1523 if (epi->event.events & EPOLLONESHOT) in ep_send_events_proc()
1524 epi->event.events &= EP_PRIVATE_BITS; in ep_send_events_proc()
1525 else if (!(epi->event.events & EPOLLET)) { in ep_send_events_proc()
1537 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events_proc()
1538 ep_pm_stay_awake(epi); in ep_send_events_proc()
1686 struct epitem *epi; in ep_loop_check_proc() local
1692 epi = rb_entry(rbp, struct epitem, rbn); in ep_loop_check_proc()
1693 if (unlikely(is_file_epoll(epi->ffd.file))) { in ep_loop_check_proc()
1694 ep_tovisit = epi->ffd.file->private_data; in ep_loop_check_proc()
1698 ep_loop_check_proc, epi->ffd.file, in ep_loop_check_proc()
1711 if (list_empty(&epi->ffd.file->f_tfile_llink)) in ep_loop_check_proc()
1712 list_add(&epi->ffd.file->f_tfile_llink, in ep_loop_check_proc()
1827 struct epitem *epi; in SYSCALL_DEFINE4() local
1914 epi = ep_find(ep, tf.file, fd); in SYSCALL_DEFINE4()
1919 if (!epi) { in SYSCALL_DEFINE4()
1928 if (epi) in SYSCALL_DEFINE4()
1929 error = ep_remove(ep, epi); in SYSCALL_DEFINE4()
1934 if (epi) { in SYSCALL_DEFINE4()
1936 error = ep_modify(ep, epi, &epds); in SYSCALL_DEFINE4()