Lines Matching refs:ep
163 struct eventpoll *ep; member
377 static inline int ep_events_available(struct eventpoll *ep) in ep_events_available() argument
379 return !list_empty(&ep->rdllist) || ep->ovflist != EP_UNACTIVE_PTR; in ep_events_available()
533 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) in ep_unregister_pollwait() argument
550 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
592 static int ep_scan_ready_list(struct eventpoll *ep, in ep_scan_ready_list() argument
608 mutex_lock_nested(&ep->mtx, depth); in ep_scan_ready_list()
618 spin_lock_irqsave(&ep->lock, flags); in ep_scan_ready_list()
619 list_splice_init(&ep->rdllist, &txlist); in ep_scan_ready_list()
620 ep->ovflist = NULL; in ep_scan_ready_list()
621 spin_unlock_irqrestore(&ep->lock, flags); in ep_scan_ready_list()
626 error = (*sproc)(ep, &txlist, priv); in ep_scan_ready_list()
628 spin_lock_irqsave(&ep->lock, flags); in ep_scan_ready_list()
634 for (nepi = ep->ovflist; (epi = nepi) != NULL; in ep_scan_ready_list()
643 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_scan_ready_list()
652 ep->ovflist = EP_UNACTIVE_PTR; in ep_scan_ready_list()
657 list_splice(&txlist, &ep->rdllist); in ep_scan_ready_list()
658 __pm_relax(ep->ws); in ep_scan_ready_list()
660 if (!list_empty(&ep->rdllist)) { in ep_scan_ready_list()
665 if (waitqueue_active(&ep->wq)) in ep_scan_ready_list()
666 wake_up_locked(&ep->wq); in ep_scan_ready_list()
667 if (waitqueue_active(&ep->poll_wait)) in ep_scan_ready_list()
670 spin_unlock_irqrestore(&ep->lock, flags); in ep_scan_ready_list()
673 mutex_unlock(&ep->mtx); in ep_scan_ready_list()
677 ep_poll_safewake(&ep->poll_wait); in ep_scan_ready_list()
692 static int ep_remove(struct eventpoll *ep, struct epitem *epi) in ep_remove() argument
705 ep_unregister_pollwait(ep, epi); in ep_remove()
712 rb_erase(&epi->rbn, &ep->rbr); in ep_remove()
714 spin_lock_irqsave(&ep->lock, flags); in ep_remove()
717 spin_unlock_irqrestore(&ep->lock, flags); in ep_remove()
729 atomic_long_dec(&ep->user->epoll_watches); in ep_remove()
734 static void ep_free(struct eventpoll *ep) in ep_free() argument
740 if (waitqueue_active(&ep->poll_wait)) in ep_free()
741 ep_poll_safewake(&ep->poll_wait); in ep_free()
756 for (rbp = rb_first(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_free()
759 ep_unregister_pollwait(ep, epi); in ep_free()
771 mutex_lock(&ep->mtx); in ep_free()
772 while ((rbp = rb_first(&ep->rbr)) != NULL) { in ep_free()
774 ep_remove(ep, epi); in ep_free()
777 mutex_unlock(&ep->mtx); in ep_free()
780 mutex_destroy(&ep->mtx); in ep_free()
781 free_uid(ep->user); in ep_free()
782 wakeup_source_unregister(ep->ws); in ep_free()
783 kfree(ep); in ep_free()
788 struct eventpoll *ep = file->private_data; in ep_eventpoll_release() local
790 if (ep) in ep_eventpoll_release()
791 ep_free(ep); in ep_eventpoll_release()
803 static int ep_read_events_proc(struct eventpoll *ep, struct list_head *head, in ep_read_events_proc() argument
832 struct eventpoll *ep; member
840 return ep_scan_ready_list(arg->ep, ep_read_events_proc, NULL, in ep_poll_readyevents_proc()
847 struct eventpoll *ep = file->private_data; in ep_eventpoll_poll() local
855 arg.ep = ep; in ep_eventpoll_poll()
858 poll_wait(file, &ep->poll_wait, wait); in ep_eventpoll_poll()
867 ep_poll_readyevents_proc, &arg, ep, current); in ep_eventpoll_poll()
875 struct eventpoll *ep = f->private_data; in ep_show_fdinfo() local
878 mutex_lock(&ep->mtx); in ep_show_fdinfo()
879 for (rbp = rb_first(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_show_fdinfo()
888 mutex_unlock(&ep->mtx); in ep_show_fdinfo()
909 struct eventpoll *ep; in eventpoll_release_file() local
927 ep = epi->ep; in eventpoll_release_file()
928 mutex_lock_nested(&ep->mtx, 0); in eventpoll_release_file()
929 ep_remove(ep, epi); in eventpoll_release_file()
930 mutex_unlock(&ep->mtx); in eventpoll_release_file()
939 struct eventpoll *ep; in ep_alloc() local
943 ep = kzalloc(sizeof(*ep), GFP_KERNEL); in ep_alloc()
944 if (unlikely(!ep)) in ep_alloc()
947 spin_lock_init(&ep->lock); in ep_alloc()
948 mutex_init(&ep->mtx); in ep_alloc()
949 init_waitqueue_head(&ep->wq); in ep_alloc()
950 init_waitqueue_head(&ep->poll_wait); in ep_alloc()
951 INIT_LIST_HEAD(&ep->rdllist); in ep_alloc()
952 ep->rbr = RB_ROOT; in ep_alloc()
953 ep->ovflist = EP_UNACTIVE_PTR; in ep_alloc()
954 ep->user = user; in ep_alloc()
956 *pep = ep; in ep_alloc()
970 static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd) in ep_find() argument
978 for (rbp = ep->rbr.rb_node; rbp; ) { in ep_find()
1004 struct eventpoll *ep = epi->ep; in ep_poll_callback() local
1017 spin_lock_irqsave(&ep->lock, flags); in ep_poll_callback()
1043 if (unlikely(ep->ovflist != EP_UNACTIVE_PTR)) { in ep_poll_callback()
1045 epi->next = ep->ovflist; in ep_poll_callback()
1046 ep->ovflist = epi; in ep_poll_callback()
1052 __pm_stay_awake(ep->ws); in ep_poll_callback()
1061 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_poll_callback()
1069 if (waitqueue_active(&ep->wq)) in ep_poll_callback()
1070 wake_up_locked(&ep->wq); in ep_poll_callback()
1071 if (waitqueue_active(&ep->poll_wait)) in ep_poll_callback()
1075 spin_unlock_irqrestore(&ep->lock, flags); in ep_poll_callback()
1079 ep_poll_safewake(&ep->poll_wait); in ep_poll_callback()
1107 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) in ep_rbtree_insert() argument
1110 struct rb_node **p = &ep->rbr.rb_node, *parent = NULL; in ep_rbtree_insert()
1123 rb_insert_color(&epi->rbn, &ep->rbr); in ep_rbtree_insert()
1172 child_file = epi->ep->file; in reverse_path_check_proc()
1229 if (!epi->ep->ws) { in ep_create_wakeup_source()
1230 epi->ep->ws = wakeup_source_register("eventpoll"); in ep_create_wakeup_source()
1231 if (!epi->ep->ws) in ep_create_wakeup_source()
1264 static int ep_insert(struct eventpoll *ep, struct epoll_event *event, in ep_insert() argument
1273 user_watches = atomic_long_read(&ep->user->epoll_watches); in ep_insert()
1283 epi->ep = ep; in ep_insert()
1327 ep_rbtree_insert(ep, epi); in ep_insert()
1335 spin_lock_irqsave(&ep->lock, flags); in ep_insert()
1339 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1343 if (waitqueue_active(&ep->wq)) in ep_insert()
1344 wake_up_locked(&ep->wq); in ep_insert()
1345 if (waitqueue_active(&ep->poll_wait)) in ep_insert()
1349 spin_unlock_irqrestore(&ep->lock, flags); in ep_insert()
1351 atomic_long_inc(&ep->user->epoll_watches); in ep_insert()
1355 ep_poll_safewake(&ep->poll_wait); in ep_insert()
1364 rb_erase(&epi->rbn, &ep->rbr); in ep_insert()
1367 ep_unregister_pollwait(ep, epi); in ep_insert()
1375 spin_lock_irqsave(&ep->lock, flags); in ep_insert()
1378 spin_unlock_irqrestore(&ep->lock, flags); in ep_insert()
1392 static int ep_modify(struct eventpoll *ep, struct epitem *epi, struct epoll_event *event) in ep_modify() argument
1445 spin_lock_irq(&ep->lock); in ep_modify()
1447 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1451 if (waitqueue_active(&ep->wq)) in ep_modify()
1452 wake_up_locked(&ep->wq); in ep_modify()
1453 if (waitqueue_active(&ep->poll_wait)) in ep_modify()
1456 spin_unlock_irq(&ep->lock); in ep_modify()
1461 ep_poll_safewake(&ep->poll_wait); in ep_modify()
1466 static int ep_send_events_proc(struct eventpoll *ep, struct list_head *head, in ep_send_events_proc() argument
1500 __pm_stay_awake(ep->ws); in ep_send_events_proc()
1537 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events_proc()
1546 static int ep_send_events(struct eventpoll *ep, in ep_send_events() argument
1554 return ep_scan_ready_list(ep, ep_send_events_proc, &esed, 0, false); in ep_send_events()
1585 static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, in ep_poll() argument
1606 spin_lock_irqsave(&ep->lock, flags); in ep_poll()
1611 spin_lock_irqsave(&ep->lock, flags); in ep_poll()
1613 if (!ep_events_available(ep)) { in ep_poll()
1620 __add_wait_queue_exclusive(&ep->wq, &wait); in ep_poll()
1629 if (ep_events_available(ep) || timed_out) in ep_poll()
1636 spin_unlock_irqrestore(&ep->lock, flags); in ep_poll()
1640 spin_lock_irqsave(&ep->lock, flags); in ep_poll()
1643 __remove_wait_queue(&ep->wq, &wait); in ep_poll()
1648 eavail = ep_events_available(ep); in ep_poll()
1650 spin_unlock_irqrestore(&ep->lock, flags); in ep_poll()
1658 !(res = ep_send_events(ep, events, maxevents)) && !timed_out) in ep_poll()
1683 struct eventpoll *ep = file->private_data; in ep_loop_check_proc() local
1688 mutex_lock_nested(&ep->mtx, call_nests + 1); in ep_loop_check_proc()
1689 ep->visited = 1; in ep_loop_check_proc()
1690 list_add(&ep->visited_list_link, &visited_list); in ep_loop_check_proc()
1691 for (rbp = rb_first(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_loop_check_proc()
1716 mutex_unlock(&ep->mtx); in ep_loop_check_proc()
1732 static int ep_loop_check(struct eventpoll *ep, struct file *file) in ep_loop_check() argument
1738 ep_loop_check_proc, file, ep, current); in ep_loop_check()
1767 struct eventpoll *ep = NULL; in SYSCALL_DEFINE1() local
1778 error = ep_alloc(&ep); in SYSCALL_DEFINE1()
1790 file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep, in SYSCALL_DEFINE1()
1796 ep->file = file; in SYSCALL_DEFINE1()
1803 ep_free(ep); in SYSCALL_DEFINE1()
1826 struct eventpoll *ep; in SYSCALL_DEFINE4() local
1868 ep = f.file->private_data; in SYSCALL_DEFINE4()
1885 mutex_lock_nested(&ep->mtx, 0); in SYSCALL_DEFINE4()
1890 mutex_unlock(&ep->mtx); in SYSCALL_DEFINE4()
1894 if (ep_loop_check(ep, tf.file) != 0) { in SYSCALL_DEFINE4()
1901 mutex_lock_nested(&ep->mtx, 0); in SYSCALL_DEFINE4()
1914 epi = ep_find(ep, tf.file, fd); in SYSCALL_DEFINE4()
1921 error = ep_insert(ep, &epds, tf.file, fd, full_check); in SYSCALL_DEFINE4()
1929 error = ep_remove(ep, epi); in SYSCALL_DEFINE4()
1936 error = ep_modify(ep, epi, &epds); in SYSCALL_DEFINE4()
1943 mutex_unlock(&ep->mtx); in SYSCALL_DEFINE4()
1966 struct eventpoll *ep; in SYSCALL_DEFINE4() local
1993 ep = f.file->private_data; in SYSCALL_DEFINE4()
1996 error = ep_poll(ep, events, maxevents, timeout); in SYSCALL_DEFINE4()