1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * fs/eventpoll.c (Efficient event retrieval implementation)
4 * Copyright (C) 2001,...,2009 Davide Libenzi
5 *
6 * Davide Libenzi <davidel@xmailserver.org>
7 */
8
9 #include <linux/init.h>
10 #include <linux/kernel.h>
11 #include <linux/sched/signal.h>
12 #include <linux/fs.h>
13 #include <linux/file.h>
14 #include <linux/signal.h>
15 #include <linux/errno.h>
16 #include <linux/mm.h>
17 #include <linux/slab.h>
18 #include <linux/poll.h>
19 #include <linux/string.h>
20 #include <linux/list.h>
21 #include <linux/hash.h>
22 #include <linux/spinlock.h>
23 #include <linux/syscalls.h>
24 #include <linux/rbtree.h>
25 #include <linux/wait.h>
26 #include <linux/eventpoll.h>
27 #include <linux/mount.h>
28 #include <linux/bitops.h>
29 #include <linux/mutex.h>
30 #include <linux/anon_inodes.h>
31 #include <linux/device.h>
32 #include <linux/uaccess.h>
33 #include <asm/io.h>
34 #include <asm/mman.h>
35 #include <linux/atomic.h>
36 #include <linux/proc_fs.h>
37 #include <linux/seq_file.h>
38 #include <linux/compat.h>
39 #include <linux/rculist.h>
40 #include <net/busy_poll.h>
41
42 /*
43 * LOCKING:
44 * There are three level of locking required by epoll :
45 *
46 * 1) epmutex (mutex)
47 * 2) ep->mtx (mutex)
48 * 3) ep->lock (rwlock)
49 *
50 * The acquire order is the one listed above, from 1 to 3.
51 * We need a rwlock (ep->lock) because we manipulate objects
52 * from inside the poll callback, that might be triggered from
53 * a wake_up() that in turn might be called from IRQ context.
54 * So we can't sleep inside the poll callback and hence we need
55 * a spinlock. During the event transfer loop (from kernel to
56 * user space) we could end up sleeping due a copy_to_user(), so
57 * we need a lock that will allow us to sleep. This lock is a
58 * mutex (ep->mtx). It is acquired during the event transfer loop,
59 * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file().
60 * Then we also need a global mutex to serialize eventpoll_release_file()
61 * and ep_free().
62 * This mutex is acquired by ep_free() during the epoll file
63 * cleanup path and it is also acquired by eventpoll_release_file()
64 * if a file has been pushed inside an epoll set and it is then
65 * close()d without a previous call to epoll_ctl(EPOLL_CTL_DEL).
66 * It is also acquired when inserting an epoll fd onto another epoll
67 * fd. We do this so that we walk the epoll tree and ensure that this
68 * insertion does not create a cycle of epoll file descriptors, which
69 * could lead to deadlock. We need a global mutex to prevent two
70 * simultaneous inserts (A into B and B into A) from racing and
71 * constructing a cycle without either insert observing that it is
72 * going to.
73 * It is necessary to acquire multiple "ep->mtx"es at once in the
74 * case when one epoll fd is added to another. In this case, we
75 * always acquire the locks in the order of nesting (i.e. after
76 * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
77 * before e2->mtx). Since we disallow cycles of epoll file
78 * descriptors, this ensures that the mutexes are well-ordered. In
79 * order to communicate this nesting to lockdep, when walking a tree
80 * of epoll file descriptors, we use the current recursion depth as
81 * the lockdep subkey.
82 * It is possible to drop the "ep->mtx" and to use the global
83 * mutex "epmutex" (together with "ep->lock") to have it working,
84 * but having "ep->mtx" will make the interface more scalable.
85 * Events that require holding "epmutex" are very rare, while for
86 * normal operations the epoll private "ep->mtx" will guarantee
87 * a better scalability.
88 */
89
90 /* Epoll private bits inside the event mask */
91 #define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE)
92
93 #define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT)
94
95 #define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \
96 EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE)
97
98 /* Maximum number of nesting allowed inside epoll sets */
99 #define EP_MAX_NESTS 4
100
101 #define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))
102
103 #define EP_UNACTIVE_PTR ((void *) -1L)
104
105 #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry))
106
107 struct epoll_filefd {
108 struct file *file;
109 int fd;
110 } __packed;
111
112 /*
113 * Structure used to track possible nested calls, for too deep recursions
114 * and loop cycles.
115 */
116 struct nested_call_node {
117 struct list_head llink;
118 void *cookie;
119 void *ctx;
120 };
121
122 /*
123 * This structure is used as collector for nested calls, to check for
124 * maximum recursion dept and loop cycles.
125 */
126 struct nested_calls {
127 struct list_head tasks_call_list;
128 spinlock_t lock;
129 };
130
131 /*
132 * Each file descriptor added to the eventpoll interface will
133 * have an entry of this type linked to the "rbr" RB tree.
134 * Avoid increasing the size of this struct, there can be many thousands
135 * of these on a server and we do not want this to take another cache line.
136 */
137 struct epitem {
138 union {
139 /* RB tree node links this structure to the eventpoll RB tree */
140 struct rb_node rbn;
141 /* Used to free the struct epitem */
142 struct rcu_head rcu;
143 };
144
145 /* List header used to link this structure to the eventpoll ready list */
146 struct list_head rdllink;
147
148 /*
149 * Works together "struct eventpoll"->ovflist in keeping the
150 * single linked chain of items.
151 */
152 struct epitem *next;
153
154 /* The file descriptor information this item refers to */
155 struct epoll_filefd ffd;
156
157 /* Number of active wait queue attached to poll operations */
158 int nwait;
159
160 /* List containing poll wait queues */
161 struct list_head pwqlist;
162
163 /* The "container" of this item */
164 struct eventpoll *ep;
165
166 /* List header used to link this item to the "struct file" items list */
167 struct list_head fllink;
168
169 /* wakeup_source used when EPOLLWAKEUP is set */
170 struct wakeup_source __rcu *ws;
171
172 /* The structure that describe the interested events and the source fd */
173 struct epoll_event event;
174 };
175
176 /*
177 * This structure is stored inside the "private_data" member of the file
178 * structure and represents the main data structure for the eventpoll
179 * interface.
180 */
181 struct eventpoll {
182 /*
183 * This mutex is used to ensure that files are not removed
184 * while epoll is using them. This is held during the event
185 * collection loop, the file cleanup path, the epoll file exit
186 * code and the ctl operations.
187 */
188 struct mutex mtx;
189
190 /* Wait queue used by sys_epoll_wait() */
191 wait_queue_head_t wq;
192
193 /* Wait queue used by file->poll() */
194 wait_queue_head_t poll_wait;
195
196 /* List of ready file descriptors */
197 struct list_head rdllist;
198
199 /* Lock which protects rdllist and ovflist */
200 rwlock_t lock;
201
202 /* RB tree root used to store monitored fd structs */
203 struct rb_root_cached rbr;
204
205 /*
206 * This is a single linked list that chains all the "struct epitem" that
207 * happened while transferring ready events to userspace w/out
208 * holding ->lock.
209 */
210 struct epitem *ovflist;
211
212 /* wakeup_source used when ep_scan_ready_list is running */
213 struct wakeup_source *ws;
214
215 /* The user that created the eventpoll descriptor */
216 struct user_struct *user;
217
218 struct file *file;
219
220 /* used to optimize loop detection check */
221 int visited;
222 struct list_head visited_list_link;
223
224 #ifdef CONFIG_NET_RX_BUSY_POLL
225 /* used to track busy poll napi_id */
226 unsigned int napi_id;
227 #endif
228 };
229
230 /* Wait structure used by the poll hooks */
231 struct eppoll_entry {
232 /* List header used to link this structure to the "struct epitem" */
233 struct list_head llink;
234
235 /* The "base" pointer is set to the container "struct epitem" */
236 struct epitem *base;
237
238 /*
239 * Wait queue item that will be linked to the target file wait
240 * queue head.
241 */
242 wait_queue_entry_t wait;
243
244 /* The wait queue head that linked the "wait" wait queue item */
245 wait_queue_head_t *whead;
246 };
247
248 /* Wrapper struct used by poll queueing */
249 struct ep_pqueue {
250 poll_table pt;
251 struct epitem *epi;
252 };
253
254 /* Used by the ep_send_events() function as callback private data */
255 struct ep_send_events_data {
256 int maxevents;
257 struct epoll_event __user *events;
258 int res;
259 };
260
261 /*
262 * Configuration options available inside /proc/sys/fs/epoll/
263 */
264 /* Maximum number of epoll watched descriptors, per user */
265 static long max_user_watches __read_mostly;
266
267 /*
268 * This mutex is used to serialize ep_free() and eventpoll_release_file().
269 */
270 static DEFINE_MUTEX(epmutex);
271
272 /* Used to check for epoll file descriptor inclusion loops */
273 static struct nested_calls poll_loop_ncalls;
274
275 /* Slab cache used to allocate "struct epitem" */
276 static struct kmem_cache *epi_cache __read_mostly;
277
278 /* Slab cache used to allocate "struct eppoll_entry" */
279 static struct kmem_cache *pwq_cache __read_mostly;
280
281 /* Visited nodes during ep_loop_check(), so we can unset them when we finish */
282 static LIST_HEAD(visited_list);
283
284 /*
285 * List of files with newly added links, where we may need to limit the number
286 * of emanating paths. Protected by the epmutex.
287 */
288 static LIST_HEAD(tfile_check_list);
289
290 #ifdef CONFIG_SYSCTL
291
292 #include <linux/sysctl.h>
293
294 static long long_zero;
295 static long long_max = LONG_MAX;
296
297 struct ctl_table epoll_table[] = {
298 {
299 .procname = "max_user_watches",
300 .data = &max_user_watches,
301 .maxlen = sizeof(max_user_watches),
302 .mode = 0644,
303 .proc_handler = proc_doulongvec_minmax,
304 .extra1 = &long_zero,
305 .extra2 = &long_max,
306 },
307 { }
308 };
309 #endif /* CONFIG_SYSCTL */
310
311 static const struct file_operations eventpoll_fops;
312
313 static inline int is_file_epoll(struct file *f)
314 {
315 return f->f_op == &eventpoll_fops;
316 }
317
318 /* Setup the structure that is used as key for the RB tree */
319 static inline void ep_set_ffd(struct epoll_filefd *ffd,
320 struct file *file, int fd)
321 {
322 ffd->file = file;
323 ffd->fd = fd;
324 }
325
326 /* Compare RB tree keys */
327 static inline int ep_cmp_ffd(struct epoll_filefd *p1,
328 struct epoll_filefd *p2)
329 {
330 return (p1->file > p2->file ? +1:
331 (p1->file < p2->file ? -1 : p1->fd - p2->fd));
332 }
333
334 /* Tells us if the item is currently linked */
335 static inline int ep_is_linked(struct epitem *epi)
336 {
337 return !list_empty(&epi->rdllink);
338 }
339
340 static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_entry_t *p)
341 {
342 return container_of(p, struct eppoll_entry, wait);
343 }
344
345 /* Get the "struct epitem" from a wait queue pointer */
346 static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p)
347 {
348 return container_of(p, struct eppoll_entry, wait)->base;
349 }
350
351 /* Get the "struct epitem" from an epoll queue wrapper */
352 static inline struct epitem *ep_item_from_epqueue(poll_table *p)
353 {
354 return container_of(p, struct ep_pqueue, pt)->epi;
355 }
356
357 /* Tells if the epoll_ctl(2) operation needs an event copy from userspace */
358 static inline int ep_op_has_event(int op)
359 {
360 return op != EPOLL_CTL_DEL;
361 }
362
363 /* Initialize the poll safe wake up structure */
364 static void ep_nested_calls_init(struct nested_calls *ncalls)
365 {
366 INIT_LIST_HEAD(&ncalls->tasks_call_list);
367 spin_lock_init(&ncalls->lock);
368 }
369
370 /**
371 * ep_events_available - Checks if ready events might be available.
372 *
373 * @ep: Pointer to the eventpoll context.
374 *
375 * Returns: Returns a value different than zero if ready events are available,
376 * or zero otherwise.
377 */
378 static inline int ep_events_available(struct eventpoll *ep)
379 {
380 return !list_empty_careful(&ep->rdllist) ||
381 READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;
382 }
383
384 #ifdef CONFIG_NET_RX_BUSY_POLL
385 static bool ep_busy_loop_end(void *p, unsigned long start_time)
386 {
387 struct eventpoll *ep = p;
388
389 return ep_events_available(ep) || busy_loop_timeout(start_time);
390 }
391
392 /*
393 * Busy poll if globally on and supporting sockets found && no events,
394 * busy loop will return if need_resched or ep_events_available.
395 *
396 * we must do our busy polling with irqs enabled
397 */
398 static void ep_busy_loop(struct eventpoll *ep, int nonblock)
399 {
400 unsigned int napi_id = READ_ONCE(ep->napi_id);
401
402 if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on())
403 napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep);
404 }
405
406 static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
407 {
408 if (ep->napi_id)
409 ep->napi_id = 0;
410 }
411
412 /*
413 * Set epoll busy poll NAPI ID from sk.
414 */
415 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
416 {
417 struct eventpoll *ep;
418 unsigned int napi_id;
419 struct socket *sock;
420 struct sock *sk;
421 int err;
422
423 if (!net_busy_loop_on())
424 return;
425
426 sock = sock_from_file(epi->ffd.file, &err);
427 if (!sock)
428 return;
429
430 sk = sock->sk;
431 if (!sk)
432 return;
433
434 napi_id = READ_ONCE(sk->sk_napi_id);
435 ep = epi->ep;
436
437 /* Non-NAPI IDs can be rejected
438 * or
439 * Nothing to do if we already have this ID
440 */
441 if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id)
442 return;
443
444 /* record NAPI ID for use in next busy poll */
445 ep->napi_id = napi_id;
446 }
447
448 #else
449
450 static inline void ep_busy_loop(struct eventpoll *ep, int nonblock)
451 {
452 }
453
454 static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
455 {
456 }
457
458 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
459 {
460 }
461
462 #endif /* CONFIG_NET_RX_BUSY_POLL */
463
464 /**
465 * ep_call_nested - Perform a bound (possibly) nested call, by checking
466 * that the recursion limit is not exceeded, and that
467 * the same nested call (by the meaning of same cookie) is
468 * no re-entered.
469 *
470 * @ncalls: Pointer to the nested_calls structure to be used for this call.
471 * @nproc: Nested call core function pointer.
472 * @priv: Opaque data to be passed to the @nproc callback.
473 * @cookie: Cookie to be used to identify this nested call.
474 * @ctx: This instance context.
475 *
476 * Returns: Returns the code returned by the @nproc callback, or -1 if
477 * the maximum recursion limit has been exceeded.
478 */
479 static int ep_call_nested(struct nested_calls *ncalls,
480 int (*nproc)(void *, void *, int), void *priv,
481 void *cookie, void *ctx)
482 {
483 int error, call_nests = 0;
484 unsigned long flags;
485 struct list_head *lsthead = &ncalls->tasks_call_list;
486 struct nested_call_node *tncur;
487 struct nested_call_node tnode;
488
489 spin_lock_irqsave(&ncalls->lock, flags);
490
491 /*
492 * Try to see if the current task is already inside this wakeup call.
493 * We use a list here, since the population inside this set is always
494 * very much limited.
495 */
496 list_for_each_entry(tncur, lsthead, llink) {
497 if (tncur->ctx == ctx &&
498 (tncur->cookie == cookie || ++call_nests > EP_MAX_NESTS)) {
499 /*
500 * Ops ... loop detected or maximum nest level reached.
501 * We abort this wake by breaking the cycle itself.
502 */
503 error = -1;
504 goto out_unlock;
505 }
506 }
507
508 /* Add the current task and cookie to the list */
509 tnode.ctx = ctx;
510 tnode.cookie = cookie;
511 list_add(&tnode.llink, lsthead);
512
513 spin_unlock_irqrestore(&ncalls->lock, flags);
514
515 /* Call the nested function */
516 error = (*nproc)(priv, cookie, call_nests);
517
518 /* Remove the current task from the list */
519 spin_lock_irqsave(&ncalls->lock, flags);
520 list_del(&tnode.llink);
521 out_unlock:
522 spin_unlock_irqrestore(&ncalls->lock, flags);
523
524 return error;
525 }
526
527 /*
528 * As described in commit 0ccf831cb lockdep: annotate epoll
529 * the use of wait queues used by epoll is done in a very controlled
530 * manner. Wake ups can nest inside each other, but are never done
531 * with the same locking. For example:
532 *
533 * dfd = socket(...);
534 * efd1 = epoll_create();
535 * efd2 = epoll_create();
536 * epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...);
537 * epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...);
538 *
539 * When a packet arrives to the device underneath "dfd", the net code will
540 * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a
541 * callback wakeup entry on that queue, and the wake_up() performed by the
542 * "dfd" net code will end up in ep_poll_callback(). At this point epoll
543 * (efd1) notices that it may have some event ready, so it needs to wake up
544 * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()
545 * that ends up in another wake_up(), after having checked about the
546 * recursion constraints. That are, no more than EP_MAX_POLLWAKE_NESTS, to
547 * avoid stack blasting.
548 *
549 * When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle
550 * this special case of epoll.
551 */
552 #ifdef CONFIG_DEBUG_LOCK_ALLOC
553
554 static struct nested_calls poll_safewake_ncalls;
555
556 static int ep_poll_wakeup_proc(void *priv, void *cookie, int call_nests)
557 {
558 unsigned long flags;
559 wait_queue_head_t *wqueue = (wait_queue_head_t *)cookie;
560
561 spin_lock_irqsave_nested(&wqueue->lock, flags, call_nests + 1);
562 wake_up_locked_poll(wqueue, EPOLLIN);
563 spin_unlock_irqrestore(&wqueue->lock, flags);
564
565 return 0;
566 }
567
568 static void ep_poll_safewake(wait_queue_head_t *wq)
569 {
570 int this_cpu = get_cpu();
571
572 ep_call_nested(&poll_safewake_ncalls,
573 ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu);
574
575 put_cpu();
576 }
577
578 #else
579
580 static void ep_poll_safewake(wait_queue_head_t *wq)
581 {
582 wake_up_poll(wq, EPOLLIN);
583 }
584
585 #endif
586
587 static void ep_remove_wait_queue(struct eppoll_entry *pwq)
588 {
589 wait_queue_head_t *whead;
590
591 rcu_read_lock();
592 /*
593 * If it is cleared by POLLFREE, it should be rcu-safe.
594 * If we read NULL we need a barrier paired with
595 * smp_store_release() in ep_poll_callback(), otherwise
596 * we rely on whead->lock.
597 */
598 whead = smp_load_acquire(&pwq->whead);
599 if (whead)
600 remove_wait_queue(whead, &pwq->wait);
601 rcu_read_unlock();
602 }
603
604 /*
605 * This function unregisters poll callbacks from the associated file
606 * descriptor. Must be called with "mtx" held (or "epmutex" if called from
607 * ep_free).
608 */
609 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
610 {
611 struct list_head *lsthead = &epi->pwqlist;
612 struct eppoll_entry *pwq;
613
614 while (!list_empty(lsthead)) {
615 pwq = list_first_entry(lsthead, struct eppoll_entry, llink);
616
617 list_del(&pwq->llink);
618 ep_remove_wait_queue(pwq);
619 kmem_cache_free(pwq_cache, pwq);
620 }
621 }
622
623 /* call only when ep->mtx is held */
624 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)
625 {
626 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));
627 }
628
629 /* call only when ep->mtx is held */
630 static inline void ep_pm_stay_awake(struct epitem *epi)
631 {
632 struct wakeup_source *ws = ep_wakeup_source(epi);
633
634 if (ws)
635 __pm_stay_awake(ws);
636 }
637
638 static inline bool ep_has_wakeup_source(struct epitem *epi)
639 {
640 return rcu_access_pointer(epi->ws) ? true : false;
641 }
642
643 /* call when ep->mtx cannot be held (ep_poll_callback) */
644 static inline void ep_pm_stay_awake_rcu(struct epitem *epi)
645 {
646 struct wakeup_source *ws;
647
648 rcu_read_lock();
649 ws = rcu_dereference(epi->ws);
650 if (ws)
651 __pm_stay_awake(ws);
652 rcu_read_unlock();
653 }
654
655 /**
656 * ep_scan_ready_list - Scans the ready list in a way that makes possible for
657 * the scan code, to call f_op->poll(). Also allows for
658 * O(NumReady) performance.
659 *
660 * @ep: Pointer to the epoll private data structure.
661 * @sproc: Pointer to the scan callback.
662 * @priv: Private opaque data passed to the @sproc callback.
663 * @depth: The current depth of recursive f_op->poll calls.
664 * @ep_locked: caller already holds ep->mtx
665 *
666 * Returns: The same integer error code returned by the @sproc callback.
667 */
668 static __poll_t ep_scan_ready_list(struct eventpoll *ep,
669 __poll_t (*sproc)(struct eventpoll *,
670 struct list_head *, void *),
671 void *priv, int depth, bool ep_locked)
672 {
673 __poll_t res;
674 int pwake = 0;
675 struct epitem *epi, *nepi;
676 LIST_HEAD(txlist);
677
678 lockdep_assert_irqs_enabled();
679
680 /*
681 * We need to lock this because we could be hit by
682 * eventpoll_release_file() and epoll_ctl().
683 */
684
685 if (!ep_locked)
686 mutex_lock_nested(&ep->mtx, depth);
687
688 /*
689 * Steal the ready list, and re-init the original one to the
690 * empty list. Also, set ep->ovflist to NULL so that events
691 * happening while looping w/out locks, are not lost. We cannot
692 * have the poll callback to queue directly on ep->rdllist,
693 * because we want the "sproc" callback to be able to do it
694 * in a lockless way.
695 */
696 write_lock_irq(&ep->lock);
697 list_splice_init(&ep->rdllist, &txlist);
698 WRITE_ONCE(ep->ovflist, NULL);
699 write_unlock_irq(&ep->lock);
700
701 /*
702 * Now call the callback function.
703 */
704 res = (*sproc)(ep, &txlist, priv);
705
706 write_lock_irq(&ep->lock);
707 /*
708 * During the time we spent inside the "sproc" callback, some
709 * other events might have been queued by the poll callback.
710 * We re-insert them inside the main ready-list here.
711 */
712 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;
713 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {
714 /*
715 * We need to check if the item is already in the list.
716 * During the "sproc" callback execution time, items are
717 * queued into ->ovflist but the "txlist" might already
718 * contain them, and the list_splice() below takes care of them.
719 */
720 if (!ep_is_linked(epi)) {
721 /*
722 * ->ovflist is LIFO, so we have to reverse it in order
723 * to keep in FIFO.
724 */
725 list_add(&epi->rdllink, &ep->rdllist);
726 ep_pm_stay_awake(epi);
727 }
728 }
729 /*
730 * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after
731 * releasing the lock, events will be queued in the normal way inside
732 * ep->rdllist.
733 */
734 WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);
735
736 /*
737 * Quickly re-inject items left on "txlist".
738 */
739 list_splice(&txlist, &ep->rdllist);
740 __pm_relax(ep->ws);
741
742 if (!list_empty(&ep->rdllist)) {
743 /*
744 * Wake up (if active) both the eventpoll wait list and
745 * the ->poll() wait list (delayed after we release the lock).
746 */
747 if (waitqueue_active(&ep->wq))
748 wake_up(&ep->wq);
749 if (waitqueue_active(&ep->poll_wait))
750 pwake++;
751 }
752 write_unlock_irq(&ep->lock);
753
754 if (!ep_locked)
755 mutex_unlock(&ep->mtx);
756
757 /* We have to call this outside the lock */
758 if (pwake)
759 ep_poll_safewake(&ep->poll_wait);
760
761 return res;
762 }
763
764 static void epi_rcu_free(struct rcu_head *head)
765 {
766 struct epitem *epi = container_of(head, struct epitem, rcu);
767 kmem_cache_free(epi_cache, epi);
768 }
769
770 /*
771 * Removes a "struct epitem" from the eventpoll RB tree and deallocates
772 * all the associated resources. Must be called with "mtx" held.
773 */
774 static int ep_remove(struct eventpoll *ep, struct epitem *epi)
775 {
776 struct file *file = epi->ffd.file;
777
778 lockdep_assert_irqs_enabled();
779
780 /*
781 * Removes poll wait queue hooks.
782 */
783 ep_unregister_pollwait(ep, epi);
784
785 /* Remove the current item from the list of epoll hooks */
786 spin_lock(&file->f_lock);
787 list_del_rcu(&epi->fllink);
788 spin_unlock(&file->f_lock);
789
790 rb_erase_cached(&epi->rbn, &ep->rbr);
791
792 write_lock_irq(&ep->lock);
793 if (ep_is_linked(epi))
794 list_del_init(&epi->rdllink);
795 write_unlock_irq(&ep->lock);
796
797 wakeup_source_unregister(ep_wakeup_source(epi));
798 /*
799 * At this point it is safe to free the eventpoll item. Use the union
800 * field epi->rcu, since we are trying to minimize the size of
801 * 'struct epitem'. The 'rbn' field is no longer in use. Protected by
802 * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make
803 * use of the rbn field.
804 */
805 call_rcu(&epi->rcu, epi_rcu_free);
806
807 atomic_long_dec(&ep->user->epoll_watches);
808
809 return 0;
810 }
811
812 static void ep_free(struct eventpoll *ep)
813 {
814 struct rb_node *rbp;
815 struct epitem *epi;
816
817 /* We need to release all tasks waiting for these file */
818 if (waitqueue_active(&ep->poll_wait))
819 ep_poll_safewake(&ep->poll_wait);
820
821 /*
822 * We need to lock this because we could be hit by
823 * eventpoll_release_file() while we're freeing the "struct eventpoll".
824 * We do not need to hold "ep->mtx" here because the epoll file
825 * is on the way to be removed and no one has references to it
826 * anymore. The only hit might come from eventpoll_release_file() but
827 * holding "epmutex" is sufficient here.
828 */
829 mutex_lock(&epmutex);
830
831 /*
832 * Walks through the whole tree by unregistering poll callbacks.
833 */
834 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
835 epi = rb_entry(rbp, struct epitem, rbn);
836
837 ep_unregister_pollwait(ep, epi);
838 cond_resched();
839 }
840
841 /*
842 * Walks through the whole tree by freeing each "struct epitem". At this
843 * point we are sure no poll callbacks will be lingering around, and also by
844 * holding "epmutex" we can be sure that no file cleanup code will hit
845 * us during this operation. So we can avoid the lock on "ep->lock".
846 * We do not need to lock ep->mtx, either, we only do it to prevent
847 * a lockdep warning.
848 */
849 mutex_lock(&ep->mtx);
850 while ((rbp = rb_first_cached(&ep->rbr)) != NULL) {
851 epi = rb_entry(rbp, struct epitem, rbn);
852 ep_remove(ep, epi);
853 cond_resched();
854 }
855 mutex_unlock(&ep->mtx);
856
857 mutex_unlock(&epmutex);
858 mutex_destroy(&ep->mtx);
859 free_uid(ep->user);
860 wakeup_source_unregister(ep->ws);
861 kfree(ep);
862 }
863
864 static int ep_eventpoll_release(struct inode *inode, struct file *file)
865 {
866 struct eventpoll *ep = file->private_data;
867
868 if (ep)
869 ep_free(ep);
870
871 return 0;
872 }
873
874 static __poll_t ep_read_events_proc(struct eventpoll *ep, struct list_head *head,
875 void *priv);
876 static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
877 poll_table *pt);
878
879 /*
880 * Differs from ep_eventpoll_poll() in that internal callers already have
881 * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
882 * is correctly annotated.
883 */
884 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,
885 int depth)
886 {
887 struct eventpoll *ep;
888 bool locked;
889
890 pt->_key = epi->event.events;
891 if (!is_file_epoll(epi->ffd.file))
892 return vfs_poll(epi->ffd.file, pt) & epi->event.events;
893
894 ep = epi->ffd.file->private_data;
895 poll_wait(epi->ffd.file, &ep->poll_wait, pt);
896 locked = pt && (pt->_qproc == ep_ptable_queue_proc);
897
898 return ep_scan_ready_list(epi->ffd.file->private_data,
899 ep_read_events_proc, &depth, depth,
900 locked) & epi->event.events;
901 }
902
903 static __poll_t ep_read_events_proc(struct eventpoll *ep, struct list_head *head,
904 void *priv)
905 {
906 struct epitem *epi, *tmp;
907 poll_table pt;
908 int depth = *(int *)priv;
909
910 init_poll_funcptr(&pt, NULL);
911 depth++;
912
913 list_for_each_entry_safe(epi, tmp, head, rdllink) {
914 if (ep_item_poll(epi, &pt, depth)) {
915 return EPOLLIN | EPOLLRDNORM;
916 } else {
917 /*
918 * Item has been dropped into the ready list by the poll
919 * callback, but it's not actually ready, as far as
920 * caller requested events goes. We can remove it here.
921 */
922 __pm_relax(ep_wakeup_source(epi));
923 list_del_init(&epi->rdllink);
924 }
925 }
926
927 return 0;
928 }
929
930 static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait)
931 {
932 struct eventpoll *ep = file->private_data;
933 int depth = 0;
934
935 /* Insert inside our poll wait queue */
936 poll_wait(file, &ep->poll_wait, wait);
937
938 /*
939 * Proceed to find out if wanted events are really available inside
940 * the ready list.
941 */
942 return ep_scan_ready_list(ep, ep_read_events_proc,
943 &depth, depth, false);
944 }
945
946 #ifdef CONFIG_PROC_FS
947 static void ep_show_fdinfo(struct seq_file *m, struct file *f)
948 {
949 struct eventpoll *ep = f->private_data;
950 struct rb_node *rbp;
951
952 mutex_lock(&ep->mtx);
953 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
954 struct epitem *epi = rb_entry(rbp, struct epitem, rbn);
955 struct inode *inode = file_inode(epi->ffd.file);
956
957 seq_printf(m, "tfd: %8d events: %8x data: %16llx "
958 " pos:%lli ino:%lx sdev:%x\n",
959 epi->ffd.fd, epi->event.events,
960 (long long)epi->event.data,
961 (long long)epi->ffd.file->f_pos,
962 inode->i_ino, inode->i_sb->s_dev);
963 if (seq_has_overflowed(m))
964 break;
965 }
966 mutex_unlock(&ep->mtx);
967 }
968 #endif
969
970 /* File callbacks that implement the eventpoll file behaviour */
971 static const struct file_operations eventpoll_fops = {
972 #ifdef CONFIG_PROC_FS
973 .show_fdinfo = ep_show_fdinfo,
974 #endif
975 .release = ep_eventpoll_release,
976 .poll = ep_eventpoll_poll,
977 .llseek = noop_llseek,
978 };
979
980 /*
981 * This is called from eventpoll_release() to unlink files from the eventpoll
982 * interface. We need to have this facility to cleanup correctly files that are
983 * closed without being removed from the eventpoll interface.
984 */
985 void eventpoll_release_file(struct file *file)
986 {
987 struct eventpoll *ep;
988 struct epitem *epi, *next;
989
990 /*
991 * We don't want to get "file->f_lock" because it is not
992 * necessary. It is not necessary because we're in the "struct file"
993 * cleanup path, and this means that no one is using this file anymore.
994 * So, for example, epoll_ctl() cannot hit here since if we reach this
995 * point, the file counter already went to zero and fget() would fail.
996 * The only hit might come from ep_free() but by holding the mutex
997 * will correctly serialize the operation. We do need to acquire
998 * "ep->mtx" after "epmutex" because ep_remove() requires it when called
999 * from anywhere but ep_free().
1000 *
1001 * Besides, ep_remove() acquires the lock, so we can't hold it here.
1002 */
1003 mutex_lock(&epmutex);
1004 list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) {
1005 ep = epi->ep;
1006 mutex_lock_nested(&ep->mtx, 0);
1007 ep_remove(ep, epi);
1008 mutex_unlock(&ep->mtx);
1009 }
1010 mutex_unlock(&epmutex);
1011 }
1012
1013 static int ep_alloc(struct eventpoll **pep)
1014 {
1015 int error;
1016 struct user_struct *user;
1017 struct eventpoll *ep;
1018
1019 user = get_current_user();
1020 error = -ENOMEM;
1021 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1022 if (unlikely(!ep))
1023 goto free_uid;
1024
1025 mutex_init(&ep->mtx);
1026 rwlock_init(&ep->lock);
1027 init_waitqueue_head(&ep->wq);
1028 init_waitqueue_head(&ep->poll_wait);
1029 INIT_LIST_HEAD(&ep->rdllist);
1030 ep->rbr = RB_ROOT_CACHED;
1031 ep->ovflist = EP_UNACTIVE_PTR;
1032 ep->user = user;
1033
1034 *pep = ep;
1035
1036 return 0;
1037
1038 free_uid:
1039 free_uid(user);
1040 return error;
1041 }
1042
1043 /*
1044 * Search the file inside the eventpoll tree. The RB tree operations
1045 * are protected by the "mtx" mutex, and ep_find() must be called with
1046 * "mtx" held.
1047 */
1048 static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
1049 {
1050 int kcmp;
1051 struct rb_node *rbp;
1052 struct epitem *epi, *epir = NULL;
1053 struct epoll_filefd ffd;
1054
1055 ep_set_ffd(&ffd, file, fd);
1056 for (rbp = ep->rbr.rb_root.rb_node; rbp; ) {
1057 epi = rb_entry(rbp, struct epitem, rbn);
1058 kcmp = ep_cmp_ffd(&ffd, &epi->ffd);
1059 if (kcmp > 0)
1060 rbp = rbp->rb_right;
1061 else if (kcmp < 0)
1062 rbp = rbp->rb_left;
1063 else {
1064 epir = epi;
1065 break;
1066 }
1067 }
1068
1069 return epir;
1070 }
1071
1072 #ifdef CONFIG_CHECKPOINT_RESTORE
1073 static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)
1074 {
1075 struct rb_node *rbp;
1076 struct epitem *epi;
1077
1078 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
1079 epi = rb_entry(rbp, struct epitem, rbn);
1080 if (epi->ffd.fd == tfd) {
1081 if (toff == 0)
1082 return epi;
1083 else
1084 toff--;
1085 }
1086 cond_resched();
1087 }
1088
1089 return NULL;
1090 }
1091
1092 struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
1093 unsigned long toff)
1094 {
1095 struct file *file_raw;
1096 struct eventpoll *ep;
1097 struct epitem *epi;
1098
1099 if (!is_file_epoll(file))
1100 return ERR_PTR(-EINVAL);
1101
1102 ep = file->private_data;
1103
1104 mutex_lock(&ep->mtx);
1105 epi = ep_find_tfd(ep, tfd, toff);
1106 if (epi)
1107 file_raw = epi->ffd.file;
1108 else
1109 file_raw = ERR_PTR(-ENOENT);
1110 mutex_unlock(&ep->mtx);
1111
1112 return file_raw;
1113 }
1114 #endif /* CONFIG_CHECKPOINT_RESTORE */
1115
1116 /**
1117 * Adds a new entry to the tail of the list in a lockless way, i.e.
1118 * multiple CPUs are allowed to call this function concurrently.
1119 *
1120 * Beware: it is necessary to prevent any other modifications of the
1121 * existing list until all changes are completed, in other words
1122 * concurrent list_add_tail_lockless() calls should be protected
1123 * with a read lock, where write lock acts as a barrier which
1124 * makes sure all list_add_tail_lockless() calls are fully
1125 * completed.
1126 *
1127 * Also an element can be locklessly added to the list only in one
1128 * direction i.e. either to the tail either to the head, otherwise
1129 * concurrent access will corrupt the list.
1130 *
1131 * Returns %false if element has been already added to the list, %true
1132 * otherwise.
1133 */
1134 static inline bool list_add_tail_lockless(struct list_head *new,
1135 struct list_head *head)
1136 {
1137 struct list_head *prev;
1138
1139 /*
1140 * This is simple 'new->next = head' operation, but cmpxchg()
1141 * is used in order to detect that same element has been just
1142 * added to the list from another CPU: the winner observes
1143 * new->next == new.
1144 */
1145 if (cmpxchg(&new->next, new, head) != new)
1146 return false;
1147
1148 /*
1149 * Initially ->next of a new element must be updated with the head
1150 * (we are inserting to the tail) and only then pointers are atomically
1151 * exchanged. XCHG guarantees memory ordering, thus ->next should be
1152 * updated before pointers are actually swapped and pointers are
1153 * swapped before prev->next is updated.
1154 */
1155
1156 prev = xchg(&head->prev, new);
1157
1158 /*
1159 * It is safe to modify prev->next and new->prev, because a new element
1160 * is added only to the tail and new->next is updated before XCHG.
1161 */
1162
1163 prev->next = new;
1164 new->prev = prev;
1165
1166 return true;
1167 }
1168
1169 /**
1170 * Chains a new epi entry to the tail of the ep->ovflist in a lockless way,
1171 * i.e. multiple CPUs are allowed to call this function concurrently.
1172 *
1173 * Returns %false if epi element has been already chained, %true otherwise.
1174 */
1175 static inline bool chain_epi_lockless(struct epitem *epi)
1176 {
1177 struct eventpoll *ep = epi->ep;
1178
1179 /* Fast preliminary check */
1180 if (epi->next != EP_UNACTIVE_PTR)
1181 return false;
1182
1183 /* Check that the same epi has not been just chained from another CPU */
1184 if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)
1185 return false;
1186
1187 /* Atomically exchange tail */
1188 epi->next = xchg(&ep->ovflist, epi);
1189
1190 return true;
1191 }
1192
1193 /*
1194 * This is the callback that is passed to the wait queue wakeup
1195 * mechanism. It is called by the stored file descriptors when they
1196 * have events to report.
1197 *
1198 * This callback takes a read lock in order not to content with concurrent
1199 * events from another file descriptors, thus all modifications to ->rdllist
1200 * or ->ovflist are lockless. Read lock is paired with the write lock from
1201 * ep_scan_ready_list(), which stops all list modifications and guarantees
1202 * that lists state is seen correctly.
1203 *
1204 * Another thing worth to mention is that ep_poll_callback() can be called
1205 * concurrently for the same @epi from different CPUs if poll table was inited
1206 * with several wait queues entries. Plural wakeup from different CPUs of a
1207 * single wait queue is serialized by wq.lock, but the case when multiple wait
1208 * queues are used should be detected accordingly. This is detected using
1209 * cmpxchg() operation.
1210 */
1211 static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1212 {
1213 int pwake = 0;
1214 struct epitem *epi = ep_item_from_wait(wait);
1215 struct eventpoll *ep = epi->ep;
1216 __poll_t pollflags = key_to_poll(key);
1217 unsigned long flags;
1218 int ewake = 0;
1219
1220 read_lock_irqsave(&ep->lock, flags);
1221
1222 ep_set_busy_poll_napi_id(epi);
1223
1224 /*
1225 * If the event mask does not contain any poll(2) event, we consider the
1226 * descriptor to be disabled. This condition is likely the effect of the
1227 * EPOLLONESHOT bit that disables the descriptor when an event is received,
1228 * until the next EPOLL_CTL_MOD will be issued.
1229 */
1230 if (!(epi->event.events & ~EP_PRIVATE_BITS))
1231 goto out_unlock;
1232
1233 /*
1234 * Check the events coming with the callback. At this stage, not
1235 * every device reports the events in the "key" parameter of the
1236 * callback. We need to be able to handle both cases here, hence the
1237 * test for "key" != NULL before the event match test.
1238 */
1239 if (pollflags && !(pollflags & epi->event.events))
1240 goto out_unlock;
1241
1242 /*
1243 * If we are transferring events to userspace, we can hold no locks
1244 * (because we're accessing user memory, and because of linux f_op->poll()
1245 * semantics). All the events that happen during that period of time are
1246 * chained in ep->ovflist and requeued later on.
1247 */
1248 if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) {
1249 if (chain_epi_lockless(epi))
1250 ep_pm_stay_awake_rcu(epi);
1251 } else if (!ep_is_linked(epi)) {
1252 /* In the usual case, add event to ready list. */
1253 if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist))
1254 ep_pm_stay_awake_rcu(epi);
1255 }
1256
1257 /*
1258 * Wake up ( if active ) both the eventpoll wait list and the ->poll()
1259 * wait list.
1260 */
1261 if (waitqueue_active(&ep->wq)) {
1262 if ((epi->event.events & EPOLLEXCLUSIVE) &&
1263 !(pollflags & POLLFREE)) {
1264 switch (pollflags & EPOLLINOUT_BITS) {
1265 case EPOLLIN:
1266 if (epi->event.events & EPOLLIN)
1267 ewake = 1;
1268 break;
1269 case EPOLLOUT:
1270 if (epi->event.events & EPOLLOUT)
1271 ewake = 1;
1272 break;
1273 case 0:
1274 ewake = 1;
1275 break;
1276 }
1277 }
1278 wake_up(&ep->wq);
1279 }
1280 if (waitqueue_active(&ep->poll_wait))
1281 pwake++;
1282
1283 out_unlock:
1284 read_unlock_irqrestore(&ep->lock, flags);
1285
1286 /* We have to call this outside the lock */
1287 if (pwake)
1288 ep_poll_safewake(&ep->poll_wait);
1289
1290 if (!(epi->event.events & EPOLLEXCLUSIVE))
1291 ewake = 1;
1292
1293 if (pollflags & POLLFREE) {
1294 /*
1295 * If we race with ep_remove_wait_queue() it can miss
1296 * ->whead = NULL and do another remove_wait_queue() after
1297 * us, so we can't use __remove_wait_queue().
1298 */
1299 list_del_init(&wait->entry);
1300 /*
1301 * ->whead != NULL protects us from the race with ep_free()
1302 * or ep_remove(), ep_remove_wait_queue() takes whead->lock
1303 * held by the caller. Once we nullify it, nothing protects
1304 * ep/epi or even wait.
1305 */
1306 smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL);
1307 }
1308
1309 return ewake;
1310 }
1311
1312 /*
1313 * This is the callback that is used to add our wait queue to the
1314 * target file wakeup lists.
1315 */
1316 static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
1317 poll_table *pt)
1318 {
1319 struct epitem *epi = ep_item_from_epqueue(pt);
1320 struct eppoll_entry *pwq;
1321
1322 if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) {
1323 init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
1324 pwq->whead = whead;
1325 pwq->base = epi;
1326 if (epi->event.events & EPOLLEXCLUSIVE)
1327 add_wait_queue_exclusive(whead, &pwq->wait);
1328 else
1329 add_wait_queue(whead, &pwq->wait);
1330 list_add_tail(&pwq->llink, &epi->pwqlist);
1331 epi->nwait++;
1332 } else {
1333 /* We have to signal that an error occurred */
1334 epi->nwait = -1;
1335 }
1336 }
1337
1338 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)
1339 {
1340 int kcmp;
1341 struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL;
1342 struct epitem *epic;
1343 bool leftmost = true;
1344
1345 while (*p) {
1346 parent = *p;
1347 epic = rb_entry(parent, struct epitem, rbn);
1348 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);
1349 if (kcmp > 0) {
1350 p = &parent->rb_right;
1351 leftmost = false;
1352 } else
1353 p = &parent->rb_left;
1354 }
1355 rb_link_node(&epi->rbn, parent, p);
1356 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);
1357 }
1358
1359
1360
1361 #define PATH_ARR_SIZE 5
1362 /*
1363 * These are the number paths of length 1 to 5, that we are allowing to emanate
1364 * from a single file of interest. For example, we allow 1000 paths of length
1365 * 1, to emanate from each file of interest. This essentially represents the
1366 * potential wakeup paths, which need to be limited in order to avoid massive
1367 * uncontrolled wakeup storms. The common use case should be a single ep which
1368 * is connected to n file sources. In this case each file source has 1 path
1369 * of length 1. Thus, the numbers below should be more than sufficient. These
1370 * path limits are enforced during an EPOLL_CTL_ADD operation, since a modify
1371 * and delete can't add additional paths. Protected by the epmutex.
1372 */
1373 static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 };
1374 static int path_count[PATH_ARR_SIZE];
1375
1376 static int path_count_inc(int nests)
1377 {
1378 /* Allow an arbitrary number of depth 1 paths */
1379 if (nests == 0)
1380 return 0;
1381
1382 if (++path_count[nests] > path_limits[nests])
1383 return -1;
1384 return 0;
1385 }
1386
1387 static void path_count_init(void)
1388 {
1389 int i;
1390
1391 for (i = 0; i < PATH_ARR_SIZE; i++)
1392 path_count[i] = 0;
1393 }
1394
1395 static int reverse_path_check_proc(void *priv, void *cookie, int call_nests)
1396 {
1397 int error = 0;
1398 struct file *file = priv;
1399 struct file *child_file;
1400 struct epitem *epi;
1401
1402 /* CTL_DEL can remove links here, but that can't increase our count */
1403 rcu_read_lock();
1404 list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) {
1405 child_file = epi->ep->file;
1406 if (is_file_epoll(child_file)) {
1407 if (list_empty(&child_file->f_ep_links)) {
1408 if (path_count_inc(call_nests)) {
1409 error = -1;
1410 break;
1411 }
1412 } else {
1413 error = ep_call_nested(&poll_loop_ncalls,
1414 reverse_path_check_proc,
1415 child_file, child_file,
1416 current);
1417 }
1418 if (error != 0)
1419 break;
1420 } else {
1421 printk(KERN_ERR "reverse_path_check_proc: "
1422 "file is not an ep!\n");
1423 }
1424 }
1425 rcu_read_unlock();
1426 return error;
1427 }
1428
1429 /**
1430 * reverse_path_check - The tfile_check_list is list of file *, which have
1431 * links that are proposed to be newly added. We need to
1432 * make sure that those added links don't add too many
1433 * paths such that we will spend all our time waking up
1434 * eventpoll objects.
1435 *
1436 * Returns: Returns zero if the proposed links don't create too many paths,
1437 * -1 otherwise.
1438 */
1439 static int reverse_path_check(void)
1440 {
1441 int error = 0;
1442 struct file *current_file;
1443
1444 /* let's call this for all tfiles */
1445 list_for_each_entry(current_file, &tfile_check_list, f_tfile_llink) {
1446 path_count_init();
1447 error = ep_call_nested(&poll_loop_ncalls,
1448 reverse_path_check_proc, current_file,
1449 current_file, current);
1450 if (error)
1451 break;
1452 }
1453 return error;
1454 }
1455
1456 static int ep_create_wakeup_source(struct epitem *epi)
1457 {
1458 const char *name;
1459 struct wakeup_source *ws;
1460
1461 if (!epi->ep->ws) {
1462 epi->ep->ws = wakeup_source_register(NULL, "eventpoll");
1463 if (!epi->ep->ws)
1464 return -ENOMEM;
1465 }
1466
1467 name = epi->ffd.file->f_path.dentry->d_name.name;
1468 ws = wakeup_source_register(NULL, name);
1469
1470 if (!ws)
1471 return -ENOMEM;
1472 rcu_assign_pointer(epi->ws, ws);
1473
1474 return 0;
1475 }
1476
1477 /* rare code path, only used when EPOLL_CTL_MOD removes a wakeup source */
1478 static noinline void ep_destroy_wakeup_source(struct epitem *epi)
1479 {
1480 struct wakeup_source *ws = ep_wakeup_source(epi);
1481
1482 RCU_INIT_POINTER(epi->ws, NULL);
1483
1484 /*
1485 * wait for ep_pm_stay_awake_rcu to finish, synchronize_rcu is
1486 * used internally by wakeup_source_remove, too (called by
1487 * wakeup_source_unregister), so we cannot use call_rcu
1488 */
1489 synchronize_rcu();
1490 wakeup_source_unregister(ws);
1491 }
1492
1493 /*
1494 * Must be called with "mtx" held.
1495 */
1496 static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
1497 struct file *tfile, int fd, int full_check)
1498 {
1499 int error, pwake = 0;
1500 __poll_t revents;
1501 long user_watches;
1502 struct epitem *epi;
1503 struct ep_pqueue epq;
1504
1505 lockdep_assert_irqs_enabled();
1506
1507 user_watches = atomic_long_read(&ep->user->epoll_watches);
1508 if (unlikely(user_watches >= max_user_watches))
1509 return -ENOSPC;
1510 if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))
1511 return -ENOMEM;
1512
1513 /* Item initialization follow here ... */
1514 INIT_LIST_HEAD(&epi->rdllink);
1515 INIT_LIST_HEAD(&epi->fllink);
1516 INIT_LIST_HEAD(&epi->pwqlist);
1517 epi->ep = ep;
1518 ep_set_ffd(&epi->ffd, tfile, fd);
1519 epi->event = *event;
1520 epi->nwait = 0;
1521 epi->next = EP_UNACTIVE_PTR;
1522 if (epi->event.events & EPOLLWAKEUP) {
1523 error = ep_create_wakeup_source(epi);
1524 if (error)
1525 goto error_create_wakeup_source;
1526 } else {
1527 RCU_INIT_POINTER(epi->ws, NULL);
1528 }
1529
1530 /* Initialize the poll table using the queue callback */
1531 epq.epi = epi;
1532 init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
1533
1534 /*
1535 * Attach the item to the poll hooks and get current event bits.
1536 * We can safely use the file* here because its usage count has
1537 * been increased by the caller of this function. Note that after
1538 * this operation completes, the poll callback can start hitting
1539 * the new item.
1540 */
1541 revents = ep_item_poll(epi, &epq.pt, 1);
1542
1543 /*
1544 * We have to check if something went wrong during the poll wait queue
1545 * install process. Namely an allocation for a wait queue failed due
1546 * high memory pressure.
1547 */
1548 error = -ENOMEM;
1549 if (epi->nwait < 0)
1550 goto error_unregister;
1551
1552 /* Add the current item to the list of active epoll hook for this file */
1553 spin_lock(&tfile->f_lock);
1554 list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);
1555 spin_unlock(&tfile->f_lock);
1556
1557 /*
1558 * Add the current item to the RB tree. All RB tree operations are
1559 * protected by "mtx", and ep_insert() is called with "mtx" held.
1560 */
1561 ep_rbtree_insert(ep, epi);
1562
1563 /* now check if we've created too many backpaths */
1564 error = -EINVAL;
1565 if (full_check && reverse_path_check())
1566 goto error_remove_epi;
1567
1568 /* We have to drop the new item inside our item list to keep track of it */
1569 write_lock_irq(&ep->lock);
1570
1571 /* record NAPI ID of new item if present */
1572 ep_set_busy_poll_napi_id(epi);
1573
1574 /* If the file is already "ready" we drop it inside the ready list */
1575 if (revents && !ep_is_linked(epi)) {
1576 list_add_tail(&epi->rdllink, &ep->rdllist);
1577 ep_pm_stay_awake(epi);
1578
1579 /* Notify waiting tasks that events are available */
1580 if (waitqueue_active(&ep->wq))
1581 wake_up(&ep->wq);
1582 if (waitqueue_active(&ep->poll_wait))
1583 pwake++;
1584 }
1585
1586 write_unlock_irq(&ep->lock);
1587
1588 atomic_long_inc(&ep->user->epoll_watches);
1589
1590 /* We have to call this outside the lock */
1591 if (pwake)
1592 ep_poll_safewake(&ep->poll_wait);
1593
1594 return 0;
1595
1596 error_remove_epi:
1597 spin_lock(&tfile->f_lock);
1598 list_del_rcu(&epi->fllink);
1599 spin_unlock(&tfile->f_lock);
1600
1601 rb_erase_cached(&epi->rbn, &ep->rbr);
1602
1603 error_unregister:
1604 ep_unregister_pollwait(ep, epi);
1605
1606 /*
1607 * We need to do this because an event could have been arrived on some
1608 * allocated wait queue. Note that we don't care about the ep->ovflist
1609 * list, since that is used/cleaned only inside a section bound by "mtx".
1610 * And ep_insert() is called with "mtx" held.
1611 */
1612 write_lock_irq(&ep->lock);
1613 if (ep_is_linked(epi))
1614 list_del_init(&epi->rdllink);
1615 write_unlock_irq(&ep->lock);
1616
1617 wakeup_source_unregister(ep_wakeup_source(epi));
1618
1619 error_create_wakeup_source:
1620 kmem_cache_free(epi_cache, epi);
1621
1622 return error;
1623 }
1624
1625 /*
1626 * Modify the interest event mask by dropping an event if the new mask
1627 * has a match in the current file status. Must be called with "mtx" held.
1628 */
1629 static int ep_modify(struct eventpoll *ep, struct epitem *epi,
1630 const struct epoll_event *event)
1631 {
1632 int pwake = 0;
1633 poll_table pt;
1634
1635 lockdep_assert_irqs_enabled();
1636
1637 init_poll_funcptr(&pt, NULL);
1638
1639 /*
1640 * Set the new event interest mask before calling f_op->poll();
1641 * otherwise we might miss an event that happens between the
1642 * f_op->poll() call and the new event set registering.
1643 */
1644 epi->event.events = event->events; /* need barrier below */
1645 epi->event.data = event->data; /* protected by mtx */
1646 if (epi->event.events & EPOLLWAKEUP) {
1647 if (!ep_has_wakeup_source(epi))
1648 ep_create_wakeup_source(epi);
1649 } else if (ep_has_wakeup_source(epi)) {
1650 ep_destroy_wakeup_source(epi);
1651 }
1652
1653 /*
1654 * The following barrier has two effects:
1655 *
1656 * 1) Flush epi changes above to other CPUs. This ensures
1657 * we do not miss events from ep_poll_callback if an
1658 * event occurs immediately after we call f_op->poll().
1659 * We need this because we did not take ep->lock while
1660 * changing epi above (but ep_poll_callback does take
1661 * ep->lock).
1662 *
1663 * 2) We also need to ensure we do not miss _past_ events
1664 * when calling f_op->poll(). This barrier also
1665 * pairs with the barrier in wq_has_sleeper (see
1666 * comments for wq_has_sleeper).
1667 *
1668 * This barrier will now guarantee ep_poll_callback or f_op->poll
1669 * (or both) will notice the readiness of an item.
1670 */
1671 smp_mb();
1672
1673 /*
1674 * Get current event bits. We can safely use the file* here because
1675 * its usage count has been increased by the caller of this function.
1676 * If the item is "hot" and it is not registered inside the ready
1677 * list, push it inside.
1678 */
1679 if (ep_item_poll(epi, &pt, 1)) {
1680 write_lock_irq(&ep->lock);
1681 if (!ep_is_linked(epi)) {
1682 list_add_tail(&epi->rdllink, &ep->rdllist);
1683 ep_pm_stay_awake(epi);
1684
1685 /* Notify waiting tasks that events are available */
1686 if (waitqueue_active(&ep->wq))
1687 wake_up(&ep->wq);
1688 if (waitqueue_active(&ep->poll_wait))
1689 pwake++;
1690 }
1691 write_unlock_irq(&ep->lock);
1692 }
1693
1694 /* We have to call this outside the lock */
1695 if (pwake)
1696 ep_poll_safewake(&ep->poll_wait);
1697
1698 return 0;
1699 }
1700
1701 static __poll_t ep_send_events_proc(struct eventpoll *ep, struct list_head *head,
1702 void *priv)
1703 {
1704 struct ep_send_events_data *esed = priv;
1705 __poll_t revents;
1706 struct epitem *epi, *tmp;
1707 struct epoll_event __user *uevent = esed->events;
1708 struct wakeup_source *ws;
1709 poll_table pt;
1710
1711 init_poll_funcptr(&pt, NULL);
1712 esed->res = 0;
1713
1714 /*
1715 * We can loop without lock because we are passed a task private list.
1716 * Items cannot vanish during the loop because ep_scan_ready_list() is
1717 * holding "mtx" during this call.
1718 */
1719 lockdep_assert_held(&ep->mtx);
1720
1721 list_for_each_entry_safe(epi, tmp, head, rdllink) {
1722 if (esed->res >= esed->maxevents)
1723 break;
1724
1725 /*
1726 * Activate ep->ws before deactivating epi->ws to prevent
1727 * triggering auto-suspend here (in case we reactive epi->ws
1728 * below).
1729 *
1730 * This could be rearranged to delay the deactivation of epi->ws
1731 * instead, but then epi->ws would temporarily be out of sync
1732 * with ep_is_linked().
1733 */
1734 ws = ep_wakeup_source(epi);
1735 if (ws) {
1736 if (ws->active)
1737 __pm_stay_awake(ep->ws);
1738 __pm_relax(ws);
1739 }
1740
1741 list_del_init(&epi->rdllink);
1742
1743 /*
1744 * If the event mask intersect the caller-requested one,
1745 * deliver the event to userspace. Again, ep_scan_ready_list()
1746 * is holding ep->mtx, so no operations coming from userspace
1747 * can change the item.
1748 */
1749 revents = ep_item_poll(epi, &pt, 1);
1750 if (!revents)
1751 continue;
1752
1753 if (__put_user(revents, &uevent->events) ||
1754 __put_user(epi->event.data, &uevent->data)) {
1755 list_add(&epi->rdllink, head);
1756 ep_pm_stay_awake(epi);
1757 if (!esed->res)
1758 esed->res = -EFAULT;
1759 return 0;
1760 }
1761 esed->res++;
1762 uevent++;
1763 if (epi->event.events & EPOLLONESHOT)
1764 epi->event.events &= EP_PRIVATE_BITS;
1765 else if (!(epi->event.events & EPOLLET)) {
1766 /*
1767 * If this file has been added with Level
1768 * Trigger mode, we need to insert back inside
1769 * the ready list, so that the next call to
1770 * epoll_wait() will check again the events
1771 * availability. At this point, no one can insert
1772 * into ep->rdllist besides us. The epoll_ctl()
1773 * callers are locked out by
1774 * ep_scan_ready_list() holding "mtx" and the
1775 * poll callback will queue them in ep->ovflist.
1776 */
1777 list_add_tail(&epi->rdllink, &ep->rdllist);
1778 ep_pm_stay_awake(epi);
1779 }
1780 }
1781
1782 return 0;
1783 }
1784
1785 static int ep_send_events(struct eventpoll *ep,
1786 struct epoll_event __user *events, int maxevents)
1787 {
1788 struct ep_send_events_data esed;
1789
1790 esed.maxevents = maxevents;
1791 esed.events = events;
1792
1793 ep_scan_ready_list(ep, ep_send_events_proc, &esed, 0, false);
1794 return esed.res;
1795 }
1796
1797 static inline struct timespec64 ep_set_mstimeout(long ms)
1798 {
1799 struct timespec64 now, ts = {
1800 .tv_sec = ms / MSEC_PER_SEC,
1801 .tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC),
1802 };
1803
1804 ktime_get_ts64(&now);
1805 return timespec64_add_safe(now, ts);
1806 }
1807
1808 /**
1809 * ep_poll - Retrieves ready events, and delivers them to the caller supplied
1810 * event buffer.
1811 *
1812 * @ep: Pointer to the eventpoll context.
1813 * @events: Pointer to the userspace buffer where the ready events should be
1814 * stored.
1815 * @maxevents: Size (in terms of number of events) of the caller event buffer.
1816 * @timeout: Maximum timeout for the ready events fetch operation, in
1817 * milliseconds. If the @timeout is zero, the function will not block,
1818 * while if the @timeout is less than zero, the function will block
1819 * until at least one event has been retrieved (or an error
1820 * occurred).
1821 *
1822 * Returns: Returns the number of ready events which have been fetched, or an
1823 * error code, in case of error.
1824 */
1825 static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
1826 int maxevents, long timeout)
1827 {
1828 int res = 0, eavail, timed_out = 0;
1829 u64 slack = 0;
1830 wait_queue_entry_t wait;
1831 ktime_t expires, *to = NULL;
1832
1833 lockdep_assert_irqs_enabled();
1834
1835 if (timeout > 0) {
1836 struct timespec64 end_time = ep_set_mstimeout(timeout);
1837
1838 slack = select_estimate_accuracy(&end_time);
1839 to = &expires;
1840 *to = timespec64_to_ktime(end_time);
1841 } else if (timeout == 0) {
1842 /*
1843 * Avoid the unnecessary trip to the wait queue loop, if the
1844 * caller specified a non blocking operation. We still need
1845 * lock because we could race and not see an epi being added
1846 * to the ready list while in irq callback. Thus incorrectly
1847 * returning 0 back to userspace.
1848 */
1849 timed_out = 1;
1850
1851 write_lock_irq(&ep->lock);
1852 eavail = ep_events_available(ep);
1853 write_unlock_irq(&ep->lock);
1854
1855 goto send_events;
1856 }
1857
1858 fetch_events:
1859
1860 if (!ep_events_available(ep))
1861 ep_busy_loop(ep, timed_out);
1862
1863 eavail = ep_events_available(ep);
1864 if (eavail)
1865 goto send_events;
1866
1867 /*
1868 * Busy poll timed out. Drop NAPI ID for now, we can add
1869 * it back in when we have moved a socket with a valid NAPI
1870 * ID onto the ready list.
1871 */
1872 ep_reset_busy_poll_napi_id(ep);
1873
1874 do {
1875 /*
1876 * Internally init_wait() uses autoremove_wake_function(),
1877 * thus wait entry is removed from the wait queue on each
1878 * wakeup. Why it is important? In case of several waiters
1879 * each new wakeup will hit the next waiter, giving it the
1880 * chance to harvest new event. Otherwise wakeup can be
1881 * lost. This is also good performance-wise, because on
1882 * normal wakeup path no need to call __remove_wait_queue()
1883 * explicitly, thus ep->lock is not taken, which halts the
1884 * event delivery.
1885 */
1886 init_wait(&wait);
1887 write_lock_irq(&ep->lock);
1888 __add_wait_queue_exclusive(&ep->wq, &wait);
1889 write_unlock_irq(&ep->lock);
1890
1891 /*
1892 * We don't want to sleep if the ep_poll_callback() sends us
1893 * a wakeup in between. That's why we set the task state
1894 * to TASK_INTERRUPTIBLE before doing the checks.
1895 */
1896 set_current_state(TASK_INTERRUPTIBLE);
1897 /*
1898 * Always short-circuit for fatal signals to allow
1899 * threads to make a timely exit without the chance of
1900 * finding more events available and fetching
1901 * repeatedly.
1902 */
1903 if (fatal_signal_pending(current)) {
1904 res = -EINTR;
1905 break;
1906 }
1907
1908 eavail = ep_events_available(ep);
1909 if (eavail)
1910 break;
1911 if (signal_pending(current)) {
1912 res = -EINTR;
1913 break;
1914 }
1915
1916 if (!schedule_hrtimeout_range(to, slack, HRTIMER_MODE_ABS)) {
1917 timed_out = 1;
1918 break;
1919 }
1920
1921 /* We were woken up, thus go and try to harvest some events */
1922 eavail = 1;
1923
1924 } while (0);
1925
1926 __set_current_state(TASK_RUNNING);
1927
1928 if (!list_empty_careful(&wait.entry)) {
1929 write_lock_irq(&ep->lock);
1930 __remove_wait_queue(&ep->wq, &wait);
1931 write_unlock_irq(&ep->lock);
1932 }
1933
1934 send_events:
1935 /*
1936 * Try to transfer events to user space. In case we get 0 events and
1937 * there's still timeout left over, we go trying again in search of
1938 * more luck.
1939 */
1940 if (!res && eavail &&
1941 !(res = ep_send_events(ep, events, maxevents)) && !timed_out)
1942 goto fetch_events;
1943
1944 return res;
1945 }
1946
1947 /**
1948 * ep_loop_check_proc - Callback function to be passed to the @ep_call_nested()
1949 * API, to verify that adding an epoll file inside another
1950 * epoll structure, does not violate the constraints, in
1951 * terms of closed loops, or too deep chains (which can
1952 * result in excessive stack usage).
1953 *
1954 * @priv: Pointer to the epoll file to be currently checked.
1955 * @cookie: Original cookie for this call. This is the top-of-the-chain epoll
1956 * data structure pointer.
1957 * @call_nests: Current dept of the @ep_call_nested() call stack.
1958 *
1959 * Returns: Returns zero if adding the epoll @file inside current epoll
1960 * structure @ep does not violate the constraints, or -1 otherwise.
1961 */
1962 static int ep_loop_check_proc(void *priv, void *cookie, int call_nests)
1963 {
1964 int error = 0;
1965 struct file *file = priv;
1966 struct eventpoll *ep = file->private_data;
1967 struct eventpoll *ep_tovisit;
1968 struct rb_node *rbp;
1969 struct epitem *epi;
1970
1971 mutex_lock_nested(&ep->mtx, call_nests + 1);
1972 ep->visited = 1;
1973 list_add(&ep->visited_list_link, &visited_list);
1974 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
1975 epi = rb_entry(rbp, struct epitem, rbn);
1976 if (unlikely(is_file_epoll(epi->ffd.file))) {
1977 ep_tovisit = epi->ffd.file->private_data;
1978 if (ep_tovisit->visited)
1979 continue;
1980 error = ep_call_nested(&poll_loop_ncalls,
1981 ep_loop_check_proc, epi->ffd.file,
1982 ep_tovisit, current);
1983 if (error != 0)
1984 break;
1985 } else {
1986 /*
1987 * If we've reached a file that is not associated with
1988 * an ep, then we need to check if the newly added
1989 * links are going to add too many wakeup paths. We do
1990 * this by adding it to the tfile_check_list, if it's
1991 * not already there, and calling reverse_path_check()
1992 * during ep_insert().
1993 */
1994 if (list_empty(&epi->ffd.file->f_tfile_llink))
1995 list_add(&epi->ffd.file->f_tfile_llink,
1996 &tfile_check_list);
1997 }
1998 }
1999 mutex_unlock(&ep->mtx);
2000
2001 return error;
2002 }
2003
2004 /**
2005 * ep_loop_check - Performs a check to verify that adding an epoll file (@file)
2006 * another epoll file (represented by @ep) does not create
2007 * closed loops or too deep chains.
2008 *
2009 * @ep: Pointer to the epoll private data structure.
2010 * @file: Pointer to the epoll file to be checked.
2011 *
2012 * Returns: Returns zero if adding the epoll @file inside current epoll
2013 * structure @ep does not violate the constraints, or -1 otherwise.
2014 */
2015 static int ep_loop_check(struct eventpoll *ep, struct file *file)
2016 {
2017 int ret;
2018 struct eventpoll *ep_cur, *ep_next;
2019
2020 ret = ep_call_nested(&poll_loop_ncalls,
2021 ep_loop_check_proc, file, ep, current);
2022 /* clear visited list */
2023 list_for_each_entry_safe(ep_cur, ep_next, &visited_list,
2024 visited_list_link) {
2025 ep_cur->visited = 0;
2026 list_del(&ep_cur->visited_list_link);
2027 }
2028 return ret;
2029 }
2030
2031 static void clear_tfile_check_list(void)
2032 {
2033 struct file *file;
2034
2035 /* first clear the tfile_check_list */
2036 while (!list_empty(&tfile_check_list)) {
2037 file = list_first_entry(&tfile_check_list, struct file,
2038 f_tfile_llink);
2039 list_del_init(&file->f_tfile_llink);
2040 }
2041 INIT_LIST_HEAD(&tfile_check_list);
2042 }
2043
2044 /*
2045 * Open an eventpoll file descriptor.
2046 */
2047 static int do_epoll_create(int flags)
2048 {
2049 int error, fd;
2050 struct eventpoll *ep = NULL;
2051 struct file *file;
2052
2053 /* Check the EPOLL_* constant for consistency. */
2054 BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC);
2055
2056 if (flags & ~EPOLL_CLOEXEC)
2057 return -EINVAL;
2058 /*
2059 * Create the internal data structure ("struct eventpoll").
2060 */
2061 error = ep_alloc(&ep);
2062 if (error < 0)
2063 return error;
2064 /*
2065 * Creates all the items needed to setup an eventpoll file. That is,
2066 * a file structure and a free file descriptor.
2067 */
2068 fd = get_unused_fd_flags(O_RDWR | (flags & O_CLOEXEC));
2069 if (fd < 0) {
2070 error = fd;
2071 goto out_free_ep;
2072 }
2073 file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,
2074 O_RDWR | (flags & O_CLOEXEC));
2075 if (IS_ERR(file)) {
2076 error = PTR_ERR(file);
2077 goto out_free_fd;
2078 }
2079 ep->file = file;
2080 fd_install(fd, file);
2081 return fd;
2082
2083 out_free_fd:
2084 put_unused_fd(fd);
2085 out_free_ep:
2086 ep_free(ep);
2087 return error;
2088 }
2089
2090 SYSCALL_DEFINE1(epoll_create1, int, flags)
2091 {
2092 return do_epoll_create(flags);
2093 }
2094
2095 SYSCALL_DEFINE1(epoll_create, int, size)
2096 {
2097 if (size <= 0)
2098 return -EINVAL;
2099
2100 return do_epoll_create(0);
2101 }
2102
2103 /*
2104 * The following function implements the controller interface for
2105 * the eventpoll file that enables the insertion/removal/change of
2106 * file descriptors inside the interest set.
2107 */
2108 SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
2109 struct epoll_event __user *, event)
2110 {
2111 int error;
2112 int full_check = 0;
2113 struct fd f, tf;
2114 struct eventpoll *ep;
2115 struct epitem *epi;
2116 struct epoll_event epds;
2117 struct eventpoll *tep = NULL;
2118
2119 error = -EFAULT;
2120 if (ep_op_has_event(op) &&
2121 copy_from_user(&epds, event, sizeof(struct epoll_event)))
2122 goto error_return;
2123
2124 error = -EBADF;
2125 f = fdget(epfd);
2126 if (!f.file)
2127 goto error_return;
2128
2129 /* Get the "struct file *" for the target file */
2130 tf = fdget(fd);
2131 if (!tf.file)
2132 goto error_fput;
2133
2134 /* The target file descriptor must support poll */
2135 error = -EPERM;
2136 if (!file_can_poll(tf.file))
2137 goto error_tgt_fput;
2138
2139 /* Check if EPOLLWAKEUP is allowed */
2140 if (ep_op_has_event(op))
2141 ep_take_care_of_epollwakeup(&epds);
2142
2143 /*
2144 * We have to check that the file structure underneath the file descriptor
2145 * the user passed to us _is_ an eventpoll file. And also we do not permit
2146 * adding an epoll file descriptor inside itself.
2147 */
2148 error = -EINVAL;
2149 if (f.file == tf.file || !is_file_epoll(f.file))
2150 goto error_tgt_fput;
2151
2152 /*
2153 * epoll adds to the wakeup queue at EPOLL_CTL_ADD time only,
2154 * so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation.
2155 * Also, we do not currently supported nested exclusive wakeups.
2156 */
2157 if (ep_op_has_event(op) && (epds.events & EPOLLEXCLUSIVE)) {
2158 if (op == EPOLL_CTL_MOD)
2159 goto error_tgt_fput;
2160 if (op == EPOLL_CTL_ADD && (is_file_epoll(tf.file) ||
2161 (epds.events & ~EPOLLEXCLUSIVE_OK_BITS)))
2162 goto error_tgt_fput;
2163 }
2164
2165 /*
2166 * At this point it is safe to assume that the "private_data" contains
2167 * our own data structure.
2168 */
2169 ep = f.file->private_data;
2170
2171 /*
2172 * When we insert an epoll file descriptor, inside another epoll file
2173 * descriptor, there is the change of creating closed loops, which are
2174 * better be handled here, than in more critical paths. While we are
2175 * checking for loops we also determine the list of files reachable
2176 * and hang them on the tfile_check_list, so we can check that we
2177 * haven't created too many possible wakeup paths.
2178 *
2179 * We do not need to take the global 'epumutex' on EPOLL_CTL_ADD when
2180 * the epoll file descriptor is attaching directly to a wakeup source,
2181 * unless the epoll file descriptor is nested. The purpose of taking the
2182 * 'epmutex' on add is to prevent complex toplogies such as loops and
2183 * deep wakeup paths from forming in parallel through multiple
2184 * EPOLL_CTL_ADD operations.
2185 */
2186 mutex_lock_nested(&ep->mtx, 0);
2187 if (op == EPOLL_CTL_ADD) {
2188 if (!list_empty(&f.file->f_ep_links) ||
2189 is_file_epoll(tf.file)) {
2190 full_check = 1;
2191 mutex_unlock(&ep->mtx);
2192 mutex_lock(&epmutex);
2193 if (is_file_epoll(tf.file)) {
2194 error = -ELOOP;
2195 if (ep_loop_check(ep, tf.file) != 0) {
2196 clear_tfile_check_list();
2197 goto error_tgt_fput;
2198 }
2199 } else
2200 list_add(&tf.file->f_tfile_llink,
2201 &tfile_check_list);
2202 mutex_lock_nested(&ep->mtx, 0);
2203 if (is_file_epoll(tf.file)) {
2204 tep = tf.file->private_data;
2205 mutex_lock_nested(&tep->mtx, 1);
2206 }
2207 }
2208 }
2209
2210 /*
2211 * Try to lookup the file inside our RB tree, Since we grabbed "mtx"
2212 * above, we can be sure to be able to use the item looked up by
2213 * ep_find() till we release the mutex.
2214 */
2215 epi = ep_find(ep, tf.file, fd);
2216
2217 error = -EINVAL;
2218 switch (op) {
2219 case EPOLL_CTL_ADD:
2220 if (!epi) {
2221 epds.events |= EPOLLERR | EPOLLHUP;
2222 error = ep_insert(ep, &epds, tf.file, fd, full_check);
2223 } else
2224 error = -EEXIST;
2225 if (full_check)
2226 clear_tfile_check_list();
2227 break;
2228 case EPOLL_CTL_DEL:
2229 if (epi)
2230 error = ep_remove(ep, epi);
2231 else
2232 error = -ENOENT;
2233 break;
2234 case EPOLL_CTL_MOD:
2235 if (epi) {
2236 if (!(epi->event.events & EPOLLEXCLUSIVE)) {
2237 epds.events |= EPOLLERR | EPOLLHUP;
2238 error = ep_modify(ep, epi, &epds);
2239 }
2240 } else
2241 error = -ENOENT;
2242 break;
2243 }
2244 if (tep != NULL)
2245 mutex_unlock(&tep->mtx);
2246 mutex_unlock(&ep->mtx);
2247
2248 error_tgt_fput:
2249 if (full_check)
2250 mutex_unlock(&epmutex);
2251
2252 fdput(tf);
2253 error_fput:
2254 fdput(f);
2255 error_return:
2256
2257 return error;
2258 }
2259
2260 /*
2261 * Implement the event wait interface for the eventpoll file. It is the kernel
2262 * part of the user space epoll_wait(2).
2263 */
2264 static int do_epoll_wait(int epfd, struct epoll_event __user *events,
2265 int maxevents, int timeout)
2266 {
2267 int error;
2268 struct fd f;
2269 struct eventpoll *ep;
2270
2271 /* The maximum number of event must be greater than zero */
2272 if (maxevents <= 0 || maxevents > EP_MAX_EVENTS)
2273 return -EINVAL;
2274
2275 /* Verify that the area passed by the user is writeable */
2276 if (!access_ok(events, maxevents * sizeof(struct epoll_event)))
2277 return -EFAULT;
2278
2279 /* Get the "struct file *" for the eventpoll file */
2280 f = fdget(epfd);
2281 if (!f.file)
2282 return -EBADF;
2283
2284 /*
2285 * We have to check that the file structure underneath the fd
2286 * the user passed to us _is_ an eventpoll file.
2287 */
2288 error = -EINVAL;
2289 if (!is_file_epoll(f.file))
2290 goto error_fput;
2291
2292 /*
2293 * At this point it is safe to assume that the "private_data" contains
2294 * our own data structure.
2295 */
2296 ep = f.file->private_data;
2297
2298 /* Time to fish for events ... */
2299 error = ep_poll(ep, events, maxevents, timeout);
2300
2301 error_fput:
2302 fdput(f);
2303 return error;
2304 }
2305
2306 SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
2307 int, maxevents, int, timeout)
2308 {
2309 return do_epoll_wait(epfd, events, maxevents, timeout);
2310 }
2311
2312 /*
2313 * Implement the event wait interface for the eventpoll file. It is the kernel
2314 * part of the user space epoll_pwait(2).
2315 */
2316 SYSCALL_DEFINE6(epoll_pwait, int, epfd, struct epoll_event __user *, events,
2317 int, maxevents, int, timeout, const sigset_t __user *, sigmask,
2318 size_t, sigsetsize)
2319 {
2320 int error;
2321
2322 /*
2323 * If the caller wants a certain signal mask to be set during the wait,
2324 * we apply it here.
2325 */
2326 error = set_user_sigmask(sigmask, sigsetsize);
2327 if (error)
2328 return error;
2329
2330 error = do_epoll_wait(epfd, events, maxevents, timeout);
2331 restore_saved_sigmask_unless(error == -EINTR);
2332
2333 return error;
2334 }
2335
2336 #ifdef CONFIG_COMPAT
2337 COMPAT_SYSCALL_DEFINE6(epoll_pwait, int, epfd,
2338 struct epoll_event __user *, events,
2339 int, maxevents, int, timeout,
2340 const compat_sigset_t __user *, sigmask,
2341 compat_size_t, sigsetsize)
2342 {
2343 long err;
2344
2345 /*
2346 * If the caller wants a certain signal mask to be set during the wait,
2347 * we apply it here.
2348 */
2349 err = set_compat_user_sigmask(sigmask, sigsetsize);
2350 if (err)
2351 return err;
2352
2353 err = do_epoll_wait(epfd, events, maxevents, timeout);
2354 restore_saved_sigmask_unless(err == -EINTR);
2355
2356 return err;
2357 }
2358 #endif
2359
2360 static int __init eventpoll_init(void)
2361 {
2362 struct sysinfo si;
2363
2364 si_meminfo(&si);
2365 /*
2366 * Allows top 4% of lomem to be allocated for epoll watches (per user).
2367 */
2368 max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) /
2369 EP_ITEM_COST;
2370 BUG_ON(max_user_watches < 0);
2371
2372 /*
2373 * Initialize the structure used to perform epoll file descriptor
2374 * inclusion loops checks.
2375 */
2376 ep_nested_calls_init(&poll_loop_ncalls);
2377
2378 #ifdef CONFIG_DEBUG_LOCK_ALLOC
2379 /* Initialize the structure used to perform safe poll wait head wake ups */
2380 ep_nested_calls_init(&poll_safewake_ncalls);
2381 #endif
2382
2383 /*
2384 * We can have many thousands of epitems, so prevent this from
2385 * using an extra cache line on 64-bit (and smaller) CPUs
2386 */
2387 BUILD_BUG_ON(sizeof(void *) <= 8 && sizeof(struct epitem) > 128);
2388
2389 /* Allocates slab cache used to allocate "struct epitem" items */
2390 epi_cache = kmem_cache_create("eventpoll_epi", sizeof(struct epitem),
2391 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);
2392
2393 /* Allocates slab cache used to allocate "struct eppoll_entry" */
2394 pwq_cache = kmem_cache_create("eventpoll_pwq",
2395 sizeof(struct eppoll_entry), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);
2396
2397 return 0;
2398 }
2399 fs_initcall(eventpoll_init);