1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Generic wait-for-completion handler;
4 *
5 * It differs from semaphores in that their default case is the opposite,
6 * wait_for_completion default blocks whereas semaphore default non-block. The
7 * interface also makes it easy to 'complete' multiple waiting threads,
8 * something which isn't entirely natural for semaphores.
9 *
10 * But more importantly, the primitive documents the usage. Semaphores would
11 * typically be used for exclusion which gives rise to priority inversion.
12 * Waiting for completion is a typically sync point, but not an exclusion point.
13 */
14 #include "sched.h"
15
16 /**
17 * complete: - signals a single thread waiting on this completion
18 * @x: holds the state of this particular completion
19 *
20 * This will wake up a single thread waiting on this completion. Threads will be
21 * awakened in the same order in which they were queued.
22 *
23 * See also complete_all(), wait_for_completion() and related routines.
24 *
25 * If this function wakes up a task, it executes a full memory barrier before
26 * accessing the task state.
27 */
28 void complete(struct completion *x)
29 {
30 unsigned long flags;
31
32 spin_lock_irqsave(&x->wait.lock, flags);
33
34 if (x->done != UINT_MAX)
35 x->done++;
36 __wake_up_locked(&x->wait, TASK_NORMAL, 1);
37 spin_unlock_irqrestore(&x->wait.lock, flags);
38 }
39 EXPORT_SYMBOL(complete);
40
41 /**
42 * complete_all: - signals all threads waiting on this completion
43 * @x: holds the state of this particular completion
44 *
45 * This will wake up all threads waiting on this particular completion event.
46 *
47 * If this function wakes up a task, it executes a full memory barrier before
48 * accessing the task state.
49 *
50 * Since complete_all() sets the completion of @x permanently to done
51 * to allow multiple waiters to finish, a call to reinit_completion()
52 * must be used on @x if @x is to be used again. The code must make
53 * sure that all waiters have woken and finished before reinitializing
54 * @x. Also note that the function completion_done() can not be used
55 * to know if there are still waiters after complete_all() has been called.
56 */
57 void complete_all(struct completion *x)
58 {
59 unsigned long flags;
60
61 spin_lock_irqsave(&x->wait.lock, flags);
62 x->done = UINT_MAX;
63 __wake_up_locked(&x->wait, TASK_NORMAL, 0);
64 spin_unlock_irqrestore(&x->wait.lock, flags);
65 }
66 EXPORT_SYMBOL(complete_all);
67
68 static inline long __sched
69 do_wait_for_common(struct completion *x,
70 long (*action)(long), long timeout, int state)
71 {
72 if (!x->done) {
73 DECLARE_WAITQUEUE(wait, current);
74
75 __add_wait_queue_entry_tail_exclusive(&x->wait, &wait);
76 do {
77 if (signal_pending_state(state, current)) {
78 timeout = -ERESTARTSYS;
79 break;
80 }
81 __set_current_state(state);
82 spin_unlock_irq(&x->wait.lock);
83 timeout = action(timeout);
84 spin_lock_irq(&x->wait.lock);
85 } while (!x->done && timeout);
86 __remove_wait_queue(&x->wait, &wait);
87 if (!x->done)
88 return timeout;
89 }
90 if (x->done != UINT_MAX)
91 x->done--;
92 return timeout ?: 1;
93 }
94
95 static inline long __sched
96 __wait_for_common(struct completion *x,
97 long (*action)(long), long timeout, int state)
98 {
99 might_sleep();
100
101 complete_acquire(x);
102
103 spin_lock_irq(&x->wait.lock);
104 timeout = do_wait_for_common(x, action, timeout, state);
105 spin_unlock_irq(&x->wait.lock);
106
107 complete_release(x);
108
109 return timeout;
110 }
111
112 static long __sched
113 wait_for_common(struct completion *x, long timeout, int state)
114 {
115 return __wait_for_common(x, schedule_timeout, timeout, state);
116 }
117
118 static long __sched
119 wait_for_common_io(struct completion *x, long timeout, int state)
120 {
121 return __wait_for_common(x, io_schedule_timeout, timeout, state);
122 }
123
124 /**
125 * wait_for_completion: - waits for completion of a task
126 * @x: holds the state of this particular completion
127 *
128 * This waits to be signaled for completion of a specific task. It is NOT
129 * interruptible and there is no timeout.
130 *
131 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
132 * and interrupt capability. Also see complete().
133 */
134 void __sched wait_for_completion(struct completion *x)
135 {
136 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
137 }
138 EXPORT_SYMBOL(wait_for_completion);
139
140 /**
141 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
142 * @x: holds the state of this particular completion
143 * @timeout: timeout value in jiffies
144 *
145 * This waits for either a completion of a specific task to be signaled or for a
146 * specified timeout to expire. The timeout is in jiffies. It is not
147 * interruptible.
148 *
149 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
150 * till timeout) if completed.
151 */
152 unsigned long __sched
153 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
154 {
155 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
156 }
157 EXPORT_SYMBOL(wait_for_completion_timeout);
158
159 /**
160 * wait_for_completion_io: - waits for completion of a task
161 * @x: holds the state of this particular completion
162 *
163 * This waits to be signaled for completion of a specific task. It is NOT
164 * interruptible and there is no timeout. The caller is accounted as waiting
165 * for IO (which traditionally means blkio only).
166 */
167 void __sched wait_for_completion_io(struct completion *x)
168 {
169 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
170 }
171 EXPORT_SYMBOL(wait_for_completion_io);
172
173 /**
174 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
175 * @x: holds the state of this particular completion
176 * @timeout: timeout value in jiffies
177 *
178 * This waits for either a completion of a specific task to be signaled or for a
179 * specified timeout to expire. The timeout is in jiffies. It is not
180 * interruptible. The caller is accounted as waiting for IO (which traditionally
181 * means blkio only).
182 *
183 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
184 * till timeout) if completed.
185 */
186 unsigned long __sched
187 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
188 {
189 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
190 }
191 EXPORT_SYMBOL(wait_for_completion_io_timeout);
192
193 /**
194 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
195 * @x: holds the state of this particular completion
196 *
197 * This waits for completion of a specific task to be signaled. It is
198 * interruptible.
199 *
200 * Return: -ERESTARTSYS if interrupted, 0 if completed.
201 */
202 int __sched wait_for_completion_interruptible(struct completion *x)
203 {
204 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
205 if (t == -ERESTARTSYS)
206 return t;
207 return 0;
208 }
209 EXPORT_SYMBOL(wait_for_completion_interruptible);
210
211 /**
212 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
213 * @x: holds the state of this particular completion
214 * @timeout: timeout value in jiffies
215 *
216 * This waits for either a completion of a specific task to be signaled or for a
217 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
218 *
219 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
220 * or number of jiffies left till timeout) if completed.
221 */
222 long __sched
223 wait_for_completion_interruptible_timeout(struct completion *x,
224 unsigned long timeout)
225 {
226 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
227 }
228 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
229
230 /**
231 * wait_for_completion_killable: - waits for completion of a task (killable)
232 * @x: holds the state of this particular completion
233 *
234 * This waits to be signaled for completion of a specific task. It can be
235 * interrupted by a kill signal.
236 *
237 * Return: -ERESTARTSYS if interrupted, 0 if completed.
238 */
239 int __sched wait_for_completion_killable(struct completion *x)
240 {
241 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
242 if (t == -ERESTARTSYS)
243 return t;
244 return 0;
245 }
246 EXPORT_SYMBOL(wait_for_completion_killable);
247
248 /**
249 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
250 * @x: holds the state of this particular completion
251 * @timeout: timeout value in jiffies
252 *
253 * This waits for either a completion of a specific task to be
254 * signaled or for a specified timeout to expire. It can be
255 * interrupted by a kill signal. The timeout is in jiffies.
256 *
257 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
258 * or number of jiffies left till timeout) if completed.
259 */
260 long __sched
261 wait_for_completion_killable_timeout(struct completion *x,
262 unsigned long timeout)
263 {
264 return wait_for_common(x, timeout, TASK_KILLABLE);
265 }
266 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
267
268 /**
269 * try_wait_for_completion - try to decrement a completion without blocking
270 * @x: completion structure
271 *
272 * Return: 0 if a decrement cannot be done without blocking
273 * 1 if a decrement succeeded.
274 *
275 * If a completion is being used as a counting completion,
276 * attempt to decrement the counter without blocking. This
277 * enables us to avoid waiting if the resource the completion
278 * is protecting is not available.
279 */
280 bool try_wait_for_completion(struct completion *x)
281 {
282 unsigned long flags;
283 bool ret = true;
284
285 /*
286 * Since x->done will need to be locked only
287 * in the non-blocking case, we check x->done
288 * first without taking the lock so we can
289 * return early in the blocking case.
290 */
291 if (!READ_ONCE(x->done))
292 return false;
293
294 spin_lock_irqsave(&x->wait.lock, flags);
295 if (!x->done)
296 ret = false;
297 else if (x->done != UINT_MAX)
298 x->done--;
299 spin_unlock_irqrestore(&x->wait.lock, flags);
300 return ret;
301 }
302 EXPORT_SYMBOL(try_wait_for_completion);
303
304 /**
305 * completion_done - Test to see if a completion has any waiters
306 * @x: completion structure
307 *
308 * Return: 0 if there are waiters (wait_for_completion() in progress)
309 * 1 if there are no waiters.
310 *
311 * Note, this will always return true if complete_all() was called on @X.
312 */
313 bool completion_done(struct completion *x)
314 {
315 unsigned long flags;
316
317 if (!READ_ONCE(x->done))
318 return false;
319
320 /*
321 * If ->done, we need to wait for complete() to release ->wait.lock
322 * otherwise we can end up freeing the completion before complete()
323 * is done referencing it.
324 */
325 spin_lock_irqsave(&x->wait.lock, flags);
326 spin_unlock_irqrestore(&x->wait.lock, flags);
327 return true;
328 }
329 EXPORT_SYMBOL(completion_done);