23 void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)  in add_wait_queue()  argument
27 	wait->flags &= ~WQ_FLAG_EXCLUSIVE;  in add_wait_queue()
29 	__add_wait_queue(q, wait);  in add_wait_queue()
34 void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)  in add_wait_queue_exclusive()  argument
38 	wait->flags |= WQ_FLAG_EXCLUSIVE;  in add_wait_queue_exclusive()
40 	__add_wait_queue_tail(q, wait);  in add_wait_queue_exclusive()
45 void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)  in remove_wait_queue()  argument
50 	__remove_wait_queue(q, wait);  in remove_wait_queue()
172 prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state)  in prepare_to_wait()  argument
176 	wait->flags &= ~WQ_FLAG_EXCLUSIVE;  in prepare_to_wait()
178 	if (list_empty(&wait->task_list))  in prepare_to_wait()
179 		__add_wait_queue(q, wait);  in prepare_to_wait()
186 prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state)  in prepare_to_wait_exclusive()  argument
190 	wait->flags |= WQ_FLAG_EXCLUSIVE;  in prepare_to_wait_exclusive()
192 	if (list_empty(&wait->task_list))  in prepare_to_wait_exclusive()
193 		__add_wait_queue_tail(q, wait);  in prepare_to_wait_exclusive()
199 long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state)  in prepare_to_wait_event()  argument
206 	wait->private = current;  in prepare_to_wait_event()
207 	wait->func = autoremove_wake_function;  in prepare_to_wait_event()
210 	if (list_empty(&wait->task_list)) {  in prepare_to_wait_event()
211 		if (wait->flags & WQ_FLAG_EXCLUSIVE)  in prepare_to_wait_event()
212 			__add_wait_queue_tail(q, wait);  in prepare_to_wait_event()
214 			__add_wait_queue(q, wait);  in prepare_to_wait_event()
232 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait)  in finish_wait()  argument
250 	if (!list_empty_careful(&wait->task_list)) {  in finish_wait()
252 		list_del_init(&wait->task_list);  in finish_wait()
276 void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait,  in abort_exclusive_wait()  argument
283 	if (!list_empty(&wait->task_list))  in abort_exclusive_wait()
284 		list_del_init(&wait->task_list);  in abort_exclusive_wait()
291 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)  in autoremove_wake_function()  argument
293 	int ret = default_wake_function(wait, mode, sync, key);  in autoremove_wake_function()
296 		list_del_init(&wait->task_list);  in autoremove_wake_function()
326 long wait_woken(wait_queue_t *wait, unsigned mode, long timeout)  in wait_woken()  argument
334 	if (!(wait->flags & WQ_FLAG_WOKEN) && !is_kthread_should_stop())  in wait_woken()
344 	smp_store_mb(wait->flags, wait->flags & ~WQ_FLAG_WOKEN); /* B */  in wait_woken()
350 int woken_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)  in woken_wake_function()  argument
360 	wait->flags |= WQ_FLAG_WOKEN;  in woken_wake_function()
362 	return default_wake_function(wait, mode, sync, key);  in woken_wake_function()
366 int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *arg)  in wake_bit_function()  argument
370 		= container_of(wait, struct wait_bit_queue, wait);  in wake_bit_function()
377 		return autoremove_wake_function(wait, mode, sync, key);  in wake_bit_function()
393 		prepare_to_wait(wq, &q->wait, mode);  in __wait_on_bit()
397 	finish_wait(wq, &q->wait);  in __wait_on_bit()
406 	DEFINE_WAIT_BIT(wait, word, bit);  in out_of_line_wait_on_bit()
408 	return __wait_on_bit(wq, &wait, action, mode);  in out_of_line_wait_on_bit()
417 	DEFINE_WAIT_BIT(wait, word, bit);  in out_of_line_wait_on_bit_timeout()
419 	wait.key.timeout = jiffies + timeout;  in out_of_line_wait_on_bit_timeout()
420 	return __wait_on_bit(wq, &wait, action, mode);  in out_of_line_wait_on_bit_timeout()
431 		prepare_to_wait_exclusive(wq, &q->wait, mode);  in __wait_on_bit_lock()
437 		abort_exclusive_wait(wq, &q->wait, mode, &q->key);  in __wait_on_bit_lock()
440 	finish_wait(wq, &q->wait);  in __wait_on_bit_lock()
449 	DEFINE_WAIT_BIT(wait, word, bit);  in out_of_line_wait_on_bit_lock()
451 	return __wait_on_bit_lock(wq, &wait, action, mode);  in out_of_line_wait_on_bit_lock()
510 static int wake_atomic_t_function(wait_queue_t *wait, unsigned mode, int sync,  in wake_atomic_t_function()  argument
515 		= container_of(wait, struct wait_bit_queue, wait);  in wake_atomic_t_function()
522 	return autoremove_wake_function(wait, mode, sync, key);  in wake_atomic_t_function()
538 		prepare_to_wait(wq, &q->wait, mode);  in __wait_on_atomic_t()
544 	finish_wait(wq, &q->wait);  in __wait_on_atomic_t()
551 		.wait	= {						\
555 				LIST_HEAD_INIT((name).wait.task_list),	\
563 	DEFINE_WAIT_ATOMIC_T(wait, p);  in out_of_line_wait_on_atomic_t()
565 	return __wait_on_atomic_t(wq, &wait, action, mode);  in out_of_line_wait_on_atomic_t()