14 void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *key)  in __init_waitqueue_head()  argument
16 	spin_lock_init(&q->lock);  in __init_waitqueue_head()
17 	lockdep_set_class_and_name(&q->lock, key, name);  in __init_waitqueue_head()
18 	INIT_LIST_HEAD(&q->task_list);  in __init_waitqueue_head()
23 void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)  in add_wait_queue()  argument
28 	spin_lock_irqsave(&q->lock, flags);  in add_wait_queue()
29 	__add_wait_queue(q, wait);  in add_wait_queue()
30 	spin_unlock_irqrestore(&q->lock, flags);  in add_wait_queue()
34 void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)  in add_wait_queue_exclusive()  argument
39 	spin_lock_irqsave(&q->lock, flags);  in add_wait_queue_exclusive()
40 	__add_wait_queue_tail(q, wait);  in add_wait_queue_exclusive()
41 	spin_unlock_irqrestore(&q->lock, flags);  in add_wait_queue_exclusive()
45 void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)  in remove_wait_queue()  argument
49 	spin_lock_irqsave(&q->lock, flags);  in remove_wait_queue()
50 	__remove_wait_queue(q, wait);  in remove_wait_queue()
51 	spin_unlock_irqrestore(&q->lock, flags);  in remove_wait_queue()
65 static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,  in __wake_up_common()  argument
70 	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {  in __wake_up_common()
89 void __wake_up(wait_queue_head_t *q, unsigned int mode,  in __wake_up()  argument
94 	spin_lock_irqsave(&q->lock, flags);  in __wake_up()
95 	__wake_up_common(q, mode, nr_exclusive, 0, key);  in __wake_up()
96 	spin_unlock_irqrestore(&q->lock, flags);  in __wake_up()
103 void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr)  in __wake_up_locked()  argument
105 	__wake_up_common(q, mode, nr, 0, NULL);  in __wake_up_locked()
109 void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)  in __wake_up_locked_key()  argument
111 	__wake_up_common(q, mode, 1, 0, key);  in __wake_up_locked_key()
132 void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,  in __wake_up_sync_key()  argument
138 	if (unlikely(!q))  in __wake_up_sync_key()
144 	spin_lock_irqsave(&q->lock, flags);  in __wake_up_sync_key()
145 	__wake_up_common(q, mode, nr_exclusive, wake_flags, key);  in __wake_up_sync_key()
146 	spin_unlock_irqrestore(&q->lock, flags);  in __wake_up_sync_key()
153 void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)  in __wake_up_sync()  argument
155 	__wake_up_sync_key(q, mode, nr_exclusive, NULL);  in __wake_up_sync()
172 prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state)  in prepare_to_wait()  argument
177 	spin_lock_irqsave(&q->lock, flags);  in prepare_to_wait()
179 		__add_wait_queue(q, wait);  in prepare_to_wait()
181 	spin_unlock_irqrestore(&q->lock, flags);  in prepare_to_wait()
186 prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state)  in prepare_to_wait_exclusive()  argument
191 	spin_lock_irqsave(&q->lock, flags);  in prepare_to_wait_exclusive()
193 		__add_wait_queue_tail(q, wait);  in prepare_to_wait_exclusive()
195 	spin_unlock_irqrestore(&q->lock, flags);  in prepare_to_wait_exclusive()
199 long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state)  in prepare_to_wait_event()  argument
209 	spin_lock_irqsave(&q->lock, flags);  in prepare_to_wait_event()
212 			__add_wait_queue_tail(q, wait);  in prepare_to_wait_event()
214 			__add_wait_queue(q, wait);  in prepare_to_wait_event()
217 	spin_unlock_irqrestore(&q->lock, flags);  in prepare_to_wait_event()
232 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait)  in finish_wait()  argument
251 		spin_lock_irqsave(&q->lock, flags);  in finish_wait()
253 		spin_unlock_irqrestore(&q->lock, flags);  in finish_wait()
276 void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait,  in abort_exclusive_wait()  argument
282 	spin_lock_irqsave(&q->lock, flags);  in abort_exclusive_wait()
285 	else if (waitqueue_active(q))  in abort_exclusive_wait()
286 		__wake_up_locked_key(q, mode, key);  in abort_exclusive_wait()
287 	spin_unlock_irqrestore(&q->lock, flags);  in abort_exclusive_wait()
387 __wait_on_bit(wait_queue_head_t *wq, struct wait_bit_queue *q,  in __wait_on_bit()  argument
393 		prepare_to_wait(wq, &q->wait, mode);  in __wait_on_bit()
394 		if (test_bit(q->key.bit_nr, q->key.flags))  in __wait_on_bit()
395 			ret = (*action)(&q->key);  in __wait_on_bit()
396 	} while (test_bit(q->key.bit_nr, q->key.flags) && !ret);  in __wait_on_bit()
397 	finish_wait(wq, &q->wait);  in __wait_on_bit()
425 __wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q,  in __wait_on_bit_lock()  argument
431 		prepare_to_wait_exclusive(wq, &q->wait, mode);  in __wait_on_bit_lock()
432 		if (!test_bit(q->key.bit_nr, q->key.flags))  in __wait_on_bit_lock()
434 		ret = action(&q->key);  in __wait_on_bit_lock()
437 		abort_exclusive_wait(wq, &q->wait, mode, &q->key);  in __wait_on_bit_lock()
439 	} while (test_and_set_bit(q->key.bit_nr, q->key.flags));  in __wait_on_bit_lock()
440 	finish_wait(wq, &q->wait);  in __wait_on_bit_lock()
504 		unsigned long q = (unsigned long)p;  in atomic_t_waitqueue()  local
505 		return bit_waitqueue((void *)(q & ~1), q & 1);  in atomic_t_waitqueue()
531 int __wait_on_atomic_t(wait_queue_head_t *wq, struct wait_bit_queue *q,  in __wait_on_atomic_t()  argument
538 		prepare_to_wait(wq, &q->wait, mode);  in __wait_on_atomic_t()
539 		val = q->key.flags;  in __wait_on_atomic_t()
544 	finish_wait(wq, &q->wait);  in __wait_on_atomic_t()