65 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_disable_timer()  argument
72 	if (list_empty(&queue->timer_list.list))  in __rpc_disable_timer()
73 		del_timer(&queue->timer_list.timer);  in __rpc_disable_timer()
77 rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)  in rpc_set_queue_timer()  argument
79 	queue->timer_list.expires = expires;  in rpc_set_queue_timer()
80 	mod_timer(&queue->timer_list.timer, expires);  in rpc_set_queue_timer()
87 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_add_timer()  argument
96 …if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.…  in __rpc_add_timer()
97 		rpc_set_queue_timer(queue, task->u.tk_wait.expires);  in __rpc_add_timer()
98 	list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);  in __rpc_add_timer()
101 static void rpc_rotate_queue_owner(struct rpc_wait_queue *queue)  in rpc_rotate_queue_owner()  argument
103 	struct list_head *q = &queue->tasks[queue->priority];  in rpc_rotate_queue_owner()
108 		if (task->tk_owner == queue->owner)  in rpc_rotate_queue_owner()
113 static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)  in rpc_set_waitqueue_priority()  argument
115 	if (queue->priority != priority) {  in rpc_set_waitqueue_priority()
117 		rpc_rotate_queue_owner(queue);  in rpc_set_waitqueue_priority()
118 		queue->priority = priority;  in rpc_set_waitqueue_priority()
122 static void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)  in rpc_set_waitqueue_owner()  argument
124 	queue->owner = pid;  in rpc_set_waitqueue_owner()
125 	queue->nr = RPC_BATCH_COUNT;  in rpc_set_waitqueue_owner()
128 static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)  in rpc_reset_waitqueue_priority()  argument
130 	rpc_set_waitqueue_priority(queue, queue->maxpriority);  in rpc_reset_waitqueue_priority()
131 	rpc_set_waitqueue_owner(queue, 0);  in rpc_reset_waitqueue_priority()
137 static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue_priority()  argument
145 	if (unlikely(queue_priority > queue->maxpriority))  in __rpc_add_wait_queue_priority()
146 		queue_priority = queue->maxpriority;  in __rpc_add_wait_queue_priority()
147 	if (queue_priority > queue->priority)  in __rpc_add_wait_queue_priority()
148 		rpc_set_waitqueue_priority(queue, queue_priority);  in __rpc_add_wait_queue_priority()
149 	q = &queue->tasks[queue_priority];  in __rpc_add_wait_queue_priority()
167 static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue()  argument
175 	if (RPC_IS_PRIORITY(queue))  in __rpc_add_wait_queue()
176 		__rpc_add_wait_queue_priority(queue, task, queue_priority);  in __rpc_add_wait_queue()
178 		list_add(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
180 		list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
181 	task->tk_waitqueue = queue;  in __rpc_add_wait_queue()
182 	queue->qlen++;  in __rpc_add_wait_queue()
188 			task->tk_pid, queue, rpc_qname(queue));  in __rpc_add_wait_queue()
209 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_remove_wait_queue()  argument
211 	__rpc_disable_timer(queue, task);  in __rpc_remove_wait_queue()
212 	if (RPC_IS_PRIORITY(queue))  in __rpc_remove_wait_queue()
215 	queue->qlen--;  in __rpc_remove_wait_queue()
217 			task->tk_pid, queue, rpc_qname(queue));  in __rpc_remove_wait_queue()
220 static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigne…  in __rpc_init_priority_wait_queue()  argument
224 	spin_lock_init(&queue->lock);  in __rpc_init_priority_wait_queue()
225 	for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)  in __rpc_init_priority_wait_queue()
226 		INIT_LIST_HEAD(&queue->tasks[i]);  in __rpc_init_priority_wait_queue()
227 	queue->maxpriority = nr_queues - 1;  in __rpc_init_priority_wait_queue()
228 	rpc_reset_waitqueue_priority(queue);  in __rpc_init_priority_wait_queue()
229 	queue->qlen = 0;  in __rpc_init_priority_wait_queue()
230 	setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);  in __rpc_init_priority_wait_queue()
231 	INIT_LIST_HEAD(&queue->timer_list.list);  in __rpc_init_priority_wait_queue()
232 	rpc_assign_waitqueue_name(queue, qname);  in __rpc_init_priority_wait_queue()
235 void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_priority_wait_queue()  argument
237 	__rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);  in rpc_init_priority_wait_queue()
241 void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_wait_queue()  argument
243 	__rpc_init_priority_wait_queue(queue, qname, 1);  in rpc_init_wait_queue()
247 void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)  in rpc_destroy_wait_queue()  argument
249 	del_timer_sync(&queue->timer_list.timer);  in rpc_destroy_wait_queue()
416 static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_do_wake_up_task()  argument
427 	trace_rpc_task_wakeup(task->tk_client, task, queue);  in __rpc_do_wake_up_task()
429 	__rpc_remove_wait_queue(queue, task);  in __rpc_do_wake_up_task()
439 static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_task_queue_locked()  argument
443 		if (task->tk_waitqueue == queue)  in rpc_wake_up_task_queue_locked()
444 			__rpc_do_wake_up_task(queue, task);  in rpc_wake_up_task_queue_locked()
451 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_queued_task()  argument
453 	spin_lock_bh(&queue->lock);  in rpc_wake_up_queued_task()
454 	rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_queued_task()
455 	spin_unlock_bh(&queue->lock);  in rpc_wake_up_queued_task()
462 static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue)  in __rpc_find_next_queued_priority()  argument
470 	q = &queue->tasks[queue->priority];  in __rpc_find_next_queued_priority()
473 		if (queue->owner == task->tk_owner) {  in __rpc_find_next_queued_priority()
474 			if (--queue->nr)  in __rpc_find_next_queued_priority()
488 		if (q == &queue->tasks[0])  in __rpc_find_next_queued_priority()
489 			q = &queue->tasks[queue->maxpriority];  in __rpc_find_next_queued_priority()
496 	} while (q != &queue->tasks[queue->priority]);  in __rpc_find_next_queued_priority()
498 	rpc_reset_waitqueue_priority(queue);  in __rpc_find_next_queued_priority()
502 	rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));  in __rpc_find_next_queued_priority()
504 	rpc_set_waitqueue_owner(queue, task->tk_owner);  in __rpc_find_next_queued_priority()
509 static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue)  in __rpc_find_next_queued()  argument
511 	if (RPC_IS_PRIORITY(queue))  in __rpc_find_next_queued()
512 		return __rpc_find_next_queued_priority(queue);  in __rpc_find_next_queued()
513 	if (!list_empty(&queue->tasks[0]))  in __rpc_find_next_queued()
514 		return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued()
521 struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue,  in rpc_wake_up_first()  argument
527 			queue, rpc_qname(queue));  in rpc_wake_up_first()
528 	spin_lock_bh(&queue->lock);  in rpc_wake_up_first()
529 	task = __rpc_find_next_queued(queue);  in rpc_wake_up_first()
532 			rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_first()
536 	spin_unlock_bh(&queue->lock);  in rpc_wake_up_first()
550 struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue)  in rpc_wake_up_next()  argument
552 	return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL);  in rpc_wake_up_next()
562 void rpc_wake_up(struct rpc_wait_queue *queue)  in rpc_wake_up()  argument
566 	spin_lock_bh(&queue->lock);  in rpc_wake_up()
567 	head = &queue->tasks[queue->maxpriority];  in rpc_wake_up()
574 			rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up()
576 		if (head == &queue->tasks[0])  in rpc_wake_up()
580 	spin_unlock_bh(&queue->lock);  in rpc_wake_up()
591 void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)  in rpc_wake_up_status()  argument
595 	spin_lock_bh(&queue->lock);  in rpc_wake_up_status()
596 	head = &queue->tasks[queue->maxpriority];  in rpc_wake_up_status()
604 			rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_status()
606 		if (head == &queue->tasks[0])  in rpc_wake_up_status()
610 	spin_unlock_bh(&queue->lock);  in rpc_wake_up_status()
616 	struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;  in __rpc_queue_timer_fn()  local
620 	spin_lock(&queue->lock);  in __rpc_queue_timer_fn()
622 	list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {  in __rpc_queue_timer_fn()
627 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_queue_timer_fn()
633 	if (!list_empty(&queue->timer_list.list))  in __rpc_queue_timer_fn()
634 		rpc_set_queue_timer(queue, expires);  in __rpc_queue_timer_fn()
635 	spin_unlock(&queue->lock);  in __rpc_queue_timer_fn()
720 	struct rpc_wait_queue *queue;  in __rpc_execute()  local
767 		queue = task->tk_waitqueue;  in __rpc_execute()
768 		spin_lock_bh(&queue->lock);  in __rpc_execute()
770 			spin_unlock_bh(&queue->lock);  in __rpc_execute()
774 		spin_unlock_bh(&queue->lock);  in __rpc_execute()