321 static u64 fuse_get_unique(struct fuse_iqueue *fiq)  in fuse_get_unique()  argument
323 	return ++fiq->reqctr;  in fuse_get_unique()
326 static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)  in queue_request()  argument
330 	list_add_tail(&req->list, &fiq->pending);  in queue_request()
331 	wake_up_locked(&fiq->waitq);  in queue_request()
332 	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in queue_request()
338 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_queue_forget()  local
343 	spin_lock(&fiq->waitq.lock);  in fuse_queue_forget()
344 	if (fiq->connected) {  in fuse_queue_forget()
345 		fiq->forget_list_tail->next = forget;  in fuse_queue_forget()
346 		fiq->forget_list_tail = forget;  in fuse_queue_forget()
347 		wake_up_locked(&fiq->waitq);  in fuse_queue_forget()
348 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_queue_forget()
352 	spin_unlock(&fiq->waitq.lock);  in fuse_queue_forget()
360 		struct fuse_iqueue *fiq = &fc->iq;  in flush_bg_queue()  local
365 		spin_lock(&fiq->waitq.lock);  in flush_bg_queue()
366 		req->in.h.unique = fuse_get_unique(fiq);  in flush_bg_queue()
367 		queue_request(fiq, req);  in flush_bg_queue()
368 		spin_unlock(&fiq->waitq.lock);  in flush_bg_queue()
382 	struct fuse_iqueue *fiq = &fc->iq;  in request_end()  local
387 	spin_lock(&fiq->waitq.lock);  in request_end()
389 	spin_unlock(&fiq->waitq.lock);  in request_end()
418 static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)  in queue_interrupt()  argument
420 	spin_lock(&fiq->waitq.lock);  in queue_interrupt()
422 		list_add_tail(&req->intr_entry, &fiq->interrupts);  in queue_interrupt()
423 		wake_up_locked(&fiq->waitq);  in queue_interrupt()
425 	spin_unlock(&fiq->waitq.lock);  in queue_interrupt()
426 	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in queue_interrupt()
431 	struct fuse_iqueue *fiq = &fc->iq;  in request_wait_answer()  local
445 			queue_interrupt(fiq, req);  in request_wait_answer()
460 		spin_lock(&fiq->waitq.lock);  in request_wait_answer()
464 			spin_unlock(&fiq->waitq.lock);  in request_wait_answer()
469 		spin_unlock(&fiq->waitq.lock);  in request_wait_answer()
481 	struct fuse_iqueue *fiq = &fc->iq;  in __fuse_request_send()  local
484 	spin_lock(&fiq->waitq.lock);  in __fuse_request_send()
485 	if (!fiq->connected) {  in __fuse_request_send()
486 		spin_unlock(&fiq->waitq.lock);  in __fuse_request_send()
489 		req->in.h.unique = fuse_get_unique(fiq);  in __fuse_request_send()
490 		queue_request(fiq, req);  in __fuse_request_send()
494 		spin_unlock(&fiq->waitq.lock);  in __fuse_request_send()
624 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_request_send_notify_reply()  local
628 	spin_lock(&fiq->waitq.lock);  in fuse_request_send_notify_reply()
629 	if (fiq->connected) {  in fuse_request_send_notify_reply()
630 		queue_request(fiq, req);  in fuse_request_send_notify_reply()
633 	spin_unlock(&fiq->waitq.lock);  in fuse_request_send_notify_reply()
1061 static int forget_pending(struct fuse_iqueue *fiq)  in forget_pending()  argument
1063 	return fiq->forget_list_head.next != NULL;  in forget_pending()
1066 static int request_pending(struct fuse_iqueue *fiq)  in request_pending()  argument
1068 	return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||  in request_pending()
1069 		forget_pending(fiq);  in request_pending()
1080 static int fuse_read_interrupt(struct fuse_iqueue *fiq,  in fuse_read_interrupt()  argument
1083 __releases(fiq->waitq.lock)  in fuse_read_interrupt()
1091 	req->intr_unique = fuse_get_unique(fiq);  in fuse_read_interrupt()
1099 	spin_unlock(&fiq->waitq.lock);  in fuse_read_interrupt()
1111 static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,  in dequeue_forget()  argument
1115 	struct fuse_forget_link *head = fiq->forget_list_head.next;  in dequeue_forget()
1122 	fiq->forget_list_head.next = *newhead;  in dequeue_forget()
1124 	if (fiq->forget_list_head.next == NULL)  in dequeue_forget()
1125 		fiq->forget_list_tail = &fiq->forget_list_head;  in dequeue_forget()
1133 static int fuse_read_single_forget(struct fuse_iqueue *fiq,  in fuse_read_single_forget()  argument
1136 __releases(fiq->waitq.lock)  in fuse_read_single_forget()
1139 	struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);  in fuse_read_single_forget()
1146 		.unique = fuse_get_unique(fiq),  in fuse_read_single_forget()
1150 	spin_unlock(&fiq->waitq.lock);  in fuse_read_single_forget()
1166 static int fuse_read_batch_forget(struct fuse_iqueue *fiq,  in fuse_read_batch_forget()  argument
1168 __releases(fiq->waitq.lock)  in fuse_read_batch_forget()
1177 		.unique = fuse_get_unique(fiq),  in fuse_read_batch_forget()
1182 		spin_unlock(&fiq->waitq.lock);  in fuse_read_batch_forget()
1187 	head = dequeue_forget(fiq, max_forgets, &count);  in fuse_read_batch_forget()
1188 	spin_unlock(&fiq->waitq.lock);  in fuse_read_batch_forget()
1215 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,  in fuse_read_forget()  argument
1218 __releases(fiq->waitq.lock)  in fuse_read_forget()
1220 	if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)  in fuse_read_forget()
1221 		return fuse_read_single_forget(fiq, cs, nbytes);  in fuse_read_forget()
1223 		return fuse_read_batch_forget(fiq, cs, nbytes);  in fuse_read_forget()
1240 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_dev_do_read()  local
1247 	spin_lock(&fiq->waitq.lock);  in fuse_dev_do_read()
1249 	if ((file->f_flags & O_NONBLOCK) && fiq->connected &&  in fuse_dev_do_read()
1250 	    !request_pending(fiq))  in fuse_dev_do_read()
1253 	err = wait_event_interruptible_exclusive_locked(fiq->waitq,  in fuse_dev_do_read()
1254 				!fiq->connected || request_pending(fiq));  in fuse_dev_do_read()
1259 	if (!fiq->connected)  in fuse_dev_do_read()
1262 	if (!list_empty(&fiq->interrupts)) {  in fuse_dev_do_read()
1263 		req = list_entry(fiq->interrupts.next, struct fuse_req,  in fuse_dev_do_read()
1265 		return fuse_read_interrupt(fiq, cs, nbytes, req);  in fuse_dev_do_read()
1268 	if (forget_pending(fiq)) {  in fuse_dev_do_read()
1269 		if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)  in fuse_dev_do_read()
1270 			return fuse_read_forget(fc, fiq, cs, nbytes);  in fuse_dev_do_read()
1272 		if (fiq->forget_batch <= -8)  in fuse_dev_do_read()
1273 			fiq->forget_batch = 16;  in fuse_dev_do_read()
1276 	req = list_entry(fiq->pending.next, struct fuse_req, list);  in fuse_dev_do_read()
1279 	spin_unlock(&fiq->waitq.lock);  in fuse_dev_do_read()
1321 		queue_interrupt(fiq, req);  in fuse_dev_do_read()
1333 	spin_unlock(&fiq->waitq.lock);  in fuse_dev_do_read()
2056 	struct fuse_iqueue *fiq;  in fuse_dev_poll()  local
2062 	fiq = &fud->fc->iq;  in fuse_dev_poll()
2063 	poll_wait(file, &fiq->waitq, wait);  in fuse_dev_poll()
2065 	spin_lock(&fiq->waitq.lock);  in fuse_dev_poll()
2066 	if (!fiq->connected)  in fuse_dev_poll()
2068 	else if (request_pending(fiq))  in fuse_dev_poll()
2070 	spin_unlock(&fiq->waitq.lock);  in fuse_dev_poll()
2128 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_abort_conn()  local
2161 		spin_lock(&fiq->waitq.lock);  in fuse_abort_conn()
2162 		fiq->connected = 0;  in fuse_abort_conn()
2163 		list_splice_init(&fiq->pending, &to_end2);  in fuse_abort_conn()
2164 		while (forget_pending(fiq))  in fuse_abort_conn()
2165 			kfree(dequeue_forget(fiq, 1, NULL));  in fuse_abort_conn()
2166 		wake_up_all_locked(&fiq->waitq);  in fuse_abort_conn()
2167 		spin_unlock(&fiq->waitq.lock);  in fuse_abort_conn()
2168 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_abort_conn()