97 static unsigned int blk_flush_policy(unsigned int fflags, struct request *rq)  in blk_flush_policy()  argument
101 	if (blk_rq_sectors(rq))  in blk_flush_policy()
105 		if (rq->cmd_flags & REQ_FLUSH)  in blk_flush_policy()
107 		if (!(fflags & REQ_FUA) && (rq->cmd_flags & REQ_FUA))  in blk_flush_policy()
113 static unsigned int blk_flush_cur_seq(struct request *rq)  in blk_flush_cur_seq()  argument
115 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
118 static void blk_flush_restore_request(struct request *rq)  in blk_flush_restore_request()  argument
125 	rq->bio = rq->biotail;  in blk_flush_restore_request()
128 	rq->cmd_flags &= ~REQ_FLUSH_SEQ;  in blk_flush_restore_request()
129 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
132 static bool blk_flush_queue_rq(struct request *rq, bool add_front)  in blk_flush_queue_rq()  argument
134 	if (rq->q->mq_ops) {  in blk_flush_queue_rq()
135 		struct request_queue *q = rq->q;  in blk_flush_queue_rq()
137 		blk_mq_add_to_requeue_list(rq, add_front);  in blk_flush_queue_rq()
142 			list_add(&rq->queuelist, &rq->q->queue_head);  in blk_flush_queue_rq()
144 			list_add_tail(&rq->queuelist, &rq->q->queue_head);  in blk_flush_queue_rq()
165 static bool blk_flush_complete_seq(struct request *rq,  in blk_flush_complete_seq()  argument
169 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
173 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
174 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
177 		seq = blk_flush_cur_seq(rq);  in blk_flush_complete_seq()
187 		list_move_tail(&rq->flush.list, pending);  in blk_flush_complete_seq()
191 		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);  in blk_flush_complete_seq()
192 		queued = blk_flush_queue_rq(rq, true);  in blk_flush_complete_seq()
202 		BUG_ON(!list_empty(&rq->queuelist));  in blk_flush_complete_seq()
203 		list_del_init(&rq->flush.list);  in blk_flush_complete_seq()
204 		blk_flush_restore_request(rq);  in blk_flush_complete_seq()
206 			blk_mq_end_request(rq, error);  in blk_flush_complete_seq()
208 			__blk_end_request_all(rq, error);  in blk_flush_complete_seq()
224 	struct request *rq, *n;  in flush_end_io()  local
243 	list_for_each_entry_safe(rq, n, running, flush.list) {  in flush_end_io()
244 		unsigned int seq = blk_flush_cur_seq(rq);  in flush_end_io()
247 		queued |= blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
326 static void flush_data_end_io(struct request *rq, int error)  in flush_data_end_io()  argument
328 	struct request_queue *q = rq->q;  in flush_data_end_io()
335 	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))  in flush_data_end_io()
339 static void mq_flush_data_end_io(struct request *rq, int error)  in mq_flush_data_end_io()  argument
341 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
343 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
354 	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))  in mq_flush_data_end_io()
371 void blk_insert_flush(struct request *rq)  in blk_insert_flush()  argument
373 	struct request_queue *q = rq->q;  in blk_insert_flush()
375 	unsigned int policy = blk_flush_policy(fflags, rq);  in blk_insert_flush()
376 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()
382 	rq->cmd_flags &= ~REQ_FLUSH;  in blk_insert_flush()
384 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
394 			blk_mq_end_request(rq, 0);  in blk_insert_flush()
396 			__blk_end_bidi_request(rq, 0, 0, 0);  in blk_insert_flush()
400 	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */  in blk_insert_flush()
410 			blk_mq_insert_request(rq, false, false, true);  in blk_insert_flush()
412 			list_add_tail(&rq->queuelist, &q->queue_head);  in blk_insert_flush()
420 	memset(&rq->flush, 0, sizeof(rq->flush));  in blk_insert_flush()
421 	INIT_LIST_HEAD(&rq->flush.list);  in blk_insert_flush()
422 	rq->cmd_flags |= REQ_FLUSH_SEQ;  in blk_insert_flush()
423 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_insert_flush()
425 		rq->end_io = mq_flush_data_end_io;  in blk_insert_flush()
428 		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()
432 	rq->end_io = flush_data_end_io;  in blk_insert_flush()
434 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()