1380 void __blk_put_request(struct request_queue *q, struct request *req)  in __blk_put_request()  argument
1386 		blk_mq_free_request(req);  in __blk_put_request()
1390 	blk_pm_put_request(req);  in __blk_put_request()
1392 	elv_completed_request(q, req);  in __blk_put_request()
1395 	WARN_ON(req->bio != NULL);  in __blk_put_request()
1401 	if (req->cmd_flags & REQ_ALLOCED) {  in __blk_put_request()
1402 		unsigned int flags = req->cmd_flags;  in __blk_put_request()
1403 		struct request_list *rl = blk_rq_rl(req);  in __blk_put_request()
1405 		BUG_ON(!list_empty(&req->queuelist));  in __blk_put_request()
1406 		BUG_ON(ELV_ON_HASH(req));  in __blk_put_request()
1408 		blk_free_request(rl, req);  in __blk_put_request()
1415 void blk_put_request(struct request *req)  in blk_put_request()  argument
1417 	struct request_queue *q = req->q;  in blk_put_request()
1420 		blk_mq_free_request(req);  in blk_put_request()
1425 		__blk_put_request(q, req);  in blk_put_request()
1462 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,  in bio_attempt_back_merge()  argument
1467 	if (!ll_back_merge_fn(q, req, bio))  in bio_attempt_back_merge()
1470 	trace_block_bio_backmerge(q, req, bio);  in bio_attempt_back_merge()
1472 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_back_merge()
1473 		blk_rq_set_mixed_merge(req);  in bio_attempt_back_merge()
1475 	req->biotail->bi_next = bio;  in bio_attempt_back_merge()
1476 	req->biotail = bio;  in bio_attempt_back_merge()
1477 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_back_merge()
1478 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));  in bio_attempt_back_merge()
1480 	blk_account_io_start(req, false);  in bio_attempt_back_merge()
1484 bool bio_attempt_front_merge(struct request_queue *q, struct request *req,  in bio_attempt_front_merge()  argument
1489 	if (!ll_front_merge_fn(q, req, bio))  in bio_attempt_front_merge()
1492 	trace_block_bio_frontmerge(q, req, bio);  in bio_attempt_front_merge()
1494 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_front_merge()
1495 		blk_rq_set_mixed_merge(req);  in bio_attempt_front_merge()
1497 	bio->bi_next = req->bio;  in bio_attempt_front_merge()
1498 	req->bio = bio;  in bio_attempt_front_merge()
1500 	req->__sector = bio->bi_iter.bi_sector;  in bio_attempt_front_merge()
1501 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_front_merge()
1502 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));  in bio_attempt_front_merge()
1504 	blk_account_io_start(req, false);  in bio_attempt_front_merge()
1569 void init_request_from_bio(struct request *req, struct bio *bio)  in init_request_from_bio()  argument
1571 	req->cmd_type = REQ_TYPE_FS;  in init_request_from_bio()
1573 	req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;  in init_request_from_bio()
1575 		req->cmd_flags |= REQ_FAILFAST_MASK;  in init_request_from_bio()
1577 	req->errors = 0;  in init_request_from_bio()
1578 	req->__sector = bio->bi_iter.bi_sector;  in init_request_from_bio()
1579 	req->ioprio = bio_prio(bio);  in init_request_from_bio()
1580 	blk_rq_bio_prep(req->q, req, bio);  in init_request_from_bio()
1588 	struct request *req;  in blk_queue_bio()  local
1619 	el_ret = elv_merge(q, &req, bio);  in blk_queue_bio()
1621 		if (bio_attempt_back_merge(q, req, bio)) {  in blk_queue_bio()
1622 			elv_bio_merged(q, req, bio);  in blk_queue_bio()
1623 			if (!attempt_back_merge(q, req))  in blk_queue_bio()
1624 				elv_merged_request(q, req, el_ret);  in blk_queue_bio()
1628 		if (bio_attempt_front_merge(q, req, bio)) {  in blk_queue_bio()
1629 			elv_bio_merged(q, req, bio);  in blk_queue_bio()
1630 			if (!attempt_front_merge(q, req))  in blk_queue_bio()
1631 				elv_merged_request(q, req, el_ret);  in blk_queue_bio()
1650 	req = get_request(q, rw_flags, bio, GFP_NOIO);  in blk_queue_bio()
1651 	if (IS_ERR(req)) {  in blk_queue_bio()
1652 		bio_endio(bio, PTR_ERR(req));	/* @q is dead */  in blk_queue_bio()
1662 	init_request_from_bio(req, bio);  in blk_queue_bio()
1665 		req->cpu = raw_smp_processor_id();  in blk_queue_bio()
1681 		list_add_tail(&req->queuelist, &plug->list);  in blk_queue_bio()
1682 		blk_account_io_start(req, true);  in blk_queue_bio()
1685 		add_acct_request(q, req, where);  in blk_queue_bio()
2142 void blk_account_io_completion(struct request *req, unsigned int bytes)  in blk_account_io_completion()  argument
2144 	if (blk_do_io_stat(req)) {  in blk_account_io_completion()
2145 		const int rw = rq_data_dir(req);  in blk_account_io_completion()
2150 		part = req->part;  in blk_account_io_completion()
2156 void blk_account_io_done(struct request *req)  in blk_account_io_done()  argument
2163 	if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {  in blk_account_io_done()
2164 		unsigned long duration = jiffies - req->start_time;  in blk_account_io_done()
2165 		const int rw = rq_data_dir(req);  in blk_account_io_done()
2170 		part = req->part;  in blk_account_io_done()
2379 void blk_start_request(struct request *req)  in blk_start_request()  argument
2381 	blk_dequeue_request(req);  in blk_start_request()
2387 	req->resid_len = blk_rq_bytes(req);  in blk_start_request()
2388 	if (unlikely(blk_bidi_rq(req)))  in blk_start_request()
2389 		req->next_rq->resid_len = blk_rq_bytes(req->next_rq);  in blk_start_request()
2391 	BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));  in blk_start_request()
2392 	blk_add_timer(req);  in blk_start_request()
2444 bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)  in blk_update_request()  argument
2448 	trace_block_rq_complete(req->q, req, nr_bytes);  in blk_update_request()
2450 	if (!req->bio)  in blk_update_request()
2461 	if (req->cmd_type == REQ_TYPE_FS)  in blk_update_request()
2462 		req->errors = 0;  in blk_update_request()
2464 	if (error && req->cmd_type == REQ_TYPE_FS &&  in blk_update_request()
2465 	    !(req->cmd_flags & REQ_QUIET)) {  in blk_update_request()
2493 				   __func__, error_type, req->rq_disk ?  in blk_update_request()
2494 				   req->rq_disk->disk_name : "?",  in blk_update_request()
2495 				   (unsigned long long)blk_rq_pos(req));  in blk_update_request()
2499 	blk_account_io_completion(req, nr_bytes);  in blk_update_request()
2502 	while (req->bio) {  in blk_update_request()
2503 		struct bio *bio = req->bio;  in blk_update_request()
2507 			req->bio = bio->bi_next;  in blk_update_request()
2509 		req_bio_endio(req, bio, bio_bytes, error);  in blk_update_request()
2521 	if (!req->bio) {  in blk_update_request()
2527 		req->__data_len = 0;  in blk_update_request()
2531 	req->__data_len -= total_bytes;  in blk_update_request()
2534 	if (req->cmd_type == REQ_TYPE_FS)  in blk_update_request()
2535 		req->__sector += total_bytes >> 9;  in blk_update_request()
2538 	if (req->cmd_flags & REQ_MIXED_MERGE) {  in blk_update_request()
2539 		req->cmd_flags &= ~REQ_FAILFAST_MASK;  in blk_update_request()
2540 		req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;  in blk_update_request()
2547 	if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {  in blk_update_request()
2548 		blk_dump_rq_flags(req, "request botched");  in blk_update_request()
2549 		req->__data_len = blk_rq_cur_bytes(req);  in blk_update_request()
2553 	blk_recalc_rq_segments(req);  in blk_update_request()
2587 void blk_unprep_request(struct request *req)  in blk_unprep_request()  argument
2589 	struct request_queue *q = req->q;  in blk_unprep_request()
2591 	req->cmd_flags &= ~REQ_DONTPREP;  in blk_unprep_request()
2593 		q->unprep_rq_fn(q, req);  in blk_unprep_request()
2600 void blk_finish_request(struct request *req, int error)  in blk_finish_request()  argument
2602 	if (req->cmd_flags & REQ_QUEUED)  in blk_finish_request()
2603 		blk_queue_end_tag(req->q, req);  in blk_finish_request()
2605 	BUG_ON(blk_queued_rq(req));  in blk_finish_request()
2607 	if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)  in blk_finish_request()
2608 		laptop_io_completion(&req->q->backing_dev_info);  in blk_finish_request()
2610 	blk_delete_timer(req);  in blk_finish_request()
2612 	if (req->cmd_flags & REQ_DONTPREP)  in blk_finish_request()
2613 		blk_unprep_request(req);  in blk_finish_request()
2615 	blk_account_io_done(req);  in blk_finish_request()
2617 	if (req->end_io)  in blk_finish_request()
2618 		req->end_io(req, error);  in blk_finish_request()
2620 		if (blk_bidi_rq(req))  in blk_finish_request()
2621 			__blk_put_request(req->next_rq->q, req->next_rq);  in blk_finish_request()
2623 		__blk_put_request(req->q, req);  in blk_finish_request()