1461 void __blk_put_request(struct request_queue *q, struct request *req)  in __blk_put_request()  argument
1467 		blk_mq_free_request(req);  in __blk_put_request()
1471 	blk_pm_put_request(req);  in __blk_put_request()
1473 	elv_completed_request(q, req);  in __blk_put_request()
1476 	WARN_ON(req->bio != NULL);  in __blk_put_request()
1482 	if (req->cmd_flags & REQ_ALLOCED) {  in __blk_put_request()
1483 		unsigned int flags = req->cmd_flags;  in __blk_put_request()
1484 		struct request_list *rl = blk_rq_rl(req);  in __blk_put_request()
1486 		BUG_ON(!list_empty(&req->queuelist));  in __blk_put_request()
1487 		BUG_ON(ELV_ON_HASH(req));  in __blk_put_request()
1489 		blk_free_request(rl, req);  in __blk_put_request()
1496 void blk_put_request(struct request *req)  in blk_put_request()  argument
1498 	struct request_queue *q = req->q;  in blk_put_request()
1501 		blk_mq_free_request(req);  in blk_put_request()
1506 		__blk_put_request(q, req);  in blk_put_request()
1543 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,  in bio_attempt_back_merge()  argument
1548 	if (!ll_back_merge_fn(q, req, bio))  in bio_attempt_back_merge()
1551 	trace_block_bio_backmerge(q, req, bio);  in bio_attempt_back_merge()
1553 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_back_merge()
1554 		blk_rq_set_mixed_merge(req);  in bio_attempt_back_merge()
1556 	req->biotail->bi_next = bio;  in bio_attempt_back_merge()
1557 	req->biotail = bio;  in bio_attempt_back_merge()
1558 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_back_merge()
1559 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));  in bio_attempt_back_merge()
1561 	blk_account_io_start(req, false);  in bio_attempt_back_merge()
1565 bool bio_attempt_front_merge(struct request_queue *q, struct request *req,  in bio_attempt_front_merge()  argument
1570 	if (!ll_front_merge_fn(q, req, bio))  in bio_attempt_front_merge()
1573 	trace_block_bio_frontmerge(q, req, bio);  in bio_attempt_front_merge()
1575 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_front_merge()
1576 		blk_rq_set_mixed_merge(req);  in bio_attempt_front_merge()
1578 	bio->bi_next = req->bio;  in bio_attempt_front_merge()
1579 	req->bio = bio;  in bio_attempt_front_merge()
1581 	req->__sector = bio->bi_iter.bi_sector;  in bio_attempt_front_merge()
1582 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_front_merge()
1583 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));  in bio_attempt_front_merge()
1585 	blk_account_io_start(req, false);  in bio_attempt_front_merge()
1686 void init_request_from_bio(struct request *req, struct bio *bio)  in init_request_from_bio()  argument
1688 	req->cmd_type = REQ_TYPE_FS;  in init_request_from_bio()
1690 	req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;  in init_request_from_bio()
1692 		req->cmd_flags |= REQ_FAILFAST_MASK;  in init_request_from_bio()
1694 	req->errors = 0;  in init_request_from_bio()
1695 	req->__sector = bio->bi_iter.bi_sector;  in init_request_from_bio()
1696 	req->ioprio = bio_prio(bio);  in init_request_from_bio()
1697 	blk_rq_bio_prep(req->q, req, bio);  in init_request_from_bio()
1705 	struct request *req;  in blk_queue_bio()  local
1741 	el_ret = elv_merge(q, &req, bio);  in blk_queue_bio()
1743 		if (bio_attempt_back_merge(q, req, bio)) {  in blk_queue_bio()
1744 			elv_bio_merged(q, req, bio);  in blk_queue_bio()
1745 			if (!attempt_back_merge(q, req))  in blk_queue_bio()
1746 				elv_merged_request(q, req, el_ret);  in blk_queue_bio()
1750 		if (bio_attempt_front_merge(q, req, bio)) {  in blk_queue_bio()
1751 			elv_bio_merged(q, req, bio);  in blk_queue_bio()
1752 			if (!attempt_front_merge(q, req))  in blk_queue_bio()
1753 				elv_merged_request(q, req, el_ret);  in blk_queue_bio()
1772 	req = get_request(q, rw_flags, bio, GFP_NOIO);  in blk_queue_bio()
1773 	if (IS_ERR(req)) {  in blk_queue_bio()
1774 		bio->bi_error = PTR_ERR(req);  in blk_queue_bio()
1785 	init_request_from_bio(req, bio);  in blk_queue_bio()
1788 		req->cpu = raw_smp_processor_id();  in blk_queue_bio()
1804 		list_add_tail(&req->queuelist, &plug->list);  in blk_queue_bio()
1805 		blk_account_io_start(req, true);  in blk_queue_bio()
1808 		add_acct_request(q, req, where);  in blk_queue_bio()
2264 void blk_account_io_completion(struct request *req, unsigned int bytes)  in blk_account_io_completion()  argument
2266 	if (blk_do_io_stat(req)) {  in blk_account_io_completion()
2267 		const int rw = rq_data_dir(req);  in blk_account_io_completion()
2272 		part = req->part;  in blk_account_io_completion()
2278 void blk_account_io_done(struct request *req)  in blk_account_io_done()  argument
2285 	if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {  in blk_account_io_done()
2286 		unsigned long duration = jiffies - req->start_time;  in blk_account_io_done()
2287 		const int rw = rq_data_dir(req);  in blk_account_io_done()
2292 		part = req->part;  in blk_account_io_done()
2501 void blk_start_request(struct request *req)  in blk_start_request()  argument
2503 	blk_dequeue_request(req);  in blk_start_request()
2509 	req->resid_len = blk_rq_bytes(req);  in blk_start_request()
2510 	if (unlikely(blk_bidi_rq(req)))  in blk_start_request()
2511 		req->next_rq->resid_len = blk_rq_bytes(req->next_rq);  in blk_start_request()
2513 	BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));  in blk_start_request()
2514 	blk_add_timer(req);  in blk_start_request()
2566 bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)  in blk_update_request()  argument
2570 	trace_block_rq_complete(req->q, req, nr_bytes);  in blk_update_request()
2572 	if (!req->bio)  in blk_update_request()
2583 	if (req->cmd_type == REQ_TYPE_FS)  in blk_update_request()
2584 		req->errors = 0;  in blk_update_request()
2586 	if (error && req->cmd_type == REQ_TYPE_FS &&  in blk_update_request()
2587 	    !(req->cmd_flags & REQ_QUIET)) {  in blk_update_request()
2615 				   __func__, error_type, req->rq_disk ?  in blk_update_request()
2616 				   req->rq_disk->disk_name : "?",  in blk_update_request()
2617 				   (unsigned long long)blk_rq_pos(req));  in blk_update_request()
2621 	blk_account_io_completion(req, nr_bytes);  in blk_update_request()
2624 	while (req->bio) {  in blk_update_request()
2625 		struct bio *bio = req->bio;  in blk_update_request()
2629 			req->bio = bio->bi_next;  in blk_update_request()
2631 		req_bio_endio(req, bio, bio_bytes, error);  in blk_update_request()
2643 	if (!req->bio) {  in blk_update_request()
2649 		req->__data_len = 0;  in blk_update_request()
2653 	req->__data_len -= total_bytes;  in blk_update_request()
2656 	if (req->cmd_type == REQ_TYPE_FS)  in blk_update_request()
2657 		req->__sector += total_bytes >> 9;  in blk_update_request()
2660 	if (req->cmd_flags & REQ_MIXED_MERGE) {  in blk_update_request()
2661 		req->cmd_flags &= ~REQ_FAILFAST_MASK;  in blk_update_request()
2662 		req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;  in blk_update_request()
2669 	if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {  in blk_update_request()
2670 		blk_dump_rq_flags(req, "request botched");  in blk_update_request()
2671 		req->__data_len = blk_rq_cur_bytes(req);  in blk_update_request()
2675 	blk_recalc_rq_segments(req);  in blk_update_request()
2709 void blk_unprep_request(struct request *req)  in blk_unprep_request()  argument
2711 	struct request_queue *q = req->q;  in blk_unprep_request()
2713 	req->cmd_flags &= ~REQ_DONTPREP;  in blk_unprep_request()
2715 		q->unprep_rq_fn(q, req);  in blk_unprep_request()
2722 void blk_finish_request(struct request *req, int error)  in blk_finish_request()  argument
2724 	if (req->cmd_flags & REQ_QUEUED)  in blk_finish_request()
2725 		blk_queue_end_tag(req->q, req);  in blk_finish_request()
2727 	BUG_ON(blk_queued_rq(req));  in blk_finish_request()
2729 	if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)  in blk_finish_request()
2730 		laptop_io_completion(&req->q->backing_dev_info);  in blk_finish_request()
2732 	blk_delete_timer(req);  in blk_finish_request()
2734 	if (req->cmd_flags & REQ_DONTPREP)  in blk_finish_request()
2735 		blk_unprep_request(req);  in blk_finish_request()
2737 	blk_account_io_done(req);  in blk_finish_request()
2739 	if (req->end_io)  in blk_finish_request()
2740 		req->end_io(req, error);  in blk_finish_request()
2742 		if (blk_bidi_rq(req))  in blk_finish_request()
2743 			__blk_put_request(req->next_rq->q, req->next_rq);  in blk_finish_request()
2745 		__blk_put_request(req->q, req);  in blk_finish_request()