120 	struct virtio_blk *vblk = req->q->queuedata;  in virtblk_request_done()  local
124 		req->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual);  in virtblk_request_done()
125 		req->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len);  in virtblk_request_done()
126 		req->errors = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors);  in virtblk_request_done()
136 	struct virtio_blk *vblk = vq->vdev->priv;  in virtblk_done()  local
143 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtblk_done()
146 		while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {  in virtblk_done()
156 		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_done()
157 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtblk_done()
163 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_queue_rq()  local
172 	BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);  in virtio_queue_rq()
176 		vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_FLUSH);  in virtio_queue_rq()
178 		vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));  in virtio_queue_rq()
183 			vbr->out_hdr.sector = cpu_to_virtio64(vblk->vdev, blk_rq_pos(vbr->req));  in virtio_queue_rq()
184 			vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));  in virtio_queue_rq()
187 			vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_SCSI_CMD);  in virtio_queue_rq()
189 			vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));  in virtio_queue_rq()
192 			vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_GET_ID);  in virtio_queue_rq()
194 			vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));  in virtio_queue_rq()
207 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);  in virtio_queue_rq()
209 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);  in virtio_queue_rq()
212 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
213 	err = __virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);  in virtio_queue_rq()
215 		virtqueue_kick(vblk->vqs[qid].vq);  in virtio_queue_rq()
217 		spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
225 	if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))  in virtio_queue_rq()
227 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
230 		virtqueue_notify(vblk->vqs[qid].vq);  in virtio_queue_rq()
238 	struct virtio_blk *vblk = disk->private_data;  in virtblk_get_id()  local
243 	bio = bio_map_kern(vblk->disk->queue, id_str, VIRTIO_BLK_ID_BYTES,  in virtblk_get_id()
248 	req = blk_make_request(vblk->disk->queue, bio, GFP_KERNEL);  in virtblk_get_id()
255 	err = blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);  in virtblk_get_id()
265 	struct virtio_blk *vblk = disk->private_data;  in virtblk_ioctl()  local
270 	if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI))  in virtblk_ioctl()
280 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_getgeo()  local
283 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {  in virtblk_getgeo()
284 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
286 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
288 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
339 	struct virtio_blk *vblk =  in virtblk_config_changed_work()  local
341 	struct virtio_device *vdev = vblk->vdev;  in virtblk_config_changed_work()
342 	struct request_queue *q = vblk->disk->queue;  in virtblk_config_changed_work()
368 	set_capacity(vblk->disk, capacity);  in virtblk_config_changed_work()
369 	revalidate_disk(vblk->disk);  in virtblk_config_changed_work()
370 	kobject_uevent_env(&disk_to_dev(vblk->disk)->kobj, KOBJ_CHANGE, envp);  in virtblk_config_changed_work()
375 	struct virtio_blk *vblk = vdev->priv;  in virtblk_config_changed()  local
377 	queue_work(virtblk_wq, &vblk->config_work);  in virtblk_config_changed()
380 static int init_vq(struct virtio_blk *vblk)  in init_vq()  argument
388 	struct virtio_device *vdev = vblk->vdev;  in init_vq()
396 	vblk->vqs = kmalloc(sizeof(*vblk->vqs) * num_vqs, GFP_KERNEL);  in init_vq()
397 	if (!vblk->vqs) {  in init_vq()
416 		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);  in init_vq()
417 		names[i] = vblk->vqs[i].name;  in init_vq()
426 		spin_lock_init(&vblk->vqs[i].lock);  in init_vq()
427 		vblk->vqs[i].vq = vqs[i];  in init_vq()
429 	vblk->num_vqs = num_vqs;  in init_vq()
439 		kfree(vblk->vqs);  in init_vq()
489 	struct virtio_blk *vblk = vdev->priv;  in virtblk_update_cache_mode()  local
492 		blk_queue_flush(vblk->disk->queue, REQ_FLUSH);  in virtblk_update_cache_mode()
494 		blk_queue_flush(vblk->disk->queue, 0);  in virtblk_update_cache_mode()
496 	revalidate_disk(vblk->disk);  in virtblk_update_cache_mode()
508 	struct virtio_blk *vblk = disk->private_data;  in virtblk_cache_type_store()  local
509 	struct virtio_device *vdev = vblk->vdev;  in virtblk_cache_type_store()
512 	BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));  in virtblk_cache_type_store()
530 	struct virtio_blk *vblk = disk->private_data;  in virtblk_cache_type_show()  local
531 	u8 writeback = virtblk_get_cache_mode(vblk->vdev);  in virtblk_cache_type_show()
548 	struct virtio_blk *vblk = data;  in virtblk_init_request()  local
551 	sg_init_table(vbr->sg, vblk->sg_elems);  in virtblk_init_request()
567 	struct virtio_blk *vblk;  in virtblk_probe()  local
599 	vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);  in virtblk_probe()
600 	if (!vblk) {  in virtblk_probe()
605 	vblk->vdev = vdev;  in virtblk_probe()
606 	vblk->sg_elems = sg_elems;  in virtblk_probe()
608 	INIT_WORK(&vblk->config_work, virtblk_config_changed_work);  in virtblk_probe()
610 	err = init_vq(vblk);  in virtblk_probe()
615 	vblk->disk = alloc_disk(1 << PART_BITS);  in virtblk_probe()
616 	if (!vblk->disk) {  in virtblk_probe()
623 		virtblk_queue_depth = vblk->vqs[0].vq->num_free;  in virtblk_probe()
629 	memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));  in virtblk_probe()
630 	vblk->tag_set.ops = &virtio_mq_ops;  in virtblk_probe()
631 	vblk->tag_set.queue_depth = virtblk_queue_depth;  in virtblk_probe()
632 	vblk->tag_set.numa_node = NUMA_NO_NODE;  in virtblk_probe()
633 	vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in virtblk_probe()
634 	vblk->tag_set.cmd_size =  in virtblk_probe()
637 	vblk->tag_set.driver_data = vblk;  in virtblk_probe()
638 	vblk->tag_set.nr_hw_queues = vblk->num_vqs;  in virtblk_probe()
640 	err = blk_mq_alloc_tag_set(&vblk->tag_set);  in virtblk_probe()
644 	q = vblk->disk->queue = blk_mq_init_queue(&vblk->tag_set);  in virtblk_probe()
650 	q->queuedata = vblk;  in virtblk_probe()
652 	virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);  in virtblk_probe()
654 	vblk->disk->major = major;  in virtblk_probe()
655 	vblk->disk->first_minor = index_to_minor(index);  in virtblk_probe()
656 	vblk->disk->private_data = vblk;  in virtblk_probe()
657 	vblk->disk->fops = &virtblk_fops;  in virtblk_probe()
658 	vblk->disk->driverfs_dev = &vdev->dev;  in virtblk_probe()
659 	vblk->disk->flags |= GENHD_FL_EXT_DEVT;  in virtblk_probe()
660 	vblk->index = index;  in virtblk_probe()
667 		set_disk_ro(vblk->disk, 1);  in virtblk_probe()
678 	set_capacity(vblk->disk, cap);  in virtblk_probe()
681 	blk_queue_max_segments(q, vblk->sg_elems-2);  in virtblk_probe()
735 	add_disk(vblk->disk);  in virtblk_probe()
736 	err = device_create_file(disk_to_dev(vblk->disk), &dev_attr_serial);  in virtblk_probe()
741 		err = device_create_file(disk_to_dev(vblk->disk),  in virtblk_probe()
744 		err = device_create_file(disk_to_dev(vblk->disk),  in virtblk_probe()
751 	del_gendisk(vblk->disk);  in virtblk_probe()
752 	blk_cleanup_queue(vblk->disk->queue);  in virtblk_probe()
754 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_probe()
756 	put_disk(vblk->disk);  in virtblk_probe()
760 	kfree(vblk);  in virtblk_probe()
769 	struct virtio_blk *vblk = vdev->priv;  in virtblk_remove()  local
770 	int index = vblk->index;  in virtblk_remove()
774 	flush_work(&vblk->config_work);  in virtblk_remove()
776 	del_gendisk(vblk->disk);  in virtblk_remove()
777 	blk_cleanup_queue(vblk->disk->queue);  in virtblk_remove()
779 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_remove()
784 	refc = atomic_read(&disk_to_dev(vblk->disk)->kobj.kref.refcount);  in virtblk_remove()
785 	put_disk(vblk->disk);  in virtblk_remove()
787 	kfree(vblk->vqs);  in virtblk_remove()
788 	kfree(vblk);  in virtblk_remove()
798 	struct virtio_blk *vblk = vdev->priv;  in virtblk_freeze()  local
804 	flush_work(&vblk->config_work);  in virtblk_freeze()
806 	blk_mq_stop_hw_queues(vblk->disk->queue);  in virtblk_freeze()
814 	struct virtio_blk *vblk = vdev->priv;  in virtblk_restore()  local
823 	blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_restore()