This source file includes following definitions.
- virtblk_result
- virtblk_add_req_scsi
- virtblk_scsi_request_done
- virtblk_ioctl
- virtblk_add_req_scsi
- virtblk_scsi_request_done
- virtblk_add_req
- virtblk_setup_discard_write_zeroes
- virtblk_request_done
- virtblk_done
- virtio_commit_rqs
- virtio_queue_rq
- virtblk_get_id
- virtblk_get
- virtblk_put
- virtblk_open
- virtblk_release
- virtblk_getgeo
- index_to_minor
- minor_to_index
- serial_show
- virtblk_update_capacity
- virtblk_config_changed_work
- virtblk_config_changed
- init_vq
- virtblk_name_format
- virtblk_get_cache_mode
- virtblk_update_cache_mode
- cache_type_store
- cache_type_show
- virtblk_attrs_are_visible
- virtblk_init_request
- virtblk_map_queues
- virtblk_initialize_rq
- virtblk_probe
- virtblk_remove
- virtblk_freeze
- virtblk_restore
- fini
1
2
3 #include <linux/spinlock.h>
4 #include <linux/slab.h>
5 #include <linux/blkdev.h>
6 #include <linux/hdreg.h>
7 #include <linux/module.h>
8 #include <linux/mutex.h>
9 #include <linux/interrupt.h>
10 #include <linux/virtio.h>
11 #include <linux/virtio_blk.h>
12 #include <linux/scatterlist.h>
13 #include <linux/string_helpers.h>
14 #include <scsi/scsi_cmnd.h>
15 #include <linux/idr.h>
16 #include <linux/blk-mq.h>
17 #include <linux/blk-mq-virtio.h>
18 #include <linux/numa.h>
19
20 #define PART_BITS 4
21 #define VQ_NAME_LEN 16
22 #define MAX_DISCARD_SEGMENTS 256u
23
24 static int major;
25 static DEFINE_IDA(vd_index_ida);
26
27 static struct workqueue_struct *virtblk_wq;
28
29 struct virtio_blk_vq {
30 struct virtqueue *vq;
31 spinlock_t lock;
32 char name[VQ_NAME_LEN];
33 } ____cacheline_aligned_in_smp;
34
35 struct virtio_blk {
36
37
38
39
40
41
42
43
44 struct mutex vdev_mutex;
45 struct virtio_device *vdev;
46
47
48 struct gendisk *disk;
49
50
51 struct blk_mq_tag_set tag_set;
52
53
54 struct work_struct config_work;
55
56
57
58
59
60
61 refcount_t refs;
62
63
64 unsigned int sg_elems;
65
66
67 int index;
68
69
70 int num_vqs;
71 struct virtio_blk_vq *vqs;
72 };
73
74 struct virtblk_req {
75 #ifdef CONFIG_VIRTIO_BLK_SCSI
76 struct scsi_request sreq;
77 u8 sense[SCSI_SENSE_BUFFERSIZE];
78 struct virtio_scsi_inhdr in_hdr;
79 #endif
80 struct virtio_blk_outhdr out_hdr;
81 u8 status;
82 struct scatterlist sg[];
83 };
84
85 static inline blk_status_t virtblk_result(struct virtblk_req *vbr)
86 {
87 switch (vbr->status) {
88 case VIRTIO_BLK_S_OK:
89 return BLK_STS_OK;
90 case VIRTIO_BLK_S_UNSUPP:
91 return BLK_STS_NOTSUPP;
92 default:
93 return BLK_STS_IOERR;
94 }
95 }
96
97
98
99
100
101
102
103 #ifdef CONFIG_VIRTIO_BLK_SCSI
104 static int virtblk_add_req_scsi(struct virtqueue *vq, struct virtblk_req *vbr,
105 struct scatterlist *data_sg, bool have_data)
106 {
107 struct scatterlist hdr, status, cmd, sense, inhdr, *sgs[6];
108 unsigned int num_out = 0, num_in = 0;
109
110 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));
111 sgs[num_out++] = &hdr;
112 sg_init_one(&cmd, vbr->sreq.cmd, vbr->sreq.cmd_len);
113 sgs[num_out++] = &cmd;
114
115 if (have_data) {
116 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))
117 sgs[num_out++] = data_sg;
118 else
119 sgs[num_out + num_in++] = data_sg;
120 }
121
122 sg_init_one(&sense, vbr->sense, SCSI_SENSE_BUFFERSIZE);
123 sgs[num_out + num_in++] = &sense;
124 sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr));
125 sgs[num_out + num_in++] = &inhdr;
126 sg_init_one(&status, &vbr->status, sizeof(vbr->status));
127 sgs[num_out + num_in++] = &status;
128
129 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);
130 }
131
132 static inline void virtblk_scsi_request_done(struct request *req)
133 {
134 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
135 struct virtio_blk *vblk = req->q->queuedata;
136 struct scsi_request *sreq = &vbr->sreq;
137
138 sreq->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual);
139 sreq->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len);
140 sreq->result = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors);
141 }
142
143 static int virtblk_ioctl(struct block_device *bdev, fmode_t mode,
144 unsigned int cmd, unsigned long data)
145 {
146 struct gendisk *disk = bdev->bd_disk;
147 struct virtio_blk *vblk = disk->private_data;
148
149
150
151
152 if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI))
153 return -ENOTTY;
154
155 return scsi_cmd_blk_ioctl(bdev, mode, cmd,
156 (void __user *)data);
157 }
158 #else
159 static inline int virtblk_add_req_scsi(struct virtqueue *vq,
160 struct virtblk_req *vbr, struct scatterlist *data_sg,
161 bool have_data)
162 {
163 return -EIO;
164 }
165 static inline void virtblk_scsi_request_done(struct request *req)
166 {
167 }
168 #define virtblk_ioctl NULL
169 #endif
170
171 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr,
172 struct scatterlist *data_sg, bool have_data)
173 {
174 struct scatterlist hdr, status, *sgs[3];
175 unsigned int num_out = 0, num_in = 0;
176
177 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));
178 sgs[num_out++] = &hdr;
179
180 if (have_data) {
181 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))
182 sgs[num_out++] = data_sg;
183 else
184 sgs[num_out + num_in++] = data_sg;
185 }
186
187 sg_init_one(&status, &vbr->status, sizeof(vbr->status));
188 sgs[num_out + num_in++] = &status;
189
190 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);
191 }
192
193 static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap)
194 {
195 unsigned short segments = blk_rq_nr_discard_segments(req);
196 unsigned short n = 0;
197 struct virtio_blk_discard_write_zeroes *range;
198 struct bio *bio;
199 u32 flags = 0;
200
201 if (unmap)
202 flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP;
203
204 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
205 if (!range)
206 return -ENOMEM;
207
208 __rq_for_each_bio(bio, req) {
209 u64 sector = bio->bi_iter.bi_sector;
210 u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT;
211
212 range[n].flags = cpu_to_le32(flags);
213 range[n].num_sectors = cpu_to_le32(num_sectors);
214 range[n].sector = cpu_to_le64(sector);
215 n++;
216 }
217
218 req->special_vec.bv_page = virt_to_page(range);
219 req->special_vec.bv_offset = offset_in_page(range);
220 req->special_vec.bv_len = sizeof(*range) * segments;
221 req->rq_flags |= RQF_SPECIAL_PAYLOAD;
222
223 return 0;
224 }
225
226 static inline void virtblk_request_done(struct request *req)
227 {
228 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
229
230 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
231 kfree(page_address(req->special_vec.bv_page) +
232 req->special_vec.bv_offset);
233 }
234
235 switch (req_op(req)) {
236 case REQ_OP_SCSI_IN:
237 case REQ_OP_SCSI_OUT:
238 virtblk_scsi_request_done(req);
239 break;
240 }
241
242 blk_mq_end_request(req, virtblk_result(vbr));
243 }
244
245 static void virtblk_done(struct virtqueue *vq)
246 {
247 struct virtio_blk *vblk = vq->vdev->priv;
248 bool req_done = false;
249 int qid = vq->index;
250 struct virtblk_req *vbr;
251 unsigned long flags;
252 unsigned int len;
253
254 spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
255 do {
256 virtqueue_disable_cb(vq);
257 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {
258 struct request *req = blk_mq_rq_from_pdu(vbr);
259
260 blk_mq_complete_request(req);
261 req_done = true;
262 }
263 if (unlikely(virtqueue_is_broken(vq)))
264 break;
265 } while (!virtqueue_enable_cb(vq));
266
267
268 if (req_done)
269 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
270 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
271 }
272
273 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx)
274 {
275 struct virtio_blk *vblk = hctx->queue->queuedata;
276 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];
277 bool kick;
278
279 spin_lock_irq(&vq->lock);
280 kick = virtqueue_kick_prepare(vq->vq);
281 spin_unlock_irq(&vq->lock);
282
283 if (kick)
284 virtqueue_notify(vq->vq);
285 }
286
287 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,
288 const struct blk_mq_queue_data *bd)
289 {
290 struct virtio_blk *vblk = hctx->queue->queuedata;
291 struct request *req = bd->rq;
292 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
293 unsigned long flags;
294 unsigned int num;
295 int qid = hctx->queue_num;
296 int err;
297 bool notify = false;
298 bool unmap = false;
299 u32 type;
300
301 BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);
302
303 switch (req_op(req)) {
304 case REQ_OP_READ:
305 case REQ_OP_WRITE:
306 type = 0;
307 break;
308 case REQ_OP_FLUSH:
309 type = VIRTIO_BLK_T_FLUSH;
310 break;
311 case REQ_OP_DISCARD:
312 type = VIRTIO_BLK_T_DISCARD;
313 break;
314 case REQ_OP_WRITE_ZEROES:
315 type = VIRTIO_BLK_T_WRITE_ZEROES;
316 unmap = !(req->cmd_flags & REQ_NOUNMAP);
317 break;
318 case REQ_OP_SCSI_IN:
319 case REQ_OP_SCSI_OUT:
320 type = VIRTIO_BLK_T_SCSI_CMD;
321 break;
322 case REQ_OP_DRV_IN:
323 type = VIRTIO_BLK_T_GET_ID;
324 break;
325 default:
326 WARN_ON_ONCE(1);
327 return BLK_STS_IOERR;
328 }
329
330 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type);
331 vbr->out_hdr.sector = type ?
332 0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req));
333 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req));
334
335 blk_mq_start_request(req);
336
337 if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) {
338 err = virtblk_setup_discard_write_zeroes(req, unmap);
339 if (err)
340 return BLK_STS_RESOURCE;
341 }
342
343 num = blk_rq_map_sg(hctx->queue, req, vbr->sg);
344 if (num) {
345 if (rq_data_dir(req) == WRITE)
346 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);
347 else
348 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);
349 }
350
351 spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
352 if (blk_rq_is_scsi(req))
353 err = virtblk_add_req_scsi(vblk->vqs[qid].vq, vbr, vbr->sg, num);
354 else
355 err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);
356 if (err) {
357 virtqueue_kick(vblk->vqs[qid].vq);
358
359
360
361 if (err == -ENOSPC)
362 blk_mq_stop_hw_queue(hctx);
363 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
364 switch (err) {
365 case -ENOSPC:
366 return BLK_STS_DEV_RESOURCE;
367 case -ENOMEM:
368 return BLK_STS_RESOURCE;
369 default:
370 return BLK_STS_IOERR;
371 }
372 }
373
374 if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))
375 notify = true;
376 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
377
378 if (notify)
379 virtqueue_notify(vblk->vqs[qid].vq);
380 return BLK_STS_OK;
381 }
382
383
384
385 static int virtblk_get_id(struct gendisk *disk, char *id_str)
386 {
387 struct virtio_blk *vblk = disk->private_data;
388 struct request_queue *q = vblk->disk->queue;
389 struct request *req;
390 int err;
391
392 req = blk_get_request(q, REQ_OP_DRV_IN, 0);
393 if (IS_ERR(req))
394 return PTR_ERR(req);
395
396 err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL);
397 if (err)
398 goto out;
399
400 blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);
401 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req)));
402 out:
403 blk_put_request(req);
404 return err;
405 }
406
407 static void virtblk_get(struct virtio_blk *vblk)
408 {
409 refcount_inc(&vblk->refs);
410 }
411
412 static void virtblk_put(struct virtio_blk *vblk)
413 {
414 if (refcount_dec_and_test(&vblk->refs)) {
415 ida_simple_remove(&vd_index_ida, vblk->index);
416 mutex_destroy(&vblk->vdev_mutex);
417 kfree(vblk);
418 }
419 }
420
421 static int virtblk_open(struct block_device *bd, fmode_t mode)
422 {
423 struct virtio_blk *vblk = bd->bd_disk->private_data;
424 int ret = 0;
425
426 mutex_lock(&vblk->vdev_mutex);
427
428 if (vblk->vdev)
429 virtblk_get(vblk);
430 else
431 ret = -ENXIO;
432
433 mutex_unlock(&vblk->vdev_mutex);
434 return ret;
435 }
436
437 static void virtblk_release(struct gendisk *disk, fmode_t mode)
438 {
439 struct virtio_blk *vblk = disk->private_data;
440
441 virtblk_put(vblk);
442 }
443
444
445 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
446 {
447 struct virtio_blk *vblk = bd->bd_disk->private_data;
448 int ret = 0;
449
450 mutex_lock(&vblk->vdev_mutex);
451
452 if (!vblk->vdev) {
453 ret = -ENXIO;
454 goto out;
455 }
456
457
458 if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
459 virtio_cread(vblk->vdev, struct virtio_blk_config,
460 geometry.cylinders, &geo->cylinders);
461 virtio_cread(vblk->vdev, struct virtio_blk_config,
462 geometry.heads, &geo->heads);
463 virtio_cread(vblk->vdev, struct virtio_blk_config,
464 geometry.sectors, &geo->sectors);
465 } else {
466
467 geo->heads = 1 << 6;
468 geo->sectors = 1 << 5;
469 geo->cylinders = get_capacity(bd->bd_disk) >> 11;
470 }
471 out:
472 mutex_unlock(&vblk->vdev_mutex);
473 return ret;
474 }
475
476 static const struct block_device_operations virtblk_fops = {
477 .ioctl = virtblk_ioctl,
478 .owner = THIS_MODULE,
479 .open = virtblk_open,
480 .release = virtblk_release,
481 .getgeo = virtblk_getgeo,
482 };
483
484 static int index_to_minor(int index)
485 {
486 return index << PART_BITS;
487 }
488
489 static int minor_to_index(int minor)
490 {
491 return minor >> PART_BITS;
492 }
493
494 static ssize_t serial_show(struct device *dev,
495 struct device_attribute *attr, char *buf)
496 {
497 struct gendisk *disk = dev_to_disk(dev);
498 int err;
499
500
501 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES);
502
503 buf[VIRTIO_BLK_ID_BYTES] = '\0';
504 err = virtblk_get_id(disk, buf);
505 if (!err)
506 return strlen(buf);
507
508 if (err == -EIO)
509 return 0;
510
511 return err;
512 }
513
514 static DEVICE_ATTR_RO(serial);
515
516
517 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)
518 {
519 struct virtio_device *vdev = vblk->vdev;
520 struct request_queue *q = vblk->disk->queue;
521 char cap_str_2[10], cap_str_10[10];
522 unsigned long long nblocks;
523 u64 capacity;
524
525
526 virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity);
527
528
529 if ((sector_t)capacity != capacity) {
530 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n",
531 (unsigned long long)capacity);
532 capacity = (sector_t)-1;
533 }
534
535 nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9);
536
537 string_get_size(nblocks, queue_logical_block_size(q),
538 STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
539 string_get_size(nblocks, queue_logical_block_size(q),
540 STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
541
542 dev_notice(&vdev->dev,
543 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n",
544 vblk->disk->disk_name,
545 resize ? "new size: " : "",
546 nblocks,
547 queue_logical_block_size(q),
548 cap_str_10,
549 cap_str_2);
550
551 set_capacity(vblk->disk, capacity);
552 }
553
554 static void virtblk_config_changed_work(struct work_struct *work)
555 {
556 struct virtio_blk *vblk =
557 container_of(work, struct virtio_blk, config_work);
558 char *envp[] = { "RESIZE=1", NULL };
559
560 virtblk_update_capacity(vblk, true);
561 revalidate_disk(vblk->disk);
562 kobject_uevent_env(&disk_to_dev(vblk->disk)->kobj, KOBJ_CHANGE, envp);
563 }
564
565 static void virtblk_config_changed(struct virtio_device *vdev)
566 {
567 struct virtio_blk *vblk = vdev->priv;
568
569 queue_work(virtblk_wq, &vblk->config_work);
570 }
571
572 static int init_vq(struct virtio_blk *vblk)
573 {
574 int err;
575 int i;
576 vq_callback_t **callbacks;
577 const char **names;
578 struct virtqueue **vqs;
579 unsigned short num_vqs;
580 struct virtio_device *vdev = vblk->vdev;
581 struct irq_affinity desc = { 0, };
582
583 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ,
584 struct virtio_blk_config, num_queues,
585 &num_vqs);
586 if (err)
587 num_vqs = 1;
588
589 num_vqs = min_t(unsigned int, nr_cpu_ids, num_vqs);
590
591 vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);
592 if (!vblk->vqs)
593 return -ENOMEM;
594
595 names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL);
596 callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL);
597 vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL);
598 if (!names || !callbacks || !vqs) {
599 err = -ENOMEM;
600 goto out;
601 }
602
603 for (i = 0; i < num_vqs; i++) {
604 callbacks[i] = virtblk_done;
605 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);
606 names[i] = vblk->vqs[i].name;
607 }
608
609
610 err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);
611 if (err)
612 goto out;
613
614 for (i = 0; i < num_vqs; i++) {
615 spin_lock_init(&vblk->vqs[i].lock);
616 vblk->vqs[i].vq = vqs[i];
617 }
618 vblk->num_vqs = num_vqs;
619
620 out:
621 kfree(vqs);
622 kfree(callbacks);
623 kfree(names);
624 if (err)
625 kfree(vblk->vqs);
626 return err;
627 }
628
629
630
631
632
633 static int virtblk_name_format(char *prefix, int index, char *buf, int buflen)
634 {
635 const int base = 'z' - 'a' + 1;
636 char *begin = buf + strlen(prefix);
637 char *end = buf + buflen;
638 char *p;
639 int unit;
640
641 p = end - 1;
642 *p = '\0';
643 unit = base;
644 do {
645 if (p == begin)
646 return -EINVAL;
647 *--p = 'a' + (index % unit);
648 index = (index / unit) - 1;
649 } while (index >= 0);
650
651 memmove(begin, p, end - p);
652 memcpy(buf, prefix, strlen(prefix));
653
654 return 0;
655 }
656
657 static int virtblk_get_cache_mode(struct virtio_device *vdev)
658 {
659 u8 writeback;
660 int err;
661
662 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE,
663 struct virtio_blk_config, wce,
664 &writeback);
665
666
667
668
669
670 if (err)
671 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH);
672
673 return writeback;
674 }
675
676 static void virtblk_update_cache_mode(struct virtio_device *vdev)
677 {
678 u8 writeback = virtblk_get_cache_mode(vdev);
679 struct virtio_blk *vblk = vdev->priv;
680
681 blk_queue_write_cache(vblk->disk->queue, writeback, false);
682 revalidate_disk(vblk->disk);
683 }
684
685 static const char *const virtblk_cache_types[] = {
686 "write through", "write back"
687 };
688
689 static ssize_t
690 cache_type_store(struct device *dev, struct device_attribute *attr,
691 const char *buf, size_t count)
692 {
693 struct gendisk *disk = dev_to_disk(dev);
694 struct virtio_blk *vblk = disk->private_data;
695 struct virtio_device *vdev = vblk->vdev;
696 int i;
697
698 BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));
699 i = sysfs_match_string(virtblk_cache_types, buf);
700 if (i < 0)
701 return i;
702
703 virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i);
704 virtblk_update_cache_mode(vdev);
705 return count;
706 }
707
708 static ssize_t
709 cache_type_show(struct device *dev, struct device_attribute *attr, char *buf)
710 {
711 struct gendisk *disk = dev_to_disk(dev);
712 struct virtio_blk *vblk = disk->private_data;
713 u8 writeback = virtblk_get_cache_mode(vblk->vdev);
714
715 BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types));
716 return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]);
717 }
718
719 static DEVICE_ATTR_RW(cache_type);
720
721 static struct attribute *virtblk_attrs[] = {
722 &dev_attr_serial.attr,
723 &dev_attr_cache_type.attr,
724 NULL,
725 };
726
727 static umode_t virtblk_attrs_are_visible(struct kobject *kobj,
728 struct attribute *a, int n)
729 {
730 struct device *dev = container_of(kobj, struct device, kobj);
731 struct gendisk *disk = dev_to_disk(dev);
732 struct virtio_blk *vblk = disk->private_data;
733 struct virtio_device *vdev = vblk->vdev;
734
735 if (a == &dev_attr_cache_type.attr &&
736 !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE))
737 return S_IRUGO;
738
739 return a->mode;
740 }
741
742 static const struct attribute_group virtblk_attr_group = {
743 .attrs = virtblk_attrs,
744 .is_visible = virtblk_attrs_are_visible,
745 };
746
747 static const struct attribute_group *virtblk_attr_groups[] = {
748 &virtblk_attr_group,
749 NULL,
750 };
751
752 static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq,
753 unsigned int hctx_idx, unsigned int numa_node)
754 {
755 struct virtio_blk *vblk = set->driver_data;
756 struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq);
757
758 #ifdef CONFIG_VIRTIO_BLK_SCSI
759 vbr->sreq.sense = vbr->sense;
760 #endif
761 sg_init_table(vbr->sg, vblk->sg_elems);
762 return 0;
763 }
764
765 static int virtblk_map_queues(struct blk_mq_tag_set *set)
766 {
767 struct virtio_blk *vblk = set->driver_data;
768
769 return blk_mq_virtio_map_queues(&set->map[HCTX_TYPE_DEFAULT],
770 vblk->vdev, 0);
771 }
772
773 #ifdef CONFIG_VIRTIO_BLK_SCSI
774 static void virtblk_initialize_rq(struct request *req)
775 {
776 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
777
778 scsi_req_init(&vbr->sreq);
779 }
780 #endif
781
782 static const struct blk_mq_ops virtio_mq_ops = {
783 .queue_rq = virtio_queue_rq,
784 .commit_rqs = virtio_commit_rqs,
785 .complete = virtblk_request_done,
786 .init_request = virtblk_init_request,
787 #ifdef CONFIG_VIRTIO_BLK_SCSI
788 .initialize_rq_fn = virtblk_initialize_rq,
789 #endif
790 .map_queues = virtblk_map_queues,
791 };
792
793 static unsigned int virtblk_queue_depth;
794 module_param_named(queue_depth, virtblk_queue_depth, uint, 0444);
795
796 static int virtblk_probe(struct virtio_device *vdev)
797 {
798 struct virtio_blk *vblk;
799 struct request_queue *q;
800 int err, index;
801
802 u32 v, blk_size, max_size, sg_elems, opt_io_size;
803 u16 min_io_size;
804 u8 physical_block_exp, alignment_offset;
805
806 if (!vdev->config->get) {
807 dev_err(&vdev->dev, "%s failure: config access disabled\n",
808 __func__);
809 return -EINVAL;
810 }
811
812 err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS),
813 GFP_KERNEL);
814 if (err < 0)
815 goto out;
816 index = err;
817
818
819 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX,
820 struct virtio_blk_config, seg_max,
821 &sg_elems);
822
823
824 if (err || !sg_elems)
825 sg_elems = 1;
826
827
828 sg_elems += 2;
829 vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);
830 if (!vblk) {
831 err = -ENOMEM;
832 goto out_free_index;
833 }
834
835
836 refcount_set(&vblk->refs, 1);
837 mutex_init(&vblk->vdev_mutex);
838
839 vblk->vdev = vdev;
840 vblk->sg_elems = sg_elems;
841
842 INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
843
844 err = init_vq(vblk);
845 if (err)
846 goto out_free_vblk;
847
848
849 vblk->disk = alloc_disk(1 << PART_BITS);
850 if (!vblk->disk) {
851 err = -ENOMEM;
852 goto out_free_vq;
853 }
854
855
856 if (!virtblk_queue_depth) {
857 virtblk_queue_depth = vblk->vqs[0].vq->num_free;
858
859 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
860 virtblk_queue_depth /= 2;
861 }
862
863 memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));
864 vblk->tag_set.ops = &virtio_mq_ops;
865 vblk->tag_set.queue_depth = virtblk_queue_depth;
866 vblk->tag_set.numa_node = NUMA_NO_NODE;
867 vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
868 vblk->tag_set.cmd_size =
869 sizeof(struct virtblk_req) +
870 sizeof(struct scatterlist) * sg_elems;
871 vblk->tag_set.driver_data = vblk;
872 vblk->tag_set.nr_hw_queues = vblk->num_vqs;
873
874 err = blk_mq_alloc_tag_set(&vblk->tag_set);
875 if (err)
876 goto out_put_disk;
877
878 q = blk_mq_init_queue(&vblk->tag_set);
879 if (IS_ERR(q)) {
880 err = -ENOMEM;
881 goto out_free_tags;
882 }
883 vblk->disk->queue = q;
884
885 q->queuedata = vblk;
886
887 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);
888
889 vblk->disk->major = major;
890 vblk->disk->first_minor = index_to_minor(index);
891 vblk->disk->private_data = vblk;
892 vblk->disk->fops = &virtblk_fops;
893 vblk->disk->flags |= GENHD_FL_EXT_DEVT;
894 vblk->index = index;
895
896
897 virtblk_update_cache_mode(vdev);
898
899
900 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO))
901 set_disk_ro(vblk->disk, 1);
902
903
904 blk_queue_max_segments(q, vblk->sg_elems-2);
905
906
907 blk_queue_max_hw_sectors(q, -1U);
908
909 max_size = virtio_max_dma_size(vdev);
910
911
912
913 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX,
914 struct virtio_blk_config, size_max, &v);
915 if (!err)
916 max_size = min(max_size, v);
917
918 blk_queue_max_segment_size(q, max_size);
919
920
921 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE,
922 struct virtio_blk_config, blk_size,
923 &blk_size);
924 if (!err)
925 blk_queue_logical_block_size(q, blk_size);
926 else
927 blk_size = queue_logical_block_size(q);
928
929
930 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
931 struct virtio_blk_config, physical_block_exp,
932 &physical_block_exp);
933 if (!err && physical_block_exp)
934 blk_queue_physical_block_size(q,
935 blk_size * (1 << physical_block_exp));
936
937 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
938 struct virtio_blk_config, alignment_offset,
939 &alignment_offset);
940 if (!err && alignment_offset)
941 blk_queue_alignment_offset(q, blk_size * alignment_offset);
942
943 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
944 struct virtio_blk_config, min_io_size,
945 &min_io_size);
946 if (!err && min_io_size)
947 blk_queue_io_min(q, blk_size * min_io_size);
948
949 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
950 struct virtio_blk_config, opt_io_size,
951 &opt_io_size);
952 if (!err && opt_io_size)
953 blk_queue_io_opt(q, blk_size * opt_io_size);
954
955 if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) {
956 q->limits.discard_granularity = blk_size;
957
958 virtio_cread(vdev, struct virtio_blk_config,
959 discard_sector_alignment, &v);
960 q->limits.discard_alignment = v ? v << SECTOR_SHIFT : 0;
961
962 virtio_cread(vdev, struct virtio_blk_config,
963 max_discard_sectors, &v);
964 blk_queue_max_discard_sectors(q, v ? v : UINT_MAX);
965
966 virtio_cread(vdev, struct virtio_blk_config, max_discard_seg,
967 &v);
968 blk_queue_max_discard_segments(q,
969 min_not_zero(v,
970 MAX_DISCARD_SEGMENTS));
971
972 blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
973 }
974
975 if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) {
976 virtio_cread(vdev, struct virtio_blk_config,
977 max_write_zeroes_sectors, &v);
978 blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX);
979 }
980
981 virtblk_update_capacity(vblk, false);
982 virtio_device_ready(vdev);
983
984 device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);
985 return 0;
986
987 out_free_tags:
988 blk_mq_free_tag_set(&vblk->tag_set);
989 out_put_disk:
990 put_disk(vblk->disk);
991 out_free_vq:
992 vdev->config->del_vqs(vdev);
993 out_free_vblk:
994 kfree(vblk);
995 out_free_index:
996 ida_simple_remove(&vd_index_ida, index);
997 out:
998 return err;
999 }
1000
1001 static void virtblk_remove(struct virtio_device *vdev)
1002 {
1003 struct virtio_blk *vblk = vdev->priv;
1004
1005
1006 flush_work(&vblk->config_work);
1007
1008 del_gendisk(vblk->disk);
1009 blk_cleanup_queue(vblk->disk->queue);
1010
1011 blk_mq_free_tag_set(&vblk->tag_set);
1012
1013 mutex_lock(&vblk->vdev_mutex);
1014
1015
1016 vdev->config->reset(vdev);
1017
1018
1019 vblk->vdev = NULL;
1020
1021 put_disk(vblk->disk);
1022 vdev->config->del_vqs(vdev);
1023 kfree(vblk->vqs);
1024
1025 mutex_unlock(&vblk->vdev_mutex);
1026
1027 virtblk_put(vblk);
1028 }
1029
1030 #ifdef CONFIG_PM_SLEEP
1031 static int virtblk_freeze(struct virtio_device *vdev)
1032 {
1033 struct virtio_blk *vblk = vdev->priv;
1034
1035
1036 vdev->config->reset(vdev);
1037
1038
1039 flush_work(&vblk->config_work);
1040
1041 blk_mq_quiesce_queue(vblk->disk->queue);
1042
1043 vdev->config->del_vqs(vdev);
1044 return 0;
1045 }
1046
1047 static int virtblk_restore(struct virtio_device *vdev)
1048 {
1049 struct virtio_blk *vblk = vdev->priv;
1050 int ret;
1051
1052 ret = init_vq(vdev->priv);
1053 if (ret)
1054 return ret;
1055
1056 virtio_device_ready(vdev);
1057
1058 blk_mq_unquiesce_queue(vblk->disk->queue);
1059 return 0;
1060 }
1061 #endif
1062
1063 static const struct virtio_device_id id_table[] = {
1064 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1065 { 0 },
1066 };
1067
1068 static unsigned int features_legacy[] = {
1069 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
1070 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
1071 #ifdef CONFIG_VIRTIO_BLK_SCSI
1072 VIRTIO_BLK_F_SCSI,
1073 #endif
1074 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
1075 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
1076 }
1077 ;
1078 static unsigned int features[] = {
1079 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
1080 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
1081 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
1082 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
1083 };
1084
1085 static struct virtio_driver virtio_blk = {
1086 .feature_table = features,
1087 .feature_table_size = ARRAY_SIZE(features),
1088 .feature_table_legacy = features_legacy,
1089 .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
1090 .driver.name = KBUILD_MODNAME,
1091 .driver.owner = THIS_MODULE,
1092 .id_table = id_table,
1093 .probe = virtblk_probe,
1094 .remove = virtblk_remove,
1095 .config_changed = virtblk_config_changed,
1096 #ifdef CONFIG_PM_SLEEP
1097 .freeze = virtblk_freeze,
1098 .restore = virtblk_restore,
1099 #endif
1100 };
1101
1102 static int __init init(void)
1103 {
1104 int error;
1105
1106 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0);
1107 if (!virtblk_wq)
1108 return -ENOMEM;
1109
1110 major = register_blkdev(0, "virtblk");
1111 if (major < 0) {
1112 error = major;
1113 goto out_destroy_workqueue;
1114 }
1115
1116 error = register_virtio_driver(&virtio_blk);
1117 if (error)
1118 goto out_unregister_blkdev;
1119 return 0;
1120
1121 out_unregister_blkdev:
1122 unregister_blkdev(major, "virtblk");
1123 out_destroy_workqueue:
1124 destroy_workqueue(virtblk_wq);
1125 return error;
1126 }
1127
1128 static void __exit fini(void)
1129 {
1130 unregister_virtio_driver(&virtio_blk);
1131 unregister_blkdev(major, "virtblk");
1132 destroy_workqueue(virtblk_wq);
1133 }
1134 module_init(init);
1135 module_exit(fini);
1136
1137 MODULE_DEVICE_TABLE(virtio, id_table);
1138 MODULE_DESCRIPTION("Virtio block driver");
1139 MODULE_LICENSE("GPL");