Lines Matching refs:vq
36 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) argument
37 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) argument
178 struct vhost_virtqueue *vq) in vhost_vq_reset() argument
180 vq->num = 1; in vhost_vq_reset()
181 vq->desc = NULL; in vhost_vq_reset()
182 vq->avail = NULL; in vhost_vq_reset()
183 vq->used = NULL; in vhost_vq_reset()
184 vq->last_avail_idx = 0; in vhost_vq_reset()
185 vq->avail_idx = 0; in vhost_vq_reset()
186 vq->last_used_idx = 0; in vhost_vq_reset()
187 vq->signalled_used = 0; in vhost_vq_reset()
188 vq->signalled_used_valid = false; in vhost_vq_reset()
189 vq->used_flags = 0; in vhost_vq_reset()
190 vq->log_used = false; in vhost_vq_reset()
191 vq->log_addr = -1ull; in vhost_vq_reset()
192 vq->private_data = NULL; in vhost_vq_reset()
193 vq->acked_features = 0; in vhost_vq_reset()
194 vq->log_base = NULL; in vhost_vq_reset()
195 vq->error_ctx = NULL; in vhost_vq_reset()
196 vq->error = NULL; in vhost_vq_reset()
197 vq->kick = NULL; in vhost_vq_reset()
198 vq->call_ctx = NULL; in vhost_vq_reset()
199 vq->call = NULL; in vhost_vq_reset()
200 vq->log_ctx = NULL; in vhost_vq_reset()
201 vq->memory = NULL; in vhost_vq_reset()
253 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) in vhost_vq_free_iovecs() argument
255 kfree(vq->indirect); in vhost_vq_free_iovecs()
256 vq->indirect = NULL; in vhost_vq_free_iovecs()
257 kfree(vq->log); in vhost_vq_free_iovecs()
258 vq->log = NULL; in vhost_vq_free_iovecs()
259 kfree(vq->heads); in vhost_vq_free_iovecs()
260 vq->heads = NULL; in vhost_vq_free_iovecs()
266 struct vhost_virtqueue *vq; in vhost_dev_alloc_iovecs() local
270 vq = dev->vqs[i]; in vhost_dev_alloc_iovecs()
271 vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV, in vhost_dev_alloc_iovecs()
273 vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL); in vhost_dev_alloc_iovecs()
274 vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL); in vhost_dev_alloc_iovecs()
275 if (!vq->indirect || !vq->log || !vq->heads) in vhost_dev_alloc_iovecs()
297 struct vhost_virtqueue *vq; in vhost_dev_init() local
312 vq = dev->vqs[i]; in vhost_dev_init()
313 vq->log = NULL; in vhost_dev_init()
314 vq->indirect = NULL; in vhost_dev_init()
315 vq->heads = NULL; in vhost_dev_init()
316 vq->dev = dev; in vhost_dev_init()
317 mutex_init(&vq->mutex); in vhost_dev_init()
318 vhost_vq_reset(dev, vq); in vhost_dev_init()
319 if (vq->handle_kick) in vhost_dev_init()
320 vhost_poll_init(&vq->poll, vq->handle_kick, in vhost_dev_init()
549 static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, in vq_access_ok() argument
554 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vq_access_ok()
572 static int vq_log_access_ok(struct vhost_virtqueue *vq, in vq_log_access_ok() argument
575 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vq_log_access_ok()
577 return vq_memory_access_ok(log_base, vq->memory, in vq_log_access_ok()
578 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && in vq_log_access_ok()
579 (!vq->log_used || log_access_ok(log_base, vq->log_addr, in vq_log_access_ok()
580 sizeof *vq->used + in vq_log_access_ok()
581 vq->num * sizeof *vq->used->ring + s)); in vq_log_access_ok()
586 int vhost_vq_access_ok(struct vhost_virtqueue *vq) in vhost_vq_access_ok() argument
588 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) && in vhost_vq_access_ok()
589 vq_log_access_ok(vq, vq->log_base); in vhost_vq_access_ok()
639 struct vhost_virtqueue *vq; in vhost_vring_ioctl() local
652 vq = d->vqs[idx]; in vhost_vring_ioctl()
654 mutex_lock(&vq->mutex); in vhost_vring_ioctl()
660 if (vq->private_data) { in vhost_vring_ioctl()
672 vq->num = s.num; in vhost_vring_ioctl()
677 if (vq->private_data) { in vhost_vring_ioctl()
689 vq->last_avail_idx = s.num; in vhost_vring_ioctl()
691 vq->avail_idx = vq->last_avail_idx; in vhost_vring_ioctl()
695 s.num = vq->last_avail_idx; in vhost_vring_ioctl()
718 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); in vhost_vring_ioctl()
719 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); in vhost_vring_ioctl()
730 if (vq->private_data) { in vhost_vring_ioctl()
731 if (!vq_access_ok(vq, vq->num, in vhost_vring_ioctl()
741 !log_access_ok(vq->log_base, a.log_guest_addr, in vhost_vring_ioctl()
742 sizeof *vq->used + in vhost_vring_ioctl()
743 vq->num * sizeof *vq->used->ring)) { in vhost_vring_ioctl()
749 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); in vhost_vring_ioctl()
750 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; in vhost_vring_ioctl()
751 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; in vhost_vring_ioctl()
752 vq->log_addr = a.log_guest_addr; in vhost_vring_ioctl()
753 vq->used = (void __user *)(unsigned long)a.used_user_addr; in vhost_vring_ioctl()
765 if (eventfp != vq->kick) { in vhost_vring_ioctl()
766 pollstop = (filep = vq->kick) != NULL; in vhost_vring_ioctl()
767 pollstart = (vq->kick = eventfp) != NULL; in vhost_vring_ioctl()
781 if (eventfp != vq->call) { in vhost_vring_ioctl()
782 filep = vq->call; in vhost_vring_ioctl()
783 ctx = vq->call_ctx; in vhost_vring_ioctl()
784 vq->call = eventfp; in vhost_vring_ioctl()
785 vq->call_ctx = eventfp ? in vhost_vring_ioctl()
800 if (eventfp != vq->error) { in vhost_vring_ioctl()
801 filep = vq->error; in vhost_vring_ioctl()
802 vq->error = eventfp; in vhost_vring_ioctl()
803 ctx = vq->error_ctx; in vhost_vring_ioctl()
804 vq->error_ctx = eventfp ? in vhost_vring_ioctl()
813 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
814 vhost_poll_stop(&vq->poll); in vhost_vring_ioctl()
821 if (pollstart && vq->handle_kick) in vhost_vring_ioctl()
822 r = vhost_poll_start(&vq->poll, vq->kick); in vhost_vring_ioctl()
824 mutex_unlock(&vq->mutex); in vhost_vring_ioctl()
826 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
827 vhost_poll_flush(&vq->poll); in vhost_vring_ioctl()
866 struct vhost_virtqueue *vq; in vhost_dev_ioctl() local
868 vq = d->vqs[i]; in vhost_dev_ioctl()
869 mutex_lock(&vq->mutex); in vhost_dev_ioctl()
871 if (vq->private_data && !vq_log_access_ok(vq, base)) in vhost_dev_ioctl()
874 vq->log_base = base; in vhost_dev_ioctl()
875 mutex_unlock(&vq->mutex); in vhost_dev_ioctl()
981 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, in vhost_log_write() argument
990 r = log_write(vq->log_base, log[i].addr, l); in vhost_log_write()
995 if (vq->log_ctx) in vhost_log_write()
996 eventfd_signal(vq->log_ctx, 1); in vhost_log_write()
1006 static int vhost_update_used_flags(struct vhost_virtqueue *vq) in vhost_update_used_flags() argument
1009 if (__put_user(cpu_to_vhost16(vq, vq->used_flags), &vq->used->flags) < 0) in vhost_update_used_flags()
1011 if (unlikely(vq->log_used)) { in vhost_update_used_flags()
1015 used = &vq->used->flags; in vhost_update_used_flags()
1016 log_write(vq->log_base, vq->log_addr + in vhost_update_used_flags()
1017 (used - (void __user *)vq->used), in vhost_update_used_flags()
1018 sizeof vq->used->flags); in vhost_update_used_flags()
1019 if (vq->log_ctx) in vhost_update_used_flags()
1020 eventfd_signal(vq->log_ctx, 1); in vhost_update_used_flags()
1025 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) in vhost_update_avail_event() argument
1027 if (__put_user(cpu_to_vhost16(vq, vq->avail_idx), vhost_avail_event(vq))) in vhost_update_avail_event()
1029 if (unlikely(vq->log_used)) { in vhost_update_avail_event()
1034 used = vhost_avail_event(vq); in vhost_update_avail_event()
1035 log_write(vq->log_base, vq->log_addr + in vhost_update_avail_event()
1036 (used - (void __user *)vq->used), in vhost_update_avail_event()
1037 sizeof *vhost_avail_event(vq)); in vhost_update_avail_event()
1038 if (vq->log_ctx) in vhost_update_avail_event()
1039 eventfd_signal(vq->log_ctx, 1); in vhost_update_avail_event()
1044 int vhost_init_used(struct vhost_virtqueue *vq) in vhost_init_used() argument
1048 if (!vq->private_data) in vhost_init_used()
1051 r = vhost_update_used_flags(vq); in vhost_init_used()
1054 vq->signalled_used_valid = false; in vhost_init_used()
1055 if (!access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) in vhost_init_used()
1057 r = __get_user(last_used_idx, &vq->used->idx); in vhost_init_used()
1060 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); in vhost_init_used()
1065 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, in translate_desc() argument
1074 mem = vq->memory; in translate_desc()
1102 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc) in next_desc() argument
1107 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT))) in next_desc()
1111 next = vhost16_to_cpu(vq, desc->next); in next_desc()
1120 static int get_indirect(struct vhost_virtqueue *vq, in get_indirect() argument
1128 u32 len = vhost32_to_cpu(vq, indirect->len); in get_indirect()
1134 vq_err(vq, "Invalid length in indirect descriptor: " in get_indirect()
1141 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, in get_indirect()
1144 vq_err(vq, "Translation failure %d in indirect.\n", ret); in get_indirect()
1147 iov_iter_init(&from, READ, vq->indirect, ret, len); in get_indirect()
1157 vq_err(vq, "Indirect buffer length too big: %d\n", in get_indirect()
1165 vq_err(vq, "Loop detected: last one at %u " in get_indirect()
1172 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", in get_indirect()
1173 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); in get_indirect()
1176 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) { in get_indirect()
1177 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", in get_indirect()
1178 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); in get_indirect()
1182 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), in get_indirect()
1183 vhost32_to_cpu(vq, desc.len), iov + iov_count, in get_indirect()
1186 vq_err(vq, "Translation failure %d indirect idx %d\n", in get_indirect()
1191 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) { in get_indirect()
1194 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); in get_indirect()
1195 log[*log_num].len = vhost32_to_cpu(vq, desc.len); in get_indirect()
1202 vq_err(vq, "Indirect descriptor " in get_indirect()
1208 } while ((i = next_desc(vq, &desc)) != -1); in get_indirect()
1220 int vhost_get_vq_desc(struct vhost_virtqueue *vq, in vhost_get_vq_desc() argument
1233 last_avail_idx = vq->last_avail_idx; in vhost_get_vq_desc()
1234 if (unlikely(__get_user(avail_idx, &vq->avail->idx))) { in vhost_get_vq_desc()
1235 vq_err(vq, "Failed to access avail idx at %p\n", in vhost_get_vq_desc()
1236 &vq->avail->idx); in vhost_get_vq_desc()
1239 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); in vhost_get_vq_desc()
1241 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { in vhost_get_vq_desc()
1242 vq_err(vq, "Guest moved used index from %u to %u", in vhost_get_vq_desc()
1243 last_avail_idx, vq->avail_idx); in vhost_get_vq_desc()
1248 if (vq->avail_idx == last_avail_idx) in vhost_get_vq_desc()
1249 return vq->num; in vhost_get_vq_desc()
1257 &vq->avail->ring[last_avail_idx % vq->num]))) { in vhost_get_vq_desc()
1258 vq_err(vq, "Failed to read head: idx %d address %p\n", in vhost_get_vq_desc()
1260 &vq->avail->ring[last_avail_idx % vq->num]); in vhost_get_vq_desc()
1264 head = vhost16_to_cpu(vq, ring_head); in vhost_get_vq_desc()
1267 if (unlikely(head >= vq->num)) { in vhost_get_vq_desc()
1268 vq_err(vq, "Guest says index %u > %u is available", in vhost_get_vq_desc()
1269 head, vq->num); in vhost_get_vq_desc()
1281 if (unlikely(i >= vq->num)) { in vhost_get_vq_desc()
1282 vq_err(vq, "Desc index is %u > %u, head = %u", in vhost_get_vq_desc()
1283 i, vq->num, head); in vhost_get_vq_desc()
1286 if (unlikely(++found > vq->num)) { in vhost_get_vq_desc()
1287 vq_err(vq, "Loop detected: last one at %u " in vhost_get_vq_desc()
1289 i, vq->num, head); in vhost_get_vq_desc()
1292 ret = __copy_from_user(&desc, vq->desc + i, sizeof desc); in vhost_get_vq_desc()
1294 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", in vhost_get_vq_desc()
1295 i, vq->desc + i); in vhost_get_vq_desc()
1298 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) { in vhost_get_vq_desc()
1299 ret = get_indirect(vq, iov, iov_size, in vhost_get_vq_desc()
1303 vq_err(vq, "Failure detected " in vhost_get_vq_desc()
1310 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), in vhost_get_vq_desc()
1311 vhost32_to_cpu(vq, desc.len), iov + iov_count, in vhost_get_vq_desc()
1314 vq_err(vq, "Translation failure %d descriptor idx %d\n", in vhost_get_vq_desc()
1318 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) { in vhost_get_vq_desc()
1323 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); in vhost_get_vq_desc()
1324 log[*log_num].len = vhost32_to_cpu(vq, desc.len); in vhost_get_vq_desc()
1331 vq_err(vq, "Descriptor has out after in: " in vhost_get_vq_desc()
1337 } while ((i = next_desc(vq, &desc)) != -1); in vhost_get_vq_desc()
1340 vq->last_avail_idx++; in vhost_get_vq_desc()
1344 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); in vhost_get_vq_desc()
1350 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) in vhost_discard_vq_desc() argument
1352 vq->last_avail_idx -= n; in vhost_discard_vq_desc()
1358 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) in vhost_add_used() argument
1361 cpu_to_vhost32(vq, head), in vhost_add_used()
1362 cpu_to_vhost32(vq, len) in vhost_add_used()
1365 return vhost_add_used_n(vq, &heads, 1); in vhost_add_used()
1369 static int __vhost_add_used_n(struct vhost_virtqueue *vq, in __vhost_add_used_n() argument
1377 start = vq->last_used_idx % vq->num; in __vhost_add_used_n()
1378 used = vq->used->ring + start; in __vhost_add_used_n()
1381 vq_err(vq, "Failed to write used id"); in __vhost_add_used_n()
1385 vq_err(vq, "Failed to write used len"); in __vhost_add_used_n()
1389 vq_err(vq, "Failed to write used"); in __vhost_add_used_n()
1392 if (unlikely(vq->log_used)) { in __vhost_add_used_n()
1396 log_write(vq->log_base, in __vhost_add_used_n()
1397 vq->log_addr + in __vhost_add_used_n()
1398 ((void __user *)used - (void __user *)vq->used), in __vhost_add_used_n()
1401 old = vq->last_used_idx; in __vhost_add_used_n()
1402 new = (vq->last_used_idx += count); in __vhost_add_used_n()
1407 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) in __vhost_add_used_n()
1408 vq->signalled_used_valid = false; in __vhost_add_used_n()
1414 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, in vhost_add_used_n() argument
1419 start = vq->last_used_idx % vq->num; in vhost_add_used_n()
1420 n = vq->num - start; in vhost_add_used_n()
1422 r = __vhost_add_used_n(vq, heads, n); in vhost_add_used_n()
1428 r = __vhost_add_used_n(vq, heads, count); in vhost_add_used_n()
1432 if (__put_user(cpu_to_vhost16(vq, vq->last_used_idx), &vq->used->idx)) { in vhost_add_used_n()
1433 vq_err(vq, "Failed to increment used idx"); in vhost_add_used_n()
1436 if (unlikely(vq->log_used)) { in vhost_add_used_n()
1438 log_write(vq->log_base, in vhost_add_used_n()
1439 vq->log_addr + offsetof(struct vring_used, idx), in vhost_add_used_n()
1440 sizeof vq->used->idx); in vhost_add_used_n()
1441 if (vq->log_ctx) in vhost_add_used_n()
1442 eventfd_signal(vq->log_ctx, 1); in vhost_add_used_n()
1448 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_notify() argument
1458 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && in vhost_notify()
1459 unlikely(vq->avail_idx == vq->last_avail_idx)) in vhost_notify()
1462 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_notify()
1464 if (__get_user(flags, &vq->avail->flags)) { in vhost_notify()
1465 vq_err(vq, "Failed to get flags"); in vhost_notify()
1468 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT)); in vhost_notify()
1470 old = vq->signalled_used; in vhost_notify()
1471 v = vq->signalled_used_valid; in vhost_notify()
1472 new = vq->signalled_used = vq->last_used_idx; in vhost_notify()
1473 vq->signalled_used_valid = true; in vhost_notify()
1478 if (__get_user(event, vhost_used_event(vq))) { in vhost_notify()
1479 vq_err(vq, "Failed to get used event idx"); in vhost_notify()
1482 return vring_need_event(vhost16_to_cpu(vq, event), new, old); in vhost_notify()
1486 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_signal() argument
1489 if (vq->call_ctx && vhost_notify(dev, vq)) in vhost_signal()
1490 eventfd_signal(vq->call_ctx, 1); in vhost_signal()
1496 struct vhost_virtqueue *vq, in vhost_add_used_and_signal() argument
1499 vhost_add_used(vq, head, len); in vhost_add_used_and_signal()
1500 vhost_signal(dev, vq); in vhost_add_used_and_signal()
1506 struct vhost_virtqueue *vq, in vhost_add_used_and_signal_n() argument
1509 vhost_add_used_n(vq, heads, count); in vhost_add_used_and_signal_n()
1510 vhost_signal(dev, vq); in vhost_add_used_and_signal_n()
1515 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_enable_notify() argument
1520 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) in vhost_enable_notify()
1522 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; in vhost_enable_notify()
1523 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_enable_notify()
1524 r = vhost_update_used_flags(vq); in vhost_enable_notify()
1526 vq_err(vq, "Failed to enable notification at %p: %d\n", in vhost_enable_notify()
1527 &vq->used->flags, r); in vhost_enable_notify()
1531 r = vhost_update_avail_event(vq, vq->avail_idx); in vhost_enable_notify()
1533 vq_err(vq, "Failed to update avail event index at %p: %d\n", in vhost_enable_notify()
1534 vhost_avail_event(vq), r); in vhost_enable_notify()
1541 r = __get_user(avail_idx, &vq->avail->idx); in vhost_enable_notify()
1543 vq_err(vq, "Failed to check avail idx at %p: %d\n", in vhost_enable_notify()
1544 &vq->avail->idx, r); in vhost_enable_notify()
1548 return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx; in vhost_enable_notify()
1553 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_disable_notify() argument
1557 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) in vhost_disable_notify()
1559 vq->used_flags |= VRING_USED_F_NO_NOTIFY; in vhost_disable_notify()
1560 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_disable_notify()
1561 r = vhost_update_used_flags(vq); in vhost_disable_notify()
1563 vq_err(vq, "Failed to enable notification at %p: %d\n", in vhost_disable_notify()
1564 &vq->used->flags, r); in vhost_disable_notify()