187 	struct vhost_virtqueue vq;  member
244 	struct vhost_virtqueue *vq;  in vhost_scsi_init_inflight()  local
248 		vq = &vs->vqs[i].vq;  in vhost_scsi_init_inflight()
250 		mutex_lock(&vq->mutex);  in vhost_scsi_init_inflight()
263 		mutex_unlock(&vq->mutex);  in vhost_scsi_init_inflight()
268 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)  in vhost_scsi_get_inflight()  argument
273 	svq = container_of(vq, struct vhost_scsi_virtqueue, vq);  in vhost_scsi_get_inflight()
577 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_allocate_evt()  local
587 		vq_err(vq, "Failed to allocate vhost_scsi_evt\n");  in vhost_scsi_allocate_evt()
592 	evt->event.event = cpu_to_vhost32(vq, event);  in vhost_scsi_allocate_evt()
593 	evt->event.reason = cpu_to_vhost32(vq, reason);  in vhost_scsi_allocate_evt()
616 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_do_evt_work()  local
622 	if (!vq->private_data) {  in vhost_scsi_do_evt_work()
628 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_do_evt_work()
629 	head = vhost_get_vq_desc(vq, vq->iov,  in vhost_scsi_do_evt_work()
630 			ARRAY_SIZE(vq->iov), &out, &in,  in vhost_scsi_do_evt_work()
636 	if (head == vq->num) {  in vhost_scsi_do_evt_work()
637 		if (vhost_enable_notify(&vs->dev, vq))  in vhost_scsi_do_evt_work()
643 	if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {  in vhost_scsi_do_evt_work()
644 		vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",  in vhost_scsi_do_evt_work()
645 				vq->iov[out].iov_len);  in vhost_scsi_do_evt_work()
651 		event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);  in vhost_scsi_do_evt_work()
655 	eventp = vq->iov[out].iov_base;  in vhost_scsi_do_evt_work()
658 		vhost_add_used_and_signal(&vs->dev, vq, head, 0);  in vhost_scsi_do_evt_work()
660 		vq_err(vq, "Faulted on vhost_scsi_send_event\n");  in vhost_scsi_do_evt_work()
667 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_evt_work()  local
671 	mutex_lock(&vq->mutex);  in vhost_scsi_evt_work()
679 	mutex_unlock(&vq->mutex);  in vhost_scsi_evt_work()
697 	int ret, vq;  in vhost_scsi_complete_cmd_work()  local
725 			q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);  in vhost_scsi_complete_cmd_work()
726 			vq = q - vs->vqs;  in vhost_scsi_complete_cmd_work()
727 			__set_bit(vq, signal);  in vhost_scsi_complete_cmd_work()
734 	vq = -1;  in vhost_scsi_complete_cmd_work()
735 	while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1))  in vhost_scsi_complete_cmd_work()
737 		vhost_signal(&vs->dev, &vs->vqs[vq].vq);  in vhost_scsi_complete_cmd_work()
741 vhost_scsi_get_tag(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,  in vhost_scsi_get_tag()  argument
781 	cmd->inflight = vhost_scsi_get_inflight(vq);  in vhost_scsi_get_tag()
987 			   struct vhost_virtqueue *vq,  in vhost_scsi_send_bad_target()  argument
996 	resp = vq->iov[out].iov_base;  in vhost_scsi_send_bad_target()
999 		vhost_add_used_and_signal(&vs->dev, vq, head, 0);  in vhost_scsi_send_bad_target()
1005 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)  in vhost_scsi_handle_vq()  argument
1020 	bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);  in vhost_scsi_handle_vq()
1023 	mutex_lock(&vq->mutex);  in vhost_scsi_handle_vq()
1028 	vs_tpg = vq->private_data;  in vhost_scsi_handle_vq()
1032 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_handle_vq()
1035 		head = vhost_get_vq_desc(vq, vq->iov,  in vhost_scsi_handle_vq()
1036 					 ARRAY_SIZE(vq->iov), &out, &in,  in vhost_scsi_handle_vq()
1044 		if (head == vq->num) {  in vhost_scsi_handle_vq()
1045 			if (unlikely(vhost_enable_notify(&vs->dev, vq))) {  in vhost_scsi_handle_vq()
1046 				vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_handle_vq()
1055 		if (unlikely(vq->iov[out].iov_len < rsp_size)) {  in vhost_scsi_handle_vq()
1056 			vq_err(vq, "Expecting at least virtio_scsi_cmd_resp"  in vhost_scsi_handle_vq()
1057 				" size, got %zu bytes\n", vq->iov[out].iov_len);  in vhost_scsi_handle_vq()
1078 		out_size = iov_length(vq->iov, out);  in vhost_scsi_handle_vq()
1079 		in_size = iov_length(&vq->iov[out], in);  in vhost_scsi_handle_vq()
1091 		iov_iter_init(&out_iter, WRITE, vq->iov, out, out_size);  in vhost_scsi_handle_vq()
1095 			vq_err(vq, "Faulted on copy_from_iter\n");  in vhost_scsi_handle_vq()
1096 			vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1101 			vq_err(vq, "Illegal virtio-scsi lun: %u\n", *lunp);  in vhost_scsi_handle_vq()
1102 			vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1109 			vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1137 			iov_iter_init(&in_iter, READ, &vq->iov[out], in,  in vhost_scsi_handle_vq()
1153 					vq_err(vq, "Received non zero pi_bytesout,"  in vhost_scsi_handle_vq()
1155 					vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1158 				prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);  in vhost_scsi_handle_vq()
1161 					vq_err(vq, "Received non zero pi_bytesin,"  in vhost_scsi_handle_vq()
1163 					vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1166 				prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);  in vhost_scsi_handle_vq()
1180 			tag = vhost64_to_cpu(vq, v_req_pi.tag);  in vhost_scsi_handle_vq()
1185 			tag = vhost64_to_cpu(vq, v_req.tag);  in vhost_scsi_handle_vq()
1198 			vq_err(vq, "Received SCSI CDB with command_size: %d that"  in vhost_scsi_handle_vq()
1201 			vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1204 		cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr,  in vhost_scsi_handle_vq()
1208 			vq_err(vq, "vhost_scsi_get_tag failed %ld\n",  in vhost_scsi_handle_vq()
1210 			vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1214 		cmd->tvc_vq = vq;  in vhost_scsi_handle_vq()
1215 		cmd->tvc_resp_iov = &vq->iov[out];  in vhost_scsi_handle_vq()
1228 				vq_err(vq, "Failed to map iov to sgl\n");  in vhost_scsi_handle_vq()
1230 				vhost_scsi_send_bad_target(vs, vq, head, out);  in vhost_scsi_handle_vq()
1250 	mutex_unlock(&vq->mutex);  in vhost_scsi_handle_vq()
1290 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_evt_handle_kick()  local
1292 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_evt_handle_kick()
1294 	mutex_lock(&vq->mutex);  in vhost_scsi_evt_handle_kick()
1295 	if (!vq->private_data)  in vhost_scsi_evt_handle_kick()
1301 	mutex_unlock(&vq->mutex);  in vhost_scsi_evt_handle_kick()
1306 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_handle_kick()  local
1308 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_handle_kick()
1310 	vhost_scsi_handle_vq(vs, vq);  in vhost_scsi_handle_kick()
1315 	vhost_poll_flush(&vs->vqs[index].vq.poll);  in vhost_scsi_flush_vq()
1361 	struct vhost_virtqueue *vq;  in vhost_scsi_set_endpoint()  local
1371 		if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {  in vhost_scsi_set_endpoint()
1432 			vq = &vs->vqs[i].vq;  in vhost_scsi_set_endpoint()
1433 			mutex_lock(&vq->mutex);  in vhost_scsi_set_endpoint()
1434 			vq->private_data = vs_tpg;  in vhost_scsi_set_endpoint()
1435 			vhost_init_used(vq);  in vhost_scsi_set_endpoint()
1436 			mutex_unlock(&vq->mutex);  in vhost_scsi_set_endpoint()
1464 	struct vhost_virtqueue *vq;  in vhost_scsi_clear_endpoint()  local
1473 		if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {  in vhost_scsi_clear_endpoint()
1519 			vq = &vs->vqs[i].vq;  in vhost_scsi_clear_endpoint()
1520 			mutex_lock(&vq->mutex);  in vhost_scsi_clear_endpoint()
1521 			vq->private_data = NULL;  in vhost_scsi_clear_endpoint()
1522 			mutex_unlock(&vq->mutex);  in vhost_scsi_clear_endpoint()
1547 	struct vhost_virtqueue *vq;  in vhost_scsi_set_features()  local
1561 		vq = &vs->vqs[i].vq;  in vhost_scsi_set_features()
1562 		mutex_lock(&vq->mutex);  in vhost_scsi_set_features()
1563 		vq->acked_features = features;  in vhost_scsi_set_features()
1564 		mutex_unlock(&vq->mutex);  in vhost_scsi_set_features()
1593 	vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;  in vhost_scsi_open()
1594 	vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_open()
1595 	vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;  in vhost_scsi_open()
1596 	vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;  in vhost_scsi_open()
1598 		vqs[i] = &vs->vqs[i].vq;  in vhost_scsi_open()
1599 		vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;  in vhost_scsi_open()
1645 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_ioctl()  local
1669 		mutex_lock(&vq->mutex);  in vhost_scsi_ioctl()
1671 		mutex_unlock(&vq->mutex);  in vhost_scsi_ioctl()
1674 		mutex_lock(&vq->mutex);  in vhost_scsi_ioctl()
1676 		mutex_unlock(&vq->mutex);  in vhost_scsi_ioctl()
1757 	struct vhost_virtqueue *vq;  in vhost_scsi_do_plug()  local
1770 	vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_do_plug()
1771 	mutex_lock(&vq->mutex);  in vhost_scsi_do_plug()
1772 	if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))  in vhost_scsi_do_plug()
1775 	mutex_unlock(&vq->mutex);  in vhost_scsi_do_plug()