75 static int mic_virtio_copy_to_user(struct mic_vdev *mvdev, void __user *ubuf,  in mic_virtio_copy_to_user()  argument
79 	struct mic_device *mdev = mvdev->mdev;  in mic_virtio_copy_to_user()
81 	struct mic_vringh *mvr = &mvdev->mvr[vr_idx];  in mic_virtio_copy_to_user()
107 		mvdev->in_bytes_dma += partlen;  in mic_virtio_copy_to_user()
108 		mvdev->in_bytes += partlen;  in mic_virtio_copy_to_user()
114 	dev_err(mic_dev(mvdev), "%s %d err %d\n", __func__, __LINE__, err);  in mic_virtio_copy_to_user()
124 static int mic_virtio_copy_from_user(struct mic_vdev *mvdev, void __user *ubuf,  in mic_virtio_copy_from_user()  argument
128 	struct mic_device *mdev = mvdev->mdev;  in mic_virtio_copy_from_user()
130 	struct mic_vringh *mvr = &mvdev->mvr[vr_idx];  in mic_virtio_copy_from_user()
136 		mvdev->tx_dst_unaligned += len;  in mic_virtio_copy_from_user()
139 		mvdev->tx_len_unaligned += len;  in mic_virtio_copy_from_user()
157 		mvdev->out_bytes_dma += partlen;  in mic_virtio_copy_from_user()
158 		mvdev->out_bytes += partlen;  in mic_virtio_copy_from_user()
170 	mvdev->out_bytes += len;  in mic_virtio_copy_from_user()
173 	dev_err(mic_dev(mvdev), "%s %d err %d\n", __func__, __LINE__, err);  in mic_virtio_copy_from_user()
183 	struct mic_vdev *mvdev = mvrh->mvdev;  in mic_notify()  local
184 	s8 db = mvdev->dc->h2c_vdev_db;  in mic_notify()
187 		mvdev->mdev->ops->send_intr(mvdev->mdev, db);  in mic_notify()
208 static int mic_vringh_copy(struct mic_vdev *mvdev, struct vringh_kiov *iov,  in mic_vringh_copy()  argument
218 			ret = mic_virtio_copy_to_user(mvdev, ubuf, partlen,  in mic_vringh_copy()
223 			ret = mic_virtio_copy_from_user(mvdev, ubuf, partlen,  in mic_vringh_copy()
228 			dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_vringh_copy()
255 static int _mic_virtio_copy(struct mic_vdev *mvdev,  in _mic_virtio_copy()  argument
263 	struct mic_vringh *mvr = &mvdev->mvr[copy->vr_idx];  in _mic_virtio_copy()
286 				dev_err(mic_dev(mvdev), "%s %d err %d\n",  in _mic_virtio_copy()
294 		ret = mic_vringh_copy(mvdev, riov, ubuf, len, MIC_VRINGH_READ,  in _mic_virtio_copy()
297 			dev_err(mic_dev(mvdev), "%s %d err %d\n",  in _mic_virtio_copy()
305 		ret = mic_vringh_copy(mvdev, wiov, ubuf, len, !MIC_VRINGH_READ,  in _mic_virtio_copy()
308 			dev_err(mic_dev(mvdev), "%s %d err %d\n",  in _mic_virtio_copy()
346 static inline int mic_verify_copy_args(struct mic_vdev *mvdev,  in mic_verify_copy_args()  argument
349 	if (copy->vr_idx >= mvdev->dd->num_vq) {  in mic_verify_copy_args()
350 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_verify_copy_args()
358 int mic_virtio_copy_desc(struct mic_vdev *mvdev,  in mic_virtio_copy_desc()  argument
362 	struct mic_vringh *mvr = &mvdev->mvr[copy->vr_idx];  in mic_virtio_copy_desc()
364 	err = mic_verify_copy_args(mvdev, copy);  in mic_virtio_copy_desc()
369 	if (!mic_vdevup(mvdev)) {  in mic_virtio_copy_desc()
371 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_virtio_copy_desc()
375 	err = _mic_virtio_copy(mvdev, copy);  in mic_virtio_copy_desc()
377 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_virtio_copy_desc()
385 static void mic_virtio_init_post(struct mic_vdev *mvdev)  in mic_virtio_init_post()  argument
387 	struct mic_vqconfig *vqconfig = mic_vq_config(mvdev->dd);  in mic_virtio_init_post()
390 	for (i = 0; i < mvdev->dd->num_vq; i++) {  in mic_virtio_init_post()
392 			dev_warn(mic_dev(mvdev), "used_address zero??\n");  in mic_virtio_init_post()
395 		mvdev->mvr[i].vrh.vring.used =  in mic_virtio_init_post()
396 			(void __force *)mvdev->mdev->aper.va +  in mic_virtio_init_post()
400 	mvdev->dc->used_address_updated = 0;  in mic_virtio_init_post()
402 	dev_dbg(mic_dev(mvdev), "%s: device type %d LINKUP\n",  in mic_virtio_init_post()
403 		__func__, mvdev->virtio_id);  in mic_virtio_init_post()
406 static inline void mic_virtio_device_reset(struct mic_vdev *mvdev)  in mic_virtio_device_reset()  argument
410 	dev_dbg(mic_dev(mvdev), "%s: status %d device type %d RESET\n",  in mic_virtio_device_reset()
411 		__func__, mvdev->dd->status, mvdev->virtio_id);  in mic_virtio_device_reset()
413 	for (i = 0; i < mvdev->dd->num_vq; i++)  in mic_virtio_device_reset()
418 		mutex_lock_nested(&mvdev->mvr[i].vr_mutex, i + 1);  in mic_virtio_device_reset()
421 	mvdev->dd->status = 0;  in mic_virtio_device_reset()
422 	mvdev->dc->vdev_reset = 0;  in mic_virtio_device_reset()
423 	mvdev->dc->host_ack = 1;  in mic_virtio_device_reset()
425 	for (i = 0; i < mvdev->dd->num_vq; i++) {  in mic_virtio_device_reset()
426 		struct vringh *vrh = &mvdev->mvr[i].vrh;  in mic_virtio_device_reset()
427 		mvdev->mvr[i].vring.info->avail_idx = 0;  in mic_virtio_device_reset()
433 	for (i = 0; i < mvdev->dd->num_vq; i++)  in mic_virtio_device_reset()
434 		mutex_unlock(&mvdev->mvr[i].vr_mutex);  in mic_virtio_device_reset()
440 	struct mic_vdev *mvdev;  in mic_virtio_reset_devices()  local
445 		mvdev = list_entry(pos, struct mic_vdev, list);  in mic_virtio_reset_devices()
446 		mic_virtio_device_reset(mvdev);  in mic_virtio_reset_devices()
447 		mvdev->poll_wake = 1;  in mic_virtio_reset_devices()
448 		wake_up(&mvdev->waitq);  in mic_virtio_reset_devices()
454 	struct mic_vdev *mvdev = container_of(work, struct mic_vdev,  in mic_bh_handler()  local
457 	if (mvdev->dc->used_address_updated)  in mic_bh_handler()
458 		mic_virtio_init_post(mvdev);  in mic_bh_handler()
460 	if (mvdev->dc->vdev_reset)  in mic_bh_handler()
461 		mic_virtio_device_reset(mvdev);  in mic_bh_handler()
463 	mvdev->poll_wake = 1;  in mic_bh_handler()
464 	wake_up(&mvdev->waitq);  in mic_bh_handler()
469 	struct mic_vdev *mvdev = data;  in mic_virtio_intr_handler()  local
470 	struct mic_device *mdev = mvdev->mdev;  in mic_virtio_intr_handler()
473 	schedule_work(&mvdev->virtio_bh_work);  in mic_virtio_intr_handler()
477 int mic_virtio_config_change(struct mic_vdev *mvdev,  in mic_virtio_config_change()  argument
482 	struct mic_bootparam *bootparam = mvdev->mdev->dp;  in mic_virtio_config_change()
485 	mutex_lock(&mvdev->mdev->mic_mutex);  in mic_virtio_config_change()
486 	for (i = 0; i < mvdev->dd->num_vq; i++)  in mic_virtio_config_change()
487 		mutex_lock_nested(&mvdev->mvr[i].vr_mutex, i + 1);  in mic_virtio_config_change()
489 	if (db == -1 || mvdev->dd->type == -1) {  in mic_virtio_config_change()
494 	if (copy_from_user(mic_vq_configspace(mvdev->dd),  in mic_virtio_config_change()
495 			   argp, mvdev->dd->config_len)) {  in mic_virtio_config_change()
496 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_virtio_config_change()
501 	mvdev->dc->config_change = MIC_VIRTIO_PARAM_CONFIG_CHANGED;  in mic_virtio_config_change()
502 	mvdev->mdev->ops->send_intr(mvdev->mdev, db);  in mic_virtio_config_change()
506 			mvdev->dc->guest_ack, msecs_to_jiffies(100));  in mic_virtio_config_change()
511 	dev_dbg(mic_dev(mvdev),  in mic_virtio_config_change()
513 	mvdev->dc->config_change = 0;  in mic_virtio_config_change()
514 	mvdev->dc->guest_ack = 0;  in mic_virtio_config_change()
516 	for (i = 0; i < mvdev->dd->num_vq; i++)  in mic_virtio_config_change()
517 		mutex_unlock(&mvdev->mvr[i].vr_mutex);  in mic_virtio_config_change()
518 	mutex_unlock(&mvdev->mdev->mic_mutex);  in mic_virtio_config_change()
522 static int mic_copy_dp_entry(struct mic_vdev *mvdev,  in mic_copy_dp_entry()  argument
527 	struct mic_device *mdev = mvdev->mdev;  in mic_copy_dp_entry()
534 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_copy_dp_entry()
541 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_copy_dp_entry()
548 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_copy_dp_entry()
554 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_copy_dp_entry()
563 			dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_copy_dp_entry()
581 		dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_copy_dp_entry()
599 static void mic_init_device_ctrl(struct mic_vdev *mvdev,  in mic_init_device_ctrl()  argument
613 	mvdev->dc = dc;  in mic_init_device_ctrl()
616 int mic_virtio_add_device(struct mic_vdev *mvdev,  in mic_virtio_add_device()  argument
619 	struct mic_device *mdev = mvdev->mdev;  in mic_virtio_add_device()
632 	ret = mic_copy_dp_entry(mvdev, argp, &type, &dd);  in mic_virtio_add_device()
638 	mic_init_device_ctrl(mvdev, dd);  in mic_virtio_add_device()
640 	mvdev->dd = dd;  in mic_virtio_add_device()
641 	mvdev->virtio_id = type;  in mic_virtio_add_device()
643 	INIT_WORK(&mvdev->virtio_bh_work, mic_bh_handler);  in mic_virtio_add_device()
646 		struct mic_vringh *mvr = &mvdev->mvr[i];  in mic_virtio_add_device()
647 		struct mic_vring *vr = &mvdev->mvr[i].vring;  in mic_virtio_add_device()
657 			dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_virtio_add_device()
663 		vr->info->magic = cpu_to_le32(MIC_MAGIC + mvdev->virtio_id + i);  in mic_virtio_add_device()
668 			dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_virtio_add_device()
676 			*(u32 *)mic_vq_features(mvdev->dd), num, false,  in mic_virtio_add_device()
679 			dev_err(mic_dev(mvdev), "%s %d err %d\n",  in mic_virtio_add_device()
686 		mvr->mvdev = mvdev;  in mic_virtio_add_device()
693 		mvr->buf_da = mic_map_single(mvdev->mdev, mvr->buf,  in mic_virtio_add_device()
698 		 mvdev->virtio_id);  in mic_virtio_add_device()
699 	mvdev->virtio_db = mic_next_db(mdev);  in mic_virtio_add_device()
700 	mvdev->virtio_cookie = mic_request_threaded_irq(mdev,  in mic_virtio_add_device()
702 					       NULL, irqname, mvdev,  in mic_virtio_add_device()
703 					       mvdev->virtio_db, MIC_INTR_DB);  in mic_virtio_add_device()
704 	if (IS_ERR(mvdev->virtio_cookie)) {  in mic_virtio_add_device()
705 		ret = PTR_ERR(mvdev->virtio_cookie);  in mic_virtio_add_device()
710 	mvdev->dc->c2h_vdev_db = mvdev->virtio_db;  in mic_virtio_add_device()
712 	list_add_tail(&mvdev->list, &mdev->vdev_list);  in mic_virtio_add_device()
732 		struct mic_vringh *mvr = &mvdev->mvr[j];  in mic_virtio_add_device()
742 void mic_virtio_del_device(struct mic_vdev *mvdev)  in mic_virtio_del_device()  argument
746 	struct mic_device *mdev = mvdev->mdev;  in mic_virtio_del_device()
758 		"Requesting hot remove id %d\n", mvdev->virtio_id);  in mic_virtio_del_device()
759 	mvdev->dc->config_change = MIC_VIRTIO_PARAM_DEV_REMOVE;  in mic_virtio_del_device()
763 			mvdev->dc->guest_ack, msecs_to_jiffies(100));  in mic_virtio_del_device()
769 		mvdev->virtio_id, mvdev->dc->config_change,  in mic_virtio_del_device()
770 		mvdev->dc->guest_ack, retry);  in mic_virtio_del_device()
771 	mvdev->dc->config_change = 0;  in mic_virtio_del_device()
772 	mvdev->dc->guest_ack = 0;  in mic_virtio_del_device()
774 	mic_free_irq(mdev, mvdev->virtio_cookie, mvdev);  in mic_virtio_del_device()
775 	flush_work(&mvdev->virtio_bh_work);  in mic_virtio_del_device()
776 	vqconfig = mic_vq_config(mvdev->dd);  in mic_virtio_del_device()
777 	for (i = 0; i < mvdev->dd->num_vq; i++) {  in mic_virtio_del_device()
778 		struct mic_vringh *mvr = &mvdev->mvr[i];  in mic_virtio_del_device()
780 		mic_unmap_single(mvdev->mdev, mvr->buf_da,  in mic_virtio_del_device()
794 		if (tmp_mvdev == mvdev) {  in mic_virtio_del_device()
798 				mvdev->virtio_id);  in mic_virtio_del_device()
809 	mvdev->dd->type = -1;  in mic_virtio_del_device()