34 void scif_rma_ep_init(struct scif_endpt *ep)  in scif_rma_ep_init()  argument
36 	struct scif_endpt_rma_info *rma = &ep->rma_info;  in scif_rma_ep_init()
63 int scif_rma_ep_can_uninit(struct scif_endpt *ep)  in scif_rma_ep_can_uninit()  argument
67 	mutex_lock(&ep->rma_info.rma_lock);  in scif_rma_ep_can_uninit()
69 	if (list_empty(&ep->rma_info.reg_list) &&  in scif_rma_ep_can_uninit()
70 	    list_empty(&ep->rma_info.remote_reg_list) &&  in scif_rma_ep_can_uninit()
71 	    list_empty(&ep->rma_info.mmn_list) &&  in scif_rma_ep_can_uninit()
72 	    !atomic_read(&ep->rma_info.tw_refcount) &&  in scif_rma_ep_can_uninit()
73 	    !atomic_read(&ep->rma_info.tcw_refcount) &&  in scif_rma_ep_can_uninit()
74 	    !atomic_read(&ep->rma_info.fence_refcount))  in scif_rma_ep_can_uninit()
76 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_ep_can_uninit()
146 struct scif_window *scif_create_window(struct scif_endpt *ep, int nr_pages,  in scif_create_window()  argument
165 	window->ep = (u64)ep;  in scif_create_window()
191 static void scif_destroy_incomplete_window(struct scif_endpt *ep,  in scif_destroy_incomplete_window()  argument
204 	if (!err && scifdev_alive(ep))  in scif_destroy_incomplete_window()
207 	mutex_lock(&ep->rma_info.rma_lock);  in scif_destroy_incomplete_window()
210 		msg.src = ep->port;  in scif_destroy_incomplete_window()
211 		msg.payload[0] = ep->remote_ep;  in scif_destroy_incomplete_window()
215 		_scif_nodeqp_send(ep->remote_dev, &msg);  in scif_destroy_incomplete_window()
217 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_destroy_incomplete_window()
219 	scif_free_window_offset(ep, window, window->offset);  in scif_destroy_incomplete_window()
319 int scif_destroy_window(struct scif_endpt *ep, struct scif_window *window)  in scif_destroy_window()  argument
332 	scif_free_window_offset(ep, window, window->offset);  in scif_destroy_window()
333 	scif_unmap_window(ep->remote_dev, window);  in scif_destroy_window()
643 static int scif_send_scif_unregister(struct scif_endpt *ep,  in scif_send_scif_unregister()  argument
649 	msg.src = ep->port;  in scif_send_scif_unregister()
652 	return scif_nodeqp_send(ep->remote_dev, &msg);  in scif_send_scif_unregister()
664 	struct scif_endpt *ep = (struct scif_endpt *)window->ep;  in scif_unregister_window()  local
678 		mutex_unlock(&ep->rma_info.rma_lock);  in scif_unregister_window()
680 			err = scif_send_scif_unregister(ep, window);  in scif_unregister_window()
687 			mutex_lock(&ep->rma_info.rma_lock);  in scif_unregister_window()
695 		if (!err && scifdev_alive(ep))  in scif_unregister_window()
706 		mutex_lock(&ep->rma_info.rma_lock);  in scif_unregister_window()
712 		if (!scifdev_alive(ep)) {  in scif_unregister_window()
728 		atomic_inc(&ep->rma_info.tw_refcount);  in scif_unregister_window()
730 		scif_free_window_offset(ep, window, window->offset);  in scif_unregister_window()
731 		mutex_unlock(&ep->rma_info.rma_lock);  in scif_unregister_window()
733 		    scifdev_alive(ep)) {  in scif_unregister_window()
734 			scif_drain_dma_intr(ep->remote_dev->sdev,  in scif_unregister_window()
735 					    ep->rma_info.dma_chan);  in scif_unregister_window()
744 		mutex_lock(&ep->rma_info.rma_lock);  in scif_unregister_window()
756 static int scif_send_alloc_request(struct scif_endpt *ep,  in scif_send_alloc_request()  argument
770 	return _scif_nodeqp_send(ep->remote_dev, &msg);  in scif_send_alloc_request()
781 static int scif_prep_remote_window(struct scif_endpt *ep,  in scif_prep_remote_window()  argument
793 	map_err = scif_map_window(ep->remote_dev, window);  in scif_prep_remote_window()
795 		dev_err(&ep->remote_dev->sdev->dev,  in scif_prep_remote_window()
804 	mutex_lock(&ep->rma_info.rma_lock);  in scif_prep_remote_window()
806 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_prep_remote_window()
807 	if (!err && scifdev_alive(ep))  in scif_prep_remote_window()
823 		dev_err(&ep->remote_dev->sdev->dev,  in scif_prep_remote_window()
826 		msg.src = ep->port;  in scif_prep_remote_window()
827 		msg.payload[0] = ep->remote_ep;  in scif_prep_remote_window()
831 		spin_lock(&ep->lock);  in scif_prep_remote_window()
832 		if (ep->state == SCIFEP_CONNECTED)  in scif_prep_remote_window()
833 			err = _scif_nodeqp_send(ep->remote_dev, &msg);  in scif_prep_remote_window()
836 		spin_unlock(&ep->lock);  in scif_prep_remote_window()
841 				     ep->remote_dev);  in scif_prep_remote_window()
851 			     ep->remote_dev);  in scif_prep_remote_window()
856 			     ep->remote_dev);  in scif_prep_remote_window()
872 				   ep->remote_dev);  in scif_prep_remote_window()
876 				    ep->remote_dev);  in scif_prep_remote_window()
885 			if (scifdev_is_p2p(ep->remote_dev)) {  in scif_prep_remote_window()
895 						ep->remote_dev->base_addr;  in scif_prep_remote_window()
925 			     sizeof(*window->dma_addr), ep->remote_dev);  in scif_prep_remote_window()
927 			     sizeof(*window->num_pages), ep->remote_dev);  in scif_prep_remote_window()
935 	remote_window->ep = ep->remote_ep;  in scif_prep_remote_window()
939 		     ep->remote_dev);  in scif_prep_remote_window()
943 		     ep->remote_dev);  in scif_prep_remote_window()
944 	scif_iounmap(remote_window, sizeof(*remote_window), ep->remote_dev);  in scif_prep_remote_window()
958 static int scif_send_scif_register(struct scif_endpt *ep,  in scif_send_scif_register()  argument
964 	msg.src = ep->port;  in scif_send_scif_register()
965 	msg.payload[0] = ep->remote_ep;  in scif_send_scif_register()
968 	spin_lock(&ep->lock);  in scif_send_scif_register()
969 	if (ep->state == SCIFEP_CONNECTED) {  in scif_send_scif_register()
972 		err = _scif_nodeqp_send(ep->remote_dev, &msg);  in scif_send_scif_register()
973 		spin_unlock(&ep->lock);  in scif_send_scif_register()
981 			if (!err && scifdev_alive(ep))  in scif_send_scif_register()
990 		err = _scif_nodeqp_send(ep->remote_dev, &msg);  in scif_send_scif_register()
991 		spin_unlock(&ep->lock);  in scif_send_scif_register()
1008 int scif_get_window_offset(struct scif_endpt *ep, int flags, s64 offset,  in scif_get_window_offset()  argument
1017 		iova_ptr = reserve_iova(&ep->rma_info.iovad, page_index,  in scif_get_window_offset()
1022 		iova_ptr = alloc_iova(&ep->rma_info.iovad, num_pages,  in scif_get_window_offset()
1041 void scif_free_window_offset(struct scif_endpt *ep,  in scif_free_window_offset()  argument
1045 		free_iova(&ep->rma_info.iovad, offset >> PAGE_SHIFT);  in scif_free_window_offset()
1097 	struct scif_endpt *ep = (struct scif_endpt *)window->ep;  in scif_alloc_gnt_rej()  local
1099 	mutex_lock(&ep->rma_info.rma_lock);  in scif_alloc_gnt_rej()
1107 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_alloc_gnt_rej()
1156 	struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];  in scif_recv_reg()  local
1160 	mutex_lock(&ep->rma_info.rma_lock);  in scif_recv_reg()
1161 	spin_lock(&ep->lock);  in scif_recv_reg()
1162 	if (ep->state == SCIFEP_CONNECTED) {  in scif_recv_reg()
1164 		scif_nodeqp_send(ep->remote_dev, msg);  in scif_recv_reg()
1165 		scif_fixup_aper_base(ep->remote_dev, window);  in scif_recv_reg()
1167 		scif_insert_window(window, &ep->rma_info.remote_reg_list);  in scif_recv_reg()
1170 		scif_nodeqp_send(ep->remote_dev, msg);  in scif_recv_reg()
1172 	spin_unlock(&ep->lock);  in scif_recv_reg()
1173 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_recv_reg()
1175 	scif_destroy_remote_lookup(ep->remote_dev, window);  in scif_recv_reg()
1196 	struct scif_endpt *ep;  in scif_recv_unreg()  local
1199 	ep = (struct scif_endpt *)recv_window->ep;  in scif_recv_unreg()
1205 	req.head = &ep->rma_info.remote_reg_list;  in scif_recv_unreg()
1206 	msg->payload[0] = ep->remote_ep;  in scif_recv_unreg()
1208 	mutex_lock(&ep->rma_info.rma_lock);  in scif_recv_unreg()
1226 			atomic_inc(&ep->rma_info.tw_refcount);  in scif_recv_unreg()
1227 			ep->rma_info.async_list_del = 1;  in scif_recv_unreg()
1240 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_recv_unreg()
1242 		scif_drain_dma_intr(ep->remote_dev->sdev,  in scif_recv_unreg()
1243 				    ep->rma_info.dma_chan);  in scif_recv_unreg()
1244 	scif_nodeqp_send(ep->remote_dev, msg);  in scif_recv_unreg()
1259 	struct scif_endpt *ep = (struct scif_endpt *)window->ep;  in scif_recv_reg_ack()  local
1261 	mutex_lock(&ep->rma_info.rma_lock);  in scif_recv_reg_ack()
1264 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_recv_reg_ack()
1278 	struct scif_endpt *ep = (struct scif_endpt *)window->ep;  in scif_recv_reg_nack()  local
1280 	mutex_lock(&ep->rma_info.rma_lock);  in scif_recv_reg_nack()
1283 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_recv_reg_nack()
1296 	struct scif_endpt *ep = (struct scif_endpt *)window->ep;  in scif_recv_unreg_ack()  local
1298 	mutex_lock(&ep->rma_info.rma_lock);  in scif_recv_unreg_ack()
1301 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_recv_unreg_ack()
1315 	struct scif_endpt *ep = (struct scif_endpt *)window->ep;  in scif_recv_unreg_nack()  local
1317 	mutex_lock(&ep->rma_info.rma_lock);  in scif_recv_unreg_nack()
1320 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_recv_unreg_nack()
1483 scif_insert_local_window(struct scif_window *window, struct scif_endpt *ep)  in scif_insert_local_window()  argument
1485 	mutex_lock(&ep->rma_info.rma_lock);  in scif_insert_local_window()
1486 	scif_insert_window(window, &ep->rma_info.reg_list);  in scif_insert_local_window()
1487 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_insert_local_window()
1494 	struct scif_endpt *ep = (struct scif_endpt *)epd;  in scif_register_pinned_pages()  local
1519 	err = scif_verify_epd(ep);  in scif_register_pinned_pages()
1530 	err = scif_get_window_offset(ep, map_flags, offset,  in scif_register_pinned_pages()
1538 	window = scif_create_window(ep, pinned_pages->nr_pages,  in scif_register_pinned_pages()
1542 		scif_free_window_offset(ep, NULL, computed_offset);  in scif_register_pinned_pages()
1550 	spdev = scif_get_peer_dev(ep->remote_dev);  in scif_register_pinned_pages()
1553 		scif_destroy_window(ep, window);  in scif_register_pinned_pages()
1556 	err = scif_send_alloc_request(ep, window);  in scif_register_pinned_pages()
1558 		dev_err(&ep->remote_dev->sdev->dev,  in scif_register_pinned_pages()
1564 	err = scif_prep_remote_window(ep, window);  in scif_register_pinned_pages()
1566 		dev_err(&ep->remote_dev->sdev->dev,  in scif_register_pinned_pages()
1572 	err = scif_send_scif_register(ep, window);  in scif_register_pinned_pages()
1574 		dev_err(&ep->remote_dev->sdev->dev,  in scif_register_pinned_pages()
1581 	scif_insert_local_window(window, ep);  in scif_register_pinned_pages()
1584 	scif_destroy_window(ep, window);  in scif_register_pinned_pages()
1586 	dev_err(&ep->remote_dev->sdev->dev,  in scif_register_pinned_pages()
1597 	struct scif_endpt *ep = (struct scif_endpt *)epd;  in scif_register()  local
1631 	err = scif_verify_epd(ep);  in scif_register()
1636 	err = scif_get_window_offset(ep, map_flags, offset,  in scif_register()
1641 	spdev = scif_get_peer_dev(ep->remote_dev);  in scif_register()
1644 		scif_free_window_offset(ep, NULL, computed_offset);  in scif_register()
1648 	window = scif_create_window(ep, len >> PAGE_SHIFT,  in scif_register()
1651 		scif_free_window_offset(ep, NULL, computed_offset);  in scif_register()
1658 	err = scif_send_alloc_request(ep, window);  in scif_register()
1660 		scif_destroy_incomplete_window(ep, window);  in scif_register()
1674 		scif_destroy_incomplete_window(ep, window);  in scif_register()
1684 	err = scif_prep_remote_window(ep, window);  in scif_register()
1686 		dev_err(&ep->remote_dev->sdev->dev,  in scif_register()
1692 	err = scif_send_scif_register(ep, window);  in scif_register()
1694 		dev_err(&ep->remote_dev->sdev->dev,  in scif_register()
1701 	scif_insert_local_window(window, ep);  in scif_register()
1702 	dev_dbg(&ep->remote_dev->sdev->dev,  in scif_register()
1707 	scif_destroy_window(ep, window);  in scif_register()
1710 	dev_err(&ep->remote_dev->sdev->dev,  in scif_register()
1719 	struct scif_endpt *ep = (struct scif_endpt *)epd;  in scif_unregister()  local
1727 		ep, offset, len);  in scif_unregister()
1739 	err = scif_verify_epd(ep);  in scif_unregister()
1751 	req.head = &ep->rma_info.reg_list;  in scif_unregister()
1753 	spdev = scif_get_peer_dev(ep->remote_dev);  in scif_unregister()
1758 	mutex_lock(&ep->rma_info.rma_lock);  in scif_unregister()
1762 		dev_err(&ep->remote_dev->sdev->dev,  in scif_unregister()
1769 		dev_err(&ep->remote_dev->sdev->dev,  in scif_unregister()
1772 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_unregister()