90 int scif_reserve_dma_chan(struct scif_endpt *ep)  in scif_reserve_dma_chan()  argument
98 	if (!scif_info.nodeid && scifdev_self(ep->remote_dev))  in scif_reserve_dma_chan()
103 		scifdev = ep->remote_dev;  in scif_reserve_dma_chan()
109 	mutex_lock(&ep->rma_info.rma_lock);  in scif_reserve_dma_chan()
110 	ep->rma_info.dma_chan = chan;  in scif_reserve_dma_chan()
111 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_reserve_dma_chan()
123 			    struct scif_endpt *ep,  in __scif_rma_destroy_tcw()  argument
136 		ep = (struct scif_endpt *)window->ep;  in __scif_rma_destroy_tcw()
151 	struct scif_endpt *ep = mmn->ep;  in scif_rma_destroy_tcw()  local
153 	spin_lock(&ep->rma_info.tc_lock);  in scif_rma_destroy_tcw()
154 	__scif_rma_destroy_tcw(mmn, ep, start, len);  in scif_rma_destroy_tcw()
155 	spin_unlock(&ep->rma_info.tc_lock);  in scif_rma_destroy_tcw()
158 static void scif_rma_destroy_tcw_ep(struct scif_endpt *ep)  in scif_rma_destroy_tcw_ep()  argument
163 	list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {  in scif_rma_destroy_tcw_ep()
169 static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)  in __scif_rma_destroy_tcw_ep()  argument
174 	spin_lock(&ep->rma_info.tc_lock);  in __scif_rma_destroy_tcw_ep()
175 	list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {  in __scif_rma_destroy_tcw_ep()
177 		__scif_rma_destroy_tcw(mmn, ep, 0, ULONG_MAX);  in __scif_rma_destroy_tcw_ep()
179 	spin_unlock(&ep->rma_info.tc_lock);  in __scif_rma_destroy_tcw_ep()
182 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)  in scif_rma_tc_can_cache()  argument
186 	if ((atomic_read(&ep->rma_info.tcw_total_pages)  in scif_rma_tc_can_cache()
192 			 atomic_read(&ep->rma_info.tcw_total_pages),  in scif_rma_tc_can_cache()
195 		__scif_rma_destroy_tcw_ep(ep);  in scif_rma_tc_can_cache()
249 static void scif_ep_unregister_mmu_notifier(struct scif_endpt *ep)  in scif_ep_unregister_mmu_notifier()  argument
251 	struct scif_endpt_rma_info *rma = &ep->rma_info;  in scif_ep_unregister_mmu_notifier()
255 	mutex_lock(&ep->rma_info.mmn_lock);  in scif_ep_unregister_mmu_notifier()
262 	mutex_unlock(&ep->rma_info.mmn_lock);  in scif_ep_unregister_mmu_notifier()
266 				   struct mm_struct *mm, struct scif_endpt *ep)  in scif_init_mmu_notifier()  argument
268 	mmn->ep = ep;  in scif_init_mmu_notifier()
290 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)  in scif_add_mmu_notifier()  argument
298 	scif_init_mmu_notifier(mmn, current->mm, ep);  in scif_add_mmu_notifier()
304 	list_add(&mmn->list, &ep->rma_info.mmn_list);  in scif_add_mmu_notifier()
315 	struct scif_endpt *ep;  in scif_mmu_notif_handler()  local
320 		ep = list_entry(pos, struct scif_endpt, mmu_list);  in scif_mmu_notif_handler()
321 		list_del(&ep->mmu_list);  in scif_mmu_notif_handler()
323 		scif_rma_destroy_tcw_ep(ep);  in scif_mmu_notif_handler()
324 		scif_ep_unregister_mmu_notifier(ep);  in scif_mmu_notif_handler()
343 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)  in scif_add_mmu_notifier()  argument
357 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)  in scif_rma_tc_can_cache()  argument
378 	struct scif_endpt *ep = (struct scif_endpt *)epd;  in scif_register_temp()  local
393 	err = scif_get_window_offset(ep, 0, 0,  in scif_register_temp()
400 	*out_window = scif_create_window(ep, aligned_len >> PAGE_SHIFT,  in scif_register_temp()
403 		scif_free_window_offset(ep, NULL, *out_offset);  in scif_register_temp()
413 	err = scif_map_window(ep->remote_dev, *out_window);  in scif_register_temp()
416 		scif_destroy_window(ep, *out_window);  in scif_register_temp()
424 		dev_err(&ep->remote_dev->sdev->dev,  in scif_register_temp()
604 	struct scif_endpt *ep;  in scif_rma_destroy_windows()  local
613 		ep = (struct scif_endpt *)window->ep;  in scif_rma_destroy_windows()
614 		chan = ep->rma_info.dma_chan;  in scif_rma_destroy_windows()
618 		if (!chan || !scifdev_alive(ep) ||  in scif_rma_destroy_windows()
619 		    !scif_drain_dma_intr(ep->remote_dev->sdev,  in scif_rma_destroy_windows()
620 					 ep->rma_info.dma_chan))  in scif_rma_destroy_windows()
624 			dev_warn(&ep->remote_dev->sdev->dev,  in scif_rma_destroy_windows()
628 				scif_destroy_window(ep, window);  in scif_rma_destroy_windows()
631 			atomic_dec(&ep->rma_info.tw_refcount);  in scif_rma_destroy_windows()
648 	struct scif_endpt *ep;  in scif_rma_destroy_tcw_invalid()  local
656 		ep = (struct scif_endpt *)window->ep;  in scif_rma_destroy_tcw_invalid()
657 		chan = ep->rma_info.dma_chan;  in scif_rma_destroy_tcw_invalid()
660 		mutex_lock(&ep->rma_info.rma_lock);  in scif_rma_destroy_tcw_invalid()
661 		if (!chan || !scifdev_alive(ep) ||  in scif_rma_destroy_tcw_invalid()
662 		    !scif_drain_dma_intr(ep->remote_dev->sdev,  in scif_rma_destroy_tcw_invalid()
663 					 ep->rma_info.dma_chan)) {  in scif_rma_destroy_tcw_invalid()
665 				   &ep->rma_info.tcw_total_pages);  in scif_rma_destroy_tcw_invalid()
666 			scif_destroy_window(ep, window);  in scif_rma_destroy_tcw_invalid()
667 			atomic_dec(&ep->rma_info.tcw_refcount);  in scif_rma_destroy_tcw_invalid()
669 			dev_warn(&ep->remote_dev->sdev->dev,  in scif_rma_destroy_tcw_invalid()
672 		mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_destroy_tcw_invalid()
1678 	struct scif_endpt *ep = (struct scif_endpt *)epd;  in scif_rma_copy()  local
1691 	err = scif_verify_epd(ep);  in scif_rma_copy()
1699 	loopback = scifdev_self(ep->remote_dev) ? true : false;  in scif_rma_copy()
1721 	remote_req.head = &ep->rma_info.remote_reg_list;  in scif_rma_copy()
1723 	spdev = scif_get_peer_dev(ep->remote_dev);  in scif_rma_copy()
1730 		mutex_lock(&ep->rma_info.mmn_lock);  in scif_rma_copy()
1731 		mmn = scif_find_mmu_notifier(current->mm, &ep->rma_info);  in scif_rma_copy()
1733 			scif_add_mmu_notifier(current->mm, ep);  in scif_rma_copy()
1734 		mutex_unlock(&ep->rma_info.mmn_lock);  in scif_rma_copy()
1739 		cache = cache && !scif_rma_tc_can_cache(ep, len);  in scif_rma_copy()
1741 	mutex_lock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1751 			spin_lock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1753 			err = scif_query_tcw(ep, &req);  in scif_rma_copy()
1754 			spin_unlock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1761 				mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1766 			atomic_inc(&ep->rma_info.tcw_refcount);  in scif_rma_copy()
1768 					  &ep->rma_info.tcw_total_pages);  in scif_rma_copy()
1770 				spin_lock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1773 				spin_unlock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1789 		req.head = &ep->rma_info.reg_list;  in scif_rma_copy()
1793 			mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1801 		mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1811 	copy_work.remote_dev = ep->remote_dev;  in scif_rma_copy()
1827 		chan = ep->rma_info.dma_chan;  in scif_rma_copy()
1832 		atomic_inc(&ep->rma_info.tw_refcount);  in scif_rma_copy()
1834 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1837 		struct scif_dev *rdev = ep->remote_dev;  in scif_rma_copy()
1841 						  ep->rma_info.dma_chan);  in scif_rma_copy()
1844 						  ep->rma_info.dma_chan);  in scif_rma_copy()
1854 			scif_destroy_window(ep, local_window);  in scif_rma_copy()