312 void scif_mmu_notif_handler(struct work_struct *work)  in scif_mmu_notif_handler()  argument
348 void scif_mmu_notif_handler(struct work_struct *work)  in scif_mmu_notif_handler()  argument
712 iounmap_remote(void *virt, size_t size, struct scif_copy_work *work)  in iounmap_remote()  argument
714 	scif_iounmap(virt, size, work->remote_dev);  in iounmap_remote()
900 scif_rma_list_dma_copy_unaligned(struct scif_copy_work *work,  in scif_rma_list_dma_copy_unaligned()  argument
904 	struct scif_dma_comp_cb *comp_cb = work->comp_cb;  in scif_rma_list_dma_copy_unaligned()
907 	size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;  in scif_rma_list_dma_copy_unaligned()
918 		offset = work->dst_offset;  in scif_rma_list_dma_copy_unaligned()
919 		window = work->dst_window;  in scif_rma_list_dma_copy_unaligned()
921 		offset = work->src_offset;  in scif_rma_list_dma_copy_unaligned()
922 		window = work->src_window;  in scif_rma_list_dma_copy_unaligned()
931 						  work->remote_dev,  in scif_rma_list_dma_copy_unaligned()
938 						work->ordered &&  in scif_rma_list_dma_copy_unaligned()
942 						  loop_len, work->ordered &&  in scif_rma_list_dma_copy_unaligned()
944 		iounmap_remote(window_virt_addr, loop_len, work);  in scif_rma_list_dma_copy_unaligned()
974 			if (work->ordered && !tail_len &&  in scif_rma_list_dma_copy_unaligned()
1072 						  work->remote_dev,  in scif_rma_list_dma_copy_unaligned()
1081 		if (work->ordered) {  in scif_rma_list_dma_copy_unaligned()
1082 			struct scif_dev *rdev = work->remote_dev;  in scif_rma_list_dma_copy_unaligned()
1090 						tail_len, work->ordered);  in scif_rma_list_dma_copy_unaligned()
1093 						  tail_len, work->ordered);  in scif_rma_list_dma_copy_unaligned()
1094 		iounmap_remote(window_virt_addr, tail_len, work);  in scif_rma_list_dma_copy_unaligned()
1123 static int _scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,  in _scif_rma_list_dma_copy_aligned()  argument
1132 	struct scif_window *src_window = work->src_window;  in _scif_rma_list_dma_copy_aligned()
1133 	struct scif_window *dst_window = work->dst_window;  in _scif_rma_list_dma_copy_aligned()
1134 	s64 src_offset = work->src_offset, dst_offset = work->dst_offset;  in _scif_rma_list_dma_copy_aligned()
1140 	remaining_len = work->len;  in _scif_rma_list_dma_copy_aligned()
1171 		if (work->ordered && !(remaining_len - loop_len)) {  in _scif_rma_list_dma_copy_aligned()
1241 static int scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,  in scif_rma_list_dma_copy_aligned()  argument
1252 	struct scif_window *src_window = work->src_window;  in scif_rma_list_dma_copy_aligned()
1253 	struct scif_window *dst_window = work->dst_window;  in scif_rma_list_dma_copy_aligned()
1254 	s64 src_offset = work->src_offset, dst_offset = work->dst_offset;  in scif_rma_list_dma_copy_aligned()
1260 	remaining_len = work->len;  in scif_rma_list_dma_copy_aligned()
1277 						  work->remote_dev, NULL);  in scif_rma_list_dma_copy_aligned()
1286 						  work->remote_dev, NULL);  in scif_rma_list_dma_copy_aligned()
1289 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1295 						work->ordered : false);  in scif_rma_list_dma_copy_aligned()
1299 						  work->ordered : false);  in scif_rma_list_dma_copy_aligned()
1301 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1303 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1338 		if (work->ordered && !tail_len &&  in scif_rma_list_dma_copy_aligned()
1419 		if (work->ordered) {  in scif_rma_list_dma_copy_aligned()
1420 			struct scif_dev *rdev = work->remote_dev;  in scif_rma_list_dma_copy_aligned()
1432 						  work->remote_dev, NULL);  in scif_rma_list_dma_copy_aligned()
1442 						  work->remote_dev, NULL);  in scif_rma_list_dma_copy_aligned()
1445 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1451 						work->ordered);  in scif_rma_list_dma_copy_aligned()
1454 						  loop_len, work->ordered);  in scif_rma_list_dma_copy_aligned()
1456 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1459 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1475 static int scif_rma_list_cpu_copy(struct scif_copy_work *work)  in scif_rma_list_cpu_copy()  argument
1480 	s64 src_offset = work->src_offset, dst_offset = work->dst_offset;  in scif_rma_list_cpu_copy()
1481 	struct scif_window *src_window = work->src_window;  in scif_rma_list_cpu_copy()
1482 	struct scif_window *dst_window = work->dst_window;  in scif_rma_list_cpu_copy()
1488 	remaining_len = work->len;  in scif_rma_list_cpu_copy()
1505 						  work->remote_dev,  in scif_rma_list_cpu_copy()
1518 						  work->remote_dev,  in scif_rma_list_cpu_copy()
1522 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1527 		if (work->loopback) {  in scif_rma_list_cpu_copy()
1539 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1542 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1569 					  struct scif_copy_work *work,  in scif_rma_list_dma_copy_wrapper()  argument
1573 	s64 src_offset = work->src_offset, dst_offset = work->dst_offset;  in scif_rma_list_dma_copy_wrapper()
1581 		return _scif_rma_list_dma_copy_aligned(work, chan);  in scif_rma_list_dma_copy_wrapper()
1587 		return scif_rma_list_dma_copy_aligned(work, chan);  in scif_rma_list_dma_copy_wrapper()
1589 	if (work->loopback)  in scif_rma_list_dma_copy_wrapper()
1590 		return scif_rma_list_cpu_copy(work);  in scif_rma_list_dma_copy_wrapper()
1591 	src_dma_addr = __scif_off_to_dma_addr(work->src_window, src_offset);  in scif_rma_list_dma_copy_wrapper()
1592 	dst_dma_addr = __scif_off_to_dma_addr(work->dst_window, dst_offset);  in scif_rma_list_dma_copy_wrapper()
1593 	src_local = work->src_window->type == SCIF_WINDOW_SELF;  in scif_rma_list_dma_copy_wrapper()
1594 	dst_local = work->dst_window->type == SCIF_WINDOW_SELF;  in scif_rma_list_dma_copy_wrapper()
1602 	work->comp_cb = comp_cb;  in scif_rma_list_dma_copy_wrapper()
1606 	if (work->len + (L1_CACHE_BYTES << 1) < SCIF_KMEM_UNALIGNED_BUF_SIZE) {  in scif_rma_list_dma_copy_wrapper()
1609 		temp = kmalloc(work->len + (L1_CACHE_BYTES << 1),  in scif_rma_list_dma_copy_wrapper()
1627 		scif_rma_local_cpu_copy(work->src_offset, work->src_window,  in scif_rma_list_dma_copy_wrapper()
1628 					temp, work->len, true);  in scif_rma_list_dma_copy_wrapper()
1630 		comp_cb->dst_window = work->dst_window;  in scif_rma_list_dma_copy_wrapper()
1631 		comp_cb->dst_offset = work->dst_offset;  in scif_rma_list_dma_copy_wrapper()
1632 		work->src_offset = work->src_offset - src_cache_off;  in scif_rma_list_dma_copy_wrapper()
1633 		comp_cb->len = work->len;  in scif_rma_list_dma_copy_wrapper()
1634 		work->len = ALIGN(work->len + src_cache_off, L1_CACHE_BYTES);  in scif_rma_list_dma_copy_wrapper()
1640 			      work->remote_dev, SCIF_KMEM_UNALIGNED_BUF_SIZE);  in scif_rma_list_dma_copy_wrapper()
1643 	comp_cb->sdev = work->remote_dev;  in scif_rma_list_dma_copy_wrapper()
1644 	if (scif_rma_list_dma_copy_unaligned(work, temp, chan, src_local) < 0)  in scif_rma_list_dma_copy_wrapper()
1647 		work->fence_type = SCIF_DMA_INTR;  in scif_rma_list_dma_copy_wrapper()