828 	size_t loop_len;  in scif_rma_local_cpu_copy()  local
833 	loop_len = PAGE_SIZE - offset_in_page;  in scif_rma_local_cpu_copy()
835 	if (rem_len < loop_len)  in scif_rma_local_cpu_copy()
836 		loop_len = rem_len;  in scif_rma_local_cpu_copy()
838 	window_virt = _get_local_va(offset, window, loop_len);  in scif_rma_local_cpu_copy()
842 		memcpy(temp, window_virt, loop_len);  in scif_rma_local_cpu_copy()
844 		memcpy(window_virt, temp, loop_len);  in scif_rma_local_cpu_copy()
846 	offset += loop_len;  in scif_rma_local_cpu_copy()
847 	temp += loop_len;  in scif_rma_local_cpu_copy()
848 	rem_len -= loop_len;  in scif_rma_local_cpu_copy()
858 		loop_len = min(PAGE_SIZE, rem_len);  in scif_rma_local_cpu_copy()
859 		window_virt = _get_local_va(offset, window, loop_len);  in scif_rma_local_cpu_copy()
863 			memcpy(temp, window_virt, loop_len);  in scif_rma_local_cpu_copy()
865 			memcpy(window_virt, temp, loop_len);  in scif_rma_local_cpu_copy()
866 		offset	+= loop_len;  in scif_rma_local_cpu_copy()
867 		temp	+= loop_len;  in scif_rma_local_cpu_copy()
868 		rem_len	-= loop_len;  in scif_rma_local_cpu_copy()
907 	size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;  in scif_rma_list_dma_copy_unaligned()  local
927 		loop_len = L1_CACHE_BYTES - offset_in_ca;  in scif_rma_list_dma_copy_unaligned()
928 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_unaligned()
930 						  loop_len,  in scif_rma_list_dma_copy_unaligned()
937 						loop_len,  in scif_rma_list_dma_copy_unaligned()
939 						!(remaining_len - loop_len));  in scif_rma_list_dma_copy_unaligned()
942 						  loop_len, work->ordered &&  in scif_rma_list_dma_copy_unaligned()
943 						  !(remaining_len - loop_len));  in scif_rma_list_dma_copy_unaligned()
944 		iounmap_remote(window_virt_addr, loop_len, work);  in scif_rma_list_dma_copy_unaligned()
946 		offset += loop_len;  in scif_rma_list_dma_copy_unaligned()
947 		temp += loop_len;  in scif_rma_list_dma_copy_unaligned()
948 		temp_phys += loop_len;  in scif_rma_list_dma_copy_unaligned()
949 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_unaligned()
972 		loop_len = min(nr_contig_bytes, remaining_len);  in scif_rma_list_dma_copy_unaligned()
975 			    !(remaining_len - loop_len) &&  in scif_rma_list_dma_copy_unaligned()
976 			    loop_len != L1_CACHE_BYTES) {  in scif_rma_list_dma_copy_unaligned()
991 							    loop_len -  in scif_rma_list_dma_copy_unaligned()
1004 				offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
1005 				temp_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
1006 				window_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
1007 				remaining_len -= (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
1008 				loop_len = remaining_len;  in scif_rma_list_dma_copy_unaligned()
1015 							    loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1031 							    loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1045 					window_dma_addr, loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1059 		offset += loop_len;  in scif_rma_list_dma_copy_unaligned()
1060 		temp += loop_len;  in scif_rma_list_dma_copy_unaligned()
1061 		temp_phys += loop_len;  in scif_rma_list_dma_copy_unaligned()
1062 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_unaligned()
1127 	size_t loop_len, remaining_len, src_contig_bytes = 0;  in _scif_rma_list_dma_copy_aligned()  local
1169 		loop_len = min(src_contig_bytes, dst_contig_bytes);  in _scif_rma_list_dma_copy_aligned()
1170 		loop_len = min(loop_len, remaining_len);  in _scif_rma_list_dma_copy_aligned()
1171 		if (work->ordered && !(remaining_len - loop_len)) {  in _scif_rma_list_dma_copy_aligned()
1180 							 loop_len - 1,  in _scif_rma_list_dma_copy_aligned()
1191 			src_offset += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1192 			dst_offset += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1193 			src_dma_addr += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1194 			dst_dma_addr += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1195 			remaining_len -= (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1196 			loop_len = remaining_len;  in _scif_rma_list_dma_copy_aligned()
1200 					src_dma_addr, loop_len, 0);  in _scif_rma_list_dma_copy_aligned()
1213 					src_dma_addr, loop_len, 0);  in _scif_rma_list_dma_copy_aligned()
1224 		src_offset += loop_len;  in _scif_rma_list_dma_copy_aligned()
1225 		dst_offset += loop_len;  in _scif_rma_list_dma_copy_aligned()
1226 		remaining_len -= loop_len;  in _scif_rma_list_dma_copy_aligned()
1245 	size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;  in scif_rma_list_dma_copy_aligned()  local
1267 		loop_len = L1_CACHE_BYTES - src_cache_off;  in scif_rma_list_dma_copy_aligned()
1268 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_aligned()
1273 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1276 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1282 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1285 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1289 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1293 			scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1294 						remaining_len == loop_len ?  in scif_rma_list_dma_copy_aligned()
1297 			scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1298 						  remaining_len == loop_len ?  in scif_rma_list_dma_copy_aligned()
1301 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1303 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1304 		src_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1305 		dst_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1306 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1336 		loop_len = min(src_contig_bytes, dst_contig_bytes);  in scif_rma_list_dma_copy_aligned()
1337 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_aligned()
1339 		    !(remaining_len - loop_len)) {  in scif_rma_list_dma_copy_aligned()
1351 							 loop_len -  in scif_rma_list_dma_copy_aligned()
1364 			src_offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1365 			dst_offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1366 			src_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1367 			dst_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1368 			remaining_len -= (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1369 			loop_len = remaining_len;  in scif_rma_list_dma_copy_aligned()
1374 							 loop_len, 0);  in scif_rma_list_dma_copy_aligned()
1388 							 loop_len, 0);  in scif_rma_list_dma_copy_aligned()
1400 		src_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1401 		dst_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1402 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1406 		loop_len = remaining_len;  in scif_rma_list_dma_copy_aligned()
1428 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1431 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1438 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1441 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1445 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1450 			scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1454 						  loop_len, work->ordered);  in scif_rma_list_dma_copy_aligned()
1456 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1459 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1460 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1478 	size_t loop_len, remaining_len;  in scif_rma_list_cpu_copy()  local
1495 		loop_len = min(PAGE_SIZE -  in scif_rma_list_cpu_copy()
1501 						 loop_len);  in scif_rma_list_cpu_copy()
1504 						  loop_len,  in scif_rma_list_cpu_copy()
1514 						 loop_len);  in scif_rma_list_cpu_copy()
1517 						  loop_len,  in scif_rma_list_cpu_copy()
1522 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1528 			memcpy(dst_virt, src_virt, loop_len);  in scif_rma_list_cpu_copy()
1532 					    src_virt, loop_len);  in scif_rma_list_cpu_copy()
1536 					      loop_len);  in scif_rma_list_cpu_copy()
1539 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1542 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1544 		src_offset += loop_len;  in scif_rma_list_cpu_copy()
1545 		dst_offset += loop_len;  in scif_rma_list_cpu_copy()
1546 		remaining_len -= loop_len;  in scif_rma_list_cpu_copy()