173 static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args,  in __rds_rdma_map()  argument
197 	nr_pages = rds_pages_in_vec(&args->vec);  in __rds_rdma_map()
204 		args->vec.addr, args->vec.bytes, nr_pages);  in __rds_rdma_map()
224 	if (args->flags & RDS_RDMA_USE_ONCE)  in __rds_rdma_map()
226 	if (args->flags & RDS_RDMA_INVALIDATE)  in __rds_rdma_map()
228 	if (args->flags & RDS_RDMA_READWRITE)  in __rds_rdma_map()
241 	ret = rds_pin_pages(args->vec.addr, nr_pages, pages, 1);  in __rds_rdma_map()
278 	       mr->r_key, (void *)(unsigned long) args->cookie_addr);  in __rds_rdma_map()
284 	cookie = rds_rdma_make_cookie(mr->r_key, args->vec.addr & ~PAGE_MASK);  in __rds_rdma_map()
288 	if (args->cookie_addr && put_user(cookie, (u64 __user *)(unsigned long) args->cookie_addr)) {  in __rds_rdma_map()
317 	struct rds_get_mr_args args;  in rds_get_mr()  local
322 	if (copy_from_user(&args, (struct rds_get_mr_args __user *)optval,  in rds_get_mr()
326 	return __rds_rdma_map(rs, &args, NULL, NULL);  in rds_get_mr()
331 	struct rds_get_mr_for_dest_args args;  in rds_get_mr_for_dest()  local
337 	if (copy_from_user(&args, (struct rds_get_mr_for_dest_args __user *)optval,  in rds_get_mr_for_dest()
346 	new_args.vec = args.vec;  in rds_get_mr_for_dest()
347 	new_args.cookie_addr = args.cookie_addr;  in rds_get_mr_for_dest()
348 	new_args.flags = args.flags;  in rds_get_mr_for_dest()
358 	struct rds_free_mr_args args;  in rds_free_mr()  local
365 	if (copy_from_user(&args, (struct rds_free_mr_args __user *)optval,  in rds_free_mr()
370 	if (args.cookie == 0) {  in rds_free_mr()
382 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);  in rds_free_mr()
386 		if (args.flags & RDS_RDMA_INVALIDATE)  in rds_free_mr()
509 int rds_rdma_extra_size(struct rds_rdma_args *args)  in rds_rdma_extra_size()  argument
517 	local_vec = (struct rds_iovec __user *)(unsigned long) args->local_vec_addr;  in rds_rdma_extra_size()
520 	for (i = 0; i < args->nr_local; i++) {  in rds_rdma_extra_size()
549 	struct rds_rdma_args *args;  in rds_cmsg_rdma_args()  local
563 	args = CMSG_DATA(cmsg);  in rds_cmsg_rdma_args()
570 	if (args->nr_local > UIO_MAXIOV) {  in rds_cmsg_rdma_args()
576 	iov_size = args->nr_local * sizeof(struct rds_iovec);  in rds_cmsg_rdma_args()
577 	if (args->nr_local > UIO_FASTIOV) {  in rds_cmsg_rdma_args()
585 …if (copy_from_user(iovs, (struct rds_iovec __user *)(unsigned long) args->local_vec_addr, iov_size…  in rds_cmsg_rdma_args()
590 	nr_pages = rds_rdma_pages(iovs, args->nr_local);  in rds_cmsg_rdma_args()
602 	op->op_write = !!(args->flags & RDS_RDMA_READWRITE);  in rds_cmsg_rdma_args()
603 	op->op_fence = !!(args->flags & RDS_RDMA_FENCE);  in rds_cmsg_rdma_args()
604 	op->op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);  in rds_cmsg_rdma_args()
605 	op->op_silent = !!(args->flags & RDS_RDMA_SILENT);  in rds_cmsg_rdma_args()
626 		op->op_notifier->n_user_token = args->user_token;  in rds_cmsg_rdma_args()
637 	op->op_rkey = rds_rdma_cookie_key(args->cookie);  in rds_cmsg_rdma_args()
638 	op->op_remote_addr = args->remote_vec.addr + rds_rdma_cookie_offset(args->cookie);  in rds_cmsg_rdma_args()
643 	       (unsigned long long)args->nr_local,  in rds_cmsg_rdma_args()
644 	       (unsigned long long)args->remote_vec.addr,  in rds_cmsg_rdma_args()
647 	for (i = 0; i < args->nr_local; i++) {  in rds_cmsg_rdma_args()
686 	if (nr_bytes > args->remote_vec.bytes) {  in rds_cmsg_rdma_args()
689 				(unsigned int) args->remote_vec.bytes);  in rds_cmsg_rdma_args()
771 	struct rds_atomic_args *args;  in rds_cmsg_atomic()  local
778 	args = CMSG_DATA(cmsg);  in rds_cmsg_atomic()
784 		rm->atomic.op_m_fadd.add = args->fadd.add;  in rds_cmsg_atomic()
789 		rm->atomic.op_m_fadd.add = args->m_fadd.add;  in rds_cmsg_atomic()
790 		rm->atomic.op_m_fadd.nocarry_mask = args->m_fadd.nocarry_mask;  in rds_cmsg_atomic()
794 		rm->atomic.op_m_cswp.compare = args->cswp.compare;  in rds_cmsg_atomic()
795 		rm->atomic.op_m_cswp.swap = args->cswp.swap;  in rds_cmsg_atomic()
801 		rm->atomic.op_m_cswp.compare = args->m_cswp.compare;  in rds_cmsg_atomic()
802 		rm->atomic.op_m_cswp.swap = args->m_cswp.swap;  in rds_cmsg_atomic()
803 		rm->atomic.op_m_cswp.compare_mask = args->m_cswp.compare_mask;  in rds_cmsg_atomic()
804 		rm->atomic.op_m_cswp.swap_mask = args->m_cswp.swap_mask;  in rds_cmsg_atomic()
810 	rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);  in rds_cmsg_atomic()
811 	rm->atomic.op_silent = !!(args->flags & RDS_RDMA_SILENT);  in rds_cmsg_atomic()
821 	if (args->local_addr & 0x7) {  in rds_cmsg_atomic()
826 	ret = rds_pin_pages(args->local_addr, 1, &page, 1);  in rds_cmsg_atomic()
831 	sg_set_page(rm->atomic.op_sg, page, 8, offset_in_page(args->local_addr));  in rds_cmsg_atomic()
845 		rm->atomic.op_notifier->n_user_token = args->user_token;  in rds_cmsg_atomic()
849 	rm->atomic.op_rkey = rds_rdma_cookie_key(args->cookie);  in rds_cmsg_atomic()
850 	rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie);  in rds_cmsg_atomic()