98 	struct rds_sock *rs = mr->r_sock;  in rds_destroy_mr()  local
108 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_destroy_mr()
110 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_destroy_mr()
113 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_destroy_mr()
129 void rds_rdma_drop_keys(struct rds_sock *rs)  in rds_rdma_drop_keys()  argument
136 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
137 	while ((node = rb_first(&rs->rs_rdma_keys))) {  in rds_rdma_drop_keys()
139 		if (mr->r_trans == rs->rs_transport)  in rds_rdma_drop_keys()
141 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_drop_keys()
143 		spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
146 		spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
148 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
150 	if (rs->rs_transport && rs->rs_transport->flush_mrs)  in rds_rdma_drop_keys()
151 		rs->rs_transport->flush_mrs();  in rds_rdma_drop_keys()
173 static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args,  in __rds_rdma_map()  argument
187 	if (rs->rs_bound_addr == 0) {  in __rds_rdma_map()
192 	if (!rs->rs_transport->get_mr) {  in __rds_rdma_map()
221 	mr->r_trans = rs->rs_transport;  in __rds_rdma_map()
222 	mr->r_sock = rs;  in __rds_rdma_map()
264 	trans_private = rs->rs_transport->get_mr(sg, nents, rs,  in __rds_rdma_map()
295 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in __rds_rdma_map()
296 	found = rds_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr);  in __rds_rdma_map()
297 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in __rds_rdma_map()
315 int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen)  in rds_get_mr()  argument
326 	return __rds_rdma_map(rs, &args, NULL, NULL);  in rds_get_mr()
329 int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen)  in rds_get_mr_for_dest()  argument
350 	return __rds_rdma_map(rs, &new_args, NULL, NULL);  in rds_get_mr_for_dest()
356 int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen)  in rds_free_mr()  argument
371 		if (!rs->rs_transport || !rs->rs_transport->flush_mrs)  in rds_free_mr()
373 		rs->rs_transport->flush_mrs();  in rds_free_mr()
381 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_free_mr()
382 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);  in rds_free_mr()
384 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_free_mr()
389 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_free_mr()
409 void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force)  in rds_rdma_unuse()  argument
415 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
416 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_rdma_unuse()
419 		spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
424 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_unuse()
428 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
547 int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_args()  argument
566 	if (rs->rs_bound_addr == 0) {  in rds_cmsg_rdma_args()
579 		iovs = sock_kmalloc(rds_rs_to_sk(rs), iov_size, GFP_KERNEL);  in rds_cmsg_rdma_args()
608 	op->op_recverr = rs->rs_recverr;  in rds_cmsg_rdma_args()
653 		rs->rs_user_addr = iov->addr;  in rds_cmsg_rdma_args()
654 		rs->rs_user_bytes = iov->bytes;  in rds_cmsg_rdma_args()
700 		sock_kfree_s(rds_rs_to_sk(rs), iovs, iov_size);  in rds_cmsg_rdma_args()
715 int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_dest()  argument
736 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_cmsg_rdma_dest()
737 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_cmsg_rdma_dest()
742 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_cmsg_rdma_dest()
757 int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_map()  argument
764 	return __rds_rdma_map(rs, CMSG_DATA(cmsg), &rm->m_rdma_cookie, &rm->rdma.op_rdma_mr);  in rds_cmsg_rdma_map()
770 int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_atomic()  argument
816 	rm->atomic.op_recverr = rs->rs_recverr;  in rds_cmsg_atomic()