424 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm)  in rds_send_sndbuf_remove()  argument
428 	assert_spin_locked(&rs->rs_lock);  in rds_send_sndbuf_remove()
430 	BUG_ON(rs->rs_snd_bytes < len);  in rds_send_sndbuf_remove()
431 	rs->rs_snd_bytes -= len;  in rds_send_sndbuf_remove()
433 	if (rs->rs_snd_bytes == 0)  in rds_send_sndbuf_remove()
453 	struct rds_sock *rs = NULL;  in rds_rdma_send_complete()  local
464 		rs = rm->m_rs;  in rds_rdma_send_complete()
465 		sock_hold(rds_rs_to_sk(rs));  in rds_rdma_send_complete()
468 		spin_lock(&rs->rs_lock);  in rds_rdma_send_complete()
469 		list_add_tail(&notifier->n_list, &rs->rs_notify_queue);  in rds_rdma_send_complete()
470 		spin_unlock(&rs->rs_lock);  in rds_rdma_send_complete()
477 	if (rs) {  in rds_rdma_send_complete()
478 		rds_wake_sk_sleep(rs);  in rds_rdma_send_complete()
479 		sock_put(rds_rs_to_sk(rs));  in rds_rdma_send_complete()
489 	struct rds_sock *rs = NULL;  in rds_atomic_send_complete()  local
500 		rs = rm->m_rs;  in rds_atomic_send_complete()
501 		sock_hold(rds_rs_to_sk(rs));  in rds_atomic_send_complete()
504 		spin_lock(&rs->rs_lock);  in rds_atomic_send_complete()
505 		list_add_tail(&notifier->n_list, &rs->rs_notify_queue);  in rds_atomic_send_complete()
506 		spin_unlock(&rs->rs_lock);  in rds_atomic_send_complete()
513 	if (rs) {  in rds_atomic_send_complete()
514 		rds_wake_sk_sleep(rs);  in rds_atomic_send_complete()
515 		sock_put(rds_rs_to_sk(rs));  in rds_atomic_send_complete()
526 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status)  in __rds_send_complete()  argument
534 		list_add_tail(&ro->op_notifier->n_list, &rs->rs_notify_queue);  in __rds_send_complete()
541 		list_add_tail(&ao->op_notifier->n_list, &rs->rs_notify_queue);  in __rds_send_complete()
595 	struct rds_sock *rs = NULL;  in rds_send_remove_from_sock()  local
619 		if (rs != rm->m_rs) {  in rds_send_remove_from_sock()
620 			if (rs) {  in rds_send_remove_from_sock()
621 				rds_wake_sk_sleep(rs);  in rds_send_remove_from_sock()
622 				sock_put(rds_rs_to_sk(rs));  in rds_send_remove_from_sock()
624 			rs = rm->m_rs;  in rds_send_remove_from_sock()
625 			if (rs)  in rds_send_remove_from_sock()
626 				sock_hold(rds_rs_to_sk(rs));  in rds_send_remove_from_sock()
628 		if (!rs)  in rds_send_remove_from_sock()
630 		spin_lock(&rs->rs_lock);  in rds_send_remove_from_sock()
637 			rds_send_sndbuf_remove(rs, rm);  in rds_send_remove_from_sock()
643 						&rs->rs_notify_queue);  in rds_send_remove_from_sock()
651 		spin_unlock(&rs->rs_lock);  in rds_send_remove_from_sock()
660 	if (rs) {  in rds_send_remove_from_sock()
661 		rds_wake_sk_sleep(rs);  in rds_send_remove_from_sock()
662 		sock_put(rds_rs_to_sk(rs));  in rds_send_remove_from_sock()
702 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest)  in rds_send_drop_to()  argument
710 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_send_drop_to()
712 	list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) {  in rds_send_drop_to()
718 		rds_send_sndbuf_remove(rs, rm);  in rds_send_drop_to()
725 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_send_drop_to()
757 		spin_lock(&rs->rs_lock);  in rds_send_drop_to()
758 		__rds_send_complete(rs, rm, RDS_RDMA_CANCELED);  in rds_send_drop_to()
759 		spin_unlock(&rs->rs_lock);  in rds_send_drop_to()
767 	rds_wake_sk_sleep(rs);  in rds_send_drop_to()
783 static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn,  in rds_send_queue_rm()  argument
797 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_send_queue_rm()
807 	if (rs->rs_snd_bytes < rds_sk_sndbuf(rs)) {  in rds_send_queue_rm()
808 		rs->rs_snd_bytes += len;  in rds_send_queue_rm()
815 		if (rs->rs_snd_bytes >= rds_sk_sndbuf(rs) / 2)  in rds_send_queue_rm()
818 		list_add_tail(&rm->m_sock_item, &rs->rs_send_queue);  in rds_send_queue_rm()
821 		rm->m_rs = rs;  in rds_send_queue_rm()
836 			 rm, len, rs, rs->rs_snd_bytes,  in rds_send_queue_rm()
842 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_send_queue_rm()
904 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_send()  argument
922 			ret = rds_cmsg_rdma_args(rs, rm, cmsg);  in rds_cmsg_send()
926 			ret = rds_cmsg_rdma_dest(rs, rm, cmsg);  in rds_cmsg_send()
930 			ret = rds_cmsg_rdma_map(rs, rm, cmsg);  in rds_cmsg_send()
938 			ret = rds_cmsg_atomic(rs, rm, cmsg);  in rds_cmsg_send()
955 	struct rds_sock *rs = rds_sk_to_rs(sk);  in rds_sendmsg()  local
984 		daddr = rs->rs_conn_addr;  in rds_sendmsg()
985 		dport = rs->rs_conn_port;  in rds_sendmsg()
990 	if (daddr == 0 || rs->rs_bound_addr == 0) {  in rds_sendmsg()
1025 	if (rs->rs_conn && rs->rs_conn->c_faddr == daddr)  in rds_sendmsg()
1026 		conn = rs->rs_conn;  in rds_sendmsg()
1028 		conn = rds_conn_create_outgoing(rs->rs_bound_addr, daddr,  in rds_sendmsg()
1029 					rs->rs_transport,  in rds_sendmsg()
1035 		rs->rs_conn = conn;  in rds_sendmsg()
1039 	ret = rds_cmsg_send(rs, rm, msg, &allocated_mr);  in rds_sendmsg()
1059 	ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs);  in rds_sendmsg()
1061 		rs->rs_seen_congestion = 1;  in rds_sendmsg()
1065 	while (!rds_send_queue_rm(rs, conn, rm, rs->rs_bound_port,  in rds_sendmsg()
1069 		if (payload_len > rds_sk_sndbuf(rs)) {  in rds_sendmsg()
1079 					rds_send_queue_rm(rs, conn, rm,  in rds_sendmsg()
1080 							  rs->rs_bound_port,  in rds_sendmsg()
1111 		rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1);  in rds_sendmsg()