437 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm)  in rds_send_sndbuf_remove()  argument
441 	assert_spin_locked(&rs->rs_lock);  in rds_send_sndbuf_remove()
443 	BUG_ON(rs->rs_snd_bytes < len);  in rds_send_sndbuf_remove()
444 	rs->rs_snd_bytes -= len;  in rds_send_sndbuf_remove()
446 	if (rs->rs_snd_bytes == 0)  in rds_send_sndbuf_remove()
466 	struct rds_sock *rs = NULL;  in rds_rdma_send_complete()  local
477 		rs = rm->m_rs;  in rds_rdma_send_complete()
478 		sock_hold(rds_rs_to_sk(rs));  in rds_rdma_send_complete()
481 		spin_lock(&rs->rs_lock);  in rds_rdma_send_complete()
482 		list_add_tail(&notifier->n_list, &rs->rs_notify_queue);  in rds_rdma_send_complete()
483 		spin_unlock(&rs->rs_lock);  in rds_rdma_send_complete()
490 	if (rs) {  in rds_rdma_send_complete()
491 		rds_wake_sk_sleep(rs);  in rds_rdma_send_complete()
492 		sock_put(rds_rs_to_sk(rs));  in rds_rdma_send_complete()
502 	struct rds_sock *rs = NULL;  in rds_atomic_send_complete()  local
513 		rs = rm->m_rs;  in rds_atomic_send_complete()
514 		sock_hold(rds_rs_to_sk(rs));  in rds_atomic_send_complete()
517 		spin_lock(&rs->rs_lock);  in rds_atomic_send_complete()
518 		list_add_tail(&notifier->n_list, &rs->rs_notify_queue);  in rds_atomic_send_complete()
519 		spin_unlock(&rs->rs_lock);  in rds_atomic_send_complete()
526 	if (rs) {  in rds_atomic_send_complete()
527 		rds_wake_sk_sleep(rs);  in rds_atomic_send_complete()
528 		sock_put(rds_rs_to_sk(rs));  in rds_atomic_send_complete()
539 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status)  in __rds_send_complete()  argument
547 		list_add_tail(&ro->op_notifier->n_list, &rs->rs_notify_queue);  in __rds_send_complete()
554 		list_add_tail(&ao->op_notifier->n_list, &rs->rs_notify_queue);  in __rds_send_complete()
608 	struct rds_sock *rs = NULL;  in rds_send_remove_from_sock()  local
632 		if (rs != rm->m_rs) {  in rds_send_remove_from_sock()
633 			if (rs) {  in rds_send_remove_from_sock()
634 				rds_wake_sk_sleep(rs);  in rds_send_remove_from_sock()
635 				sock_put(rds_rs_to_sk(rs));  in rds_send_remove_from_sock()
637 			rs = rm->m_rs;  in rds_send_remove_from_sock()
638 			if (rs)  in rds_send_remove_from_sock()
639 				sock_hold(rds_rs_to_sk(rs));  in rds_send_remove_from_sock()
641 		if (!rs)  in rds_send_remove_from_sock()
643 		spin_lock(&rs->rs_lock);  in rds_send_remove_from_sock()
650 			rds_send_sndbuf_remove(rs, rm);  in rds_send_remove_from_sock()
656 						&rs->rs_notify_queue);  in rds_send_remove_from_sock()
664 		spin_unlock(&rs->rs_lock);  in rds_send_remove_from_sock()
673 	if (rs) {  in rds_send_remove_from_sock()
674 		rds_wake_sk_sleep(rs);  in rds_send_remove_from_sock()
675 		sock_put(rds_rs_to_sk(rs));  in rds_send_remove_from_sock()
715 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest)  in rds_send_drop_to()  argument
723 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_send_drop_to()
725 	list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) {  in rds_send_drop_to()
731 		rds_send_sndbuf_remove(rs, rm);  in rds_send_drop_to()
738 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_send_drop_to()
770 		spin_lock(&rs->rs_lock);  in rds_send_drop_to()
771 		__rds_send_complete(rs, rm, RDS_RDMA_CANCELED);  in rds_send_drop_to()
772 		spin_unlock(&rs->rs_lock);  in rds_send_drop_to()
780 	rds_wake_sk_sleep(rs);  in rds_send_drop_to()
794 		spin_lock(&rs->rs_lock);  in rds_send_drop_to()
795 		__rds_send_complete(rs, rm, RDS_RDMA_CANCELED);  in rds_send_drop_to()
796 		spin_unlock(&rs->rs_lock);  in rds_send_drop_to()
810 static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn,  in rds_send_queue_rm()  argument
824 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_send_queue_rm()
834 	if (rs->rs_snd_bytes < rds_sk_sndbuf(rs)) {  in rds_send_queue_rm()
835 		rs->rs_snd_bytes += len;  in rds_send_queue_rm()
842 		if (rs->rs_snd_bytes >= rds_sk_sndbuf(rs) / 2)  in rds_send_queue_rm()
845 		list_add_tail(&rm->m_sock_item, &rs->rs_send_queue);  in rds_send_queue_rm()
848 		rm->m_rs = rs;  in rds_send_queue_rm()
863 			 rm, len, rs, rs->rs_snd_bytes,  in rds_send_queue_rm()
869 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_send_queue_rm()
931 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_send()  argument
949 			ret = rds_cmsg_rdma_args(rs, rm, cmsg);  in rds_cmsg_send()
953 			ret = rds_cmsg_rdma_dest(rs, rm, cmsg);  in rds_cmsg_send()
957 			ret = rds_cmsg_rdma_map(rs, rm, cmsg);  in rds_cmsg_send()
965 			ret = rds_cmsg_atomic(rs, rm, cmsg);  in rds_cmsg_send()
982 	struct rds_sock *rs = rds_sk_to_rs(sk);  in rds_sendmsg()  local
1011 		daddr = rs->rs_conn_addr;  in rds_sendmsg()
1012 		dport = rs->rs_conn_port;  in rds_sendmsg()
1017 	if (daddr == 0 || rs->rs_bound_addr == 0) {  in rds_sendmsg()
1024 	if (payload_len > rds_sk_sndbuf(rs)) {  in rds_sendmsg()
1057 	if (rs->rs_conn && rs->rs_conn->c_faddr == daddr)  in rds_sendmsg()
1058 		conn = rs->rs_conn;  in rds_sendmsg()
1061 						rs->rs_bound_addr, daddr,  in rds_sendmsg()
1062 					rs->rs_transport,  in rds_sendmsg()
1068 		rs->rs_conn = conn;  in rds_sendmsg()
1072 	ret = rds_cmsg_send(rs, rm, msg, &allocated_mr);  in rds_sendmsg()
1092 	ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs);  in rds_sendmsg()
1094 		rs->rs_seen_congestion = 1;  in rds_sendmsg()
1098 	while (!rds_send_queue_rm(rs, conn, rm, rs->rs_bound_port,  in rds_sendmsg()
1108 					rds_send_queue_rm(rs, conn, rm,  in rds_sendmsg()
1109 							  rs->rs_bound_port,  in rds_sendmsg()
1141 		rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1);  in rds_sendmsg()