66 	struct rds_message *rm, *tmp;  in rds_send_reset()  local
70 		rm = conn->c_xmit_rm;  in rds_send_reset()
76 		rds_message_unmapped(rm);  in rds_send_reset()
77 		rds_message_put(rm);  in rds_send_reset()
94 	list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {  in rds_send_reset()
95 		set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);  in rds_send_reset()
96 		set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags);  in rds_send_reset()
137 	struct rds_message *rm;  in rds_send_xmit()  local
192 		rm = conn->c_xmit_rm;  in rds_send_xmit()
198 		if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) {  in rds_send_xmit()
199 			rm = rds_cong_update_alloc(conn);  in rds_send_xmit()
200 			if (IS_ERR(rm)) {  in rds_send_xmit()
201 				ret = PTR_ERR(rm);  in rds_send_xmit()
204 			rm->data.op_active = 1;  in rds_send_xmit()
206 			conn->c_xmit_rm = rm;  in rds_send_xmit()
216 		if (!rm) {  in rds_send_xmit()
232 				rm = list_entry(conn->c_send_queue.next,  in rds_send_xmit()
235 				rds_message_addref(rm);  in rds_send_xmit()
241 				list_move_tail(&rm->m_conn_item, &conn->c_retrans);  in rds_send_xmit()
246 			if (!rm)  in rds_send_xmit()
256 			if (rm->rdma.op_active &&  in rds_send_xmit()
257 			    test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags)) {  in rds_send_xmit()
259 				if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags))  in rds_send_xmit()
260 					list_move(&rm->m_conn_item, &to_be_dropped);  in rds_send_xmit()
266 			len = ntohl(rm->m_inc.i_hdr.h_len);  in rds_send_xmit()
269 				__set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);  in rds_send_xmit()
279 			conn->c_xmit_rm = rm;  in rds_send_xmit()
283 		if (rm->rdma.op_active && !conn->c_xmit_rdma_sent) {  in rds_send_xmit()
284 			rm->m_final_op = &rm->rdma;  in rds_send_xmit()
285 			ret = conn->c_trans->xmit_rdma(conn, &rm->rdma);  in rds_send_xmit()
292 			set_bit(RDS_MSG_MAPPED, &rm->m_flags);  in rds_send_xmit()
295 		if (rm->atomic.op_active && !conn->c_xmit_atomic_sent) {  in rds_send_xmit()
296 			rm->m_final_op = &rm->atomic;  in rds_send_xmit()
297 			ret = conn->c_trans->xmit_atomic(conn, &rm->atomic);  in rds_send_xmit()
304 			set_bit(RDS_MSG_MAPPED, &rm->m_flags);  in rds_send_xmit()
314 		if (rm->data.op_nents == 0) {  in rds_send_xmit()
318 			ops_present = (rm->atomic.op_active || rm->rdma.op_active);  in rds_send_xmit()
319 			if (rm->atomic.op_active && !rm->atomic.op_silent)  in rds_send_xmit()
321 			if (rm->rdma.op_active && !rm->rdma.op_silent)  in rds_send_xmit()
325 			    && !rm->m_rdma_cookie)  in rds_send_xmit()
326 				rm->data.op_active = 0;  in rds_send_xmit()
329 		if (rm->data.op_active && !conn->c_xmit_data_sent) {  in rds_send_xmit()
330 			rm->m_final_op = &rm->data;  in rds_send_xmit()
331 			ret = conn->c_trans->xmit(conn, rm,  in rds_send_xmit()
346 			sg = &rm->data.op_sg[conn->c_xmit_sg];  in rds_send_xmit()
357 					       conn->c_xmit_sg == rm->data.op_nents);  in rds_send_xmit()
362 			    (conn->c_xmit_sg == rm->data.op_nents))  in rds_send_xmit()
371 		if (!rm->data.op_active || conn->c_xmit_data_sent) {  in rds_send_xmit()
380 			rds_message_put(rm);  in rds_send_xmit()
392 		list_for_each_entry(rm, &to_be_dropped, m_conn_item)  in rds_send_xmit()
393 			rds_message_put(rm);  in rds_send_xmit()
424 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm)  in rds_send_sndbuf_remove()  argument
426 	u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len);  in rds_send_sndbuf_remove()
437 static inline int rds_send_is_acked(struct rds_message *rm, u64 ack,  in rds_send_is_acked()  argument
441 		return is_acked(rm, ack);  in rds_send_is_acked()
442 	return be64_to_cpu(rm->m_inc.i_hdr.h_sequence) <= ack;  in rds_send_is_acked()
451 void rds_rdma_send_complete(struct rds_message *rm, int status)  in rds_rdma_send_complete()  argument
458 	spin_lock_irqsave(&rm->m_rs_lock, flags);  in rds_rdma_send_complete()
460 	ro = &rm->rdma;  in rds_rdma_send_complete()
461 	if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) &&  in rds_rdma_send_complete()
464 		rs = rm->m_rs;  in rds_rdma_send_complete()
475 	spin_unlock_irqrestore(&rm->m_rs_lock, flags);  in rds_rdma_send_complete()
487 void rds_atomic_send_complete(struct rds_message *rm, int status)  in rds_atomic_send_complete()  argument
494 	spin_lock_irqsave(&rm->m_rs_lock, flags);  in rds_atomic_send_complete()
496 	ao = &rm->atomic;  in rds_atomic_send_complete()
497 	if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags)  in rds_atomic_send_complete()
500 		rs = rm->m_rs;  in rds_atomic_send_complete()
511 	spin_unlock_irqrestore(&rm->m_rs_lock, flags);  in rds_atomic_send_complete()
526 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status)  in __rds_send_complete()  argument
531 	ro = &rm->rdma;  in __rds_send_complete()
538 	ao = &rm->atomic;  in __rds_send_complete()
556 	struct rds_message *rm, *tmp, *found = NULL;  in rds_send_get_message()  local
561 	list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {  in rds_send_get_message()
562 		if (&rm->rdma == op) {  in rds_send_get_message()
563 			atomic_inc(&rm->m_refcount);  in rds_send_get_message()
564 			found = rm;  in rds_send_get_message()
569 	list_for_each_entry_safe(rm, tmp, &conn->c_send_queue, m_conn_item) {  in rds_send_get_message()
570 		if (&rm->rdma == op) {  in rds_send_get_message()
571 			atomic_inc(&rm->m_refcount);  in rds_send_get_message()
572 			found = rm;  in rds_send_get_message()
596 	struct rds_message *rm;  in rds_send_remove_from_sock()  local
601 		rm = list_entry(messages->next, struct rds_message,  in rds_send_remove_from_sock()
603 		list_del_init(&rm->m_conn_item);  in rds_send_remove_from_sock()
615 		spin_lock_irqsave(&rm->m_rs_lock, flags);  in rds_send_remove_from_sock()
616 		if (!test_bit(RDS_MSG_ON_SOCK, &rm->m_flags))  in rds_send_remove_from_sock()
619 		if (rs != rm->m_rs) {  in rds_send_remove_from_sock()
624 			rs = rm->m_rs;  in rds_send_remove_from_sock()
632 		if (test_and_clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) {  in rds_send_remove_from_sock()
633 			struct rm_rdma_op *ro = &rm->rdma;  in rds_send_remove_from_sock()
636 			list_del_init(&rm->m_sock_item);  in rds_send_remove_from_sock()
637 			rds_send_sndbuf_remove(rs, rm);  in rds_send_remove_from_sock()
646 				rm->rdma.op_notifier = NULL;  in rds_send_remove_from_sock()
649 			rm->m_rs = NULL;  in rds_send_remove_from_sock()
654 		spin_unlock_irqrestore(&rm->m_rs_lock, flags);  in rds_send_remove_from_sock()
655 		rds_message_put(rm);  in rds_send_remove_from_sock()
657 			rds_message_put(rm);  in rds_send_remove_from_sock()
677 	struct rds_message *rm, *tmp;  in rds_send_drop_acked()  local
683 	list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {  in rds_send_drop_acked()
684 		if (!rds_send_is_acked(rm, ack, is_acked))  in rds_send_drop_acked()
687 		list_move(&rm->m_conn_item, &list);  in rds_send_drop_acked()
688 		clear_bit(RDS_MSG_ON_CONN, &rm->m_flags);  in rds_send_drop_acked()
704 	struct rds_message *rm, *tmp;  in rds_send_drop_to()  local
712 	list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) {  in rds_send_drop_to()
713 		if (dest && (dest->sin_addr.s_addr != rm->m_daddr ||  in rds_send_drop_to()
714 			     dest->sin_port != rm->m_inc.i_hdr.h_dport))  in rds_send_drop_to()
717 		list_move(&rm->m_sock_item, &list);  in rds_send_drop_to()
718 		rds_send_sndbuf_remove(rs, rm);  in rds_send_drop_to()
719 		clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags);  in rds_send_drop_to()
731 	list_for_each_entry(rm, &list, m_sock_item) {  in rds_send_drop_to()
733 		conn = rm->m_inc.i_conn;  in rds_send_drop_to()
741 		if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) {  in rds_send_drop_to()
743 			spin_lock_irqsave(&rm->m_rs_lock, flags);  in rds_send_drop_to()
744 			rm->m_rs = NULL;  in rds_send_drop_to()
745 			spin_unlock_irqrestore(&rm->m_rs_lock, flags);  in rds_send_drop_to()
748 		list_del_init(&rm->m_conn_item);  in rds_send_drop_to()
755 		spin_lock_irqsave(&rm->m_rs_lock, flags);  in rds_send_drop_to()
758 		__rds_send_complete(rs, rm, RDS_RDMA_CANCELED);  in rds_send_drop_to()
761 		rm->m_rs = NULL;  in rds_send_drop_to()
762 		spin_unlock_irqrestore(&rm->m_rs_lock, flags);  in rds_send_drop_to()
764 		rds_message_put(rm);  in rds_send_drop_to()
770 		rm = list_entry(list.next, struct rds_message, m_sock_item);  in rds_send_drop_to()
771 		list_del_init(&rm->m_sock_item);  in rds_send_drop_to()
773 		rds_message_wait(rm);  in rds_send_drop_to()
774 		rds_message_put(rm);  in rds_send_drop_to()
784 			     struct rds_message *rm, __be16 sport,  in rds_send_queue_rm()  argument
793 	len = be32_to_cpu(rm->m_inc.i_hdr.h_len);  in rds_send_queue_rm()
816 			__set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);  in rds_send_queue_rm()
818 		list_add_tail(&rm->m_sock_item, &rs->rs_send_queue);  in rds_send_queue_rm()
819 		set_bit(RDS_MSG_ON_SOCK, &rm->m_flags);  in rds_send_queue_rm()
820 		rds_message_addref(rm);  in rds_send_queue_rm()
821 		rm->m_rs = rs;  in rds_send_queue_rm()
825 		rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0);  in rds_send_queue_rm()
826 		rm->m_inc.i_conn = conn;  in rds_send_queue_rm()
827 		rds_message_addref(rm);  in rds_send_queue_rm()
830 		rm->m_inc.i_hdr.h_sequence = cpu_to_be64(conn->c_next_tx_seq++);  in rds_send_queue_rm()
831 		list_add_tail(&rm->m_conn_item, &conn->c_send_queue);  in rds_send_queue_rm()
832 		set_bit(RDS_MSG_ON_CONN, &rm->m_flags);  in rds_send_queue_rm()
836 			 rm, len, rs, rs->rs_snd_bytes,  in rds_send_queue_rm()
837 			 (unsigned long long)be64_to_cpu(rm->m_inc.i_hdr.h_sequence));  in rds_send_queue_rm()
904 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_send()  argument
922 			ret = rds_cmsg_rdma_args(rs, rm, cmsg);  in rds_cmsg_send()
926 			ret = rds_cmsg_rdma_dest(rs, rm, cmsg);  in rds_cmsg_send()
930 			ret = rds_cmsg_rdma_map(rs, rm, cmsg);  in rds_cmsg_send()
938 			ret = rds_cmsg_atomic(rs, rm, cmsg);  in rds_cmsg_send()
959 	struct rds_message *rm = NULL;  in rds_sendmsg()  local
1002 	rm = rds_message_alloc(ret, GFP_KERNEL);  in rds_sendmsg()
1003 	if (!rm) {  in rds_sendmsg()
1010 		rm->data.op_sg = rds_message_alloc_sgs(rm, ceil(payload_len, PAGE_SIZE));  in rds_sendmsg()
1011 		if (!rm->data.op_sg) {  in rds_sendmsg()
1015 		ret = rds_message_copy_from_user(rm, &msg->msg_iter);  in rds_sendmsg()
1019 	rm->data.op_active = 1;  in rds_sendmsg()
1021 	rm->m_daddr = daddr;  in rds_sendmsg()
1039 	ret = rds_cmsg_send(rs, rm, msg, &allocated_mr);  in rds_sendmsg()
1043 	if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) {  in rds_sendmsg()
1045 			       &rm->rdma, conn->c_trans->xmit_rdma);  in rds_sendmsg()
1050 	if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) {  in rds_sendmsg()
1052 			       &rm->atomic, conn->c_trans->xmit_atomic);  in rds_sendmsg()
1065 	while (!rds_send_queue_rm(rs, conn, rm, rs->rs_bound_port,  in rds_sendmsg()
1079 					rds_send_queue_rm(rs, conn, rm,  in rds_sendmsg()
1103 	rds_message_put(rm);  in rds_sendmsg()
1111 		rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1);  in rds_sendmsg()
1113 	if (rm)  in rds_sendmsg()
1114 		rds_message_put(rm);  in rds_sendmsg()
1124 	struct rds_message *rm;  in rds_send_pong()  local
1128 	rm = rds_message_alloc(0, GFP_ATOMIC);  in rds_send_pong()
1129 	if (!rm) {  in rds_send_pong()
1134 	rm->m_daddr = conn->c_faddr;  in rds_send_pong()
1135 	rm->data.op_active = 1;  in rds_send_pong()
1144 	list_add_tail(&rm->m_conn_item, &conn->c_send_queue);  in rds_send_pong()
1145 	set_bit(RDS_MSG_ON_CONN, &rm->m_flags);  in rds_send_pong()
1146 	rds_message_addref(rm);  in rds_send_pong()
1147 	rm->m_inc.i_conn = conn;  in rds_send_pong()
1149 	rds_message_populate_header(&rm->m_inc.i_hdr, 0, dport,  in rds_send_pong()
1160 	rds_message_put(rm);  in rds_send_pong()
1164 	if (rm)  in rds_send_pong()
1165 		rds_message_put(rm);  in rds_send_pong()