41 void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,  in rds_inc_init()  argument
44 	atomic_set(&inc->i_refcount, 1);  in rds_inc_init()
45 	INIT_LIST_HEAD(&inc->i_item);  in rds_inc_init()
46 	inc->i_conn = conn;  in rds_inc_init()
47 	inc->i_saddr = saddr;  in rds_inc_init()
48 	inc->i_rdma_cookie = 0;  in rds_inc_init()
52 static void rds_inc_addref(struct rds_incoming *inc)  in rds_inc_addref()  argument
54 	rdsdebug("addref inc %p ref %d\n", inc, atomic_read(&inc->i_refcount));  in rds_inc_addref()
55 	atomic_inc(&inc->i_refcount);  in rds_inc_addref()
58 void rds_inc_put(struct rds_incoming *inc)  in rds_inc_put()  argument
60 	rdsdebug("put inc %p ref %d\n", inc, atomic_read(&inc->i_refcount));  in rds_inc_put()
61 	if (atomic_dec_and_test(&inc->i_refcount)) {  in rds_inc_put()
62 		BUG_ON(!list_empty(&inc->i_item));  in rds_inc_put()
64 		inc->i_conn->c_trans->inc_free(inc);  in rds_inc_put()
108 static void rds_recv_incoming_exthdrs(struct rds_incoming *inc, struct rds_sock *rs)  in rds_recv_incoming_exthdrs()  argument
110 	struct rds_header *hdr = &inc->i_hdr;  in rds_recv_incoming_exthdrs()
132 			inc->i_rdma_cookie = rds_rdma_make_cookie(  in rds_recv_incoming_exthdrs()
158 		       struct rds_incoming *inc, gfp_t gfp)  in rds_recv_incoming()  argument
164 	inc->i_conn = conn;  in rds_recv_incoming()
165 	inc->i_rx_jiffies = jiffies;  in rds_recv_incoming()
170 		 inc,  in rds_recv_incoming()
171 		 (unsigned long long)be64_to_cpu(inc->i_hdr.h_sequence),  in rds_recv_incoming()
172 		 be32_to_cpu(inc->i_hdr.h_len),  in rds_recv_incoming()
173 		 be16_to_cpu(inc->i_hdr.h_sport),  in rds_recv_incoming()
174 		 be16_to_cpu(inc->i_hdr.h_dport),  in rds_recv_incoming()
175 		 inc->i_hdr.h_flags,  in rds_recv_incoming()
176 		 inc->i_rx_jiffies);  in rds_recv_incoming()
198 	if (be64_to_cpu(inc->i_hdr.h_sequence) < conn->c_next_rx_seq &&  in rds_recv_incoming()
199 	    (inc->i_hdr.h_flags & RDS_FLAG_RETRANSMITTED)) {  in rds_recv_incoming()
203 	conn->c_next_rx_seq = be64_to_cpu(inc->i_hdr.h_sequence) + 1;  in rds_recv_incoming()
205 	if (rds_sysctl_ping_enable && inc->i_hdr.h_dport == 0) {  in rds_recv_incoming()
207 		rds_send_pong(conn, inc->i_hdr.h_sport);  in rds_recv_incoming()
211 	rs = rds_find_bound(daddr, inc->i_hdr.h_dport);  in rds_recv_incoming()
218 	rds_recv_incoming_exthdrs(inc, rs);  in rds_recv_incoming()
226 		rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs);  in rds_recv_incoming()
228 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_recv_incoming()
229 				      be32_to_cpu(inc->i_hdr.h_len),  in rds_recv_incoming()
230 				      inc->i_hdr.h_dport);  in rds_recv_incoming()
231 		rds_inc_addref(inc);  in rds_recv_incoming()
232 		list_add_tail(&inc->i_item, &rs->rs_recv_queue);  in rds_recv_incoming()
249 static int rds_next_incoming(struct rds_sock *rs, struct rds_incoming **inc)  in rds_next_incoming()  argument
253 	if (!*inc) {  in rds_next_incoming()
256 			*inc = list_entry(rs->rs_recv_queue.next,  in rds_next_incoming()
259 			rds_inc_addref(*inc);  in rds_next_incoming()
264 	return *inc != NULL;  in rds_next_incoming()
267 static int rds_still_queued(struct rds_sock *rs, struct rds_incoming *inc,  in rds_still_queued()  argument
275 	if (!list_empty(&inc->i_item)) {  in rds_still_queued()
279 			rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_still_queued()
280 					      -be32_to_cpu(inc->i_hdr.h_len),  in rds_still_queued()
281 					      inc->i_hdr.h_dport);  in rds_still_queued()
282 			list_del_init(&inc->i_item);  in rds_still_queued()
283 			rds_inc_put(inc);  in rds_still_queued()
288 	rdsdebug("inc %p rs %p still %d dropped %d\n", inc, rs, ret, drop);  in rds_still_queued()
384 static int rds_cmsg_recv(struct rds_incoming *inc, struct msghdr *msg)  in rds_cmsg_recv()  argument
388 	if (inc->i_rdma_cookie) {  in rds_cmsg_recv()
390 				sizeof(inc->i_rdma_cookie), &inc->i_rdma_cookie);  in rds_cmsg_recv()
406 	struct rds_incoming *inc = NULL;  in rds_recvmsg()  local
429 		if (!rds_next_incoming(rs, &inc)) {  in rds_recvmsg()
438 					 rds_next_incoming(rs, &inc)), timeo);  in rds_recvmsg()
439 			rdsdebug("recvmsg woke inc %p timeo %ld\n", inc,  in rds_recvmsg()
450 		rdsdebug("copying inc %p from %pI4:%u to user\n", inc,  in rds_recvmsg()
451 			 &inc->i_conn->c_faddr,  in rds_recvmsg()
452 			 ntohs(inc->i_hdr.h_sport));  in rds_recvmsg()
454 		ret = inc->i_conn->c_trans->inc_copy_to_user(inc, &msg->msg_iter);  in rds_recvmsg()
463 		if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) {  in rds_recvmsg()
464 			rds_inc_put(inc);  in rds_recvmsg()
465 			inc = NULL;  in rds_recvmsg()
471 		if (ret < be32_to_cpu(inc->i_hdr.h_len)) {  in rds_recvmsg()
473 				ret = be32_to_cpu(inc->i_hdr.h_len);  in rds_recvmsg()
477 		if (rds_cmsg_recv(inc, msg)) {  in rds_recvmsg()
486 			sin->sin_port = inc->i_hdr.h_sport;  in rds_recvmsg()
487 			sin->sin_addr.s_addr = inc->i_saddr;  in rds_recvmsg()
494 	if (inc)  in rds_recvmsg()
495 		rds_inc_put(inc);  in rds_recvmsg()
509 	struct rds_incoming *inc, *tmp;  in rds_clear_recv_queue()  local
513 	list_for_each_entry_safe(inc, tmp, &rs->rs_recv_queue, i_item) {  in rds_clear_recv_queue()
514 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_clear_recv_queue()
515 				      -be32_to_cpu(inc->i_hdr.h_len),  in rds_clear_recv_queue()
516 				      inc->i_hdr.h_dport);  in rds_clear_recv_queue()
517 		list_del_init(&inc->i_item);  in rds_clear_recv_queue()
518 		rds_inc_put(inc);  in rds_clear_recv_queue()
527 void rds_inc_info_copy(struct rds_incoming *inc,  in rds_inc_info_copy()  argument
533 	minfo.seq = be64_to_cpu(inc->i_hdr.h_sequence);  in rds_inc_info_copy()
534 	minfo.len = be32_to_cpu(inc->i_hdr.h_len);  in rds_inc_info_copy()
539 		minfo.lport = inc->i_hdr.h_dport;  in rds_inc_info_copy()
540 		minfo.fport = inc->i_hdr.h_sport;  in rds_inc_info_copy()
544 		minfo.lport = inc->i_hdr.h_sport;  in rds_inc_info_copy()
545 		minfo.fport = inc->i_hdr.h_dport;  in rds_inc_info_copy()