42 static void rds_iw_send_rdma_complete(struct rds_message *rm,  in rds_iw_send_rdma_complete()  argument
63 	rds_rdma_send_complete(rm, notify_status);  in rds_iw_send_rdma_complete()
81 	struct rds_message *rm = send->s_rm;  in rds_iw_send_unmap_rm()  local
83 	rdsdebug("ic %p send %p rm %p\n", ic, send, rm);  in rds_iw_send_unmap_rm()
86 		     rm->data.op_sg, rm->data.op_nents,  in rds_iw_send_unmap_rm()
89 	if (rm->rdma.op_active) {  in rds_iw_send_unmap_rm()
90 		rds_iw_send_unmap_rdma(ic, &rm->rdma);  in rds_iw_send_unmap_rm()
112 		rds_iw_send_rdma_complete(rm, wc_status);  in rds_iw_send_unmap_rm()
114 		if (rm->rdma.op_write)  in rds_iw_send_unmap_rm()
115 			rds_stats_add(s_send_rdma_bytes, rm->rdma.op_bytes);  in rds_iw_send_unmap_rm()
117 			rds_stats_add(s_recv_rdma_bytes, rm->rdma.op_bytes);  in rds_iw_send_unmap_rm()
122 	rds_message_unmapped(rm);  in rds_iw_send_unmap_rm()
124 	rds_message_put(rm);  in rds_iw_send_unmap_rm()
270 				struct rds_message *rm;  in rds_iw_send_cq_comp_handler()  local
272 				rm = rds_send_get_message(conn, send->s_op);  in rds_iw_send_cq_comp_handler()
273 				if (rm)  in rds_iw_send_cq_comp_handler()
274 					rds_iw_send_rdma_complete(rm, wc.status);  in rds_iw_send_cq_comp_handler()
488 int rds_iw_xmit(struct rds_connection *conn, struct rds_message *rm,  in rds_iw_xmit()  argument
513 	if (rds_rdma_cookie_key(rm->m_rdma_cookie) && !ic->i_fastreg_posted) {  in rds_iw_xmit()
519 	if (be32_to_cpu(rm->m_inc.i_hdr.h_len) == 0)  in rds_iw_xmit()
522 		i = ceil(be32_to_cpu(rm->m_inc.i_hdr.h_len), RDS_FRAG_SIZE);  in rds_iw_xmit()
557 		if (rm->data.op_nents) {  in rds_iw_xmit()
558 			rm->data.op_count = ib_dma_map_sg(dev,  in rds_iw_xmit()
559 							  rm->data.op_sg,  in rds_iw_xmit()
560 							  rm->data.op_nents,  in rds_iw_xmit()
562 			rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);  in rds_iw_xmit()
563 			if (rm->data.op_count == 0) {  in rds_iw_xmit()
570 			rm->data.op_count = 0;  in rds_iw_xmit()
575 		rds_message_addref(rm);  in rds_iw_xmit()
576 		rm->data.op_dmasg = 0;  in rds_iw_xmit()
577 		rm->data.op_dmaoff = 0;  in rds_iw_xmit()
578 		ic->i_rm = rm;  in rds_iw_xmit()
581 		if (test_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags))  in rds_iw_xmit()
582 			rm->m_inc.i_hdr.h_flags |= RDS_FLAG_ACK_REQUIRED;  in rds_iw_xmit()
583 		if (test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags))  in rds_iw_xmit()
584 			rm->m_inc.i_hdr.h_flags |= RDS_FLAG_RETRANSMITTED;  in rds_iw_xmit()
588 		if (rm->rdma.op_active) {  in rds_iw_xmit()
591 			ext_hdr.h_rdma_rkey = cpu_to_be32(rm->rdma.op_rkey);  in rds_iw_xmit()
592 			rds_message_add_extension(&rm->m_inc.i_hdr,  in rds_iw_xmit()
595 		if (rm->m_rdma_cookie) {  in rds_iw_xmit()
596 			rds_message_add_rdma_dest_extension(&rm->m_inc.i_hdr,  in rds_iw_xmit()
597 					rds_rdma_cookie_key(rm->m_rdma_cookie),  in rds_iw_xmit()
598 					rds_rdma_cookie_offset(rm->m_rdma_cookie));  in rds_iw_xmit()
605 		rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_iw_piggyb_ack(ic));  in rds_iw_xmit()
606 		rds_message_make_checksum(&rm->m_inc.i_hdr);  in rds_iw_xmit()
619 	scat = &rm->data.op_sg[rm->data.op_dmasg];  in rds_iw_xmit()
629 	if (rm->rdma.op_active && rm->rdma.op_fence)  in rds_iw_xmit()
642 	if (be32_to_cpu(rm->m_inc.i_hdr.h_len) == 0) {  in rds_iw_xmit()
648 	for (; i < work_alloc && scat != &rm->data.op_sg[rm->data.op_count]; i++) {  in rds_iw_xmit()
654 			  ib_sg_dma_len(dev, scat) - rm->data.op_dmaoff);  in rds_iw_xmit()
656 			ib_sg_dma_address(dev, scat) + rm->data.op_dmaoff, len,  in rds_iw_xmit()
685 		rm->data.op_dmaoff += len;  in rds_iw_xmit()
686 		if (rm->data.op_dmaoff == ib_sg_dma_len(dev, scat)) {  in rds_iw_xmit()
688 			rm->data.op_dmaoff = 0;  in rds_iw_xmit()
689 			rm->data.op_dmasg++;  in rds_iw_xmit()
695 		memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr, sizeof(struct rds_header));  in rds_iw_xmit()
728 	if (scat == &rm->data.op_sg[rm->data.op_count]) {  in rds_iw_xmit()