78 			  struct rds_iw_send_work *send,  in rds_iw_send_unmap_rm()  argument
81 	struct rds_message *rm = send->s_rm;  in rds_iw_send_unmap_rm()
83 	rdsdebug("ic %p send %p rm %p\n", ic, send, rm);  in rds_iw_send_unmap_rm()
125 	send->s_rm = NULL;  in rds_iw_send_unmap_rm()
130 	struct rds_iw_send_work *send;  in rds_iw_send_init_ring()  local
133 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_iw_send_init_ring()
136 		send->s_rm = NULL;  in rds_iw_send_init_ring()
137 		send->s_op = NULL;  in rds_iw_send_init_ring()
138 		send->s_mapping = NULL;  in rds_iw_send_init_ring()
140 		send->s_send_wr.next = NULL;  in rds_iw_send_init_ring()
141 		send->s_send_wr.wr_id = i;  in rds_iw_send_init_ring()
142 		send->s_send_wr.sg_list = send->s_sge;  in rds_iw_send_init_ring()
143 		send->s_send_wr.num_sge = 1;  in rds_iw_send_init_ring()
144 		send->s_send_wr.opcode = IB_WR_SEND;  in rds_iw_send_init_ring()
145 		send->s_send_wr.send_flags = 0;  in rds_iw_send_init_ring()
146 		send->s_send_wr.ex.imm_data = 0;  in rds_iw_send_init_ring()
148 		sge = rds_iw_data_sge(ic, send->s_sge);  in rds_iw_send_init_ring()
151 		sge = rds_iw_header_sge(ic, send->s_sge);  in rds_iw_send_init_ring()
156 		send->s_mr = ib_alloc_mr(ic->i_pd, IB_MR_TYPE_MEM_REG,  in rds_iw_send_init_ring()
158 		if (IS_ERR(send->s_mr)) {  in rds_iw_send_init_ring()
167 	struct rds_iw_send_work *send;  in rds_iw_send_clear_ring()  local
170 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_iw_send_clear_ring()
171 		BUG_ON(!send->s_mr);  in rds_iw_send_clear_ring()
172 		ib_dereg_mr(send->s_mr);  in rds_iw_send_clear_ring()
173 		if (send->s_send_wr.opcode == 0xdead)  in rds_iw_send_clear_ring()
175 		if (send->s_rm)  in rds_iw_send_clear_ring()
176 			rds_iw_send_unmap_rm(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_iw_send_clear_ring()
177 		if (send->s_op)  in rds_iw_send_clear_ring()
178 			rds_iw_send_unmap_rdma(ic, send->s_op);  in rds_iw_send_clear_ring()
193 	struct rds_iw_send_work *send;  in rds_iw_send_cq_comp_handler()  local
238 			send = &ic->i_sends[oldest];  in rds_iw_send_cq_comp_handler()
241 			switch (send->s_send_wr.opcode) {  in rds_iw_send_cq_comp_handler()
243 				if (send->s_rm)  in rds_iw_send_cq_comp_handler()
244 					rds_iw_send_unmap_rm(ic, send, wc.status);  in rds_iw_send_cq_comp_handler()
256 						__func__, send->s_send_wr.opcode);  in rds_iw_send_cq_comp_handler()
260 			send->s_send_wr.opcode = 0xdead;  in rds_iw_send_cq_comp_handler()
261 			send->s_send_wr.num_sge = 1;  in rds_iw_send_cq_comp_handler()
262 			if (time_after(jiffies, send->s_queued + HZ/2))  in rds_iw_send_cq_comp_handler()
269 			if (unlikely(wc.status == IB_WC_REM_ACCESS_ERR && send->s_op)) {  in rds_iw_send_cq_comp_handler()
272 				rm = rds_send_get_message(conn, send->s_op);  in rds_iw_send_cq_comp_handler()
441 		struct rds_iw_send_work *send, unsigned int pos,  in rds_iw_xmit_populate_wr()  argument
447 	WARN_ON(pos != send - ic->i_sends);  in rds_iw_xmit_populate_wr()
449 	send->s_send_wr.send_flags = send_flags;  in rds_iw_xmit_populate_wr()
450 	send->s_send_wr.opcode = IB_WR_SEND;  in rds_iw_xmit_populate_wr()
451 	send->s_send_wr.num_sge = 2;  in rds_iw_xmit_populate_wr()
452 	send->s_send_wr.next = NULL;  in rds_iw_xmit_populate_wr()
453 	send->s_queued = jiffies;  in rds_iw_xmit_populate_wr()
454 	send->s_op = NULL;  in rds_iw_xmit_populate_wr()
457 		sge = rds_iw_data_sge(ic, send->s_sge);  in rds_iw_xmit_populate_wr()
462 		sge = rds_iw_header_sge(ic, send->s_sge);  in rds_iw_xmit_populate_wr()
466 		send->s_send_wr.num_sge = 1;  in rds_iw_xmit_populate_wr()
467 		sge = &send->s_sge[0];  in rds_iw_xmit_populate_wr()
493 	struct rds_iw_send_work *send = NULL;  in rds_iw_xmit()  local
616 	send = &ic->i_sends[pos];  in rds_iw_xmit()
617 	first = send;  in rds_iw_xmit()
643 		rds_iw_xmit_populate_wr(ic, send, pos, 0, 0, send_flags);  in rds_iw_xmit()
651 		send = &ic->i_sends[pos];  in rds_iw_xmit()
655 		rds_iw_xmit_populate_wr(ic, send, pos,  in rds_iw_xmit()
666 			send->s_send_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;  in rds_iw_xmit()
672 			send->s_send_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;  in rds_iw_xmit()
679 			send->s_send_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;  in rds_iw_xmit()
681 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_iw_xmit()
682 			 &send->s_send_wr, send->s_send_wr.num_sge, send->s_send_wr.next);  in rds_iw_xmit()
716 			prev->s_send_wr.next = &send->s_send_wr;  in rds_iw_xmit()
717 		prev = send;  in rds_iw_xmit()
764 static int rds_iw_build_send_reg(struct rds_iw_send_work *send,  in rds_iw_build_send_reg()  argument
770 	n = ib_map_mr_sg(send->s_mr, sg, sg_nents, PAGE_SIZE);  in rds_iw_build_send_reg()
774 	send->s_reg_wr.wr.opcode = IB_WR_REG_MR;  in rds_iw_build_send_reg()
775 	send->s_reg_wr.wr.wr_id = 0;  in rds_iw_build_send_reg()
776 	send->s_reg_wr.wr.num_sge = 0;  in rds_iw_build_send_reg()
777 	send->s_reg_wr.mr = send->s_mr;  in rds_iw_build_send_reg()
778 	send->s_reg_wr.key = send->s_mr->rkey;  in rds_iw_build_send_reg()
779 	send->s_reg_wr.access = IB_ACCESS_REMOTE_WRITE;  in rds_iw_build_send_reg()
781 	ib_update_fast_reg_key(send->s_mr, send->s_remap_count++);  in rds_iw_build_send_reg()
789 	struct rds_iw_send_work *send = NULL;  in rds_iw_xmit_rdma()  local
848 	send = &ic->i_sends[pos];  in rds_iw_xmit_rdma()
852 		first = send;  in rds_iw_xmit_rdma()
861 		send->s_rdma_wr.wr.send_flags = 0;  in rds_iw_xmit_rdma()
862 		send->s_queued = jiffies;  in rds_iw_xmit_rdma()
870 			send->s_rdma_wr.wr.send_flags = IB_SEND_SIGNALED;  in rds_iw_xmit_rdma()
878 			send->s_rdma_wr.wr.opcode = IB_WR_RDMA_WRITE;  in rds_iw_xmit_rdma()
880 			send->s_rdma_wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV;  in rds_iw_xmit_rdma()
882 		send->s_rdma_wr.remote_addr = remote_addr;  in rds_iw_xmit_rdma()
883 		send->s_rdma_wr.rkey = op->op_rkey;  in rds_iw_xmit_rdma()
884 		send->s_op = op;  in rds_iw_xmit_rdma()
887 			send->s_rdma_wr.wr.num_sge = rds_iwdev->max_sge;  in rds_iw_xmit_rdma()
890 			send->s_rdma_wr.wr.num_sge = num_sge;  in rds_iw_xmit_rdma()
892 		send->s_rdma_wr.wr.next = NULL;  in rds_iw_xmit_rdma()
895 			prev->s_send_wr.next = &send->s_rdma_wr.wr;  in rds_iw_xmit_rdma()
897 		for (j = 0; j < send->s_rdma_wr.wr.num_sge &&  in rds_iw_xmit_rdma()
901 			if (send->s_rdma_wr.wr.opcode == IB_WR_RDMA_READ_WITH_INV)  in rds_iw_xmit_rdma()
904 				send->s_sge[j].addr = ib_sg_dma_address(ic->i_cm_id->device, scat);  in rds_iw_xmit_rdma()
905 				send->s_sge[j].length = len;  in rds_iw_xmit_rdma()
906 				send->s_sge[j].lkey = rds_iw_local_dma_lkey(ic);  in rds_iw_xmit_rdma()
916 		if (send->s_rdma_wr.wr.opcode == IB_WR_RDMA_READ_WITH_INV) {  in rds_iw_xmit_rdma()
917 			send->s_rdma_wr.wr.num_sge = 1;  in rds_iw_xmit_rdma()
918 			send->s_sge[0].addr = conn->c_xmit_rm->m_rs->rs_user_addr;  in rds_iw_xmit_rdma()
919 			send->s_sge[0].length = conn->c_xmit_rm->m_rs->rs_user_bytes;  in rds_iw_xmit_rdma()
920 			send->s_sge[0].lkey = ic->i_sends[fr_pos].s_mr->lkey;  in rds_iw_xmit_rdma()
923 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_iw_xmit_rdma()
924 			&send->s_rdma_wr,  in rds_iw_xmit_rdma()
925 			send->s_rdma_wr.wr.num_sge,  in rds_iw_xmit_rdma()
926 			send->s_rdma_wr.wr.next);  in rds_iw_xmit_rdma()
928 		prev = send;  in rds_iw_xmit_rdma()
929 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_iw_xmit_rdma()
930 			send = ic->i_sends;  in rds_iw_xmit_rdma()