149 						struct rds_ib_send_work *send,  in rds_ib_send_unmap_op()  argument
155 	switch (send->s_wr.opcode) {  in rds_ib_send_unmap_op()
157 		if (send->s_op) {  in rds_ib_send_unmap_op()
158 			rm = container_of(send->s_op, struct rds_message, data);  in rds_ib_send_unmap_op()
159 			rds_ib_send_unmap_data(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
164 		if (send->s_op) {  in rds_ib_send_unmap_op()
165 			rm = container_of(send->s_op, struct rds_message, rdma);  in rds_ib_send_unmap_op()
166 			rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
171 		if (send->s_op) {  in rds_ib_send_unmap_op()
172 			rm = container_of(send->s_op, struct rds_message, atomic);  in rds_ib_send_unmap_op()
173 			rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
179 			       __func__, send->s_wr.opcode);  in rds_ib_send_unmap_op()
183 	send->s_wr.opcode = 0xdead;  in rds_ib_send_unmap_op()
190 	struct rds_ib_send_work *send;  in rds_ib_send_init_ring()  local
193 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_init_ring()
196 		send->s_op = NULL;  in rds_ib_send_init_ring()
198 		send->s_wr.wr_id = i | RDS_IB_SEND_OP;  in rds_ib_send_init_ring()
199 		send->s_wr.sg_list = send->s_sge;  in rds_ib_send_init_ring()
200 		send->s_wr.ex.imm_data = 0;  in rds_ib_send_init_ring()
202 		sge = &send->s_sge[0];  in rds_ib_send_init_ring()
207 		send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
213 	struct rds_ib_send_work *send;  in rds_ib_send_clear_ring()  local
216 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_clear_ring()
217 		if (send->s_op && send->s_wr.opcode != 0xdead)  in rds_ib_send_clear_ring()
218 			rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_ib_send_clear_ring()
244 	struct rds_ib_send_work *send;  in rds_ib_send_cqe_handler()  local
271 		send = &ic->i_sends[oldest];  in rds_ib_send_cqe_handler()
272 		if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_send_cqe_handler()
275 		rm = rds_ib_send_unmap_op(ic, send, wc->status);  in rds_ib_send_cqe_handler()
277 		if (time_after(jiffies, send->s_queued + HZ / 2))  in rds_ib_send_cqe_handler()
280 		if (send->s_op) {  in rds_ib_send_cqe_handler()
281 			if (send->s_op == rm->m_final_op) {  in rds_ib_send_cqe_handler()
288 			send->s_op = NULL;  in rds_ib_send_cqe_handler()
454 					     struct rds_ib_send_work *send,  in rds_ib_set_wr_signal_state()  argument
464 		send->s_wr.send_flags |= IB_SEND_SIGNALED;  in rds_ib_set_wr_signal_state()
488 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit()  local
618 	send = &ic->i_sends[pos];  in rds_ib_xmit()
619 	first = send;  in rds_ib_xmit()
627 		send->s_wr.send_flags = send_flags;  in rds_ib_xmit()
628 		send->s_wr.opcode = IB_WR_SEND;  in rds_ib_xmit()
629 		send->s_wr.num_sge = 1;  in rds_ib_xmit()
630 		send->s_wr.next = NULL;  in rds_ib_xmit()
631 		send->s_queued = jiffies;  in rds_ib_xmit()
632 		send->s_op = NULL;  in rds_ib_xmit()
634 		send->s_sge[0].addr = ic->i_send_hdrs_dma  in rds_ib_xmit()
636 		send->s_sge[0].length = sizeof(struct rds_header);  in rds_ib_xmit()
645 			send->s_wr.num_sge = 2;  in rds_ib_xmit()
647 			send->s_sge[1].addr = ib_sg_dma_address(dev, scat);  in rds_ib_xmit()
648 			send->s_sge[1].addr += rm->data.op_dmaoff;  in rds_ib_xmit()
649 			send->s_sge[1].length = len;  in rds_ib_xmit()
660 		rds_ib_set_wr_signal_state(ic, send, 0);  in rds_ib_xmit()
666 			send->s_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;  in rds_ib_xmit()
668 		if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_xmit()
671 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit()
672 			 &send->s_wr, send->s_wr.num_sge, send->s_wr.next);  in rds_ib_xmit()
685 			prev->s_wr.next = &send->s_wr;  in rds_ib_xmit()
686 		prev = send;  in rds_ib_xmit()
689 		send = &ic->i_sends[pos];  in rds_ib_xmit()
757 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_atomic()  local
776 	send = &ic->i_sends[pos];  in rds_ib_xmit_atomic()
777 	send->s_queued = jiffies;  in rds_ib_xmit_atomic()
780 		send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;  in rds_ib_xmit_atomic()
781 		send->s_atomic_wr.compare_add = op->op_m_cswp.compare;  in rds_ib_xmit_atomic()
782 		send->s_atomic_wr.swap = op->op_m_cswp.swap;  in rds_ib_xmit_atomic()
783 		send->s_atomic_wr.compare_add_mask = op->op_m_cswp.compare_mask;  in rds_ib_xmit_atomic()
784 		send->s_atomic_wr.swap_mask = op->op_m_cswp.swap_mask;  in rds_ib_xmit_atomic()
786 		send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;  in rds_ib_xmit_atomic()
787 		send->s_atomic_wr.compare_add = op->op_m_fadd.add;  in rds_ib_xmit_atomic()
788 		send->s_atomic_wr.swap = 0;  in rds_ib_xmit_atomic()
789 		send->s_atomic_wr.compare_add_mask = op->op_m_fadd.nocarry_mask;  in rds_ib_xmit_atomic()
790 		send->s_atomic_wr.swap_mask = 0;  in rds_ib_xmit_atomic()
792 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
793 	send->s_atomic_wr.wr.num_sge = 1;  in rds_ib_xmit_atomic()
794 	send->s_atomic_wr.wr.next = NULL;  in rds_ib_xmit_atomic()
795 	send->s_atomic_wr.remote_addr = op->op_remote_addr;  in rds_ib_xmit_atomic()
796 	send->s_atomic_wr.rkey = op->op_rkey;  in rds_ib_xmit_atomic()
797 	send->s_op = op;  in rds_ib_xmit_atomic()
798 	rds_message_addref(container_of(send->s_op, struct rds_message, atomic));  in rds_ib_xmit_atomic()
811 	send->s_sge[0].addr = ib_sg_dma_address(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
812 	send->s_sge[0].length = ib_sg_dma_len(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
813 	send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_atomic()
816 		 send->s_sge[0].addr, send->s_sge[0].length);  in rds_ib_xmit_atomic()
821 	failed_wr = &send->s_atomic_wr.wr;  in rds_ib_xmit_atomic()
822 	ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);  in rds_ib_xmit_atomic()
824 		 send, &send->s_atomic_wr, ret, failed_wr);  in rds_ib_xmit_atomic()
825 	BUG_ON(failed_wr != &send->s_atomic_wr.wr);  in rds_ib_xmit_atomic()
834 	if (unlikely(failed_wr != &send->s_atomic_wr.wr)) {  in rds_ib_xmit_atomic()
836 		BUG_ON(failed_wr != &send->s_atomic_wr.wr);  in rds_ib_xmit_atomic()
846 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_rdma()  local
892 	send = &ic->i_sends[pos];  in rds_ib_xmit_rdma()
893 	first = send;  in rds_ib_xmit_rdma()
900 		send->s_wr.send_flags = 0;  in rds_ib_xmit_rdma()
901 		send->s_queued = jiffies;  in rds_ib_xmit_rdma()
902 		send->s_op = NULL;  in rds_ib_xmit_rdma()
904 		nr_sig += rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_rdma()
906 		send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;  in rds_ib_xmit_rdma()
907 		send->s_rdma_wr.remote_addr = remote_addr;  in rds_ib_xmit_rdma()
908 		send->s_rdma_wr.rkey = op->op_rkey;  in rds_ib_xmit_rdma()
911 			send->s_rdma_wr.wr.num_sge = max_sge;  in rds_ib_xmit_rdma()
914 			send->s_rdma_wr.wr.num_sge = num_sge;  in rds_ib_xmit_rdma()
917 		send->s_rdma_wr.wr.next = NULL;  in rds_ib_xmit_rdma()
920 			prev->s_rdma_wr.wr.next = &send->s_rdma_wr.wr;  in rds_ib_xmit_rdma()
922 		for (j = 0; j < send->s_rdma_wr.wr.num_sge &&  in rds_ib_xmit_rdma()
925 			send->s_sge[j].addr =  in rds_ib_xmit_rdma()
927 			send->s_sge[j].length = len;  in rds_ib_xmit_rdma()
928 			send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_rdma()
937 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit_rdma()
938 			&send->s_rdma_wr.wr,  in rds_ib_xmit_rdma()
939 			send->s_rdma_wr.wr.num_sge,  in rds_ib_xmit_rdma()
940 			send->s_rdma_wr.wr.next);  in rds_ib_xmit_rdma()
942 		prev = send;  in rds_ib_xmit_rdma()
943 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_ib_xmit_rdma()
944 			send = ic->i_sends;  in rds_ib_xmit_rdma()