106 				   struct rm_data_op *op,  in rds_ib_send_unmap_data()  argument
109 	if (op->op_nents)  in rds_ib_send_unmap_data()
111 				op->op_sg, op->op_nents,  in rds_ib_send_unmap_data()
116 				   struct rm_rdma_op *op,  in rds_ib_send_unmap_rdma()  argument
119 	if (op->op_mapped) {  in rds_ib_send_unmap_rdma()
121 				op->op_sg, op->op_nents,  in rds_ib_send_unmap_rdma()
122 				op->op_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);  in rds_ib_send_unmap_rdma()
123 		op->op_mapped = 0;  in rds_ib_send_unmap_rdma()
146 	rds_ib_send_complete(container_of(op, struct rds_message, rdma),  in rds_ib_send_unmap_rdma()
149 	if (op->op_write)  in rds_ib_send_unmap_rdma()
150 		rds_stats_add(s_send_rdma_bytes, op->op_bytes);  in rds_ib_send_unmap_rdma()
152 		rds_stats_add(s_recv_rdma_bytes, op->op_bytes);  in rds_ib_send_unmap_rdma()
156 				     struct rm_atomic_op *op,  in rds_ib_send_unmap_atomic()  argument
160 	if (op->op_mapped) {  in rds_ib_send_unmap_atomic()
161 		ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,  in rds_ib_send_unmap_atomic()
163 		op->op_mapped = 0;  in rds_ib_send_unmap_atomic()
166 	rds_ib_send_complete(container_of(op, struct rds_message, atomic),  in rds_ib_send_unmap_atomic()
169 	if (op->op_type == RDS_ATOMIC_TYPE_CSWP)  in rds_ib_send_unmap_atomic()
786 int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)  in rds_ib_xmit_atomic()  argument
811 	if (op->op_type == RDS_ATOMIC_TYPE_CSWP) {  in rds_ib_xmit_atomic()
813 		send->s_wr.wr.atomic.compare_add = op->op_m_cswp.compare;  in rds_ib_xmit_atomic()
814 		send->s_wr.wr.atomic.swap = op->op_m_cswp.swap;  in rds_ib_xmit_atomic()
815 		send->s_wr.wr.atomic.compare_add_mask = op->op_m_cswp.compare_mask;  in rds_ib_xmit_atomic()
816 		send->s_wr.wr.atomic.swap_mask = op->op_m_cswp.swap_mask;  in rds_ib_xmit_atomic()
819 		send->s_wr.wr.atomic.compare_add = op->op_m_fadd.add;  in rds_ib_xmit_atomic()
821 		send->s_wr.wr.atomic.compare_add_mask = op->op_m_fadd.nocarry_mask;  in rds_ib_xmit_atomic()
824 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
827 	send->s_wr.wr.atomic.remote_addr = op->op_remote_addr;  in rds_ib_xmit_atomic()
828 	send->s_wr.wr.atomic.rkey = op->op_rkey;  in rds_ib_xmit_atomic()
829 	send->s_op = op;  in rds_ib_xmit_atomic()
833 	ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);  in rds_ib_xmit_atomic()
834 	rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);  in rds_ib_xmit_atomic()
843 	send->s_sge[0].addr = ib_sg_dma_address(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
844 	send->s_sge[0].length = ib_sg_dma_len(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
847 	rdsdebug("rva %Lx rpa %Lx len %u\n", op->op_remote_addr,  in rds_ib_xmit_atomic()
875 int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)  in rds_ib_xmit_rdma()  argument
884 	u64 remote_addr = op->op_remote_addr;  in rds_ib_xmit_rdma()
896 	if (!op->op_mapped) {  in rds_ib_xmit_rdma()
897 		op->op_count = ib_dma_map_sg(ic->i_cm_id->device,  in rds_ib_xmit_rdma()
898 					     op->op_sg, op->op_nents, (op->op_write) ?  in rds_ib_xmit_rdma()
900 		rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count);  in rds_ib_xmit_rdma()
901 		if (op->op_count == 0) {  in rds_ib_xmit_rdma()
907 		op->op_mapped = 1;  in rds_ib_xmit_rdma()
914 	i = ceil(op->op_count, max_sge);  in rds_ib_xmit_rdma()
927 	scat = &op->op_sg[0];  in rds_ib_xmit_rdma()
929 	num_sge = op->op_count;  in rds_ib_xmit_rdma()
931 	for (i = 0; i < work_alloc && scat != &op->op_sg[op->op_count]; i++) {  in rds_ib_xmit_rdma()
936 		nr_sig += rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_rdma()
938 		send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;  in rds_ib_xmit_rdma()
940 		send->s_wr.wr.rdma.rkey = op->op_rkey;  in rds_ib_xmit_rdma()
954 		for (j = 0; j < send->s_wr.num_sge && scat != &op->op_sg[op->op_count]; j++) {  in rds_ib_xmit_rdma()
977 	if (scat == &op->op_sg[op->op_count]) {  in rds_ib_xmit_rdma()
978 		prev->s_op = op;  in rds_ib_xmit_rdma()
979 		rds_message_addref(container_of(op, struct rds_message, rdma));  in rds_ib_xmit_rdma()