71 static void rds_ib_send_unmap_data(struct rds_ib_connection *ic,  in rds_ib_send_unmap_data()  argument
76 		ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_ib_send_unmap_data()
81 static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,  in rds_ib_send_unmap_rdma()  argument
86 		ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_ib_send_unmap_rdma()
121 static void rds_ib_send_unmap_atomic(struct rds_ib_connection *ic,  in rds_ib_send_unmap_atomic()  argument
127 		ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,  in rds_ib_send_unmap_atomic()
148 static struct rds_message *rds_ib_send_unmap_op(struct rds_ib_connection *ic,  in rds_ib_send_unmap_op()  argument
159 			rds_ib_send_unmap_data(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
166 			rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
173 			rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
188 void rds_ib_send_init_ring(struct rds_ib_connection *ic)  in rds_ib_send_init_ring()  argument
193 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_init_ring()
203 		sge->addr = ic->i_send_hdrs_dma + (i * sizeof(struct rds_header));  in rds_ib_send_init_ring()
205 		sge->lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
207 		send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
211 void rds_ib_send_clear_ring(struct rds_ib_connection *ic)  in rds_ib_send_clear_ring()  argument
216 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_clear_ring()
218 			rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_ib_send_clear_ring()
226 static void rds_ib_sub_signaled(struct rds_ib_connection *ic, int nr)  in rds_ib_sub_signaled()  argument
228 	if ((atomic_sub_return(nr, &ic->i_signaled_sends) == 0) &&  in rds_ib_sub_signaled()
231 	BUG_ON(atomic_read(&ic->i_signaled_sends) < 0);  in rds_ib_sub_signaled()
240 void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc)  in rds_ib_send_cqe_handler()  argument
243 	struct rds_connection *conn = ic->conn;  in rds_ib_send_cqe_handler()
258 		if (time_after(jiffies, ic->i_ack_queued + HZ / 2))  in rds_ib_send_cqe_handler()
260 		rds_ib_ack_send_complete(ic);  in rds_ib_send_cqe_handler()
264 	oldest = rds_ib_ring_oldest(&ic->i_send_ring);  in rds_ib_send_cqe_handler()
266 	completed = rds_ib_ring_completed(&ic->i_send_ring,  in rds_ib_send_cqe_handler()
271 		send = &ic->i_sends[oldest];  in rds_ib_send_cqe_handler()
275 		rm = rds_ib_send_unmap_op(ic, send, wc->status);  in rds_ib_send_cqe_handler()
291 		oldest = (oldest + 1) % ic->i_send_ring.w_nr;  in rds_ib_send_cqe_handler()
294 	rds_ib_ring_free(&ic->i_send_ring, completed);  in rds_ib_send_cqe_handler()
295 	rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_send_cqe_handler()
354 int rds_ib_send_grab_credits(struct rds_ib_connection *ic,  in rds_ib_send_grab_credits()  argument
361 	if (!ic->i_flowctl)  in rds_ib_send_grab_credits()
366 	oldval = newval = atomic_read(&ic->i_credits);  in rds_ib_send_grab_credits()
378 		struct rds_connection *conn = ic->i_cm_id->context;  in rds_ib_send_grab_credits()
400 	if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)  in rds_ib_send_grab_credits()
409 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_send_add_credits()  local
416 			IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)),  in rds_ib_send_add_credits()
419 	atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits);  in rds_ib_send_add_credits()
430 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_advertise_credits()  local
435 	atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits);  in rds_ib_advertise_credits()
449 	if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16)  in rds_ib_advertise_credits()
450 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_advertise_credits()
453 static inline int rds_ib_set_wr_signal_state(struct rds_ib_connection *ic,  in rds_ib_set_wr_signal_state()  argument
462 	if (ic->i_unsignaled_wrs-- == 0 || notify) {  in rds_ib_set_wr_signal_state()
463 		ic->i_unsignaled_wrs = rds_ib_sysctl_max_unsig_wrs;  in rds_ib_set_wr_signal_state()
486 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit()  local
487 	struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_xmit()
523 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_ib_xmit()
531 	if (ic->i_flowctl) {  in rds_ib_xmit()
532 		credit_alloc = rds_ib_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);  in rds_ib_xmit()
535 			rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);  in rds_ib_xmit()
548 	if (!ic->i_data_op) {  in rds_ib_xmit()
554 			rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);  in rds_ib_xmit()
557 				rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit()
568 		ic->i_data_op = &rm->data;  in rds_ib_xmit()
595 		rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_ib_piggyb_ack(ic));  in rds_ib_xmit()
601 		if (ic->i_flowctl) {  in rds_ib_xmit()
602 			rds_ib_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);  in rds_ib_xmit()
618 	send = &ic->i_sends[pos];  in rds_ib_xmit()
621 	scat = &ic->i_data_op->op_sg[rm->data.op_dmasg];  in rds_ib_xmit()
634 		send->s_sge[0].addr = ic->i_send_hdrs_dma  in rds_ib_xmit()
638 		memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr, sizeof(struct rds_header));  in rds_ib_xmit()
660 		rds_ib_set_wr_signal_state(ic, send, 0);  in rds_ib_xmit()
665 		if (ic->i_flowctl && flow_controlled && i == (work_alloc-1))  in rds_ib_xmit()
674 		if (ic->i_flowctl && adv_credits) {  in rds_ib_xmit()
675 			struct rds_header *hdr = &ic->i_send_hdrs[pos];  in rds_ib_xmit()
688 		pos = (pos + 1) % ic->i_send_ring.w_nr;  in rds_ib_xmit()
689 		send = &ic->i_sends[pos];  in rds_ib_xmit()
702 		prev->s_op = ic->i_data_op;  in rds_ib_xmit()
705 			ic->i_unsignaled_wrs = rds_ib_sysctl_max_unsig_wrs;  in rds_ib_xmit()
709 		ic->i_data_op = NULL;  in rds_ib_xmit()
714 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_ib_xmit()
717 	if (ic->i_flowctl && i < credit_alloc)  in rds_ib_xmit()
721 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit()
725 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);  in rds_ib_xmit()
726 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit()
732 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit()
733 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit()
735 			ic->i_data_op = prev->s_op;  in rds_ib_xmit()
739 		rds_ib_conn_error(ic->conn, "ib_post_send failed\n");  in rds_ib_xmit()
756 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_atomic()  local
765 	rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rds_ib_client);  in rds_ib_xmit_atomic()
767 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, 1, &pos);  in rds_ib_xmit_atomic()
769 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_atomic()
776 	send = &ic->i_sends[pos];  in rds_ib_xmit_atomic()
792 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
801 	ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);  in rds_ib_xmit_atomic()
802 	rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);  in rds_ib_xmit_atomic()
804 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_atomic()
811 	send->s_sge[0].addr = ib_sg_dma_address(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
812 	send->s_sge[0].length = ib_sg_dma_len(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
813 	send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_atomic()
819 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit_atomic()
822 	ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);  in rds_ib_xmit_atomic()
823 	rdsdebug("ic %p send %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit_atomic()
829 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_atomic()
830 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit_atomic()
845 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_rdma()  local
853 	u32 max_sge = ic->rds_ibdev->max_sge;  in rds_ib_xmit_rdma()
865 		op->op_count = ib_dma_map_sg(ic->i_cm_id->device,  in rds_ib_xmit_rdma()
868 		rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count);  in rds_ib_xmit_rdma()
884 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_ib_xmit_rdma()
886 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_rdma()
892 	send = &ic->i_sends[pos];  in rds_ib_xmit_rdma()
904 		nr_sig += rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_rdma()
924 			len = ib_sg_dma_len(ic->i_cm_id->device, scat);  in rds_ib_xmit_rdma()
926 				 ib_sg_dma_address(ic->i_cm_id->device, scat);  in rds_ib_xmit_rdma()
928 			send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_rdma()
931 			rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr);  in rds_ib_xmit_rdma()
943 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_ib_xmit_rdma()
944 			send = ic->i_sends;  in rds_ib_xmit_rdma()
954 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_ib_xmit_rdma()
959 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit_rdma()
962 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr);  in rds_ib_xmit_rdma()
963 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit_rdma()
969 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_rdma()
970 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit_rdma()
986 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_complete()  local
990 	rds_ib_attempt_ack(ic);  in rds_ib_xmit_complete()