66 static void rds_iw_send_unmap_rdma(struct rds_iw_connection *ic,  in rds_iw_send_unmap_rdma()  argument
70 		ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_iw_send_unmap_rdma()
77 static void rds_iw_send_unmap_rm(struct rds_iw_connection *ic,  in rds_iw_send_unmap_rm()  argument
83 	rdsdebug("ic %p send %p rm %p\n", ic, send, rm);  in rds_iw_send_unmap_rm()
85 	ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_iw_send_unmap_rm()
90 		rds_iw_send_unmap_rdma(ic, &rm->rdma);  in rds_iw_send_unmap_rm()
128 void rds_iw_send_init_ring(struct rds_iw_connection *ic)  in rds_iw_send_init_ring()  argument
133 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_iw_send_init_ring()
148 		sge = rds_iw_data_sge(ic, send->s_sge);  in rds_iw_send_init_ring()
151 		sge = rds_iw_header_sge(ic, send->s_sge);  in rds_iw_send_init_ring()
152 		sge->addr = ic->i_send_hdrs_dma + (i * sizeof(struct rds_header));  in rds_iw_send_init_ring()
156 		send->s_mr = ib_alloc_mr(ic->i_pd, IB_MR_TYPE_MEM_REG,  in rds_iw_send_init_ring()
165 void rds_iw_send_clear_ring(struct rds_iw_connection *ic)  in rds_iw_send_clear_ring()  argument
170 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_iw_send_clear_ring()
176 			rds_iw_send_unmap_rm(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_iw_send_clear_ring()
178 			rds_iw_send_unmap_rdma(ic, send->s_op);  in rds_iw_send_clear_ring()
191 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_send_cq_comp_handler()  local
217 			ic->i_fastreg_posted = 0;  in rds_iw_send_cq_comp_handler()
222 			ic->i_fastreg_posted = 1;  in rds_iw_send_cq_comp_handler()
227 			if (time_after(jiffies, ic->i_ack_queued + HZ/2))  in rds_iw_send_cq_comp_handler()
229 			rds_iw_ack_send_complete(ic);  in rds_iw_send_cq_comp_handler()
233 		oldest = rds_iw_ring_oldest(&ic->i_send_ring);  in rds_iw_send_cq_comp_handler()
235 		completed = rds_iw_ring_completed(&ic->i_send_ring, wc.wr_id, oldest);  in rds_iw_send_cq_comp_handler()
238 			send = &ic->i_sends[oldest];  in rds_iw_send_cq_comp_handler()
244 					rds_iw_send_unmap_rm(ic, send, wc.status);  in rds_iw_send_cq_comp_handler()
277 			oldest = (oldest + 1) % ic->i_send_ring.w_nr;  in rds_iw_send_cq_comp_handler()
280 		rds_iw_ring_free(&ic->i_send_ring, completed);  in rds_iw_send_cq_comp_handler()
340 int rds_iw_send_grab_credits(struct rds_iw_connection *ic,  in rds_iw_send_grab_credits()  argument
347 	if (!ic->i_flowctl)  in rds_iw_send_grab_credits()
352 	oldval = newval = atomic_read(&ic->i_credits);  in rds_iw_send_grab_credits()
364 		struct rds_connection *conn = ic->i_cm_id->context;  in rds_iw_send_grab_credits()
386 	if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)  in rds_iw_send_grab_credits()
395 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_send_add_credits()  local
402 			IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)),  in rds_iw_send_add_credits()
405 	atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits);  in rds_iw_send_add_credits()
416 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_advertise_credits()  local
421 	atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits);  in rds_iw_advertise_credits()
435 	if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16)  in rds_iw_advertise_credits()
436 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_iw_advertise_credits()
440 rds_iw_xmit_populate_wr(struct rds_iw_connection *ic,  in rds_iw_xmit_populate_wr()  argument
447 	WARN_ON(pos != send - ic->i_sends);  in rds_iw_xmit_populate_wr()
457 		sge = rds_iw_data_sge(ic, send->s_sge);  in rds_iw_xmit_populate_wr()
460 		sge->lkey = rds_iw_local_dma_lkey(ic);  in rds_iw_xmit_populate_wr()
462 		sge = rds_iw_header_sge(ic, send->s_sge);  in rds_iw_xmit_populate_wr()
470 	sge->addr = ic->i_send_hdrs_dma + (pos * sizeof(struct rds_header));  in rds_iw_xmit_populate_wr()
472 	sge->lkey = rds_iw_local_dma_lkey(ic);  in rds_iw_xmit_populate_wr()
491 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_xmit()  local
492 	struct ib_device *dev = ic->i_cm_id->device;  in rds_iw_xmit()
513 	if (rds_rdma_cookie_key(rm->m_rdma_cookie) && !ic->i_fastreg_posted) {  in rds_iw_xmit()
524 	work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_iw_xmit()
533 	if (ic->i_flowctl) {  in rds_iw_xmit()
534 		credit_alloc = rds_iw_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);  in rds_iw_xmit()
537 			rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);  in rds_iw_xmit()
550 	if (!ic->i_rm) {  in rds_iw_xmit()
562 			rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);  in rds_iw_xmit()
565 				rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_iw_xmit()
573 		ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;  in rds_iw_xmit()
574 		ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes;  in rds_iw_xmit()
578 		ic->i_rm = rm;  in rds_iw_xmit()
605 		rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_iw_piggyb_ack(ic));  in rds_iw_xmit()
611 		rds_iw_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);  in rds_iw_xmit()
616 	send = &ic->i_sends[pos];  in rds_iw_xmit()
643 		rds_iw_xmit_populate_wr(ic, send, pos, 0, 0, send_flags);  in rds_iw_xmit()
651 		send = &ic->i_sends[pos];  in rds_iw_xmit()
655 		rds_iw_xmit_populate_wr(ic, send, pos,  in rds_iw_xmit()
664 		if (ic->i_unsignaled_wrs-- == 0) {  in rds_iw_xmit()
665 			ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;  in rds_iw_xmit()
669 		ic->i_unsignaled_bytes -= len;  in rds_iw_xmit()
670 		if (ic->i_unsignaled_bytes <= 0) {  in rds_iw_xmit()
671 			ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes;  in rds_iw_xmit()
695 		memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr, sizeof(struct rds_header));  in rds_iw_xmit()
698 			struct rds_header *hdr = &ic->i_send_hdrs[pos];  in rds_iw_xmit()
706 			struct rds_header *hdr = &ic->i_send_hdrs[pos];  in rds_iw_xmit()
719 		pos = (pos + 1) % ic->i_send_ring.w_nr;  in rds_iw_xmit()
729 		prev->s_rm = ic->i_rm;  in rds_iw_xmit()
731 		ic->i_rm = NULL;  in rds_iw_xmit()
735 		rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_iw_xmit()
738 	if (ic->i_flowctl && i < credit_alloc)  in rds_iw_xmit()
743 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_send_wr, &failed_wr);  in rds_iw_xmit()
744 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_iw_xmit()
750 		rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_iw_xmit()
752 			ic->i_rm = prev->s_rm;  in rds_iw_xmit()
788 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_xmit_rdma()  local
806 	rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client);  in rds_iw_xmit_rdma()
810 		op->op_count = ib_dma_map_sg(ic->i_cm_id->device,  in rds_iw_xmit_rdma()
813 		rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count);  in rds_iw_xmit_rdma()
825 		work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, 1, &fr_pos);  in rds_iw_xmit_rdma()
827 			rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_iw_xmit_rdma()
840 	work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_iw_xmit_rdma()
842 		rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_iw_xmit_rdma()
848 	send = &ic->i_sends[pos];  in rds_iw_xmit_rdma()
850 		first = prev = &ic->i_sends[fr_pos];  in rds_iw_xmit_rdma()
868 		if (ic->i_unsignaled_wrs-- == 0) {  in rds_iw_xmit_rdma()
869 			ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;  in rds_iw_xmit_rdma()
899 			len = ib_sg_dma_len(ic->i_cm_id->device, scat);  in rds_iw_xmit_rdma()
904 				send->s_sge[j].addr = ib_sg_dma_address(ic->i_cm_id->device, scat);  in rds_iw_xmit_rdma()
906 				send->s_sge[j].lkey = rds_iw_local_dma_lkey(ic);  in rds_iw_xmit_rdma()
910 			rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr);  in rds_iw_xmit_rdma()
920 			send->s_sge[0].lkey = ic->i_sends[fr_pos].s_mr->lkey;  in rds_iw_xmit_rdma()
929 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_iw_xmit_rdma()
930 			send = ic->i_sends;  in rds_iw_xmit_rdma()
938 		rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_iw_xmit_rdma()
949 		ret = rds_iw_build_send_reg(&ic->i_sends[fr_pos],  in rds_iw_xmit_rdma()
959 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr);  in rds_iw_xmit_rdma()
960 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_iw_xmit_rdma()
966 		rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_iw_xmit_rdma()
976 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_xmit_complete()  local
980 	rds_iw_attempt_ack(ic);  in rds_iw_xmit_complete()