69 static void rds_iw_recv_unmap_page(struct rds_iw_connection *ic,  in rds_iw_recv_unmap_page()  argument
76 		ib_dma_unmap_page(ic->i_cm_id->device,  in rds_iw_recv_unmap_page()
82 void rds_iw_recv_init_ring(struct rds_iw_connection *ic)  in rds_iw_recv_init_ring()  argument
87 	for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {  in rds_iw_recv_init_ring()
98 		sge = rds_iw_data_sge(ic, recv->r_sge);  in rds_iw_recv_init_ring()
103 		sge = rds_iw_header_sge(ic, recv->r_sge);  in rds_iw_recv_init_ring()
104 		sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));  in rds_iw_recv_init_ring()
110 static void rds_iw_recv_clear_one(struct rds_iw_connection *ic,  in rds_iw_recv_clear_one()  argument
118 		rds_iw_recv_unmap_page(ic, recv);  in rds_iw_recv_clear_one()
126 void rds_iw_recv_clear_ring(struct rds_iw_connection *ic)  in rds_iw_recv_clear_ring()  argument
130 	for (i = 0; i < ic->i_recv_ring.w_nr; i++)  in rds_iw_recv_clear_ring()
131 		rds_iw_recv_clear_one(ic, &ic->i_recvs[i]);  in rds_iw_recv_clear_ring()
133 	if (ic->i_frag.f_page)  in rds_iw_recv_clear_ring()
134 		rds_iw_frag_drop_page(&ic->i_frag);  in rds_iw_recv_clear_ring()
141 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_recv_refill_one()  local
169 	if (!ic->i_frag.f_page) {  in rds_iw_recv_refill_one()
170 		ic->i_frag.f_page = alloc_page(page_gfp);  in rds_iw_recv_refill_one()
171 		if (!ic->i_frag.f_page)  in rds_iw_recv_refill_one()
173 		ic->i_frag.f_offset = 0;  in rds_iw_recv_refill_one()
176 	dma_addr = ib_dma_map_page(ic->i_cm_id->device,  in rds_iw_recv_refill_one()
177 				  ic->i_frag.f_page,  in rds_iw_recv_refill_one()
178 				  ic->i_frag.f_offset,  in rds_iw_recv_refill_one()
181 	if (ib_dma_mapping_error(ic->i_cm_id->device, dma_addr))  in rds_iw_recv_refill_one()
189 	recv->r_frag->f_page = ic->i_frag.f_page;  in rds_iw_recv_refill_one()
190 	recv->r_frag->f_offset = ic->i_frag.f_offset;  in rds_iw_recv_refill_one()
193 	sge = rds_iw_data_sge(ic, recv->r_sge);  in rds_iw_recv_refill_one()
197 	sge = rds_iw_header_sge(ic, recv->r_sge);  in rds_iw_recv_refill_one()
198 	sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);  in rds_iw_recv_refill_one()
203 	if (ic->i_frag.f_offset < RDS_PAGE_LAST_OFF) {  in rds_iw_recv_refill_one()
204 		ic->i_frag.f_offset += RDS_FRAG_SIZE;  in rds_iw_recv_refill_one()
206 		put_page(ic->i_frag.f_page);  in rds_iw_recv_refill_one()
207 		ic->i_frag.f_page = NULL;  in rds_iw_recv_refill_one()
208 		ic->i_frag.f_offset = 0;  in rds_iw_recv_refill_one()
227 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_recv_refill()  local
235 	       rds_iw_ring_alloc(&ic->i_recv_ring, 1, &pos)) {  in rds_iw_recv_refill()
236 		if (pos >= ic->i_recv_ring.w_nr) {  in rds_iw_recv_refill()
243 		recv = &ic->i_recvs[pos];  in rds_iw_recv_refill()
251 		ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, &failed_wr);  in rds_iw_recv_refill()
268 	if (ic->i_flowctl && posted)  in rds_iw_recv_refill()
272 		rds_iw_ring_unalloc(&ic->i_recv_ring, 1);  in rds_iw_recv_refill()
347 void rds_iw_recv_init_ack(struct rds_iw_connection *ic)  in rds_iw_recv_init_ack()  argument
349 	struct ib_send_wr *wr = &ic->i_ack_wr;  in rds_iw_recv_init_ack()
350 	struct ib_sge *sge = &ic->i_ack_sge;  in rds_iw_recv_init_ack()
352 	sge->addr = ic->i_ack_dma;  in rds_iw_recv_init_ack()
354 	sge->lkey = rds_iw_local_dma_lkey(ic);  in rds_iw_recv_init_ack()
386 static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq,  in rds_iw_set_ack()  argument
391 	spin_lock_irqsave(&ic->i_ack_lock, flags);  in rds_iw_set_ack()
392 	ic->i_ack_next = seq;  in rds_iw_set_ack()
394 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_iw_set_ack()
395 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);  in rds_iw_set_ack()
398 static u64 rds_iw_get_ack(struct rds_iw_connection *ic)  in rds_iw_get_ack()  argument
403 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_iw_get_ack()
405 	spin_lock_irqsave(&ic->i_ack_lock, flags);  in rds_iw_get_ack()
406 	seq = ic->i_ack_next;  in rds_iw_get_ack()
407 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);  in rds_iw_get_ack()
412 static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq,  in rds_iw_set_ack()  argument
415 	atomic64_set(&ic->i_ack_next, seq);  in rds_iw_set_ack()
418 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_iw_set_ack()
422 static u64 rds_iw_get_ack(struct rds_iw_connection *ic)  in rds_iw_get_ack()  argument
424 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_iw_get_ack()
427 	return atomic64_read(&ic->i_ack_next);  in rds_iw_get_ack()
432 static void rds_iw_send_ack(struct rds_iw_connection *ic, unsigned int adv_credits)  in rds_iw_send_ack()  argument
434 	struct rds_header *hdr = ic->i_ack;  in rds_iw_send_ack()
439 	seq = rds_iw_get_ack(ic);  in rds_iw_send_ack()
441 	rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq);  in rds_iw_send_ack()
446 	ic->i_ack_queued = jiffies;  in rds_iw_send_ack()
448 	ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, &failed_wr);  in rds_iw_send_ack()
453 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_iw_send_ack()
454 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_iw_send_ack()
458 		rds_iw_conn_error(ic->conn, "sending ack failed\n");  in rds_iw_send_ack()
501 void rds_iw_attempt_ack(struct rds_iw_connection *ic)  in rds_iw_attempt_ack()  argument
505 	if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))  in rds_iw_attempt_ack()
508 	if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) {  in rds_iw_attempt_ack()
514 	if (!rds_iw_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {  in rds_iw_attempt_ack()
516 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_iw_attempt_ack()
520 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_iw_attempt_ack()
521 	rds_iw_send_ack(ic, adv_credits);  in rds_iw_attempt_ack()
528 void rds_iw_ack_send_complete(struct rds_iw_connection *ic)  in rds_iw_ack_send_complete()  argument
530 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_iw_ack_send_complete()
531 	rds_iw_attempt_ack(ic);  in rds_iw_ack_send_complete()
538 u64 rds_iw_piggyb_ack(struct rds_iw_connection *ic)  in rds_iw_piggyb_ack()  argument
540 	if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))  in rds_iw_piggyb_ack()
542 	return rds_iw_get_ack(ic);  in rds_iw_piggyb_ack()
638 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_process_recv()  local
639 	struct rds_iw_incoming *iwinc = ic->i_iwinc;  in rds_iw_process_recv()
644 	rdsdebug("ic %p iwinc %p recv %p byte len %u\n", ic, iwinc, recv,  in rds_iw_process_recv()
657 	ihdr = &ic->i_recv_hdrs[recv - ic->i_recvs];  in rds_iw_process_recv()
707 		ic->i_iwinc = iwinc;  in rds_iw_process_recv()
711 		ic->i_recv_data_rem = be32_to_cpu(hdr->h_len);  in rds_iw_process_recv()
713 		rdsdebug("ic %p iwinc %p rem %u flag 0x%x\n", ic, iwinc,  in rds_iw_process_recv()
714 			 ic->i_recv_data_rem, hdr->h_flags);  in rds_iw_process_recv()
732 	if (ic->i_recv_data_rem > RDS_FRAG_SIZE)  in rds_iw_process_recv()
733 		ic->i_recv_data_rem -= RDS_FRAG_SIZE;  in rds_iw_process_recv()
735 		ic->i_recv_data_rem = 0;  in rds_iw_process_recv()
736 		ic->i_iwinc = NULL;  in rds_iw_process_recv()
771 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_recv_cq_comp_handler()  local
777 	tasklet_schedule(&ic->i_recv_tasklet);  in rds_iw_recv_cq_comp_handler()
780 static inline void rds_poll_cq(struct rds_iw_connection *ic,  in rds_poll_cq()  argument
783 	struct rds_connection *conn = ic->conn;  in rds_poll_cq()
787 	while (ib_poll_cq(ic->i_recv_cq, 1, &wc) > 0) {  in rds_poll_cq()
793 		recv = &ic->i_recvs[rds_iw_ring_oldest(&ic->i_recv_ring)];  in rds_poll_cq()
795 		rds_iw_recv_unmap_page(ic, recv);  in rds_poll_cq()
814 		rds_iw_ring_free(&ic->i_recv_ring, 1);  in rds_poll_cq()
820 	struct rds_iw_connection *ic = (struct rds_iw_connection *) data;  in rds_iw_recv_tasklet_fn()  local
821 	struct rds_connection *conn = ic->conn;  in rds_iw_recv_tasklet_fn()
824 	rds_poll_cq(ic, &state);  in rds_iw_recv_tasklet_fn()
825 	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_iw_recv_tasklet_fn()
826 	rds_poll_cq(ic, &state);  in rds_iw_recv_tasklet_fn()
829 		rds_iw_set_ack(ic, state.ack_next, state.ack_required);  in rds_iw_recv_tasklet_fn()
830 	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {  in rds_iw_recv_tasklet_fn()
832 		ic->i_ack_recv = state.ack_recv;  in rds_iw_recv_tasklet_fn()
835 		rds_iw_attempt_ack(ic);  in rds_iw_recv_tasklet_fn()
840 	if (rds_iw_ring_empty(&ic->i_recv_ring))  in rds_iw_recv_tasklet_fn()
846 	if (rds_iw_ring_low(&ic->i_recv_ring))  in rds_iw_recv_tasklet_fn()
852 	struct rds_iw_connection *ic = conn->c_transport_data;  in rds_iw_recv()  local
861 	mutex_lock(&ic->i_recv_mutex);  in rds_iw_recv()
866 	mutex_unlock(&ic->i_recv_mutex);  in rds_iw_recv()
869 		rds_iw_attempt_ack(ic);  in rds_iw_recv()