46 void rds_ib_recv_init_ring(struct rds_ib_connection *ic)  in rds_ib_recv_init_ring()  argument
51 	for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {  in rds_ib_recv_init_ring()
63 		sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));  in rds_ib_recv_init_ring()
65 		sge->lkey = ic->i_pd->local_dma_lkey;  in rds_ib_recv_init_ring()
70 		sge->lkey = ic->i_pd->local_dma_lkey;  in rds_ib_recv_init_ring()
120 int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic)  in rds_ib_recv_alloc_caches()  argument
124 	ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs);  in rds_ib_recv_alloc_caches()
126 		ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags);  in rds_ib_recv_alloc_caches()
128 			free_percpu(ic->i_cache_incs.percpu);  in rds_ib_recv_alloc_caches()
154 void rds_ib_recv_free_caches(struct rds_ib_connection *ic)  in rds_ib_recv_free_caches()  argument
162 	rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);  in rds_ib_recv_free_caches()
163 	rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list);  in rds_ib_recv_free_caches()
164 	free_percpu(ic->i_cache_incs.percpu);  in rds_ib_recv_free_caches()
172 	rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);  in rds_ib_recv_free_caches()
173 	rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list);  in rds_ib_recv_free_caches()
174 	free_percpu(ic->i_cache_frags.percpu);  in rds_ib_recv_free_caches()
190 static void rds_ib_frag_free(struct rds_ib_connection *ic,  in rds_ib_frag_free()  argument
195 	rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags);  in rds_ib_frag_free()
204 	struct rds_ib_connection *ic = inc->i_conn->c_transport_data;  in rds_ib_inc_free()  local
211 		rds_ib_frag_free(ic, frag);  in rds_ib_inc_free()
216 	rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs);  in rds_ib_inc_free()
219 static void rds_ib_recv_clear_one(struct rds_ib_connection *ic,  in rds_ib_recv_clear_one()  argument
227 		ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE);  in rds_ib_recv_clear_one()
228 		rds_ib_frag_free(ic, recv->r_frag);  in rds_ib_recv_clear_one()
233 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)  in rds_ib_recv_clear_ring()  argument
237 	for (i = 0; i < ic->i_recv_ring.w_nr; i++)  in rds_ib_recv_clear_ring()
238 		rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);  in rds_ib_recv_clear_ring()
241 static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic,  in rds_ib_refill_one_inc()  argument
248 	cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs);  in rds_ib_refill_one_inc()
265 	rds_inc_init(&ibinc->ii_inc, ic->conn, ic->conn->c_faddr);  in rds_ib_refill_one_inc()
270 static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic,  in rds_ib_refill_one_frag()  argument
277 	cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags);  in rds_ib_refill_one_frag()
302 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_recv_refill_one()  local
313 	if (!ic->i_cache_incs.ready)  in rds_ib_recv_refill_one()
314 		rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);  in rds_ib_recv_refill_one()
315 	if (!ic->i_cache_frags.ready)  in rds_ib_recv_refill_one()
316 		rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);  in rds_ib_recv_refill_one()
323 		recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask);  in rds_ib_recv_refill_one()
329 	recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask);  in rds_ib_recv_refill_one()
333 	ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg,  in rds_ib_recv_refill_one()
338 	sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);  in rds_ib_recv_refill_one()
342 	sge->addr = ib_sg_dma_address(ic->i_cm_id->device, &recv->r_frag->f_sg);  in rds_ib_recv_refill_one()
343 	sge->length = ib_sg_dma_len(ic->i_cm_id->device, &recv->r_frag->f_sg);  in rds_ib_recv_refill_one()
377 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_recv_refill()  local
393 	       rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) {  in rds_ib_recv_refill()
394 		if (pos >= ic->i_recv_ring.w_nr) {  in rds_ib_recv_refill()
400 		recv = &ic->i_recvs[pos];  in rds_ib_recv_refill()
407 		ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, &failed_wr);  in rds_ib_recv_refill()
411 				ic->i_cm_id->device,  in rds_ib_recv_refill()
426 	if (ic->i_flowctl && posted)  in rds_ib_recv_refill()
430 		rds_ib_ring_unalloc(&ic->i_recv_ring, 1);  in rds_ib_recv_refill()
445 	    ((can_wait && rds_ib_ring_low(&ic->i_recv_ring)) ||  in rds_ib_recv_refill()
446 	    rds_ib_ring_empty(&ic->i_recv_ring))) {  in rds_ib_recv_refill()
560 void rds_ib_recv_init_ack(struct rds_ib_connection *ic)  in rds_ib_recv_init_ack()  argument
562 	struct ib_send_wr *wr = &ic->i_ack_wr;  in rds_ib_recv_init_ack()
563 	struct ib_sge *sge = &ic->i_ack_sge;  in rds_ib_recv_init_ack()
565 	sge->addr = ic->i_ack_dma;  in rds_ib_recv_init_ack()
567 	sge->lkey = ic->i_pd->local_dma_lkey;  in rds_ib_recv_init_ack()
599 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required)  in rds_ib_set_ack()  argument
603 	spin_lock_irqsave(&ic->i_ack_lock, flags);  in rds_ib_set_ack()
604 	ic->i_ack_next = seq;  in rds_ib_set_ack()
606 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_set_ack()
607 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);  in rds_ib_set_ack()
610 static u64 rds_ib_get_ack(struct rds_ib_connection *ic)  in rds_ib_get_ack()  argument
615 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_get_ack()
617 	spin_lock_irqsave(&ic->i_ack_lock, flags);  in rds_ib_get_ack()
618 	seq = ic->i_ack_next;  in rds_ib_get_ack()
619 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);  in rds_ib_get_ack()
624 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required)  in rds_ib_set_ack()  argument
626 	atomic64_set(&ic->i_ack_next, seq);  in rds_ib_set_ack()
629 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_set_ack()
633 static u64 rds_ib_get_ack(struct rds_ib_connection *ic)  in rds_ib_get_ack()  argument
635 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_get_ack()
638 	return atomic64_read(&ic->i_ack_next);  in rds_ib_get_ack()
643 static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits)  in rds_ib_send_ack()  argument
645 	struct rds_header *hdr = ic->i_ack;  in rds_ib_send_ack()
650 	seq = rds_ib_get_ack(ic);  in rds_ib_send_ack()
652 	rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq);  in rds_ib_send_ack()
657 	ic->i_ack_queued = jiffies;  in rds_ib_send_ack()
659 	ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, &failed_wr);  in rds_ib_send_ack()
664 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_ib_send_ack()
665 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_send_ack()
669 		rds_ib_conn_error(ic->conn, "sending ack failed\n");  in rds_ib_send_ack()
712 void rds_ib_attempt_ack(struct rds_ib_connection *ic)  in rds_ib_attempt_ack()  argument
716 	if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))  in rds_ib_attempt_ack()
719 	if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) {  in rds_ib_attempt_ack()
725 	if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {  in rds_ib_attempt_ack()
727 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_ib_attempt_ack()
731 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_attempt_ack()
732 	rds_ib_send_ack(ic, adv_credits);  in rds_ib_attempt_ack()
739 void rds_ib_ack_send_complete(struct rds_ib_connection *ic)  in rds_ib_ack_send_complete()  argument
741 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_ib_ack_send_complete()
742 	rds_ib_attempt_ack(ic);  in rds_ib_ack_send_complete()
749 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic)  in rds_ib_piggyb_ack()  argument
751 	if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))  in rds_ib_piggyb_ack()
753 	return rds_ib_get_ack(ic);  in rds_ib_piggyb_ack()
835 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_process_recv()  local
836 	struct rds_ib_incoming *ibinc = ic->i_ibinc;  in rds_ib_process_recv()
841 	rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv,  in rds_ib_process_recv()
854 	ihdr = &ic->i_recv_hdrs[recv - ic->i_recvs];  in rds_ib_process_recv()
890 		rds_ib_frag_free(ic, recv->r_frag);  in rds_ib_process_recv()
904 		ic->i_ibinc = ibinc;  in rds_ib_process_recv()
908 		ic->i_recv_data_rem = be32_to_cpu(hdr->h_len);  in rds_ib_process_recv()
910 		rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc,  in rds_ib_process_recv()
911 			 ic->i_recv_data_rem, hdr->h_flags);  in rds_ib_process_recv()
929 	if (ic->i_recv_data_rem > RDS_FRAG_SIZE)  in rds_ib_process_recv()
930 		ic->i_recv_data_rem -= RDS_FRAG_SIZE;  in rds_ib_process_recv()
932 		ic->i_recv_data_rem = 0;  in rds_ib_process_recv()
933 		ic->i_ibinc = NULL;  in rds_ib_process_recv()
956 void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic,  in rds_ib_recv_cqe_handler()  argument
960 	struct rds_connection *conn = ic->conn;  in rds_ib_recv_cqe_handler()
969 	recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)];  in rds_ib_recv_cqe_handler()
970 	ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1,  in rds_ib_recv_cqe_handler()
996 		rds_ib_frag_free(ic, recv->r_frag);  in rds_ib_recv_cqe_handler()
999 	rds_ib_ring_free(&ic->i_recv_ring, 1);  in rds_ib_recv_cqe_handler()
1004 	if (rds_ib_ring_empty(&ic->i_recv_ring))  in rds_ib_recv_cqe_handler()
1007 	if (rds_ib_ring_low(&ic->i_recv_ring))  in rds_ib_recv_cqe_handler()
1013 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_recv()  local
1018 		rds_ib_attempt_ack(ic);  in rds_ib_recv()