183 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq)  in insert_recv_cqe()  argument
188 	     wq, cq, cq->sw_cidx, cq->sw_pidx);  in insert_recv_cqe()
194 				 CQE_QPID_V(wq->sq.qid));  in insert_recv_cqe()
200 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)  in c4iw_flush_rq()  argument
203 	int in_use = wq->rq.in_use - count;  in c4iw_flush_rq()
207 	     wq, cq, wq->rq.in_use, count);  in c4iw_flush_rq()
209 		insert_recv_cqe(wq, cq);  in c4iw_flush_rq()
215 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,  in insert_sq_cqe()  argument
221 	     wq, cq, cq->sw_cidx, cq->sw_pidx);  in insert_sq_cqe()
227 				 CQE_QPID_V(wq->sq.qid));  in insert_sq_cqe()
234 static void advance_oldest_read(struct t4_wq *wq);
239 	struct t4_wq *wq = &qhp->wq;  in c4iw_flush_sq()  local
245 	if (wq->sq.flush_cidx == -1)  in c4iw_flush_sq()
246 		wq->sq.flush_cidx = wq->sq.cidx;  in c4iw_flush_sq()
247 	idx = wq->sq.flush_cidx;  in c4iw_flush_sq()
248 	BUG_ON(idx >= wq->sq.size);  in c4iw_flush_sq()
249 	while (idx != wq->sq.pidx) {  in c4iw_flush_sq()
250 		swsqe = &wq->sq.sw_sq[idx];  in c4iw_flush_sq()
253 		insert_sq_cqe(wq, cq, swsqe);  in c4iw_flush_sq()
254 		if (wq->sq.oldest_read == swsqe) {  in c4iw_flush_sq()
256 			advance_oldest_read(wq);  in c4iw_flush_sq()
259 		if (++idx == wq->sq.size)  in c4iw_flush_sq()
262 	wq->sq.flush_cidx += flushed;  in c4iw_flush_sq()
263 	if (wq->sq.flush_cidx >= wq->sq.size)  in c4iw_flush_sq()
264 		wq->sq.flush_cidx -= wq->sq.size;  in c4iw_flush_sq()
268 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)  in flush_completed_wrs()  argument
273 	if (wq->sq.flush_cidx == -1)  in flush_completed_wrs()
274 		wq->sq.flush_cidx = wq->sq.cidx;  in flush_completed_wrs()
275 	cidx = wq->sq.flush_cidx;  in flush_completed_wrs()
276 	BUG_ON(cidx > wq->sq.size);  in flush_completed_wrs()
278 	while (cidx != wq->sq.pidx) {  in flush_completed_wrs()
279 		swsqe = &wq->sq.sw_sq[cidx];  in flush_completed_wrs()
281 			if (++cidx == wq->sq.size)  in flush_completed_wrs()
296 			if (++cidx == wq->sq.size)  in flush_completed_wrs()
298 			wq->sq.flush_cidx = cidx;  in flush_completed_wrs()
304 static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,  in create_read_req_cqe()  argument
307 	read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;  in create_read_req_cqe()
308 	read_cqe->len = htonl(wq->sq.oldest_read->read_len);  in create_read_req_cqe()
316 static void advance_oldest_read(struct t4_wq *wq)  in advance_oldest_read()  argument
319 	u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;  in advance_oldest_read()
321 	if (rptr == wq->sq.size)  in advance_oldest_read()
323 	while (rptr != wq->sq.pidx) {  in advance_oldest_read()
324 		wq->sq.oldest_read = &wq->sq.sw_sq[rptr];  in advance_oldest_read()
326 		if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)  in advance_oldest_read()
328 		if (++rptr == wq->sq.size)  in advance_oldest_read()
331 	wq->sq.oldest_read = NULL;  in advance_oldest_read()
383 			if (!qhp->wq.sq.oldest_read->signaled) {  in c4iw_flush_hw_cq()
384 				advance_oldest_read(&qhp->wq);  in c4iw_flush_hw_cq()
392 			create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);  in c4iw_flush_hw_cq()
394 			advance_oldest_read(&qhp->wq);  in c4iw_flush_hw_cq()
401 			swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];  in c4iw_flush_hw_cq()
404 			flush_completed_wrs(&qhp->wq, &chp->cq);  in c4iw_flush_hw_cq()
417 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)  in cqe_completes_wr()  argument
428 	if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))  in cqe_completes_wr()
433 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)  in c4iw_count_rcqes()  argument
444 		    (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))  in c4iw_count_rcqes()
468 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,  in poll_cq()  argument
490 	if (wq == NULL) {  in poll_cq()
498 	if (wq->flushed && !SW_CQE(hw_cqe)) {  in poll_cq()
526 				t4_set_wq_in_error(wq);  in poll_cq()
537 				t4_set_wq_in_error(wq);  in poll_cq()
545 		if (!wq->sq.oldest_read->signaled) {  in poll_cq()
546 			advance_oldest_read(wq);  in poll_cq()
555 		create_read_req_cqe(wq, hw_cqe, &read_cqe);  in poll_cq()
557 		advance_oldest_read(wq);  in poll_cq()
560 	if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {  in poll_cq()
562 		t4_set_wq_in_error(wq);  in poll_cq()
577 		if (t4_rq_empty(wq)) {  in poll_cq()
578 			t4_set_wq_in_error(wq);  in poll_cq()
582 		if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) {  in poll_cq()
583 			t4_set_wq_in_error(wq);  in poll_cq()
601 	if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) {  in poll_cq()
606 		swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];  in poll_cq()
622 		BUG_ON(idx >= wq->sq.size);  in poll_cq()
632 		if (idx < wq->sq.cidx)  in poll_cq()
633 			wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;  in poll_cq()
635 			wq->sq.in_use -= idx - wq->sq.cidx;  in poll_cq()
636 		BUG_ON(wq->sq.in_use <= 0 && wq->sq.in_use >= wq->sq.size);  in poll_cq()
638 		wq->sq.cidx = (uint16_t)idx;  in poll_cq()
639 		PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx);  in poll_cq()
640 		*cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;  in poll_cq()
642 			c4iw_log_wr_stats(wq, hw_cqe);  in poll_cq()
643 		t4_sq_consume(wq);  in poll_cq()
645 		PDBG("%s completing rq idx %u\n", __func__, wq->rq.cidx);  in poll_cq()
646 		*cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;  in poll_cq()
647 		BUG_ON(t4_rq_empty(wq));  in poll_cq()
649 			c4iw_log_wr_stats(wq, hw_cqe);  in poll_cq()
650 		t4_rq_consume(wq);  in poll_cq()
658 	flush_completed_wrs(wq, cq);  in poll_cq()
686 	struct t4_wq *wq;  in c4iw_poll_cq_one()  local
699 		wq = NULL;  in c4iw_poll_cq_one()
702 		wq = &(qhp->wq);  in c4iw_poll_cq_one()
704 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);  in c4iw_poll_cq_one()
821 	if (wq)  in c4iw_poll_cq_one()