Lines Matching refs:qp
1196 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) in ocrdma_add_qpn_map() argument
1200 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { in ocrdma_add_qpn_map()
1201 dev->qp_tbl[qp->id] = qp; in ocrdma_add_qpn_map()
1207 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) in ocrdma_del_qpn_map() argument
1209 dev->qp_tbl[qp->id] = NULL; in ocrdma_del_qpn_map()
1285 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, in ocrdma_copy_qp_uresp() argument
1292 struct ocrdma_pd *pd = qp->pd; in ocrdma_copy_qp_uresp()
1298 uresp.qp_id = qp->id; in ocrdma_copy_qp_uresp()
1299 uresp.sq_dbid = qp->sq.dbid; in ocrdma_copy_qp_uresp()
1301 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); in ocrdma_copy_qp_uresp()
1302 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); in ocrdma_copy_qp_uresp()
1303 uresp.num_wqe_allocated = qp->sq.max_cnt; in ocrdma_copy_qp_uresp()
1305 uresp.rq_dbid = qp->rq.dbid; in ocrdma_copy_qp_uresp()
1307 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); in ocrdma_copy_qp_uresp()
1308 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); in ocrdma_copy_qp_uresp()
1309 uresp.num_rqe_allocated = qp->rq.max_cnt; in ocrdma_copy_qp_uresp()
1317 if (qp->dpp_enabled) { in ocrdma_copy_qp_uresp()
1344 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, in ocrdma_set_qp_db() argument
1348 qp->sq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1351 qp->rq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1355 qp->sq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1358 qp->rq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1364 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) in ocrdma_alloc_wr_id_tbl() argument
1366 qp->wqe_wr_id_tbl = in ocrdma_alloc_wr_id_tbl()
1367 kzalloc(sizeof(*(qp->wqe_wr_id_tbl)) * qp->sq.max_cnt, in ocrdma_alloc_wr_id_tbl()
1369 if (qp->wqe_wr_id_tbl == NULL) in ocrdma_alloc_wr_id_tbl()
1371 qp->rqe_wr_id_tbl = in ocrdma_alloc_wr_id_tbl()
1372 kzalloc(sizeof(u64) * qp->rq.max_cnt, GFP_KERNEL); in ocrdma_alloc_wr_id_tbl()
1373 if (qp->rqe_wr_id_tbl == NULL) in ocrdma_alloc_wr_id_tbl()
1379 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, in ocrdma_set_qp_init_params() argument
1383 qp->pd = pd; in ocrdma_set_qp_init_params()
1384 spin_lock_init(&qp->q_lock); in ocrdma_set_qp_init_params()
1385 INIT_LIST_HEAD(&qp->sq_entry); in ocrdma_set_qp_init_params()
1386 INIT_LIST_HEAD(&qp->rq_entry); in ocrdma_set_qp_init_params()
1388 qp->qp_type = attrs->qp_type; in ocrdma_set_qp_init_params()
1389 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; in ocrdma_set_qp_init_params()
1390 qp->max_inline_data = attrs->cap.max_inline_data; in ocrdma_set_qp_init_params()
1391 qp->sq.max_sges = attrs->cap.max_send_sge; in ocrdma_set_qp_init_params()
1392 qp->rq.max_sges = attrs->cap.max_recv_sge; in ocrdma_set_qp_init_params()
1393 qp->state = OCRDMA_QPS_RST; in ocrdma_set_qp_init_params()
1394 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; in ocrdma_set_qp_init_params()
1413 struct ocrdma_qp *qp; in ocrdma_create_qp() local
1427 qp = kzalloc(sizeof(*qp), GFP_KERNEL); in ocrdma_create_qp()
1428 if (!qp) { in ocrdma_create_qp()
1432 ocrdma_set_qp_init_params(qp, pd, attrs); in ocrdma_create_qp()
1434 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | in ocrdma_create_qp()
1438 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, in ocrdma_create_qp()
1446 status = ocrdma_alloc_wr_id_tbl(qp); in ocrdma_create_qp()
1451 status = ocrdma_add_qpn_map(dev, qp); in ocrdma_create_qp()
1454 ocrdma_set_qp_db(dev, qp, pd); in ocrdma_create_qp()
1456 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, in ocrdma_create_qp()
1463 qp->ibqp.qp_num = qp->id; in ocrdma_create_qp()
1465 return &qp->ibqp; in ocrdma_create_qp()
1468 ocrdma_del_qpn_map(dev, qp); in ocrdma_create_qp()
1470 ocrdma_mbx_destroy_qp(dev, qp); in ocrdma_create_qp()
1473 kfree(qp->wqe_wr_id_tbl); in ocrdma_create_qp()
1474 kfree(qp->rqe_wr_id_tbl); in ocrdma_create_qp()
1475 kfree(qp); in ocrdma_create_qp()
1485 struct ocrdma_qp *qp; in _ocrdma_modify_qp() local
1489 qp = get_ocrdma_qp(ibqp); in _ocrdma_modify_qp()
1492 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); in _ocrdma_modify_qp()
1498 status = ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); in _ocrdma_modify_qp()
1508 struct ocrdma_qp *qp; in ocrdma_modify_qp() local
1512 qp = get_ocrdma_qp(ibqp); in ocrdma_modify_qp()
1518 spin_lock_irqsave(&qp->q_lock, flags); in ocrdma_modify_qp()
1519 old_qps = get_ibqp_state(qp->state); in ocrdma_modify_qp()
1524 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_modify_qp()
1530 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, in ocrdma_modify_qp()
1579 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); in ocrdma_query_qp() local
1584 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); in ocrdma_query_qp()
1588 if (qp->qp_type == IB_QPT_UD) in ocrdma_query_qp()
1600 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); in ocrdma_query_qp()
1601 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; in ocrdma_query_qp()
1602 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; in ocrdma_query_qp()
1603 qp_attr->cap.max_send_sge = qp->sq.max_sges; in ocrdma_query_qp()
1604 qp_attr->cap.max_recv_sge = qp->rq.max_sges; in ocrdma_query_qp()
1605 qp_attr->cap.max_inline_data = qp->max_inline_data; in ocrdma_query_qp()
1611 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; in ocrdma_query_qp()
1654 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); in ocrdma_query_qp()
1672 static int is_hw_sq_empty(struct ocrdma_qp *qp) in is_hw_sq_empty() argument
1674 return (qp->sq.tail == qp->sq.head); in is_hw_sq_empty()
1677 static int is_hw_rq_empty(struct ocrdma_qp *qp) in is_hw_rq_empty() argument
1679 return (qp->rq.tail == qp->rq.head); in is_hw_rq_empty()
1704 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) in ocrdma_discard_cqes() argument
1726 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) in ocrdma_discard_cqes()
1737 if (qpn == 0 || qpn != qp->id) in ocrdma_discard_cqes()
1741 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_discard_cqes()
1743 if (qp->srq) { in ocrdma_discard_cqes()
1746 qp->srq->rq.max_wqe_idx; in ocrdma_discard_cqes()
1749 spin_lock_irqsave(&qp->srq->q_lock, flags); in ocrdma_discard_cqes()
1750 ocrdma_hwq_inc_tail(&qp->srq->rq); in ocrdma_discard_cqes()
1751 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); in ocrdma_discard_cqes()
1752 spin_unlock_irqrestore(&qp->srq->q_lock, flags); in ocrdma_discard_cqes()
1755 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_discard_cqes()
1769 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) in ocrdma_del_flush_qp() argument
1773 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); in ocrdma_del_flush_qp()
1777 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); in ocrdma_del_flush_qp()
1779 list_del(&qp->sq_entry); in ocrdma_del_flush_qp()
1780 if (!qp->srq) { in ocrdma_del_flush_qp()
1781 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); in ocrdma_del_flush_qp()
1783 list_del(&qp->rq_entry); in ocrdma_del_flush_qp()
1791 struct ocrdma_qp *qp; in ocrdma_destroy_qp() local
1797 qp = get_ocrdma_qp(ibqp); in ocrdma_destroy_qp()
1800 pd = qp->pd; in ocrdma_destroy_qp()
1803 if (qp->state != OCRDMA_QPS_RST) { in ocrdma_destroy_qp()
1813 (void) ocrdma_mbx_destroy_qp(dev, qp); in ocrdma_destroy_qp()
1819 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); in ocrdma_destroy_qp()
1820 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) in ocrdma_destroy_qp()
1821 spin_lock(&qp->rq_cq->cq_lock); in ocrdma_destroy_qp()
1823 ocrdma_del_qpn_map(dev, qp); in ocrdma_destroy_qp()
1825 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) in ocrdma_destroy_qp()
1826 spin_unlock(&qp->rq_cq->cq_lock); in ocrdma_destroy_qp()
1827 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); in ocrdma_destroy_qp()
1830 ocrdma_discard_cqes(qp, qp->sq_cq); in ocrdma_destroy_qp()
1831 ocrdma_discard_cqes(qp, qp->rq_cq); in ocrdma_destroy_qp()
1836 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, in ocrdma_destroy_qp()
1837 PAGE_ALIGN(qp->sq.len)); in ocrdma_destroy_qp()
1838 if (!qp->srq) in ocrdma_destroy_qp()
1839 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, in ocrdma_destroy_qp()
1840 PAGE_ALIGN(qp->rq.len)); in ocrdma_destroy_qp()
1843 ocrdma_del_flush_qp(qp); in ocrdma_destroy_qp()
1845 kfree(qp->wqe_wr_id_tbl); in ocrdma_destroy_qp()
1846 kfree(qp->rqe_wr_id_tbl); in ocrdma_destroy_qp()
1847 kfree(qp); in ocrdma_destroy_qp()
1995 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, in ocrdma_build_ud_hdr() argument
2004 if (qp->qp_type == IB_QPT_GSI) in ocrdma_build_ud_hdr()
2005 ud_hdr->qkey = qp->qkey; in ocrdma_build_ud_hdr()
2040 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, in ocrdma_build_inline_sges() argument
2048 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { in ocrdma_build_inline_sges()
2050 if (unlikely(hdr->total_len > qp->max_inline_data)) { in ocrdma_build_inline_sges()
2053 qp->max_inline_data, hdr->total_len); in ocrdma_build_inline_sges()
2080 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, in ocrdma_build_send() argument
2087 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { in ocrdma_build_send()
2088 ocrdma_build_ud_hdr(qp, hdr, wr); in ocrdma_build_send()
2095 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); in ocrdma_build_send()
2099 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, in ocrdma_build_write() argument
2107 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); in ocrdma_build_write()
2117 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, in ocrdma_build_read() argument
2146 static int ocrdma_build_reg(struct ocrdma_qp *qp, in ocrdma_build_reg() argument
2202 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) in ocrdma_ring_sq_db() argument
2204 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); in ocrdma_ring_sq_db()
2206 iowrite32(val, qp->sq_db); in ocrdma_ring_sq_db()
2213 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); in ocrdma_post_send() local
2217 spin_lock_irqsave(&qp->q_lock, flags); in ocrdma_post_send()
2218 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { in ocrdma_post_send()
2219 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_send()
2225 if (qp->qp_type == IB_QPT_UD && in ocrdma_post_send()
2232 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || in ocrdma_post_send()
2233 wr->num_sge > qp->sq.max_sges) { in ocrdma_post_send()
2238 hdr = ocrdma_hwq_head(&qp->sq); in ocrdma_post_send()
2240 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) in ocrdma_post_send()
2255 ocrdma_build_send(qp, hdr, wr); in ocrdma_post_send()
2261 status = ocrdma_build_send(qp, hdr, wr); in ocrdma_post_send()
2268 status = ocrdma_build_write(qp, hdr, wr); in ocrdma_post_send()
2271 ocrdma_build_read(qp, hdr, wr); in ocrdma_post_send()
2282 status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); in ocrdma_post_send()
2292 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) in ocrdma_post_send()
2293 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; in ocrdma_post_send()
2295 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; in ocrdma_post_send()
2296 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; in ocrdma_post_send()
2302 ocrdma_ring_sq_db(qp); in ocrdma_post_send()
2305 ocrdma_hwq_inc_head(&qp->sq); in ocrdma_post_send()
2308 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_send()
2312 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) in ocrdma_ring_rq_db() argument
2314 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); in ocrdma_ring_rq_db()
2316 iowrite32(val, qp->rq_db); in ocrdma_ring_rq_db()
2345 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); in ocrdma_post_recv() local
2348 spin_lock_irqsave(&qp->q_lock, flags); in ocrdma_post_recv()
2349 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { in ocrdma_post_recv()
2350 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_recv()
2355 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || in ocrdma_post_recv()
2356 wr->num_sge > qp->rq.max_sges) { in ocrdma_post_recv()
2361 rqe = ocrdma_hwq_head(&qp->rq); in ocrdma_post_recv()
2364 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; in ocrdma_post_recv()
2369 ocrdma_ring_rq_db(qp); in ocrdma_post_recv()
2372 ocrdma_hwq_inc_head(&qp->rq); in ocrdma_post_recv()
2375 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_recv()
2523 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, in ocrdma_update_wc() argument
2530 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); in ocrdma_update_wc()
2532 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; in ocrdma_update_wc()
2561 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, in ocrdma_set_cqe_status_flushed() argument
2573 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { in ocrdma_set_cqe_status_flushed()
2594 struct ocrdma_qp *qp, int status) in ocrdma_update_err_cqe() argument
2599 ibwc->qp = &qp->ibqp; in ocrdma_update_err_cqe()
2602 ocrdma_flush_qp(qp); in ocrdma_update_err_cqe()
2603 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); in ocrdma_update_err_cqe()
2608 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { in ocrdma_update_err_cqe()
2610 ocrdma_set_cqe_status_flushed(qp, cqe); in ocrdma_update_err_cqe()
2616 struct ocrdma_qp *qp, int status) in ocrdma_update_err_rcqe() argument
2619 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; in ocrdma_update_err_rcqe()
2620 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_update_err_rcqe()
2622 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); in ocrdma_update_err_rcqe()
2626 struct ocrdma_qp *qp, int status) in ocrdma_update_err_scqe() argument
2628 ocrdma_update_wc(qp, ibwc, qp->sq.tail); in ocrdma_update_err_scqe()
2629 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_update_err_scqe()
2631 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); in ocrdma_update_err_scqe()
2635 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, in ocrdma_poll_err_scqe() argument
2640 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); in ocrdma_poll_err_scqe()
2649 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { in ocrdma_poll_err_scqe()
2653 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { in ocrdma_poll_err_scqe()
2656 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); in ocrdma_poll_err_scqe()
2667 } else if (is_hw_sq_empty(qp)) { in ocrdma_poll_err_scqe()
2674 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); in ocrdma_poll_err_scqe()
2679 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, in ocrdma_poll_success_scqe() argument
2684 int tail = qp->sq.tail; in ocrdma_poll_success_scqe()
2687 if (!qp->wqe_wr_id_tbl[tail].signaled) { in ocrdma_poll_success_scqe()
2692 ibwc->qp = &qp->ibqp; in ocrdma_poll_success_scqe()
2693 ocrdma_update_wc(qp, ibwc, tail); in ocrdma_poll_success_scqe()
2697 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; in ocrdma_poll_success_scqe()
2701 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_poll_success_scqe()
2705 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, in ocrdma_poll_scqe() argument
2715 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); in ocrdma_poll_scqe()
2717 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); in ocrdma_poll_scqe()
2739 struct ocrdma_qp *qp) in ocrdma_update_free_srq_cqe() argument
2745 srq = get_ocrdma_srq(qp->ibqp.srq); in ocrdma_update_free_srq_cqe()
2758 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, in ocrdma_poll_err_rcqe() argument
2763 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); in ocrdma_poll_err_rcqe()
2771 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { in ocrdma_poll_err_rcqe()
2772 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { in ocrdma_poll_err_rcqe()
2775 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); in ocrdma_poll_err_rcqe()
2781 } else if (is_hw_rq_empty(qp)) { in ocrdma_poll_err_rcqe()
2788 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); in ocrdma_poll_err_rcqe()
2793 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, in ocrdma_poll_success_rcqe() argument
2797 ibwc->qp = &qp->ibqp; in ocrdma_poll_success_rcqe()
2800 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) in ocrdma_poll_success_rcqe()
2816 if (qp->ibqp.srq) { in ocrdma_poll_success_rcqe()
2817 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); in ocrdma_poll_success_rcqe()
2819 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; in ocrdma_poll_success_rcqe()
2820 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_poll_success_rcqe()
2824 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, in ocrdma_poll_rcqe() argument
2831 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { in ocrdma_poll_rcqe()
2842 ocrdma_poll_success_rcqe(qp, cqe, ibwc); in ocrdma_poll_rcqe()
2844 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, in ocrdma_poll_rcqe()
2869 struct ocrdma_qp *qp = NULL; in ocrdma_poll_hwcq() local
2884 qp = dev->qp_tbl[qpn]; in ocrdma_poll_hwcq()
2885 BUG_ON(qp == NULL); in ocrdma_poll_hwcq()
2888 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, in ocrdma_poll_hwcq()
2891 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, in ocrdma_poll_hwcq()
2926 struct ocrdma_qp *qp, struct ib_wc *ibwc) in ocrdma_add_err_cqe() argument
2931 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) in ocrdma_add_err_cqe()
2933 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { in ocrdma_add_err_cqe()
2934 ocrdma_update_wc(qp, ibwc, qp->sq.tail); in ocrdma_add_err_cqe()
2935 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_add_err_cqe()
2936 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { in ocrdma_add_err_cqe()
2937 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; in ocrdma_add_err_cqe()
2938 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_add_err_cqe()
2957 struct ocrdma_qp *qp; in ocrdma_poll_cq() local
2974 list_for_each_entry(qp, &cq->sq_head, sq_entry) { in ocrdma_poll_cq()
2977 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); in ocrdma_poll_cq()