Lines Matching refs:qp
1125 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) in ocrdma_add_qpn_map() argument
1129 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { in ocrdma_add_qpn_map()
1130 dev->qp_tbl[qp->id] = qp; in ocrdma_add_qpn_map()
1136 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) in ocrdma_del_qpn_map() argument
1138 dev->qp_tbl[qp->id] = NULL; in ocrdma_del_qpn_map()
1214 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, in ocrdma_copy_qp_uresp() argument
1221 struct ocrdma_pd *pd = qp->pd; in ocrdma_copy_qp_uresp()
1227 uresp.qp_id = qp->id; in ocrdma_copy_qp_uresp()
1228 uresp.sq_dbid = qp->sq.dbid; in ocrdma_copy_qp_uresp()
1230 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); in ocrdma_copy_qp_uresp()
1231 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); in ocrdma_copy_qp_uresp()
1232 uresp.num_wqe_allocated = qp->sq.max_cnt; in ocrdma_copy_qp_uresp()
1234 uresp.rq_dbid = qp->rq.dbid; in ocrdma_copy_qp_uresp()
1236 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); in ocrdma_copy_qp_uresp()
1237 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); in ocrdma_copy_qp_uresp()
1238 uresp.num_rqe_allocated = qp->rq.max_cnt; in ocrdma_copy_qp_uresp()
1246 if (qp->dpp_enabled) { in ocrdma_copy_qp_uresp()
1273 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, in ocrdma_set_qp_db() argument
1277 qp->sq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1280 qp->rq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1284 qp->sq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1287 qp->rq_db = dev->nic_info.db + in ocrdma_set_qp_db()
1293 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) in ocrdma_alloc_wr_id_tbl() argument
1295 qp->wqe_wr_id_tbl = in ocrdma_alloc_wr_id_tbl()
1296 kzalloc(sizeof(*(qp->wqe_wr_id_tbl)) * qp->sq.max_cnt, in ocrdma_alloc_wr_id_tbl()
1298 if (qp->wqe_wr_id_tbl == NULL) in ocrdma_alloc_wr_id_tbl()
1300 qp->rqe_wr_id_tbl = in ocrdma_alloc_wr_id_tbl()
1301 kzalloc(sizeof(u64) * qp->rq.max_cnt, GFP_KERNEL); in ocrdma_alloc_wr_id_tbl()
1302 if (qp->rqe_wr_id_tbl == NULL) in ocrdma_alloc_wr_id_tbl()
1308 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, in ocrdma_set_qp_init_params() argument
1312 qp->pd = pd; in ocrdma_set_qp_init_params()
1313 spin_lock_init(&qp->q_lock); in ocrdma_set_qp_init_params()
1314 INIT_LIST_HEAD(&qp->sq_entry); in ocrdma_set_qp_init_params()
1315 INIT_LIST_HEAD(&qp->rq_entry); in ocrdma_set_qp_init_params()
1317 qp->qp_type = attrs->qp_type; in ocrdma_set_qp_init_params()
1318 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; in ocrdma_set_qp_init_params()
1319 qp->max_inline_data = attrs->cap.max_inline_data; in ocrdma_set_qp_init_params()
1320 qp->sq.max_sges = attrs->cap.max_send_sge; in ocrdma_set_qp_init_params()
1321 qp->rq.max_sges = attrs->cap.max_recv_sge; in ocrdma_set_qp_init_params()
1322 qp->state = OCRDMA_QPS_RST; in ocrdma_set_qp_init_params()
1323 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; in ocrdma_set_qp_init_params()
1342 struct ocrdma_qp *qp; in ocrdma_create_qp() local
1356 qp = kzalloc(sizeof(*qp), GFP_KERNEL); in ocrdma_create_qp()
1357 if (!qp) { in ocrdma_create_qp()
1361 ocrdma_set_qp_init_params(qp, pd, attrs); in ocrdma_create_qp()
1363 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | in ocrdma_create_qp()
1367 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, in ocrdma_create_qp()
1375 status = ocrdma_alloc_wr_id_tbl(qp); in ocrdma_create_qp()
1380 status = ocrdma_add_qpn_map(dev, qp); in ocrdma_create_qp()
1383 ocrdma_set_qp_db(dev, qp, pd); in ocrdma_create_qp()
1385 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, in ocrdma_create_qp()
1392 qp->ibqp.qp_num = qp->id; in ocrdma_create_qp()
1394 return &qp->ibqp; in ocrdma_create_qp()
1397 ocrdma_del_qpn_map(dev, qp); in ocrdma_create_qp()
1399 ocrdma_mbx_destroy_qp(dev, qp); in ocrdma_create_qp()
1402 kfree(qp->wqe_wr_id_tbl); in ocrdma_create_qp()
1403 kfree(qp->rqe_wr_id_tbl); in ocrdma_create_qp()
1404 kfree(qp); in ocrdma_create_qp()
1414 struct ocrdma_qp *qp; in _ocrdma_modify_qp() local
1418 qp = get_ocrdma_qp(ibqp); in _ocrdma_modify_qp()
1421 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); in _ocrdma_modify_qp()
1427 status = ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); in _ocrdma_modify_qp()
1437 struct ocrdma_qp *qp; in ocrdma_modify_qp() local
1441 qp = get_ocrdma_qp(ibqp); in ocrdma_modify_qp()
1447 spin_lock_irqsave(&qp->q_lock, flags); in ocrdma_modify_qp()
1448 old_qps = get_ibqp_state(qp->state); in ocrdma_modify_qp()
1453 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_modify_qp()
1459 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, in ocrdma_modify_qp()
1508 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); in ocrdma_query_qp() local
1513 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); in ocrdma_query_qp()
1517 if (qp->qp_type == IB_QPT_UD) in ocrdma_query_qp()
1529 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); in ocrdma_query_qp()
1530 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; in ocrdma_query_qp()
1531 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; in ocrdma_query_qp()
1532 qp_attr->cap.max_send_sge = qp->sq.max_sges; in ocrdma_query_qp()
1533 qp_attr->cap.max_recv_sge = qp->rq.max_sges; in ocrdma_query_qp()
1534 qp_attr->cap.max_inline_data = qp->max_inline_data; in ocrdma_query_qp()
1540 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; in ocrdma_query_qp()
1583 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); in ocrdma_query_qp()
1601 static int is_hw_sq_empty(struct ocrdma_qp *qp) in is_hw_sq_empty() argument
1603 return (qp->sq.tail == qp->sq.head); in is_hw_sq_empty()
1606 static int is_hw_rq_empty(struct ocrdma_qp *qp) in is_hw_rq_empty() argument
1608 return (qp->rq.tail == qp->rq.head); in is_hw_rq_empty()
1633 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) in ocrdma_discard_cqes() argument
1655 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) in ocrdma_discard_cqes()
1666 if (qpn == 0 || qpn != qp->id) in ocrdma_discard_cqes()
1670 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_discard_cqes()
1672 if (qp->srq) { in ocrdma_discard_cqes()
1675 qp->srq->rq.max_wqe_idx; in ocrdma_discard_cqes()
1678 spin_lock_irqsave(&qp->srq->q_lock, flags); in ocrdma_discard_cqes()
1679 ocrdma_hwq_inc_tail(&qp->srq->rq); in ocrdma_discard_cqes()
1680 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); in ocrdma_discard_cqes()
1681 spin_unlock_irqrestore(&qp->srq->q_lock, flags); in ocrdma_discard_cqes()
1684 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_discard_cqes()
1698 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) in ocrdma_del_flush_qp() argument
1702 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); in ocrdma_del_flush_qp()
1706 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); in ocrdma_del_flush_qp()
1708 list_del(&qp->sq_entry); in ocrdma_del_flush_qp()
1709 if (!qp->srq) { in ocrdma_del_flush_qp()
1710 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); in ocrdma_del_flush_qp()
1712 list_del(&qp->rq_entry); in ocrdma_del_flush_qp()
1720 struct ocrdma_qp *qp; in ocrdma_destroy_qp() local
1726 qp = get_ocrdma_qp(ibqp); in ocrdma_destroy_qp()
1729 pd = qp->pd; in ocrdma_destroy_qp()
1732 if (qp->state != OCRDMA_QPS_RST) { in ocrdma_destroy_qp()
1742 (void) ocrdma_mbx_destroy_qp(dev, qp); in ocrdma_destroy_qp()
1748 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); in ocrdma_destroy_qp()
1749 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) in ocrdma_destroy_qp()
1750 spin_lock(&qp->rq_cq->cq_lock); in ocrdma_destroy_qp()
1752 ocrdma_del_qpn_map(dev, qp); in ocrdma_destroy_qp()
1754 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) in ocrdma_destroy_qp()
1755 spin_unlock(&qp->rq_cq->cq_lock); in ocrdma_destroy_qp()
1756 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); in ocrdma_destroy_qp()
1759 ocrdma_discard_cqes(qp, qp->sq_cq); in ocrdma_destroy_qp()
1760 ocrdma_discard_cqes(qp, qp->rq_cq); in ocrdma_destroy_qp()
1765 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, in ocrdma_destroy_qp()
1766 PAGE_ALIGN(qp->sq.len)); in ocrdma_destroy_qp()
1767 if (!qp->srq) in ocrdma_destroy_qp()
1768 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, in ocrdma_destroy_qp()
1769 PAGE_ALIGN(qp->rq.len)); in ocrdma_destroy_qp()
1772 ocrdma_del_flush_qp(qp); in ocrdma_destroy_qp()
1774 kfree(qp->wqe_wr_id_tbl); in ocrdma_destroy_qp()
1775 kfree(qp->rqe_wr_id_tbl); in ocrdma_destroy_qp()
1776 kfree(qp); in ocrdma_destroy_qp()
1924 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, in ocrdma_build_ud_hdr() argument
1933 if (qp->qp_type == IB_QPT_GSI) in ocrdma_build_ud_hdr()
1934 ud_hdr->qkey = qp->qkey; in ocrdma_build_ud_hdr()
1969 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, in ocrdma_build_inline_sges() argument
1977 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { in ocrdma_build_inline_sges()
1979 if (unlikely(hdr->total_len > qp->max_inline_data)) { in ocrdma_build_inline_sges()
1982 qp->max_inline_data, hdr->total_len); in ocrdma_build_inline_sges()
2009 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, in ocrdma_build_send() argument
2016 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { in ocrdma_build_send()
2017 ocrdma_build_ud_hdr(qp, hdr, wr); in ocrdma_build_send()
2024 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); in ocrdma_build_send()
2028 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, in ocrdma_build_write() argument
2036 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); in ocrdma_build_write()
2046 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, in ocrdma_build_read() argument
2110 static int ocrdma_build_fr(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, in ocrdma_build_fr() argument
2116 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); in ocrdma_build_fr()
2154 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) in ocrdma_ring_sq_db() argument
2156 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); in ocrdma_ring_sq_db()
2158 iowrite32(val, qp->sq_db); in ocrdma_ring_sq_db()
2165 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); in ocrdma_post_send() local
2169 spin_lock_irqsave(&qp->q_lock, flags); in ocrdma_post_send()
2170 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { in ocrdma_post_send()
2171 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_send()
2177 if (qp->qp_type == IB_QPT_UD && in ocrdma_post_send()
2184 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || in ocrdma_post_send()
2185 wr->num_sge > qp->sq.max_sges) { in ocrdma_post_send()
2190 hdr = ocrdma_hwq_head(&qp->sq); in ocrdma_post_send()
2192 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) in ocrdma_post_send()
2207 ocrdma_build_send(qp, hdr, wr); in ocrdma_post_send()
2213 status = ocrdma_build_send(qp, hdr, wr); in ocrdma_post_send()
2220 status = ocrdma_build_write(qp, hdr, wr); in ocrdma_post_send()
2223 ocrdma_build_read(qp, hdr, wr); in ocrdma_post_send()
2234 status = ocrdma_build_fr(qp, hdr, wr); in ocrdma_post_send()
2244 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) in ocrdma_post_send()
2245 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; in ocrdma_post_send()
2247 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; in ocrdma_post_send()
2248 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; in ocrdma_post_send()
2254 ocrdma_ring_sq_db(qp); in ocrdma_post_send()
2257 ocrdma_hwq_inc_head(&qp->sq); in ocrdma_post_send()
2260 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_send()
2264 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) in ocrdma_ring_rq_db() argument
2266 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); in ocrdma_ring_rq_db()
2268 iowrite32(val, qp->rq_db); in ocrdma_ring_rq_db()
2297 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); in ocrdma_post_recv() local
2300 spin_lock_irqsave(&qp->q_lock, flags); in ocrdma_post_recv()
2301 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { in ocrdma_post_recv()
2302 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_recv()
2307 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || in ocrdma_post_recv()
2308 wr->num_sge > qp->rq.max_sges) { in ocrdma_post_recv()
2313 rqe = ocrdma_hwq_head(&qp->rq); in ocrdma_post_recv()
2316 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; in ocrdma_post_recv()
2321 ocrdma_ring_rq_db(qp); in ocrdma_post_recv()
2324 ocrdma_hwq_inc_head(&qp->rq); in ocrdma_post_recv()
2327 spin_unlock_irqrestore(&qp->q_lock, flags); in ocrdma_post_recv()
2475 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, in ocrdma_update_wc() argument
2482 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); in ocrdma_update_wc()
2484 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; in ocrdma_update_wc()
2513 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, in ocrdma_set_cqe_status_flushed() argument
2525 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { in ocrdma_set_cqe_status_flushed()
2546 struct ocrdma_qp *qp, int status) in ocrdma_update_err_cqe() argument
2551 ibwc->qp = &qp->ibqp; in ocrdma_update_err_cqe()
2554 ocrdma_flush_qp(qp); in ocrdma_update_err_cqe()
2555 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); in ocrdma_update_err_cqe()
2560 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { in ocrdma_update_err_cqe()
2562 ocrdma_set_cqe_status_flushed(qp, cqe); in ocrdma_update_err_cqe()
2568 struct ocrdma_qp *qp, int status) in ocrdma_update_err_rcqe() argument
2571 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; in ocrdma_update_err_rcqe()
2572 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_update_err_rcqe()
2574 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); in ocrdma_update_err_rcqe()
2578 struct ocrdma_qp *qp, int status) in ocrdma_update_err_scqe() argument
2580 ocrdma_update_wc(qp, ibwc, qp->sq.tail); in ocrdma_update_err_scqe()
2581 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_update_err_scqe()
2583 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); in ocrdma_update_err_scqe()
2587 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, in ocrdma_poll_err_scqe() argument
2592 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); in ocrdma_poll_err_scqe()
2601 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { in ocrdma_poll_err_scqe()
2605 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { in ocrdma_poll_err_scqe()
2608 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); in ocrdma_poll_err_scqe()
2619 } else if (is_hw_sq_empty(qp)) { in ocrdma_poll_err_scqe()
2626 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); in ocrdma_poll_err_scqe()
2631 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, in ocrdma_poll_success_scqe() argument
2636 int tail = qp->sq.tail; in ocrdma_poll_success_scqe()
2639 if (!qp->wqe_wr_id_tbl[tail].signaled) { in ocrdma_poll_success_scqe()
2644 ibwc->qp = &qp->ibqp; in ocrdma_poll_success_scqe()
2645 ocrdma_update_wc(qp, ibwc, tail); in ocrdma_poll_success_scqe()
2649 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; in ocrdma_poll_success_scqe()
2653 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_poll_success_scqe()
2657 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, in ocrdma_poll_scqe() argument
2667 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); in ocrdma_poll_scqe()
2669 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); in ocrdma_poll_scqe()
2691 struct ocrdma_qp *qp) in ocrdma_update_free_srq_cqe() argument
2697 srq = get_ocrdma_srq(qp->ibqp.srq); in ocrdma_update_free_srq_cqe()
2710 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, in ocrdma_poll_err_rcqe() argument
2715 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); in ocrdma_poll_err_rcqe()
2723 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { in ocrdma_poll_err_rcqe()
2724 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { in ocrdma_poll_err_rcqe()
2727 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); in ocrdma_poll_err_rcqe()
2733 } else if (is_hw_rq_empty(qp)) { in ocrdma_poll_err_rcqe()
2740 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); in ocrdma_poll_err_rcqe()
2745 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, in ocrdma_poll_success_rcqe() argument
2749 ibwc->qp = &qp->ibqp; in ocrdma_poll_success_rcqe()
2752 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) in ocrdma_poll_success_rcqe()
2768 if (qp->ibqp.srq) { in ocrdma_poll_success_rcqe()
2769 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); in ocrdma_poll_success_rcqe()
2771 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; in ocrdma_poll_success_rcqe()
2772 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_poll_success_rcqe()
2776 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, in ocrdma_poll_rcqe() argument
2783 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { in ocrdma_poll_rcqe()
2794 ocrdma_poll_success_rcqe(qp, cqe, ibwc); in ocrdma_poll_rcqe()
2796 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, in ocrdma_poll_rcqe()
2821 struct ocrdma_qp *qp = NULL; in ocrdma_poll_hwcq() local
2836 qp = dev->qp_tbl[qpn]; in ocrdma_poll_hwcq()
2837 BUG_ON(qp == NULL); in ocrdma_poll_hwcq()
2840 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, in ocrdma_poll_hwcq()
2843 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, in ocrdma_poll_hwcq()
2883 struct ocrdma_qp *qp, struct ib_wc *ibwc) in ocrdma_add_err_cqe() argument
2888 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) in ocrdma_add_err_cqe()
2890 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { in ocrdma_add_err_cqe()
2891 ocrdma_update_wc(qp, ibwc, qp->sq.tail); in ocrdma_add_err_cqe()
2892 ocrdma_hwq_inc_tail(&qp->sq); in ocrdma_add_err_cqe()
2893 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { in ocrdma_add_err_cqe()
2894 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; in ocrdma_add_err_cqe()
2895 ocrdma_hwq_inc_tail(&qp->rq); in ocrdma_add_err_cqe()
2914 struct ocrdma_qp *qp; in ocrdma_poll_cq() local
2931 list_for_each_entry(qp, &cq->sq_head, sq_entry) { in ocrdma_poll_cq()
2934 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); in ocrdma_poll_cq()