Lines Matching refs:wq
275 struct t3_wq *wq, struct cxio_ucontext *uctx) in cxio_create_qp() argument
277 int depth = 1UL << wq->size_log2; in cxio_create_qp()
278 int rqsize = 1UL << wq->rq_size_log2; in cxio_create_qp()
280 wq->qpid = get_qpid(rdev_p, uctx); in cxio_create_qp()
281 if (!wq->qpid) in cxio_create_qp()
284 wq->rq = kzalloc(depth * sizeof(struct t3_swrq), GFP_KERNEL); in cxio_create_qp()
285 if (!wq->rq) in cxio_create_qp()
288 wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize); in cxio_create_qp()
289 if (!wq->rq_addr) in cxio_create_qp()
292 wq->sq = kzalloc(depth * sizeof(struct t3_swsq), GFP_KERNEL); in cxio_create_qp()
293 if (!wq->sq) in cxio_create_qp()
296 wq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev), in cxio_create_qp()
298 &(wq->dma_addr), GFP_KERNEL); in cxio_create_qp()
299 if (!wq->queue) in cxio_create_qp()
302 memset(wq->queue, 0, depth * sizeof(union t3_wr)); in cxio_create_qp()
303 dma_unmap_addr_set(wq, mapping, wq->dma_addr); in cxio_create_qp()
304 wq->doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr; in cxio_create_qp()
306 wq->udb = (u64)rdev_p->rnic_info.udbell_physbase + in cxio_create_qp()
307 (wq->qpid << rdev_p->qpshift); in cxio_create_qp()
308 wq->rdev = rdev_p; in cxio_create_qp()
310 wq->qpid, wq->doorbell, (unsigned long long) wq->udb); in cxio_create_qp()
313 kfree(wq->sq); in cxio_create_qp()
315 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize); in cxio_create_qp()
317 kfree(wq->rq); in cxio_create_qp()
319 put_qpid(rdev_p, wq->qpid, uctx); in cxio_create_qp()
336 int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq, in cxio_destroy_qp() argument
340 (1UL << (wq->size_log2)) in cxio_destroy_qp()
341 * sizeof(union t3_wr), wq->queue, in cxio_destroy_qp()
342 dma_unmap_addr(wq, mapping)); in cxio_destroy_qp()
343 kfree(wq->sq); in cxio_destroy_qp()
344 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2)); in cxio_destroy_qp()
345 kfree(wq->rq); in cxio_destroy_qp()
346 put_qpid(rdev_p, wq->qpid, uctx); in cxio_destroy_qp()
350 static void insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq) in insert_recv_cqe() argument
355 wq, cq, cq->sw_rptr, cq->sw_wptr); in insert_recv_cqe()
361 V_CQE_QPID(wq->qpid) | in insert_recv_cqe()
368 int cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count) in cxio_flush_rq() argument
373 PDBG("%s wq %p cq %p\n", __func__, wq, cq); in cxio_flush_rq()
377 wq->rq_rptr, wq->rq_wptr, count); in cxio_flush_rq()
378 ptr = wq->rq_rptr + count; in cxio_flush_rq()
379 while (ptr++ != wq->rq_wptr) { in cxio_flush_rq()
380 insert_recv_cqe(wq, cq); in cxio_flush_rq()
386 static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq, in insert_sq_cqe() argument
392 wq, cq, cq->sw_rptr, cq->sw_wptr); in insert_sq_cqe()
398 V_CQE_QPID(wq->qpid) | in insert_sq_cqe()
407 int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count) in cxio_flush_sq() argument
411 struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2); in cxio_flush_sq()
413 ptr = wq->sq_rptr + count; in cxio_flush_sq()
414 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); in cxio_flush_sq()
415 while (ptr != wq->sq_wptr) { in cxio_flush_sq()
417 insert_sq_cqe(wq, cq, sqp); in cxio_flush_sq()
419 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); in cxio_flush_sq()
446 static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq) in cqe_completes_wr() argument
458 Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) in cqe_completes_wr()
464 void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count) in cxio_count_scqes() argument
474 ((CQE_OPCODE(*cqe) == T3_READ_RESP) && wq->oldest_read)) && in cxio_count_scqes()
475 (CQE_QPID(*cqe) == wq->qpid)) in cxio_count_scqes()
482 void cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count) in cxio_count_rcqes() argument
493 (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq)) in cxio_count_rcqes()
1073 static void flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq) in flush_completed_wrs() argument
1076 __u32 ptr = wq->sq_rptr; in flush_completed_wrs()
1077 int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr); in flush_completed_wrs()
1079 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); in flush_completed_wrs()
1083 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); in flush_completed_wrs()
1090 __func__, Q_PTR2IDX(ptr, wq->sq_size_log2), in flush_completed_wrs()
1102 static void create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe, in create_read_req_cqe() argument
1105 read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr; in create_read_req_cqe()
1106 read_cqe->len = wq->oldest_read->read_len; in create_read_req_cqe()
1116 static void advance_oldest_read(struct t3_wq *wq) in advance_oldest_read() argument
1119 u32 rptr = wq->oldest_read - wq->sq + 1; in advance_oldest_read()
1120 u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2); in advance_oldest_read()
1122 while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) { in advance_oldest_read()
1123 wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2); in advance_oldest_read()
1125 if (wq->oldest_read->opcode == T3_READ_REQ) in advance_oldest_read()
1129 wq->oldest_read = NULL; in advance_oldest_read()
1147 int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe, in cxio_poll_cq() argument
1167 if (wq == NULL) { in cxio_poll_cq()
1186 if (!wq->oldest_read) { in cxio_poll_cq()
1188 wq->error = 1; in cxio_poll_cq()
1197 create_read_req_cqe(wq, hw_cqe, &read_cqe); in cxio_poll_cq()
1199 advance_oldest_read(wq); in cxio_poll_cq()
1207 wq->error = 1; in cxio_poll_cq()
1211 if (CQE_STATUS(*hw_cqe) || wq->error) { in cxio_poll_cq()
1212 *cqe_flushed = wq->error; in cxio_poll_cq()
1213 wq->error = 1; in cxio_poll_cq()
1233 Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) { in cxio_poll_cq()
1253 if (Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) { in cxio_poll_cq()
1254 wq->error = 1; in cxio_poll_cq()
1259 if (unlikely((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) { in cxio_poll_cq()
1260 wq->error = 1; in cxio_poll_cq()
1278 if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) { in cxio_poll_cq()
1283 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2)); in cxio_poll_cq()
1284 sqp = wq->sq + in cxio_poll_cq()
1285 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2); in cxio_poll_cq()
1300 wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe); in cxio_poll_cq()
1302 Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2)); in cxio_poll_cq()
1303 *cookie = wq->sq[Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2)].wr_id; in cxio_poll_cq()
1304 wq->sq_rptr++; in cxio_poll_cq()
1307 Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)); in cxio_poll_cq()
1308 *cookie = wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].wr_id; in cxio_poll_cq()
1309 if (wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].pbl_addr) in cxio_poll_cq()
1310 cxio_hal_pblpool_free(wq->rdev, in cxio_poll_cq()
1311 wq->rq[Q_PTR2IDX(wq->rq_rptr, in cxio_poll_cq()
1312 wq->rq_size_log2)].pbl_addr, T3_STAG0_PBL_SIZE); in cxio_poll_cq()
1313 BUG_ON(Q_EMPTY(wq->rq_rptr, wq->rq_wptr)); in cxio_poll_cq()
1314 wq->rq_rptr++; in cxio_poll_cq()
1321 flush_completed_wrs(wq, cq); in cxio_poll_cq()