qp 33 arch/ia64/include/asm/kprobes.h unsigned long long qp : 6; qp 59 arch/ia64/kernel/brl_emu.c unsigned long opcode, btype, qp, offset, cpl; qp 83 arch/ia64/kernel/brl_emu.c qp = ((bundle[1] >> 23) & 0x3f); qp 89 arch/ia64/kernel/brl_emu.c tmp_taken = regs->pr & (1L << qp); qp 179 arch/ia64/kernel/kprobes.c int qp; qp 181 arch/ia64/kernel/kprobes.c qp = kprobe_inst & 0x3f; qp 183 arch/ia64/kernel/kprobes.c if (slot == 1 && qp) { qp 190 arch/ia64/kernel/kprobes.c qp = 0; qp 222 arch/ia64/kernel/kprobes.c if (slot == 1 && qp) { qp 228 arch/ia64/kernel/kprobes.c qp = 0; qp 257 arch/ia64/kernel/kprobes.c if (slot == 1 && qp) { qp 264 arch/ia64/kernel/kprobes.c qp = 0; qp 269 arch/ia64/kernel/kprobes.c if (slot == 1 && qp) { qp 275 arch/ia64/kernel/kprobes.c qp = 0; qp 278 arch/ia64/kernel/kprobes.c return qp; qp 289 arch/ia64/kernel/kprobes.c int qp) qp 298 arch/ia64/kernel/kprobes.c break_inst |= qp; qp 571 arch/ia64/kernel/kprobes.c int qp; qp 586 arch/ia64/kernel/kprobes.c qp = unsupported_inst(template, slot, major_opcode, kprobe_inst, addr); qp 587 arch/ia64/kernel/kprobes.c if (qp < 0) qp 596 arch/ia64/kernel/kprobes.c prepare_break_inst(template, slot, major_opcode, kprobe_inst, p, qp); qp 169 arch/ia64/kernel/unaligned.c unsigned long qp:6; /* [0:5] */ qp 1391 arch/ia64/kernel/unaligned.c "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm, qp 1072 arch/ia64/kernel/unwind.c desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr) qp 1076 arch/ia64/kernel/unwind.c if (qp > 0) { qp 1077 arch/ia64/kernel/unwind.c if ((sr->pr_val & (1UL << qp)) == 0) qp 1079 arch/ia64/kernel/unwind.c sr->pr_mask |= (1UL << qp); qp 1085 arch/ia64/kernel/unwind.c desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr) qp 1089 arch/ia64/kernel/unwind.c if (!desc_is_active(qp, t, sr)) qp 1099 arch/ia64/kernel/unwind.c desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x, qp 1105 arch/ia64/kernel/unwind.c if (!desc_is_active(qp, t, sr)) qp 1120 arch/ia64/kernel/unwind.c desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff, qp 1125 arch/ia64/kernel/unwind.c if (!desc_is_active(qp, t, sr)) qp 1135 arch/ia64/kernel/unwind.c desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff, qp 1140 arch/ia64/kernel/unwind.c if (!desc_is_active(qp, t, sr)) qp 123 arch/ia64/kernel/unwind_decoder.c unsigned char byte1, byte2, abreg, qp; qp 130 arch/ia64/kernel/unwind_decoder.c qp = (byte1 & 0x3f); qp 134 arch/ia64/kernel/unwind_decoder.c UNW_DEC_SPILL_SPREL_P(X3, qp, t, abreg, off, arg); qp 136 arch/ia64/kernel/unwind_decoder.c UNW_DEC_SPILL_PSPREL_P(X3, qp, t, abreg, off, arg); qp 143 arch/ia64/kernel/unwind_decoder.c unsigned char byte1, byte2, byte3, qp, abreg, x, ytreg; qp 149 arch/ia64/kernel/unwind_decoder.c qp = (byte1 & 0x3f); qp 155 arch/ia64/kernel/unwind_decoder.c UNW_DEC_RESTORE_P(X4, qp, t, abreg, arg); qp 157 arch/ia64/kernel/unwind_decoder.c UNW_DEC_SPILL_REG_P(X4, qp, t, abreg, x, ytreg, arg); qp 2091 arch/powerpc/platforms/cell/spufs/file.c struct mfc_cq_sr *qp, *spuqp; qp 2100 arch/powerpc/platforms/cell/spufs/file.c qp = &info.dma_info_command_data[i]; qp 2103 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data0_RW = spuqp->mfc_cq_data0_RW; qp 2104 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data1_RW = spuqp->mfc_cq_data1_RW; qp 2105 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data2_RW = spuqp->mfc_cq_data2_RW; qp 2106 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data3_RW = spuqp->mfc_cq_data3_RW; qp 2143 arch/powerpc/platforms/cell/spufs/file.c struct mfc_cq_sr *qp, *puqp; qp 2157 arch/powerpc/platforms/cell/spufs/file.c qp = &info.proxydma_info_command_data[i]; qp 2160 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data0_RW = puqp->mfc_cq_data0_RW; qp 2161 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data1_RW = puqp->mfc_cq_data1_RW; qp 2162 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data2_RW = puqp->mfc_cq_data2_RW; qp 2163 arch/powerpc/platforms/cell/spufs/file.c qp->mfc_cq_data3_RW = puqp->mfc_cq_data3_RW; qp 997 arch/sparc/kernel/ds.c struct ds_queue_entry *qp, *tmp; qp 1005 arch/sparc/kernel/ds.c list_for_each_entry_safe(qp, tmp, &todo, list) { qp 1006 arch/sparc/kernel/ds.c struct ds_data *dpkt = (struct ds_data *) qp->req; qp 1007 arch/sparc/kernel/ds.c struct ds_info *dp = qp->dp; qp 1009 arch/sparc/kernel/ds.c int req_len = qp->req_len; qp 1023 arch/sparc/kernel/ds.c list_del(&qp->list); qp 1024 arch/sparc/kernel/ds.c kfree(qp); qp 1050 arch/sparc/kernel/ds.c struct ds_queue_entry *qp; qp 1052 arch/sparc/kernel/ds.c qp = kmalloc(sizeof(struct ds_queue_entry) + len, GFP_ATOMIC); qp 1053 arch/sparc/kernel/ds.c if (!qp) { qp 1056 arch/sparc/kernel/ds.c qp->dp = dp; qp 1057 arch/sparc/kernel/ds.c memcpy(&qp->req, pkt, len); qp 1058 arch/sparc/kernel/ds.c list_add_tail(&qp->list, &ds_work_list); qp 620 drivers/atm/firestream.c static int qp; qp 640 drivers/atm/firestream.c pq[qp].cmd = cmd; qp 641 drivers/atm/firestream.c pq[qp].p0 = p1; qp 642 drivers/atm/firestream.c pq[qp].p1 = p2; qp 643 drivers/atm/firestream.c pq[qp].p2 = p3; qp 644 drivers/atm/firestream.c qp++; qp 645 drivers/atm/firestream.c if (qp >= 60) qp = 0; qp 1940 drivers/atm/firestream.c i, pq[qp].cmd, pq[qp].p0, pq[qp].p1, pq[qp].p2); qp 1941 drivers/atm/firestream.c qp++; qp 1942 drivers/atm/firestream.c if (qp >= 60) qp = 0; qp 450 drivers/crypto/hisilicon/qm.c static void qm_cq_head_update(struct hisi_qp *qp) qp 452 drivers/crypto/hisilicon/qm.c if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) { qp 453 drivers/crypto/hisilicon/qm.c qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase; qp 454 drivers/crypto/hisilicon/qm.c qp->qp_status.cq_head = 0; qp 456 drivers/crypto/hisilicon/qm.c qp->qp_status.cq_head++; qp 460 drivers/crypto/hisilicon/qm.c static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) qp 462 drivers/crypto/hisilicon/qm.c struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; qp 464 drivers/crypto/hisilicon/qm.c if (qp->req_cb) { qp 465 drivers/crypto/hisilicon/qm.c while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { qp 467 drivers/crypto/hisilicon/qm.c qp->req_cb(qp, qp->sqe + qm->sqe_size * cqe->sq_head); qp 468 drivers/crypto/hisilicon/qm.c qm_cq_head_update(qp); qp 469 drivers/crypto/hisilicon/qm.c cqe = qp->cqe + qp->qp_status.cq_head; qp 470 drivers/crypto/hisilicon/qm.c qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp 471 drivers/crypto/hisilicon/qm.c qp->qp_status.cq_head, 0); qp 472 drivers/crypto/hisilicon/qm.c atomic_dec(&qp->qp_status.used); qp 476 drivers/crypto/hisilicon/qm.c qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp 477 drivers/crypto/hisilicon/qm.c qp->qp_status.cq_head, 1); qp 483 drivers/crypto/hisilicon/qm.c struct hisi_qp *qp; qp 485 drivers/crypto/hisilicon/qm.c qp = container_of(work, struct hisi_qp, work); qp 486 drivers/crypto/hisilicon/qm.c qm_poll_qp(qp, qp->qm); qp 493 drivers/crypto/hisilicon/qm.c struct hisi_qp *qp; qp 498 drivers/crypto/hisilicon/qm.c qp = qm_to_hisi_qp(qm, eqe); qp 499 drivers/crypto/hisilicon/qm.c if (qp) qp 500 drivers/crypto/hisilicon/qm.c queue_work(qp->wq, &qp->work); qp 643 drivers/crypto/hisilicon/qm.c static void qm_init_qp_status(struct hisi_qp *qp) qp 645 drivers/crypto/hisilicon/qm.c struct hisi_qp_status *qp_status = &qp->qp_status; qp 1086 drivers/crypto/hisilicon/qm.c static void *qm_get_avail_sqe(struct hisi_qp *qp) qp 1088 drivers/crypto/hisilicon/qm.c struct hisi_qp_status *qp_status = &qp->qp_status; qp 1091 drivers/crypto/hisilicon/qm.c if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH)) qp 1094 drivers/crypto/hisilicon/qm.c return qp->sqe + sq_tail * qp->qm->sqe_size; qp 1108 drivers/crypto/hisilicon/qm.c struct hisi_qp *qp; qp 1111 drivers/crypto/hisilicon/qm.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 1112 drivers/crypto/hisilicon/qm.c if (!qp) qp 1125 drivers/crypto/hisilicon/qm.c qm->qp_array[qp_id] = qp; qp 1129 drivers/crypto/hisilicon/qm.c qp->qm = qm; qp 1132 drivers/crypto/hisilicon/qm.c qp->qdma.size = qm->sqe_size * QM_Q_DEPTH + qp 1134 drivers/crypto/hisilicon/qm.c qp->qdma.va = dma_alloc_coherent(dev, qp->qdma.size, qp 1135 drivers/crypto/hisilicon/qm.c &qp->qdma.dma, GFP_KERNEL); qp 1136 drivers/crypto/hisilicon/qm.c if (!qp->qdma.va) { qp 1142 drivers/crypto/hisilicon/qm.c qp->qdma.va, &qp->qdma.dma, qp->qdma.size); qp 1145 drivers/crypto/hisilicon/qm.c qp->qp_id = qp_id; qp 1146 drivers/crypto/hisilicon/qm.c qp->alg_type = alg_type; qp 1147 drivers/crypto/hisilicon/qm.c INIT_WORK(&qp->work, qm_qp_work_func); qp 1148 drivers/crypto/hisilicon/qm.c qp->wq = alloc_workqueue("hisi_qm", WQ_UNBOUND | WQ_HIGHPRI | qp 1150 drivers/crypto/hisilicon/qm.c if (!qp->wq) { qp 1155 drivers/crypto/hisilicon/qm.c return qp; qp 1159 drivers/crypto/hisilicon/qm.c dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp 1160 drivers/crypto/hisilicon/qm.c qp->qdma.dma); qp 1167 drivers/crypto/hisilicon/qm.c kfree(qp); qp 1178 drivers/crypto/hisilicon/qm.c void hisi_qm_release_qp(struct hisi_qp *qp) qp 1180 drivers/crypto/hisilicon/qm.c struct hisi_qm *qm = qp->qm; qp 1181 drivers/crypto/hisilicon/qm.c struct qm_dma *qdma = &qp->qdma; qp 1188 drivers/crypto/hisilicon/qm.c qm->qp_array[qp->qp_id] = NULL; qp 1189 drivers/crypto/hisilicon/qm.c clear_bit(qp->qp_id, qm->qp_bitmap); qp 1192 drivers/crypto/hisilicon/qm.c kfree(qp); qp 1196 drivers/crypto/hisilicon/qm.c static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) qp 1198 drivers/crypto/hisilicon/qm.c struct hisi_qm *qm = qp->qm; qp 1207 drivers/crypto/hisilicon/qm.c qm_init_qp_status(qp); qp 1219 drivers/crypto/hisilicon/qm.c INIT_QC_COMMON(sqc, qp->sqe_dma, pasid); qp 1228 drivers/crypto/hisilicon/qm.c sqc->w13 = QM_MK_SQC_W13(0, 1, qp->alg_type); qp 1246 drivers/crypto/hisilicon/qm.c INIT_QC_COMMON(cqc, qp->cqe_dma, pasid); qp 1271 drivers/crypto/hisilicon/qm.c int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) qp 1273 drivers/crypto/hisilicon/qm.c struct hisi_qm *qm = qp->qm; qp 1276 drivers/crypto/hisilicon/qm.c int qp_id = qp->qp_id; qp 1281 drivers/crypto/hisilicon/qm.c #define QP_INIT_BUF(qp, type, size) do { \ qp 1282 drivers/crypto/hisilicon/qm.c (qp)->type = ((qp)->qdma.va + (off)); \ qp 1283 drivers/crypto/hisilicon/qm.c (qp)->type##_dma = (qp)->qdma.dma + (off); \ qp 1287 drivers/crypto/hisilicon/qm.c if (!qp->qdma.dma) { qp 1293 drivers/crypto/hisilicon/qm.c if (qp->qdma.dma & QM_SQE_DATA_ALIGN_MASK) { qp 1298 drivers/crypto/hisilicon/qm.c QP_INIT_BUF(qp, sqe, qm->sqe_size * QM_Q_DEPTH); qp 1299 drivers/crypto/hisilicon/qm.c QP_INIT_BUF(qp, cqe, sizeof(struct qm_cqe) * QM_Q_DEPTH); qp 1304 drivers/crypto/hisilicon/qm.c ver, qp->sqe, (unsigned long)qp->sqe_dma, qp 1305 drivers/crypto/hisilicon/qm.c qp->cqe, (unsigned long)qp->cqe_dma); qp 1307 drivers/crypto/hisilicon/qm.c ret = qm_qp_ctx_cfg(qp, qp_id, pasid); qp 1323 drivers/crypto/hisilicon/qm.c int hisi_qm_stop_qp(struct hisi_qp *qp) qp 1325 drivers/crypto/hisilicon/qm.c struct device *dev = &qp->qm->pdev->dev; qp 1329 drivers/crypto/hisilicon/qm.c if (test_bit(QP_STOP, &qp->qp_status.flags)) qp 1332 drivers/crypto/hisilicon/qm.c while (atomic_read(&qp->qp_status.used)) { qp 1341 drivers/crypto/hisilicon/qm.c set_bit(QP_STOP, &qp->qp_status.flags); qp 1343 drivers/crypto/hisilicon/qm.c dev_dbg(dev, "stop queue %u!", qp->qp_id); qp 1357 drivers/crypto/hisilicon/qm.c int hisi_qp_send(struct hisi_qp *qp, const void *msg) qp 1359 drivers/crypto/hisilicon/qm.c struct hisi_qp_status *qp_status = &qp->qp_status; qp 1362 drivers/crypto/hisilicon/qm.c void *sqe = qm_get_avail_sqe(qp); qp 1364 drivers/crypto/hisilicon/qm.c if (unlikely(test_bit(QP_STOP, &qp->qp_status.flags))) { qp 1365 drivers/crypto/hisilicon/qm.c dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); qp 1372 drivers/crypto/hisilicon/qm.c memcpy(sqe, msg, qp->qm->sqe_size); qp 1374 drivers/crypto/hisilicon/qm.c qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); qp 1375 drivers/crypto/hisilicon/qm.c atomic_inc(&qp->qp_status.used); qp 1739 drivers/crypto/hisilicon/qm.c struct hisi_qp *qp; qp 1755 drivers/crypto/hisilicon/qm.c qp = qm->qp_array[i]; qp 1756 drivers/crypto/hisilicon/qm.c if (qp) { qp 1757 drivers/crypto/hisilicon/qm.c ret = hisi_qm_stop_qp(qp); qp 191 drivers/crypto/hisilicon/qm.h void (*req_cb)(struct hisi_qp *qp, void *data); qp 203 drivers/crypto/hisilicon/qm.h int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg); qp 204 drivers/crypto/hisilicon/qm.h int hisi_qm_stop_qp(struct hisi_qp *qp); qp 205 drivers/crypto/hisilicon/qm.h void hisi_qm_release_qp(struct hisi_qp *qp); qp 206 drivers/crypto/hisilicon/qm.h int hisi_qp_send(struct hisi_qp *qp, const void *msg); qp 65 drivers/crypto/hisilicon/zip/zip_crypto.c struct hisi_qp *qp; qp 113 drivers/crypto/hisilicon/zip/zip_crypto.c struct hisi_qp *qp; qp 116 drivers/crypto/hisilicon/zip/zip_crypto.c qp = hisi_qm_create_qp(qm, alg_type); qp 117 drivers/crypto/hisilicon/zip/zip_crypto.c if (IS_ERR(qp)) qp 118 drivers/crypto/hisilicon/zip/zip_crypto.c return PTR_ERR(qp); qp 120 drivers/crypto/hisilicon/zip/zip_crypto.c qp->req_type = req_type; qp 121 drivers/crypto/hisilicon/zip/zip_crypto.c qp->qp_ctx = ctx; qp 122 drivers/crypto/hisilicon/zip/zip_crypto.c ctx->qp = qp; qp 124 drivers/crypto/hisilicon/zip/zip_crypto.c ret = hisi_qm_start_qp(qp, 0); qp 131 drivers/crypto/hisilicon/zip/zip_crypto.c hisi_qm_release_qp(qp); qp 137 drivers/crypto/hisilicon/zip/zip_crypto.c hisi_qm_stop_qp(ctx->qp); qp 138 drivers/crypto/hisilicon/zip/zip_crypto.c hisi_qm_release_qp(ctx->qp); qp 272 drivers/crypto/hisilicon/zip/zip_crypto.c ret = hisi_acc_create_sgl_pool(&tmp->qp->qm->pdev->dev, qp 285 drivers/crypto/hisilicon/zip/zip_crypto.c hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev, qp 295 drivers/crypto/hisilicon/zip/zip_crypto.c hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, qp 310 drivers/crypto/hisilicon/zip/zip_crypto.c static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) qp 313 drivers/crypto/hisilicon/zip/zip_crypto.c struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; qp 317 drivers/crypto/hisilicon/zip/zip_crypto.c struct device *dev = &qp->qm->pdev->dev; qp 325 drivers/crypto/hisilicon/zip/zip_crypto.c (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, qp 334 drivers/crypto/hisilicon/zip/zip_crypto.c head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; qp 349 drivers/crypto/hisilicon/zip/zip_crypto.c ctx->qp_ctx[i].qp->req_cb = fn; qp 440 drivers/crypto/hisilicon/zip/zip_crypto.c dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); qp 467 drivers/crypto/hisilicon/zip/zip_crypto.c struct hisi_qp *qp = qp_ctx->qp; qp 468 drivers/crypto/hisilicon/zip/zip_crypto.c struct device *dev = &qp->qm->pdev->dev; qp 492 drivers/crypto/hisilicon/zip/zip_crypto.c hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, a_req->slen, qp 498 drivers/crypto/hisilicon/zip/zip_crypto.c ret = hisi_qp_send(qp, zip_sqe); qp 520 drivers/crypto/hisilicon/zip/zip_crypto.c head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); qp 543 drivers/crypto/hisilicon/zip/zip_crypto.c head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type); qp 239 drivers/crypto/n2_core.c static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) qp 242 drivers/crypto/n2_core.c qp->head != qp->tail) qp 491 drivers/crypto/n2_core.c static unsigned long wait_for_tail(struct spu_queue *qp) qp 496 drivers/crypto/n2_core.c hv_ret = sun4v_ncs_gethead(qp->qhandle, &head); qp 501 drivers/crypto/n2_core.c if (head == qp->tail) { qp 502 drivers/crypto/n2_core.c qp->head = head; qp 509 drivers/crypto/n2_core.c static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, qp 512 drivers/crypto/n2_core.c unsigned long hv_ret = spu_queue_submit(qp, ent); qp 515 drivers/crypto/n2_core.c hv_ret = wait_for_tail(qp); qp 528 drivers/crypto/n2_core.c struct spu_queue *qp; qp 553 drivers/crypto/n2_core.c qp = cpu_to_cwq[cpu]; qp 554 drivers/crypto/n2_core.c if (!qp) qp 557 drivers/crypto/n2_core.c spin_lock_irqsave(&qp->lock, flags); qp 562 drivers/crypto/n2_core.c ent = qp->q + qp->tail; qp 579 drivers/crypto/n2_core.c ent = spu_queue_next(qp, ent); qp 594 drivers/crypto/n2_core.c if (submit_and_wait_for_tail(qp, ent) != HV_EOK) qp 599 drivers/crypto/n2_core.c spin_unlock_irqrestore(&qp->lock, flags); qp 834 drivers/crypto/n2_core.c struct spu_queue *qp, bool encrypt) qp 841 drivers/crypto/n2_core.c ent = spu_queue_alloc(qp, cp->arr_len); qp 864 drivers/crypto/n2_core.c ent = spu_queue_next(qp, ent); qp 877 drivers/crypto/n2_core.c return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; qp 985 drivers/crypto/n2_core.c struct spu_queue *qp; qp 990 drivers/crypto/n2_core.c qp = cpu_to_cwq[get_cpu()]; qp 992 drivers/crypto/n2_core.c if (!qp) qp 995 drivers/crypto/n2_core.c spin_lock_irqsave(&qp->lock, flags); qp 998 drivers/crypto/n2_core.c err = __n2_crypt_chunk(tfm, c, qp, encrypt); qp 1006 drivers/crypto/n2_core.c hv_ret = wait_for_tail(qp); qp 1011 drivers/crypto/n2_core.c spin_unlock_irqrestore(&qp->lock, flags); qp 1037 drivers/crypto/n2_core.c struct spu_queue *qp; qp 1045 drivers/crypto/n2_core.c qp = cpu_to_cwq[get_cpu()]; qp 1047 drivers/crypto/n2_core.c if (!qp) qp 1050 drivers/crypto/n2_core.c spin_lock_irqsave(&qp->lock, flags); qp 1057 drivers/crypto/n2_core.c err = __n2_crypt_chunk(tfm, c, qp, true); qp 1087 drivers/crypto/n2_core.c err = __n2_crypt_chunk(tfm, c, qp, false); qp 1096 drivers/crypto/n2_core.c hv_ret = wait_for_tail(qp); qp 1101 drivers/crypto/n2_core.c spin_unlock_irqrestore(&qp->lock, flags); qp 102 drivers/infiniband/core/agent.c ah = ib_create_ah_from_wc(agent->qp->pd, wc, grh, port_num); qp 347 drivers/infiniband/core/cm.c ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr, 0); qp 392 drivers/infiniband/core/cm.c ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc, qp 911 drivers/infiniband/core/cma.c static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) qp 921 drivers/infiniband/core/cma.c ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); qp 926 drivers/infiniband/core/cma.c ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); qp 932 drivers/infiniband/core/cma.c ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); qp 937 drivers/infiniband/core/cma.c static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) qp 947 drivers/infiniband/core/cma.c return ib_modify_qp(qp, &qp_attr, qp_attr_mask); qp 954 drivers/infiniband/core/cma.c struct ib_qp *qp; qp 962 drivers/infiniband/core/cma.c qp = ib_create_qp(pd, qp_init_attr); qp 963 drivers/infiniband/core/cma.c if (IS_ERR(qp)) qp 964 drivers/infiniband/core/cma.c return PTR_ERR(qp); qp 967 drivers/infiniband/core/cma.c ret = cma_init_ud_qp(id_priv, qp); qp 969 drivers/infiniband/core/cma.c ret = cma_init_conn_qp(id_priv, qp); qp 973 drivers/infiniband/core/cma.c id->qp = qp; qp 974 drivers/infiniband/core/cma.c id_priv->qp_num = qp->qp_num; qp 975 drivers/infiniband/core/cma.c id_priv->srq = (qp->srq != NULL); qp 978 drivers/infiniband/core/cma.c ib_destroy_qp(qp); qp 989 drivers/infiniband/core/cma.c ib_destroy_qp(id_priv->id.qp); qp 990 drivers/infiniband/core/cma.c id_priv->id.qp = NULL; qp 1002 drivers/infiniband/core/cma.c if (!id_priv->id.qp) { qp 1013 drivers/infiniband/core/cma.c ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); qp 1026 drivers/infiniband/core/cma.c ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); qp 1039 drivers/infiniband/core/cma.c if (!id_priv->id.qp) { qp 1051 drivers/infiniband/core/cma.c ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); qp 1063 drivers/infiniband/core/cma.c if (!id_priv->id.qp) { qp 1069 drivers/infiniband/core/cma.c ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); qp 1944 drivers/infiniband/core/cma.c if (id_priv->id.qp) { qp 3899 drivers/infiniband/core/cma.c iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num; qp 3922 drivers/infiniband/core/cma.c if (!id->qp) { qp 3994 drivers/infiniband/core/cma.c if (id_priv->id.qp) { qp 4037 drivers/infiniband/core/cma.c if (!id->qp && conn_param) { qp 4168 drivers/infiniband/core/cma.c if (!status && id_priv->id.qp) { qp 4169 drivers/infiniband/core/cma.c status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, qp 4485 drivers/infiniband/core/cma.c if (id->qp) qp 4486 drivers/infiniband/core/cma.c ib_detach_mcast(id->qp, qp 227 drivers/infiniband/core/core_priv.h int ib_security_modify_qp(struct ib_qp *qp, qp 232 drivers/infiniband/core/core_priv.h int ib_create_qp_security(struct ib_qp *qp, struct ib_device *dev); qp 236 drivers/infiniband/core/core_priv.h int ib_open_shared_qp_security(struct ib_qp *qp, struct ib_device *dev); qp 254 drivers/infiniband/core/core_priv.h static inline int ib_security_modify_qp(struct ib_qp *qp, qp 259 drivers/infiniband/core/core_priv.h return qp->device->ops.modify_qp(qp->real_qp, qp 265 drivers/infiniband/core/core_priv.h static inline int ib_create_qp_security(struct ib_qp *qp, qp 283 drivers/infiniband/core/core_priv.h static inline int ib_open_shared_qp_security(struct ib_qp *qp, qp 327 drivers/infiniband/core/core_priv.h struct ib_qp *qp; qp 333 drivers/infiniband/core/core_priv.h qp = dev->ops.create_qp(pd, attr, udata); qp 334 drivers/infiniband/core/core_priv.h if (IS_ERR(qp)) qp 335 drivers/infiniband/core/core_priv.h return qp; qp 337 drivers/infiniband/core/core_priv.h qp->device = dev; qp 338 drivers/infiniband/core/core_priv.h qp->pd = pd; qp 339 drivers/infiniband/core/core_priv.h qp->uobject = uobj; qp 340 drivers/infiniband/core/core_priv.h qp->real_qp = qp; qp 342 drivers/infiniband/core/core_priv.h qp->qp_type = attr->qp_type; qp 343 drivers/infiniband/core/core_priv.h qp->qp_context = attr->qp_context; qp 344 drivers/infiniband/core/core_priv.h qp->rwq_ind_tbl = attr->rwq_ind_tbl; qp 345 drivers/infiniband/core/core_priv.h qp->send_cq = attr->send_cq; qp 346 drivers/infiniband/core/core_priv.h qp->recv_cq = attr->recv_cq; qp 347 drivers/infiniband/core/core_priv.h qp->srq = attr->srq; qp 348 drivers/infiniband/core/core_priv.h qp->rwq_ind_tbl = attr->rwq_ind_tbl; qp 349 drivers/infiniband/core/core_priv.h qp->event_handler = attr->event_handler; qp 351 drivers/infiniband/core/core_priv.h atomic_set(&qp->usecnt, 0); qp 352 drivers/infiniband/core/core_priv.h spin_lock_init(&qp->mr_lock); qp 353 drivers/infiniband/core/core_priv.h INIT_LIST_HEAD(&qp->rdma_mrs); qp 354 drivers/infiniband/core/core_priv.h INIT_LIST_HEAD(&qp->sig_mrs); qp 363 drivers/infiniband/core/core_priv.h qp->res.type = RDMA_RESTRACK_QP; qp 365 drivers/infiniband/core/core_priv.h rdma_restrack_uadd(&qp->res); qp 367 drivers/infiniband/core/core_priv.h rdma_restrack_kadd(&qp->res); qp 369 drivers/infiniband/core/core_priv.h qp->res.valid = false; qp 371 drivers/infiniband/core/core_priv.h return qp; qp 134 drivers/infiniband/core/counters.c const struct ib_qp *qp, qp 143 drivers/infiniband/core/counters.c param->qp_type = qp->qp_type; qp 146 drivers/infiniband/core/counters.c static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter, qp 152 drivers/infiniband/core/counters.c if (!rdma_is_visible_in_pid_ns(&qp->res)) qp 156 drivers/infiniband/core/counters.c if (task_pid_nr(counter->res.task) != task_pid_nr(qp->res.task)) qp 160 drivers/infiniband/core/counters.c match &= (param->qp_type == qp->qp_type); qp 166 drivers/infiniband/core/counters.c struct ib_qp *qp) qp 170 drivers/infiniband/core/counters.c if (qp->counter) qp 173 drivers/infiniband/core/counters.c if (!qp->device->ops.counter_bind_qp) qp 177 drivers/infiniband/core/counters.c ret = qp->device->ops.counter_bind_qp(counter, qp); qp 183 drivers/infiniband/core/counters.c static int __rdma_counter_unbind_qp(struct ib_qp *qp) qp 185 drivers/infiniband/core/counters.c struct rdma_counter *counter = qp->counter; qp 188 drivers/infiniband/core/counters.c if (!qp->device->ops.counter_unbind_qp) qp 192 drivers/infiniband/core/counters.c ret = qp->device->ops.counter_unbind_qp(qp); qp 218 drivers/infiniband/core/counters.c static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp, qp 223 drivers/infiniband/core/counters.c struct ib_device *dev = qp->device; qp 236 drivers/infiniband/core/counters.c if ((counter->device != qp->device) || (counter->port != port)) qp 239 drivers/infiniband/core/counters.c if (auto_mode_match(qp, counter, port_counter->mode.mask)) qp 253 drivers/infiniband/core/counters.c struct ib_qp *qp) qp 255 drivers/infiniband/core/counters.c if (rdma_is_kernel_res(&qp->res)) { qp 256 drivers/infiniband/core/counters.c rdma_restrack_set_task(&counter->res, qp->res.kern_name); qp 259 drivers/infiniband/core/counters.c rdma_restrack_attach_task(&counter->res, qp->res.task); qp 278 drivers/infiniband/core/counters.c int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port) qp 281 drivers/infiniband/core/counters.c struct ib_device *dev = qp->device; qp 285 drivers/infiniband/core/counters.c if (!qp->res.valid) qp 295 drivers/infiniband/core/counters.c counter = rdma_get_counter_auto_mode(qp, port); qp 297 drivers/infiniband/core/counters.c ret = __rdma_counter_bind_qp(counter, qp); qp 307 drivers/infiniband/core/counters.c auto_mode_init_counter(counter, qp, port_counter->mode.mask); qp 309 drivers/infiniband/core/counters.c ret = __rdma_counter_bind_qp(counter, qp); qp 315 drivers/infiniband/core/counters.c rdma_counter_res_add(counter, qp); qp 326 drivers/infiniband/core/counters.c int rdma_counter_unbind_qp(struct ib_qp *qp, bool force) qp 328 drivers/infiniband/core/counters.c struct rdma_counter *counter = qp->counter; qp 334 drivers/infiniband/core/counters.c ret = __rdma_counter_unbind_qp(qp); qp 412 drivers/infiniband/core/counters.c struct ib_qp *qp = NULL; qp 421 drivers/infiniband/core/counters.c qp = container_of(res, struct ib_qp, res); qp 422 drivers/infiniband/core/counters.c if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) qp 425 drivers/infiniband/core/counters.c return qp; qp 433 drivers/infiniband/core/counters.c struct ib_qp *qp) qp 435 drivers/infiniband/core/counters.c if ((counter->device != qp->device) || (counter->port != qp->port)) qp 438 drivers/infiniband/core/counters.c return __rdma_counter_bind_qp(counter, qp); qp 471 drivers/infiniband/core/counters.c struct ib_qp *qp; qp 478 drivers/infiniband/core/counters.c qp = rdma_counter_get_qp(dev, qp_num); qp 479 drivers/infiniband/core/counters.c if (!qp) qp 488 drivers/infiniband/core/counters.c if (counter->res.task != qp->res.task) { qp 493 drivers/infiniband/core/counters.c ret = rdma_counter_bind_qp_manual(counter, qp); qp 497 drivers/infiniband/core/counters.c rdma_restrack_put(&qp->res); qp 503 drivers/infiniband/core/counters.c rdma_restrack_put(&qp->res); qp 516 drivers/infiniband/core/counters.c struct ib_qp *qp; qp 529 drivers/infiniband/core/counters.c qp = rdma_counter_get_qp(dev, qp_num); qp 530 drivers/infiniband/core/counters.c if (!qp) qp 533 drivers/infiniband/core/counters.c if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { qp 544 drivers/infiniband/core/counters.c ret = rdma_counter_bind_qp_manual(counter, qp); qp 551 drivers/infiniband/core/counters.c rdma_counter_res_add(counter, qp); qp 553 drivers/infiniband/core/counters.c rdma_restrack_put(&qp->res); qp 559 drivers/infiniband/core/counters.c rdma_restrack_put(&qp->res); qp 570 drivers/infiniband/core/counters.c struct ib_qp *qp; qp 576 drivers/infiniband/core/counters.c qp = rdma_counter_get_qp(dev, qp_num); qp 577 drivers/infiniband/core/counters.c if (!qp) qp 580 drivers/infiniband/core/counters.c if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { qp 586 drivers/infiniband/core/counters.c if (!qp->counter || qp->counter->id != counter_id || qp 592 drivers/infiniband/core/counters.c ret = rdma_counter_unbind_qp(qp, false); qp 595 drivers/infiniband/core/counters.c rdma_restrack_put(&qp->res); qp 271 drivers/infiniband/core/iwcm.c static int iwcm_modify_qp_err(struct ib_qp *qp) qp 275 drivers/infiniband/core/iwcm.c if (!qp) qp 279 drivers/infiniband/core/iwcm.c return ib_modify_qp(qp, &qp_attr, IB_QP_STATE); qp 286 drivers/infiniband/core/iwcm.c static int iwcm_modify_qp_sqd(struct ib_qp *qp) qp 290 drivers/infiniband/core/iwcm.c BUG_ON(qp == NULL); qp 292 drivers/infiniband/core/iwcm.c return ib_modify_qp(qp, &qp_attr, IB_QP_STATE); qp 312 drivers/infiniband/core/iwcm.c struct ib_qp *qp = NULL; qp 325 drivers/infiniband/core/iwcm.c if (cm_id_priv->qp) qp 326 drivers/infiniband/core/iwcm.c qp = cm_id_priv->qp; qp 351 drivers/infiniband/core/iwcm.c if (qp) { qp 353 drivers/infiniband/core/iwcm.c ret = iwcm_modify_qp_err(qp); qp 355 drivers/infiniband/core/iwcm.c ret = iwcm_modify_qp_sqd(qp); qp 377 drivers/infiniband/core/iwcm.c struct ib_qp *qp; qp 395 drivers/infiniband/core/iwcm.c qp = cm_id_priv->qp; qp 396 drivers/infiniband/core/iwcm.c cm_id_priv->qp = NULL; qp 410 drivers/infiniband/core/iwcm.c (void)iwcm_modify_qp_err(qp); qp 436 drivers/infiniband/core/iwcm.c if (qp) qp 437 drivers/infiniband/core/iwcm.c cm_id_priv->id.device->ops.iw_rem_ref(qp); qp 646 drivers/infiniband/core/iwcm.c struct ib_qp *qp; qp 661 drivers/infiniband/core/iwcm.c qp = cm_id->device->ops.iw_get_qp(cm_id->device, iw_param->qpn); qp 662 drivers/infiniband/core/iwcm.c if (!qp) { qp 668 drivers/infiniband/core/iwcm.c cm_id->device->ops.iw_add_ref(qp); qp 669 drivers/infiniband/core/iwcm.c cm_id_priv->qp = qp; qp 678 drivers/infiniband/core/iwcm.c qp = cm_id_priv->qp; qp 679 drivers/infiniband/core/iwcm.c cm_id_priv->qp = NULL; qp 681 drivers/infiniband/core/iwcm.c if (qp) qp 682 drivers/infiniband/core/iwcm.c cm_id->device->ops.iw_rem_ref(qp); qp 703 drivers/infiniband/core/iwcm.c struct ib_qp *qp = NULL; qp 720 drivers/infiniband/core/iwcm.c qp = cm_id->device->ops.iw_get_qp(cm_id->device, iw_param->qpn); qp 721 drivers/infiniband/core/iwcm.c if (!qp) { qp 725 drivers/infiniband/core/iwcm.c cm_id->device->ops.iw_add_ref(qp); qp 726 drivers/infiniband/core/iwcm.c cm_id_priv->qp = qp; qp 737 drivers/infiniband/core/iwcm.c qp = cm_id_priv->qp; qp 738 drivers/infiniband/core/iwcm.c cm_id_priv->qp = NULL; qp 742 drivers/infiniband/core/iwcm.c if (qp) qp 743 drivers/infiniband/core/iwcm.c cm_id->device->ops.iw_rem_ref(qp); qp 885 drivers/infiniband/core/iwcm.c struct ib_qp *qp = NULL; qp 904 drivers/infiniband/core/iwcm.c qp = cm_id_priv->qp; qp 905 drivers/infiniband/core/iwcm.c cm_id_priv->qp = NULL; qp 909 drivers/infiniband/core/iwcm.c if (qp) qp 910 drivers/infiniband/core/iwcm.c cm_id_priv->id.device->ops.iw_rem_ref(qp); qp 952 drivers/infiniband/core/iwcm.c struct ib_qp *qp; qp 956 drivers/infiniband/core/iwcm.c qp = cm_id_priv->qp; qp 957 drivers/infiniband/core/iwcm.c cm_id_priv->qp = NULL; qp 972 drivers/infiniband/core/iwcm.c if (qp) qp 973 drivers/infiniband/core/iwcm.c cm_id_priv->id.device->ops.iw_rem_ref(qp); qp 50 drivers/infiniband/core/iwcm.h struct ib_qp *qp; qp 365 drivers/infiniband/core/mad.c if (!port_priv->qp_info[qpn].qp) { qp 395 drivers/infiniband/core/mad.c mad_agent_priv->agent.qp = port_priv->qp_info[qpn].qp; qp 579 drivers/infiniband/core/mad.c mad_snoop_priv->agent.qp = port_priv->qp_info[qpn].qp; qp 760 drivers/infiniband/core/mad.c static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid, qp 770 drivers/infiniband/core/mad.c wc->qp = qp; qp 903 drivers/infiniband/core/mad.c build_smp_wc(mad_agent_priv->agent.qp, qp 1097 drivers/infiniband/core/mad.c mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey; qp 1106 drivers/infiniband/core/mad.c mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey; qp 1251 drivers/infiniband/core/mad.c ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr, qp 1400 drivers/infiniband/core/mad.c struct ib_mad_agent *ib_redirect_mad_qp(struct ib_qp *qp, qp 1858 drivers/infiniband/core/mad.c u32 qp_num = qp_info->qp->qp_num; qp 2137 drivers/infiniband/core/mad.c qp_info->qp->qp_num, qp 2227 drivers/infiniband/core/mad.c qp_info->qp->qp_num, qp 2358 drivers/infiniband/core/mad.c qp_info->qp->qp_num, qp 2378 drivers/infiniband/core/mad.c qp_info->qp->qp_num, mad_size, opa); qp 2567 drivers/infiniband/core/mad.c ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr, qp 2615 drivers/infiniband/core/mad.c ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr, qp 2628 drivers/infiniband/core/mad.c ret = ib_modify_qp(qp_info->qp, attr, qp 2781 drivers/infiniband/core/mad.c build_smp_wc(recv_mad_agent->agent.qp, qp 2976 drivers/infiniband/core/mad.c ret = ib_post_recv(qp_info->qp, &recv_wr, NULL); qp 3005 drivers/infiniband/core/mad.c if (!qp_info->qp) qp 3038 drivers/infiniband/core/mad.c struct ib_qp *qp; qp 3051 drivers/infiniband/core/mad.c qp = port_priv->qp_info[i].qp; qp 3052 drivers/infiniband/core/mad.c if (!qp) qp 3061 drivers/infiniband/core/mad.c attr->qkey = (qp->qp_num == 0) ? 0 : IB_QP1_QKEY; qp 3062 drivers/infiniband/core/mad.c ret = ib_modify_qp(qp, attr, IB_QP_STATE | qp 3072 drivers/infiniband/core/mad.c ret = ib_modify_qp(qp, attr, IB_QP_STATE); qp 3082 drivers/infiniband/core/mad.c ret = ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_SQ_PSN); qp 3100 drivers/infiniband/core/mad.c if (!port_priv->qp_info[i].qp) qp 3122 drivers/infiniband/core/mad.c event->event, qp_info->qp->qp_num); qp 3165 drivers/infiniband/core/mad.c qp_info->qp = ib_create_qp(qp_info->port_priv->pd, &qp_init_attr); qp 3166 drivers/infiniband/core/mad.c if (IS_ERR(qp_info->qp)) { qp 3170 drivers/infiniband/core/mad.c ret = PTR_ERR(qp_info->qp); qp 3184 drivers/infiniband/core/mad.c if (!qp_info->qp) qp 3187 drivers/infiniband/core/mad.c ib_destroy_qp(qp_info->qp); qp 189 drivers/infiniband/core/mad_priv.h struct ib_qp *qp; qp 163 drivers/infiniband/core/mad_rmpp.c ah = ib_create_ah_from_wc(agent->qp->pd, recv_wc->wc, qp 295 drivers/infiniband/core/mad_rmpp.c rmpp_recv->ah = ib_create_ah_from_wc(agent->agent.qp->pd, qp 8 drivers/infiniband/core/mr_pool.c struct ib_mr *ib_mr_pool_get(struct ib_qp *qp, struct list_head *list) qp 13 drivers/infiniband/core/mr_pool.c spin_lock_irqsave(&qp->mr_lock, flags); qp 17 drivers/infiniband/core/mr_pool.c qp->mrs_used++; qp 19 drivers/infiniband/core/mr_pool.c spin_unlock_irqrestore(&qp->mr_lock, flags); qp 25 drivers/infiniband/core/mr_pool.c void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr) qp 29 drivers/infiniband/core/mr_pool.c spin_lock_irqsave(&qp->mr_lock, flags); qp 31 drivers/infiniband/core/mr_pool.c qp->mrs_used--; qp 32 drivers/infiniband/core/mr_pool.c spin_unlock_irqrestore(&qp->mr_lock, flags); qp 36 drivers/infiniband/core/mr_pool.c int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr, qp 45 drivers/infiniband/core/mr_pool.c mr = ib_alloc_mr_integrity(qp->pd, max_num_sg, qp 48 drivers/infiniband/core/mr_pool.c mr = ib_alloc_mr(qp->pd, type, max_num_sg); qp 54 drivers/infiniband/core/mr_pool.c spin_lock_irqsave(&qp->mr_lock, flags); qp 56 drivers/infiniband/core/mr_pool.c spin_unlock_irqrestore(&qp->mr_lock, flags); qp 61 drivers/infiniband/core/mr_pool.c ib_mr_pool_destroy(qp, list); qp 66 drivers/infiniband/core/mr_pool.c void ib_mr_pool_destroy(struct ib_qp *qp, struct list_head *list) qp 71 drivers/infiniband/core/mr_pool.c spin_lock_irqsave(&qp->mr_lock, flags); qp 76 drivers/infiniband/core/mr_pool.c spin_unlock_irqrestore(&qp->mr_lock, flags); qp 78 drivers/infiniband/core/mr_pool.c spin_lock_irqsave(&qp->mr_lock, flags); qp 80 drivers/infiniband/core/mr_pool.c spin_unlock_irqrestore(&qp->mr_lock, flags); qp 429 drivers/infiniband/core/nldev.c struct ib_qp *qp = container_of(res, struct ib_qp, res); qp 430 drivers/infiniband/core/nldev.c struct ib_device *dev = qp->device; qp 435 drivers/infiniband/core/nldev.c ret = ib_query_qp(qp, &qp_attr, 0, &qp_init_attr); qp 447 drivers/infiniband/core/nldev.c if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num)) qp 449 drivers/infiniband/core/nldev.c if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) { qp 461 drivers/infiniband/core/nldev.c if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC || qp 462 drivers/infiniband/core/nldev.c qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) { qp 467 drivers/infiniband/core/nldev.c if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, qp->qp_type)) qp 473 drivers/infiniband/core/nldev.c nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id)) qp 692 drivers/infiniband/core/nldev.c struct ib_qp *qp = NULL; qp 704 drivers/infiniband/core/nldev.c qp = container_of(res, struct ib_qp, res); qp 705 drivers/infiniband/core/nldev.c if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) qp 708 drivers/infiniband/core/nldev.c if (!qp->counter || (qp->counter->id != counter->id)) qp 711 drivers/infiniband/core/nldev.c ret = fill_stat_counter_qp_entry(msg, qp->qp_num); qp 1414 drivers/infiniband/core/nldev.c RES_GET_FUNCS(qp, RDMA_RESTRACK_QP); qp 223 drivers/infiniband/core/restrack.c struct ib_qp *qp = container_of(res, struct ib_qp, res); qp 225 drivers/infiniband/core/restrack.c ret = xa_insert(&rt->xa, qp->qp_num, res, GFP_KERNEL); qp 226 drivers/infiniband/core/restrack.c res->id = ret ? 0 : qp->qp_num; qp 85 drivers/infiniband/core/rw.c static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num, qp 89 drivers/infiniband/core/rw.c u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device, qp 90 drivers/infiniband/core/rw.c qp->integrity_en); qp 94 drivers/infiniband/core/rw.c reg->mr = ib_mr_pool_get(qp, &qp->rdma_mrs); qp 102 drivers/infiniband/core/rw.c ib_mr_pool_put(qp, &qp->rdma_mrs, reg->mr); qp 109 drivers/infiniband/core/rw.c if (rdma_protocol_iwarp(qp->device, port_num)) qp 118 drivers/infiniband/core/rw.c static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 123 drivers/infiniband/core/rw.c u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device, qp 124 drivers/infiniband/core/rw.c qp->integrity_en); qp 138 drivers/infiniband/core/rw.c ret = rdma_rw_init_one_mr(qp, port_num, reg, sg, sg_cnt, qp 159 drivers/infiniband/core/rw.c } else if (!rdma_cap_read_inv(qp->device, port_num)) { qp 183 drivers/infiniband/core/rw.c ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr); qp 189 drivers/infiniband/core/rw.c static int rdma_rw_init_map_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 193 drivers/infiniband/core/rw.c u32 max_sge = dir == DMA_TO_DEVICE ? qp->max_write_sge : qp 194 drivers/infiniband/core/rw.c qp->max_read_sge; qp 224 drivers/infiniband/core/rw.c sge->lkey = qp->pd->local_dma_lkey; qp 245 drivers/infiniband/core/rw.c static int rdma_rw_init_single_wr(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 253 drivers/infiniband/core/rw.c ctx->single.sge.lkey = qp->pd->local_dma_lkey; qp 303 drivers/infiniband/core/rw.c int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, qp 307 drivers/infiniband/core/rw.c struct ib_device *dev = qp->pd->device; qp 333 drivers/infiniband/core/rw.c if (rdma_rw_io_needs_mr(qp->device, port_num, dir, sg_cnt)) { qp 334 drivers/infiniband/core/rw.c ret = rdma_rw_init_mr_wrs(ctx, qp, port_num, sg, sg_cnt, qp 337 drivers/infiniband/core/rw.c ret = rdma_rw_init_map_wrs(ctx, qp, sg, sg_cnt, sg_offset, qp 340 drivers/infiniband/core/rw.c ret = rdma_rw_init_single_wr(ctx, qp, sg, sg_offset, qp 371 drivers/infiniband/core/rw.c int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 377 drivers/infiniband/core/rw.c struct ib_device *dev = qp->pd->device; qp 378 drivers/infiniband/core/rw.c u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device, qp 379 drivers/infiniband/core/rw.c qp->integrity_en); qp 411 drivers/infiniband/core/rw.c ctx->reg->mr = ib_mr_pool_get(qp, &qp->sig_mrs); qp 433 drivers/infiniband/core/rw.c if (rdma_protocol_iwarp(qp->device, port_num)) qp 459 drivers/infiniband/core/rw.c ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg->mr); qp 499 drivers/infiniband/core/rw.c struct ib_send_wr *rdma_rw_ctx_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 558 drivers/infiniband/core/rw.c int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, qp 563 drivers/infiniband/core/rw.c first_wr = rdma_rw_ctx_wrs(ctx, qp, port_num, cqe, chain_wr); qp 564 drivers/infiniband/core/rw.c return ib_post_send(qp, first_wr, NULL); qp 577 drivers/infiniband/core/rw.c void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, qp 585 drivers/infiniband/core/rw.c ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr); qp 599 drivers/infiniband/core/rw.c rdma_rw_unmap_sg(qp->pd->device, sg, sg_cnt, dir); qp 615 drivers/infiniband/core/rw.c void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 623 drivers/infiniband/core/rw.c ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg->mr); qp 626 drivers/infiniband/core/rw.c ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir); qp 628 drivers/infiniband/core/rw.c ib_dma_unmap_sg(qp->pd->device, prot_sg, prot_sg_cnt, dir); qp 689 drivers/infiniband/core/rw.c int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr) qp 691 drivers/infiniband/core/rw.c struct ib_device *dev = qp->pd->device; qp 705 drivers/infiniband/core/rw.c ret = ib_mr_pool_init(qp, &qp->rdma_mrs, nr_mrs, qp 716 drivers/infiniband/core/rw.c ret = ib_mr_pool_init(qp, &qp->sig_mrs, nr_sig_mrs, qp 728 drivers/infiniband/core/rw.c ib_mr_pool_destroy(qp, &qp->rdma_mrs); qp 732 drivers/infiniband/core/rw.c void rdma_rw_cleanup_mrs(struct ib_qp *qp) qp 734 drivers/infiniband/core/rw.c ib_mr_pool_destroy(qp, &qp->sig_mrs); qp 735 drivers/infiniband/core/rw.c ib_mr_pool_destroy(qp, &qp->rdma_mrs); qp 2271 drivers/infiniband/core/sa_query.c new_ah->ah = rdma_create_ah(port->agent->qp->pd, &ah_attr, qp 170 drivers/infiniband/core/security.c ib_modify_qp(sec->qp, qp 174 drivers/infiniband/core/security.c if (sec->qp->event_handler && sec->qp->qp_context) { qp 175 drivers/infiniband/core/security.c event.element.qp = sec->qp; qp 176 drivers/infiniband/core/security.c sec->qp->event_handler(&event, qp 177 drivers/infiniband/core/security.c sec->qp->qp_context); qp 183 drivers/infiniband/core/security.c struct ib_qp *qp = shared_qp_sec->qp; qp 185 drivers/infiniband/core/security.c if (qp->event_handler && qp->qp_context) { qp 186 drivers/infiniband/core/security.c event.element.qp = qp; qp 187 drivers/infiniband/core/security.c event.device = qp->device; qp 188 drivers/infiniband/core/security.c qp->event_handler(&event, qp 189 drivers/infiniband/core/security.c qp->qp_context); qp 331 drivers/infiniband/core/security.c static struct ib_ports_pkeys *get_new_pps(const struct ib_qp *qp, qp 336 drivers/infiniband/core/security.c struct ib_ports_pkeys *qp_pps = qp->qp_sec->ports_pkeys; qp 368 drivers/infiniband/core/security.c new_pps->main.sec = qp->qp_sec; qp 369 drivers/infiniband/core/security.c new_pps->alt.sec = qp->qp_sec; qp 373 drivers/infiniband/core/security.c int ib_open_shared_qp_security(struct ib_qp *qp, struct ib_device *dev) qp 375 drivers/infiniband/core/security.c struct ib_qp *real_qp = qp->real_qp; qp 378 drivers/infiniband/core/security.c ret = ib_create_qp_security(qp, dev); qp 383 drivers/infiniband/core/security.c if (!qp->qp_sec) qp 388 drivers/infiniband/core/security.c qp->qp_sec); qp 393 drivers/infiniband/core/security.c if (qp != real_qp) qp 394 drivers/infiniband/core/security.c list_add(&qp->qp_sec->shared_qp_list, qp 399 drivers/infiniband/core/security.c destroy_qp_security(qp->qp_sec); qp 406 drivers/infiniband/core/security.c struct ib_qp *real_qp = sec->qp->real_qp; qp 415 drivers/infiniband/core/security.c int ib_create_qp_security(struct ib_qp *qp, struct ib_device *dev) qp 431 drivers/infiniband/core/security.c qp->qp_sec = kzalloc(sizeof(*qp->qp_sec), GFP_KERNEL); qp 432 drivers/infiniband/core/security.c if (!qp->qp_sec) qp 435 drivers/infiniband/core/security.c qp->qp_sec->qp = qp; qp 436 drivers/infiniband/core/security.c qp->qp_sec->dev = dev; qp 437 drivers/infiniband/core/security.c mutex_init(&qp->qp_sec->mutex); qp 438 drivers/infiniband/core/security.c INIT_LIST_HEAD(&qp->qp_sec->shared_qp_list); qp 439 drivers/infiniband/core/security.c atomic_set(&qp->qp_sec->error_list_count, 0); qp 440 drivers/infiniband/core/security.c init_completion(&qp->qp_sec->error_complete); qp 441 drivers/infiniband/core/security.c ret = security_ib_alloc_security(&qp->qp_sec->security); qp 443 drivers/infiniband/core/security.c kfree(qp->qp_sec); qp 444 drivers/infiniband/core/security.c qp->qp_sec = NULL; qp 571 drivers/infiniband/core/security.c int ib_security_modify_qp(struct ib_qp *qp, qp 579 drivers/infiniband/core/security.c struct ib_qp *real_qp = qp->real_qp; qp 729 drivers/infiniband/core/security.c if (agent->qp->qp_type == IB_QPT_SMI) { qp 743 drivers/infiniband/core/security.c if (map->agent.qp->qp_type == IB_QPT_SMI) { qp 547 drivers/infiniband/core/user_mad.c ah = rdma_create_user_ah(agent->qp->pd, &ah_attr, NULL); qp 246 drivers/infiniband/core/uverbs.h void ib_uverbs_detach_umcast(struct ib_qp *qp, qp 1150 drivers/infiniband/core/uverbs_cmd.c tmp.qp_num = wc->qp->qp_num; qp 1279 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 1419 drivers/infiniband/core/uverbs_cmd.c qp = ib_create_qp(pd, &attr); qp 1421 drivers/infiniband/core/uverbs_cmd.c qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, qp 1424 drivers/infiniband/core/uverbs_cmd.c if (IS_ERR(qp)) { qp 1425 drivers/infiniband/core/uverbs_cmd.c ret = PTR_ERR(qp); qp 1430 drivers/infiniband/core/uverbs_cmd.c ret = ib_create_qp_security(qp, device); qp 1445 drivers/infiniband/core/uverbs_cmd.c qp->uobject = &obj->uevent.uobject; qp 1448 drivers/infiniband/core/uverbs_cmd.c obj->uevent.uobject.object = qp; qp 1451 drivers/infiniband/core/uverbs_cmd.c resp.base.qpn = qp->qp_num; qp 1484 drivers/infiniband/core/uverbs_cmd.c ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); qp 1557 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 1592 drivers/infiniband/core/uverbs_cmd.c qp = ib_open_qp(xrcd, &attr); qp 1593 drivers/infiniband/core/uverbs_cmd.c if (IS_ERR(qp)) { qp 1594 drivers/infiniband/core/uverbs_cmd.c ret = PTR_ERR(qp); qp 1598 drivers/infiniband/core/uverbs_cmd.c obj->uevent.uobject.object = qp; qp 1602 drivers/infiniband/core/uverbs_cmd.c resp.qpn = qp->qp_num; qp 1611 drivers/infiniband/core/uverbs_cmd.c qp->uobject = &obj->uevent.uobject; qp 1617 drivers/infiniband/core/uverbs_cmd.c ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); qp 1651 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 1667 drivers/infiniband/core/uverbs_cmd.c qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); qp 1668 drivers/infiniband/core/uverbs_cmd.c if (!qp) { qp 1673 drivers/infiniband/core/uverbs_cmd.c ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); qp 1675 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 1764 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 1771 drivers/infiniband/core/uverbs_cmd.c qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, qp 1773 drivers/infiniband/core/uverbs_cmd.c if (!qp) { qp 1779 drivers/infiniband/core/uverbs_cmd.c !rdma_is_port_valid(qp->device, cmd->base.port_num)) { qp 1785 drivers/infiniband/core/uverbs_cmd.c if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) { qp 1806 drivers/infiniband/core/uverbs_cmd.c if (cmd->base.dest.port_num != qp->real_qp->port) { qp 1839 drivers/infiniband/core/uverbs_cmd.c (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) || qp 1840 drivers/infiniband/core/uverbs_cmd.c !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) || qp 1899 drivers/infiniband/core/uverbs_cmd.c copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr, qp 1903 drivers/infiniband/core/uverbs_cmd.c copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr, qp 1906 drivers/infiniband/core/uverbs_cmd.c ret = ib_modify_qp_with_udata(qp, attr, qp 1907 drivers/infiniband/core/uverbs_cmd.c modify_qp_mask(qp->qp_type, qp 1912 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 2006 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 2033 drivers/infiniband/core/uverbs_cmd.c qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); qp 2034 drivers/infiniband/core/uverbs_cmd.c if (!qp) { qp 2039 drivers/infiniband/core/uverbs_cmd.c is_ud = qp->qp_type == IB_QPT_UD; qp 2163 drivers/infiniband/core/uverbs_cmd.c ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr); qp 2176 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 2296 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 2309 drivers/infiniband/core/uverbs_cmd.c qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); qp 2310 drivers/infiniband/core/uverbs_cmd.c if (!qp) { qp 2316 drivers/infiniband/core/uverbs_cmd.c ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr); qp 2318 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 2485 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 2494 drivers/infiniband/core/uverbs_cmd.c qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); qp 2495 drivers/infiniband/core/uverbs_cmd.c if (!qp) qp 2498 drivers/infiniband/core/uverbs_cmd.c obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); qp 2517 drivers/infiniband/core/uverbs_cmd.c ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); qp 2525 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 2534 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 2543 drivers/infiniband/core/uverbs_cmd.c qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); qp 2544 drivers/infiniband/core/uverbs_cmd.c if (!qp) qp 2547 drivers/infiniband/core/uverbs_cmd.c obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); qp 2564 drivers/infiniband/core/uverbs_cmd.c ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid); qp 2568 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 3180 drivers/infiniband/core/uverbs_cmd.c struct ib_qp *qp; qp 3243 drivers/infiniband/core/uverbs_cmd.c qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); qp 3244 drivers/infiniband/core/uverbs_cmd.c if (!qp) { qp 3249 drivers/infiniband/core/uverbs_cmd.c if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) { qp 3298 drivers/infiniband/core/uverbs_cmd.c flow_id = qp->device->ops.create_flow( qp 3299 drivers/infiniband/core/uverbs_cmd.c qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata); qp 3306 drivers/infiniband/core/uverbs_cmd.c ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res); qp 3315 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 3321 drivers/infiniband/core/uverbs_cmd.c if (!qp->device->ops.destroy_flow(flow_id)) qp 3322 drivers/infiniband/core/uverbs_cmd.c atomic_dec(&qp->usecnt); qp 3328 drivers/infiniband/core/uverbs_cmd.c uobj_put_obj_read(qp); qp 174 drivers/infiniband/core/uverbs_main.c void ib_uverbs_detach_umcast(struct ib_qp *qp, qp 180 drivers/infiniband/core/uverbs_main.c ib_detach_mcast(qp, &mcast->gid, mcast->lid); qp 501 drivers/infiniband/core/uverbs_main.c if (!event->element.qp->uobject) qp 504 drivers/infiniband/core/uverbs_main.c uobj = container_of(event->element.qp->uobject, qp 58 drivers/infiniband/core/uverbs_std_types.c struct ib_qp *qp = flow->qp; qp 63 drivers/infiniband/core/uverbs_std_types.c if (qp) qp 64 drivers/infiniband/core/uverbs_std_types.c atomic_dec(&qp->usecnt); qp 82 drivers/infiniband/core/uverbs_std_types.c struct ib_qp *qp = uobject->object; qp 97 drivers/infiniband/core/uverbs_std_types.c } else if (qp == qp->real_qp) { qp 98 drivers/infiniband/core/uverbs_std_types.c ib_uverbs_detach_umcast(qp, uqp); qp 101 drivers/infiniband/core/uverbs_std_types.c ret = ib_destroy_qp_user(qp, &attrs->driver_udata); qp 1051 drivers/infiniband/core/verbs.c struct ib_qp *qp = context; qp 1054 drivers/infiniband/core/verbs.c spin_lock_irqsave(&qp->device->event_handler_lock, flags); qp 1055 drivers/infiniband/core/verbs.c list_for_each_entry(event->element.qp, &qp->open_list, open_list) qp 1056 drivers/infiniband/core/verbs.c if (event->element.qp->event_handler) qp 1057 drivers/infiniband/core/verbs.c event->element.qp->event_handler(event, event->element.qp->qp_context); qp 1058 drivers/infiniband/core/verbs.c spin_unlock_irqrestore(&qp->device->event_handler_lock, flags); qp 1061 drivers/infiniband/core/verbs.c static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) qp 1064 drivers/infiniband/core/verbs.c list_add(&qp->xrcd_list, &xrcd->tgt_qp_list); qp 1072 drivers/infiniband/core/verbs.c struct ib_qp *qp; qp 1076 drivers/infiniband/core/verbs.c qp = kzalloc(sizeof *qp, GFP_KERNEL); qp 1077 drivers/infiniband/core/verbs.c if (!qp) qp 1080 drivers/infiniband/core/verbs.c qp->real_qp = real_qp; qp 1081 drivers/infiniband/core/verbs.c err = ib_open_shared_qp_security(qp, real_qp->device); qp 1083 drivers/infiniband/core/verbs.c kfree(qp); qp 1087 drivers/infiniband/core/verbs.c qp->real_qp = real_qp; qp 1089 drivers/infiniband/core/verbs.c qp->device = real_qp->device; qp 1090 drivers/infiniband/core/verbs.c qp->event_handler = event_handler; qp 1091 drivers/infiniband/core/verbs.c qp->qp_context = qp_context; qp 1092 drivers/infiniband/core/verbs.c qp->qp_num = real_qp->qp_num; qp 1093 drivers/infiniband/core/verbs.c qp->qp_type = real_qp->qp_type; qp 1096 drivers/infiniband/core/verbs.c list_add(&qp->open_list, &real_qp->open_list); qp 1099 drivers/infiniband/core/verbs.c return qp; qp 1105 drivers/infiniband/core/verbs.c struct ib_qp *qp, *real_qp; qp 1110 drivers/infiniband/core/verbs.c qp = ERR_PTR(-EINVAL); qp 1114 drivers/infiniband/core/verbs.c qp = __ib_open_qp(real_qp, qp_open_attr->event_handler, qp 1120 drivers/infiniband/core/verbs.c return qp; qp 1124 drivers/infiniband/core/verbs.c static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, qp 1128 drivers/infiniband/core/verbs.c struct ib_qp *real_qp = qp; qp 1130 drivers/infiniband/core/verbs.c qp->event_handler = __ib_shared_qp_event_handler; qp 1131 drivers/infiniband/core/verbs.c qp->qp_context = qp; qp 1132 drivers/infiniband/core/verbs.c qp->pd = NULL; qp 1133 drivers/infiniband/core/verbs.c qp->send_cq = qp->recv_cq = NULL; qp 1134 drivers/infiniband/core/verbs.c qp->srq = NULL; qp 1135 drivers/infiniband/core/verbs.c qp->xrcd = qp_init_attr->xrcd; qp 1137 drivers/infiniband/core/verbs.c INIT_LIST_HEAD(&qp->open_list); qp 1139 drivers/infiniband/core/verbs.c qp = __ib_open_qp(real_qp, qp_init_attr->event_handler, qp 1141 drivers/infiniband/core/verbs.c if (IS_ERR(qp)) qp 1142 drivers/infiniband/core/verbs.c return qp; qp 1145 drivers/infiniband/core/verbs.c return qp; qp 1153 drivers/infiniband/core/verbs.c struct ib_qp *qp; qp 1175 drivers/infiniband/core/verbs.c qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL); qp 1176 drivers/infiniband/core/verbs.c if (IS_ERR(qp)) qp 1177 drivers/infiniband/core/verbs.c return qp; qp 1179 drivers/infiniband/core/verbs.c ret = ib_create_qp_security(qp, device); qp 1185 drivers/infiniband/core/verbs.c create_xrc_qp_user(qp, qp_init_attr, udata); qp 1194 drivers/infiniband/core/verbs.c qp->event_handler = qp_init_attr->event_handler; qp 1195 drivers/infiniband/core/verbs.c qp->qp_context = qp_init_attr->qp_context; qp 1197 drivers/infiniband/core/verbs.c qp->recv_cq = NULL; qp 1198 drivers/infiniband/core/verbs.c qp->srq = NULL; qp 1200 drivers/infiniband/core/verbs.c qp->recv_cq = qp_init_attr->recv_cq; qp 1203 drivers/infiniband/core/verbs.c qp->srq = qp_init_attr->srq; qp 1204 drivers/infiniband/core/verbs.c if (qp->srq) qp 1208 drivers/infiniband/core/verbs.c qp->send_cq = qp_init_attr->send_cq; qp 1209 drivers/infiniband/core/verbs.c qp->xrcd = NULL; qp 1215 drivers/infiniband/core/verbs.c atomic_inc(&qp->rwq_ind_tbl->usecnt); qp 1218 drivers/infiniband/core/verbs.c ret = rdma_rw_init_mrs(qp, qp_init_attr); qp 1228 drivers/infiniband/core/verbs.c qp->max_write_sge = qp_init_attr->cap.max_send_sge; qp 1229 drivers/infiniband/core/verbs.c qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge, qp 1232 drivers/infiniband/core/verbs.c qp->integrity_en = true; qp 1234 drivers/infiniband/core/verbs.c return qp; qp 1237 drivers/infiniband/core/verbs.c ib_destroy_qp(qp); qp 1606 drivers/infiniband/core/verbs.c static bool is_qp_type_connected(const struct ib_qp *qp) qp 1608 drivers/infiniband/core/verbs.c return (qp->qp_type == IB_QPT_UC || qp 1609 drivers/infiniband/core/verbs.c qp->qp_type == IB_QPT_RC || qp 1610 drivers/infiniband/core/verbs.c qp->qp_type == IB_QPT_XRC_INI || qp 1611 drivers/infiniband/core/verbs.c qp->qp_type == IB_QPT_XRC_TGT); qp 1617 drivers/infiniband/core/verbs.c static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, qp 1620 drivers/infiniband/core/verbs.c u8 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; qp 1626 drivers/infiniband/core/verbs.c ret = rdma_fill_sgid_attr(qp->device, &attr->ah_attr, qp 1639 drivers/infiniband/core/verbs.c ret = rdma_fill_sgid_attr(qp->device, &attr->alt_ah_attr, qp 1648 drivers/infiniband/core/verbs.c if (!(rdma_protocol_ib(qp->device, qp 1650 drivers/infiniband/core/verbs.c rdma_protocol_ib(qp->device, port))) { qp 1662 drivers/infiniband/core/verbs.c is_qp_type_connected(qp)) { qp 1663 drivers/infiniband/core/verbs.c ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr); qp 1668 drivers/infiniband/core/verbs.c if (rdma_ib_or_roce(qp->device, port)) { qp 1670 drivers/infiniband/core/verbs.c dev_warn(&qp->device->dev, qp 1677 drivers/infiniband/core/verbs.c dev_warn(&qp->device->dev, qp 1688 drivers/infiniband/core/verbs.c if (!qp->counter && (attr_mask & IB_QP_PORT) && qp 1690 drivers/infiniband/core/verbs.c rdma_counter_bind_qp_auto(qp, attr->port_num); qp 1692 drivers/infiniband/core/verbs.c ret = ib_security_modify_qp(qp, attr, attr_mask, udata); qp 1697 drivers/infiniband/core/verbs.c qp->port = attr->port_num; qp 1699 drivers/infiniband/core/verbs.c qp->av_sgid_attr = qp 1700 drivers/infiniband/core/verbs.c rdma_update_sgid_attr(&attr->ah_attr, qp->av_sgid_attr); qp 1702 drivers/infiniband/core/verbs.c qp->alt_path_sgid_attr = rdma_update_sgid_attr( qp 1703 drivers/infiniband/core/verbs.c &attr->alt_ah_attr, qp->alt_path_sgid_attr); qp 1784 drivers/infiniband/core/verbs.c int ib_modify_qp(struct ib_qp *qp, qp 1788 drivers/infiniband/core/verbs.c return _ib_modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL); qp 1792 drivers/infiniband/core/verbs.c int ib_query_qp(struct ib_qp *qp, qp 1800 drivers/infiniband/core/verbs.c return qp->device->ops.query_qp ? qp 1801 drivers/infiniband/core/verbs.c qp->device->ops.query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp 1806 drivers/infiniband/core/verbs.c int ib_close_qp(struct ib_qp *qp) qp 1811 drivers/infiniband/core/verbs.c real_qp = qp->real_qp; qp 1812 drivers/infiniband/core/verbs.c if (real_qp == qp) qp 1816 drivers/infiniband/core/verbs.c list_del(&qp->open_list); qp 1820 drivers/infiniband/core/verbs.c if (qp->qp_sec) qp 1821 drivers/infiniband/core/verbs.c ib_close_shared_qp_security(qp->qp_sec); qp 1822 drivers/infiniband/core/verbs.c kfree(qp); qp 1828 drivers/infiniband/core/verbs.c static int __ib_destroy_shared_qp(struct ib_qp *qp) qp 1834 drivers/infiniband/core/verbs.c real_qp = qp->real_qp; qp 1838 drivers/infiniband/core/verbs.c ib_close_qp(qp); qp 1856 drivers/infiniband/core/verbs.c int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) qp 1858 drivers/infiniband/core/verbs.c const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr; qp 1859 drivers/infiniband/core/verbs.c const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr; qp 1867 drivers/infiniband/core/verbs.c WARN_ON_ONCE(qp->mrs_used > 0); qp 1869 drivers/infiniband/core/verbs.c if (atomic_read(&qp->usecnt)) qp 1872 drivers/infiniband/core/verbs.c if (qp->real_qp != qp) qp 1873 drivers/infiniband/core/verbs.c return __ib_destroy_shared_qp(qp); qp 1875 drivers/infiniband/core/verbs.c pd = qp->pd; qp 1876 drivers/infiniband/core/verbs.c scq = qp->send_cq; qp 1877 drivers/infiniband/core/verbs.c rcq = qp->recv_cq; qp 1878 drivers/infiniband/core/verbs.c srq = qp->srq; qp 1879 drivers/infiniband/core/verbs.c ind_tbl = qp->rwq_ind_tbl; qp 1880 drivers/infiniband/core/verbs.c sec = qp->qp_sec; qp 1884 drivers/infiniband/core/verbs.c if (!qp->uobject) qp 1885 drivers/infiniband/core/verbs.c rdma_rw_cleanup_mrs(qp); qp 1887 drivers/infiniband/core/verbs.c rdma_counter_unbind_qp(qp, true); qp 1888 drivers/infiniband/core/verbs.c rdma_restrack_del(&qp->res); qp 1889 drivers/infiniband/core/verbs.c ret = qp->device->ops.destroy_qp(qp, udata); qp 2145 drivers/infiniband/core/verbs.c static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid) qp 2155 drivers/infiniband/core/verbs.c if (!ib_query_qp(qp, &attr, IB_QP_STATE | IB_QP_PORT, &init_attr)) { qp 2157 drivers/infiniband/core/verbs.c if (rdma_port_get_link_layer(qp->device, attr.port_num) != qp 2165 drivers/infiniband/core/verbs.c for (port = 0; port < qp->device->phys_port_cnt; port++) qp 2166 drivers/infiniband/core/verbs.c if (rdma_port_get_link_layer(qp->device, port) != qp 2183 drivers/infiniband/core/verbs.c int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) qp 2187 drivers/infiniband/core/verbs.c if (!qp->device->ops.attach_mcast) qp 2191 drivers/infiniband/core/verbs.c qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid)) qp 2194 drivers/infiniband/core/verbs.c ret = qp->device->ops.attach_mcast(qp, gid, lid); qp 2196 drivers/infiniband/core/verbs.c atomic_inc(&qp->usecnt); qp 2201 drivers/infiniband/core/verbs.c int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) qp 2205 drivers/infiniband/core/verbs.c if (!qp->device->ops.detach_mcast) qp 2209 drivers/infiniband/core/verbs.c qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid)) qp 2212 drivers/infiniband/core/verbs.c ret = qp->device->ops.detach_mcast(qp, gid, lid); qp 2214 drivers/infiniband/core/verbs.c atomic_dec(&qp->usecnt); qp 2241 drivers/infiniband/core/verbs.c struct ib_qp *qp; qp 2248 drivers/infiniband/core/verbs.c qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list); qp 2249 drivers/infiniband/core/verbs.c ret = ib_destroy_qp(qp); qp 2630 drivers/infiniband/core/verbs.c static void __ib_drain_sq(struct ib_qp *qp) qp 2632 drivers/infiniband/core/verbs.c struct ib_cq *cq = qp->send_cq; qp 2644 drivers/infiniband/core/verbs.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE); qp 2653 drivers/infiniband/core/verbs.c ret = ib_post_send(qp, &swr.wr, NULL); qp 2669 drivers/infiniband/core/verbs.c static void __ib_drain_rq(struct ib_qp *qp) qp 2671 drivers/infiniband/core/verbs.c struct ib_cq *cq = qp->recv_cq; qp 2677 drivers/infiniband/core/verbs.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE); qp 2687 drivers/infiniband/core/verbs.c ret = ib_post_recv(qp, &rwr, NULL); qp 2719 drivers/infiniband/core/verbs.c void ib_drain_sq(struct ib_qp *qp) qp 2721 drivers/infiniband/core/verbs.c if (qp->device->ops.drain_sq) qp 2722 drivers/infiniband/core/verbs.c qp->device->ops.drain_sq(qp); qp 2724 drivers/infiniband/core/verbs.c __ib_drain_sq(qp); qp 2747 drivers/infiniband/core/verbs.c void ib_drain_rq(struct ib_qp *qp) qp 2749 drivers/infiniband/core/verbs.c if (qp->device->ops.drain_rq) qp 2750 drivers/infiniband/core/verbs.c qp->device->ops.drain_rq(qp); qp 2752 drivers/infiniband/core/verbs.c __ib_drain_rq(qp); qp 2771 drivers/infiniband/core/verbs.c void ib_drain_qp(struct ib_qp *qp) qp 2773 drivers/infiniband/core/verbs.c ib_drain_sq(qp); qp 2774 drivers/infiniband/core/verbs.c if (!qp->srq) qp 2775 drivers/infiniband/core/verbs.c ib_drain_rq(qp); qp 431 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, qp 433 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct ib_pd *ib_pd = qp->ib_qp.pd; qp 444 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_dbg(rdev_to_dev(qp->rdev), qp 446 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.bind.r_key, qp->qplib_qp.id, pd); qp 447 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); qp 449 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); qp 452 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_post_send_db(&qp->qplib_qp); qp 738 drivers/infiniband/hw/bnxt_re/ib_verbs.c unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) qp 739 drivers/infiniband/hw/bnxt_re/ib_verbs.c __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock) qp 743 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_irqsave(&qp->scq->cq_lock, flags); qp 744 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->rcq != qp->scq) qp 745 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock(&qp->rcq->cq_lock); qp 747 drivers/infiniband/hw/bnxt_re/ib_verbs.c __acquire(&qp->rcq->cq_lock); qp 752 drivers/infiniband/hw/bnxt_re/ib_verbs.c void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, qp 754 drivers/infiniband/hw/bnxt_re/ib_verbs.c __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock) qp 756 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->rcq != qp->scq) qp 757 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_unlock(&qp->rcq->cq_lock); qp 759 drivers/infiniband/hw/bnxt_re/ib_verbs.c __release(&qp->rcq->cq_lock); qp 760 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_unlock_irqrestore(&qp->scq->cq_lock, flags); qp 766 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); qp 767 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_dev *rdev = qp->rdev; qp 771 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_flush_cqn_wq(&qp->qplib_qp); qp 772 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); qp 778 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (rdma_is_kernel_res(&qp->ib_qp.res)) { qp 779 drivers/infiniband/hw/bnxt_re/ib_verbs.c flags = bnxt_re_lock_cqs(qp); qp 780 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_clean_qp(&qp->qplib_qp); qp 781 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_re_unlock_cqs(qp, flags); qp 784 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); qp 790 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_clean_qp(&qp->qplib_qp); qp 811 drivers/infiniband/hw/bnxt_re/ib_verbs.c ib_umem_release(qp->rumem); qp 812 drivers/infiniband/hw/bnxt_re/ib_verbs.c ib_umem_release(qp->sumem); qp 815 drivers/infiniband/hw/bnxt_re/ib_verbs.c list_del(&qp->list); qp 818 drivers/infiniband/hw/bnxt_re/ib_verbs.c kfree(qp); qp 837 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp, struct ib_udata *udata) qp 840 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; qp 862 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->sumem = umem; qp 868 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (!qp->qplib_qp.srq) { qp 875 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->rumem = umem; qp 884 drivers/infiniband/hw/bnxt_re/ib_verbs.c ib_umem_release(qp->sumem); qp 885 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->sumem = NULL; qp 944 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp; qp 947 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 948 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (!qp) qp 951 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->rdev = rdev; qp 954 drivers/infiniband/hw/bnxt_re/ib_verbs.c ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); qp 956 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.pd = &pd->qplib_pd; qp 957 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); qp 958 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.type = IB_QPT_UD; qp 960 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.max_inline_data = 0; qp 961 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sig_type = true; qp 964 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; qp 965 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_sge = 2; qp 967 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.q_full_delta = 1; qp 969 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.scq = qp1_qp->scq; qp 970 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rcq = qp1_qp->rcq; qp 972 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; qp 973 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; qp 975 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.q_full_delta = 1; qp 977 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.mtu = qp1_qp->mtu; qp 979 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq_hdr_buf_size = 0; qp 980 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; qp 981 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.dpi = &rdev->dpi_privileged; qp 983 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); qp 987 drivers/infiniband/hw/bnxt_re/ib_verbs.c rdev->sqp_id = qp->qplib_qp.id; qp 989 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_init(&qp->sq_lock); qp 990 drivers/infiniband/hw/bnxt_re/ib_verbs.c INIT_LIST_HEAD(&qp->list); qp 992 drivers/infiniband/hw/bnxt_re/ib_verbs.c list_add_tail(&qp->list, &rdev->qp_list); qp 995 drivers/infiniband/hw/bnxt_re/ib_verbs.c return qp; qp 997 drivers/infiniband/hw/bnxt_re/ib_verbs.c kfree(qp); qp 1008 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp; qp 1020 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 1021 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (!qp) qp 1024 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->rdev = rdev; qp 1025 drivers/infiniband/hw/bnxt_re/ib_verbs.c ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); qp 1026 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.pd = &pd->qplib_pd; qp 1027 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); qp 1028 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type); qp 1032 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.type = CMDQ_CREATE_QP_TYPE_GSI; qp 1033 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.type == IB_QPT_MAX) { qp 1035 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.type); qp 1040 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data; qp 1041 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type == qp 1044 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge; qp 1045 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) qp 1046 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; qp 1056 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.scq = &cq->qplib_cq; qp 1057 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->scq = cq; qp 1068 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rcq = &cq->qplib_cq; qp 1069 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->rcq = cq; qp 1080 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.srq = &srq->qplib_srq; qp 1081 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_wqe = 0; qp 1087 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_wqe = min_t(u32, entries, qp 1090 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - qp 1093 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge; qp 1094 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) qp 1095 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; qp 1098 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); qp 1104 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_wqe = min_t(u32, entries, qp 1106 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - qp 1108 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; qp 1109 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) qp 1110 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; qp 1111 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_sge++; qp 1112 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) qp 1113 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; qp 1115 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq_hdr_buf_size = qp 1118 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq_hdr_buf_size = qp 1120 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.dpi = &rdev->dpi_privileged; qp 1121 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp); qp 1128 drivers/infiniband/hw/bnxt_re/ib_verbs.c &qp->qplib_qp); qp 1136 drivers/infiniband/hw/bnxt_re/ib_verbs.c &qp->qplib_qp); qp 1150 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_wqe = min_t(u32, entries, qp 1153 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1; qp 1161 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.q_full_delta -= 1; qp 1163 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom; qp 1164 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; qp 1166 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); qp 1170 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.dpi = &rdev->dpi_privileged; qp 1173 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); qp 1180 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->ib_qp.qp_num = qp->qplib_qp.id; qp 1181 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_init(&qp->sq_lock); qp 1182 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_init(&qp->rq_lock); qp 1187 drivers/infiniband/hw/bnxt_re/ib_verbs.c resp.qpid = qp->ib_qp.qp_num; qp 1195 drivers/infiniband/hw/bnxt_re/ib_verbs.c INIT_LIST_HEAD(&qp->list); qp 1197 drivers/infiniband/hw/bnxt_re/ib_verbs.c list_add_tail(&qp->list, &rdev->qp_list); qp 1201 drivers/infiniband/hw/bnxt_re/ib_verbs.c return &qp->ib_qp; qp 1203 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); qp 1205 drivers/infiniband/hw/bnxt_re/ib_verbs.c ib_umem_release(qp->rumem); qp 1206 drivers/infiniband/hw/bnxt_re/ib_verbs.c ib_umem_release(qp->sumem); qp 1208 drivers/infiniband/hw/bnxt_re/ib_verbs.c kfree(qp); qp 1506 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = rdev->qp1_sqp; qp 1510 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; qp 1511 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.state = qp1_qp->qplib_qp.state; qp 1514 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; qp 1515 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; qp 1519 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; qp 1521 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.qkey = 0x81818181; qp 1524 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; qp 1525 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; qp 1528 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); qp 1538 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); qp 1539 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_dev *rdev = qp->rdev; qp 1546 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags = 0; qp 1548 drivers/infiniband/hw/bnxt_re/ib_verbs.c curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); qp 1563 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; qp 1564 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); qp 1566 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (!qp->sumem && qp 1567 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { qp 1570 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp); qp 1571 drivers/infiniband/hw/bnxt_re/ib_verbs.c flags = bnxt_re_lock_cqs(qp); qp 1572 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_add_flush_qp(&qp->qplib_qp); qp 1573 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_re_unlock_cqs(qp, flags); qp 1575 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (!qp->sumem && qp 1576 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { qp 1579 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp); qp 1580 drivers/infiniband/hw/bnxt_re/ib_verbs.c flags = bnxt_re_lock_cqs(qp); qp 1581 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_clean_qp(&qp->qplib_qp); qp 1582 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_re_unlock_cqs(qp, flags); qp 1586 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1588 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.en_sqd_async_notify = true; qp 1591 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; qp 1592 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.access = qp 1595 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; qp 1597 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; qp 1598 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; qp 1601 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; qp 1602 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.pkey_index = qp_attr->pkey_index; qp 1605 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; qp 1606 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.qkey = qp_attr->qkey; qp 1613 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | qp 1620 drivers/infiniband/hw/bnxt_re/ib_verbs.c memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, qp 1621 drivers/infiniband/hw/bnxt_re/ib_verbs.c sizeof(qp->qplib_qp.ah.dgid.data)); qp 1622 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.ah.flow_label = grh->flow_label; qp 1627 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2; qp 1628 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; qp 1629 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.ah.hop_limit = grh->hop_limit; qp 1630 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.ah.traffic_class = grh->traffic_class; qp 1631 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); qp 1632 drivers/infiniband/hw/bnxt_re/ib_verbs.c ether_addr_copy(qp->qplib_qp.ah.dmac, qp 1637 drivers/infiniband/hw/bnxt_re/ib_verbs.c &qp->qplib_qp.smac[0]); qp 1644 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.nw_type = qp 1648 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.nw_type = qp 1652 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.nw_type = qp 1659 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1661 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); qp 1662 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu); qp 1664 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1666 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.path_mtu = qp 1668 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.mtu = qp 1673 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; qp 1674 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.timeout = qp_attr->timeout; qp 1677 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1679 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; qp 1682 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1684 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; qp 1687 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1689 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; qp 1692 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; qp 1693 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.psn = qp_attr->rq_psn; qp 1696 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1699 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, qp 1703 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; qp 1704 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.psn = qp_attr->sq_psn; qp 1716 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1718 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; qp 1721 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1738 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_wqe = min_t(u32, entries, qp 1740 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - qp 1747 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.q_full_delta -= 1; qp 1748 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; qp 1749 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.rq.max_wqe) { qp 1751 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_wqe = qp 1753 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - qp 1755 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; qp 1761 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.modify_flags |= qp 1763 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; qp 1765 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); qp 1771 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); qp 1778 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); qp 1779 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_dev *rdev = qp->rdev; qp 1787 drivers/infiniband/hw/bnxt_re/ib_verbs.c qplib_qp->id = qp->qplib_qp.id; qp 1788 drivers/infiniband/hw/bnxt_re/ib_verbs.c qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; qp 1821 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; qp 1822 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; qp 1823 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; qp 1824 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; qp 1825 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; qp 1835 drivers/infiniband/hw/bnxt_re/ib_verbs.c static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, qp 1857 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); qp 1896 drivers/infiniband/hw/bnxt_re/ib_verbs.c ip_version, is_udp, 0, &qp->qp1_hdr); qp 1899 drivers/infiniband/hw/bnxt_re/ib_verbs.c ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); qp 1900 drivers/infiniband/hw/bnxt_re/ib_verbs.c ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); qp 1905 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); qp 1907 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); qp 1908 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); qp 1912 drivers/infiniband/hw/bnxt_re/ib_verbs.c memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, qp 1914 drivers/infiniband/hw/bnxt_re/ib_verbs.c memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, qp 1916 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; qp 1920 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.ip4.tos = 0; qp 1921 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.ip4.id = 0; qp 1922 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.ip4.frag_off = htons(IP_DF); qp 1923 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; qp 1925 drivers/infiniband/hw/bnxt_re/ib_verbs.c memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); qp 1926 drivers/infiniband/hw/bnxt_re/ib_verbs.c memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); qp 1927 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); qp 1931 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); qp 1932 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.udp.sport = htons(0x8CD1); qp 1933 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.udp.csum = 0; qp 1938 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; qp 1939 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.immediate_present = 1; qp 1941 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; qp 1944 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.solicited_event = 1; qp 1946 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; qp 1949 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); qp 1950 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.destination_qpn = IB_QP1; qp 1951 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.ack_req = 0; qp 1952 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->send_psn++; qp 1953 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->send_psn &= BTH_PSN_MASK; qp 1954 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); qp 1957 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); qp 1958 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qp1_hdr.deth.source_qpn = IB_QP1; qp 1961 drivers/infiniband/hw/bnxt_re/ib_verbs.c buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); qp 1963 drivers/infiniband/hw/bnxt_re/ib_verbs.c ib_ud_header_pack(&qp->qp1_hdr, buf); qp 1997 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!"); qp 2009 drivers/infiniband/hw/bnxt_re/ib_verbs.c static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, qp 2018 drivers/infiniband/hw/bnxt_re/ib_verbs.c rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); qp 2020 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) qp 2031 drivers/infiniband/hw/bnxt_re/ib_verbs.c sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index]; qp 2049 drivers/infiniband/hw/bnxt_re/ib_verbs.c static int is_ud_qp(struct bnxt_re_qp *qp) qp 2051 drivers/infiniband/hw/bnxt_re/ib_verbs.c return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || qp 2052 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); qp 2055 drivers/infiniband/hw/bnxt_re/ib_verbs.c static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, qp 2061 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (is_ud_qp(qp)) { qp 2258 drivers/infiniband/hw/bnxt_re/ib_verbs.c static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) qp 2260 drivers/infiniband/hw/bnxt_re/ib_verbs.c if ((qp->ib_qp.qp_type == IB_QPT_UD || qp 2261 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->ib_qp.qp_type == IB_QPT_GSI || qp 2262 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && qp 2263 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { qp 2269 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); qp 2270 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.wqe_cnt = 0; qp 2275 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp, qp 2282 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_irqsave(&qp->sq_lock, flags); qp 2290 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (wr->num_sge > qp->qplib_qp.sq.max_sge) { qp 2297 drivers/infiniband/hw/bnxt_re/ib_verbs.c payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); qp 2306 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_send_wqe(qp, wr, &wqe); qp 2308 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); qp 2318 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_post_send_db(&qp->qplib_qp); qp 2319 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_ud_qp_hw_stall_workaround(qp); qp 2320 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 2327 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); qp 2332 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_irqsave(&qp->sq_lock, flags); qp 2339 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (wr->num_sge > qp->qplib_qp.sq.max_sge) { qp 2340 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_err(rdev_to_dev(qp->rdev), qp 2346 drivers/infiniband/hw/bnxt_re/ib_verbs.c payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); qp 2356 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) { qp 2357 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, qp 2374 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_send_wqe(qp, wr, &wqe); qp 2386 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_err(rdev_to_dev(qp->rdev), qp 2398 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_err(rdev_to_dev(qp->rdev), qp 2404 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); qp 2407 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_err(rdev_to_dev(qp->rdev), qp 2409 drivers/infiniband/hw/bnxt_re/ib_verbs.c wr->opcode, qp->qplib_qp.state, rc); qp 2415 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_post_send_db(&qp->qplib_qp); qp 2416 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_ud_qp_hw_stall_workaround(qp); qp 2417 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 2423 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp, qp 2436 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (wr->num_sge > qp->qplib_qp.rq.max_sge) { qp 2446 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); qp 2453 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_post_recv_db(&qp->qplib_qp); qp 2460 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); qp 2466 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_irqsave(&qp->rq_lock, flags); qp 2473 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (wr->num_sge > qp->qplib_qp.rq.max_sge) { qp 2474 drivers/infiniband/hw/bnxt_re/ib_verbs.c dev_err(rdev_to_dev(qp->rdev), qp 2487 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI) qp 2488 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, qp 2491 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); qp 2499 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_post_recv_db(&qp->qplib_qp); qp 2507 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_qplib_post_recv_db(&qp->qplib_qp); qp 2509 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_unlock_irqrestore(&qp->rq_lock, flags); qp 2856 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp = rdev->qp1_sqp; qp 2885 drivers/infiniband/hw/bnxt_re/ib_verbs.c shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp, qp 2945 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr); qp 2963 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr); qp 3017 drivers/infiniband/hw/bnxt_re/ib_verbs.c static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp, qp 3021 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_dev *rdev = qp->rdev; qp 3038 drivers/infiniband/hw/bnxt_re/ib_verbs.c wc->qp = &qp1_qp->ib_qp; qp 3063 drivers/infiniband/hw/bnxt_re/ib_verbs.c static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp, qp 3075 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) { qp 3092 drivers/infiniband/hw/bnxt_re/ib_verbs.c static int send_phantom_wqe(struct bnxt_re_qp *qp) qp 3094 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; qp 3098 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_lock_irqsave(&qp->sq_lock, flags); qp 3110 drivers/infiniband/hw/bnxt_re/ib_verbs.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 3117 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_re_qp *qp; qp 3140 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp = container_of(lib_qp, qp 3142 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (send_phantom_wqe(qp) == -ENOMEM) qp 3163 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp = container_of qp 3167 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (!qp) { qp 3172 drivers/infiniband/hw/bnxt_re/ib_verbs.c wc->qp = &qp->ib_qp; qp 3181 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->rdev->qp1_sqp && qp->qplib_qp.id == qp 3182 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->rdev->qp1_sqp->qplib_qp.id) { qp 3196 drivers/infiniband/hw/bnxt_re/ib_verbs.c (qp, cqe); qp 3216 drivers/infiniband/hw/bnxt_re/ib_verbs.c if (qp->rdev->qp1_sqp && qp->qplib_qp.id == qp 3217 drivers/infiniband/hw/bnxt_re/ib_verbs.c qp->rdev->qp1_sqp->qplib_qp.id) { qp 3225 drivers/infiniband/hw/bnxt_re/ib_verbs.c (qp, wc, cqe); qp 3229 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_re_process_res_ud_wc(qp, wc, cqe); qp 184 drivers/infiniband/hw/bnxt_re/ib_verbs.h int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 186 drivers/infiniband/hw/bnxt_re/ib_verbs.h int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 188 drivers/infiniband/hw/bnxt_re/ib_verbs.h int bnxt_re_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); qp 189 drivers/infiniband/hw/bnxt_re/ib_verbs.h int bnxt_re_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr, qp 191 drivers/infiniband/hw/bnxt_re/ib_verbs.h int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, qp 215 drivers/infiniband/hw/bnxt_re/ib_verbs.h unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp); qp 216 drivers/infiniband/hw/bnxt_re/ib_verbs.h void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); qp 787 drivers/infiniband/hw/bnxt_re/main.c struct bnxt_re_qp *qp) qp 792 drivers/infiniband/hw/bnxt_re/main.c if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { qp 793 drivers/infiniband/hw/bnxt_re/main.c flags = bnxt_re_lock_cqs(qp); qp 794 drivers/infiniband/hw/bnxt_re/main.c bnxt_qplib_add_flush_qp(&qp->qplib_qp); qp 795 drivers/infiniband/hw/bnxt_re/main.c bnxt_re_unlock_cqs(qp, flags); qp 799 drivers/infiniband/hw/bnxt_re/main.c if (qp->qplib_qp.srq) { qp 800 drivers/infiniband/hw/bnxt_re/main.c event.device = &qp->rdev->ibdev; qp 801 drivers/infiniband/hw/bnxt_re/main.c event.element.qp = &qp->ib_qp; qp 805 drivers/infiniband/hw/bnxt_re/main.c if (event.device && qp->ib_qp.event_handler) qp 806 drivers/infiniband/hw/bnxt_re/main.c qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); qp 823 drivers/infiniband/hw/bnxt_re/main.c struct bnxt_re_qp *qp = container_of(lib_qp, struct bnxt_re_qp, qp 825 drivers/infiniband/hw/bnxt_re/main.c rc = bnxt_re_handle_qp_async_event(affi_async, qp); qp 1042 drivers/infiniband/hw/bnxt_re/main.c static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, qp 1048 drivers/infiniband/hw/bnxt_re/main.c if (qp) { qp 1049 drivers/infiniband/hw/bnxt_re/main.c ib_event.element.qp = qp; qp 1051 drivers/infiniband/hw/bnxt_re/main.c if (qp->event_handler) qp 1052 drivers/infiniband/hw/bnxt_re/main.c qp->event_handler(&ib_event, qp->qp_context); qp 1106 drivers/infiniband/hw/bnxt_re/main.c struct bnxt_re_qp *qp) qp 1108 drivers/infiniband/hw/bnxt_re/main.c return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp); qp 1115 drivers/infiniband/hw/bnxt_re/main.c struct bnxt_re_qp *qp; qp 1119 drivers/infiniband/hw/bnxt_re/main.c list_for_each_entry(qp, &rdev->qp_list, list) { qp 1121 drivers/infiniband/hw/bnxt_re/main.c if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { qp 1122 drivers/infiniband/hw/bnxt_re/main.c if (qp->qplib_qp.state != qp 1124 drivers/infiniband/hw/bnxt_re/main.c qp->qplib_qp.state != qp 1126 drivers/infiniband/hw/bnxt_re/main.c bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, qp 1128 drivers/infiniband/hw/bnxt_re/main.c bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, qp 57 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); qp 60 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp) qp 62 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.condition = false; qp 63 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.send_phantom = false; qp 64 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.single = false; qp 68 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) qp 72 drivers/infiniband/hw/bnxt_re/qplib_fp.c scq = qp->scq; qp 73 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcq = qp->rcq; qp 75 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp->sq.flushed) { qp 77 drivers/infiniband/hw/bnxt_re/qplib_fp.c "FP: Adding to SQ Flush list = %p\n", qp); qp 78 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_cancel_phantom_processing(qp); qp 79 drivers/infiniband/hw/bnxt_re/qplib_fp.c list_add_tail(&qp->sq_flush, &scq->sqf_head); qp 80 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.flushed = true; qp 82 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp->srq) { qp 83 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp->rq.flushed) { qp 85 drivers/infiniband/hw/bnxt_re/qplib_fp.c "FP: Adding to RQ Flush list = %p\n", qp); qp 86 drivers/infiniband/hw/bnxt_re/qplib_fp.c list_add_tail(&qp->rq_flush, &rcq->rqf_head); qp 87 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq.flushed = true; qp 92 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp, qp 94 drivers/infiniband/hw/bnxt_re/qplib_fp.c __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) qp 96 drivers/infiniband/hw/bnxt_re/qplib_fp.c spin_lock_irqsave(&qp->scq->flush_lock, *flags); qp 97 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->scq == qp->rcq) qp 98 drivers/infiniband/hw/bnxt_re/qplib_fp.c __acquire(&qp->rcq->flush_lock); qp 100 drivers/infiniband/hw/bnxt_re/qplib_fp.c spin_lock(&qp->rcq->flush_lock); qp 103 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp, qp 105 drivers/infiniband/hw/bnxt_re/qplib_fp.c __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) qp 107 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->scq == qp->rcq) qp 108 drivers/infiniband/hw/bnxt_re/qplib_fp.c __release(&qp->rcq->flush_lock); qp 110 drivers/infiniband/hw/bnxt_re/qplib_fp.c spin_unlock(&qp->rcq->flush_lock); qp 111 drivers/infiniband/hw/bnxt_re/qplib_fp.c spin_unlock_irqrestore(&qp->scq->flush_lock, *flags); qp 114 drivers/infiniband/hw/bnxt_re/qplib_fp.c void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) qp 118 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_acquire_cq_flush_locks(qp, &flags); qp 119 drivers/infiniband/hw/bnxt_re/qplib_fp.c __bnxt_qplib_add_flush_qp(qp); qp 120 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_release_cq_flush_locks(qp, &flags); qp 123 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) qp 125 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sq.flushed) { qp 126 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.flushed = false; qp 127 drivers/infiniband/hw/bnxt_re/qplib_fp.c list_del(&qp->sq_flush); qp 129 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp->srq) { qp 130 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq.flushed) { qp 131 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq.flushed = false; qp 132 drivers/infiniband/hw/bnxt_re/qplib_fp.c list_del(&qp->rq_flush); qp 137 drivers/infiniband/hw/bnxt_re/qplib_fp.c void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp) qp 141 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_acquire_cq_flush_locks(qp, &flags); qp 142 drivers/infiniband/hw/bnxt_re/qplib_fp.c __clean_cq(qp->scq, (u64)(unsigned long)qp); qp 143 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.hwq.prod = 0; qp 144 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.hwq.cons = 0; qp 145 drivers/infiniband/hw/bnxt_re/qplib_fp.c __clean_cq(qp->rcq, (u64)(unsigned long)qp); qp 146 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq.hwq.prod = 0; qp 147 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq.hwq.cons = 0; qp 149 drivers/infiniband/hw/bnxt_re/qplib_fp.c __bnxt_qplib_del_flush_qp(qp); qp 150 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_release_cq_flush_locks(qp, &flags); qp 175 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp) qp 177 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 178 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 180 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq_hdr_buf) qp 182 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq->hwq.max_elements * qp->rq_hdr_buf_size, qp 183 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq_hdr_buf, qp->rq_hdr_buf_map); qp 184 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sq_hdr_buf) qp 186 drivers/infiniband/hw/bnxt_re/qplib_fp.c sq->hwq.max_elements * qp->sq_hdr_buf_size, qp 187 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq_hdr_buf, qp->sq_hdr_buf_map); qp 188 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq_hdr_buf = NULL; qp 189 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq_hdr_buf = NULL; qp 190 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq_hdr_buf_map = 0; qp 191 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq_hdr_buf_map = 0; qp 192 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq_hdr_buf_size = 0; qp 193 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq_hdr_buf_size = 0; qp 197 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp) qp 199 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 200 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 203 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { qp 204 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, qp 206 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq_hdr_buf_size, qp 207 drivers/infiniband/hw/bnxt_re/qplib_fp.c &qp->sq_hdr_buf_map, GFP_KERNEL); qp 208 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp->sq_hdr_buf) { qp 216 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { qp 217 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, qp 219 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq_hdr_buf_size, qp 220 drivers/infiniband/hw/bnxt_re/qplib_fp.c &qp->rq_hdr_buf_map, qp 222 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp->rq_hdr_buf) { qp 232 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_qp_hdr_buf(res, qp); qp 722 drivers/infiniband/hw/bnxt_re/qplib_fp.c int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) qp 728 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 729 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 737 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.type = qp->type; qp 738 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.dpi = cpu_to_le32(qp->dpi->dpi); qp 739 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.qp_handle = cpu_to_le64(qp->qp_handle); qp 774 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->scq) qp 775 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.scq_cid = cpu_to_le32(qp->scq->id); qp 781 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq->hwq.max_elements = qp->rq.max_wqe; qp 813 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rcq) qp 814 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.rcq_cid = cpu_to_le32(qp->rcq->id); qp 818 drivers/infiniband/hw/bnxt_re/qplib_fp.c rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); qp 834 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.pd_id = cpu_to_le32(qp->pd->id); qp 841 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->id = le32_to_cpu(resp.xid); qp 842 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; qp 843 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_id = qp->id; qp 844 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; qp 849 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_qp_hdr_buf(res, qp); qp 860 drivers/infiniband/hw/bnxt_re/qplib_fp.c int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) qp 865 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 866 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 880 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.type = qp->type; qp 881 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.dpi = cpu_to_le32(qp->dpi->dpi); qp 882 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.qp_handle = cpu_to_le64(qp->qp_handle); qp 885 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { qp 949 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->scq) qp 950 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.scq_cid = cpu_to_le32(qp->scq->id); qp 954 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sig_type) qp 994 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->srq) { qp 996 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.srq_cid = cpu_to_le32(qp->srq->id); qp 1000 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rcq) qp 1001 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.rcq_cid = cpu_to_le32(qp->rcq->id); qp 1005 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq_hdr_buf = NULL; qp 1006 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq_hdr_buf = NULL; qp 1008 drivers/infiniband/hw/bnxt_re/qplib_fp.c rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); qp 1016 drivers/infiniband/hw/bnxt_re/qplib_fp.c max_ssge = qp->max_inline_data ? 6 : sq->max_sge; qp 1026 drivers/infiniband/hw/bnxt_re/qplib_fp.c xrrq = &qp->orrq; qp 1028 drivers/infiniband/hw/bnxt_re/qplib_fp.c ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); qp 1041 drivers/infiniband/hw/bnxt_re/qplib_fp.c xrrq = &qp->irrq; qp 1043 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->max_dest_rd_atomic); qp 1058 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.pd_id = cpu_to_le32(qp->pd->id); qp 1065 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->id = le32_to_cpu(resp.xid); qp 1066 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; qp 1067 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->cctx = res->cctx; qp 1068 drivers/infiniband/hw/bnxt_re/qplib_fp.c INIT_LIST_HEAD(&qp->sq_flush); qp 1069 drivers/infiniband/hw/bnxt_re/qplib_fp.c INIT_LIST_HEAD(&qp->rq_flush); qp 1070 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_id = qp->id; qp 1071 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; qp 1076 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->irrq.max_elements) qp 1077 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_hwq(res->pdev, &qp->irrq); qp 1079 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->orrq.max_elements) qp 1080 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_hwq(res->pdev, &qp->orrq); qp 1082 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_qp_hdr_buf(res, qp); qp 1093 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) qp 1095 drivers/infiniband/hw/bnxt_re/qplib_fp.c switch (qp->state) { qp 1100 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!(qp->modify_flags & qp 1102 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->modify_flags |= qp 1104 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->path_mtu = qp 1107 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->modify_flags &= qp 1110 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->max_dest_rd_atomic < 1) qp 1111 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->max_dest_rd_atomic = 1; qp 1112 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; qp 1114 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!(qp->modify_flags & qp 1116 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->modify_flags |= qp 1118 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->ah.sgid_index = 0; qp 1126 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) qp 1128 drivers/infiniband/hw/bnxt_re/qplib_fp.c switch (qp->state) { qp 1131 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->max_rd_atomic < 1) qp 1132 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->max_rd_atomic = 1; qp 1139 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->modify_flags &= qp 1158 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void __filter_modify_flags(struct bnxt_qplib_qp *qp) qp 1160 drivers/infiniband/hw/bnxt_re/qplib_fp.c switch (qp->cur_qp_state) { qp 1164 drivers/infiniband/hw/bnxt_re/qplib_fp.c __modify_flags_from_init_state(qp); qp 1167 drivers/infiniband/hw/bnxt_re/qplib_fp.c __modify_flags_from_rtr_state(qp); qp 1182 drivers/infiniband/hw/bnxt_re/qplib_fp.c int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) qp 1195 drivers/infiniband/hw/bnxt_re/qplib_fp.c __filter_modify_flags(qp); qp 1196 drivers/infiniband/hw/bnxt_re/qplib_fp.c bmask = qp->modify_flags; qp 1197 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.modify_mask = cpu_to_le32(qp->modify_flags); qp 1198 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.qp_cid = cpu_to_le32(qp->id); qp 1201 drivers/infiniband/hw/bnxt_re/qplib_fp.c (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | qp 1202 drivers/infiniband/hw/bnxt_re/qplib_fp.c (qp->en_sqd_async_notify ? qp 1205 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; qp 1208 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.access = qp->access; qp 1212 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->pkey_index, &pkey)) qp 1216 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.qkey = cpu_to_le32(qp->qkey); qp 1219 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); qp 1226 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.flow_label = cpu_to_le32(qp->ah.flow_label); qp 1230 drivers/infiniband/hw/bnxt_re/qplib_fp.c [qp->ah.sgid_index]); qp 1233 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.hop_limit = qp->ah.hop_limit; qp 1236 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.traffic_class = qp->ah.traffic_class; qp 1239 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(req.dest_mac, qp->ah.dmac, 6); qp 1242 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.path_mtu = qp->path_mtu; qp 1245 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.timeout = qp->timeout; qp 1248 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.retry_cnt = qp->retry_cnt; qp 1251 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.rnr_retry = qp->rnr_retry; qp 1254 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.min_rnr_timer = qp->min_rnr_timer; qp 1257 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.rq_psn = cpu_to_le32(qp->rq.psn); qp 1260 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.sq_psn = cpu_to_le32(qp->sq.psn); qp 1264 drivers/infiniband/hw/bnxt_re/qplib_fp.c ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); qp 1268 drivers/infiniband/hw/bnxt_re/qplib_fp.c IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); qp 1270 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); qp 1271 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); qp 1272 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.sq_sge = cpu_to_le16(qp->sq.max_sge); qp 1273 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.rq_sge = cpu_to_le16(qp->rq.max_sge); qp 1274 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.max_inline_data = cpu_to_le32(qp->max_inline_data); qp 1276 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.dest_qp_id = cpu_to_le32(qp->dest_qpn); qp 1278 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); qp 1284 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->cur_qp_state = qp->state; qp 1288 drivers/infiniband/hw/bnxt_re/qplib_fp.c int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) qp 1306 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.qp_cid = cpu_to_le32(qp->id); qp 1313 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->state = sb->en_sqd_async_notify_state & qp 1315 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & qp 1318 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->access = sb->access; qp 1319 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->pkey_index = le16_to_cpu(sb->pkey); qp 1320 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->qkey = le32_to_cpu(sb->qkey); qp 1326 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); qp 1328 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->ah.flow_label = le32_to_cpu(sb->flow_label); qp 1330 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->ah.sgid_index = 0; qp 1333 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->ah.sgid_index = i; qp 1340 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->ah.hop_limit = sb->hop_limit; qp 1341 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->ah.traffic_class = sb->traffic_class; qp 1342 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(qp->ah.dmac, sb->dest_mac, 6); qp 1343 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & qp 1346 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & qp 1349 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->timeout = sb->timeout; qp 1350 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->retry_cnt = sb->retry_cnt; qp 1351 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rnr_retry = sb->rnr_retry; qp 1352 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->min_rnr_timer = sb->min_rnr_timer; qp 1353 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq.psn = le32_to_cpu(sb->rq_psn); qp 1354 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); qp 1355 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.psn = le32_to_cpu(sb->sq_psn); qp 1356 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->max_dest_rd_atomic = qp 1358 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.max_wqe = qp->sq.hwq.max_elements; qp 1359 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq.max_wqe = qp->rq.hwq.max_elements; qp 1360 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->sq.max_sge = le16_to_cpu(sb->sq_sge); qp 1361 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->rq.max_sge = le16_to_cpu(sb->rq_sge); qp 1362 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->max_inline_data = le32_to_cpu(sb->max_inline_data); qp 1363 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); qp 1364 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(qp->smac, sb->src_mac, 6); qp 1365 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); qp 1371 drivers/infiniband/hw/bnxt_re/qplib_fp.c static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) qp 1393 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp == le64_to_cpu(cqe->qp_handle)) qp 1403 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp == le64_to_cpu(cqe->qp_handle)) qp 1414 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp) qp 1422 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID; qp 1423 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_handle = NULL; qp 1427 drivers/infiniband/hw/bnxt_re/qplib_fp.c req.qp_cid = cpu_to_le32(qp->id); qp 1431 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_id = qp->id; qp 1432 drivers/infiniband/hw/bnxt_re/qplib_fp.c rcfw->qp_tbl[qp->id].qp_handle = qp; qp 1440 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp) qp 1442 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_qp_hdr_buf(res, qp); qp 1443 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq); qp 1444 drivers/infiniband/hw/bnxt_re/qplib_fp.c kfree(qp->sq.swq); qp 1446 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq); qp 1447 drivers/infiniband/hw/bnxt_re/qplib_fp.c kfree(qp->rq.swq); qp 1449 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->irrq.max_elements) qp 1450 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_hwq(res->pdev, &qp->irrq); qp 1451 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->orrq.max_elements) qp 1452 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_free_hwq(res->pdev, &qp->orrq); qp 1456 drivers/infiniband/hw/bnxt_re/qplib_fp.c void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, qp 1459 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 1464 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sq_hdr_buf) { qp 1466 drivers/infiniband/hw/bnxt_re/qplib_fp.c sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + qp 1467 drivers/infiniband/hw/bnxt_re/qplib_fp.c sw_prod * qp->sq_hdr_buf_size); qp 1469 drivers/infiniband/hw/bnxt_re/qplib_fp.c sge->size = qp->sq_hdr_buf_size; qp 1470 drivers/infiniband/hw/bnxt_re/qplib_fp.c return qp->sq_hdr_buf + sw_prod * sge->size; qp 1475 drivers/infiniband/hw/bnxt_re/qplib_fp.c u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) qp 1477 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 1482 drivers/infiniband/hw/bnxt_re/qplib_fp.c dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) qp 1484 drivers/infiniband/hw/bnxt_re/qplib_fp.c return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); qp 1487 drivers/infiniband/hw/bnxt_re/qplib_fp.c void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, qp 1490 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 1495 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq_hdr_buf) { qp 1497 drivers/infiniband/hw/bnxt_re/qplib_fp.c sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + qp 1498 drivers/infiniband/hw/bnxt_re/qplib_fp.c sw_prod * qp->rq_hdr_buf_size); qp 1500 drivers/infiniband/hw/bnxt_re/qplib_fp.c sge->size = qp->rq_hdr_buf_size; qp 1501 drivers/infiniband/hw/bnxt_re/qplib_fp.c return qp->rq_hdr_buf + sw_prod * sge->size; qp 1506 drivers/infiniband/hw/bnxt_re/qplib_fp.c void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) qp 1508 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 1512 drivers/infiniband/hw/bnxt_re/qplib_fp.c val = (((qp->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | qp 1518 drivers/infiniband/hw/bnxt_re/qplib_fp.c writeq(val, qp->dpi->dbr); qp 1521 drivers/infiniband/hw/bnxt_re/qplib_fp.c int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, qp 1524 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 1535 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { qp 1536 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { qp 1558 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sig_type) qp 1599 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { qp 1629 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->type == CMDQ_CREATE_QP_TYPE_UD || qp 1630 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->type == CMDQ_CREATE_QP_TYPE_GSI) { qp 1642 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->mtu) qp 1643 drivers/infiniband/hw/bnxt_re/qplib_fp.c pkt_num = (data_len + qp->mtu - 1) / qp->mtu; qp 1664 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->mtu) qp 1665 drivers/infiniband/hw/bnxt_re/qplib_fp.c pkt_num = (data_len + qp->mtu - 1) / qp->mtu; qp 1682 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->mtu) qp 1683 drivers/infiniband/hw/bnxt_re/qplib_fp.c pkt_num = (data_len + qp->mtu - 1) / qp->mtu; qp 1763 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) { qp 1783 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sig_type) qp 1788 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->wqe_cnt++; qp 1794 drivers/infiniband/hw/bnxt_re/qplib_fp.c nq_work->cq = qp->scq; qp 1795 drivers/infiniband/hw/bnxt_re/qplib_fp.c nq_work->nq = qp->scq->nq; qp 1797 drivers/infiniband/hw/bnxt_re/qplib_fp.c queue_work(qp->scq->nq->cqn_wq, &nq_work->work); qp 1807 drivers/infiniband/hw/bnxt_re/qplib_fp.c void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) qp 1809 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 1813 drivers/infiniband/hw/bnxt_re/qplib_fp.c val = (((qp->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | qp 1819 drivers/infiniband/hw/bnxt_re/qplib_fp.c writeq(val, qp->dpi->dbr); qp 1822 drivers/infiniband/hw/bnxt_re/qplib_fp.c int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, qp 1825 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *rq = &qp->rq; qp 1833 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { qp 1841 drivers/infiniband/hw/bnxt_re/qplib_fp.c "FP: QP (0x%x) RQ is full!\n", qp->id); qp 1884 drivers/infiniband/hw/bnxt_re/qplib_fp.c nq_work->cq = qp->rcq; qp 1885 drivers/infiniband/hw/bnxt_re/qplib_fp.c nq_work->nq = qp->rcq->nq; qp 1887 drivers/infiniband/hw/bnxt_re/qplib_fp.c queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); qp 2015 drivers/infiniband/hw/bnxt_re/qplib_fp.c static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, qp 2032 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_cancel_phantom_processing(qp); qp 2038 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->qp_handle = (u64)(unsigned long)qp; qp 2040 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->src_qp = qp->id; qp 2055 drivers/infiniband/hw/bnxt_re/qplib_fp.c static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, qp 2063 drivers/infiniband/hw/bnxt_re/qplib_fp.c switch (qp->type) { qp 2087 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->qp_handle = (unsigned long)qp; qp 2103 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp = qp_handle; qp 2105 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp) qp 2109 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qp 2110 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_cancel_phantom_processing(qp); qp 2116 drivers/infiniband/hw/bnxt_re/qplib_fp.c static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, qp 2119 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_q *sq = &qp->sq; qp 2139 drivers/infiniband/hw/bnxt_re/qplib_fp.c cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); qp 2208 drivers/infiniband/hw/bnxt_re/qplib_fp.c cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); qp 2220 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp; qp 2227 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp = (struct bnxt_qplib_qp *)((unsigned long) qp 2229 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp) { qp 2234 drivers/infiniband/hw/bnxt_re/qplib_fp.c sq = &qp->sq; qp 2244 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sq.flushed) { qp 2246 drivers/infiniband/hw/bnxt_re/qplib_fp.c "%s: QP in Flush QP = %p\n", __func__, qp); qp 2263 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->qp_handle = (u64)(unsigned long)qp; qp 2264 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->src_qp = qp->id; qp 2282 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_mark_qp_error(qp); qp 2284 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_add_flush_qp(qp); qp 2287 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, qp 2289 drivers/infiniband/hw/bnxt_re/qplib_fp.c *lib_qp = qp; qp 2334 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp; qp 2341 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp = (struct bnxt_qplib_qp *)((unsigned long) qp 2343 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp) { qp 2347 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq.flushed) { qp 2349 drivers/infiniband/hw/bnxt_re/qplib_fp.c "%s: QP in Flush QP = %p\n", __func__, qp); qp 2360 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->qp_handle = (u64)(unsigned long)qp; qp 2365 drivers/infiniband/hw/bnxt_re/qplib_fp.c srq = qp->srq; qp 2380 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq = &qp->rq; qp 2394 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qp 2396 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_add_flush_qp(qp); qp 2409 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp; qp 2416 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp = (struct bnxt_qplib_qp *)((unsigned long) qp 2418 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp) { qp 2422 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq.flushed) { qp 2424 drivers/infiniband/hw/bnxt_re/qplib_fp.c "%s: QP in Flush QP = %p\n", __func__, qp); qp 2434 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->qp_handle = (u64)(unsigned long)qp; qp 2445 drivers/infiniband/hw/bnxt_re/qplib_fp.c srq = qp->srq; qp 2461 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq = &qp->rq; qp 2476 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qp 2478 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_add_flush_qp(qp); qp 2506 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp; qp 2513 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp = (struct bnxt_qplib_qp *)((unsigned long) qp 2515 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp) { qp 2519 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq.flushed) { qp 2521 drivers/infiniband/hw/bnxt_re/qplib_fp.c "%s: QP in Flush QP = %p\n", __func__, qp); qp 2527 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->qp_handle = (u64)(unsigned long)qp; qp 2532 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->src_qp = qp->id; qp 2533 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->id == 1 && !cqe->length) { qp 2539 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->pkey_index = qp->pkey_index; qp 2540 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(cqe->smac, qp->smac, 6); qp 2547 drivers/infiniband/hw/bnxt_re/qplib_fp.c srq = qp->srq; qp 2565 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq = &qp->rq; qp 2579 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qp 2581 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_add_flush_qp(qp); qp 2594 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp; qp 2606 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp = (struct bnxt_qplib_qp *)((unsigned long) qp 2608 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!qp) { qp 2615 drivers/infiniband/hw/bnxt_re/qplib_fp.c qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qp 2617 drivers/infiniband/hw/bnxt_re/qplib_fp.c sq = &qp->sq; qp 2618 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq = &qp->rq; qp 2631 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->sq.flushed) { qp 2633 drivers/infiniband/hw/bnxt_re/qplib_fp.c "%s: QP in Flush QP = %p\n", __func__, qp); qp 2650 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->qp_handle = (u64)(unsigned long)qp; qp 2651 drivers/infiniband/hw/bnxt_re/qplib_fp.c cqe->src_qp = qp->id; qp 2679 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->rq.flushed) { qp 2681 drivers/infiniband/hw/bnxt_re/qplib_fp.c "%s: QP in Flush QP = %p\n", __func__, qp); qp 2692 drivers/infiniband/hw/bnxt_re/qplib_fp.c bnxt_qplib_add_flush_qp(qp); qp 2717 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_qp *qp = NULL; qp 2722 drivers/infiniband/hw/bnxt_re/qplib_fp.c list_for_each_entry(qp, &cq->sqf_head, sq_flush) { qp 2723 drivers/infiniband/hw/bnxt_re/qplib_fp.c dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp); qp 2724 drivers/infiniband/hw/bnxt_re/qplib_fp.c __flush_sq(&qp->sq, qp, &cqe, &budget); qp 2727 drivers/infiniband/hw/bnxt_re/qplib_fp.c list_for_each_entry(qp, &cq->rqf_head, rq_flush) { qp 2728 drivers/infiniband/hw/bnxt_re/qplib_fp.c dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp); qp 2729 drivers/infiniband/hw/bnxt_re/qplib_fp.c __flush_rq(&qp->rq, qp, &cqe, &budget); qp 2829 drivers/infiniband/hw/bnxt_re/qplib_fp.c void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) qp 2831 drivers/infiniband/hw/bnxt_re/qplib_fp.c flush_workqueue(qp->scq->nq->cqn_wq); qp 2832 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (qp->scq != qp->rcq) qp 2833 drivers/infiniband/hw/bnxt_re/qplib_fp.c flush_workqueue(qp->rcq->nq->cqn_wq); qp 525 drivers/infiniband/hw/bnxt_re/qplib_fp.h int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); qp 526 drivers/infiniband/hw/bnxt_re/qplib_fp.h int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); qp 527 drivers/infiniband/hw/bnxt_re/qplib_fp.h int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); qp 528 drivers/infiniband/hw/bnxt_re/qplib_fp.h int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); qp 529 drivers/infiniband/hw/bnxt_re/qplib_fp.h int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); qp 530 drivers/infiniband/hw/bnxt_re/qplib_fp.h void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp); qp 532 drivers/infiniband/hw/bnxt_re/qplib_fp.h struct bnxt_qplib_qp *qp); qp 533 drivers/infiniband/hw/bnxt_re/qplib_fp.h void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, qp 535 drivers/infiniband/hw/bnxt_re/qplib_fp.h void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, qp 537 drivers/infiniband/hw/bnxt_re/qplib_fp.h u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); qp 538 drivers/infiniband/hw/bnxt_re/qplib_fp.h dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, qp 540 drivers/infiniband/hw/bnxt_re/qplib_fp.h void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); qp 541 drivers/infiniband/hw/bnxt_re/qplib_fp.h int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, qp 543 drivers/infiniband/hw/bnxt_re/qplib_fp.h void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); qp 544 drivers/infiniband/hw/bnxt_re/qplib_fp.h int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, qp 549 drivers/infiniband/hw/bnxt_re/qplib_fp.h int num, struct bnxt_qplib_qp **qp); qp 554 drivers/infiniband/hw/bnxt_re/qplib_fp.h void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); qp 555 drivers/infiniband/hw/bnxt_re/qplib_fp.h void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp, qp 557 drivers/infiniband/hw/bnxt_re/qplib_fp.h void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp, qp 562 drivers/infiniband/hw/bnxt_re/qplib_fp.h void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); qp 299 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c struct bnxt_qplib_qp *qp; qp 309 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c qp = rcfw->qp_tbl[qp_id].qp_handle; qp 316 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c if (!qp) qp 318 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c bnxt_qplib_mark_qp_error(qp); qp 319 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c rcfw->aeq_handler(rcfw, qp_event, qp); qp 677 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp = NULL; qp 709 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp = NULL; qp 738 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp = NULL; qp 921 drivers/infiniband/hw/cxgb3/iwch_cm.c err = iwch_modify_qp(ep->com.qp->rhp, qp 922 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp, mask, &attrs, 1); qp 926 drivers/infiniband/hw/cxgb3/iwch_cm.c if (peer2peer && iwch_rqes_posted(ep->com.qp) == 0) { qp 1473 drivers/infiniband/hw/cxgb3/iwch_cm.c iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 1486 drivers/infiniband/hw/cxgb3/iwch_cm.c if (ep->com.cm_id && ep->com.qp) { qp 1488 drivers/infiniband/hw/cxgb3/iwch_cm.c iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 1581 drivers/infiniband/hw/cxgb3/iwch_cm.c if (ep->com.cm_id && ep->com.qp) { qp 1583 drivers/infiniband/hw/cxgb3/iwch_cm.c ret = iwch_modify_qp(ep->com.qp->rhp, qp 1584 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, qp 1645 drivers/infiniband/hw/cxgb3/iwch_cm.c if ((ep->com.cm_id) && (ep->com.qp)) { qp 1647 drivers/infiniband/hw/cxgb3/iwch_cm.c iwch_modify_qp(ep->com.qp->rhp, qp 1648 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp, qp 1691 drivers/infiniband/hw/cxgb3/iwch_cm.c skb_copy_from_linear_data(skb, ep->com.qp->attr.terminate_buffer, qp 1693 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp->attr.terminate_msg_len = skb->len; qp 1694 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp->attr.is_terminate_local = 0; qp 1712 drivers/infiniband/hw/cxgb3/iwch_cm.c iwch_modify_qp(ep->com.qp->rhp, qp 1713 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, qp 1740 drivers/infiniband/hw/cxgb3/iwch_cm.c if (ep->com.cm_id && ep->com.qp) { qp 1742 drivers/infiniband/hw/cxgb3/iwch_cm.c iwch_modify_qp(ep->com.qp->rhp, qp 1743 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, qp 1787 drivers/infiniband/hw/cxgb3/iwch_cm.c struct iwch_qp *qp = get_qhp(h, conn_param->qpn); qp 1796 drivers/infiniband/hw/cxgb3/iwch_cm.c BUG_ON(!qp); qp 1798 drivers/infiniband/hw/cxgb3/iwch_cm.c if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) || qp 1799 drivers/infiniband/hw/cxgb3/iwch_cm.c (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) { qp 1807 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp = qp; qp 1831 drivers/infiniband/hw/cxgb3/iwch_cm.c err = iwch_modify_qp(ep->com.qp->rhp, qp 1832 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp, mask, &attrs, 1); qp 1837 drivers/infiniband/hw/cxgb3/iwch_cm.c if (iwch_rqes_posted(qp)) { qp 1856 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp = NULL; qp 1915 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp = get_qhp(h, conn_param->qpn); qp 1916 drivers/infiniband/hw/cxgb3/iwch_cm.c BUG_ON(!ep->com.qp); qp 1918 drivers/infiniband/hw/cxgb3/iwch_cm.c ep->com.qp, cm_id); qp 156 drivers/infiniband/hw/cxgb3/iwch_cm.h struct iwch_qp *qp; qp 60 drivers/infiniband/hw/cxgb3/iwch_cq.c wc->qp = qhp ? &qhp->ibqp : NULL; qp 94 drivers/infiniband/hw/cxgb3/iwch_ev.c event.element.qp = &qhp->ibqp; qp 891 drivers/infiniband/hw/cxgb3/iwch_provider.c void iwch_qp_add_ref(struct ib_qp *qp) qp 893 drivers/infiniband/hw/cxgb3/iwch_provider.c pr_debug("%s ib_qp %p\n", __func__, qp); qp 894 drivers/infiniband/hw/cxgb3/iwch_provider.c atomic_inc(&(to_iwch_qp(qp)->refcnt)); qp 897 drivers/infiniband/hw/cxgb3/iwch_provider.c void iwch_qp_rem_ref(struct ib_qp *qp) qp 899 drivers/infiniband/hw/cxgb3/iwch_provider.c pr_debug("%s ib_qp %p\n", __func__, qp); qp 900 drivers/infiniband/hw/cxgb3/iwch_provider.c if (atomic_dec_and_test(&(to_iwch_qp(qp)->refcnt))) qp 901 drivers/infiniband/hw/cxgb3/iwch_provider.c wake_up(&(to_iwch_qp(qp)->wait)); qp 183 drivers/infiniband/hw/cxgb3/iwch_provider.h void iwch_qp_add_ref(struct ib_qp *qp); qp 184 drivers/infiniband/hw/cxgb3/iwch_provider.h void iwch_qp_rem_ref(struct ib_qp *qp); qp 686 drivers/infiniband/hw/cxgb3/iwch_qp.c return iwch_cxgb3_ofld_send(ep->com.qp->rhp->rdev.t3cdev_p, skb); qp 165 drivers/infiniband/hw/cxgb4/cm.c c4iw_qp_rem_ref(&ep->com.qp->ibqp); qp 174 drivers/infiniband/hw/cxgb4/cm.c c4iw_qp_add_ref(&ep->com.qp->ibqp); qp 704 drivers/infiniband/hw/cxgb4/cm.c if (!ep->com.qp || !ep->com.qp->srq) { qp 1643 drivers/infiniband/hw/cxgb4/cm.c err = c4iw_modify_qp(ep->com.qp->rhp, qp 1644 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp, mask, &attrs, 1); qp 1658 drivers/infiniband/hw/cxgb4/cm.c err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 1677 drivers/infiniband/hw/cxgb4/cm.c err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 1879 drivers/infiniband/hw/cxgb4/cm.c __func__, ep->com.qp->wq.sq.qid, ep, qp 1882 drivers/infiniband/hw/cxgb4/cm.c c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 1910 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.qp->ibqp.uobject) qp 1911 drivers/infiniband/hw/cxgb4/cm.c t4_set_wq_in_error(&ep->com.qp->wq, srqidx); qp 1913 drivers/infiniband/hw/cxgb4/cm.c c4iw_flush_srqidx(ep->com.qp, srqidx); qp 1931 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.qp && ep->com.qp->srq) { qp 2166 drivers/infiniband/hw/cxgb4/cm.c pr_debug("qp %p cm_id %p\n", ep->com.qp, ep->com.cm_id); qp 2756 drivers/infiniband/hw/cxgb4/cm.c ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 2772 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.cm_id && ep->com.qp) { qp 2774 drivers/infiniband/hw/cxgb4/cm.c c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 2800 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.cm_id && ep->com.qp) { qp 2804 drivers/infiniband/hw/cxgb4/cm.c c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 2890 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.qp && ep->com.qp->srq) { qp 2906 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.cm_id && ep->com.qp) { qp 2908 drivers/infiniband/hw/cxgb4/cm.c ret = c4iw_modify_qp(ep->com.qp->rhp, qp 2909 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, qp 2995 drivers/infiniband/hw/cxgb4/cm.c if ((ep->com.cm_id) && (ep->com.qp)) { qp 2997 drivers/infiniband/hw/cxgb4/cm.c c4iw_modify_qp(ep->com.qp->rhp, qp 2998 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp, qp 3030 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.qp) { qp 3032 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp->wq.sq.qid); qp 3034 drivers/infiniband/hw/cxgb4/cm.c c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, qp 3123 drivers/infiniband/hw/cxgb4/cm.c struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); qp 3134 drivers/infiniband/hw/cxgb4/cm.c if (!qp) { qp 3186 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp = qp; qp 3203 drivers/infiniband/hw/cxgb4/cm.c err = c4iw_modify_qp(ep->com.qp->rhp, qp 3204 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp, mask, &attrs, 1); qp 3342 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp = get_qhp(dev, conn_param->qpn); qp 3343 drivers/infiniband/hw/cxgb4/cm.c if (!ep->com.qp) { qp 3350 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp, cm_id); qp 3722 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.qp) { qp 3726 drivers/infiniband/hw/cxgb4/cm.c ret = c4iw_modify_qp(ep->com.qp->rhp, qp 3727 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp, qp 4241 drivers/infiniband/hw/cxgb4/cm.c if (ep->com.cm_id && ep->com.qp) { qp 4243 drivers/infiniband/hw/cxgb4/cm.c c4iw_modify_qp(ep->com.qp->rhp, qp 4244 drivers/infiniband/hw/cxgb4/cm.c ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, qp 770 drivers/infiniband/hw/cxgb4/cq.c wc->qp = qhp ? &qhp->ibqp : NULL; qp 245 drivers/infiniband/hw/cxgb4/device.c static int dump_qp(unsigned long id, struct c4iw_qp *qp, qp 250 drivers/infiniband/hw/cxgb4/device.c if (id != qp->wq.sq.qid) qp 257 drivers/infiniband/hw/cxgb4/device.c if (qp->ep) { qp 258 drivers/infiniband/hw/cxgb4/device.c struct c4iw_ep *ep = qp->ep; qp 271 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.qid, qp->srq ? "srq" : "rq", qp 272 drivers/infiniband/hw/cxgb4/device.c qp->srq ? qp->srq->idx : qp->wq.rq.qid, qp 273 drivers/infiniband/hw/cxgb4/device.c (int)qp->attr.state, qp 274 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.flags & T4_SQ_ONCHIP, qp 292 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.qid, qp->wq.rq.qid, qp 293 drivers/infiniband/hw/cxgb4/device.c (int)qp->attr.state, qp 294 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.flags & T4_SQ_ONCHIP, qp 306 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.qid, qp->wq.rq.qid, qp 307 drivers/infiniband/hw/cxgb4/device.c (int)qp->attr.state, qp 308 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.flags & T4_SQ_ONCHIP); qp 328 drivers/infiniband/hw/cxgb4/device.c struct c4iw_qp *qp; qp 344 drivers/infiniband/hw/cxgb4/device.c xa_for_each(&qpd->devp->qps, index, qp) qp 355 drivers/infiniband/hw/cxgb4/device.c xa_for_each(&qpd->devp->qps, index, qp) qp 356 drivers/infiniband/hw/cxgb4/device.c dump_qp(index, qp, qpd); qp 579 drivers/infiniband/hw/cxgb4/device.c ep, ep->com.cm_id, ep->com.qp, qp 600 drivers/infiniband/hw/cxgb4/device.c ep, ep->com.cm_id, ep->com.qp, qp 802 drivers/infiniband/hw/cxgb4/device.c if (rdev->lldi.vr->qp.start != rdev->lldi.vr->cq.start || qp 803 drivers/infiniband/hw/cxgb4/device.c rdev->lldi.vr->qp.size != rdev->lldi.vr->cq.size) { qp 805 drivers/infiniband/hw/cxgb4/device.c pci_name(rdev->lldi.pdev), rdev->lldi.vr->qp.start, qp 806 drivers/infiniband/hw/cxgb4/device.c rdev->lldi.vr->qp.size, rdev->lldi.vr->cq.size, qp 829 drivers/infiniband/hw/cxgb4/device.c rdev->lldi.vr->qp.start, qp 830 drivers/infiniband/hw/cxgb4/device.c rdev->lldi.vr->qp.size, qp 848 drivers/infiniband/hw/cxgb4/device.c rdev->stats.qid.total = rdev->lldi.vr->qp.size; qp 877 drivers/infiniband/hw/cxgb4/device.c rdev->status_page->qp_start = rdev->lldi.vr->qp.start; qp 878 drivers/infiniband/hw/cxgb4/device.c rdev->status_page->qp_size = rdev->lldi.vr->qp.size; qp 963 drivers/infiniband/hw/cxgb4/device.c infop->vr->rq.size > 0 && infop->vr->qp.size > 0 && qp 1269 drivers/infiniband/hw/cxgb4/device.c struct c4iw_qp *qp; qp 1276 drivers/infiniband/hw/cxgb4/device.c xa_for_each(&ctx->dev->qps, index, qp) qp 1277 drivers/infiniband/hw/cxgb4/device.c t4_disable_wq_db(&qp->wq); qp 1284 drivers/infiniband/hw/cxgb4/device.c static void resume_rc_qp(struct c4iw_qp *qp) qp 1286 drivers/infiniband/hw/cxgb4/device.c spin_lock(&qp->lock); qp 1287 drivers/infiniband/hw/cxgb4/device.c t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc, NULL); qp 1288 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.wq_pidx_inc = 0; qp 1289 drivers/infiniband/hw/cxgb4/device.c t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc, NULL); qp 1290 drivers/infiniband/hw/cxgb4/device.c qp->wq.rq.wq_pidx_inc = 0; qp 1291 drivers/infiniband/hw/cxgb4/device.c spin_unlock(&qp->lock); qp 1297 drivers/infiniband/hw/cxgb4/device.c struct c4iw_qp *qp; qp 1300 drivers/infiniband/hw/cxgb4/device.c qp = list_first_entry(&ctx->dev->db_fc_list, struct c4iw_qp, qp 1302 drivers/infiniband/hw/cxgb4/device.c list_del_init(&qp->db_fc_entry); qp 1303 drivers/infiniband/hw/cxgb4/device.c resume_rc_qp(qp); qp 1317 drivers/infiniband/hw/cxgb4/device.c struct c4iw_qp *qp; qp 1324 drivers/infiniband/hw/cxgb4/device.c xa_for_each(&ctx->dev->qps, index, qp) qp 1325 drivers/infiniband/hw/cxgb4/device.c t4_enable_wq_db(&qp->wq); qp 1373 drivers/infiniband/hw/cxgb4/device.c struct c4iw_qp *qp = qp_list->qps[idx]; qp 1375 drivers/infiniband/hw/cxgb4/device.c xa_lock_irq(&qp->rhp->qps); qp 1376 drivers/infiniband/hw/cxgb4/device.c spin_lock(&qp->lock); qp 1377 drivers/infiniband/hw/cxgb4/device.c ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], qp 1378 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.qid, qp 1379 drivers/infiniband/hw/cxgb4/device.c t4_sq_host_wq_pidx(&qp->wq), qp 1380 drivers/infiniband/hw/cxgb4/device.c t4_sq_wq_size(&qp->wq)); qp 1383 drivers/infiniband/hw/cxgb4/device.c pci_name(ctx->lldi.pdev), qp->wq.sq.qid); qp 1384 drivers/infiniband/hw/cxgb4/device.c spin_unlock(&qp->lock); qp 1385 drivers/infiniband/hw/cxgb4/device.c xa_unlock_irq(&qp->rhp->qps); qp 1388 drivers/infiniband/hw/cxgb4/device.c qp->wq.sq.wq_pidx_inc = 0; qp 1390 drivers/infiniband/hw/cxgb4/device.c ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], qp 1391 drivers/infiniband/hw/cxgb4/device.c qp->wq.rq.qid, qp 1392 drivers/infiniband/hw/cxgb4/device.c t4_rq_host_wq_pidx(&qp->wq), qp 1393 drivers/infiniband/hw/cxgb4/device.c t4_rq_wq_size(&qp->wq)); qp 1397 drivers/infiniband/hw/cxgb4/device.c pci_name(ctx->lldi.pdev), qp->wq.rq.qid); qp 1398 drivers/infiniband/hw/cxgb4/device.c spin_unlock(&qp->lock); qp 1399 drivers/infiniband/hw/cxgb4/device.c xa_unlock_irq(&qp->rhp->qps); qp 1402 drivers/infiniband/hw/cxgb4/device.c qp->wq.rq.wq_pidx_inc = 0; qp 1403 drivers/infiniband/hw/cxgb4/device.c spin_unlock(&qp->lock); qp 1404 drivers/infiniband/hw/cxgb4/device.c xa_unlock_irq(&qp->rhp->qps); qp 1407 drivers/infiniband/hw/cxgb4/device.c while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) { qp 1416 drivers/infiniband/hw/cxgb4/device.c struct c4iw_qp *qp; qp 1438 drivers/infiniband/hw/cxgb4/device.c xa_for_each(&ctx->dev->qps, index, qp) qp 1449 drivers/infiniband/hw/cxgb4/device.c xa_for_each(&ctx->dev->qps, index, qp) { qp 1450 drivers/infiniband/hw/cxgb4/device.c c4iw_qp_add_ref(&qp->ibqp); qp 1451 drivers/infiniband/hw/cxgb4/device.c qp_list.qps[qp_list.idx++] = qp; qp 109 drivers/infiniband/hw/cxgb4/ev.c event.element.qp = &qhp->ibqp; qp 855 drivers/infiniband/hw/cxgb4/iw_cxgb4.h struct c4iw_qp *qp; qp 980 drivers/infiniband/hw/cxgb4/iw_cxgb4.h void c4iw_qp_add_ref(struct ib_qp *qp); qp 981 drivers/infiniband/hw/cxgb4/iw_cxgb4.h void c4iw_qp_rem_ref(struct ib_qp *qp); qp 282 drivers/infiniband/hw/cxgb4/provider.c props->max_qp = dev->rdev.lldi.vr->qp.size / 2; qp 294 drivers/infiniband/hw/cxgb4/provider.c props->max_cq = dev->rdev.lldi.vr->qp.size; qp 892 drivers/infiniband/hw/cxgb4/qp.c void c4iw_qp_add_ref(struct ib_qp *qp) qp 894 drivers/infiniband/hw/cxgb4/qp.c pr_debug("ib_qp %p\n", qp); qp 895 drivers/infiniband/hw/cxgb4/qp.c refcount_inc(&to_c4iw_qp(qp)->qp_refcnt); qp 898 drivers/infiniband/hw/cxgb4/qp.c void c4iw_qp_rem_ref(struct ib_qp *qp) qp 900 drivers/infiniband/hw/cxgb4/qp.c pr_debug("ib_qp %p\n", qp); qp 901 drivers/infiniband/hw/cxgb4/qp.c if (refcount_dec_and_test(&to_c4iw_qp(qp)->qp_refcnt)) qp 902 drivers/infiniband/hw/cxgb4/qp.c complete(&to_c4iw_qp(qp)->qp_rel_comp); qp 43 drivers/infiniband/hw/cxgb4/resource.c rdev->lldi.vr->qp.start, qp 44 drivers/infiniband/hw/cxgb4/resource.c rdev->lldi.vr->qp.size, qp 45 drivers/infiniband/hw/cxgb4/resource.c rdev->lldi.vr->qp.size, 0)) qp 48 drivers/infiniband/hw/cxgb4/resource.c for (i = rdev->lldi.vr->qp.start; qp 49 drivers/infiniband/hw/cxgb4/resource.c i < rdev->lldi.vr->qp.start + rdev->lldi.vr->qp.size; i++) qp 356 drivers/infiniband/hw/efa/efa_verbs.c struct efa_qp *qp = to_eqp(ibqp); qp 373 drivers/infiniband/hw/efa/efa_verbs.c params.qp_handle = qp->qp_handle; qp 384 drivers/infiniband/hw/efa/efa_verbs.c qp_attr->cap.max_send_wr = qp->max_send_wr; qp 385 drivers/infiniband/hw/efa/efa_verbs.c qp_attr->cap.max_recv_wr = qp->max_recv_wr; qp 386 drivers/infiniband/hw/efa/efa_verbs.c qp_attr->cap.max_send_sge = qp->max_send_sge; qp 387 drivers/infiniband/hw/efa/efa_verbs.c qp_attr->cap.max_recv_sge = qp->max_recv_sge; qp 388 drivers/infiniband/hw/efa/efa_verbs.c qp_attr->cap.max_inline_data = qp->max_inline_data; qp 491 drivers/infiniband/hw/efa/efa_verbs.c struct efa_qp *qp = to_eqp(ibqp); qp 495 drivers/infiniband/hw/efa/efa_verbs.c err = efa_destroy_qp_handle(dev, qp->qp_handle); qp 499 drivers/infiniband/hw/efa/efa_verbs.c if (qp->rq_cpu_addr) { qp 502 drivers/infiniband/hw/efa/efa_verbs.c qp->rq_cpu_addr, qp->rq_size, qp 503 drivers/infiniband/hw/efa/efa_verbs.c &qp->rq_dma_addr); qp 504 drivers/infiniband/hw/efa/efa_verbs.c dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr, qp->rq_size, qp 508 drivers/infiniband/hw/efa/efa_verbs.c kfree(qp); qp 512 drivers/infiniband/hw/efa/efa_verbs.c static int qp_mmap_entries_setup(struct efa_qp *qp, qp 523 drivers/infiniband/hw/efa/efa_verbs.c mmap_entry_insert(dev, ucontext, qp, qp 532 drivers/infiniband/hw/efa/efa_verbs.c mmap_entry_insert(dev, ucontext, qp, qp 542 drivers/infiniband/hw/efa/efa_verbs.c if (qp->rq_size) { qp 544 drivers/infiniband/hw/efa/efa_verbs.c mmap_entry_insert(dev, ucontext, qp, qp 553 drivers/infiniband/hw/efa/efa_verbs.c mmap_entry_insert(dev, ucontext, qp, qp 554 drivers/infiniband/hw/efa/efa_verbs.c virt_to_phys(qp->rq_cpu_addr), qp 555 drivers/infiniband/hw/efa/efa_verbs.c qp->rq_size, EFA_MMAP_DMA_PAGE); qp 559 drivers/infiniband/hw/efa/efa_verbs.c resp->rq_mmap_size = qp->rq_size; qp 639 drivers/infiniband/hw/efa/efa_verbs.c struct efa_qp *qp; qp 684 drivers/infiniband/hw/efa/efa_verbs.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 685 drivers/infiniband/hw/efa/efa_verbs.c if (!qp) { qp 714 drivers/infiniband/hw/efa/efa_verbs.c qp->rq_size = PAGE_ALIGN(create_qp_params.rq_ring_size_in_bytes); qp 715 drivers/infiniband/hw/efa/efa_verbs.c if (qp->rq_size) { qp 716 drivers/infiniband/hw/efa/efa_verbs.c qp->rq_cpu_addr = efa_zalloc_mapped(dev, &qp->rq_dma_addr, qp 717 drivers/infiniband/hw/efa/efa_verbs.c qp->rq_size, DMA_TO_DEVICE); qp 718 drivers/infiniband/hw/efa/efa_verbs.c if (!qp->rq_cpu_addr) { qp 725 drivers/infiniband/hw/efa/efa_verbs.c qp->rq_cpu_addr, qp->rq_size, &qp->rq_dma_addr); qp 726 drivers/infiniband/hw/efa/efa_verbs.c create_qp_params.rq_base_addr = qp->rq_dma_addr; qp 740 drivers/infiniband/hw/efa/efa_verbs.c err = qp_mmap_entries_setup(qp, dev, ucontext, &create_qp_params, qp 746 drivers/infiniband/hw/efa/efa_verbs.c qp->qp_handle = create_qp_resp.qp_handle; qp 747 drivers/infiniband/hw/efa/efa_verbs.c qp->ibqp.qp_num = create_qp_resp.qp_num; qp 748 drivers/infiniband/hw/efa/efa_verbs.c qp->ibqp.qp_type = init_attr->qp_type; qp 749 drivers/infiniband/hw/efa/efa_verbs.c qp->max_send_wr = init_attr->cap.max_send_wr; qp 750 drivers/infiniband/hw/efa/efa_verbs.c qp->max_recv_wr = init_attr->cap.max_recv_wr; qp 751 drivers/infiniband/hw/efa/efa_verbs.c qp->max_send_sge = init_attr->cap.max_send_sge; qp 752 drivers/infiniband/hw/efa/efa_verbs.c qp->max_recv_sge = init_attr->cap.max_recv_sge; qp 753 drivers/infiniband/hw/efa/efa_verbs.c qp->max_inline_data = init_attr->cap.max_inline_data; qp 766 drivers/infiniband/hw/efa/efa_verbs.c ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num); qp 768 drivers/infiniband/hw/efa/efa_verbs.c return &qp->ibqp; qp 773 drivers/infiniband/hw/efa/efa_verbs.c if (qp->rq_size) { qp 774 drivers/infiniband/hw/efa/efa_verbs.c dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr, qp->rq_size, qp 777 drivers/infiniband/hw/efa/efa_verbs.c free_pages_exact(qp->rq_cpu_addr, qp->rq_size); qp 780 drivers/infiniband/hw/efa/efa_verbs.c kfree(qp); qp 786 drivers/infiniband/hw/efa/efa_verbs.c static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp, qp 826 drivers/infiniband/hw/efa/efa_verbs.c struct efa_qp *qp = to_eqp(ibqp); qp 839 drivers/infiniband/hw/efa/efa_verbs.c qp->state; qp 842 drivers/infiniband/hw/efa/efa_verbs.c err = efa_modify_qp_validate(dev, qp, qp_attr, qp_attr_mask, cur_state, qp 847 drivers/infiniband/hw/efa/efa_verbs.c params.qp_handle = qp->qp_handle; qp 876 drivers/infiniband/hw/efa/efa_verbs.c qp->state = new_state; qp 309 drivers/infiniband/hw/hfi1/driver.c struct rvt_qp *qp; qp 313 drivers/infiniband/hw/hfi1/driver.c qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); qp 314 drivers/infiniband/hw/hfi1/driver.c if (!qp) { qp 323 drivers/infiniband/hw/hfi1/driver.c spin_lock_irqsave(&qp->r_lock, flags); qp 326 drivers/infiniband/hw/hfi1/driver.c if (!(ib_rvt_state_ops[qp->state] & qp 331 drivers/infiniband/hw/hfi1/driver.c switch (qp->ibqp.qp_type) { qp 333 drivers/infiniband/hw/hfi1/driver.c hfi1_rc_hdrerr(rcd, packet, qp); qp 340 drivers/infiniband/hw/hfi1/driver.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 364 drivers/infiniband/hw/hfi1/driver.c struct rvt_qp *qp = NULL; qp 374 drivers/infiniband/hw/hfi1/driver.c qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn); qp 375 drivers/infiniband/hw/hfi1/driver.c if (!qp) { qp 380 drivers/infiniband/hw/hfi1/driver.c switch (qp->ibqp.qp_type) { qp 388 drivers/infiniband/hw/hfi1/driver.c rqpn = qp->remote_qpn; qp 444 drivers/infiniband/hw/hfi1/driver.c bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, qp 447 drivers/infiniband/hw/hfi1/driver.c struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 472 drivers/infiniband/hw/hfi1/driver.c dlid = qp->ibqp.qp_type != IB_QPT_UD ? ib_get_dlid(pkt->hdr) : qp 483 drivers/infiniband/hw/hfi1/driver.c switch (qp->ibqp.qp_type) { qp 496 drivers/infiniband/hw/hfi1/driver.c rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); qp 497 drivers/infiniband/hw/hfi1/driver.c rqpn = qp->remote_qpn; qp 501 drivers/infiniband/hw/hfi1/driver.c rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); qp 502 drivers/infiniband/hw/hfi1/driver.c rqpn = qp->remote_qpn; qp 523 drivers/infiniband/hw/hfi1/driver.c hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey, qp 626 drivers/infiniband/hw/hfi1/driver.c struct rvt_qp *qp; qp 662 drivers/infiniband/hw/hfi1/driver.c qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn); qp 664 drivers/infiniband/hw/hfi1/driver.c if (!qp) { qp 669 drivers/infiniband/hw/hfi1/driver.c hfi1_process_ecn_slowpath(qp, packet, true); qp 682 drivers/infiniband/hw/hfi1/driver.c struct rvt_qp *qp, *nqp; qp 689 drivers/infiniband/hw/hfi1/driver.c list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { qp 690 drivers/infiniband/hw/hfi1/driver.c list_del_init(&qp->rspwait); qp 691 drivers/infiniband/hw/hfi1/driver.c if (qp->r_flags & RVT_R_RSP_NAK) { qp 692 drivers/infiniband/hw/hfi1/driver.c qp->r_flags &= ~RVT_R_RSP_NAK; qp 693 drivers/infiniband/hw/hfi1/driver.c packet->qp = qp; qp 696 drivers/infiniband/hw/hfi1/driver.c if (qp->r_flags & RVT_R_RSP_SEND) { qp 699 drivers/infiniband/hw/hfi1/driver.c qp->r_flags &= ~RVT_R_RSP_SEND; qp 700 drivers/infiniband/hw/hfi1/driver.c spin_lock_irqsave(&qp->s_lock, flags); qp 701 drivers/infiniband/hw/hfi1/driver.c if (ib_rvt_state_ops[qp->state] & qp 703 drivers/infiniband/hw/hfi1/driver.c hfi1_schedule_send(qp); qp 704 drivers/infiniband/hw/hfi1/driver.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 706 drivers/infiniband/hw/hfi1/driver.c rvt_put_qp(qp); qp 349 drivers/infiniband/hw/hfi1/fault.c bool hfi1_dbg_should_fault_tx(struct rvt_qp *qp, u32 opcode) qp 351 drivers/infiniband/hw/hfi1/fault.c struct hfi1_ibdev *ibd = to_idev(qp->ibqp.device); qp 354 drivers/infiniband/hw/hfi1/fault.c trace_hfi1_fault_opcode(qp, opcode); qp 77 drivers/infiniband/hw/hfi1/fault.h bool hfi1_dbg_should_fault_tx(struct rvt_qp *qp, u32 opcode); qp 94 drivers/infiniband/hw/hfi1/fault.h static inline bool hfi1_dbg_should_fault_tx(struct rvt_qp *qp, qp 371 drivers/infiniband/hw/hfi1/hfi.h struct rvt_qp *qp; qp 978 drivers/infiniband/hw/hfi1/hfi.h typedef void (*hfi1_make_req)(struct rvt_qp *qp, qp 1502 drivers/infiniband/hw/hfi1/hfi.h void hfi1_make_ud_req_9B(struct rvt_qp *qp, qp 1506 drivers/infiniband/hw/hfi1/hfi.h void hfi1_make_ud_req_16B(struct rvt_qp *qp, qp 1617 drivers/infiniband/hw/hfi1/hfi.h void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn, qp 1620 drivers/infiniband/hw/hfi1/hfi.h void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp, qp 1623 drivers/infiniband/hw/hfi1/hfi.h typedef void (*hfi1_handle_cnp)(struct hfi1_ibport *ibp, struct rvt_qp *qp, qp 1886 drivers/infiniband/hw/hfi1/hfi.h bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, qp 1888 drivers/infiniband/hw/hfi1/hfi.h static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt) qp 1894 drivers/infiniband/hw/hfi1/hfi.h return hfi1_process_ecn_slowpath(qp, pkt, false); qp 286 drivers/infiniband/hw/hfi1/mad.c qp0 = rcu_dereference(ibp->rvp.qp[0]); qp 306 drivers/infiniband/hw/hfi1/mad.c qp0 = rcu_dereference(ibp->rvp.qp[0]); qp 17 drivers/infiniband/hw/hfi1/opfn.c bool (*request)(struct rvt_qp *qp, u64 *data); qp 18 drivers/infiniband/hw/hfi1/opfn.c bool (*response)(struct rvt_qp *qp, u64 *data); qp 19 drivers/infiniband/hw/hfi1/opfn.c bool (*reply)(struct rvt_qp *qp, u64 data); qp 20 drivers/infiniband/hw/hfi1/opfn.c void (*error)(struct rvt_qp *qp); qp 34 drivers/infiniband/hw/hfi1/opfn.c static void opfn_schedule_conn_request(struct rvt_qp *qp); qp 41 drivers/infiniband/hw/hfi1/opfn.c static void opfn_conn_request(struct rvt_qp *qp) qp 43 drivers/infiniband/hw/hfi1/opfn.c struct hfi1_qp_priv *priv = qp->priv; qp 51 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_conn_request(qp); qp 70 drivers/infiniband/hw/hfi1/opfn.c if (!extd || !extd->request || !extd->request(qp, &data)) { qp 80 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_data_conn_request(qp, capcode, data); qp 92 drivers/infiniband/hw/hfi1/opfn.c ret = ib_post_send(&qp->ibqp, &wr.wr, NULL); qp 95 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_conn_request(qp); qp 98 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_msg_opfn_conn_request(qp, "ib_ost_send failed: ret = ", qp 106 drivers/infiniband/hw/hfi1/opfn.c opfn_schedule_conn_request(qp); qp 127 drivers/infiniband/hw/hfi1/opfn.c static void opfn_schedule_conn_request(struct rvt_qp *qp) qp 129 drivers/infiniband/hw/hfi1/opfn.c struct hfi1_qp_priv *priv = qp->priv; qp 131 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_sched_conn_request(qp); qp 135 drivers/infiniband/hw/hfi1/opfn.c void opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 138 drivers/infiniband/hw/hfi1/opfn.c struct hfi1_qp_priv *priv = qp->priv; qp 144 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_conn_response(qp); qp 146 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_data_conn_response(qp, capcode, data); qp 165 drivers/infiniband/hw/hfi1/opfn.c extd->error(qp); qp 168 drivers/infiniband/hw/hfi1/opfn.c if (extd->response(qp, &data)) qp 171 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_conn_response(qp); qp 175 drivers/infiniband/hw/hfi1/opfn.c void opfn_conn_reply(struct rvt_qp *qp, u64 data) qp 177 drivers/infiniband/hw/hfi1/opfn.c struct hfi1_qp_priv *priv = qp->priv; qp 182 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_conn_reply(qp); qp 184 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_data_conn_reply(qp, capcode, data); qp 201 drivers/infiniband/hw/hfi1/opfn.c if (extd->reply(qp, data)) qp 209 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_conn_reply(qp); qp 214 drivers/infiniband/hw/hfi1/opfn.c void opfn_conn_error(struct rvt_qp *qp) qp 216 drivers/infiniband/hw/hfi1/opfn.c struct hfi1_qp_priv *priv = qp->priv; qp 221 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_opfn_state_conn_error(qp); qp 222 drivers/infiniband/hw/hfi1/opfn.c trace_hfi1_msg_opfn_conn_error(qp, "error. qp state ", (u64)qp->state); qp 233 drivers/infiniband/hw/hfi1/opfn.c extd->error(qp); qp 242 drivers/infiniband/hw/hfi1/opfn.c void opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask) qp 244 drivers/infiniband/hw/hfi1/opfn.c struct ib_qp *ibqp = &qp->ibqp; qp 245 drivers/infiniband/hw/hfi1/opfn.c struct hfi1_qp_priv *priv = qp->priv; qp 256 drivers/infiniband/hw/hfi1/opfn.c priv->tid_retry_timeout_jiffies = qp->timeout_jiffies; qp 257 drivers/infiniband/hw/hfi1/opfn.c if (qp->pmtu == enum_to_mtu(OPA_MTU_4096) || qp 258 drivers/infiniband/hw/hfi1/opfn.c qp->pmtu == enum_to_mtu(OPA_MTU_8192)) { qp 259 drivers/infiniband/hw/hfi1/opfn.c tid_rdma_opfn_init(qp, local); qp 283 drivers/infiniband/hw/hfi1/opfn.c opfn_schedule_conn_request(qp); qp 293 drivers/infiniband/hw/hfi1/opfn.c void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1) qp 295 drivers/infiniband/hw/hfi1/opfn.c struct hfi1_qp_priv *priv = qp->priv; qp 300 drivers/infiniband/hw/hfi1/opfn.c if (qp->state == IB_QPS_RTS) qp 301 drivers/infiniband/hw/hfi1/opfn.c opfn_conn_request(qp); qp 78 drivers/infiniband/hw/hfi1/opfn.h void opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 80 drivers/infiniband/hw/hfi1/opfn.h void opfn_conn_reply(struct rvt_qp *qp, u64 data); qp 81 drivers/infiniband/hw/hfi1/opfn.h void opfn_conn_error(struct rvt_qp *qp); qp 82 drivers/infiniband/hw/hfi1/opfn.h void opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask); qp 83 drivers/infiniband/hw/hfi1/opfn.h void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1); qp 959 drivers/infiniband/hw/hfi1/pio.c struct rvt_qp *qp; qp 963 drivers/infiniband/hw/hfi1/pio.c qp = iowait_to_qp(wait); qp 964 drivers/infiniband/hw/hfi1/pio.c priv = qp->priv; qp 967 drivers/infiniband/hw/hfi1/pio.c hfi1_qp_wakeup(qp, RVT_S_WAIT_PIO | HFI1_S_WAIT_PIO_DRAIN); qp 1614 drivers/infiniband/hw/hfi1/pio.c struct rvt_qp *qp; qp 1637 drivers/infiniband/hw/hfi1/pio.c qp = iowait_to_qp(wait); qp 1638 drivers/infiniband/hw/hfi1/pio.c priv = qp->priv; qp 1649 drivers/infiniband/hw/hfi1/pio.c qps[n++] = qp; qp 66 drivers/infiniband/hw/hfi1/qp.c static void flush_tx_list(struct rvt_qp *qp); qp 75 drivers/infiniband/hw/hfi1/qp.c static void qp_pio_drain(struct rvt_qp *qp); qp 164 drivers/infiniband/hw/hfi1/qp.c static void flush_tx_list(struct rvt_qp *qp) qp 166 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 172 drivers/infiniband/hw/hfi1/qp.c static void flush_iowait(struct rvt_qp *qp) qp 174 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 184 drivers/infiniband/hw/hfi1/qp.c rvt_put_qp(qp); qp 216 drivers/infiniband/hw/hfi1/qp.c int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, qp 219 drivers/infiniband/hw/hfi1/qp.c struct ib_qp *ibqp = &qp->ibqp; qp 229 drivers/infiniband/hw/hfi1/qp.c if (!qp_to_sdma_engine(qp, sc) && qp 233 drivers/infiniband/hw/hfi1/qp.c if (!qp_to_send_context(qp, sc)) qp 242 drivers/infiniband/hw/hfi1/qp.c if (!qp_to_sdma_engine(qp, sc) && qp 246 drivers/infiniband/hw/hfi1/qp.c if (!qp_to_send_context(qp, sc)) qp 258 drivers/infiniband/hw/hfi1/qp.c static inline void qp_set_16b(struct rvt_qp *qp) qp 262 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 265 drivers/infiniband/hw/hfi1/qp.c hfi1_update_ah_attr(qp->ibqp.device, &qp->remote_ah_attr); qp 268 drivers/infiniband/hw/hfi1/qp.c hfi1_make_opa_lid(&qp->remote_ah_attr); qp 270 drivers/infiniband/hw/hfi1/qp.c if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) qp 273 drivers/infiniband/hw/hfi1/qp.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 275 drivers/infiniband/hw/hfi1/qp.c priv->hdr_type = hfi1_get_hdr_type(ppd->lid, &qp->remote_ah_attr); qp 278 drivers/infiniband/hw/hfi1/qp.c void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, qp 281 drivers/infiniband/hw/hfi1/qp.c struct ib_qp *ibqp = &qp->ibqp; qp 282 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 285 drivers/infiniband/hw/hfi1/qp.c priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); qp 286 drivers/infiniband/hw/hfi1/qp.c priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); qp 287 drivers/infiniband/hw/hfi1/qp.c priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); qp 288 drivers/infiniband/hw/hfi1/qp.c qp_set_16b(qp); qp 293 drivers/infiniband/hw/hfi1/qp.c qp->s_mig_state == IB_MIG_ARMED) { qp 294 drivers/infiniband/hw/hfi1/qp.c qp->s_flags |= HFI1_S_AHG_CLEAR; qp 295 drivers/infiniband/hw/hfi1/qp.c priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); qp 296 drivers/infiniband/hw/hfi1/qp.c priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); qp 297 drivers/infiniband/hw/hfi1/qp.c priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); qp 298 drivers/infiniband/hw/hfi1/qp.c qp_set_16b(qp); qp 301 drivers/infiniband/hw/hfi1/qp.c opfn_qp_init(qp, attr, attr_mask); qp 319 drivers/infiniband/hw/hfi1/qp.c int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) qp 321 drivers/infiniband/hw/hfi1/qp.c struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 326 drivers/infiniband/hw/hfi1/qp.c switch (qp->ibqp.qp_type) { qp 328 drivers/infiniband/hw/hfi1/qp.c hfi1_setup_tid_rdma_wqe(qp, wqe); qp 333 drivers/infiniband/hw/hfi1/qp.c if (wqe->length > qp->pmtu) qp 378 drivers/infiniband/hw/hfi1/qp.c bool _hfi1_schedule_send(struct rvt_qp *qp) qp 380 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 382 drivers/infiniband/hw/hfi1/qp.c to_iport(qp->ibqp.device, qp->port_num); qp 384 drivers/infiniband/hw/hfi1/qp.c struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 392 drivers/infiniband/hw/hfi1/qp.c static void qp_pio_drain(struct rvt_qp *qp) qp 394 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 418 drivers/infiniband/hw/hfi1/qp.c bool hfi1_schedule_send(struct rvt_qp *qp) qp 420 drivers/infiniband/hw/hfi1/qp.c lockdep_assert_held(&qp->s_lock); qp 421 drivers/infiniband/hw/hfi1/qp.c if (hfi1_send_ok(qp)) { qp 422 drivers/infiniband/hw/hfi1/qp.c _hfi1_schedule_send(qp); qp 425 drivers/infiniband/hw/hfi1/qp.c if (qp->s_flags & HFI1_S_ANY_WAIT_IO) qp 426 drivers/infiniband/hw/hfi1/qp.c iowait_set_flag(&((struct hfi1_qp_priv *)qp->priv)->s_iowait, qp 431 drivers/infiniband/hw/hfi1/qp.c static void hfi1_qp_schedule(struct rvt_qp *qp) qp 433 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 437 drivers/infiniband/hw/hfi1/qp.c ret = hfi1_schedule_send(qp); qp 442 drivers/infiniband/hw/hfi1/qp.c ret = hfi1_schedule_tid_send(qp); qp 448 drivers/infiniband/hw/hfi1/qp.c void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag) qp 452 drivers/infiniband/hw/hfi1/qp.c spin_lock_irqsave(&qp->s_lock, flags); qp 453 drivers/infiniband/hw/hfi1/qp.c if (qp->s_flags & flag) { qp 454 drivers/infiniband/hw/hfi1/qp.c qp->s_flags &= ~flag; qp 455 drivers/infiniband/hw/hfi1/qp.c trace_hfi1_qpwakeup(qp, flag); qp 456 drivers/infiniband/hw/hfi1/qp.c hfi1_qp_schedule(qp); qp 458 drivers/infiniband/hw/hfi1/qp.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 460 drivers/infiniband/hw/hfi1/qp.c rvt_put_qp(qp); qp 463 drivers/infiniband/hw/hfi1/qp.c void hfi1_qp_unbusy(struct rvt_qp *qp, struct iowait_work *wait) qp 465 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 468 drivers/infiniband/hw/hfi1/qp.c qp->s_flags &= ~RVT_S_BUSY; qp 496 drivers/infiniband/hw/hfi1/qp.c struct rvt_qp *qp; qp 501 drivers/infiniband/hw/hfi1/qp.c qp = tx->qp; qp 502 drivers/infiniband/hw/hfi1/qp.c priv = qp->priv; qp 504 drivers/infiniband/hw/hfi1/qp.c spin_lock_irqsave(&qp->s_lock, flags); qp 505 drivers/infiniband/hw/hfi1/qp.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 518 drivers/infiniband/hw/hfi1/qp.c to_iport(qp->ibqp.device, qp->port_num); qp 521 drivers/infiniband/hw/hfi1/qp.c qp->s_flags |= RVT_S_WAIT_DMA_DESC; qp 526 drivers/infiniband/hw/hfi1/qp.c trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC); qp 527 drivers/infiniband/hw/hfi1/qp.c rvt_get_qp(qp); qp 530 drivers/infiniband/hw/hfi1/qp.c hfi1_qp_unbusy(qp, wait); qp 531 drivers/infiniband/hw/hfi1/qp.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 534 drivers/infiniband/hw/hfi1/qp.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 540 drivers/infiniband/hw/hfi1/qp.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 547 drivers/infiniband/hw/hfi1/qp.c struct rvt_qp *qp = iowait_to_qp(wait); qp 550 drivers/infiniband/hw/hfi1/qp.c hfi1_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC); qp 555 drivers/infiniband/hw/hfi1/qp.c struct rvt_qp *qp = iowait_to_qp(wait); qp 564 drivers/infiniband/hw/hfi1/qp.c spin_lock_irqsave(&qp->s_lock, flags); qp 565 drivers/infiniband/hw/hfi1/qp.c if (qp->s_flags & RVT_S_WAIT_DMA) { qp 566 drivers/infiniband/hw/hfi1/qp.c qp->s_flags &= ~RVT_S_WAIT_DMA; qp 567 drivers/infiniband/hw/hfi1/qp.c hfi1_schedule_send(qp); qp 569 drivers/infiniband/hw/hfi1/qp.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 574 drivers/infiniband/hw/hfi1/qp.c struct rvt_qp *qp = iowait_to_qp(w); qp 575 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 577 drivers/infiniband/hw/hfi1/qp.c if (qp->s_flags & RVT_S_ACK_PENDING) qp 591 drivers/infiniband/hw/hfi1/qp.c struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5) qp 593 drivers/infiniband/hw/hfi1/qp.c struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 598 drivers/infiniband/hw/hfi1/qp.c switch (qp->ibqp.qp_type) { qp 604 drivers/infiniband/hw/hfi1/qp.c sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5); qp 616 drivers/infiniband/hw/hfi1/qp.c struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5) qp 618 drivers/infiniband/hw/hfi1/qp.c struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 620 drivers/infiniband/hw/hfi1/qp.c switch (qp->ibqp.qp_type) { qp 628 drivers/infiniband/hw/hfi1/qp.c return pio_select_send_context_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, qp 636 drivers/infiniband/hw/hfi1/qp.c static int qp_idle(struct rvt_qp *qp) qp 639 drivers/infiniband/hw/hfi1/qp.c qp->s_last == qp->s_acked && qp 640 drivers/infiniband/hw/hfi1/qp.c qp->s_acked == qp->s_cur && qp 641 drivers/infiniband/hw/hfi1/qp.c qp->s_cur == qp->s_tail && qp 642 drivers/infiniband/hw/hfi1/qp.c qp->s_tail == qp->s_head; qp 653 drivers/infiniband/hw/hfi1/qp.c struct rvt_qp *qp = iter->qp; qp 654 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 658 drivers/infiniband/hw/hfi1/qp.c struct rvt_srq *srq = qp->ibqp.srq ? qp 659 drivers/infiniband/hw/hfi1/qp.c ibsrq_to_rvtsrq(qp->ibqp.srq) : NULL; qp 661 drivers/infiniband/hw/hfi1/qp.c sde = qp_to_sdma_engine(qp, priv->s_sc); qp 662 drivers/infiniband/hw/hfi1/qp.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 663 drivers/infiniband/hw/hfi1/qp.c send_context = qp_to_send_context(qp, priv->s_sc); qp 664 drivers/infiniband/hw/hfi1/qp.c if (qp->s_ack_queue) qp 665 drivers/infiniband/hw/hfi1/qp.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 669 drivers/infiniband/hw/hfi1/qp.c qp_idle(qp) ? "I" : "B", qp 670 drivers/infiniband/hw/hfi1/qp.c qp->ibqp.qp_num, qp 671 drivers/infiniband/hw/hfi1/qp.c atomic_read(&qp->refcount), qp 672 drivers/infiniband/hw/hfi1/qp.c qp_type_str[qp->ibqp.qp_type], qp 673 drivers/infiniband/hw/hfi1/qp.c qp->state, qp 675 drivers/infiniband/hw/hfi1/qp.c qp->s_flags, qp 679 drivers/infiniband/hw/hfi1/qp.c qp->timeout, qp 681 drivers/infiniband/hw/hfi1/qp.c qp->s_lsn, qp 682 drivers/infiniband/hw/hfi1/qp.c qp->s_last_psn, qp 683 drivers/infiniband/hw/hfi1/qp.c qp->s_psn, qp->s_next_psn, qp 684 drivers/infiniband/hw/hfi1/qp.c qp->s_sending_psn, qp->s_sending_hpsn, qp 685 drivers/infiniband/hw/hfi1/qp.c qp->r_psn, qp 686 drivers/infiniband/hw/hfi1/qp.c qp->s_last, qp->s_acked, qp->s_cur, qp 687 drivers/infiniband/hw/hfi1/qp.c qp->s_tail, qp->s_head, qp->s_size, qp 688 drivers/infiniband/hw/hfi1/qp.c qp->s_avail, qp 690 drivers/infiniband/hw/hfi1/qp.c qp->s_tail_ack_queue, qp->r_head_ack_queue, qp 691 drivers/infiniband/hw/hfi1/qp.c rvt_max_atomic(&to_idev(qp->ibqp.device)->rdi), qp 693 drivers/infiniband/hw/hfi1/qp.c qp->remote_qpn, qp 694 drivers/infiniband/hw/hfi1/qp.c rdma_ah_get_dlid(&qp->remote_ah_attr), qp 695 drivers/infiniband/hw/hfi1/qp.c rdma_ah_get_sl(&qp->remote_ah_attr), qp 696 drivers/infiniband/hw/hfi1/qp.c qp->pmtu, qp 697 drivers/infiniband/hw/hfi1/qp.c qp->s_retry, qp 698 drivers/infiniband/hw/hfi1/qp.c qp->s_retry_cnt, qp 699 drivers/infiniband/hw/hfi1/qp.c qp->s_rnr_retry_cnt, qp 700 drivers/infiniband/hw/hfi1/qp.c qp->s_rnr_retry, qp 705 drivers/infiniband/hw/hfi1/qp.c ib_cq_head(qp->ibqp.send_cq), qp 706 drivers/infiniband/hw/hfi1/qp.c ib_cq_tail(qp->ibqp.send_cq), qp 707 drivers/infiniband/hw/hfi1/qp.c qp->pid, qp 708 drivers/infiniband/hw/hfi1/qp.c qp->s_state, qp 709 drivers/infiniband/hw/hfi1/qp.c qp->s_ack_state, qp 714 drivers/infiniband/hw/hfi1/qp.c qp->r_min_rnr_timer, qp 716 drivers/infiniband/hw/hfi1/qp.c srq ? srq->rq.size : qp->r_rq.size qp 720 drivers/infiniband/hw/hfi1/qp.c void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp) qp 728 drivers/infiniband/hw/hfi1/qp.c priv->owner = qp; qp 750 drivers/infiniband/hw/hfi1/qp.c void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) qp 752 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 754 drivers/infiniband/hw/hfi1/qp.c hfi1_qp_priv_tid_free(rdi, qp); qp 774 drivers/infiniband/hw/hfi1/qp.c if (rcu_dereference(ibp->rvp.qp[0])) qp 776 drivers/infiniband/hw/hfi1/qp.c if (rcu_dereference(ibp->rvp.qp[1])) qp 784 drivers/infiniband/hw/hfi1/qp.c void flush_qp_waiters(struct rvt_qp *qp) qp 786 drivers/infiniband/hw/hfi1/qp.c lockdep_assert_held(&qp->s_lock); qp 787 drivers/infiniband/hw/hfi1/qp.c flush_iowait(qp); qp 788 drivers/infiniband/hw/hfi1/qp.c hfi1_tid_rdma_flush_wait(qp); qp 791 drivers/infiniband/hw/hfi1/qp.c void stop_send_queue(struct rvt_qp *qp) qp 793 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 797 drivers/infiniband/hw/hfi1/qp.c rvt_put_qp(qp); qp 800 drivers/infiniband/hw/hfi1/qp.c void quiesce_qp(struct rvt_qp *qp) qp 802 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 804 drivers/infiniband/hw/hfi1/qp.c hfi1_del_tid_reap_timer(qp); qp 805 drivers/infiniband/hw/hfi1/qp.c hfi1_del_tid_retry_timer(qp); qp 807 drivers/infiniband/hw/hfi1/qp.c qp_pio_drain(qp); qp 808 drivers/infiniband/hw/hfi1/qp.c flush_tx_list(qp); qp 811 drivers/infiniband/hw/hfi1/qp.c void notify_qp_reset(struct rvt_qp *qp) qp 813 drivers/infiniband/hw/hfi1/qp.c hfi1_qp_kern_exp_rcv_clear_all(qp); qp 814 drivers/infiniband/hw/hfi1/qp.c qp->r_adefered = 0; qp 815 drivers/infiniband/hw/hfi1/qp.c clear_ahg(qp); qp 818 drivers/infiniband/hw/hfi1/qp.c if (qp->ibqp.qp_type == IB_QPT_RC) qp 819 drivers/infiniband/hw/hfi1/qp.c opfn_conn_error(qp); qp 826 drivers/infiniband/hw/hfi1/qp.c void hfi1_migrate_qp(struct rvt_qp *qp) qp 828 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 831 drivers/infiniband/hw/hfi1/qp.c qp->s_mig_state = IB_MIG_MIGRATED; qp 832 drivers/infiniband/hw/hfi1/qp.c qp->remote_ah_attr = qp->alt_ah_attr; qp 833 drivers/infiniband/hw/hfi1/qp.c qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); qp 834 drivers/infiniband/hw/hfi1/qp.c qp->s_pkey_index = qp->s_alt_pkey_index; qp 835 drivers/infiniband/hw/hfi1/qp.c qp->s_flags |= HFI1_S_AHG_CLEAR; qp 836 drivers/infiniband/hw/hfi1/qp.c priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr); qp 837 drivers/infiniband/hw/hfi1/qp.c priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); qp 838 drivers/infiniband/hw/hfi1/qp.c qp_set_16b(qp); qp 840 drivers/infiniband/hw/hfi1/qp.c ev.device = qp->ibqp.device; qp 841 drivers/infiniband/hw/hfi1/qp.c ev.element.qp = &qp->ibqp; qp 843 drivers/infiniband/hw/hfi1/qp.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 851 drivers/infiniband/hw/hfi1/qp.c u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu) qp 863 drivers/infiniband/hw/hfi1/qp.c ibp = &dd->pport[qp->port_num - 1].ibport_data; qp 864 drivers/infiniband/hw/hfi1/qp.c sc = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; qp 867 drivers/infiniband/hw/hfi1/qp.c mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu); qp 873 drivers/infiniband/hw/hfi1/qp.c int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 876 drivers/infiniband/hw/hfi1/qp.c int mtu, pidx = qp->port_num - 1; qp 883 drivers/infiniband/hw/hfi1/qp.c mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu); qp 893 drivers/infiniband/hw/hfi1/qp.c void notify_error_qp(struct rvt_qp *qp) qp 895 drivers/infiniband/hw/hfi1/qp.c struct hfi1_qp_priv *priv = qp->priv; qp 901 drivers/infiniband/hw/hfi1/qp.c !(qp->s_flags & RVT_S_BUSY) && qp 903 drivers/infiniband/hw/hfi1/qp.c qp->s_flags &= ~HFI1_S_ANY_WAIT_IO; qp 908 drivers/infiniband/hw/hfi1/qp.c rvt_put_qp(qp); qp 913 drivers/infiniband/hw/hfi1/qp.c if (!(qp->s_flags & RVT_S_BUSY) && !(priv->s_flags & RVT_S_BUSY)) { qp 914 drivers/infiniband/hw/hfi1/qp.c qp->s_hdrwords = 0; qp 915 drivers/infiniband/hw/hfi1/qp.c if (qp->s_rdma_mr) { qp 916 drivers/infiniband/hw/hfi1/qp.c rvt_put_mr(qp->s_rdma_mr); qp 917 drivers/infiniband/hw/hfi1/qp.c qp->s_rdma_mr = NULL; qp 919 drivers/infiniband/hw/hfi1/qp.c flush_tx_list(qp); qp 931 drivers/infiniband/hw/hfi1/qp.c static void hfi1_qp_iter_cb(struct rvt_qp *qp, u64 v) qp 936 drivers/infiniband/hw/hfi1/qp.c to_iport(qp->ibqp.device, qp->port_num); qp 940 drivers/infiniband/hw/hfi1/qp.c if (qp->port_num != ppd->port || qp 941 drivers/infiniband/hw/hfi1/qp.c (qp->ibqp.qp_type != IB_QPT_UC && qp 942 drivers/infiniband/hw/hfi1/qp.c qp->ibqp.qp_type != IB_QPT_RC) || qp 943 drivers/infiniband/hw/hfi1/qp.c rdma_ah_get_sl(&qp->remote_ah_attr) != sl || qp 944 drivers/infiniband/hw/hfi1/qp.c !(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK)) qp 947 drivers/infiniband/hw/hfi1/qp.c spin_lock_irq(&qp->r_lock); qp 948 drivers/infiniband/hw/hfi1/qp.c spin_lock(&qp->s_hlock); qp 949 drivers/infiniband/hw/hfi1/qp.c spin_lock(&qp->s_lock); qp 950 drivers/infiniband/hw/hfi1/qp.c lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 951 drivers/infiniband/hw/hfi1/qp.c spin_unlock(&qp->s_lock); qp 952 drivers/infiniband/hw/hfi1/qp.c spin_unlock(&qp->s_hlock); qp 953 drivers/infiniband/hw/hfi1/qp.c spin_unlock_irq(&qp->r_lock); qp 955 drivers/infiniband/hw/hfi1/qp.c ev.device = qp->ibqp.device; qp 956 drivers/infiniband/hw/hfi1/qp.c ev.element.qp = &qp->ibqp; qp 958 drivers/infiniband/hw/hfi1/qp.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 91 drivers/infiniband/hw/hfi1/qp.h static inline int hfi1_send_ok(struct rvt_qp *qp) qp 93 drivers/infiniband/hw/hfi1/qp.h struct hfi1_qp_priv *priv = qp->priv; qp 95 drivers/infiniband/hw/hfi1/qp.h return !(qp->s_flags & (RVT_S_BUSY | HFI1_S_ANY_WAIT_IO)) && qp 97 drivers/infiniband/hw/hfi1/qp.h (qp->s_flags & RVT_S_RESP_PENDING) || qp 98 drivers/infiniband/hw/hfi1/qp.h !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); qp 104 drivers/infiniband/hw/hfi1/qp.h static inline void clear_ahg(struct rvt_qp *qp) qp 106 drivers/infiniband/hw/hfi1/qp.h struct hfi1_qp_priv *priv = qp->priv; qp 109 drivers/infiniband/hw/hfi1/qp.h qp->s_flags &= ~(HFI1_S_AHG_VALID | HFI1_S_AHG_CLEAR); qp 110 drivers/infiniband/hw/hfi1/qp.h if (priv->s_sde && qp->s_ahgidx >= 0) qp 111 drivers/infiniband/hw/hfi1/qp.h sdma_ahg_free(priv->s_sde, qp->s_ahgidx); qp 112 drivers/infiniband/hw/hfi1/qp.h qp->s_ahgidx = -1; qp 134 drivers/infiniband/hw/hfi1/qp.h void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag); qp 136 drivers/infiniband/hw/hfi1/qp.h struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5); qp 137 drivers/infiniband/hw/hfi1/qp.h struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5); qp 141 drivers/infiniband/hw/hfi1/qp.h bool _hfi1_schedule_send(struct rvt_qp *qp); qp 142 drivers/infiniband/hw/hfi1/qp.h bool hfi1_schedule_send(struct rvt_qp *qp); qp 144 drivers/infiniband/hw/hfi1/qp.h void hfi1_migrate_qp(struct rvt_qp *qp); qp 149 drivers/infiniband/hw/hfi1/qp.h void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp); qp 150 drivers/infiniband/hw/hfi1/qp.h void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); qp 152 drivers/infiniband/hw/hfi1/qp.h void notify_qp_reset(struct rvt_qp *qp); qp 153 drivers/infiniband/hw/hfi1/qp.h int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 155 drivers/infiniband/hw/hfi1/qp.h void flush_qp_waiters(struct rvt_qp *qp); qp 156 drivers/infiniband/hw/hfi1/qp.h void notify_error_qp(struct rvt_qp *qp); qp 157 drivers/infiniband/hw/hfi1/qp.h void stop_send_queue(struct rvt_qp *qp); qp 158 drivers/infiniband/hw/hfi1/qp.h void quiesce_qp(struct rvt_qp *qp); qp 159 drivers/infiniband/hw/hfi1/qp.h u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu); qp 162 drivers/infiniband/hw/hfi1/qp.h void hfi1_qp_unbusy(struct rvt_qp *qp, struct iowait_work *wait); qp 58 drivers/infiniband/hw/hfi1/rc.c struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev, qp 60 drivers/infiniband/hw/hfi1/rc.c __must_hold(&qp->s_lock) qp 66 drivers/infiniband/hw/hfi1/rc.c for (i = qp->r_head_ack_queue; ; i = p) { qp 67 drivers/infiniband/hw/hfi1/rc.c if (i == qp->s_tail_ack_queue) qp 72 drivers/infiniband/hw/hfi1/rc.c p = rvt_size_atomic(ib_to_rvt(qp->ibqp.device)); qp 73 drivers/infiniband/hw/hfi1/rc.c if (p == qp->r_head_ack_queue) { qp 77 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[p]; qp 83 drivers/infiniband/hw/hfi1/rc.c if (p == qp->s_tail_ack_queue && qp 109 drivers/infiniband/hw/hfi1/rc.c static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, qp 117 drivers/infiniband/hw/hfi1/rc.c u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); qp 119 drivers/infiniband/hw/hfi1/rc.c u32 pmtu = qp->pmtu; qp 120 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *qpriv = qp->priv; qp 123 drivers/infiniband/hw/hfi1/rc.c u8 next = qp->s_tail_ack_queue; qp 126 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_rsp_make_rc_ack(qp, 0); qp 127 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 129 drivers/infiniband/hw/hfi1/rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) qp 139 drivers/infiniband/hw/hfi1/rc.c switch (qp->s_ack_state) { qp 142 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 157 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 159 drivers/infiniband/hw/hfi1/rc.c qp->s_acked_ack_queue == qp->s_tail_ack_queue) qp 160 drivers/infiniband/hw/hfi1/rc.c qp->s_acked_ack_queue = next; qp 161 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue = next; qp 162 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_rsp_make_rc_ack(qp, e->psn); qp 167 drivers/infiniband/hw/hfi1/rc.c if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { qp 168 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & RVT_S_ACK_PENDING) qp 173 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 176 drivers/infiniband/hw/hfi1/rc.c hfi1_tid_rdma_ack_interlock(qp, e)) { qp 189 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked_ack_queue == qp 190 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue) qp 191 drivers/infiniband/hw/hfi1/rc.c qp->s_acked_ack_queue = qp 192 drivers/infiniband/hw/hfi1/rc.c qp->r_head_ack_queue; qp 193 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue = qp->r_head_ack_queue; qp 200 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_rdma_sge.sge = e->rdma_sge; qp 201 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_rdma_sge.num_sge = 1; qp 202 drivers/infiniband/hw/hfi1/rc.c ps->s_txreq->ss = &qp->s_ack_rdma_sge; qp 205 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); qp 207 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); qp 210 drivers/infiniband/hw/hfi1/rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 212 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_rdma_psn = e->psn; qp 213 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_ack_rdma_psn++); qp 226 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = TID_OP(WRITE_RESP); qp 227 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_rdma_psn = mask_psn(e->psn + req->cur_seg); qp 238 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked_ack_queue == qp 239 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue) qp 240 drivers/infiniband/hw/hfi1/rc.c qp->s_acked_ack_queue = qp 241 drivers/infiniband/hw/hfi1/rc.c qp->r_head_ack_queue; qp 242 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue = qp->r_head_ack_queue; qp 249 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_rdma_sge.sge = e->rdma_sge; qp 250 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_rdma_sge.num_sge = 1; qp 251 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = TID_OP(READ_RESP); qp 257 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); qp 258 drivers/infiniband/hw/hfi1/rc.c ohdr->u.at.aeth = rvt_compute_aeth(qp); qp 264 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_write_rsp_make_rc_ack(qp); qp 265 drivers/infiniband/hw/hfi1/rc.c bth0 = qp->s_ack_state << 24; qp 269 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); qp 272 drivers/infiniband/hw/hfi1/rc.c ps->s_txreq->ss = &qp->s_ack_rdma_sge; qp 273 drivers/infiniband/hw/hfi1/rc.c ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr; qp 276 drivers/infiniband/hw/hfi1/rc.c len = qp->s_ack_rdma_sge.sge.sge_length; qp 281 drivers/infiniband/hw/hfi1/rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 283 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); qp 284 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 287 drivers/infiniband/hw/hfi1/rc.c bth0 = qp->s_ack_state << 24; qp 288 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_ack_rdma_psn++); qp 312 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 321 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue == qpriv->r_tid_alloc && qp 327 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_ack_rdma_psn); qp 328 drivers/infiniband/hw/hfi1/rc.c hdrlen = hfi1_build_tid_rdma_write_resp(qp, e, ohdr, &bth1, qp 335 drivers/infiniband/hw/hfi1/rc.c bth0 = qp->s_ack_state << 24; qp 336 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_rdma_psn++; qp 337 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_rc_ack_write(qp, 0, e->opcode, e->psn, qp 344 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); qp 349 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 350 drivers/infiniband/hw/hfi1/rc.c ps->s_txreq->ss = &qp->s_ack_rdma_sge; qp 351 drivers/infiniband/hw/hfi1/rc.c delta = hfi1_build_tid_rdma_read_resp(qp, e, ohdr, &bth0, qp 363 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); qp 377 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(SEND_ONLY); qp 379 drivers/infiniband/hw/hfi1/rc.c if (qp->s_nak_state) qp 381 drivers/infiniband/hw/hfi1/rc.c cpu_to_be32((qp->r_msn & IB_MSN_MASK) | qp 382 drivers/infiniband/hw/hfi1/rc.c (qp->s_nak_state << qp 385 drivers/infiniband/hw/hfi1/rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 389 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_ack_psn); qp 390 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~RVT_S_ACK_PENDING; qp 394 drivers/infiniband/hw/hfi1/rc.c qp->s_rdma_ack_cnt++; qp 398 drivers/infiniband/hw/hfi1/rc.c hfi1_make_ruc_header(qp, ohdr, bth0, bth1, bth2, middle, ps); qp 401 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, ps->flags); qp 402 drivers/infiniband/hw/hfi1/rc.c spin_lock_irqsave(&qp->r_lock, ps->flags); qp 403 drivers/infiniband/hw/hfi1/rc.c spin_lock(&qp->s_lock); qp 404 drivers/infiniband/hw/hfi1/rc.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 405 drivers/infiniband/hw/hfi1/rc.c spin_unlock(&qp->s_lock); qp 406 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->r_lock, ps->flags); qp 407 drivers/infiniband/hw/hfi1/rc.c spin_lock_irqsave(&qp->s_lock, ps->flags); qp 409 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(ACKNOWLEDGE); qp 415 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~(RVT_S_RESP_PENDING qp 429 drivers/infiniband/hw/hfi1/rc.c int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) qp 431 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *priv = qp->priv; qp 432 drivers/infiniband/hw/hfi1/rc.c struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); qp 442 drivers/infiniband/hw/hfi1/rc.c u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); qp 443 drivers/infiniband/hw/hfi1/rc.c u32 pmtu = qp->pmtu; qp 450 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_sender_make_rc_req(qp); qp 451 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 452 drivers/infiniband/hw/hfi1/rc.c ps->s_txreq = get_txreq(ps->dev, qp); qp 459 drivers/infiniband/hw/hfi1/rc.c if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) qp 466 drivers/infiniband/hw/hfi1/rc.c if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && qp 467 drivers/infiniband/hw/hfi1/rc.c (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr)))) qp 474 drivers/infiniband/hw/hfi1/rc.c if ((qp->s_flags & RVT_S_RESP_PENDING) && qp 475 drivers/infiniband/hw/hfi1/rc.c make_rc_ack(dev, qp, ohdr, ps)) qp 478 drivers/infiniband/hw/hfi1/rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { qp 479 drivers/infiniband/hw/hfi1/rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) qp 482 drivers/infiniband/hw/hfi1/rc.c if (qp->s_last == READ_ONCE(qp->s_head)) qp 486 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_DMA; qp 489 drivers/infiniband/hw/hfi1/rc.c clear_ahg(qp); qp 490 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 491 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ? qp 497 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK | HFI1_S_WAIT_HALT)) qp 500 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) { qp 501 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { qp 502 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_PSN; qp 505 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_psn = qp->s_psn; qp 506 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_hpsn = qp->s_psn - 1; qp 510 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 512 drivers/infiniband/hw/hfi1/rc.c switch (qp->s_state) { qp 514 drivers/infiniband/hw/hfi1/rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) qp 524 drivers/infiniband/hw/hfi1/rc.c if (qp->s_cur == qp->s_tail) { qp 526 drivers/infiniband/hw/hfi1/rc.c if (qp->s_tail == READ_ONCE(qp->s_head)) { qp 527 drivers/infiniband/hw/hfi1/rc.c clear_ahg(qp); qp 537 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic && qp 539 drivers/infiniband/hw/hfi1/rc.c priv->pending_tid_r_segs < qp->s_num_rd_atomic)) { qp 540 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_FENCE; qp 552 drivers/infiniband/hw/hfi1/rc.c if (qp->s_last != qp->s_cur) qp 554 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur == qp->s_size) qp 555 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 556 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_tail == qp->s_size) qp 557 drivers/infiniband/hw/hfi1/rc.c qp->s_tail = 0; qp 561 drivers/infiniband/hw/hfi1/rc.c qp, qp 565 drivers/infiniband/hw/hfi1/rc.c rvt_send_complete(qp, wqe, qp 569 drivers/infiniband/hw/hfi1/rc.c atomic_dec(&qp->local_ops_pending); qp 574 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->psn; qp 582 drivers/infiniband/hw/hfi1/rc.c ss = &qp->s_sge; qp 583 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_psn); qp 590 drivers/infiniband/hw/hfi1/rc.c hfi1_tid_rdma_wqe_interlock(qp, wqe)) qp 598 drivers/infiniband/hw/hfi1/rc.c if (!rvt_rc_credit_avail(qp, wqe)) qp 601 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_FIRST); qp 606 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_ONLY); qp 608 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); qp 613 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE); qp 622 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur == qp->s_size) qp 623 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 627 drivers/infiniband/hw/hfi1/rc.c if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 628 drivers/infiniband/hw/hfi1/rc.c qp->s_lsn++; qp 632 drivers/infiniband/hw/hfi1/rc.c if (!rvt_rc_credit_avail(qp, wqe)) qp 643 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_WRITE_FIRST); qp 648 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_WRITE_ONLY); qp 650 drivers/infiniband/hw/hfi1/rc.c qp->s_state = qp 659 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur == qp->s_size) qp 660 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 672 drivers/infiniband/hw/hfi1/rc.c if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 673 drivers/infiniband/hw/hfi1/rc.c qp->s_lsn++; qp 676 drivers/infiniband/hw/hfi1/rc.c hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, qp 681 drivers/infiniband/hw/hfi1/rc.c priv->s_tid_cur = qp->s_cur; qp 683 drivers/infiniband/hw/hfi1/rc.c priv->s_tid_tail = qp->s_cur; qp 690 drivers/infiniband/hw/hfi1/rc.c __w = rvt_get_swqe_ptr(qp, priv->s_tid_cur); qp 723 drivers/infiniband/hw/hfi1/rc.c priv->s_tid_tail = qp->s_cur; qp 727 drivers/infiniband/hw/hfi1/rc.c priv->s_tid_cur = qp->s_cur; qp 739 drivers/infiniband/hw/hfi1/rc.c if (priv->s_tid_tail == qp->s_cur && qp 745 drivers/infiniband/hw/hfi1/rc.c priv->s_tid_head = qp->s_cur; qp 761 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_write_sender_make_req(qp, newreq); qp 762 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_write(qp, newreq, qp 766 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur == qp->s_size) qp 767 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 775 drivers/infiniband/hw/hfi1/rc.c if (qp->s_num_rd_atomic >= qp 776 drivers/infiniband/hw/hfi1/rc.c qp->s_max_rd_atomic) { qp 777 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_RDMAR; qp 780 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic++; qp 781 drivers/infiniband/hw/hfi1/rc.c if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 782 drivers/infiniband/hw/hfi1/rc.c qp->s_lsn++; qp 789 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_READ_REQUEST); qp 794 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur == qp->s_size) qp 795 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 799 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_read_sender_make_req(qp, newreq); qp 802 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_read(qp, newreq, qp 806 drivers/infiniband/hw/hfi1/rc.c delta = cmp_psn(qp->s_psn, wqe->psn); qp 818 drivers/infiniband/hw/hfi1/rc.c if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { qp 819 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_RDMAR; qp 833 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sge = wqe->sg_list[0]; qp 834 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sg_list = wqe->sg_list + 1; qp 835 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.num_sge = wqe->wr.num_sge; qp 836 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.total_len = wqe->length; qp 837 drivers/infiniband/hw/hfi1/rc.c qp->s_len = wqe->length; qp 851 drivers/infiniband/hw/hfi1/rc.c req->s_next_psn = qp->s_psn; qp 855 drivers/infiniband/hw/hfi1/rc.c delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, qp 862 drivers/infiniband/hw/hfi1/rc.c if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 863 drivers/infiniband/hw/hfi1/rc.c qp->s_lsn++; qp 868 drivers/infiniband/hw/hfi1/rc.c ++qp->s_cur == qp->s_size) qp 869 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 878 drivers/infiniband/hw/hfi1/rc.c if (qp->s_num_rd_atomic >= qp 879 drivers/infiniband/hw/hfi1/rc.c qp->s_max_rd_atomic) { qp 880 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_RDMAR; qp 883 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic++; qp 887 drivers/infiniband/hw/hfi1/rc.c if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 888 drivers/infiniband/hw/hfi1/rc.c qp->s_lsn++; qp 891 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(COMPARE_SWAP); qp 897 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(FETCH_ADD); qp 910 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur == qp->s_size) qp 911 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 918 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sge = wqe->sg_list[0]; qp 919 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sg_list = wqe->sg_list + 1; qp 920 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.num_sge = wqe->wr.num_sge; qp 921 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.total_len = wqe->length; qp 922 drivers/infiniband/hw/hfi1/rc.c qp->s_len = wqe->length; qp 925 drivers/infiniband/hw/hfi1/rc.c qp->s_tail++; qp 926 drivers/infiniband/hw/hfi1/rc.c if (qp->s_tail >= qp->s_size) qp 927 drivers/infiniband/hw/hfi1/rc.c qp->s_tail = 0; qp 931 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->lpsn + 1; qp 933 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = req->s_next_psn; qp 935 drivers/infiniband/hw/hfi1/rc.c qp->s_psn++; qp 948 drivers/infiniband/hw/hfi1/rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); qp 951 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_MIDDLE); qp 954 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_psn++); qp 955 drivers/infiniband/hw/hfi1/rc.c ss = &qp->s_sge; qp 956 drivers/infiniband/hw/hfi1/rc.c len = qp->s_len; qp 963 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST); qp 965 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); qp 970 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST_WITH_INVALIDATE); qp 978 drivers/infiniband/hw/hfi1/rc.c qp->s_cur++; qp 979 drivers/infiniband/hw/hfi1/rc.c if (qp->s_cur >= qp->s_size) qp 980 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 993 drivers/infiniband/hw/hfi1/rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); qp 996 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_WRITE_MIDDLE); qp 999 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_psn++); qp 1000 drivers/infiniband/hw/hfi1/rc.c ss = &qp->s_sge; qp 1001 drivers/infiniband/hw/hfi1/rc.c len = qp->s_len; qp 1008 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_WRITE_LAST); qp 1010 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); qp 1018 drivers/infiniband/hw/hfi1/rc.c qp->s_cur++; qp 1019 drivers/infiniband/hw/hfi1/rc.c if (qp->s_cur >= qp->s_size) qp 1020 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 1033 drivers/infiniband/hw/hfi1/rc.c len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu; qp 1040 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_READ_REQUEST); qp 1042 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_psn) | IB_BTH_REQ_ACK; qp 1043 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->lpsn + 1; qp 1046 drivers/infiniband/hw/hfi1/rc.c qp->s_cur++; qp 1047 drivers/infiniband/hw/hfi1/rc.c if (qp->s_cur == qp->s_size) qp 1048 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 1061 drivers/infiniband/hw/hfi1/rc.c req->comp_seg = delta_psn(qp->s_psn, wqe->psn); qp 1065 drivers/infiniband/hw/hfi1/rc.c bth2 = mask_psn(qp->s_psn); qp 1066 drivers/infiniband/hw/hfi1/rc.c hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1, qp 1068 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->lpsn + 1; qp 1070 drivers/infiniband/hw/hfi1/rc.c qp->s_state = TID_OP(WRITE_REQ); qp 1072 drivers/infiniband/hw/hfi1/rc.c priv->s_tid_cur = qp->s_cur; qp 1073 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur == qp->s_size) qp 1074 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 1075 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode, qp 1090 drivers/infiniband/hw/hfi1/rc.c req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps; qp 1099 drivers/infiniband/hw/hfi1/rc.c hfi1_tid_rdma_restart_req(qp, wqe, &bth2); qp 1106 drivers/infiniband/hw/hfi1/rc.c hfi1_kern_clear_hw_flow(priv->rcd, qp); qp 1108 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR); qp 1127 drivers/infiniband/hw/hfi1/rc.c ++qp->s_cur == qp->s_size) qp 1128 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 1129 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = req->s_next_psn; qp 1130 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, qp 1135 drivers/infiniband/hw/hfi1/rc.c delta = cmp_psn(qp->s_psn, wqe->psn); qp 1142 drivers/infiniband/hw/hfi1/rc.c qp->s_cur == qp->s_tail) { qp 1143 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_READ_REQUEST); qp 1144 drivers/infiniband/hw/hfi1/rc.c if (delta == 0 || qp->s_cur == qp->s_tail) qp 1151 drivers/infiniband/hw/hfi1/rc.c if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { qp 1152 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_RDMAR; qp 1160 drivers/infiniband/hw/hfi1/rc.c delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1, qp 1170 drivers/infiniband/hw/hfi1/rc.c ++qp->s_cur == qp->s_size) qp 1171 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 1172 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = req->s_next_psn; qp 1173 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, qp 1177 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_hpsn = bth2; qp 1182 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & RVT_S_SEND_ONE) { qp 1183 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~RVT_S_SEND_ONE; qp 1184 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_ACK; qp 1187 drivers/infiniband/hw/hfi1/rc.c qp->s_len -= len; qp 1193 drivers/infiniband/hw/hfi1/rc.c qp, qp 1195 drivers/infiniband/hw/hfi1/rc.c bth0 | (qp->s_state << 24), qp 1212 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~RVT_S_BUSY; qp 1222 drivers/infiniband/hw/hfi1/rc.c static inline void hfi1_make_bth_aeth(struct rvt_qp *qp, qp 1226 drivers/infiniband/hw/hfi1/rc.c if (qp->r_nak_state) qp 1227 drivers/infiniband/hw/hfi1/rc.c ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | qp 1228 drivers/infiniband/hw/hfi1/rc.c (qp->r_nak_state << qp 1231 drivers/infiniband/hw/hfi1/rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 1234 drivers/infiniband/hw/hfi1/rc.c ohdr->bth[1] = cpu_to_be32(bth1 | qp->remote_qpn); qp 1235 drivers/infiniband/hw/hfi1/rc.c ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn)); qp 1240 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp = packet->qp; qp 1244 drivers/infiniband/hw/hfi1/rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 1245 drivers/infiniband/hw/hfi1/rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) qp 1249 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; qp 1250 drivers/infiniband/hw/hfi1/rc.c qp->s_nak_state = qp->r_nak_state; qp 1251 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_psn = qp->r_ack_psn; qp 1253 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_ECN; qp 1256 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 1258 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1267 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp = packet->qp; qp 1281 drivers/infiniband/hw/hfi1/rc.c if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { qp 1283 drivers/infiniband/hw/hfi1/rc.c rdma_ah_read_grh(&qp->remote_ah_attr), qp 1292 drivers/infiniband/hw/hfi1/rc.c pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); qp 1295 drivers/infiniband/hw/hfi1/rc.c (rdma_ah_get_sl(&qp->remote_ah_attr) & IB_SL_MASK) << qp 1299 drivers/infiniband/hw/hfi1/rc.c opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), qp 1300 drivers/infiniband/hw/hfi1/rc.c ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr)); qp 1303 drivers/infiniband/hw/hfi1/rc.c if (qp->s_mig_state == IB_MIG_MIGRATED) qp 1311 drivers/infiniband/hw/hfi1/rc.c hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); qp 1320 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp = packet->qp; qp 1338 drivers/infiniband/hw/hfi1/rc.c if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && qp 1339 drivers/infiniband/hw/hfi1/rc.c hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { qp 1341 drivers/infiniband/hw/hfi1/rc.c rdma_ah_read_grh(&qp->remote_ah_attr), qp 1349 drivers/infiniband/hw/hfi1/rc.c pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); qp 1355 drivers/infiniband/hw/hfi1/rc.c (rdma_ah_get_path_bits(&qp->remote_ah_attr) & qp 1357 drivers/infiniband/hw/hfi1/rc.c opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), qp 1362 drivers/infiniband/hw/hfi1/rc.c if (qp->s_mig_state == IB_MIG_MIGRATED) qp 1364 drivers/infiniband/hw/hfi1/rc.c hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); qp 1390 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp = packet->qp; qp 1392 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *priv = qp->priv; qp 1394 drivers/infiniband/hw/hfi1/rc.c u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; qp 1403 drivers/infiniband/hw/hfi1/rc.c qp->r_adefered = 0; qp 1406 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & RVT_S_RESP_PENDING) { qp 1412 drivers/infiniband/hw/hfi1/rc.c if (qp->s_rdma_ack_cnt) { qp 1426 drivers/infiniband/hw/hfi1/rc.c pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, qp 1439 drivers/infiniband/hw/hfi1/rc.c trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), qp 1460 drivers/infiniband/hw/hfi1/rc.c static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn, qp 1468 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic++; qp 1471 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *priv = qp->priv; qp 1479 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic += req->ack_pending; qp 1480 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_update_num_rd_atomic(qp, 0, qp 1487 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic += req->total_segs; qp 1501 drivers/infiniband/hw/hfi1/rc.c static void reset_psn(struct rvt_qp *qp, u32 psn) qp 1503 drivers/infiniband/hw/hfi1/rc.c u32 n = qp->s_acked; qp 1504 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); qp 1506 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *priv = qp->priv; qp 1508 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 1509 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = n; qp 1512 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic = 0; qp 1519 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST); qp 1522 drivers/infiniband/hw/hfi1/rc.c update_num_rd_atomic(qp, psn, wqe); qp 1528 drivers/infiniband/hw/hfi1/rc.c if (++n == qp->s_size) qp 1530 drivers/infiniband/hw/hfi1/rc.c if (n == qp->s_tail) qp 1532 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, n); qp 1536 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 1539 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = n; qp 1545 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST); qp 1549 drivers/infiniband/hw/hfi1/rc.c update_num_rd_atomic(qp, psn, wqe); qp 1561 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); qp 1566 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_READ_RESPONSE_LAST); qp 1570 drivers/infiniband/hw/hfi1/rc.c qp->s_state = TID_OP(WRITE_RESP); qp 1574 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); qp 1578 drivers/infiniband/hw/hfi1/rc.c qp->s_state = TID_OP(READ_RESP); qp 1586 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST); qp 1590 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = psn; qp 1596 drivers/infiniband/hw/hfi1/rc.c if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) && qp 1597 drivers/infiniband/hw/hfi1/rc.c (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) qp 1598 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_WAIT_PSN; qp 1599 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~HFI1_S_AHG_VALID; qp 1600 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_sender_reset_psn(qp); qp 1607 drivers/infiniband/hw/hfi1/rc.c void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait) qp 1609 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *priv = qp->priv; qp 1610 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1613 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->r_lock); qp 1614 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 1615 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_sender_restart_rc(qp); qp 1616 drivers/infiniband/hw/hfi1/rc.c if (qp->s_retry == 0) { qp 1617 drivers/infiniband/hw/hfi1/rc.c if (qp->s_mig_state == IB_MIG_ARMED) { qp 1618 drivers/infiniband/hw/hfi1/rc.c hfi1_migrate_qp(qp); qp 1619 drivers/infiniband/hw/hfi1/rc.c qp->s_retry = qp->s_retry_cnt; qp 1620 drivers/infiniband/hw/hfi1/rc.c } else if (qp->s_last == qp->s_acked) { qp 1627 drivers/infiniband/hw/hfi1/rc.c to_iport(qp->ibqp.device, qp->port_num); qp 1633 drivers/infiniband/hw/hfi1/rc.c opfn_conn_reply(qp, priv->opfn.curr); qp 1634 drivers/infiniband/hw/hfi1/rc.c wqe = do_rc_completion(qp, wqe, ibp); qp 1635 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~RVT_S_WAIT_ACK; qp 1637 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_write_sender_restart_rc(qp, 0); qp 1643 drivers/infiniband/hw/hfi1/rc.c hfi1_kern_clear_hw_flow(priv->rcd, qp); qp 1646 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, qp 1648 drivers/infiniband/hw/hfi1/rc.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 1655 drivers/infiniband/hw/hfi1/rc.c qp->s_retry--; qp 1658 drivers/infiniband/hw/hfi1/rc.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 1663 drivers/infiniband/hw/hfi1/rc.c ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); qp 1665 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | qp 1669 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_SEND_ONE; qp 1670 drivers/infiniband/hw/hfi1/rc.c reset_psn(qp, psn); qp 1678 drivers/infiniband/hw/hfi1/rc.c static void reset_sending_psn(struct rvt_qp *qp, u32 psn) qp 1681 drivers/infiniband/hw/hfi1/rc.c u32 n = qp->s_last; qp 1683 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 1686 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, n); qp 1691 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_psn = wqe->lpsn + 1; qp 1693 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_psn = psn + 1; qp 1696 drivers/infiniband/hw/hfi1/rc.c if (++n == qp->s_size) qp 1698 drivers/infiniband/hw/hfi1/rc.c if (n == qp->s_tail) qp 1715 drivers/infiniband/hw/hfi1/rc.c void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah) qp 1730 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_SEND_ONE; qp 1736 drivers/infiniband/hw/hfi1/rc.c void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) qp 1739 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *priv = qp->priv; qp 1745 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 1746 drivers/infiniband/hw/hfi1/rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) qp 1755 drivers/infiniband/hw/hfi1/rc.c WARN_ON(!qp->s_rdma_ack_cnt); qp 1756 drivers/infiniband/hw/hfi1/rc.c qp->s_rdma_ack_cnt--; qp 1768 drivers/infiniband/hw/hfi1/rc.c reset_sending_psn(qp, psn); qp 1783 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, tail); qp 1787 drivers/infiniband/hw/hfi1/rc.c tail = qp->s_size - 1; qp 1792 drivers/infiniband/hw/hfi1/rc.c head = qp->s_tail; qp 1793 drivers/infiniband/hw/hfi1/rc.c tail = qp->s_acked; qp 1803 drivers/infiniband/hw/hfi1/rc.c !(qp->s_flags & qp 1805 drivers/infiniband/hw/hfi1/rc.c (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { qp 1807 drivers/infiniband/hw/hfi1/rc.c rvt_add_retry_timer_ext(qp, priv->timeout_shift); qp 1809 drivers/infiniband/hw/hfi1/rc.c rvt_add_retry_timer(qp); qp 1818 drivers/infiniband/hw/hfi1/rc.c (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { qp 1824 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1828 drivers/infiniband/hw/hfi1/rc.c hfi1_add_tid_retry_timer(qp); qp 1831 drivers/infiniband/hw/hfi1/rc.c while (qp->s_last != qp->s_acked) { qp 1832 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 1833 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && qp 1834 drivers/infiniband/hw/hfi1/rc.c cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) qp 1836 drivers/infiniband/hw/hfi1/rc.c trdma_clean_swqe(qp, wqe); qp 1837 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); qp 1838 drivers/infiniband/hw/hfi1/rc.c rvt_qp_complete_swqe(qp, qp 1847 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_sendcomplete(qp, psn); qp 1848 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & RVT_S_WAIT_PSN && qp 1849 drivers/infiniband/hw/hfi1/rc.c cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { qp 1850 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~RVT_S_WAIT_PSN; qp 1851 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_psn = qp->s_psn; qp 1852 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_hpsn = qp->s_psn - 1; qp 1853 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 1857 drivers/infiniband/hw/hfi1/rc.c static inline void update_last_psn(struct rvt_qp *qp, u32 psn) qp 1859 drivers/infiniband/hw/hfi1/rc.c qp->s_last_psn = psn; qp 1867 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, qp 1871 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *priv = qp->priv; qp 1873 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 1879 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_rc_completion(qp, wqe->lpsn); qp 1880 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || qp 1881 drivers/infiniband/hw/hfi1/rc.c cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { qp 1882 drivers/infiniband/hw/hfi1/rc.c trdma_clean_swqe(qp, wqe); qp 1883 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); qp 1884 drivers/infiniband/hw/hfi1/rc.c rvt_qp_complete_swqe(qp, qp 1898 drivers/infiniband/hw/hfi1/rc.c u8 sl = rdma_ah_get_sl(&qp->remote_ah_attr); qp 1903 drivers/infiniband/hw/hfi1/rc.c engine = qp_to_sdma_engine(qp, sc5); qp 1908 drivers/infiniband/hw/hfi1/rc.c qp->s_retry = qp->s_retry_cnt; qp 1918 drivers/infiniband/hw/hfi1/rc.c update_last_psn(qp, wqe->lpsn); qp 1925 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked == qp->s_cur) { qp 1926 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_cur >= qp->s_size) qp 1927 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 1928 drivers/infiniband/hw/hfi1/rc.c qp->s_acked = qp->s_cur; qp 1929 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 1930 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked != qp->s_tail) { qp 1931 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST); qp 1932 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->psn; qp 1935 drivers/infiniband/hw/hfi1/rc.c if (++qp->s_acked >= qp->s_size) qp 1936 drivers/infiniband/hw/hfi1/rc.c qp->s_acked = 0; qp 1937 drivers/infiniband/hw/hfi1/rc.c if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) qp 1938 drivers/infiniband/hw/hfi1/rc.c qp->s_draining = 0; qp 1939 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1943 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 1948 drivers/infiniband/hw/hfi1/rc.c static void set_restart_qp(struct rvt_qp *qp, struct hfi1_ctxtdata *rcd) qp 1951 drivers/infiniband/hw/hfi1/rc.c if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { qp 1952 drivers/infiniband/hw/hfi1/rc.c qp->r_flags |= RVT_R_RDMAR_SEQ; qp 1953 drivers/infiniband/hw/hfi1/rc.c hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); qp 1954 drivers/infiniband/hw/hfi1/rc.c if (list_empty(&qp->rspwait)) { qp 1955 drivers/infiniband/hw/hfi1/rc.c qp->r_flags |= RVT_R_RSP_SEND; qp 1956 drivers/infiniband/hw/hfi1/rc.c rvt_get_qp(qp); qp 1957 drivers/infiniband/hw/hfi1/rc.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 1973 drivers/infiniband/hw/hfi1/rc.c static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn, qp 1976 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *qpriv = qp->priv; qp 1978 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = psn + 1; qp 1986 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = qpriv->s_tid_cur + 1; qp 1987 drivers/infiniband/hw/hfi1/rc.c if (qp->s_cur >= qp->s_size) qp 1988 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = 0; qp 1989 drivers/infiniband/hw/hfi1/rc.c qp->s_state = TID_OP(WRITE_REQ); qp 1991 drivers/infiniband/hw/hfi1/rc.c qp->s_cur = qpriv->s_tid_cur; qp 1992 drivers/infiniband/hw/hfi1/rc.c qp->s_state = TID_OP(WRITE_RESP); qp 2007 drivers/infiniband/hw/hfi1/rc.c int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, qp 2012 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *qpriv = qp->priv; qp 2019 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 2029 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 2066 drivers/infiniband/hw/hfi1/rc.c (delta_psn(psn, qp->s_last_psn) != 1))) { qp 2067 drivers/infiniband/hw/hfi1/rc.c set_restart_qp(qp, rcd); qp 2080 drivers/infiniband/hw/hfi1/rc.c opfn_conn_reply(qp, val); qp 2082 drivers/infiniband/hw/hfi1/rc.c if (qp->s_num_rd_atomic && qp 2086 drivers/infiniband/hw/hfi1/rc.c qp->s_num_rd_atomic--; qp 2088 drivers/infiniband/hw/hfi1/rc.c if ((qp->s_flags & RVT_S_WAIT_FENCE) && qp 2089 drivers/infiniband/hw/hfi1/rc.c !qp->s_num_rd_atomic) { qp 2090 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~(RVT_S_WAIT_FENCE | qp 2092 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 2093 drivers/infiniband/hw/hfi1/rc.c } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { qp 2094 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~(RVT_S_WAIT_RDMAR | qp 2096 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 2107 drivers/infiniband/hw/hfi1/rc.c wqe = do_rc_completion(qp, wqe, ibp); qp 2108 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked == qp->s_tail) qp 2112 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_rc_ack_do(qp, aeth, psn, wqe); qp 2113 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_sender_do_rc_ack(qp); qp 2119 drivers/infiniband/hw/hfi1/rc.c rvt_mod_retry_timer_ext(qp, qp 2122 drivers/infiniband/hw/hfi1/rc.c rvt_stop_rc_timers(qp); qp 2123 drivers/infiniband/hw/hfi1/rc.c } else if (qp->s_acked != qp->s_tail) { qp 2127 drivers/infiniband/hw/hfi1/rc.c __w = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); qp 2147 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(psn, qp->s_last_psn + 1)) { qp 2148 drivers/infiniband/hw/hfi1/rc.c set_restart_qp(qp, rcd); qp 2155 drivers/infiniband/hw/hfi1/rc.c if (qp->s_cur != qp->s_tail && qp 2156 drivers/infiniband/hw/hfi1/rc.c cmp_psn(qp->s_psn, psn) <= 0) qp 2157 drivers/infiniband/hw/hfi1/rc.c update_qp_retry_state(qp, psn, qp 2161 drivers/infiniband/hw/hfi1/rc.c rvt_mod_retry_timer(qp); qp 2163 drivers/infiniband/hw/hfi1/rc.c rvt_stop_rc_timers(qp); qp 2169 drivers/infiniband/hw/hfi1/rc.c rvt_mod_retry_timer(qp); qp 2175 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(qp->s_psn, psn) <= 0) qp 2176 drivers/infiniband/hw/hfi1/rc.c reset_psn(qp, psn + 1); qp 2180 drivers/infiniband/hw/hfi1/rc.c rvt_stop_rc_timers(qp); qp 2181 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(qp->s_psn, psn) <= 0) { qp 2182 drivers/infiniband/hw/hfi1/rc.c qp->s_state = OP(SEND_LAST); qp 2183 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = psn + 1; qp 2186 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & RVT_S_WAIT_ACK) { qp 2187 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~RVT_S_WAIT_ACK; qp 2188 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 2190 drivers/infiniband/hw/hfi1/rc.c rvt_get_credit(qp, aeth); qp 2191 drivers/infiniband/hw/hfi1/rc.c qp->s_rnr_retry = qp->s_rnr_retry_cnt; qp 2192 drivers/infiniband/hw/hfi1/rc.c qp->s_retry = qp->s_retry_cnt; qp 2202 drivers/infiniband/hw/hfi1/rc.c update_last_psn(qp, psn); qp 2207 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked == qp->s_tail) qp 2209 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & RVT_S_WAIT_RNR) qp 2211 drivers/infiniband/hw/hfi1/rc.c rdi = ib_to_rvt(qp->ibqp.device); qp 2214 drivers/infiniband/hw/hfi1/rc.c if (qp->s_rnr_retry == 0) { qp 2218 drivers/infiniband/hw/hfi1/rc.c if (qp->s_rnr_retry_cnt < 7 && qp->s_rnr_retry_cnt > 0) qp 2219 drivers/infiniband/hw/hfi1/rc.c qp->s_rnr_retry--; qp 2229 drivers/infiniband/hw/hfi1/rc.c reset_psn(qp, qp->s_last_psn + 1); qp 2231 drivers/infiniband/hw/hfi1/rc.c update_last_psn(qp, psn - 1); qp 2232 drivers/infiniband/hw/hfi1/rc.c reset_psn(qp, psn); qp 2235 drivers/infiniband/hw/hfi1/rc.c ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); qp 2236 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); qp 2237 drivers/infiniband/hw/hfi1/rc.c rvt_stop_rc_timers(qp); qp 2238 drivers/infiniband/hw/hfi1/rc.c rvt_add_rnr_timer(qp, aeth); qp 2242 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked == qp->s_tail) qp 2245 drivers/infiniband/hw/hfi1/rc.c update_last_psn(qp, psn - 1); qp 2256 drivers/infiniband/hw/hfi1/rc.c hfi1_restart_rc(qp, psn, 0); qp 2257 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 2274 drivers/infiniband/hw/hfi1/rc.c if (qp->s_last == qp->s_acked) { qp 2276 drivers/infiniband/hw/hfi1/rc.c hfi1_kern_read_tid_flow_free(qp); qp 2278 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, status); qp 2279 drivers/infiniband/hw/hfi1/rc.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 2287 drivers/infiniband/hw/hfi1/rc.c qp->s_retry = qp->s_retry_cnt; qp 2288 drivers/infiniband/hw/hfi1/rc.c qp->s_rnr_retry = qp->s_rnr_retry_cnt; qp 2298 drivers/infiniband/hw/hfi1/rc.c rvt_stop_rc_timers(qp); qp 2306 drivers/infiniband/hw/hfi1/rc.c static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn, qp 2311 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->s_lock); qp 2313 drivers/infiniband/hw/hfi1/rc.c rvt_stop_rc_timers(qp); qp 2315 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 2324 drivers/infiniband/hw/hfi1/rc.c wqe = do_rc_completion(qp, wqe, ibp); qp 2328 drivers/infiniband/hw/hfi1/rc.c qp->r_flags |= RVT_R_RDMAR_SEQ; qp 2329 drivers/infiniband/hw/hfi1/rc.c hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); qp 2330 drivers/infiniband/hw/hfi1/rc.c if (list_empty(&qp->rspwait)) { qp 2331 drivers/infiniband/hw/hfi1/rc.c qp->r_flags |= RVT_R_RSP_SEND; qp 2332 drivers/infiniband/hw/hfi1/rc.c rvt_get_qp(qp); qp 2333 drivers/infiniband/hw/hfi1/rc.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 2350 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp = packet->qp; qp 2360 drivers/infiniband/hw/hfi1/rc.c u32 pmtu = qp->pmtu; qp 2366 drivers/infiniband/hw/hfi1/rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 2367 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_ack(qp, psn); qp 2370 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0) qp 2374 drivers/infiniband/hw/hfi1/rc.c diff = cmp_psn(psn, qp->s_last_psn); qp 2380 drivers/infiniband/hw/hfi1/rc.c rvt_get_credit(qp, aeth); qp 2389 drivers/infiniband/hw/hfi1/rc.c if (qp->r_flags & RVT_R_RDMAR_SEQ) { qp 2390 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(psn, qp->s_last_psn + 1) != 0) qp 2392 drivers/infiniband/hw/hfi1/rc.c qp->r_flags &= ~RVT_R_RDMAR_SEQ; qp 2395 drivers/infiniband/hw/hfi1/rc.c if (unlikely(qp->s_acked == qp->s_tail)) qp 2397 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 2409 drivers/infiniband/hw/hfi1/rc.c if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || qp 2412 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 2420 drivers/infiniband/hw/hfi1/rc.c qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, qp 2426 drivers/infiniband/hw/hfi1/rc.c if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) qp 2433 drivers/infiniband/hw/hfi1/rc.c if (unlikely(pmtu >= qp->s_rdma_read_len)) qp 2440 drivers/infiniband/hw/hfi1/rc.c rvt_mod_retry_timer(qp); qp 2441 drivers/infiniband/hw/hfi1/rc.c if (qp->s_flags & RVT_S_WAIT_ACK) { qp 2442 drivers/infiniband/hw/hfi1/rc.c qp->s_flags &= ~RVT_S_WAIT_ACK; qp 2443 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 2447 drivers/infiniband/hw/hfi1/rc.c qp->s_retry = qp->s_retry_cnt; qp 2453 drivers/infiniband/hw/hfi1/rc.c qp->s_rdma_read_len -= pmtu; qp 2454 drivers/infiniband/hw/hfi1/rc.c update_last_psn(qp, psn); qp 2455 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2456 drivers/infiniband/hw/hfi1/rc.c rvt_copy_sge(qp, &qp->s_rdma_read_sge, qp 2462 drivers/infiniband/hw/hfi1/rc.c if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) qp 2475 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 2476 drivers/infiniband/hw/hfi1/rc.c qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, qp 2482 drivers/infiniband/hw/hfi1/rc.c if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) qp 2494 drivers/infiniband/hw/hfi1/rc.c if (unlikely(tlen != qp->s_rdma_read_len)) qp 2497 drivers/infiniband/hw/hfi1/rc.c rvt_copy_sge(qp, &qp->s_rdma_read_sge, qp 2499 drivers/infiniband/hw/hfi1/rc.c WARN_ON(qp->s_rdma_read_sge.num_sge); qp 2500 drivers/infiniband/hw/hfi1/rc.c (void)do_rc_ack(qp, aeth, psn, qp 2511 drivers/infiniband/hw/hfi1/rc.c rdma_seq_err(qp, ibp, psn, rcd); qp 2517 drivers/infiniband/hw/hfi1/rc.c if (qp->s_last == qp->s_acked) { qp 2518 drivers/infiniband/hw/hfi1/rc.c rvt_send_complete(qp, wqe, status); qp 2519 drivers/infiniband/hw/hfi1/rc.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 2522 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2527 drivers/infiniband/hw/hfi1/rc.c static inline void rc_cancel_ack(struct rvt_qp *qp) qp 2529 drivers/infiniband/hw/hfi1/rc.c qp->r_adefered = 0; qp 2530 drivers/infiniband/hw/hfi1/rc.c if (list_empty(&qp->rspwait)) qp 2532 drivers/infiniband/hw/hfi1/rc.c list_del_init(&qp->rspwait); qp 2533 drivers/infiniband/hw/hfi1/rc.c qp->r_flags &= ~RVT_R_RSP_NAK; qp 2534 drivers/infiniband/hw/hfi1/rc.c rvt_put_qp(qp); qp 2553 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp, u32 opcode, u32 psn, qp 2563 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_rcv_error(qp, psn); qp 2570 drivers/infiniband/hw/hfi1/rc.c if (!qp->r_nak_state) { qp 2572 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = IB_NAK_PSN_ERROR; qp 2574 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = qp->r_psn; qp 2580 drivers/infiniband/hw/hfi1/rc.c rc_defered_ack(rcd, qp); qp 2605 drivers/infiniband/hw/hfi1/rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 2607 drivers/infiniband/hw/hfi1/rc.c e = find_prev_entry(qp, psn, &prev, &mra, &old_req); qp 2630 drivers/infiniband/hw/hfi1/rc.c offset = delta_psn(psn, e->psn) * qp->pmtu; qp 2640 drivers/infiniband/hw/hfi1/rc.c ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, qp 2652 drivers/infiniband/hw/hfi1/rc.c if (qp->s_acked_ack_queue == qp->s_tail_ack_queue) qp 2653 drivers/infiniband/hw/hfi1/rc.c qp->s_acked_ack_queue = prev; qp 2654 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue = prev; qp 2667 drivers/infiniband/hw/hfi1/rc.c if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) qp 2668 drivers/infiniband/hw/hfi1/rc.c qp->s_acked_ack_queue = prev; qp 2669 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue = prev; qp 2684 drivers/infiniband/hw/hfi1/rc.c if (mra == qp->r_head_ack_queue) { qp 2685 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2686 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = 0; qp 2687 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = qp->r_psn - 1; qp 2695 drivers/infiniband/hw/hfi1/rc.c if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) qp 2696 drivers/infiniband/hw/hfi1/rc.c qp->s_acked_ack_queue = mra; qp 2697 drivers/infiniband/hw/hfi1/rc.c qp->s_tail_ack_queue = mra; qp 2700 drivers/infiniband/hw/hfi1/rc.c qp->s_ack_state = OP(ACKNOWLEDGE); qp 2701 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_RESP_PENDING; qp 2702 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = 0; qp 2703 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 2706 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2810 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp = packet->qp; qp 2811 drivers/infiniband/hw/hfi1/rc.c struct hfi1_qp_priv *qpriv = qp->priv; qp 2819 drivers/infiniband/hw/hfi1/rc.c u32 pmtu = qp->pmtu; qp 2828 drivers/infiniband/hw/hfi1/rc.c lockdep_assert_held(&qp->r_lock); qp 2833 drivers/infiniband/hw/hfi1/rc.c fecn = process_ecn(qp, packet); qp 2834 drivers/infiniband/hw/hfi1/rc.c opfn_trigger_conn_request(qp, be32_to_cpu(ohdr->bth[1])); qp 2849 drivers/infiniband/hw/hfi1/rc.c diff = delta_psn(psn, qp->r_psn); qp 2851 drivers/infiniband/hw/hfi1/rc.c if (rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) qp 2857 drivers/infiniband/hw/hfi1/rc.c switch (qp->r_state) { qp 2892 drivers/infiniband/hw/hfi1/rc.c if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) qp 2893 drivers/infiniband/hw/hfi1/rc.c rvt_comm_est(qp); qp 2898 drivers/infiniband/hw/hfi1/rc.c ret = rvt_get_rwqe(qp, false); qp 2903 drivers/infiniband/hw/hfi1/rc.c qp->r_rcv_len = 0; qp 2916 drivers/infiniband/hw/hfi1/rc.c qp->r_rcv_len += pmtu; qp 2917 drivers/infiniband/hw/hfi1/rc.c if (unlikely(qp->r_rcv_len > qp->r_len)) qp 2919 drivers/infiniband/hw/hfi1/rc.c rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); qp 2924 drivers/infiniband/hw/hfi1/rc.c ret = rvt_get_rwqe(qp, true); qp 2934 drivers/infiniband/hw/hfi1/rc.c ret = rvt_get_rwqe(qp, false); qp 2939 drivers/infiniband/hw/hfi1/rc.c qp->r_rcv_len = 0; qp 2953 drivers/infiniband/hw/hfi1/rc.c if (rvt_invalidate_rkey(qp, rkey)) qp 2959 drivers/infiniband/hw/hfi1/rc.c copy_last = rvt_is_user_qp(qp); qp 2972 drivers/infiniband/hw/hfi1/rc.c wc.byte_len = tlen + qp->r_rcv_len; qp 2973 drivers/infiniband/hw/hfi1/rc.c if (unlikely(wc.byte_len > qp->r_len)) qp 2975 drivers/infiniband/hw/hfi1/rc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last); qp 2976 drivers/infiniband/hw/hfi1/rc.c rvt_put_ss(&qp->r_sge); qp 2977 drivers/infiniband/hw/hfi1/rc.c qp->r_msn++; qp 2978 drivers/infiniband/hw/hfi1/rc.c if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) qp 2980 drivers/infiniband/hw/hfi1/rc.c wc.wr_id = qp->r_wr_id; qp 2987 drivers/infiniband/hw/hfi1/rc.c wc.qp = &qp->ibqp; qp 2988 drivers/infiniband/hw/hfi1/rc.c wc.src_qp = qp->remote_qpn; qp 2989 drivers/infiniband/hw/hfi1/rc.c wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; qp 3001 drivers/infiniband/hw/hfi1/rc.c wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); qp 3008 drivers/infiniband/hw/hfi1/rc.c rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); qp 3012 drivers/infiniband/hw/hfi1/rc.c copy_last = rvt_is_user_qp(qp); qp 3016 drivers/infiniband/hw/hfi1/rc.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) qp 3020 drivers/infiniband/hw/hfi1/rc.c qp->r_len = be32_to_cpu(reth->length); qp 3021 drivers/infiniband/hw/hfi1/rc.c qp->r_rcv_len = 0; qp 3022 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.sg_list = NULL; qp 3023 drivers/infiniband/hw/hfi1/rc.c if (qp->r_len != 0) { qp 3029 drivers/infiniband/hw/hfi1/rc.c ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, qp 3033 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.num_sge = 1; qp 3035 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.num_sge = 0; qp 3036 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.sge.mr = NULL; qp 3037 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.sge.vaddr = NULL; qp 3038 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.sge.length = 0; qp 3039 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.sge.sge_length = 0; qp 3045 drivers/infiniband/hw/hfi1/rc.c ret = rvt_get_rwqe(qp, true); qp 3050 drivers/infiniband/hw/hfi1/rc.c rvt_put_ss(&qp->r_sge); qp 3062 drivers/infiniband/hw/hfi1/rc.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) qp 3064 drivers/infiniband/hw/hfi1/rc.c next = qp->r_head_ack_queue + 1; qp 3066 drivers/infiniband/hw/hfi1/rc.c if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) qp 3068 drivers/infiniband/hw/hfi1/rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 3069 drivers/infiniband/hw/hfi1/rc.c if (unlikely(next == qp->s_acked_ack_queue)) { qp 3070 drivers/infiniband/hw/hfi1/rc.c if (!qp->s_ack_queue[next].sent) qp 3072 drivers/infiniband/hw/hfi1/rc.c update_ack_queue(qp, next); qp 3074 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->r_head_ack_queue]; qp 3084 drivers/infiniband/hw/hfi1/rc.c ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, qp 3092 drivers/infiniband/hw/hfi1/rc.c qp->r_psn += rvt_div_mtu(qp, len - 1); qp 3102 drivers/infiniband/hw/hfi1/rc.c e->lpsn = qp->r_psn; qp 3108 drivers/infiniband/hw/hfi1/rc.c qp->r_msn++; qp 3109 drivers/infiniband/hw/hfi1/rc.c qp->r_psn++; qp 3110 drivers/infiniband/hw/hfi1/rc.c qp->r_state = opcode; qp 3111 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = 0; qp 3112 drivers/infiniband/hw/hfi1/rc.c qp->r_head_ack_queue = next; qp 3113 drivers/infiniband/hw/hfi1/rc.c qpriv->r_tid_alloc = qp->r_head_ack_queue; qp 3116 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_RESP_PENDING; qp 3118 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_ECN; qp 3119 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 3121 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 3137 drivers/infiniband/hw/hfi1/rc.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) && qp 3140 drivers/infiniband/hw/hfi1/rc.c next = qp->r_head_ack_queue + 1; qp 3141 drivers/infiniband/hw/hfi1/rc.c if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) qp 3143 drivers/infiniband/hw/hfi1/rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 3144 drivers/infiniband/hw/hfi1/rc.c if (unlikely(next == qp->s_acked_ack_queue)) { qp 3145 drivers/infiniband/hw/hfi1/rc.c if (!qp->s_ack_queue[next].sent) qp 3147 drivers/infiniband/hw/hfi1/rc.c update_ack_queue(qp, next); qp 3149 drivers/infiniband/hw/hfi1/rc.c e = &qp->s_ack_queue[qp->r_head_ack_queue]; qp 3153 drivers/infiniband/hw/hfi1/rc.c opfn_conn_response(qp, e, ateth); qp 3160 drivers/infiniband/hw/hfi1/rc.c if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), qp 3165 drivers/infiniband/hw/hfi1/rc.c maddr = (atomic64_t *)qp->r_sge.sge.vaddr; qp 3169 drivers/infiniband/hw/hfi1/rc.c (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr, qp 3172 drivers/infiniband/hw/hfi1/rc.c rvt_put_mr(qp->r_sge.sge.mr); qp 3173 drivers/infiniband/hw/hfi1/rc.c qp->r_sge.num_sge = 0; qp 3179 drivers/infiniband/hw/hfi1/rc.c qp->r_msn++; qp 3180 drivers/infiniband/hw/hfi1/rc.c qp->r_psn++; qp 3181 drivers/infiniband/hw/hfi1/rc.c qp->r_state = opcode; qp 3182 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = 0; qp 3183 drivers/infiniband/hw/hfi1/rc.c qp->r_head_ack_queue = next; qp 3184 drivers/infiniband/hw/hfi1/rc.c qpriv->r_tid_alloc = qp->r_head_ack_queue; qp 3187 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_RESP_PENDING; qp 3189 drivers/infiniband/hw/hfi1/rc.c qp->s_flags |= RVT_S_ECN; qp 3190 drivers/infiniband/hw/hfi1/rc.c hfi1_schedule_send(qp); qp 3192 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 3200 drivers/infiniband/hw/hfi1/rc.c qp->r_psn++; qp 3201 drivers/infiniband/hw/hfi1/rc.c qp->r_state = opcode; qp 3202 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = psn; qp 3203 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = 0; qp 3207 drivers/infiniband/hw/hfi1/rc.c qp->r_adefered >= HFI1_PSN_CREDIT) { qp 3208 drivers/infiniband/hw/hfi1/rc.c rc_cancel_ack(qp); qp 3211 drivers/infiniband/hw/hfi1/rc.c qp->r_adefered++; qp 3212 drivers/infiniband/hw/hfi1/rc.c rc_defered_ack(rcd, qp); qp 3217 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = qp->r_min_rnr_timer | IB_RNR_NAK; qp 3218 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = qp->r_psn; qp 3220 drivers/infiniband/hw/hfi1/rc.c rc_defered_ack(rcd, qp); qp 3224 drivers/infiniband/hw/hfi1/rc.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 3225 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; qp 3226 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = qp->r_psn; qp 3228 drivers/infiniband/hw/hfi1/rc.c rc_defered_ack(rcd, qp); qp 3232 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 3234 drivers/infiniband/hw/hfi1/rc.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 3235 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = IB_NAK_INVALID_REQUEST; qp 3236 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = qp->r_psn; qp 3238 drivers/infiniband/hw/hfi1/rc.c rc_defered_ack(rcd, qp); qp 3242 drivers/infiniband/hw/hfi1/rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 3244 drivers/infiniband/hw/hfi1/rc.c rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); qp 3245 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; qp 3246 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = qp->r_psn; qp 3254 drivers/infiniband/hw/hfi1/rc.c struct rvt_qp *qp) qp 3269 drivers/infiniband/hw/hfi1/rc.c diff = delta_psn(psn, qp->r_psn); qp 3270 drivers/infiniband/hw/hfi1/rc.c if (!qp->r_nak_state && diff >= 0) { qp 3272 drivers/infiniband/hw/hfi1/rc.c qp->r_nak_state = IB_NAK_PSN_ERROR; qp 3274 drivers/infiniband/hw/hfi1/rc.c qp->r_ack_psn = qp->r_psn; qp 3283 drivers/infiniband/hw/hfi1/rc.c rc_defered_ack(rcd, qp); qp 13 drivers/infiniband/hw/hfi1/rc.h static inline void update_ack_queue(struct rvt_qp *qp, unsigned int n) qp 18 drivers/infiniband/hw/hfi1/rc.h if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) qp 20 drivers/infiniband/hw/hfi1/rc.h qp->s_tail_ack_queue = next; qp 21 drivers/infiniband/hw/hfi1/rc.h qp->s_acked_ack_queue = next; qp 22 drivers/infiniband/hw/hfi1/rc.h qp->s_ack_state = OP(ACKNOWLEDGE); qp 26 drivers/infiniband/hw/hfi1/rc.h struct rvt_qp *qp) qp 28 drivers/infiniband/hw/hfi1/rc.h if (list_empty(&qp->rspwait)) { qp 29 drivers/infiniband/hw/hfi1/rc.h qp->r_flags |= RVT_R_RSP_NAK; qp 30 drivers/infiniband/hw/hfi1/rc.h rvt_get_qp(qp); qp 31 drivers/infiniband/hw/hfi1/rc.h list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 52 drivers/infiniband/hw/hfi1/rc.h struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev, qp 54 drivers/infiniband/hw/hfi1/rc.h int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, u64 val, qp 56 drivers/infiniband/hw/hfi1/rc.h struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 73 drivers/infiniband/hw/hfi1/ruc.c struct rvt_qp *qp = packet->qp; qp 74 drivers/infiniband/hw/hfi1/ruc.c u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; qp 81 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_mig_state == IB_MIG_ARMED && migrated) { qp 83 drivers/infiniband/hw/hfi1/ruc.c if ((rdma_ah_get_ah_flags(&qp->alt_ah_attr) & qp 90 drivers/infiniband/hw/hfi1/ruc.c if (!(rdma_ah_get_ah_flags(&qp->alt_ah_attr) & qp 93 drivers/infiniband/hw/hfi1/ruc.c grh = rdma_ah_read_grh(&qp->alt_ah_attr); qp 106 drivers/infiniband/hw/hfi1/ruc.c hfi1_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num, qp 111 drivers/infiniband/hw/hfi1/ruc.c if (slid != rdma_ah_get_dlid(&qp->alt_ah_attr) || qp 113 drivers/infiniband/hw/hfi1/ruc.c rdma_ah_get_port_num(&qp->alt_ah_attr)) qp 115 drivers/infiniband/hw/hfi1/ruc.c spin_lock_irqsave(&qp->s_lock, flags); qp 116 drivers/infiniband/hw/hfi1/ruc.c hfi1_migrate_qp(qp); qp 117 drivers/infiniband/hw/hfi1/ruc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 120 drivers/infiniband/hw/hfi1/ruc.c if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & qp 127 drivers/infiniband/hw/hfi1/ruc.c if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & qp 130 drivers/infiniband/hw/hfi1/ruc.c grh = rdma_ah_read_grh(&qp->remote_ah_attr); qp 143 drivers/infiniband/hw/hfi1/ruc.c hfi1_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num, qp 148 drivers/infiniband/hw/hfi1/ruc.c if ((slid != rdma_ah_get_dlid(&qp->remote_ah_attr)) || qp 149 drivers/infiniband/hw/hfi1/ruc.c ppd_from_ibp(ibp)->port != qp->port_num) qp 151 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_mig_state == IB_MIG_REARM && !migrated) qp 152 drivers/infiniband/hw/hfi1/ruc.c qp->s_mig_state = IB_MIG_ARMED; qp 205 drivers/infiniband/hw/hfi1/ruc.c static inline void build_ahg(struct rvt_qp *qp, u32 npsn) qp 207 drivers/infiniband/hw/hfi1/ruc.c struct hfi1_qp_priv *priv = qp->priv; qp 209 drivers/infiniband/hw/hfi1/ruc.c if (unlikely(qp->s_flags & HFI1_S_AHG_CLEAR)) qp 210 drivers/infiniband/hw/hfi1/ruc.c clear_ahg(qp); qp 211 drivers/infiniband/hw/hfi1/ruc.c if (!(qp->s_flags & HFI1_S_AHG_VALID)) { qp 213 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_ahgidx < 0) qp 214 drivers/infiniband/hw/hfi1/ruc.c qp->s_ahgidx = sdma_ahg_alloc(priv->s_sde); qp 215 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_ahgidx >= 0) { qp 216 drivers/infiniband/hw/hfi1/ruc.c qp->s_ahgpsn = npsn; qp 219 drivers/infiniband/hw/hfi1/ruc.c priv->s_ahg->ahgidx = qp->s_ahgidx; qp 220 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags |= HFI1_S_AHG_VALID; qp 224 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_ahgidx >= 0) { qp 226 drivers/infiniband/hw/hfi1/ruc.c priv->s_ahg->ahgidx = qp->s_ahgidx; qp 235 drivers/infiniband/hw/hfi1/ruc.c (qp->s_ahgpsn & 0xffff0000)) { qp 249 drivers/infiniband/hw/hfi1/ruc.c static inline void hfi1_make_ruc_bth(struct rvt_qp *qp, qp 272 drivers/infiniband/hw/hfi1/ruc.c static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, qp 278 drivers/infiniband/hw/hfi1/ruc.c struct hfi1_qp_priv *priv = qp->priv; qp 282 drivers/infiniband/hw/hfi1/ruc.c u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); qp 291 drivers/infiniband/hw/hfi1/ruc.c if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && qp 292 drivers/infiniband/hw/hfi1/ruc.c hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { qp 295 drivers/infiniband/hw/hfi1/ruc.c rdma_ah_retrieve_grh(&qp->remote_ah_attr); qp 311 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_mig_state == IB_MIG_MIGRATED) qp 316 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_flags & RVT_S_ECN) { qp 317 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags &= ~RVT_S_ECN; qp 323 drivers/infiniband/hw/hfi1/ruc.c build_ahg(qp, bth2); qp 325 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags &= ~HFI1_S_AHG_VALID; qp 329 drivers/infiniband/hw/hfi1/ruc.c hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); qp 335 drivers/infiniband/hw/hfi1/ruc.c (rdma_ah_get_path_bits(&qp->remote_ah_attr) & qp 340 drivers/infiniband/hw/hfi1/ruc.c opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), qp 360 drivers/infiniband/hw/hfi1/ruc.c static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp, qp 366 drivers/infiniband/hw/hfi1/ruc.c struct hfi1_qp_priv *priv = qp->priv; qp 368 drivers/infiniband/hw/hfi1/ruc.c u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); qp 374 drivers/infiniband/hw/hfi1/ruc.c if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { qp 380 drivers/infiniband/hw/hfi1/ruc.c rdma_ah_read_grh(&qp->remote_ah_attr), qp 386 drivers/infiniband/hw/hfi1/ruc.c (rdma_ah_get_sl(&qp->remote_ah_attr) & 0xf) << 4; qp 388 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_mig_state == IB_MIG_MIGRATED) qp 393 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_flags & RVT_S_ECN) { qp 394 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags &= ~RVT_S_ECN; qp 400 drivers/infiniband/hw/hfi1/ruc.c build_ahg(qp, bth2); qp 402 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags &= ~HFI1_S_AHG_VALID; qp 406 drivers/infiniband/hw/hfi1/ruc.c hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); qp 410 drivers/infiniband/hw/hfi1/ruc.c opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), qp 412 drivers/infiniband/hw/hfi1/ruc.c rdma_ah_get_path_bits(&qp->remote_ah_attr)); qp 415 drivers/infiniband/hw/hfi1/ruc.c typedef void (*hfi1_make_ruc_hdr)(struct rvt_qp *qp, qp 426 drivers/infiniband/hw/hfi1/ruc.c void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, qp 430 drivers/infiniband/hw/hfi1/ruc.c struct hfi1_qp_priv *priv = qp->priv; qp 448 drivers/infiniband/hw/hfi1/ruc.c hfi1_ruc_header_tbl[priv->hdr_type](qp, ohdr, bth0, bth1, bth2, middle, qp 470 drivers/infiniband/hw/hfi1/ruc.c bool hfi1_schedule_send_yield(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 478 drivers/infiniband/hw/hfi1/ruc.c spin_lock_irqsave(&qp->s_lock, ps->flags); qp 480 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags &= ~RVT_S_BUSY; qp 481 drivers/infiniband/hw/hfi1/ruc.c hfi1_schedule_send(qp); qp 483 drivers/infiniband/hw/hfi1/ruc.c struct hfi1_qp_priv *priv = qp->priv; qp 487 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags &= ~RVT_S_BUSY; qp 494 drivers/infiniband/hw/hfi1/ruc.c hfi1_schedule_tid_send(qp); qp 497 drivers/infiniband/hw/hfi1/ruc.c spin_unlock_irqrestore(&qp->s_lock, ps->flags); qp 499 drivers/infiniband/hw/hfi1/ruc.c trace_hfi1_rc_expired_time_slice(qp, true); qp 508 drivers/infiniband/hw/hfi1/ruc.c trace_hfi1_rc_expired_time_slice(qp, false); qp 512 drivers/infiniband/hw/hfi1/ruc.c void hfi1_do_send_from_rvt(struct rvt_qp *qp) qp 514 drivers/infiniband/hw/hfi1/ruc.c hfi1_do_send(qp, false); qp 520 drivers/infiniband/hw/hfi1/ruc.c struct rvt_qp *qp = iowait_to_qp(w->iow); qp 522 drivers/infiniband/hw/hfi1/ruc.c hfi1_do_send(qp, true); qp 534 drivers/infiniband/hw/hfi1/ruc.c void hfi1_do_send(struct rvt_qp *qp, bool in_thread) qp 537 drivers/infiniband/hw/hfi1/ruc.c struct hfi1_qp_priv *priv = qp->priv; qp 538 drivers/infiniband/hw/hfi1/ruc.c int (*make_req)(struct rvt_qp *qp, struct hfi1_pkt_state *ps); qp 540 drivers/infiniband/hw/hfi1/ruc.c ps.dev = to_idev(qp->ibqp.device); qp 541 drivers/infiniband/hw/hfi1/ruc.c ps.ibp = to_iport(qp->ibqp.device, qp->port_num); qp 546 drivers/infiniband/hw/hfi1/ruc.c trace_hfi1_rc_do_send(qp, in_thread); qp 548 drivers/infiniband/hw/hfi1/ruc.c switch (qp->ibqp.qp_type) { qp 550 drivers/infiniband/hw/hfi1/ruc.c if (!loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) & qp 553 drivers/infiniband/hw/hfi1/ruc.c rvt_ruc_loopback(qp); qp 557 drivers/infiniband/hw/hfi1/ruc.c ps.timeout_int = qp->timeout_jiffies; qp 560 drivers/infiniband/hw/hfi1/ruc.c if (!loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) & qp 563 drivers/infiniband/hw/hfi1/ruc.c rvt_ruc_loopback(qp); qp 574 drivers/infiniband/hw/hfi1/ruc.c spin_lock_irqsave(&qp->s_lock, ps.flags); qp 577 drivers/infiniband/hw/hfi1/ruc.c if (!hfi1_send_ok(qp)) { qp 578 drivers/infiniband/hw/hfi1/ruc.c if (qp->s_flags & HFI1_S_ANY_WAIT_IO) qp 580 drivers/infiniband/hw/hfi1/ruc.c spin_unlock_irqrestore(&qp->s_lock, ps.flags); qp 584 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags |= RVT_S_BUSY; qp 598 drivers/infiniband/hw/hfi1/ruc.c qp->s_flags |= RVT_S_BUSY; qp 599 drivers/infiniband/hw/hfi1/ruc.c spin_unlock_irqrestore(&qp->s_lock, ps.flags); qp 604 drivers/infiniband/hw/hfi1/ruc.c if (hfi1_verbs_send(qp, &ps)) qp 608 drivers/infiniband/hw/hfi1/ruc.c if (hfi1_schedule_send_yield(qp, &ps, false)) qp 611 drivers/infiniband/hw/hfi1/ruc.c spin_lock_irqsave(&qp->s_lock, ps.flags); qp 613 drivers/infiniband/hw/hfi1/ruc.c } while (make_req(qp, &ps)); qp 615 drivers/infiniband/hw/hfi1/ruc.c spin_unlock_irqrestore(&qp->s_lock, ps.flags); qp 114 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_init_trdma_req(struct rvt_qp *qp, qp 116 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx); qp 118 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_add_tid_reap_timer(struct rvt_qp *qp); qp 119 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp); qp 120 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp); qp 121 drivers/infiniband/hw/hfi1/tid_rdma.c static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp); qp 123 drivers/infiniband/hw/hfi1/tid_rdma.c static int make_tid_rdma_ack(struct rvt_qp *qp, qp 126 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_do_tid_send(struct rvt_qp *qp); qp 130 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp, u32 psn, int diff, bool fecn); qp 143 drivers/infiniband/hw/hfi1/tid_rdma.c static void tid_rdma_schedule_ack(struct rvt_qp *qp) qp 145 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 148 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_tid_send(qp); qp 151 drivers/infiniband/hw/hfi1/tid_rdma.c static void tid_rdma_trigger_ack(struct rvt_qp *qp) qp 153 drivers/infiniband/hw/hfi1/tid_rdma.c validate_r_tid_ack(qp->priv); qp 154 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_schedule_ack(qp); qp 160 drivers/infiniband/hw/hfi1/tid_rdma.c (((u64)p->qp & TID_OPFN_QP_CTXT_MASK) << qp 162 drivers/infiniband/hw/hfi1/tid_rdma.c ((((u64)p->qp >> 16) & TID_OPFN_QP_KDETH_MASK) << qp 185 drivers/infiniband/hw/hfi1/tid_rdma.c p->qp = qp 193 drivers/infiniband/hw/hfi1/tid_rdma.c void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p) qp 195 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 197 drivers/infiniband/hw/hfi1/tid_rdma.c p->qp = (kdeth_qp << 16) | priv->rcd->ctxt; qp 202 drivers/infiniband/hw/hfi1/tid_rdma.c p->timeout = qp->timeout; qp 206 drivers/infiniband/hw/hfi1/tid_rdma.c bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data) qp 208 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 214 drivers/infiniband/hw/hfi1/tid_rdma.c bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data) qp 216 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 246 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_opfn_param(qp, 0, &priv->tid_rdma.local); qp 247 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_opfn_param(qp, 1, remote); qp 257 drivers/infiniband/hw/hfi1/tid_rdma.c priv->pkts_ps = (u16)rvt_div_mtu(qp, remote->max_len); qp 269 drivers/infiniband/hw/hfi1/tid_rdma.c bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data) qp 273 drivers/infiniband/hw/hfi1/tid_rdma.c ret = tid_rdma_conn_reply(qp, *data); qp 281 drivers/infiniband/hw/hfi1/tid_rdma.c (void)tid_rdma_conn_req(qp, data); qp 285 drivers/infiniband/hw/hfi1/tid_rdma.c void tid_rdma_conn_error(struct rvt_qp *qp) qp 287 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 320 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp) qp 330 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->ibqp.qp_num == 0) qp 333 drivers/infiniband/hw/hfi1/tid_rdma.c ctxt = hfi1_get_qp_map(dd, qp->ibqp.qp_num >> dd->qos_shift); qp 337 drivers/infiniband/hw/hfi1/tid_rdma.c int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 340 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 343 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->rcd = qp_to_rcd(rdi, qp); qp 375 drivers/infiniband/hw/hfi1/tid_rdma.c for (i = 0; i < qp->s_size; i++) { qp 377 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); qp 384 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_init_trdma_req(qp, &priv->tid_req); qp 396 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_init_trdma_req(qp, &priv->tid_req); qp 397 drivers/infiniband/hw/hfi1/tid_rdma.c priv->tid_req.e.ack = &qp->s_ack_queue[i]; qp 405 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_ack_queue[i].priv = priv; qp 412 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) qp 414 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 418 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->ibqp.qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) { qp 419 drivers/infiniband/hw/hfi1/tid_rdma.c for (i = 0; i < qp->s_size; i++) { qp 420 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); qp 425 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ack_priv *priv = qp->s_ack_queue[i].priv; qp 430 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_ack_queue[i].priv = NULL; qp 503 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_queue *queue, struct rvt_qp *qp) qp 504 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) qp 509 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 512 drivers/infiniband/hw/hfi1/tid_rdma.c if (!fqp || (fqp == qp && (qp->s_flags & HFI1_S_WAIT_TID_SPACE))) qp 534 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_queue *queue, struct rvt_qp *qp) qp 535 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) qp 537 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 539 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 544 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; qp 546 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_put_qp(qp); qp 560 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_queue *queue, struct rvt_qp *qp) qp 561 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) qp 563 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 565 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 568 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= HFI1_S_WAIT_TID_SPACE; qp 572 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_qpsleep(qp, HFI1_S_WAIT_TID_SPACE); qp 573 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_get_qp(qp); qp 584 drivers/infiniband/hw/hfi1/tid_rdma.c static void __trigger_tid_waiter(struct rvt_qp *qp) qp 585 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 587 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 588 drivers/infiniband/hw/hfi1/tid_rdma.c if (!(qp->s_flags & HFI1_S_WAIT_TID_SPACE)) qp 590 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_qpwakeup(qp, HFI1_S_WAIT_TID_SPACE); qp 591 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 606 drivers/infiniband/hw/hfi1/tid_rdma.c static void tid_rdma_schedule_tid_wakeup(struct rvt_qp *qp) qp 614 drivers/infiniband/hw/hfi1/tid_rdma.c if (!qp) qp 617 drivers/infiniband/hw/hfi1/tid_rdma.c priv = qp->priv; qp 618 drivers/infiniband/hw/hfi1/tid_rdma.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 620 drivers/infiniband/hw/hfi1/tid_rdma.c dd = dd_from_ibdev(qp->ibqp.device); qp 628 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_put_qp(qp); qp 642 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp; qp 646 drivers/infiniband/hw/hfi1/tid_rdma.c qp = priv->owner; qp 647 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irq(&qp->s_lock); qp 648 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_flags & HFI1_S_WAIT_TID_SPACE) { qp 649 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irq(&qp->s_lock); qp 652 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irq(&qp->s_lock); qp 654 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_put_qp(qp); qp 664 drivers/infiniband/hw/hfi1/tid_rdma.c static void _tid_rdma_flush_wait(struct rvt_qp *qp, struct tid_queue *queue) qp 665 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 669 drivers/infiniband/hw/hfi1/tid_rdma.c if (!qp) qp 671 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 672 drivers/infiniband/hw/hfi1/tid_rdma.c priv = qp->priv; qp 673 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; qp 677 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; qp 679 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_put_qp(qp); qp 684 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp) qp 685 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 687 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 689 drivers/infiniband/hw/hfi1/tid_rdma.c _tid_rdma_flush_wait(qp, &priv->rcd->flow_queue); qp 690 drivers/infiniband/hw/hfi1/tid_rdma.c _tid_rdma_flush_wait(qp, &priv->rcd->rarr_queue); qp 775 drivers/infiniband/hw/hfi1/tid_rdma.c int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp) qp 777 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; qp 788 drivers/infiniband/hw/hfi1/tid_rdma.c if (kernel_tid_waiters(rcd, &rcd->flow_queue, qp)) qp 802 drivers/infiniband/hw/hfi1/tid_rdma.c dequeue_tid_waiter(rcd, &rcd->flow_queue, qp); qp 810 drivers/infiniband/hw/hfi1/tid_rdma.c queue_qp_for_tid_wait(rcd, &rcd->flow_queue, qp); qp 815 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp) qp 817 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; qp 835 drivers/infiniband/hw/hfi1/tid_rdma.c if (fqp == qp) { qp 892 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_page(flow->req->qp, flow, 0, 0, 0, vaddr); qp 895 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 0, 0, qp 929 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_pageset(flow->req->qp, setcount, qp 1027 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 0, v0); qp 1030 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 1, v1); qp 1176 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head, qp 1185 drivers/infiniband/hw/hfi1/tid_rdma.c if (flow->req->qp->pmtu == enum_to_mtu(OPA_MTU_4096)) qp 1213 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_node_add(flow->req->qp, s, flow->tnode_cnt - 1, qp 1286 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_msg_alloc_tids(flow->req->qp, " insufficient tids: needed ", qp 1301 drivers/infiniband/hw/hfi1/tid_rdma.c u32 pmtu_pg = flow->req->qp->pmtu >> PAGE_SHIFT; qp 1340 drivers/infiniband/hw/hfi1/tid_rdma.c flow->req->qp, flow->tidcnt - 1, qp 1409 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head, flow); qp 1456 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&req->qp->s_lock) qp 1460 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = req->qp->priv; qp 1465 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&req->qp->s_lock); qp 1483 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_wait_kmem(flow->req->qp); qp 1488 drivers/infiniband/hw/hfi1/tid_rdma.c if (kernel_tid_waiters(rcd, &rcd->rarr_queue, flow->req->qp)) qp 1521 drivers/infiniband/hw/hfi1/tid_rdma.c dequeue_tid_waiter(rcd, &rcd->rarr_queue, flow->req->qp); qp 1530 drivers/infiniband/hw/hfi1/tid_rdma.c queue_qp_for_tid_wait(rcd, &rcd->rarr_queue, flow->req->qp); qp 1547 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&req->qp->s_lock) qp 1555 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&req->qp->s_lock); qp 1575 drivers/infiniband/hw/hfi1/tid_rdma.c if (fqp == req->qp) { qp 1590 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&req->qp->s_lock) qp 1614 drivers/infiniband/hw/hfi1/tid_rdma.c void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) qp 1647 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_init_trdma_req(struct rvt_qp *qp, qp 1650 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 1662 drivers/infiniband/hw/hfi1/tid_rdma.c req->qp = qp; qp 1702 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = req->qp; qp 1703 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 1712 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_build_read_pkt(qp, req->flow_idx, flow); qp 1751 drivers/infiniband/hw/hfi1/tid_rdma.c cpu_to_be32(qpriv->tid_rdma.local.qp | qp 1755 drivers/infiniband/hw/hfi1/tid_rdma.c rreq->verbs_qp = cpu_to_be32(qp->remote_qpn); qp 1757 drivers/infiniband/hw/hfi1/tid_rdma.c *bth1 |= remote->qp; qp 1764 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_state = TID_OP(READ_REQ); qp 1768 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_num_rd_atomic++; qp 1780 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 1783 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 1785 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 1791 drivers/infiniband/hw/hfi1/tid_rdma.c u32 npkts = rvt_div_round_up_mtu(qp, *len); qp 1793 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_build_read_req(qp, 0, wqe->wr.opcode, wqe->psn, qp 1804 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_kern_clear_hw_flow(req->rcd, qp); qp 1822 drivers/infiniband/hw/hfi1/tid_rdma.c restart_sge(&qp->s_sge, wqe, req->s_next_psn, qp 1823 drivers/infiniband/hw/hfi1/tid_rdma.c qp->pmtu); qp 1838 drivers/infiniband/hw/hfi1/tid_rdma.c if (hfi1_kern_setup_hw_flow(qpriv->rcd, qp)) qp 1845 drivers/infiniband/hw/hfi1/tid_rdma.c if (hfi1_kern_exp_rcv_setup(req, &qp->s_sge, &last)) { qp 1882 drivers/infiniband/hw/hfi1/tid_rdma.c static int tid_rdma_rcv_read_request(struct rvt_qp *qp, qp 1888 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 1909 drivers/infiniband/hw/hfi1/tid_rdma.c flow->npkts = rvt_div_round_up_mtu(qp, len); qp 1911 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_entry_rcv_read_req(qp, i, qp 1947 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_read_req(qp, req->setup_head, flow); qp 1973 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_read_req(qp, 0, e->opcode, e->psn, e->lpsn, qp 1980 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp, u32 psn, int diff) qp 1982 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 1983 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ctxtdata *rcd = ((struct hfi1_qp_priv *)qp->priv)->rcd; qp 1984 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); qp 1985 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 1992 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_tid_rcv_error(qp, psn); qp 1993 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_rdma_rcv_err(qp, 0, psn, diff); qp 1996 drivers/infiniband/hw/hfi1/tid_rdma.c if (!qp->r_nak_state) { qp 1998 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = IB_NAK_PSN_ERROR; qp 1999 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_ack_psn = qp->r_psn; qp 2000 drivers/infiniband/hw/hfi1/tid_rdma.c rc_defered_ack(rcd, qp); qp 2007 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 2008 drivers/infiniband/hw/hfi1/tid_rdma.c e = find_prev_entry(qp, psn, &prev, NULL, &old_req); qp 2015 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, e->lpsn, req); qp 2038 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_len = len; qp 2039 drivers/infiniband/hw/hfi1/tid_rdma.c ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, qp 2055 drivers/infiniband/hw/hfi1/tid_rdma.c if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn, qp 2089 drivers/infiniband/hw/hfi1/tid_rdma.c if (i == qp->r_head_ack_queue) qp 2091 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[i]; qp 2151 drivers/infiniband/hw/hfi1/tid_rdma.c if (i == qp->r_head_ack_queue) qp 2153 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[i]; qp 2155 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, qp 2176 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_acked_ack_queue == qp->s_tail_ack_queue) qp 2177 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_acked_ack_queue = prev; qp 2178 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_tail_ack_queue = prev; qp 2185 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_ack_state = OP(ACKNOWLEDGE); qp 2192 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_nak_state = 0; qp 2194 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_psn = e->lpsn + 1; qp 2195 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_tid_write_alloc_resources(qp, true); qp 2198 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_state = e->opcode; qp 2199 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = 0; qp 2200 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_RESP_PENDING; qp 2201 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 2203 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2224 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 2225 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 2230 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 2242 drivers/infiniband/hw/hfi1/tid_rdma.c fecn = process_ecn(qp, packet); qp 2244 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_rcv_tid_read_req(qp, psn); qp 2246 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) qp 2247 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_comm_est(qp); qp 2249 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) qp 2259 drivers/infiniband/hw/hfi1/tid_rdma.c diff = delta_psn(psn, qp->r_psn); qp 2261 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn); qp 2266 drivers/infiniband/hw/hfi1/tid_rdma.c next = qp->r_head_ack_queue + 1; qp 2267 drivers/infiniband/hw/hfi1/tid_rdma.c if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) qp 2269 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 2270 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(next == qp->s_tail_ack_queue)) { qp 2271 drivers/infiniband/hw/hfi1/tid_rdma.c if (!qp->s_ack_queue[next].sent) { qp 2275 drivers/infiniband/hw/hfi1/tid_rdma.c update_ack_queue(qp, next); qp 2277 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qp->r_head_ack_queue]; qp 2281 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_len = len; qp 2283 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr, qp 2288 drivers/infiniband/hw/hfi1/tid_rdma.c if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn, vaddr, qp 2292 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_state = e->opcode; qp 2293 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = 0; qp 2299 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_msn++; qp 2300 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_psn += e->lpsn - e->psn + 1; qp 2302 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_head_ack_queue = next; qp 2310 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->r_tid_alloc = qp->r_head_ack_queue; qp 2313 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_RESP_PENDING; qp 2315 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_ECN; qp 2316 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 2318 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2322 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2324 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 2325 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = nack_state; qp 2326 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_ack_psn = qp->r_psn; qp 2328 drivers/infiniband/hw/hfi1/tid_rdma.c rc_defered_ack(rcd, qp); qp 2331 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2332 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); qp 2333 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; qp 2334 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_ack_psn = qp->r_psn; qp 2337 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 2343 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 2353 drivers/infiniband/hw/hfi1/tid_rdma.c *len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset); qp 2358 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_entry_build_read_resp(qp, flow->tid_idx, tidentry); qp 2359 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_build_read_resp(qp, req->clear_tail, flow); qp 2375 drivers/infiniband/hw/hfi1/tid_rdma.c resp->verbs_qp = cpu_to_be32(qp->remote_qpn); qp 2378 drivers/infiniband/hw/hfi1/tid_rdma.c resp->aeth = rvt_compute_aeth(qp); qp 2408 drivers/infiniband/hw/hfi1/tid_rdma.c find_tid_request(struct rvt_qp *qp, u32 psn, enum ib_wr_opcode opcode) qp 2409 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 2415 drivers/infiniband/hw/hfi1/tid_rdma.c end = qp->s_cur + 1; qp 2416 drivers/infiniband/hw/hfi1/tid_rdma.c if (end == qp->s_size) qp 2418 drivers/infiniband/hw/hfi1/tid_rdma.c for (i = qp->s_acked; i != end;) { qp 2419 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); qp 2426 drivers/infiniband/hw/hfi1/tid_rdma.c if (++i == qp->s_size) qp 2445 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 2446 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 2455 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_sender_rcv_tid_read_resp(qp); qp 2456 drivers/infiniband/hw/hfi1/tid_rdma.c fecn = process_ecn(qp, packet); qp 2461 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 2463 drivers/infiniband/hw/hfi1/tid_rdma.c req = find_tid_request(qp, ipsn, IB_WR_TID_RDMA_READ); qp 2490 drivers/infiniband/hw/hfi1/tid_rdma.c u32 pmtu = qp->pmtu; qp 2497 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_copy_sge(qp, &ss, packet->payload, pmtu, false, qp 2508 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_num_rd_atomic--; qp 2509 drivers/infiniband/hw/hfi1/tid_rdma.c if ((qp->s_flags & RVT_S_WAIT_FENCE) && qp 2510 drivers/infiniband/hw/hfi1/tid_rdma.c !qp->s_num_rd_atomic) { qp 2511 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~(RVT_S_WAIT_FENCE | qp 2513 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 2515 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_flags & RVT_S_WAIT_RDMAR) { qp 2516 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~(RVT_S_WAIT_RDMAR | RVT_S_WAIT_ACK); qp 2517 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 2520 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_ack(qp, ipsn); qp 2521 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_read_resp(qp, 0, req->e.swqe->wr.opcode, qp 2524 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_read_resp(qp, req->clear_tail, flow); qp 2529 drivers/infiniband/hw/hfi1/tid_rdma.c if (!do_rc_ack(qp, aeth, ipsn, opcode, 0, rcd)) qp 2546 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_kern_clear_hw_flow(priv->rcd, qp); qp 2552 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 2564 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_last == qp->s_acked) qp 2565 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 2568 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2571 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp) qp 2572 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 2574 drivers/infiniband/hw/hfi1/tid_rdma.c u32 n = qp->s_acked; qp 2577 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 2579 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 2581 drivers/infiniband/hw/hfi1/tid_rdma.c while (n != qp->s_tail) { qp 2582 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, n); qp 2588 drivers/infiniband/hw/hfi1/tid_rdma.c if (++n == qp->s_size) qp 2592 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_kern_clear_hw_flow(priv->rcd, qp); qp 2597 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 2602 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock(&qp->s_lock); qp 2612 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_restart_rc(qp, qp->s_last_psn + 1, 1); qp 2613 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 2617 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock(&qp->s_lock); qp 2623 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp, struct rvt_swqe *wqe) qp 2629 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_flags |= RVT_R_RDMAR_SEQ; qp 2632 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_restart_rc(qp, flow->flow_state.ib_spsn, 0); qp 2633 drivers/infiniband/hw/hfi1/tid_rdma.c if (list_empty(&qp->rspwait)) { qp 2634 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_flags |= RVT_R_RSP_SEND; qp 2635 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_get_qp(qp); qp 2636 drivers/infiniband/hw/hfi1/tid_rdma.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 2651 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&packet->qp->r_lock) __must_hold(RCU) qp 2660 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 2661 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 2666 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->r_lock); qp 2667 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_read_kdeth_eflags(qp, ibpsn); qp 2668 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_sender_read_kdeth_eflags(qp); qp 2669 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_read_sender_kdeth_eflags(qp, 0); qp 2670 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock(&qp->s_lock); qp 2672 drivers/infiniband/hw/hfi1/tid_rdma.c if (cmp_psn(ibpsn, qp->s_last_psn) < 0 || qp 2673 drivers/infiniband/hw/hfi1/tid_rdma.c cmp_psn(ibpsn, qp->s_psn) > 0) qp 2682 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 2683 drivers/infiniband/hw/hfi1/tid_rdma.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 2697 drivers/infiniband/hw/hfi1/tid_rdma.c if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { qp 2698 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_flags |= RVT_R_RDMAR_SEQ; qp 2700 drivers/infiniband/hw/hfi1/tid_rdma.c restart_tid_rdma_read_req(rcd, qp, qp 2703 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_restart_rc(qp, qp->s_last_psn + 1, qp 2705 drivers/infiniband/hw/hfi1/tid_rdma.c if (list_empty(&qp->rspwait)) { qp 2706 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_flags |= RVT_R_RSP_SEND; qp 2707 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_get_qp(qp); qp 2709 drivers/infiniband/hw/hfi1/tid_rdma.c &qp->rspwait, qp 2721 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = do_rc_completion(qp, wqe, ibp); qp 2722 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_acked == qp->s_tail) qp 2726 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_acked == qp->s_tail) qp 2734 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe->wr.opcode, wqe->psn, qp 2750 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_read_kdeth_eflags(qp, qp 2765 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->r_flags & RVT_R_RDMAR_SEQ) qp 2766 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_flags &= qp 2782 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->r_flags & RVT_R_RDMAR_SEQ) qp 2783 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_flags &= qp 2799 drivers/infiniband/hw/hfi1/tid_rdma.c if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) qp 2800 drivers/infiniband/hw/hfi1/tid_rdma.c restart_tid_rdma_read_req(rcd, qp, qp 2833 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock(&qp->s_lock); qp 2852 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp; qp 2880 drivers/infiniband/hw/hfi1/tid_rdma.c qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); qp 2881 drivers/infiniband/hw/hfi1/tid_rdma.c if (!qp) qp 2884 drivers/infiniband/hw/hfi1/tid_rdma.c packet->qp = qp; qp 2887 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->r_lock, flags); qp 2888 drivers/infiniband/hw/hfi1/tid_rdma.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { qp 2926 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock(&qp->s_lock); qp 2927 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv = qp->priv; qp 2931 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qpriv->r_tid_tail]; qp 2938 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_eflags_err_write(qp, rcv_type, rte, psn); qp 2939 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_handle_kdeth_eflags(qp, psn); qp 2940 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_handle_kdeth_eflags(qp); qp 2941 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_handle_kdeth_eflags(qp, 0, e->opcode, e->psn, qp 2943 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_handle_kdeth_eflags(qp, req->clear_tail, flow); qp 3013 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock(&qp->s_lock); qp 3015 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 3026 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_trigger_ack(qp); qp 3037 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 3042 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3048 drivers/infiniband/hw/hfi1/tid_rdma.c *bth2 = mask_psn(qp->s_psn); qp 3052 drivers/infiniband/hw/hfi1/tid_rdma.c qp, "!!!!!! Could not find flow to restart: bth2 ", qp 3054 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, qp 3072 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_restart_req(qp, fidx, flow); qp 3086 drivers/infiniband/hw/hfi1/tid_rdma.c tidnpkts = rvt_div_round_up_mtu(qp, tidlen); qp 3090 drivers/infiniband/hw/hfi1/tid_rdma.c npkts * qp->pmtu); qp 3091 drivers/infiniband/hw/hfi1/tid_rdma.c flow->tid_offset += npkts * qp->pmtu; qp 3122 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_restart_req(qp, fidx, flow); qp 3123 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe->psn, qp 3142 drivers/infiniband/hw/hfi1/tid_rdma.c i = (++i == qp->s_size ? 0 : i); qp 3143 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); qp 3154 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp) qp 3157 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3160 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->ibqp.qp_type != IB_QPT_RC || !HFI1_CAP_IS_KSET(TID_RDMA)) qp 3169 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_kern_clear_hw_flow(qpriv->rcd, qp); qp 3171 drivers/infiniband/hw/hfi1/tid_rdma.c for (i = qp->s_acked; i != qp->s_head;) { qp 3172 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); qp 3174 drivers/infiniband/hw/hfi1/tid_rdma.c if (++i == qp->s_size) qp 3185 drivers/infiniband/hw/hfi1/tid_rdma.c for (i = qp->s_acked_ack_queue; i != qp->r_head_ack_queue;) { qp 3186 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_ack_entry *e = &qp->s_ack_queue[i]; qp 3188 drivers/infiniband/hw/hfi1/tid_rdma.c if (++i == rvt_max_atomic(ib_to_rvt(qp->ibqp.device))) qp 3201 drivers/infiniband/hw/hfi1/tid_rdma.c bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) qp 3204 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 3208 drivers/infiniband/hw/hfi1/tid_rdma.c s_prev = (qp->s_cur == 0 ? qp->s_size : qp->s_cur) - 1; qp 3209 drivers/infiniband/hw/hfi1/tid_rdma.c prev = rvt_get_swqe_ptr(qp, s_prev); qp 3234 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_acked != qp->s_cur) qp 3255 drivers/infiniband/hw/hfi1/tid_rdma.c static inline bool hfi1_check_sge_align(struct rvt_qp *qp, qp 3261 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_sge_check_align(qp, i, sge); qp 3269 drivers/infiniband/hw/hfi1/tid_rdma.c void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe) qp 3271 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; qp 3278 drivers/infiniband/hw/hfi1/tid_rdma.c if ((rdma_ah_get_dlid(&qp->remote_ah_attr) & ~((1 << ppd->lmc) - 1)) == qp 3294 drivers/infiniband/hw/hfi1/tid_rdma.c if (hfi1_check_sge_align(qp, &wqe->sg_list[0], qp 3326 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn += rvt_div_round_up_mtu(qp, wqe->length) - 1; qp 3343 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_setup_tid_wqe(qp, 1, wqe->wr.opcode, qp 3353 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 3357 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3377 drivers/infiniband/hw/hfi1/tid_rdma.c ohdr->u.tid_rdma.w_req.verbs_qp = cpu_to_be32(qp->remote_qpn); qp 3379 drivers/infiniband/hw/hfi1/tid_rdma.c *bth1 |= remote->qp; qp 3380 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_state = TID_OP(WRITE_REQ); qp 3381 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= HFI1_S_WAIT_TID_RESP; qp 3389 drivers/infiniband/hw/hfi1/tid_rdma.c static u32 hfi1_compute_tid_rdma_flow_wt(struct rvt_qp *qp) qp 3399 drivers/infiniband/hw/hfi1/tid_rdma.c return (MAX_TID_FLOW_PSN * qp->pmtu) >> TID_RDMA_SEGMENT_SHIFT; qp 3413 drivers/infiniband/hw/hfi1/tid_rdma.c static u32 hfi1_compute_tid_rnr_timeout(struct rvt_qp *qp, u32 to_seg) qp 3415 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3451 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx) qp 3454 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3462 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 3465 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_tid_write_alloc_res(qp, 0); qp 3466 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_alloc_res(qp); qp 3487 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_kern_clear_hw_flow(rcd, qp); qp 3493 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qpriv->r_tid_alloc]; qp 3497 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_write_alloc_res(qp, 0, e->opcode, e->psn, qp 3513 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_kern_clear_hw_flow(rcd, qp); qp 3520 drivers/infiniband/hw/hfi1/tid_rdma.c ret = hfi1_kern_setup_hw_flow(qpriv->rcd, qp); qp 3522 drivers/infiniband/hw/hfi1/tid_rdma.c to_seg = hfi1_compute_tid_rdma_flow_wt(qp) * qp 3529 drivers/infiniband/hw/hfi1/tid_rdma.c npkts = rvt_div_round_up_mtu(qp, req->seg_len); qp 3551 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_trigger_ack(qp); qp 3568 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) qp 3577 drivers/infiniband/hw/hfi1/tid_rdma.c if (ret == -EAGAIN && intr_ctx && !qp->r_nak_state) qp 3583 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->r_lock); qp 3586 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = hfi1_compute_tid_rnr_timeout(qp, to_seg) | IB_RNR_NAK; qp 3589 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_psn = e->psn + req->alloc_seg; qp 3590 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_ack_psn = qp->r_psn; qp 3596 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_head_ack_queue = qpriv->r_tid_alloc + 1; qp 3597 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->r_head_ack_queue > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) qp 3598 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_head_ack_queue = 0; qp 3599 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->r_tid_head = qp->r_head_ack_queue; qp 3605 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_nak_state = qp->r_nak_state; qp 3606 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_ack_psn = qp->r_ack_psn; qp 3611 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~(RVT_S_ACK_PENDING); qp 3613 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_tid_write_alloc_res(qp, qp->r_psn); qp 3628 drivers/infiniband/hw/hfi1/tid_rdma.c rc_defered_ack(rcd, qp); qp 3647 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 3648 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 3653 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3665 drivers/infiniband/hw/hfi1/tid_rdma.c fecn = process_ecn(qp, packet); qp 3667 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_rcv_tid_write_req(qp, psn); qp 3669 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) qp 3670 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_comm_est(qp); qp 3672 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) qp 3680 drivers/infiniband/hw/hfi1/tid_rdma.c diff = delta_psn(psn, qp->r_psn); qp 3682 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn); qp 3692 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_head_ack_queue = qp->r_head_ack_queue ? qp 3693 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_head_ack_queue - 1 : qp 3694 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_size_atomic(ib_to_rvt(qp->ibqp.device)); qp 3697 drivers/infiniband/hw/hfi1/tid_rdma.c next = qp->r_head_ack_queue + 1; qp 3698 drivers/infiniband/hw/hfi1/tid_rdma.c if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) qp 3700 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 3701 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(next == qp->s_acked_ack_queue)) { qp 3702 drivers/infiniband/hw/hfi1/tid_rdma.c if (!qp->s_ack_queue[next].sent) qp 3704 drivers/infiniband/hw/hfi1/tid_rdma.c update_ack_queue(qp, next); qp 3706 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qp->r_head_ack_queue]; qp 3711 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = 0; qp 3712 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_nak_state = 0; qp 3714 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_psn = e->lpsn + 1; qp 3726 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_len = len; qp 3733 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr, qp 3737 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_psn += num_segs - 1; qp 3741 drivers/infiniband/hw/hfi1/tid_rdma.c e->lpsn = qp->r_psn; qp 3762 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_state = e->opcode; qp 3763 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = 0; qp 3769 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_msn++; qp 3770 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_psn++; qp 3772 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_write_req(qp, 0, e->opcode, e->psn, e->lpsn, qp 3776 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->r_tid_tail = qp->r_head_ack_queue; qp 3780 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qpriv->r_tid_tail]; qp 3786 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->r_tid_ack = qp->r_head_ack_queue; qp 3787 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->r_tid_tail = qp->r_head_ack_queue; qp 3791 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_head_ack_queue = next; qp 3792 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->r_tid_head = qp->r_head_ack_queue; qp 3794 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_tid_write_alloc_resources(qp, true); qp 3795 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_rcv_req(qp); qp 3798 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_RESP_PENDING; qp 3800 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_ECN; qp 3801 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 3803 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 3807 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 3809 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 3810 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = IB_NAK_INVALID_REQUEST; qp 3811 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_ack_psn = qp->r_psn; qp 3813 drivers/infiniband/hw/hfi1/tid_rdma.c rc_defered_ack(rcd, qp); qp 3816 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 3817 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); qp 3818 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; qp 3819 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_ack_psn = qp->r_psn; qp 3822 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 3829 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3835 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_build_write_resp(qp, 0, e->opcode, e->psn, e->lpsn, qp 3837 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_build_resp(qp); qp 3838 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_build_tid_write_resp(qp, bth2); qp 3846 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_tid_write_alloc_resources(qp, false); qp 3860 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow); qp 3862 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_add_tid_reap_timer(qp); qp 3867 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow); qp 3872 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_mod_tid_reap_timer(qp); qp 3905 drivers/infiniband/hw/hfi1/tid_rdma.c ohdr->u.tid_rdma.w_rsp.aeth = rvt_compute_aeth(qp); qp 3912 drivers/infiniband/hw/hfi1/tid_rdma.c cpu_to_be32(qpriv->tid_rdma.local.qp | qp 3916 drivers/infiniband/hw/hfi1/tid_rdma.c ohdr->u.tid_rdma.w_rsp.verbs_qp = cpu_to_be32(qp->remote_qpn); qp 3917 drivers/infiniband/hw/hfi1/tid_rdma.c *bth1 = remote->qp; qp 3925 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_add_tid_reap_timer(struct rvt_qp *qp) qp 3927 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3929 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 3938 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp) qp 3940 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3942 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 3948 drivers/infiniband/hw/hfi1/tid_rdma.c static int hfi1_stop_tid_reap_timer(struct rvt_qp *qp) qp 3950 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3953 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 3961 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_del_tid_reap_timer(struct rvt_qp *qp) qp 3963 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 3972 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = qpriv->owner; qp 3973 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 3977 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->r_lock, flags); qp 3978 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock(&qp->s_lock); qp 3980 drivers/infiniband/hw/hfi1/tid_rdma.c dd_dev_warn(dd_from_ibdev(qp->ibqp.device), "[QP%u] %s %d\n", qp 3981 drivers/infiniband/hw/hfi1/tid_rdma.c qp->ibqp.qp_num, __func__, __LINE__); qp 3983 drivers/infiniband/hw/hfi1/tid_rdma.c qp, "resource timeout = ", qp 3985 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_stop_tid_reap_timer(qp); qp 3990 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_kern_clear_hw_flow(qpriv->rcd, qp); qp 3993 drivers/infiniband/hw/hfi1/tid_rdma.c ack_to_tid_req(&qp->s_ack_queue[i]); qp 3997 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock(&qp->s_lock); qp 3998 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->ibqp.event_handler) { qp 4001 drivers/infiniband/hw/hfi1/tid_rdma.c ev.device = qp->ibqp.device; qp 4002 drivers/infiniband/hw/hfi1/tid_rdma.c ev.element.qp = &qp->ibqp; qp 4004 drivers/infiniband/hw/hfi1/tid_rdma.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 4006 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_rc_error(qp, IB_WC_RESP_TIMEOUT_ERR); qp 4009 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock(&qp->s_lock); qp 4011 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 4028 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 4029 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 4039 drivers/infiniband/hw/hfi1/tid_rdma.c fecn = process_ecn(qp, packet); qp 4044 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 4047 drivers/infiniband/hw/hfi1/tid_rdma.c if (cmp_psn(psn, qp->s_next_psn) >= 0) qp 4051 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(cmp_psn(psn, qp->s_last_psn) <= 0)) qp 4054 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(qp->s_acked == qp->s_tail)) qp 4062 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->r_flags & RVT_R_RDMAR_SEQ) { qp 4063 drivers/infiniband/hw/hfi1/tid_rdma.c if (cmp_psn(psn, qp->s_last_psn + 1) != 0) qp 4065 drivers/infiniband/hw/hfi1/tid_rdma.c qp->r_flags &= ~RVT_R_RDMAR_SEQ; qp 4068 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); qp 4088 drivers/infiniband/hw/hfi1/tid_rdma.c if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) qp 4091 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_ack(qp, psn); qp 4109 drivers/infiniband/hw/hfi1/tid_rdma.c flow->npkts = rvt_div_round_up_mtu(qp, flow->length); qp 4120 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_write_resp(qp, req->setup_head, flow); qp 4123 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_sender_rcv_resp(qp, 0); qp 4130 drivers/infiniband/hw/hfi1/tid_rdma.c qp, i, flow->tid_entry[i]); qp 4142 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_write_resp(qp, 0, wqe->wr.opcode, wqe->psn, qp 4168 drivers/infiniband/hw/hfi1/tid_rdma.c if (i == qp->s_size) qp 4170 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); qp 4178 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~HFI1_S_WAIT_TID_RESP; qp 4179 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_tid_send(qp); qp 4185 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_error_qp(qp, status); qp 4188 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_ECN; qp 4189 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 4199 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = req->qp; qp 4200 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 4208 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_trdma_send_complete(qp, wqe, IB_WC_REM_INV_RD_REQ_ERR); qp 4209 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_error_qp(qp, IB_WC_REM_INV_RD_REQ_ERR); qp 4212 drivers/infiniband/hw/hfi1/tid_rdma.c *len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset); qp 4217 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_entry_build_write_data(qp, flow->tid_idx, tidentry); qp 4218 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_build_write_data(qp, req->clear_tail, flow); qp 4230 drivers/infiniband/hw/hfi1/tid_rdma.c wd->verbs_qp = cpu_to_be32(qp->remote_qpn); qp 4241 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_div_round_up_mtu(qp, req->seg_len) > qp 4258 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 4259 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 4265 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); qp 4271 drivers/infiniband/hw/hfi1/tid_rdma.c fecn = process_ecn(qp, packet); qp 4279 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 4280 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[priv->r_tid_tail]; qp 4305 drivers/infiniband/hw/hfi1/tid_rdma.c u32 pmtu = qp->pmtu; qp 4325 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_copy_sge(qp, &ss, packet->payload, pmtu, false, qp 4347 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_rcv_tid_write_data(qp, psn); qp 4348 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_write_data(qp, 0, e->opcode, e->psn, e->lpsn, qp 4350 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_rcv_data(qp); qp 4360 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[next]; qp 4365 drivers/infiniband/hw/hfi1/tid_rdma.c if (++qp->s_acked_ack_queue > rvt_size_atomic(&dev->rdi)) qp 4366 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_acked_ack_queue = 0; qp 4369 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_tid_write_alloc_resources(qp, true); qp 4376 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_tail_ack_queue != qp->r_head_ack_queue) { qp 4377 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_RESP_PENDING; qp 4378 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 4384 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_mod_tid_reap_timer(req->qp); qp 4386 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_stop_tid_reap_timer(req->qp); qp 4390 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_schedule_ack(qp); qp 4394 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_ECN; qp 4395 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 4402 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_trigger_ack(qp); qp 4413 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 4417 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 4426 drivers/infiniband/hw/hfi1/tid_rdma.c ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn); qp 4427 drivers/infiniband/hw/hfi1/tid_rdma.c *bth1 = remote->qp; qp 4433 drivers/infiniband/hw/hfi1/tid_rdma.c ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp); qp 4437 drivers/infiniband/hw/hfi1/tid_rdma.c cpu_to_be32((qp->r_msn & IB_MSN_MASK) | qp 4442 drivers/infiniband/hw/hfi1/tid_rdma.c ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp); qp 4446 drivers/infiniband/hw/hfi1/tid_rdma.c cpu_to_be32(qpriv->tid_rdma.local.qp | qp 4487 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 4488 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 4496 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_sender_rcv_tid_ack(qp, 0); qp 4497 drivers/infiniband/hw/hfi1/tid_rdma.c process_ecn(qp, packet); qp 4503 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 4504 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rcv_tid_ack(qp, aeth, psn, req_psn, resync_psn); qp 4507 drivers/infiniband/hw/hfi1/tid_rdma.c if ((qp->s_flags & HFI1_S_WAIT_HALT) && qp 4521 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(qp->s_acked == qp->s_tail)) qp 4524 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 4530 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, qp 4533 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); qp 4547 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, qp 4551 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = do_rc_completion(qp, wqe, qp 4552 drivers/infiniband/hw/hfi1/tid_rdma.c to_iport(qp->ibqp.device, qp 4553 drivers/infiniband/hw/hfi1/tid_rdma.c qp->port_num)); qp 4554 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_sender_rcv_tid_ack(qp); qp 4556 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_acked == qp->s_tail) qp 4563 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); qp 4566 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, qp 4576 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_mod_tid_retry_timer(qp); qp 4578 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_stop_tid_retry_timer(qp); qp 4579 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 4585 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_stop_tid_retry_timer(qp); qp 4587 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~HFI1_S_WAIT_HALT; qp 4595 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 4597 drivers/infiniband/hw/hfi1/tid_rdma.c if ((qp->s_acked == qpriv->s_tid_tail && qp 4599 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_acked == qp->s_tail) { qp 4622 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 4651 drivers/infiniband/hw/hfi1/tid_rdma.c last_acked = qp->s_acked; qp 4683 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_tid_ack(qp, qp 4689 drivers/infiniband/hw/hfi1/tid_rdma.c if (last_acked == qp->s_size) qp 4691 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, last_acked); qp 4695 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->s_tid_tail = qp->s_acked; qp 4697 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_tid_send(qp); qp 4700 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->s_retry = qp->s_retry_cnt; qp 4704 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_stop_tid_retry_timer(qp); qp 4714 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, qp 4718 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->s_tid_tail = qp->s_acked; qp 4720 drivers/infiniband/hw/hfi1/tid_rdma.c qpriv->s_retry = qp->s_retry_cnt; qp 4721 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_tid_send(qp); qp 4734 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 4737 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_add_tid_retry_timer(struct rvt_qp *qp) qp 4739 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 4740 drivers/infiniband/hw/hfi1/tid_rdma.c struct ib_qp *ibqp = &qp->ibqp; qp 4743 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 4752 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp) qp 4754 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 4755 drivers/infiniband/hw/hfi1/tid_rdma.c struct ib_qp *ibqp = &qp->ibqp; qp 4758 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 4764 drivers/infiniband/hw/hfi1/tid_rdma.c static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp) qp 4766 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 4769 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 4777 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_del_tid_retry_timer(struct rvt_qp *qp) qp 4779 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 4788 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = priv->owner; qp 4793 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->r_lock, flags); qp 4794 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock(&qp->s_lock); qp 4795 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_sender_retry_timeout(qp, 0); qp 4797 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_stop_tid_retry_timer(qp); qp 4800 drivers/infiniband/hw/hfi1/tid_rdma.c qp, qp 4804 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 4805 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_trdma_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); qp 4806 drivers/infiniband/hw/hfi1/tid_rdma.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 4808 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 4811 drivers/infiniband/hw/hfi1/tid_rdma.c qp, 0, wqe->wr.opcode, wqe->psn, wqe->lpsn, req); qp 4820 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= HFI1_S_WAIT_HALT; qp 4823 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_tid_send(qp); qp 4826 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock(&qp->s_lock); qp 4827 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 4830 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 4834 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 4843 drivers/infiniband/hw/hfi1/tid_rdma.c ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn); qp 4844 drivers/infiniband/hw/hfi1/tid_rdma.c *bth1 = remote->qp; qp 4859 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = packet->qp; qp 4860 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 4862 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); qp 4871 drivers/infiniband/hw/hfi1/tid_rdma.c fecn = process_ecn(qp, packet); qp 4875 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 4908 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_rcv_resync(qp); qp 4920 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[idx]; qp 4923 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_resync(qp, 0, e->opcode, e->psn, qp 4947 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_flow_rcv_resync(qp, flow_idx, qp 4951 drivers/infiniband/hw/hfi1/tid_rdma.c if (idx == qp->s_tail_ack_queue) qp 4959 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_trigger_ack(qp); qp 4962 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_ECN; qp 4963 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 4970 drivers/infiniband/hw/hfi1/tid_rdma.c static void update_tid_tail(struct rvt_qp *qp) qp 4971 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 4973 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 4977 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 4982 drivers/infiniband/hw/hfi1/tid_rdma.c if (i == qp->s_size) qp 4987 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); qp 4995 drivers/infiniband/hw/hfi1/tid_rdma.c int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps) qp 4996 drivers/infiniband/hw/hfi1/tid_rdma.c __must_hold(&qp->s_lock) qp 4998 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 5002 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_sge_state *ss = &qp->s_sge; qp 5003 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_ack_entry *e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 5008 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 5009 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0); qp 5016 drivers/infiniband/hw/hfi1/tid_rdma.c !(qp->s_flags & (RVT_S_BUSY | RVT_S_WAIT_ACK | qp 5019 drivers/infiniband/hw/hfi1/tid_rdma.c !(qp->s_flags & (RVT_S_BUSY | HFI1_S_ANY_WAIT_IO)))) { qp 5024 drivers/infiniband/hw/hfi1/tid_rdma.c if (ps->s_txreq || hfi1_make_rc_req(qp, ps)) { qp 5030 drivers/infiniband/hw/hfi1/tid_rdma.c ps->s_txreq = get_txreq(ps->dev, qp); qp 5037 drivers/infiniband/hw/hfi1/tid_rdma.c make_tid_rdma_ack(qp, ohdr, ps)) qp 5046 drivers/infiniband/hw/hfi1/tid_rdma.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) qp 5055 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); qp 5057 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe->psn, qp 5068 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_tid_rdma_restart_req(qp, wqe, &bth2); qp 5086 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_sender_make_tid_pkt(qp); qp 5087 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0); qp 5088 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); qp 5095 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, qp 5107 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= HFI1_S_WAIT_TID_RESP; qp 5113 drivers/infiniband/hw/hfi1/tid_rdma.c update_tid_tail(qp); qp 5121 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_sender_make_tid_pkt(qp); qp 5123 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, priv->s_tid_cur); qp 5127 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp 5128 drivers/infiniband/hw/hfi1/tid_rdma.c (!priv->s_tid_cur ? qp->s_size : qp 5132 drivers/infiniband/hw/hfi1/tid_rdma.c hwords += hfi1_build_tid_rdma_resync(qp, wqe, ohdr, &bth1, qp 5149 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_len -= len; qp 5154 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_make_ruc_header(qp, ohdr, (opcode << 24), bth1, bth2, qp 5173 drivers/infiniband/hw/hfi1/tid_rdma.c static int make_tid_rdma_ack(struct rvt_qp *qp, qp 5178 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *qpriv = qp->priv; qp 5179 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); qp 5187 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_make_tid_ack(qp); qp 5189 drivers/infiniband/hw/hfi1/tid_rdma.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) qp 5195 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qpriv->r_tid_ack]; qp 5214 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qpriv->r_tid_ack]; qp 5218 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rsp_make_tid_ack(qp, e->psn); qp 5219 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn, qp 5257 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_ack_queue[next].opcode != TID_OP(WRITE_REQ)) qp 5259 drivers/infiniband/hw/hfi1/tid_rdma.c nreq = ack_to_tid_req(&qp->s_ack_queue[next]); qp 5264 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qpriv->r_tid_ack]; qp 5285 drivers/infiniband/hw/hfi1/tid_rdma.c e = &qp->s_ack_queue[qpriv->r_tid_ack]; qp 5292 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_write_rsp_make_tid_ack(qp); qp 5293 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn, qp 5295 drivers/infiniband/hw/hfi1/tid_rdma.c hwords += hfi1_build_tid_rdma_write_ack(qp, e, ohdr, flow, &bth1, qp 5303 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_make_ruc_header(qp, ohdr, (TID_OP(ACK) << 24), bth1, bth2, middle, qp 5317 drivers/infiniband/hw/hfi1/tid_rdma.c static int hfi1_send_tid_ok(struct rvt_qp *qp) qp 5319 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 5322 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags & HFI1_S_ANY_WAIT_IO) && qp 5325 drivers/infiniband/hw/hfi1/tid_rdma.c !(qp->s_flags & HFI1_S_ANY_TID_WAIT_SEND)); qp 5331 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp = iowait_to_qp(w->iow); qp 5333 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_do_tid_send(qp); qp 5336 drivers/infiniband/hw/hfi1/tid_rdma.c static void hfi1_do_tid_send(struct rvt_qp *qp) qp 5339 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 5341 drivers/infiniband/hw/hfi1/tid_rdma.c ps.dev = to_idev(qp->ibqp.device); qp 5342 drivers/infiniband/hw/hfi1/tid_rdma.c ps.ibp = to_iport(qp->ibqp.device, qp->port_num); qp 5346 drivers/infiniband/hw/hfi1/tid_rdma.c ps.timeout_int = qp->timeout_jiffies / 8; qp 5348 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_rc_do_tid_send(qp, false); qp 5349 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, ps.flags); qp 5352 drivers/infiniband/hw/hfi1/tid_rdma.c if (!hfi1_send_tid_ok(qp)) { qp 5353 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_flags & HFI1_S_ANY_WAIT_IO) qp 5355 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, ps.flags); qp 5372 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_BUSY; qp 5375 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, ps.flags); qp 5381 drivers/infiniband/hw/hfi1/tid_rdma.c if (hfi1_verbs_send(qp, &ps)) qp 5385 drivers/infiniband/hw/hfi1/tid_rdma.c if (hfi1_schedule_send_yield(qp, &ps, true)) qp 5388 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, ps.flags); qp 5390 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags &= ~RVT_S_BUSY; qp 5395 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_schedule_send(qp); qp 5398 drivers/infiniband/hw/hfi1/tid_rdma.c } while (hfi1_make_tid_rdma_pkt(qp, &ps)); qp 5400 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, ps.flags); qp 5403 drivers/infiniband/hw/hfi1/tid_rdma.c static bool _hfi1_schedule_tid_send(struct rvt_qp *qp) qp 5405 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 5407 drivers/infiniband/hw/hfi1/tid_rdma.c to_iport(qp->ibqp.device, qp->port_num); qp 5409 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 5430 drivers/infiniband/hw/hfi1/tid_rdma.c bool hfi1_schedule_tid_send(struct rvt_qp *qp) qp 5432 drivers/infiniband/hw/hfi1/tid_rdma.c lockdep_assert_held(&qp->s_lock); qp 5433 drivers/infiniband/hw/hfi1/tid_rdma.c if (hfi1_send_tid_ok(qp)) { qp 5440 drivers/infiniband/hw/hfi1/tid_rdma.c _hfi1_schedule_tid_send(qp); qp 5443 drivers/infiniband/hw/hfi1/tid_rdma.c if (qp->s_flags & HFI1_S_ANY_WAIT_IO) qp 5444 drivers/infiniband/hw/hfi1/tid_rdma.c iowait_set_flag(&((struct hfi1_qp_priv *)qp->priv)->s_iowait, qp 5449 drivers/infiniband/hw/hfi1/tid_rdma.c bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e) qp 5453 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); qp 5454 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_qp_priv *priv = qp->priv; qp 5457 drivers/infiniband/hw/hfi1/tid_rdma.c s_prev = qp->s_tail_ack_queue == 0 ? rvt_size_atomic(&dev->rdi) : qp 5458 drivers/infiniband/hw/hfi1/tid_rdma.c (qp->s_tail_ack_queue - 1); qp 5459 drivers/infiniband/hw/hfi1/tid_rdma.c prev = &qp->s_ack_queue[s_prev]; qp 5487 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp, u32 psn, int diff, bool fecn) qp 5491 drivers/infiniband/hw/hfi1/tid_rdma.c tid_rdma_rcv_error(packet, ohdr, qp, psn, diff); qp 5493 drivers/infiniband/hw/hfi1/tid_rdma.c spin_lock_irqsave(&qp->s_lock, flags); qp 5494 drivers/infiniband/hw/hfi1/tid_rdma.c qp->s_flags |= RVT_S_ECN; qp 5495 drivers/infiniband/hw/hfi1/tid_rdma.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 60 drivers/infiniband/hw/hfi1/tid_rdma.h u32 qp; qp 98 drivers/infiniband/hw/hfi1/tid_rdma.h struct rvt_qp *qp; qp 203 drivers/infiniband/hw/hfi1/tid_rdma.h bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data); qp 204 drivers/infiniband/hw/hfi1/tid_rdma.h bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data); qp 205 drivers/infiniband/hw/hfi1/tid_rdma.h bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data); qp 206 drivers/infiniband/hw/hfi1/tid_rdma.h void tid_rdma_conn_error(struct rvt_qp *qp); qp 207 drivers/infiniband/hw/hfi1/tid_rdma.h void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p); qp 214 drivers/infiniband/hw/hfi1/tid_rdma.h void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe); qp 221 drivers/infiniband/hw/hfi1/tid_rdma.h static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) qp 225 drivers/infiniband/hw/hfi1/tid_rdma.h __trdma_clean_swqe(qp, wqe); qp 228 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp); qp 230 drivers/infiniband/hw/hfi1/tid_rdma.h int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 232 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); qp 234 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp); qp 236 drivers/infiniband/hw/hfi1/tid_rdma.h int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); qp 237 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); qp 247 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 251 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 258 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 260 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp); qp 261 drivers/infiniband/hw/hfi1/tid_rdma.h bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe); qp 263 drivers/infiniband/hw/hfi1/tid_rdma.h void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); qp 264 drivers/infiniband/hw/hfi1/tid_rdma.h static inline void hfi1_setup_tid_rdma_wqe(struct rvt_qp *qp, qp 271 drivers/infiniband/hw/hfi1/tid_rdma.h setup_tid_rdma_wqe(qp, wqe); qp 274 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 280 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 285 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_del_tid_reap_timer(struct rvt_qp *qp); qp 295 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e, qp 301 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_add_tid_retry_timer(struct rvt_qp *qp); qp 302 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_del_tid_retry_timer(struct rvt_qp *qp); qp 304 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 311 drivers/infiniband/hw/hfi1/tid_rdma.h int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps); qp 315 drivers/infiniband/hw/hfi1/tid_rdma.h bool hfi1_schedule_tid_send(struct rvt_qp *qp); qp 317 drivers/infiniband/hw/hfi1/tid_rdma.h bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e); qp 97 drivers/infiniband/hw/hfi1/trace_misc.h TP_PROTO(struct rvt_qp *qp, u8 opcode), qp 98 drivers/infiniband/hw/hfi1/trace_misc.h TP_ARGS(qp, opcode), qp 99 drivers/infiniband/hw/hfi1/trace_misc.h TP_STRUCT__entry(DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 103 drivers/infiniband/hw/hfi1/trace_misc.h TP_fast_assign(DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 104 drivers/infiniband/hw/hfi1/trace_misc.h __entry->qpn = qp->ibqp.qp_num; qp 59 drivers/infiniband/hw/hfi1/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 60 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, psn), qp 62 drivers/infiniband/hw/hfi1/trace_rc.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 73 drivers/infiniband/hw/hfi1/trace_rc.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 74 drivers/infiniband/hw/hfi1/trace_rc.h __entry->qpn = qp->ibqp.qp_num; qp 75 drivers/infiniband/hw/hfi1/trace_rc.h __entry->s_flags = qp->s_flags; qp 77 drivers/infiniband/hw/hfi1/trace_rc.h __entry->s_psn = qp->s_psn; qp 78 drivers/infiniband/hw/hfi1/trace_rc.h __entry->s_next_psn = qp->s_next_psn; qp 79 drivers/infiniband/hw/hfi1/trace_rc.h __entry->s_sending_psn = qp->s_sending_psn; qp 80 drivers/infiniband/hw/hfi1/trace_rc.h __entry->s_sending_hpsn = qp->s_sending_hpsn; qp 81 drivers/infiniband/hw/hfi1/trace_rc.h __entry->r_psn = qp->r_psn; qp 98 drivers/infiniband/hw/hfi1/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 99 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, psn) qp 103 drivers/infiniband/hw/hfi1/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 104 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, psn) qp 108 drivers/infiniband/hw/hfi1/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 109 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, psn) qp 114 drivers/infiniband/hw/hfi1/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 115 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, psn) qp 120 drivers/infiniband/hw/hfi1/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 aeth, u32 psn, qp 122 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, aeth, psn, wqe), qp 124 drivers/infiniband/hw/hfi1/trace_rc.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 133 drivers/infiniband/hw/hfi1/trace_rc.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 134 drivers/infiniband/hw/hfi1/trace_rc.h __entry->qpn = qp->ibqp.qp_num; qp 155 drivers/infiniband/hw/hfi1/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 aeth, u32 psn, qp 157 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, aeth, psn, wqe) qp 195 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 196 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp), qp 198 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 205 drivers/infiniband/hw/hfi1/trace_tid.h struct hfi1_qp_priv *priv = qp->priv; qp 207 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 208 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 225 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 226 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 231 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 232 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 237 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 238 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 243 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 244 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 249 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 250 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 255 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u8 capcode, u64 data), qp 256 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, capcode, data), qp 258 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 265 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 266 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 267 drivers/infiniband/hw/hfi1/trace_tid.h __entry->state = qp->state; qp 283 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u8 capcode, u64 data), qp 284 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, capcode, data) qp 289 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u8 capcode, u64 data), qp 290 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, capcode, data) qp 295 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u8 capcode, u64 data), qp 296 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, capcode, data) qp 301 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char remote, qp 303 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, remote, param), qp 305 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 317 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 318 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 320 drivers/infiniband/hw/hfi1/trace_tid.h __entry->param_qp = param->qp; qp 345 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char remote, qp 347 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, remote, param) qp 352 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 353 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more), qp 360 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp ? qp->ibqp.qp_num : 0; qp 374 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 375 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more) qp 380 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 381 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more) qp 386 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 387 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more) qp 392 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 393 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more) qp 398 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 399 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more) qp 404 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 405 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more) qp 410 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u64 more), qp 411 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, more) qp 416 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, struct tid_rdma_flow *flow, u32 index, qp 418 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, flow, index, mtu8k, v1, vaddr), qp 420 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 429 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 430 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 450 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, struct tid_rdma_flow *flow, u32 index, qp 452 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, flow, index, mtu8k, v1, vaddr) qp 457 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 index, u16 idx, u16 count), qp 458 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, idx, count), qp 460 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 467 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 468 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 485 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 index, u16 idx, u16 count), qp 486 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, idx, count) qp 491 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 492 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow), qp 494 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 514 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 515 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 560 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 561 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 566 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 567 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 572 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 573 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 578 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 579 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 584 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 585 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 590 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 591 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 596 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 597 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 602 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 603 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 608 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 609 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 614 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 615 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 620 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 621 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 626 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 627 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 632 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct tid_rdma_flow *flow), qp 633 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, flow) qp 638 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u32 index, u32 base, qp 640 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, index, base, map, used, cnt), qp 642 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 652 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 653 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 676 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, const char *msg, u32 index, u32 base, qp 678 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, msg, index, base, map, used, cnt) qp 683 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, u32 ent), qp 684 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, ent), qp 686 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 694 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 695 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 714 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, u32 entry), qp 715 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, entry) qp 720 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, u32 ent), qp 721 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, ent) qp 726 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, u32 ent), qp 727 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, ent) qp 732 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, u32 entry), qp 733 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, entry) qp 738 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, u32 entry), qp 739 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, entry) qp 744 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 745 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn), qp 747 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 766 drivers/infiniband/hw/hfi1/trace_tid.h struct hfi1_qp_priv *priv = qp->priv; qp 768 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 769 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 770 drivers/infiniband/hw/hfi1/trace_tid.h __entry->state = qp->state; qp 771 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_state = qp->s_state; qp 773 drivers/infiniband/hw/hfi1/trace_tid.h __entry->r_psn = qp->r_psn; qp 774 drivers/infiniband/hw/hfi1/trace_tid.h __entry->r_state = qp->r_state; qp 775 drivers/infiniband/hw/hfi1/trace_tid.h __entry->r_flags = qp->r_flags; qp 776 drivers/infiniband/hw/hfi1/trace_tid.h __entry->r_head_ack_queue = qp->r_head_ack_queue; qp 777 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_tail_ack_queue = qp->s_tail_ack_queue; qp 778 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_acked_ack_queue = qp->s_acked_ack_queue; qp 779 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_ack_state = qp->s_ack_state; qp 780 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_nak_state = qp->s_nak_state; qp 781 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_flags = qp->s_flags; qp 808 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 809 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 814 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 815 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 820 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 821 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 826 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 827 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 832 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 833 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 838 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 839 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 844 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 845 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 850 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 851 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 856 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 857 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 862 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 863 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, psn) qp 868 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 869 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp), qp 871 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 889 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 890 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 891 drivers/infiniband/hw/hfi1/trace_tid.h __entry->state = qp->state; qp 892 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_cur = qp->s_cur; qp 893 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_tail = qp->s_tail; qp 894 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_head = qp->s_head; qp 895 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_acked = qp->s_acked; qp 896 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_last = qp->s_last; qp 897 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_psn = qp->s_psn; qp 898 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_last_psn = qp->s_last_psn; qp 899 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_flags = qp->s_flags; qp 900 drivers/infiniband/hw/hfi1/trace_tid.h __entry->ps_flags = ((struct hfi1_qp_priv *)qp->priv)->s_flags; qp 902 drivers/infiniband/hw/hfi1/trace_tid.h ((struct hfi1_qp_priv *)qp->priv)->s_iowait.flags; qp 903 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_state = qp->s_state; qp 904 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_num_rd = qp->s_num_rd_atomic; qp 905 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_retry = qp->s_retry; qp 930 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 931 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 936 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 937 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 942 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 943 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 948 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 949 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 954 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 955 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 960 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 961 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 966 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 967 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 972 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 973 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 978 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 979 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq), qp 981 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 996 drivers/infiniband/hw/hfi1/trace_tid.h struct hfi1_qp_priv *priv = qp->priv; qp 998 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 999 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1004 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_flags = qp->s_flags; qp 1032 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1033 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1038 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1039 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1044 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1046 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req), qp 1048 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 1070 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 1071 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1118 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1120 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1125 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1127 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1132 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1134 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1139 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1141 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1146 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1148 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1153 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1155 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1160 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1162 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1167 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1169 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1174 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1176 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1181 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1183 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1188 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1190 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1195 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1197 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1202 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1204 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1209 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1211 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1216 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1218 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1223 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1225 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1230 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1232 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1237 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1239 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1244 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1246 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1251 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1253 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1258 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1260 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1265 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq, u8 opcode, u32 psn, u32 lpsn, qp 1267 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq, opcode, psn, lpsn, req) qp 1272 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 opcode, u32 psn, int diff), qp 1273 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, opcode, psn, diff), qp 1275 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 1288 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 1289 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1290 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_flags = qp->s_flags; qp 1291 drivers/infiniband/hw/hfi1/trace_tid.h __entry->state = qp->state; qp 1292 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_acked_ack_queue = qp->s_acked_ack_queue; qp 1293 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_tail_ack_queue = qp->s_tail_ack_queue; qp 1294 drivers/infiniband/hw/hfi1/trace_tid.h __entry->r_head_ack_queue = qp->r_head_ack_queue; qp 1297 drivers/infiniband/hw/hfi1/trace_tid.h __entry->r_psn = qp->r_psn; qp 1318 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 opcode, u32 psn, int diff), qp 1319 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, opcode, psn, diff) qp 1324 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct rvt_sge *sge), qp 1325 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, sge), qp 1327 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 1334 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 1335 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1352 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, int index, struct rvt_sge *sge), qp 1353 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, index, sge) qp 1358 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1359 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp), qp 1361 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 1380 drivers/infiniband/hw/hfi1/trace_tid.h struct hfi1_qp_priv *priv = qp->priv; qp 1382 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 1383 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1424 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1425 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1430 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1431 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1436 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1437 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1442 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1443 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1448 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1449 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1454 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1455 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1460 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1461 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1466 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp), qp 1467 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp) qp 1472 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1473 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq), qp 1475 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 1491 drivers/infiniband/hw/hfi1/trace_tid.h struct hfi1_qp_priv *priv = qp->priv; qp 1493 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 1494 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1502 drivers/infiniband/hw/hfi1/trace_tid.h __entry->s_flags = qp->s_flags; qp 1529 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1530 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1535 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1536 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1541 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1542 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1547 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1548 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1553 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1554 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1559 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, char newreq), qp 1560 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, newreq) qp 1565 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 aeth, u32 psn, qp 1567 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, aeth, psn, req_psn, resync_psn), qp 1569 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 1577 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 1578 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1597 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u32 aeth, u32 psn, qp 1599 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, aeth, psn, req_psn, resync_psn) qp 1604 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u8 rcv_type, u8 rte, u32 psn), qp 1605 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, rcv_type, rte, psn), qp 1607 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 1614 drivers/infiniband/hw/hfi1/trace_tid.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)); qp 1615 drivers/infiniband/hw/hfi1/trace_tid.h __entry->qpn = qp->ibqp.qp_num; qp 1632 drivers/infiniband/hw/hfi1/trace_tid.h TP_PROTO(struct rvt_qp *qp, u8 rcv_type, u8 rte, u32 psn), qp 1633 drivers/infiniband/hw/hfi1/trace_tid.h TP_ARGS(qp, rcv_type, rte, psn) qp 110 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, u32 flags), qp 111 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, flags), qp 113 drivers/infiniband/hw/hfi1/trace_tx.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 121 drivers/infiniband/hw/hfi1/trace_tx.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 123 drivers/infiniband/hw/hfi1/trace_tx.h __entry->qpn = qp->ibqp.qp_num; qp 124 drivers/infiniband/hw/hfi1/trace_tx.h __entry->s_flags = qp->s_flags; qp 126 drivers/infiniband/hw/hfi1/trace_tx.h ((struct hfi1_qp_priv *)qp->priv)->s_flags; qp 128 drivers/infiniband/hw/hfi1/trace_tx.h ((struct hfi1_qp_priv *)qp->priv)->s_iowait.flags; qp 142 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, u32 flags), qp 143 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, flags)); qp 146 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, u32 flags), qp 147 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, flags)); qp 780 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, u32 idx), qp 781 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, wqe, idx), qp 783 drivers/infiniband/hw/hfi1/trace_tx.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 795 drivers/infiniband/hw/hfi1/trace_tx.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 798 drivers/infiniband/hw/hfi1/trace_tx.h __entry->qpn = qp->ibqp.qp_num; qp 799 drivers/infiniband/hw/hfi1/trace_tx.h __entry->qpt = qp->ibqp.qp_type; qp 823 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, bool flag), qp 824 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, flag), qp 826 drivers/infiniband/hw/hfi1/trace_tx.h DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device)) qp 831 drivers/infiniband/hw/hfi1/trace_tx.h DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device)) qp 832 drivers/infiniband/hw/hfi1/trace_tx.h __entry->qpn = qp->ibqp.qp_num; qp 845 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, bool flag), qp 846 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, flag) qp 851 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, bool flag), qp 852 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, flag) qp 857 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, bool flag), qp 858 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, flag) qp 63 drivers/infiniband/hw/hfi1/uc.c int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) qp 65 drivers/infiniband/hw/hfi1/uc.c struct hfi1_qp_priv *priv = qp->priv; qp 71 drivers/infiniband/hw/hfi1/uc.c u32 pmtu = qp->pmtu; qp 74 drivers/infiniband/hw/hfi1/uc.c ps->s_txreq = get_txreq(ps->dev, qp); qp 78 drivers/infiniband/hw/hfi1/uc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { qp 79 drivers/infiniband/hw/hfi1/uc.c if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) qp 82 drivers/infiniband/hw/hfi1/uc.c if (qp->s_last == READ_ONCE(qp->s_head)) qp 86 drivers/infiniband/hw/hfi1/uc.c qp->s_flags |= RVT_S_WAIT_DMA; qp 89 drivers/infiniband/hw/hfi1/uc.c clear_ahg(qp); qp 90 drivers/infiniband/hw/hfi1/uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 91 drivers/infiniband/hw/hfi1/uc.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); qp 98 drivers/infiniband/hw/hfi1/uc.c if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) qp 105 drivers/infiniband/hw/hfi1/uc.c if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && qp 106 drivers/infiniband/hw/hfi1/uc.c (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr)))) qp 113 drivers/infiniband/hw/hfi1/uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 114 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = NULL; qp 115 drivers/infiniband/hw/hfi1/uc.c switch (qp->s_state) { qp 117 drivers/infiniband/hw/hfi1/uc.c if (!(ib_rvt_state_ops[qp->state] & qp 121 drivers/infiniband/hw/hfi1/uc.c if (qp->s_cur == READ_ONCE(qp->s_head)) { qp 122 drivers/infiniband/hw/hfi1/uc.c clear_ahg(qp); qp 134 drivers/infiniband/hw/hfi1/uc.c if (qp->s_last != qp->s_cur) qp 136 drivers/infiniband/hw/hfi1/uc.c if (++qp->s_cur == qp->s_size) qp 137 drivers/infiniband/hw/hfi1/uc.c qp->s_cur = 0; qp 140 drivers/infiniband/hw/hfi1/uc.c qp, wqe->wr.ex.invalidate_rkey); qp 143 drivers/infiniband/hw/hfi1/uc.c rvt_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR qp 146 drivers/infiniband/hw/hfi1/uc.c atomic_dec(&qp->local_ops_pending); qp 152 drivers/infiniband/hw/hfi1/uc.c qp->s_psn = wqe->psn; qp 153 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.sge = wqe->sg_list[0]; qp 154 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.sg_list = wqe->sg_list + 1; qp 155 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.num_sge = wqe->wr.num_sge; qp 156 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.total_len = wqe->length; qp 158 drivers/infiniband/hw/hfi1/uc.c qp->s_len = len; qp 163 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(SEND_FIRST); qp 168 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(SEND_ONLY); qp 170 drivers/infiniband/hw/hfi1/uc.c qp->s_state = qp 178 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; qp 179 drivers/infiniband/hw/hfi1/uc.c if (++qp->s_cur >= qp->s_size) qp 180 drivers/infiniband/hw/hfi1/uc.c qp->s_cur = 0; qp 192 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(RDMA_WRITE_FIRST); qp 197 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(RDMA_WRITE_ONLY); qp 199 drivers/infiniband/hw/hfi1/uc.c qp->s_state = qp 207 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; qp 208 drivers/infiniband/hw/hfi1/uc.c if (++qp->s_cur >= qp->s_size) qp 209 drivers/infiniband/hw/hfi1/uc.c qp->s_cur = 0; qp 218 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(SEND_MIDDLE); qp 221 drivers/infiniband/hw/hfi1/uc.c len = qp->s_len; qp 228 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(SEND_LAST); qp 230 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); qp 237 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; qp 238 drivers/infiniband/hw/hfi1/uc.c if (++qp->s_cur >= qp->s_size) qp 239 drivers/infiniband/hw/hfi1/uc.c qp->s_cur = 0; qp 243 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(RDMA_WRITE_MIDDLE); qp 246 drivers/infiniband/hw/hfi1/uc.c len = qp->s_len; qp 253 drivers/infiniband/hw/hfi1/uc.c qp->s_state = OP(RDMA_WRITE_LAST); qp 255 drivers/infiniband/hw/hfi1/uc.c qp->s_state = qp 263 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; qp 264 drivers/infiniband/hw/hfi1/uc.c if (++qp->s_cur >= qp->s_size) qp 265 drivers/infiniband/hw/hfi1/uc.c qp->s_cur = 0; qp 268 drivers/infiniband/hw/hfi1/uc.c qp->s_len -= len; qp 271 drivers/infiniband/hw/hfi1/uc.c ps->s_txreq->ss = &qp->s_sge; qp 273 drivers/infiniband/hw/hfi1/uc.c hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), qp 274 drivers/infiniband/hw/hfi1/uc.c qp->remote_qpn, mask_psn(qp->s_psn++), qp 288 drivers/infiniband/hw/hfi1/uc.c qp->s_flags &= ~RVT_S_BUSY; qp 310 drivers/infiniband/hw/hfi1/uc.c struct rvt_qp *qp = packet->qp; qp 317 drivers/infiniband/hw/hfi1/uc.c u32 pmtu = qp->pmtu; qp 325 drivers/infiniband/hw/hfi1/uc.c process_ecn(qp, packet); qp 329 drivers/infiniband/hw/hfi1/uc.c if (unlikely(cmp_psn(psn, qp->r_psn) != 0)) { qp 334 drivers/infiniband/hw/hfi1/uc.c qp->r_psn = psn; qp 336 drivers/infiniband/hw/hfi1/uc.c if (qp->r_state == OP(SEND_FIRST) || qp 337 drivers/infiniband/hw/hfi1/uc.c qp->r_state == OP(SEND_MIDDLE)) { qp 338 drivers/infiniband/hw/hfi1/uc.c set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); qp 339 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.num_sge = 0; qp 341 drivers/infiniband/hw/hfi1/uc.c rvt_put_ss(&qp->r_sge); qp 343 drivers/infiniband/hw/hfi1/uc.c qp->r_state = OP(SEND_LAST); qp 361 drivers/infiniband/hw/hfi1/uc.c switch (qp->r_state) { qp 389 drivers/infiniband/hw/hfi1/uc.c if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) qp 390 drivers/infiniband/hw/hfi1/uc.c rvt_comm_est(qp); qp 398 drivers/infiniband/hw/hfi1/uc.c if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) { qp 399 drivers/infiniband/hw/hfi1/uc.c qp->r_sge = qp->s_rdma_read_sge; qp 401 drivers/infiniband/hw/hfi1/uc.c ret = rvt_get_rwqe(qp, false); qp 410 drivers/infiniband/hw/hfi1/uc.c qp->s_rdma_read_sge = qp->r_sge; qp 412 drivers/infiniband/hw/hfi1/uc.c qp->r_rcv_len = 0; qp 427 drivers/infiniband/hw/hfi1/uc.c qp->r_rcv_len += pmtu; qp 428 drivers/infiniband/hw/hfi1/uc.c if (unlikely(qp->r_rcv_len > qp->r_len)) qp 430 drivers/infiniband/hw/hfi1/uc.c rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false); qp 449 drivers/infiniband/hw/hfi1/uc.c wc.byte_len = tlen + qp->r_rcv_len; qp 450 drivers/infiniband/hw/hfi1/uc.c if (unlikely(wc.byte_len > qp->r_len)) qp 453 drivers/infiniband/hw/hfi1/uc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false); qp 454 drivers/infiniband/hw/hfi1/uc.c rvt_put_ss(&qp->s_rdma_read_sge); qp 456 drivers/infiniband/hw/hfi1/uc.c wc.wr_id = qp->r_wr_id; qp 458 drivers/infiniband/hw/hfi1/uc.c wc.qp = &qp->ibqp; qp 459 drivers/infiniband/hw/hfi1/uc.c wc.src_qp = qp->remote_qpn; qp 460 drivers/infiniband/hw/hfi1/uc.c wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; qp 472 drivers/infiniband/hw/hfi1/uc.c wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); qp 479 drivers/infiniband/hw/hfi1/uc.c rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); qp 486 drivers/infiniband/hw/hfi1/uc.c if (unlikely(!(qp->qp_access_flags & qp 491 drivers/infiniband/hw/hfi1/uc.c qp->r_len = be32_to_cpu(reth->length); qp 492 drivers/infiniband/hw/hfi1/uc.c qp->r_rcv_len = 0; qp 493 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.sg_list = NULL; qp 494 drivers/infiniband/hw/hfi1/uc.c if (qp->r_len != 0) { qp 500 drivers/infiniband/hw/hfi1/uc.c ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, qp 504 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.num_sge = 1; qp 506 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.num_sge = 0; qp 507 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.sge.mr = NULL; qp 508 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.sge.vaddr = NULL; qp 509 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.sge.length = 0; qp 510 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.sge.sge_length = 0; qp 523 drivers/infiniband/hw/hfi1/uc.c qp->r_rcv_len += pmtu; qp 524 drivers/infiniband/hw/hfi1/uc.c if (unlikely(qp->r_rcv_len > qp->r_len)) qp 526 drivers/infiniband/hw/hfi1/uc.c rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); qp 540 drivers/infiniband/hw/hfi1/uc.c if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) qp 542 drivers/infiniband/hw/hfi1/uc.c if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) { qp 543 drivers/infiniband/hw/hfi1/uc.c rvt_put_ss(&qp->s_rdma_read_sge); qp 545 drivers/infiniband/hw/hfi1/uc.c ret = rvt_get_rwqe(qp, true); qp 551 drivers/infiniband/hw/hfi1/uc.c wc.byte_len = qp->r_len; qp 553 drivers/infiniband/hw/hfi1/uc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); qp 554 drivers/infiniband/hw/hfi1/uc.c rvt_put_ss(&qp->r_sge); qp 565 drivers/infiniband/hw/hfi1/uc.c if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) qp 567 drivers/infiniband/hw/hfi1/uc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); qp 568 drivers/infiniband/hw/hfi1/uc.c rvt_put_ss(&qp->r_sge); qp 575 drivers/infiniband/hw/hfi1/uc.c qp->r_psn++; qp 576 drivers/infiniband/hw/hfi1/uc.c qp->r_state = opcode; qp 580 drivers/infiniband/hw/hfi1/uc.c set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); qp 581 drivers/infiniband/hw/hfi1/uc.c qp->r_sge.num_sge = 0; qp 587 drivers/infiniband/hw/hfi1/uc.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 78 drivers/infiniband/hw/hfi1/ud.c struct rvt_qp *qp; qp 89 drivers/infiniband/hw/hfi1/ud.c qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp, qp 91 drivers/infiniband/hw/hfi1/ud.c if (!qp) { qp 99 drivers/infiniband/hw/hfi1/ud.c dqptype = qp->ibqp.qp_type == IB_QPT_GSI ? qp 100 drivers/infiniband/hw/hfi1/ud.c IB_QPT_UD : qp->ibqp.qp_type; qp 103 drivers/infiniband/hw/hfi1/ud.c !(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { qp 111 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_num > 1) { qp 120 drivers/infiniband/hw/hfi1/ud.c qp->s_pkey_index, qp 124 drivers/infiniband/hw/hfi1/ud.c sqp->ibqp.qp_num, qp->ibqp.qp_num, qp 135 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_num) { qp 140 drivers/infiniband/hw/hfi1/ud.c if (unlikely(qkey != qp->qkey)) qp 157 drivers/infiniband/hw/hfi1/ud.c spin_lock_irqsave(&qp->r_lock, flags); qp 162 drivers/infiniband/hw/hfi1/ud.c if (qp->r_flags & RVT_R_REUSE_SGE) { qp 163 drivers/infiniband/hw/hfi1/ud.c qp->r_flags &= ~RVT_R_REUSE_SGE; qp 167 drivers/infiniband/hw/hfi1/ud.c ret = rvt_get_rwqe(qp, false); qp 169 drivers/infiniband/hw/hfi1/ud.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 173 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_num == 0) qp 179 drivers/infiniband/hw/hfi1/ud.c if (unlikely(wc.byte_len > qp->r_len)) { qp 180 drivers/infiniband/hw/hfi1/ud.c qp->r_flags |= RVT_R_REUSE_SGE; qp 214 drivers/infiniband/hw/hfi1/ud.c rvt_copy_sge(qp, &qp->r_sge, &grh, qp 218 drivers/infiniband/hw/hfi1/ud.c rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); qp 228 drivers/infiniband/hw/hfi1/ud.c rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false); qp 232 drivers/infiniband/hw/hfi1/ud.c rvt_put_ss(&qp->r_sge); qp 233 drivers/infiniband/hw/hfi1/ud.c if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) qp 235 drivers/infiniband/hw/hfi1/ud.c wc.wr_id = qp->r_wr_id; qp 238 drivers/infiniband/hw/hfi1/ud.c wc.qp = &qp->ibqp; qp 240 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) { qp 256 drivers/infiniband/hw/hfi1/ud.c wc.port_num = qp->port_num; qp 258 drivers/infiniband/hw/hfi1/ud.c rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); qp 261 drivers/infiniband/hw/hfi1/ud.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 266 drivers/infiniband/hw/hfi1/ud.c static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 273 drivers/infiniband/hw/hfi1/ud.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 284 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) qp 287 drivers/infiniband/hw/hfi1/ud.c *pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); qp 298 drivers/infiniband/hw/hfi1/ud.c cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : qp 300 drivers/infiniband/hw/hfi1/ud.c ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); qp 303 drivers/infiniband/hw/hfi1/ud.c void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 310 drivers/infiniband/hw/hfi1/ud.c struct hfi1_qp_priv *priv = qp->priv; qp 317 drivers/infiniband/hw/hfi1/ud.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 343 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_type == IB_QPT_SMI) { qp 365 drivers/infiniband/hw/hfi1/ud.c hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false); qp 374 drivers/infiniband/hw/hfi1/ud.c void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 377 drivers/infiniband/hw/hfi1/ud.c struct hfi1_qp_priv *priv = qp->priv; qp 384 drivers/infiniband/hw/hfi1/ud.c u32 src_qp = qp->ibqp.qp_num; qp 389 drivers/infiniband/hw/hfi1/ud.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 439 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_type == IB_QPT_SMI) qp 457 drivers/infiniband/hw/hfi1/ud.c hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true); qp 476 drivers/infiniband/hw/hfi1/ud.c int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) qp 478 drivers/infiniband/hw/hfi1/ud.c struct hfi1_qp_priv *priv = qp->priv; qp 486 drivers/infiniband/hw/hfi1/ud.c ps->s_txreq = get_txreq(ps->dev, qp); qp 490 drivers/infiniband/hw/hfi1/ud.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) { qp 491 drivers/infiniband/hw/hfi1/ud.c if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) qp 494 drivers/infiniband/hw/hfi1/ud.c if (qp->s_last == READ_ONCE(qp->s_head)) qp 498 drivers/infiniband/hw/hfi1/ud.c qp->s_flags |= RVT_S_WAIT_DMA; qp 501 drivers/infiniband/hw/hfi1/ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 502 drivers/infiniband/hw/hfi1/ud.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); qp 507 drivers/infiniband/hw/hfi1/ud.c if (qp->s_cur == READ_ONCE(qp->s_head)) qp 510 drivers/infiniband/hw/hfi1/ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 511 drivers/infiniband/hw/hfi1/ud.c next_cur = qp->s_cur + 1; qp 512 drivers/infiniband/hw/hfi1/ud.c if (next_cur >= qp->s_size) qp 516 drivers/infiniband/hw/hfi1/ud.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 526 drivers/infiniband/hw/hfi1/ud.c (qp->ibqp.qp_type == IB_QPT_GSI))))) { qp 536 drivers/infiniband/hw/hfi1/ud.c qp->s_flags |= RVT_S_WAIT_DMA; qp 539 drivers/infiniband/hw/hfi1/ud.c qp->s_cur = next_cur; qp 540 drivers/infiniband/hw/hfi1/ud.c spin_unlock_irqrestore(&qp->s_lock, tflags); qp 541 drivers/infiniband/hw/hfi1/ud.c ud_loopback(qp, wqe); qp 542 drivers/infiniband/hw/hfi1/ud.c spin_lock_irqsave(&qp->s_lock, tflags); qp 544 drivers/infiniband/hw/hfi1/ud.c rvt_send_complete(qp, wqe, IB_WC_SUCCESS); qp 549 drivers/infiniband/hw/hfi1/ud.c qp->s_cur = next_cur; qp 551 drivers/infiniband/hw/hfi1/ud.c ps->s_txreq->ss = &qp->s_sge; qp 552 drivers/infiniband/hw/hfi1/ud.c qp->s_srate = rdma_ah_get_static_rate(ah_attr); qp 553 drivers/infiniband/hw/hfi1/ud.c qp->srate_mbps = ib_rate_to_mbps(qp->s_srate); qp 554 drivers/infiniband/hw/hfi1/ud.c qp->s_wqe = wqe; qp 555 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.sge = wqe->sg_list[0]; qp 556 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.sg_list = wqe->sg_list + 1; qp 557 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.num_sge = wqe->wr.num_sge; qp 558 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.total_len = wqe->length; qp 561 drivers/infiniband/hw/hfi1/ud.c hfi1_make_ud_req_tbl[priv->hdr_type](qp, ps, qp->s_wqe); qp 562 drivers/infiniband/hw/hfi1/ud.c priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); qp 564 drivers/infiniband/hw/hfi1/ud.c priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); qp 583 drivers/infiniband/hw/hfi1/ud.c qp->s_flags &= ~RVT_S_BUSY; qp 632 drivers/infiniband/hw/hfi1/ud.c void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp, qp 643 drivers/infiniband/hw/hfi1/ud.c struct send_context *ctxt = qp_to_send_context(qp, sc5); qp 683 drivers/infiniband/hw/hfi1/ud.c pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen); qp 694 drivers/infiniband/hw/hfi1/ud.c void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn, qp 705 drivers/infiniband/hw/hfi1/ud.c struct send_context *ctxt = qp_to_send_context(qp, sc5); qp 738 drivers/infiniband/hw/hfi1/ud.c pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen); qp 768 drivers/infiniband/hw/hfi1/ud.c struct rvt_qp *qp, u16 slid, struct opa_smp *smp) qp 865 drivers/infiniband/hw/hfi1/ud.c struct rvt_qp *qp = packet->qp; qp 902 drivers/infiniband/hw/hfi1/ud.c process_ecn(qp, packet); qp 916 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_num) { qp 919 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_num > 1) { qp 929 drivers/infiniband/hw/hfi1/ud.c src_qp, qp->ibqp.qp_num, qp 940 drivers/infiniband/hw/hfi1/ud.c ib_get_qkey(packet->ohdr) != qp->qkey)) qp 944 drivers/infiniband/hw/hfi1/ud.c if (unlikely(qp->ibqp.qp_num == 1 && qp 951 drivers/infiniband/hw/hfi1/ud.c if (opa_smp_check(ibp, pkey, sc5, qp, slid, smp)) qp 966 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_num > 1 && qp 986 drivers/infiniband/hw/hfi1/ud.c if (qp->r_flags & RVT_R_REUSE_SGE) { qp 987 drivers/infiniband/hw/hfi1/ud.c qp->r_flags &= ~RVT_R_REUSE_SGE; qp 991 drivers/infiniband/hw/hfi1/ud.c ret = rvt_get_rwqe(qp, false); qp 993 drivers/infiniband/hw/hfi1/ud.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 997 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_num == 0) qp 1003 drivers/infiniband/hw/hfi1/ud.c if (unlikely(wc.byte_len > qp->r_len)) { qp 1004 drivers/infiniband/hw/hfi1/ud.c qp->r_flags |= RVT_R_REUSE_SGE; qp 1008 drivers/infiniband/hw/hfi1/ud.c rvt_copy_sge(qp, &qp->r_sge, packet->grh, qp 1019 drivers/infiniband/hw/hfi1/ud.c rvt_copy_sge(qp, &qp->r_sge, &grh, qp 1023 drivers/infiniband/hw/hfi1/ud.c rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); qp 1025 drivers/infiniband/hw/hfi1/ud.c rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), qp 1027 drivers/infiniband/hw/hfi1/ud.c rvt_put_ss(&qp->r_sge); qp 1028 drivers/infiniband/hw/hfi1/ud.c if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) qp 1030 drivers/infiniband/hw/hfi1/ud.c wc.wr_id = qp->r_wr_id; qp 1034 drivers/infiniband/hw/hfi1/ud.c wc.qp = &qp->ibqp; qp 1037 drivers/infiniband/hw/hfi1/ud.c if (qp->ibqp.qp_type == IB_QPT_GSI || qp 1038 drivers/infiniband/hw/hfi1/ud.c qp->ibqp.qp_type == IB_QPT_SMI) { qp 1044 drivers/infiniband/hw/hfi1/ud.c qp->ibqp.qp_type); qp 1062 drivers/infiniband/hw/hfi1/ud.c wc.port_num = qp->port_num; qp 1064 drivers/infiniband/hw/hfi1/ud.c rvt_recv_cq(qp, &wc, solicited); qp 142 drivers/infiniband/hw/hfi1/verbs.c static int pio_wait(struct rvt_qp *qp, qp 317 drivers/infiniband/hw/hfi1/verbs.c if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK)) qp 320 drivers/infiniband/hw/hfi1/verbs.c packet->qp->allowed_ops) || qp 327 drivers/infiniband/hw/hfi1/verbs.c static u64 hfi1_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc) qp 360 drivers/infiniband/hw/hfi1/verbs.c if (packet->qp->ibqp.qp_type != IB_QPT_RC || qp 361 drivers/infiniband/hw/hfi1/verbs.c !(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK)) qp 401 drivers/infiniband/hw/hfi1/verbs.c packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); qp 402 drivers/infiniband/hw/hfi1/verbs.c if (!packet->qp) qp 404 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&packet->qp->r_lock, flags); qp 410 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&packet->qp->r_lock, flags); qp 415 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&packet->qp->r_lock, flags); qp 455 drivers/infiniband/hw/hfi1/verbs.c packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); qp 456 drivers/infiniband/hw/hfi1/verbs.c if (!packet->qp) qp 458 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&packet->qp->r_lock, flags); qp 464 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&packet->qp->r_lock, flags); qp 469 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&packet->qp->r_lock, flags); qp 490 drivers/infiniband/hw/hfi1/verbs.c packet->qp->s_pkey_index, qp 520 drivers/infiniband/hw/hfi1/verbs.c packet->qp = p->qp; qp 523 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&packet->qp->r_lock, flags); qp 529 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&packet->qp->r_lock, flags); qp 547 drivers/infiniband/hw/hfi1/verbs.c packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); qp 548 drivers/infiniband/hw/hfi1/verbs.c if (!packet->qp) qp 554 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&packet->qp->r_lock, flags); qp 560 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&packet->qp->r_lock, flags); qp 600 drivers/infiniband/hw/hfi1/verbs.c struct rvt_qp *qp = NULL; qp 608 drivers/infiniband/hw/hfi1/verbs.c qp = iowait_to_qp(wait); qp 609 drivers/infiniband/hw/hfi1/verbs.c priv = qp->priv; qp 618 drivers/infiniband/hw/hfi1/verbs.c if (qp) qp 619 drivers/infiniband/hw/hfi1/verbs.c hfi1_qp_wakeup(qp, RVT_S_WAIT_KMEM); qp 632 drivers/infiniband/hw/hfi1/verbs.c struct rvt_qp *qp = tx->qp; qp 634 drivers/infiniband/hw/hfi1/verbs.c spin_lock(&qp->s_lock); qp 636 drivers/infiniband/hw/hfi1/verbs.c rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS); qp 637 drivers/infiniband/hw/hfi1/verbs.c } else if (qp->ibqp.qp_type == IB_QPT_RC) { qp 642 drivers/infiniband/hw/hfi1/verbs.c hfi1_rc_verbs_aborted(qp, hdr); qp 643 drivers/infiniband/hw/hfi1/verbs.c hfi1_rc_send_complete(qp, hdr); qp 645 drivers/infiniband/hw/hfi1/verbs.c spin_unlock(&qp->s_lock); qp 650 drivers/infiniband/hw/hfi1/verbs.c void hfi1_wait_kmem(struct rvt_qp *qp) qp 652 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_qp_priv *priv = qp->priv; qp 653 drivers/infiniband/hw/hfi1/verbs.c struct ib_qp *ibqp = &qp->ibqp; qp 660 drivers/infiniband/hw/hfi1/verbs.c qp->s_flags |= RVT_S_WAIT_KMEM; qp 663 drivers/infiniband/hw/hfi1/verbs.c trace_hfi1_qpsleep(qp, RVT_S_WAIT_KMEM); qp 664 drivers/infiniband/hw/hfi1/verbs.c rvt_get_qp(qp); qp 669 drivers/infiniband/hw/hfi1/verbs.c struct rvt_qp *qp, qp 675 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 676 drivers/infiniband/hw/hfi1/verbs.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 680 drivers/infiniband/hw/hfi1/verbs.c hfi1_wait_kmem(qp); qp 682 drivers/infiniband/hw/hfi1/verbs.c hfi1_qp_unbusy(qp, ps->wait); qp 685 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 738 drivers/infiniband/hw/hfi1/verbs.c static void update_tx_opstats(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 742 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 838 drivers/infiniband/hw/hfi1/verbs.c int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 841 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_qp_priv *priv = qp->priv; qp 866 drivers/infiniband/hw/hfi1/verbs.c u32 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5); qp 878 drivers/infiniband/hw/hfi1/verbs.c qp->srate_mbps, qp 882 drivers/infiniband/hw/hfi1/verbs.c if (unlikely(hfi1_dbg_should_fault_tx(qp, ps->opcode))) qp 883 drivers/infiniband/hw/hfi1/verbs.c pbc = hfi1_fault_tx(qp, ps->opcode, pbc); qp 888 drivers/infiniband/hw/hfi1/verbs.c tx->wqe = qp->s_wqe; qp 900 drivers/infiniband/hw/hfi1/verbs.c update_tx_opstats(qp, ps, plen); qp 901 drivers/infiniband/hw/hfi1/verbs.c trace_sdma_output_ibhdr(dd_from_ibdev(qp->ibqp.device), qp 909 drivers/infiniband/hw/hfi1/verbs.c ret = wait_kmem(dev, qp, ps); qp 922 drivers/infiniband/hw/hfi1/verbs.c static int pio_wait(struct rvt_qp *qp, qp 927 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_qp_priv *priv = qp->priv; qp 938 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 939 drivers/infiniband/hw/hfi1/verbs.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 949 drivers/infiniband/hw/hfi1/verbs.c qp->s_flags |= flag; qp 955 drivers/infiniband/hw/hfi1/verbs.c trace_hfi1_qpsleep(qp, RVT_S_WAIT_PIO); qp 956 drivers/infiniband/hw/hfi1/verbs.c rvt_get_qp(qp); qp 962 drivers/infiniband/hw/hfi1/verbs.c hfi1_qp_unbusy(qp, ps->wait); qp 965 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 971 drivers/infiniband/hw/hfi1/verbs.c struct rvt_qp *qp = (struct rvt_qp *)arg; qp 972 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_qp_priv *priv = qp->priv; qp 978 drivers/infiniband/hw/hfi1/verbs.c int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 981 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_qp_priv *priv = qp->priv; qp 1011 drivers/infiniband/hw/hfi1/verbs.c switch (qp->ibqp.qp_type) { qp 1025 drivers/infiniband/hw/hfi1/verbs.c u8 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5); qp 1033 drivers/infiniband/hw/hfi1/verbs.c pbc = create_pbc(ppd, pbc, qp->srate_mbps, vl, plen); qp 1034 drivers/infiniband/hw/hfi1/verbs.c if (unlikely(hfi1_dbg_should_fault_tx(qp, ps->opcode))) qp 1035 drivers/infiniband/hw/hfi1/verbs.c pbc = hfi1_fault_tx(qp, ps->opcode, pbc); qp 1042 drivers/infiniband/hw/hfi1/verbs.c pbuf = sc_buffer_alloc(sc, plen, cb, qp); qp 1045 drivers/infiniband/hw/hfi1/verbs.c verbs_pio_complete(qp, 0); qp 1065 drivers/infiniband/hw/hfi1/verbs.c ret = pio_wait(qp, sc, ps, RVT_S_WAIT_PIO); qp 1097 drivers/infiniband/hw/hfi1/verbs.c update_tx_opstats(qp, ps, plen); qp 1098 drivers/infiniband/hw/hfi1/verbs.c trace_pio_output_ibhdr(dd_from_ibdev(qp->ibqp.device), qp 1102 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 1103 drivers/infiniband/hw/hfi1/verbs.c if (qp->s_wqe) { qp 1104 drivers/infiniband/hw/hfi1/verbs.c rvt_send_complete(qp, qp->s_wqe, wc_status); qp 1105 drivers/infiniband/hw/hfi1/verbs.c } else if (qp->ibqp.qp_type == IB_QPT_RC) { qp 1107 drivers/infiniband/hw/hfi1/verbs.c hfi1_rc_verbs_aborted(qp, &ps->s_txreq->phdr.hdr); qp 1108 drivers/infiniband/hw/hfi1/verbs.c hfi1_rc_send_complete(qp, &ps->s_txreq->phdr.hdr); qp 1110 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1214 drivers/infiniband/hw/hfi1/verbs.c static inline send_routine get_send_routine(struct rvt_qp *qp, qp 1217 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 1218 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_qp_priv *priv = qp->priv; qp 1223 drivers/infiniband/hw/hfi1/verbs.c switch (qp->ibqp.qp_type) { qp 1234 drivers/infiniband/hw/hfi1/verbs.c priv->s_running_pkt_size <= min(piothreshold, qp->pmtu) && qp 1254 drivers/infiniband/hw/hfi1/verbs.c int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps) qp 1256 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 1257 drivers/infiniband/hw/hfi1/verbs.c struct hfi1_qp_priv *priv = qp->priv; qp 1294 drivers/infiniband/hw/hfi1/verbs.c sr = get_send_routine(qp, ps); qp 1296 drivers/infiniband/hw/hfi1/verbs.c priv->s_sc, qp->s_pkey_index); qp 1311 drivers/infiniband/hw/hfi1/verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 1312 drivers/infiniband/hw/hfi1/verbs.c rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR); qp 1313 drivers/infiniband/hw/hfi1/verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1318 drivers/infiniband/hw/hfi1/verbs.c return pio_wait(qp, qp 1322 drivers/infiniband/hw/hfi1/verbs.c return sr(qp, ps, 0); qp 1616 drivers/infiniband/hw/hfi1/verbs.c RCU_INIT_POINTER(ibp->rvp.qp[0], NULL); qp 1617 drivers/infiniband/hw/hfi1/verbs.c RCU_INIT_POINTER(ibp->rvp.qp[1], NULL); qp 1979 drivers/infiniband/hw/hfi1/verbs.c struct rvt_qp *qp = packet->qp; qp 1984 drivers/infiniband/hw/hfi1/verbs.c switch (packet->qp->ibqp.qp_type) { qp 1986 drivers/infiniband/hw/hfi1/verbs.c rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); qp 1987 drivers/infiniband/hw/hfi1/verbs.c rqpn = qp->remote_qpn; qp 1991 drivers/infiniband/hw/hfi1/verbs.c rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); qp 1992 drivers/infiniband/hw/hfi1/verbs.c rqpn = qp->remote_qpn; qp 2007 drivers/infiniband/hw/hfi1/verbs.c lqpn = qp->ibqp.qp_num; qp 269 drivers/infiniband/hw/hfi1/verbs.h struct rvt_qp __rcu *qp[2]; qp 404 drivers/infiniband/hw/hfi1/verbs.h int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps); qp 415 drivers/infiniband/hw/hfi1/verbs.h struct rvt_qp *qp); qp 419 drivers/infiniband/hw/hfi1/verbs.h void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah); qp 420 drivers/infiniband/hw/hfi1/verbs.h void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah); qp 426 drivers/infiniband/hw/hfi1/verbs.h void hfi1_migrate_qp(struct rvt_qp *qp); qp 428 drivers/infiniband/hw/hfi1/verbs.h int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, qp 431 drivers/infiniband/hw/hfi1/verbs.h void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, qp 433 drivers/infiniband/hw/hfi1/verbs.h void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait); qp 434 drivers/infiniband/hw/hfi1/verbs.h int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 445 drivers/infiniband/hw/hfi1/verbs.h void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, qp 449 drivers/infiniband/hw/hfi1/verbs.h bool hfi1_schedule_send_yield(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 454 drivers/infiniband/hw/hfi1/verbs.h void hfi1_do_send_from_rvt(struct rvt_qp *qp); qp 456 drivers/infiniband/hw/hfi1/verbs.h void hfi1_do_send(struct rvt_qp *qp, bool in_thread); qp 460 drivers/infiniband/hw/hfi1/verbs.h int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); qp 462 drivers/infiniband/hw/hfi1/verbs.h int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); qp 464 drivers/infiniband/hw/hfi1/verbs.h int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); qp 480 drivers/infiniband/hw/hfi1/verbs.h int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 483 drivers/infiniband/hw/hfi1/verbs.h int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, qp 491 drivers/infiniband/hw/hfi1/verbs.h void hfi1_wait_kmem(struct rvt_qp *qp); qp 493 drivers/infiniband/hw/hfi1/verbs.h static inline void hfi1_trdma_send_complete(struct rvt_qp *qp, qp 497 drivers/infiniband/hw/hfi1/verbs.h trdma_clean_swqe(qp, wqe); qp 498 drivers/infiniband/hw/hfi1/verbs.h rvt_send_complete(qp, wqe, status); qp 58 drivers/infiniband/hw/hfi1/verbs_txreq.c struct rvt_qp *qp; qp 63 drivers/infiniband/hw/hfi1/verbs_txreq.c qp = tx->qp; qp 64 drivers/infiniband/hw/hfi1/verbs_txreq.c dev = to_idev(qp->ibqp.device); qp 82 drivers/infiniband/hw/hfi1/verbs_txreq.c qp = iowait_to_qp(wait); qp 83 drivers/infiniband/hw/hfi1/verbs_txreq.c priv = qp->priv; qp 87 drivers/infiniband/hw/hfi1/verbs_txreq.c hfi1_qp_wakeup(qp, RVT_S_WAIT_TX); qp 94 drivers/infiniband/hw/hfi1/verbs_txreq.c struct rvt_qp *qp) qp 95 drivers/infiniband/hw/hfi1/verbs_txreq.c __must_hold(&qp->s_lock) qp 100 drivers/infiniband/hw/hfi1/verbs_txreq.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 106 drivers/infiniband/hw/hfi1/verbs_txreq.c priv = qp->priv; qp 109 drivers/infiniband/hw/hfi1/verbs_txreq.c qp->s_flags |= RVT_S_WAIT_TX; qp 112 drivers/infiniband/hw/hfi1/verbs_txreq.c trace_hfi1_qpsleep(qp, RVT_S_WAIT_TX); qp 113 drivers/infiniband/hw/hfi1/verbs_txreq.c rvt_get_qp(qp); qp 115 drivers/infiniband/hw/hfi1/verbs_txreq.c qp->s_flags &= ~RVT_S_BUSY; qp 61 drivers/infiniband/hw/hfi1/verbs_txreq.h struct rvt_qp *qp; qp 73 drivers/infiniband/hw/hfi1/verbs_txreq.h struct rvt_qp *qp); qp 77 drivers/infiniband/hw/hfi1/verbs_txreq.h struct rvt_qp *qp) qp 78 drivers/infiniband/hw/hfi1/verbs_txreq.h __must_hold(&qp->slock) qp 81 drivers/infiniband/hw/hfi1/verbs_txreq.h struct hfi1_qp_priv *priv = qp->priv; qp 86 drivers/infiniband/hw/hfi1/verbs_txreq.h tx = __get_txreq(dev, qp); qp 90 drivers/infiniband/hw/hfi1/verbs_txreq.h tx->qp = qp; qp 684 drivers/infiniband/hw/hns/hns_roce_device.h void (*event)(struct hns_roce_qp *qp, qp 721 drivers/infiniband/hw/hns/hns_roce_device.h __le32 qp; qp 971 drivers/infiniband/hw/hns/hns_roce_device.h int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, qp 70 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_qp *qp = to_hr_qp(ibqp); qp 90 drivers/infiniband/hw/hns/hns_roce_hw_v1.c spin_lock_irqsave(&qp->sq.lock, flags); qp 93 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { qp 99 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); qp 101 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (unlikely(wr->num_sge > qp->sq.max_gs)) { qp 103 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wr->num_sge, qp->sq.max_gs); qp 109 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wqe = get_send_wqe(qp, wqe_idx); qp 110 drivers/infiniband/hw/hns/hns_roce_hw_v1.c qp->sq.wrid[wqe_idx] = wr->wr_id; qp 141 drivers/infiniband/hw/hns/hns_roce_hw_v1.c smac = (u8 *)hr_dev->dev_addr[qp->port]; qp 187 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_get_gid_index(hr_dev, qp->phy_port, qp 317 drivers/infiniband/hw/hns/hns_roce_hw_v1.c qp->sq.head += nreq; qp 325 drivers/infiniband/hw/hns/hns_roce_hw_v1.c (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1))); qp 327 drivers/infiniband/hw/hns/hns_roce_hw_v1.c SQ_DOORBELL_U32_4_SL_S, qp->sl); qp 329 drivers/infiniband/hw/hns/hns_roce_hw_v1.c SQ_DOORBELL_U32_4_PORT_S, qp->phy_port); qp 331 drivers/infiniband/hw/hns/hns_roce_hw_v1.c SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn); qp 337 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_write64_k(doorbell, qp->sq.db_reg_l); qp 340 drivers/infiniband/hw/hns/hns_roce_hw_v1.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 681 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct ib_qp *qp; qp 689 drivers/infiniband/hw/hns/hns_roce_hw_v1.c qp = hns_roce_create_qp(pd, &init_attr, NULL); qp 690 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (IS_ERR(qp)) { qp 695 drivers/infiniband/hw/hns/hns_roce_hw_v1.c return to_hr_qp(qp); qp 2235 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wc->qp = &(*cur_qp)->ibqp; qp 3777 drivers/infiniband/hw/hns/hns_roce_hw_v1.c qpn = roce_get_field(aeqe->event.qp_event.qp, qp 3780 drivers/infiniband/hw/hns/hns_roce_hw_v1.c phy_port = roce_get_field(aeqe->event.qp_event.qp, qp 1058 drivers/infiniband/hw/hns/hns_roce_hw_v1.h struct hns_roce_qp *qp; qp 112 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, qp 124 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) qp 128 drivers/infiniband/hw/hns/hns_roce_hw_v2.c shift = qp->hr_buf.page_shift; qp 135 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1)); qp 145 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dseg = get_send_extend_sge(qp, qp 146 drivers/infiniband/hw/hns/hns_roce_hw_v2.c (*sge_ind) & (qp->sge.sge_cnt - 1)); qp 167 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_qp *qp = to_hr_qp(ibqp); qp 206 drivers/infiniband/hw/hns/hns_roce_hw_v2.c (*sge_ind) & (qp->sge.sge_cnt - 1)); qp 217 drivers/infiniband/hw/hns/hns_roce_hw_v2.c set_extend_sge(qp, wr, sge_ind, valid_num_sge); qp 241 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_qp *qp = to_hr_qp(ibqp); qp 269 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT || qp 270 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->state == IB_QPS_RTR)) { qp 271 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state); qp 276 drivers/infiniband/hw/hns/hns_roce_hw_v2.c spin_lock_irqsave(&qp->sq.lock, flags); qp 277 drivers/infiniband/hw/hns/hns_roce_hw_v2.c sge_idx = qp->next_sge; qp 280 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { qp 286 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); qp 288 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (unlikely(wr->num_sge > qp->sq.max_gs)) { qp 290 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wr->num_sge, qp->sq.max_gs); qp 296 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe = get_send_wqe(qp, wqe_idx); qp 297 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->sq.wrid[wqe_idx] = wr->wr_id; qp 299 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); qp 333 drivers/infiniband/hw/hns/hns_roce_hw_v2.c smac = (u8 *)hr_dev->dev_addr[qp->port]; qp 385 drivers/infiniband/hw/hns/hns_roce_hw_v2.c sge_idx & (qp->sge.sge_cnt - 1)); qp 392 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->qkey : ud_wr(wr)->remote_qkey); qp 421 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->port); qp 429 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_get_gid_index(hr_dev, qp->phy_port, qp 435 drivers/infiniband/hw/hns/hns_roce_hw_v2.c set_extend_sge(qp, wr, &sge_idx, valid_num_sge); qp 569 drivers/infiniband/hw/hns/hns_roce_hw_v2.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 577 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->sq.head += nreq; qp 585 drivers/infiniband/hw/hns/hns_roce_hw_v2.c V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn); qp 590 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)); qp 592 drivers/infiniband/hw/hns/hns_roce_hw_v2.c V2_DB_PARAMETER_SL_S, qp->sl); qp 594 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l); qp 596 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->next_sge = sge_idx; qp 598 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (qp->state == IB_QPS_ERR) { qp 602 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask, qp 603 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qp->state, IB_QPS_ERR); qp 605 drivers/infiniband/hw/hns/hns_roce_hw_v2.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 612 drivers/infiniband/hw/hns/hns_roce_hw_v2.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 2737 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wc->qp = &(*cur_qp)->ibqp; qp 2923 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if ((wc->qp->qp_type == IB_QPT_RC || qp 2924 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wc->qp->qp_type == IB_QPT_UC) && qp 5047 drivers/infiniband/hw/hns/hns_roce_hw_v2.c qpn = roce_get_field(aeqe->event.qp_event.qp, qp 49 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_qp *qp; qp 52 drivers/infiniband/hw/hns/hns_roce_qp.c qp = __hns_roce_qp_lookup(hr_dev, qpn); qp 53 drivers/infiniband/hw/hns/hns_roce_qp.c if (qp) qp 54 drivers/infiniband/hw/hns/hns_roce_qp.c atomic_inc(&qp->refcount); qp 57 drivers/infiniband/hw/hns/hns_roce_qp.c if (!qp) { qp 62 drivers/infiniband/hw/hns/hns_roce_qp.c qp->event(qp, (enum hns_roce_event)event_type); qp 64 drivers/infiniband/hw/hns/hns_roce_qp.c if (atomic_dec_and_test(&qp->refcount)) qp 65 drivers/infiniband/hw/hns/hns_roce_qp.c complete(&qp->free); qp 76 drivers/infiniband/hw/hns/hns_roce_qp.c event.element.qp = ibqp; qp 406 drivers/infiniband/hw/i40iw/i40iw.h struct i40iw_qp *qp); qp 536 drivers/infiniband/hw/i40iw/i40iw.h struct i40iw_sc_qp *qp, qp 573 drivers/infiniband/hw/i40iw/i40iw.h struct i40iw_sc_qp *qp, qp 578 drivers/infiniband/hw/i40iw/i40iw.h struct i40iw_sc_qp *qp, qp 3509 drivers/infiniband/hw/i40iw/i40iw_cm.c struct i40iw_sc_qp *qp = &iwqp->sc_qp; qp 3540 drivers/infiniband/hw/i40iw/i40iw_cm.c if (qp->term_flags) { qp 3548 drivers/infiniband/hw/i40iw/i40iw_cm.c i40iw_terminate_del_timer(qp); qp 3579 drivers/infiniband/hw/i40iw/i40iw_cm.c if (qp->term_flags && iwqp->ibqp.event_handler) { qp 3581 drivers/infiniband/hw/i40iw/i40iw_cm.c ibevent.event = (qp->eventtype == TERM_EVENT_QP_FATAL) ? qp 3583 drivers/infiniband/hw/i40iw/i40iw_cm.c ibevent.element.qp = &iwqp->ibqp; qp 319 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp) qp 327 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!qp) { qp 330 drivers/infiniband/hw/i40iw/i40iw_ctrl.c lastentry = &qp->list; qp 345 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp = NULL; qp 362 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp = i40iw_get_qp(&vsi->qos[i].qplist, qp); qp 363 drivers/infiniband/hw/i40iw/i40iw_ctrl.c while (qp) { qp 365 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->qs_handle = qs_handle; qp 367 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_qp_suspend_resume(dev, qp, true); qp 369 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp = i40iw_get_qp(&vsi->qos[i].qplist, qp); qp 380 drivers/infiniband/hw/i40iw/i40iw_ctrl.c void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp) qp 382 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_vsi *vsi = qp->vsi; qp 385 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!qp->on_qoslist) qp 387 drivers/infiniband/hw/i40iw/i40iw_ctrl.c spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags); qp 388 drivers/infiniband/hw/i40iw/i40iw_ctrl.c list_del(&qp->list); qp 389 drivers/infiniband/hw/i40iw/i40iw_ctrl.c spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags); qp 396 drivers/infiniband/hw/i40iw/i40iw_ctrl.c void i40iw_qp_add_qos(struct i40iw_sc_qp *qp) qp 398 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_vsi *vsi = qp->vsi; qp 401 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (qp->on_qoslist) qp 403 drivers/infiniband/hw/i40iw/i40iw_ctrl.c spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags); qp 404 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->qs_handle = vsi->qos[qp->user_pri].qs_handle; qp 405 drivers/infiniband/hw/i40iw/i40iw_ctrl.c list_add(&qp->list, &vsi->qos[qp->user_pri].qplist); qp 406 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->on_qoslist = true; qp 407 drivers/infiniband/hw/i40iw/i40iw_ctrl.c spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags); qp 1833 drivers/infiniband/hw/i40iw/i40iw_ctrl.c info->qp = true; qp 1847 drivers/infiniband/hw/i40iw/i40iw_ctrl.c info->qp = true; qp 1860 drivers/infiniband/hw/i40iw/i40iw_ctrl.c info->qp = true; qp 1867 drivers/infiniband/hw/i40iw/i40iw_ctrl.c info->qp = true; qp 1873 drivers/infiniband/hw/i40iw/i40iw_ctrl.c info->qp = true; qp 2355 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp, qp 2364 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->dev = info->pd->dev; qp 2365 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->vsi = info->vsi; qp 2366 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->sq_pa = info->sq_pa; qp 2367 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->rq_pa = info->rq_pa; qp 2368 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->hw_host_ctx_pa = info->host_ctx_pa; qp 2369 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->q2_pa = info->q2_pa; qp 2370 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->shadow_area_pa = info->shadow_area_pa; qp 2372 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->q2_buf = info->q2; qp 2373 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->pd = info->pd; qp 2374 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->hw_host_ctx = info->host_ctx; qp 2375 drivers/infiniband/hw/i40iw/i40iw_ctrl.c offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1; qp 2376 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (i40iw_get_hw_addr(qp->pd->dev)) qp 2377 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) + qp 2381 drivers/infiniband/hw/i40iw/i40iw_ctrl.c info->qp_uk_init_info.abi_ver = qp->pd->abi_ver; qp 2382 drivers/infiniband/hw/i40iw/i40iw_ctrl.c ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info); qp 2385 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->virtual_map = info->virtual_map; qp 2393 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->llp_stream_handle = (void *)(-1); qp 2394 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP; qp 2396 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, qp 2398 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n", qp 2399 drivers/infiniband/hw/i40iw/i40iw_ctrl.c __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size); qp 2401 drivers/infiniband/hw/i40iw/i40iw_ctrl.c switch (qp->pd->abi_ver) { qp 2403 drivers/infiniband/hw/i40iw/i40iw_ctrl.c ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt, qp 2410 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (qp->qp_uk.max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT) qp 2415 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size * qp 2417 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug(qp->dev, I40IW_DEBUG_WQE, qp 2419 drivers/infiniband/hw/i40iw/i40iw_ctrl.c __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size); qp 2420 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->sq_tph_val = info->sq_tph_val; qp 2421 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->rq_tph_val = info->rq_tph_val; qp 2422 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->sq_tph_en = info->sq_tph_en; qp 2423 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->rq_tph_en = info->rq_tph_en; qp 2424 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->rcv_tph_en = info->rcv_tph_en; qp 2425 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->xmit_tph_en = info->xmit_tph_en; qp 2426 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle; qp 2439 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 2448 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) || qp 2449 drivers/infiniband/hw/i40iw/i40iw_ctrl.c (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID)) qp 2452 drivers/infiniband/hw/i40iw/i40iw_ctrl.c cqp = qp->pd->dev->cqp; qp 2457 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); qp 2459 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, qp->shadow_area_pa); qp 2461 drivers/infiniband/hw/i40iw/i40iw_ctrl.c header = qp->qp_uk.qp_id | qp 2465 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) | qp 2466 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) | qp 2489 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 2500 drivers/infiniband/hw/i40iw/i40iw_ctrl.c cqp = qp->pd->dev->cqp; qp 2518 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); qp 2519 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, qp->shadow_area_pa); qp 2521 drivers/infiniband/hw/i40iw/i40iw_ctrl.c header = qp->qp_uk.qp_id | qp 2526 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) | qp 2529 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) | qp 2556 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 2566 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_qp_rem_qos(qp); qp 2567 drivers/infiniband/hw/i40iw/i40iw_ctrl.c cqp = qp->pd->dev->cqp; qp 2571 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); qp 2572 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, qp->shadow_area_pa); qp 2574 drivers/infiniband/hw/i40iw/i40iw_ctrl.c header = qp->qp_uk.qp_id | qp 2576 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) | qp 2598 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 2609 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (info->rq && !qp->flush_rq) qp 2612 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (info->sq && !qp->flush_sq) qp 2615 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->flush_sq |= flush_sq; qp 2616 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->flush_rq |= flush_rq; qp 2620 drivers/infiniband/hw/i40iw/i40iw_ctrl.c cqp = qp->pd->dev->cqp; qp 2641 drivers/infiniband/hw/i40iw/i40iw_ctrl.c header = qp->qp_uk.qp_id | qp 2667 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 2677 drivers/infiniband/hw/i40iw/i40iw_ctrl.c cqp = qp->pd->dev->cqp; qp 2687 drivers/infiniband/hw/i40iw/i40iw_ctrl.c header = qp->qp_uk.qp_id | qp 2749 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 2761 drivers/infiniband/hw/i40iw/i40iw_ctrl.c vsi = qp->vsi; qp 2762 drivers/infiniband/hw/i40iw/i40iw_ctrl.c dev = qp->dev; qp 2764 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->user_pri = info->user_pri; qp 2765 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_qp_add_qos(qp); qp 2766 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n", qp 2767 drivers/infiniband/hw/i40iw/i40iw_ctrl.c __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle); qp 2769 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) | qp 2771 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) | qp 2772 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) | qp 2773 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) | qp 2774 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) | qp 2778 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(qp_ctx, 8, qp->sq_pa); qp 2779 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(qp_ctx, 16, qp->rq_pa); qp 2781 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) | qp 2782 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) | qp 2783 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE); qp 2799 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) | qp 2800 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) | qp 2801 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) | qp 2811 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) | qp 2853 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->src_mac_addr_idx = tcp->src_mac_addr_idx; qp 2925 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE", qp 3257 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 3267 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, qp 3272 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n", qp 3274 drivers/infiniband/hw/i40iw/i40iw_ctrl.c &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); qp 3299 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); qp 3303 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE", qp 3307 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_qp_post_wr(&qp->qp_uk); qp 3318 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp, qp 3327 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp_uk = &qp->qp_uk; qp 3339 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); qp 3343 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE", qp 3353 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp, qp 3361 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp_uk = &qp->qp_uk; qp 3373 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); qp 3377 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE", qp 3386 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read) qp 3392 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp_uk = &qp->qp_uk; qp 3401 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); qp 3405 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); qp 3410 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE", qp 3419 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode) qp 3425 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp_uk = &qp->qp_uk; qp 3436 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); qp 3445 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID) | qp 3452 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_debug(qp->dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n", qp 3753 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 3762 drivers/infiniband/hw/i40iw/i40iw_ctrl.c header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) | qp 3782 drivers/infiniband/hw/i40iw/i40iw_ctrl.c struct i40iw_sc_qp *qp, qp 3793 drivers/infiniband/hw/i40iw/i40iw_ctrl.c LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE)); qp 3795 drivers/infiniband/hw/i40iw/i40iw_ctrl.c header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) | qp 4108 drivers/infiniband/hw/i40iw/i40iw_ctrl.c pcmdinfo->in.u.qp_modify.qp, qp 4138 drivers/infiniband/hw/i40iw/i40iw_ctrl.c pcmdinfo->in.u.qp_create.qp, qp 4145 drivers/infiniband/hw/i40iw/i40iw_ctrl.c pcmdinfo->in.u.qp_destroy.qp, qp 4187 drivers/infiniband/hw/i40iw/i40iw_ctrl.c pcmdinfo->in.u.qp_flush_wqes.qp, qp 4194 drivers/infiniband/hw/i40iw/i40iw_ctrl.c pcmdinfo->in.u.gen_ae.qp, qp 4233 drivers/infiniband/hw/i40iw/i40iw_ctrl.c pcmdinfo->in.u.suspend_resume.qp, qp 4239 drivers/infiniband/hw/i40iw/i40iw_ctrl.c pcmdinfo->in.u.suspend_resume.qp, qp 4358 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp, qp 4364 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->flush_code = opcode; qp 4374 drivers/infiniband/hw/i40iw/i40iw_ctrl.c static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp, qp 4377 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; qp 4384 drivers/infiniband/hw/i40iw/i40iw_ctrl.c termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf; qp 4420 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; qp 4422 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR, qp 4425 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4429 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; qp 4431 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR, qp 4434 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4440 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR, qp 4445 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4449 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4454 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; qp 4455 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4459 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR, qp 4464 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; qp 4468 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR, qp 4472 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4479 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; qp 4480 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4484 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; qp 4485 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR, qp 4489 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, qp 4494 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR, qp 4499 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR, qp 4503 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, qp 4507 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_ACCESS_ERR; qp 4508 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR, qp 4513 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, qp 4516 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, qp 4520 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, qp 4524 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR, qp 4528 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, qp 4532 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR, qp 4536 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR, qp 4540 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR, qp 4555 drivers/infiniband/hw/i40iw/i40iw_ctrl.c void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp) qp 4558 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_term_modify_qp(qp, qp 4569 drivers/infiniband/hw/i40iw/i40iw_ctrl.c void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info) qp 4573 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (qp->term_flags & I40IW_TERM_SENT) qp 4577 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_FATAL; qp 4578 drivers/infiniband/hw/i40iw/i40iw_ctrl.c termlen = i40iw_bld_terminate_hdr(qp, info); qp 4579 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_terminate_start_timer(qp); qp 4580 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->term_flags |= I40IW_TERM_SENT; qp 4581 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE, qp 4590 drivers/infiniband/hw/i40iw/i40iw_ctrl.c void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info) qp 4592 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; qp 4620 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_terminate_connection(qp, info); qp 4625 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->term_flags |= I40IW_TERM_RCVD; qp 4626 drivers/infiniband/hw/i40iw/i40iw_ctrl.c qp->eventtype = TERM_EVENT_QP_FATAL; qp 4630 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_terminate_done(qp, 0); qp 4632 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_terminate_start_timer(qp); qp 4633 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_terminate_send_fin(qp); qp 287 drivers/infiniband/hw/i40iw/i40iw_hw.c struct i40iw_sc_qp *qp = NULL; qp 305 drivers/infiniband/hw/i40iw/i40iw_hw.c __func__, info->ae_id, info->qp, info->qp_cq_id); qp 306 drivers/infiniband/hw/i40iw/i40iw_hw.c if (info->qp) { qp 318 drivers/infiniband/hw/i40iw/i40iw_hw.c qp = &iwqp->sc_qp; qp 333 drivers/infiniband/hw/i40iw/i40iw_hw.c if (qp->term_flags) qp 350 drivers/infiniband/hw/i40iw/i40iw_hw.c if (qp->term_flags) qp 351 drivers/infiniband/hw/i40iw/i40iw_hw.c i40iw_terminate_done(qp, 0); qp 370 drivers/infiniband/hw/i40iw/i40iw_hw.c i40iw_terminate_send_fin(qp); qp 373 drivers/infiniband/hw/i40iw/i40iw_hw.c i40iw_terminate_received(qp, info); qp 426 drivers/infiniband/hw/i40iw/i40iw_hw.c i40iw_terminate_connection(qp, info); qp 429 drivers/infiniband/hw/i40iw/i40iw_hw.c if (info->qp) qp 670 drivers/infiniband/hw/i40iw/i40iw_hw.c struct i40iw_sc_qp *qp, qp 678 drivers/infiniband/hw/i40iw/i40iw_hw.c struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp; qp 690 drivers/infiniband/hw/i40iw/i40iw_hw.c cqp_info->in.u.qp_flush_wqes.qp = qp; qp 727 drivers/infiniband/hw/i40iw/i40iw_hw.c struct i40iw_sc_qp *qp, qp 745 drivers/infiniband/hw/i40iw/i40iw_hw.c cqp_info->in.u.gen_ae.qp = qp; qp 840 drivers/infiniband/hw/i40iw/i40iw_hw.c struct i40iw_sc_qp *qp = &iwqp->sc_qp; qp 845 drivers/infiniband/hw/i40iw/i40iw_hw.c if (qp->term_flags) { qp 847 drivers/infiniband/hw/i40iw/i40iw_hw.c &pinfo->sq_major_code, qp->flush_code); qp 849 drivers/infiniband/hw/i40iw/i40iw_hw.c &pinfo->rq_major_code, qp->flush_code); qp 173 drivers/infiniband/hw/i40iw/i40iw_osdep.h void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); qp 201 drivers/infiniband/hw/i40iw/i40iw_osdep.h void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend); qp 203 drivers/infiniband/hw/i40iw/i40iw_osdep.h void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len); qp 204 drivers/infiniband/hw/i40iw/i40iw_osdep.h void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred); qp 205 drivers/infiniband/hw/i40iw/i40iw_osdep.h void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp); qp 206 drivers/infiniband/hw/i40iw/i40iw_osdep.h void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp); qp 58 drivers/infiniband/hw/i40iw/i40iw_p.h enum i40iw_status_code i40iw_sc_mr_fast_register(struct i40iw_sc_qp *qp, qp 88 drivers/infiniband/hw/i40iw/i40iw_p.h void i40iw_qp_add_qos(struct i40iw_sc_qp *qp); qp 89 drivers/infiniband/hw/i40iw/i40iw_p.h void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp); qp 90 drivers/infiniband/hw/i40iw/i40iw_p.h void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp); qp 92 drivers/infiniband/hw/i40iw/i40iw_p.h void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info); qp 94 drivers/infiniband/hw/i40iw/i40iw_p.h void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info); qp 97 drivers/infiniband/hw/i40iw/i40iw_p.h struct i40iw_sc_qp *qp, u64 scratch); qp 100 drivers/infiniband/hw/i40iw/i40iw_p.h struct i40iw_sc_qp *qp, u64 scratch); qp 48 drivers/infiniband/hw/i40iw/i40iw_puda.c static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx); qp 113 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_sc_qp *qp = &rsrc->qp; qp 116 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf; qp 117 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = qp->qp_uk.rq_base[wqe_idx].elem; qp 206 drivers/infiniband/hw/i40iw/i40iw_puda.c static u64 *i40iw_puda_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx) qp 211 drivers/infiniband/hw/i40iw/i40iw_puda.c *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 213 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->swqe_polarity = !qp->swqe_polarity; qp 214 drivers/infiniband/hw/i40iw/i40iw_puda.c I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); qp 217 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = qp->sq_base[*wqe_idx].elem; qp 261 drivers/infiniband/hw/i40iw/i40iw_puda.c info->qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx; qp 283 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_qp_uk *qp; qp 307 drivers/infiniband/hw/i40iw/i40iw_puda.c qp = info.qp; qp 308 drivers/infiniband/hw/i40iw/i40iw_puda.c if (!qp || !rsrc) { qp 313 drivers/infiniband/hw/i40iw/i40iw_puda.c if (qp->qp_id != rsrc->qp_id) { qp 319 drivers/infiniband/hw/i40iw/i40iw_puda.c buf = (struct i40iw_puda_buf *)(uintptr_t)qp->rq_wrid_array[info.wqe_idx]; qp 325 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_ilq_putback_rcvbuf(&rsrc->qp, qp 339 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_ilq_putback_rcvbuf(&rsrc->qp, info.wqe_idx); qp 345 drivers/infiniband/hw/i40iw/i40iw_puda.c sqwrid = (void *)(uintptr_t)qp->sq_wrtrk_array[info.wqe_idx].wrid; qp 346 drivers/infiniband/hw/i40iw/i40iw_puda.c I40IW_RING_SET_TAIL(qp->sq_ring, info.wqe_idx); qp 371 drivers/infiniband/hw/i40iw/i40iw_puda.c enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp, qp 390 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = i40iw_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx); qp 393 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch; qp 404 drivers/infiniband/hw/i40iw/i40iw_puda.c LS_64(qp->qp_uk.swqe_polarity, I40IW_UDA_QPSQ_VALID); qp 412 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_debug_buf(qp->dev, I40IW_DEBUG_PUDA, "PUDA SEND WQE", wqe, 32); qp 413 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_qp_post_wr(&qp->qp_uk); qp 459 drivers/infiniband/hw/i40iw/i40iw_puda.c ret = i40iw_puda_send(&rsrc->qp, &info); qp 480 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_sc_qp *qp = &rsrc->qp; qp 481 drivers/infiniband/hw/i40iw/i40iw_puda.c u64 *qp_ctx = qp->hw_host_ctx; qp 483 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(qp_ctx, 8, qp->sq_pa); qp 484 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(qp_ctx, 16, qp->rq_pa); qp 487 drivers/infiniband/hw/i40iw/i40iw_puda.c LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) | qp 488 drivers/infiniband/hw/i40iw/i40iw_puda.c LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE)); qp 501 drivers/infiniband/hw/i40iw/i40iw_puda.c LS_64((uintptr_t)qp, I40IWQPC_QPCOMPCTX)); qp 504 drivers/infiniband/hw/i40iw/i40iw_puda.c LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) | qp 505 drivers/infiniband/hw/i40iw/i40iw_puda.c LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) | qp 506 drivers/infiniband/hw/i40iw/i40iw_puda.c LS_64(qp->qs_handle, I40IWQPC_QSHANDLE)); qp 516 drivers/infiniband/hw/i40iw/i40iw_puda.c static enum i40iw_status_code i40iw_puda_qp_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp) qp 529 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); qp 530 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 40, qp->shadow_area_pa); qp 531 drivers/infiniband/hw/i40iw/i40iw_puda.c header = qp->qp_uk.qp_id | qp 554 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_sc_qp *qp = &rsrc->qp; qp 555 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_qp_uk *ukqp = &qp->qp_uk; qp 575 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->hw_sq_size = i40iw_get_encoded_wqe_size(rsrc->sq_size, false); qp 576 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->hw_rq_size = i40iw_get_encoded_wqe_size(rsrc->rq_size, false); qp 577 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->pd = &rsrc->sc_pd; qp 578 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->qp_type = I40IW_QP_TYPE_UDA; qp 579 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->dev = rsrc->dev; qp 580 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->back_qp = (void *)rsrc; qp 581 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->sq_pa = mem->pa; qp 582 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->rq_pa = qp->sq_pa + sq_size; qp 583 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->vsi = rsrc->vsi; qp 587 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->shadow_area_pa = qp->rq_pa + rq_size; qp 588 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->hw_host_ctx = ukqp->shadow_area + I40IW_SHADOW_AREA_SIZE; qp 589 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->hw_host_ctx_pa = qp 590 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->shadow_area_pa + (I40IW_SHADOW_AREA_SIZE << 3); qp 603 drivers/infiniband/hw/i40iw/i40iw_puda.c if (qp->pd->dev->is_pf) qp 604 drivers/infiniband/hw/i40iw/i40iw_puda.c ukqp->wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) + qp 607 drivers/infiniband/hw/i40iw/i40iw_puda.c ukqp->wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) + qp 610 drivers/infiniband/hw/i40iw/i40iw_puda.c qp->user_pri = 0; qp 611 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_qp_add_qos(qp); qp 614 drivers/infiniband/hw/i40iw/i40iw_puda.c ret = i40iw_cqp_qp_create_cmd(rsrc->dev, qp); qp 616 drivers/infiniband/hw/i40iw/i40iw_puda.c ret = i40iw_puda_qp_wqe(rsrc->dev, qp); qp 618 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_qp_rem_qos(qp); qp 729 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_cqp_qp_destroy_cmd(dev, &rsrc->qp); qp 733 drivers/infiniband/hw/i40iw/i40iw_puda.c ret = dev->iw_priv_qp_ops->qp_destroy(&rsrc->qp, qp 989 drivers/infiniband/hw/i40iw/i40iw_puda.c static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx) qp 994 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = qp->qp_uk.rq_base[wqe_idx].elem; qp 1327 drivers/infiniband/hw/i40iw/i40iw_puda.c static void i40iw_ieq_process_fpdus(struct i40iw_sc_qp *qp, qp 1330 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_pfpdu *pfpdu = &qp->pfpdu; qp 1360 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_ieq_mpa_crc_ae(ieq->dev, qp); qp 1372 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_sc_qp *qp, qp 1376 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_pfpdu *pfpdu = &qp->pfpdu; qp 1377 drivers/infiniband/hw/i40iw/i40iw_puda.c u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx; qp 1380 drivers/infiniband/hw/i40iw/i40iw_puda.c u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET); qp 1392 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_ieq_cleanup_qp(ieq, qp); qp 1399 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_debug_buf(ieq->dev, I40IW_DEBUG_IEQ, "Q2 BUFFER", (u64 *)qp->q2_buf, 128); qp 1429 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_ieq_process_fpdus(qp, ieq); qp 1444 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_sc_qp *qp = NULL; qp 1447 drivers/infiniband/hw/i40iw/i40iw_puda.c qp = i40iw_ieq_get_qp(vsi->dev, buf); qp 1448 drivers/infiniband/hw/i40iw/i40iw_puda.c if (!qp) { qp 1452 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_ieq_handle_exception(ieq, qp, buf); qp 1481 drivers/infiniband/hw/i40iw/i40iw_puda.c void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp) qp 1484 drivers/infiniband/hw/i40iw/i40iw_puda.c struct i40iw_pfpdu *pfpdu = &qp->pfpdu; qp 58 drivers/infiniband/hw/i40iw/i40iw_puda.h struct i40iw_qp_uk *qp; qp 116 drivers/infiniband/hw/i40iw/i40iw_puda.h struct i40iw_sc_qp qp; qp 162 drivers/infiniband/hw/i40iw/i40iw_puda.h enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp, qp 179 drivers/infiniband/hw/i40iw/i40iw_puda.h void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); qp 183 drivers/infiniband/hw/i40iw/i40iw_puda.h enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); qp 185 drivers/infiniband/hw/i40iw/i40iw_puda.h void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); qp 187 drivers/infiniband/hw/i40iw/i40iw_puda.h void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp); qp 762 drivers/infiniband/hw/i40iw/i40iw_type.h bool qp; qp 1164 drivers/infiniband/hw/i40iw/i40iw_type.h struct i40iw_sc_qp *qp; qp 1170 drivers/infiniband/hw/i40iw/i40iw_type.h struct i40iw_sc_qp *qp; qp 1176 drivers/infiniband/hw/i40iw/i40iw_type.h struct i40iw_sc_qp *qp; qp 1293 drivers/infiniband/hw/i40iw/i40iw_type.h struct i40iw_sc_qp *qp; qp 1299 drivers/infiniband/hw/i40iw/i40iw_type.h struct i40iw_sc_qp *qp; qp 1340 drivers/infiniband/hw/i40iw/i40iw_type.h struct i40iw_sc_qp *qp; qp 47 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_nop_1(struct i40iw_qp_uk *qp) qp 54 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!qp->sq_ring.head) qp 57 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 58 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = qp->sq_base[wqe_idx].elem; qp 60 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->sq_wrtrk_array[wqe_idx].wqe_size = I40IW_QP_WQE_MIN_SIZE; qp 62 drivers/infiniband/hw/i40iw/i40iw_uk.c peek_head = (qp->sq_ring.head + 1) % qp->sq_ring.size; qp 63 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe_0 = qp->sq_base[peek_head].elem; qp 65 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID); qp 67 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe_0[3] = LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 75 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID) | nop_signature++; qp 87 drivers/infiniband/hw/i40iw/i40iw_uk.c void i40iw_qp_post_wr(struct i40iw_qp_uk *qp) qp 96 drivers/infiniband/hw/i40iw/i40iw_uk.c get_64bit_val(qp->shadow_area, 0, &temp); qp 99 drivers/infiniband/hw/i40iw/i40iw_uk.c sw_sq_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 101 drivers/infiniband/hw/i40iw/i40iw_uk.c if (sw_sq_head > qp->initial_ring.head) { qp 102 drivers/infiniband/hw/i40iw/i40iw_uk.c if ((hw_sq_tail >= qp->initial_ring.head) && qp 104 drivers/infiniband/hw/i40iw/i40iw_uk.c writel(qp->qp_id, qp->wqe_alloc_reg); qp 106 drivers/infiniband/hw/i40iw/i40iw_uk.c } else if (sw_sq_head != qp->initial_ring.head) { qp 107 drivers/infiniband/hw/i40iw/i40iw_uk.c if ((hw_sq_tail >= qp->initial_ring.head) || qp 109 drivers/infiniband/hw/i40iw/i40iw_uk.c writel(qp->qp_id, qp->wqe_alloc_reg); qp 114 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->initial_ring.head = qp->sq_ring.head; qp 122 drivers/infiniband/hw/i40iw/i40iw_uk.c static void i40iw_qp_ring_push_db(struct i40iw_qp_uk *qp, u32 wqe_idx) qp 124 drivers/infiniband/hw/i40iw/i40iw_uk.c set_32bit_val(qp->push_db, 0, LS_32((wqe_idx >> 2), I40E_PFPE_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id); qp 125 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->initial_ring.head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 134 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, qp 149 drivers/infiniband/hw/i40iw/i40iw_uk.c *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 152 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->swqe_polarity = !qp->swqe_polarity; qp 153 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe_ptr = (uintptr_t)qp->sq_base[*wqe_idx].elem; qp 158 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_nop_1(qp); qp 159 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); qp 164 drivers/infiniband/hw/i40iw/i40iw_uk.c *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 166 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->swqe_polarity = !qp->swqe_polarity; qp 170 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_nop_1(qp); qp 171 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); qp 174 drivers/infiniband/hw/i40iw/i40iw_uk.c *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 176 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->swqe_polarity = !qp->swqe_polarity; qp 178 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_MOVE_HEAD_BY_COUNT(qp->sq_ring, qp 183 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = qp->sq_base[*wqe_idx].elem; qp 185 drivers/infiniband/hw/i40iw/i40iw_uk.c peek_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); qp 186 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe_0 = qp->sq_base[peek_head].elem; qp 189 drivers/infiniband/hw/i40iw/i40iw_uk.c if (RS_64(wqe_0[3], I40IWQPSQ_VALID) != !qp->swqe_polarity) qp 190 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID); qp 193 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->sq_wrtrk_array[*wqe_idx].wrid = wr_id; qp 194 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; qp 195 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->sq_wrtrk_array[*wqe_idx].wqe_size = wqe_size; qp 220 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx) qp 225 drivers/infiniband/hw/i40iw/i40iw_uk.c if (I40IW_RING_FULL_ERR(qp->rq_ring)) qp 228 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); qp 232 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rwqe_polarity = !qp->rwqe_polarity; qp 234 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = qp->rq_base[*wqe_idx * (qp->rq_wqe_size_multiplier >> 2)].elem; qp 245 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_rdma_write(struct i40iw_qp_uk *qp, qp 259 drivers/infiniband/hw/i40iw/i40iw_uk.c if (op_info->num_lo_sges > qp->max_sq_frag_cnt) qp 274 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); qp 288 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 302 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 314 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_rdma_read(struct i40iw_qp_uk *qp, qp 331 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->lo_addr.len, info->wr_id); qp 342 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 350 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 362 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_send(struct i40iw_qp_uk *qp, qp 376 drivers/infiniband/hw/i40iw/i40iw_uk.c if (qp->max_sq_frag_cnt < op_info->num_sges) qp 385 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); qp 398 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 411 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 422 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp, qp 444 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); qp 456 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) | qp 460 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 478 drivers/infiniband/hw/i40iw/i40iw_uk.c if (qp->push_db) { qp 479 drivers/infiniband/hw/i40iw/i40iw_uk.c push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20); qp 481 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_ring_push_db(qp, wqe_idx); qp 484 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 497 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp, qp 520 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); qp 529 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) | qp 533 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 551 drivers/infiniband/hw/i40iw/i40iw_uk.c if (qp->push_db) { qp 552 drivers/infiniband/hw/i40iw/i40iw_uk.c push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20); qp 554 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_ring_push_db(qp, wqe_idx); qp 557 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 569 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_stag_local_invalidate(struct i40iw_qp_uk *qp, qp 582 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); qp 593 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 600 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 611 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_mw_bind(struct i40iw_qp_uk *qp, qp 624 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); qp 641 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 648 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 658 drivers/infiniband/hw/i40iw/i40iw_uk.c static enum i40iw_status_code i40iw_post_receive(struct i40iw_qp_uk *qp, qp 665 drivers/infiniband/hw/i40iw/i40iw_uk.c if (qp->max_rq_frag_cnt < info->num_sges) qp 669 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_recv_wqe(qp, &wqe_idx); qp 673 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rq_wrid_array[wqe_idx] = info->wr_id; qp 678 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->rwqe_polarity, I40IWQPSQ_VALID); qp 754 drivers/infiniband/hw/i40iw/i40iw_uk.c struct i40iw_qp_uk *qp; qp 796 drivers/infiniband/hw/i40iw/i40iw_uk.c qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx; qp 797 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!qp) { qp 802 drivers/infiniband/hw/i40iw/i40iw_uk.c info->qp_handle = (i40iw_qp_handle)(unsigned long)qp; qp 805 drivers/infiniband/hw/i40iw/i40iw_uk.c array_idx = (wqe_idx * 4) / qp->rq_wqe_size_multiplier; qp 807 drivers/infiniband/hw/i40iw/i40iw_uk.c info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail]; qp 808 drivers/infiniband/hw/i40iw/i40iw_uk.c array_idx = qp->rq_ring.tail; qp 810 drivers/infiniband/hw/i40iw/i40iw_uk.c info->wr_id = qp->rq_wrid_array[array_idx]; qp 821 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_SET_TAIL(qp->rq_ring, array_idx + 1); qp 822 drivers/infiniband/hw/i40iw/i40iw_uk.c pring = &qp->rq_ring; qp 824 drivers/infiniband/hw/i40iw/i40iw_uk.c if (qp->first_sq_wq) { qp 825 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->first_sq_wq = false; qp 826 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe_idx && (qp->sq_ring.head == qp->sq_ring.tail)) { qp 837 drivers/infiniband/hw/i40iw/i40iw_uk.c info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; qp 838 drivers/infiniband/hw/i40iw/i40iw_uk.c info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; qp 841 drivers/infiniband/hw/i40iw/i40iw_uk.c sw_wqe = qp->sq_base[wqe_idx].elem; qp 844 drivers/infiniband/hw/i40iw/i40iw_uk.c addl_wqes = qp->sq_wrtrk_array[wqe_idx].wqe_size / I40IW_QP_WQE_MIN_SIZE; qp 845 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_SET_TAIL(qp->sq_ring, (wqe_idx + addl_wqes)); qp 851 drivers/infiniband/hw/i40iw/i40iw_uk.c tail = qp->sq_ring.tail; qp 852 drivers/infiniband/hw/i40iw/i40iw_uk.c sw_wqe = qp->sq_base[tail].elem; qp 856 drivers/infiniband/hw/i40iw/i40iw_uk.c addl_wqes = qp->sq_wrtrk_array[tail].wqe_size / I40IW_QP_WQE_MIN_SIZE; qp 857 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_SET_TAIL(qp->sq_ring, (tail + addl_wqes)); qp 859 drivers/infiniband/hw/i40iw/i40iw_uk.c info->wr_id = qp->sq_wrtrk_array[tail].wrid; qp 860 drivers/infiniband/hw/i40iw/i40iw_uk.c info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; qp 865 drivers/infiniband/hw/i40iw/i40iw_uk.c pring = &qp->sq_ring; qp 990 drivers/infiniband/hw/i40iw/i40iw_uk.c enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp, qp 1004 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->sq_base = info->sq; qp 1005 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rq_base = info->rq; qp 1006 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->shadow_area = info->shadow_area; qp 1007 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->sq_wrtrk_array = info->sq_wrtrk_array; qp 1008 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rq_wrid_array = info->rq_wrid_array; qp 1010 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->wqe_alloc_reg = info->wqe_alloc_reg; qp 1011 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->qp_id = info->qp_id; qp 1013 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->sq_size = info->sq_size; qp 1014 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->push_db = info->push_db; qp 1015 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->push_wqe = info->push_wqe; qp 1017 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->max_sq_frag_cnt = info->max_sq_frag_cnt; qp 1018 drivers/infiniband/hw/i40iw/i40iw_uk.c sq_ring_size = qp->sq_size << sqshift; qp 1020 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_INIT(qp->sq_ring, sq_ring_size); qp 1021 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_INIT(qp->initial_ring, sq_ring_size); qp 1022 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); qp 1023 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_MOVE_TAIL(qp->sq_ring); qp 1024 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_MOVE_HEAD(qp->initial_ring, ret_code); qp 1025 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->swqe_polarity = 1; qp 1026 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->first_sq_wq = true; qp 1027 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->swqe_polarity_deferred = 1; qp 1028 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rwqe_polarity = 0; qp 1030 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!qp->use_srq) { qp 1031 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rq_size = info->rq_size; qp 1032 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->max_rq_frag_cnt = info->max_rq_frag_cnt; qp 1033 drivers/infiniband/hw/i40iw/i40iw_uk.c I40IW_RING_INIT(qp->rq_ring, qp->rq_size); qp 1043 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rq_wqe_size = rqshift; qp 1044 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->rq_wqe_size_multiplier = 4 << rqshift; qp 1046 drivers/infiniband/hw/i40iw/i40iw_uk.c qp->ops = iw_qp_uk_ops; qp 1127 drivers/infiniband/hw/i40iw/i40iw_uk.c enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, qp 1135 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, wr_id); qp 1144 drivers/infiniband/hw/i40iw/i40iw_uk.c LS_64(qp->swqe_polarity, I40IWQPSQ_VALID); qp 1150 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_qp_post_wr(qp); qp 406 drivers/infiniband/hw/i40iw/i40iw_user.h void i40iw_qp_post_wr(struct i40iw_qp_uk *qp); qp 407 drivers/infiniband/hw/i40iw/i40iw_user.h u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx, qp 412 drivers/infiniband/hw/i40iw/i40iw_user.h u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx); qp 417 drivers/infiniband/hw/i40iw/i40iw_user.h enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp, qp 421 drivers/infiniband/hw/i40iw/i40iw_user.h enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id, qp 489 drivers/infiniband/hw/i40iw/i40iw_utils.c struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)cqp_request->param; qp 490 drivers/infiniband/hw/i40iw/i40iw_utils.c struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp; qp 665 drivers/infiniband/hw/i40iw/i40iw_utils.c cqp_info->in.u.qp_destroy.qp = &iwqp->sc_qp; qp 861 drivers/infiniband/hw/i40iw/i40iw_utils.c void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend) qp 876 drivers/infiniband/hw/i40iw/i40iw_utils.c cqp_info->in.u.suspend_resume.qp = qp; qp 890 drivers/infiniband/hw/i40iw/i40iw_utils.c void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len) qp 894 drivers/infiniband/hw/i40iw/i40iw_utils.c iwqp = (struct i40iw_qp *)qp->back_qp; qp 903 drivers/infiniband/hw/i40iw/i40iw_utils.c void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred) qp 911 drivers/infiniband/hw/i40iw/i40iw_utils.c iwqp = (struct i40iw_qp *)qp->back_qp; qp 917 drivers/infiniband/hw/i40iw/i40iw_utils.c first_time = !(qp->term_flags & I40IW_TERM_DONE); qp 918 drivers/infiniband/hw/i40iw/i40iw_utils.c qp->term_flags |= I40IW_TERM_DONE; qp 922 drivers/infiniband/hw/i40iw/i40iw_utils.c i40iw_terminate_del_timer(qp); qp 938 drivers/infiniband/hw/i40iw/i40iw_utils.c struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)&iwqp->sc_qp; qp 940 drivers/infiniband/hw/i40iw/i40iw_utils.c i40iw_terminate_done(qp, 1); qp 948 drivers/infiniband/hw/i40iw/i40iw_utils.c void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp) qp 952 drivers/infiniband/hw/i40iw/i40iw_utils.c iwqp = (struct i40iw_qp *)qp->back_qp; qp 963 drivers/infiniband/hw/i40iw/i40iw_utils.c void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp) qp 967 drivers/infiniband/hw/i40iw/i40iw_utils.c iwqp = (struct i40iw_qp *)qp->back_qp; qp 1219 drivers/infiniband/hw/i40iw/i40iw_utils.c struct i40iw_sc_qp *qp) qp 1242 drivers/infiniband/hw/i40iw/i40iw_utils.c cqp_info->in.u.qp_create.qp = qp; qp 1267 drivers/infiniband/hw/i40iw/i40iw_utils.c void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp) qp 1284 drivers/infiniband/hw/i40iw/i40iw_utils.c cqp_info->in.u.qp_destroy.qp = qp; qp 1298 drivers/infiniband/hw/i40iw/i40iw_utils.c void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp) qp 1306 drivers/infiniband/hw/i40iw/i40iw_utils.c i40iw_gen_ae(iwdev, qp, &info, false); qp 213 drivers/infiniband/hw/i40iw/i40iw_verbs.c static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp) qp 219 drivers/infiniband/hw/i40iw/i40iw_verbs.c if (qp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX) qp 232 drivers/infiniband/hw/i40iw/i40iw_verbs.c cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; qp 239 drivers/infiniband/hw/i40iw/i40iw_verbs.c qp->push_idx = cqp_request->compl_info.op_ret_val; qp 250 drivers/infiniband/hw/i40iw/i40iw_verbs.c static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp) qp 256 drivers/infiniband/hw/i40iw/i40iw_verbs.c if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) qp 267 drivers/infiniband/hw/i40iw/i40iw_verbs.c cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; qp 268 drivers/infiniband/hw/i40iw/i40iw_verbs.c cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; qp 275 drivers/infiniband/hw/i40iw/i40iw_verbs.c qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX; qp 535 drivers/infiniband/hw/i40iw/i40iw_verbs.c struct i40iw_sc_qp *qp; qp 578 drivers/infiniband/hw/i40iw/i40iw_verbs.c qp = &iwqp->sc_qp; qp 579 drivers/infiniband/hw/i40iw/i40iw_verbs.c qp->back_qp = (void *)iwqp; qp 580 drivers/infiniband/hw/i40iw/i40iw_verbs.c qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX; qp 609 drivers/infiniband/hw/i40iw/i40iw_verbs.c qp = &iwqp->sc_qp; qp 619 drivers/infiniband/hw/i40iw/i40iw_verbs.c iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; qp 626 drivers/infiniband/hw/i40iw/i40iw_verbs.c i40iw_alloc_push_page(iwdev, qp); qp 665 drivers/infiniband/hw/i40iw/i40iw_verbs.c ret = dev->iw_priv_qp_ops->qp_init(qp, &init_info); qp 685 drivers/infiniband/hw/i40iw/i40iw_verbs.c if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) { qp 689 drivers/infiniband/hw/i40iw/i40iw_verbs.c ctx_info->push_idx = qp->push_idx; qp 711 drivers/infiniband/hw/i40iw/i40iw_verbs.c cqp_info->in.u.qp_create.qp = qp; qp 731 drivers/infiniband/hw/i40iw/i40iw_verbs.c uresp.push_idx = qp->push_idx; qp 762 drivers/infiniband/hw/i40iw/i40iw_verbs.c struct i40iw_sc_qp *qp = &iwqp->sc_qp; qp 767 drivers/infiniband/hw/i40iw/i40iw_verbs.c attr->cap.max_send_wr = qp->qp_uk.sq_size; qp 768 drivers/infiniband/hw/i40iw/i40iw_verbs.c attr->cap.max_recv_wr = qp->qp_uk.rq_size; qp 807 drivers/infiniband/hw/i40iw/i40iw_verbs.c cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; qp 1647 drivers/infiniband/hw/i40iw/i40iw_verbs.c struct i40iw_sc_qp *qp = &iwqp->sc_qp; qp 1649 drivers/infiniband/hw/i40iw/i40iw_verbs.c if (I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring)) qp 1660 drivers/infiniband/hw/i40iw/i40iw_verbs.c struct i40iw_sc_qp *qp = &iwqp->sc_qp; qp 1662 drivers/infiniband/hw/i40iw/i40iw_verbs.c if (I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring)) qp 2351 drivers/infiniband/hw/i40iw/i40iw_verbs.c struct i40iw_sc_qp *qp; qp 2402 drivers/infiniband/hw/i40iw/i40iw_verbs.c qp = (struct i40iw_sc_qp *)cq_poll_info.qp_handle; qp 2403 drivers/infiniband/hw/i40iw/i40iw_verbs.c entry->qp = (struct ib_qp *)qp->back_qp; qp 2405 drivers/infiniband/hw/i40iw/i40iw_verbs.c iwqp = (struct i40iw_qp *)qp->back_qp; qp 2407 drivers/infiniband/hw/i40iw/i40iw_verbs.c if (!I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring)) qp 2409 drivers/infiniband/hw/i40iw/i40iw_verbs.c if (!I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring)) qp 585 drivers/infiniband/hw/mlx4/cq.c static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, qp 590 drivers/infiniband/hw/mlx4/cq.c ib_dma_sync_single_for_cpu(qp->ibqp.device, qp 591 drivers/infiniband/hw/mlx4/cq.c qp->sqp_proxy_rcv[tail].map, qp 594 drivers/infiniband/hw/mlx4/cq.c hdr = (struct mlx4_ib_proxy_sqp_hdr *) (qp->sqp_proxy_rcv[tail].addr); qp 612 drivers/infiniband/hw/mlx4/cq.c static void mlx4_ib_qp_sw_comp(struct mlx4_ib_qp *qp, int num_entries, qp 619 drivers/infiniband/hw/mlx4/cq.c wq = is_send ? &qp->sq : &qp->rq; qp 631 drivers/infiniband/hw/mlx4/cq.c wc->qp = &qp->ibqp; qp 639 drivers/infiniband/hw/mlx4/cq.c struct mlx4_ib_qp *qp; qp 645 drivers/infiniband/hw/mlx4/cq.c list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) { qp 646 drivers/infiniband/hw/mlx4/cq.c mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 1); qp 651 drivers/infiniband/hw/mlx4/cq.c list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) { qp 652 drivers/infiniband/hw/mlx4/cq.c mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 0); qp 725 drivers/infiniband/hw/mlx4/cq.c wc->qp = &(*cur_qp)->ibqp; qp 727 drivers/infiniband/hw/mlx4/cq.c if (wc->qp->qp_type == IB_QPT_XRC_TGT) { qp 839 drivers/infiniband/hw/mlx4/cq.c is_eth = (rdma_port_get_link_layer(wc->qp->device, qp 160 drivers/infiniband/hw/mlx4/mad.c ext_info->my_qpn = cpu_to_be32(in_wc->qp->qp_num); qp 204 drivers/infiniband/hw/mlx4/mad.c new_ah = rdma_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, qp 533 drivers/infiniband/hw/mlx4/mad.c tun_qp = &tun_ctx->qp[0]; qp 535 drivers/infiniband/hw/mlx4/mad.c tun_qp = &tun_ctx->qp[1]; qp 554 drivers/infiniband/hw/mlx4/mad.c src_qp = tun_qp->qp; qp 716 drivers/infiniband/hw/mlx4/mad.c err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); qp 795 drivers/infiniband/hw/mlx4/mad.c err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); qp 810 drivers/infiniband/hw/mlx4/mad.c if (in_wc && in_wc->qp) { qp 816 drivers/infiniband/hw/mlx4/mad.c in_wc->qp->qp_num, qp 1319 drivers/infiniband/hw/mlx4/mad.c size = (tun_qp->qp->qp_type == IB_QPT_UD) ? qp 1333 drivers/infiniband/hw/mlx4/mad.c return ib_post_recv(tun_qp->qp, &recv_wr, &bad_recv_wr); qp 1386 drivers/infiniband/hw/mlx4/mad.c sqp = &sqp_ctx->qp[0]; qp 1390 drivers/infiniband/hw/mlx4/mad.c sqp = &sqp_ctx->qp[1]; qp 1394 drivers/infiniband/hw/mlx4/mad.c send_qp = sqp->qp; qp 1484 drivers/infiniband/hw/mlx4/mad.c struct mlx4_ib_demux_pv_qp *tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc->wr_id)]; qp 1610 drivers/infiniband/hw/mlx4/mad.c tun_qp = &ctx->qp[qp_type]; qp 1705 drivers/infiniband/hw/mlx4/mad.c tun_qp = &ctx->qp[qp_type]; qp 1742 drivers/infiniband/hw/mlx4/mad.c tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; qp 1808 drivers/infiniband/hw/mlx4/mad.c tun_qp = &ctx->qp[qp_type]; qp 1834 drivers/infiniband/hw/mlx4/mad.c tun_qp->qp = ib_create_qp(ctx->pd, &qp_init_attr.init_attr); qp 1835 drivers/infiniband/hw/mlx4/mad.c if (IS_ERR(tun_qp->qp)) { qp 1836 drivers/infiniband/hw/mlx4/mad.c ret = PTR_ERR(tun_qp->qp); qp 1837 drivers/infiniband/hw/mlx4/mad.c tun_qp->qp = NULL; qp 1855 drivers/infiniband/hw/mlx4/mad.c ret = ib_modify_qp(tun_qp->qp, &attr, qp_attr_mask_INIT); qp 1862 drivers/infiniband/hw/mlx4/mad.c ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE); qp 1870 drivers/infiniband/hw/mlx4/mad.c ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN); qp 1888 drivers/infiniband/hw/mlx4/mad.c ib_destroy_qp(tun_qp->qp); qp 1889 drivers/infiniband/hw/mlx4/mad.c tun_qp->qp = NULL; qp 1908 drivers/infiniband/hw/mlx4/mad.c sqp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; qp 2060 drivers/infiniband/hw/mlx4/mad.c ib_destroy_qp(ctx->qp[1].qp); qp 2061 drivers/infiniband/hw/mlx4/mad.c ctx->qp[1].qp = NULL; qp 2066 drivers/infiniband/hw/mlx4/mad.c ib_destroy_qp(ctx->qp[0].qp); qp 2067 drivers/infiniband/hw/mlx4/mad.c ctx->qp[0].qp = NULL; qp 2098 drivers/infiniband/hw/mlx4/mad.c ib_destroy_qp(ctx->qp[0].qp); qp 2099 drivers/infiniband/hw/mlx4/mad.c ctx->qp[0].qp = NULL; qp 2102 drivers/infiniband/hw/mlx4/mad.c ib_destroy_qp(ctx->qp[1].qp); qp 2103 drivers/infiniband/hw/mlx4/mad.c ctx->qp[1].qp = NULL; qp 2230 drivers/infiniband/hw/mlx4/mad.c ib_destroy_qp(sqp_ctx->qp[0].qp); qp 2231 drivers/infiniband/hw/mlx4/mad.c sqp_ctx->qp[0].qp = NULL; qp 2234 drivers/infiniband/hw/mlx4/mad.c ib_destroy_qp(sqp_ctx->qp[1].qp); qp 2235 drivers/infiniband/hw/mlx4/mad.c sqp_ctx->qp[1].qp = NULL; qp 530 drivers/infiniband/hw/mlx4/main.c props->max_qp = dev->dev->quotas.qp; qp 1432 drivers/infiniband/hw/mlx4/main.c static int __mlx4_ib_default_rules_match(struct ib_qp *qp, qp 1438 drivers/infiniband/hw/mlx4/main.c u8 link_layer = rdma_port_get_link_layer(qp->device, flow_attr->port); qp 1488 drivers/infiniband/hw/mlx4/main.c struct ib_qp *qp, qp 1525 drivers/infiniband/hw/mlx4/main.c static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr, qp 1533 drivers/infiniband/hw/mlx4/main.c struct mlx4_ib_dev *mdev = to_mdev(qp->device); qp 1567 drivers/infiniband/hw/mlx4/main.c ctrl->qpn = cpu_to_be32(qp->qp_num); qp 1572 drivers/infiniband/hw/mlx4/main.c default_flow = __mlx4_ib_default_rules_match(qp, flow_attr); qp 1575 drivers/infiniband/hw/mlx4/main.c mdev, qp, default_table + default_flow, qp 1584 drivers/infiniband/hw/mlx4/main.c ret = parse_flow_attr(mdev->dev, qp->qp_num, ib_flow, qp 1630 drivers/infiniband/hw/mlx4/main.c static int mlx4_ib_tunnel_steer_add(struct ib_qp *qp, struct ib_flow_attr *flow_attr, qp 1635 drivers/infiniband/hw/mlx4/main.c struct mlx4_dev *dev = to_mdev(qp->device)->dev; qp 1648 drivers/infiniband/hw/mlx4/main.c err = mlx4_tunnel_steer_add(to_mdev(qp->device)->dev, ib_spec->eth.val.dst_mac, qp 1649 drivers/infiniband/hw/mlx4/main.c flow_attr->port, qp->qp_num, qp 1705 drivers/infiniband/hw/mlx4/main.c static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, qp 1712 drivers/infiniband/hw/mlx4/main.c struct mlx4_dev *dev = (to_mdev(qp->device))->dev; qp 1715 drivers/infiniband/hw/mlx4/main.c if (flow_attr->port < 1 || flow_attr->port > qp->device->phys_port_cnt) qp 1773 drivers/infiniband/hw/mlx4/main.c err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i], qp 1782 drivers/infiniband/hw/mlx4/main.c err = __mlx4_ib_create_flow(qp, flow_attr, qp 1795 drivers/infiniband/hw/mlx4/main.c err = mlx4_ib_tunnel_steer_add(qp, flow_attr, qp 1802 drivers/infiniband/hw/mlx4/main.c err = mlx4_ib_tunnel_steer_add(qp, flow_attr, qp 1817 drivers/infiniband/hw/mlx4/main.c (void)__mlx4_ib_destroy_flow(to_mdev(qp->device)->dev, qp 1823 drivers/infiniband/hw/mlx4/main.c (void)__mlx4_ib_destroy_flow(to_mdev(qp->device)->dev, qp 1836 drivers/infiniband/hw/mlx4/main.c struct mlx4_ib_dev *mdev = to_mdev(flow_id->qp->device); qp 1918 drivers/infiniband/hw/mlx4/main.c static struct mlx4_ib_gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw) qp 1924 drivers/infiniband/hw/mlx4/main.c list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { qp 2278 drivers/infiniband/hw/mlx4/main.c struct mlx4_ib_qp *qp; qp 2291 drivers/infiniband/hw/mlx4/main.c qp = ibdev->qp1_proxy[port - 1]; qp 2292 drivers/infiniband/hw/mlx4/main.c if (qp) { qp 2297 drivers/infiniband/hw/mlx4/main.c mutex_lock(&qp->mutex); qp 2298 drivers/infiniband/hw/mlx4/main.c old_smac = qp->pri.smac; qp 2308 drivers/infiniband/hw/mlx4/main.c if (mlx4_update_qp(ibdev->dev, qp->mqp.qpn, MLX4_UPDATE_QP_SMAC, qp 2314 drivers/infiniband/hw/mlx4/main.c if (qp->pri.smac_port) qp 2316 drivers/infiniband/hw/mlx4/main.c qp->pri.smac = new_smac; qp 2317 drivers/infiniband/hw/mlx4/main.c qp->pri.smac_port = port; qp 2318 drivers/infiniband/hw/mlx4/main.c qp->pri.smac_index = new_smac_index; qp 2324 drivers/infiniband/hw/mlx4/main.c if (qp) qp 2325 drivers/infiniband/hw/mlx4/main.c mutex_unlock(&qp->mutex); qp 436 drivers/infiniband/hw/mlx4/mlx4_ib.h struct ib_qp *qp; qp 462 drivers/infiniband/hw/mlx4/mlx4_ib.h struct mlx4_ib_demux_pv_qp qp[2]; qp 774 drivers/infiniband/hw/mlx4/mlx4_ib.h int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); qp 775 drivers/infiniband/hw/mlx4/mlx4_ib.h void mlx4_ib_drain_sq(struct ib_qp *qp); qp 776 drivers/infiniband/hw/mlx4/mlx4_ib.h void mlx4_ib_drain_rq(struct ib_qp *qp); qp 82 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp qp; qp 128 drivers/infiniband/hw/mlx4/qp.c return container_of(mqp, struct mlx4_ib_sqp, qp); qp 131 drivers/infiniband/hw/mlx4/qp.c static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) qp 136 drivers/infiniband/hw/mlx4/qp.c return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn && qp 137 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn + qp 141 drivers/infiniband/hw/mlx4/qp.c static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) qp 148 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && qp 149 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3); qp 155 drivers/infiniband/hw/mlx4/qp.c if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy || qp 156 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp1_proxy) { qp 165 drivers/infiniband/hw/mlx4/qp.c return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP); qp 169 drivers/infiniband/hw/mlx4/qp.c static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) qp 176 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && qp 177 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1); qp 183 drivers/infiniband/hw/mlx4/qp.c if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy) { qp 192 drivers/infiniband/hw/mlx4/qp.c static void *get_wqe(struct mlx4_ib_qp *qp, int offset) qp 194 drivers/infiniband/hw/mlx4/qp.c return mlx4_buf_offset(&qp->buf, offset); qp 197 drivers/infiniband/hw/mlx4/qp.c static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n) qp 199 drivers/infiniband/hw/mlx4/qp.c return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); qp 202 drivers/infiniband/hw/mlx4/qp.c static void *get_send_wqe(struct mlx4_ib_qp *qp, int n) qp 204 drivers/infiniband/hw/mlx4/qp.c return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); qp 212 drivers/infiniband/hw/mlx4/qp.c static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n) qp 220 drivers/infiniband/hw/mlx4/qp.c buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); qp 229 drivers/infiniband/hw/mlx4/qp.c static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type) qp 232 drivers/infiniband/hw/mlx4/qp.c struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; qp 235 drivers/infiniband/hw/mlx4/qp.c to_mibqp(qp)->port = to_mibqp(qp)->alt_port; qp 239 drivers/infiniband/hw/mlx4/qp.c event.element.qp = ibqp; qp 267 drivers/infiniband/hw/mlx4/qp.c "on QP %06x\n", type, qp->qpn); qp 275 drivers/infiniband/hw/mlx4/qp.c static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type) qp 278 drivers/infiniband/hw/mlx4/qp.c type, qp->qpn); qp 328 drivers/infiniband/hw/mlx4/qp.c bool is_user, bool has_rq, struct mlx4_ib_qp *qp, qp 340 drivers/infiniband/hw/mlx4/qp.c qp->rq.wqe_cnt = qp->rq.max_gs = 0; qp 351 drivers/infiniband/hw/mlx4/qp.c qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr)); qp 352 drivers/infiniband/hw/mlx4/qp.c qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); qp 353 drivers/infiniband/hw/mlx4/qp.c wqe_size = qp->rq.max_gs * sizeof(struct mlx4_wqe_data_seg); qp 354 drivers/infiniband/hw/mlx4/qp.c qp->rq.wqe_shift = ilog2(max_t(u32, wqe_size, inl_recv_sz)); qp 359 drivers/infiniband/hw/mlx4/qp.c cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; qp 360 drivers/infiniband/hw/mlx4/qp.c cap->max_recv_sge = qp->rq.max_gs; qp 362 drivers/infiniband/hw/mlx4/qp.c cap->max_recv_wr = qp->rq.max_post = qp 363 drivers/infiniband/hw/mlx4/qp.c min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt); qp 364 drivers/infiniband/hw/mlx4/qp.c cap->max_recv_sge = min(qp->rq.max_gs, qp 373 drivers/infiniband/hw/mlx4/qp.c enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp) qp 380 drivers/infiniband/hw/mlx4/qp.c cap->max_inline_data + send_wqe_overhead(type, qp->flags) + qp 395 drivers/infiniband/hw/mlx4/qp.c send_wqe_overhead(type, qp->flags); qp 400 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s)); qp 406 drivers/infiniband/hw/mlx4/qp.c qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift); qp 407 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr + qp 408 drivers/infiniband/hw/mlx4/qp.c qp->sq_spare_wqes); qp 410 drivers/infiniband/hw/mlx4/qp.c qp->sq.max_gs = qp 412 drivers/infiniband/hw/mlx4/qp.c (1 << qp->sq.wqe_shift)) - qp 413 drivers/infiniband/hw/mlx4/qp.c send_wqe_overhead(type, qp->flags)) / qp 416 drivers/infiniband/hw/mlx4/qp.c qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + qp 417 drivers/infiniband/hw/mlx4/qp.c (qp->sq.wqe_cnt << qp->sq.wqe_shift); qp 418 drivers/infiniband/hw/mlx4/qp.c if (qp->rq.wqe_shift > qp->sq.wqe_shift) { qp 419 drivers/infiniband/hw/mlx4/qp.c qp->rq.offset = 0; qp 420 drivers/infiniband/hw/mlx4/qp.c qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; qp 422 drivers/infiniband/hw/mlx4/qp.c qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift; qp 423 drivers/infiniband/hw/mlx4/qp.c qp->sq.offset = 0; qp 426 drivers/infiniband/hw/mlx4/qp.c cap->max_send_wr = qp->sq.max_post = qp 427 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_cnt - qp->sq_spare_wqes; qp 428 drivers/infiniband/hw/mlx4/qp.c cap->max_send_sge = min(qp->sq.max_gs, qp 438 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp, qp 448 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; qp 449 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_shift = ucmd->log_sq_stride; qp 451 drivers/infiniband/hw/mlx4/qp.c qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + qp 452 drivers/infiniband/hw/mlx4/qp.c (qp->sq.wqe_cnt << qp->sq.wqe_shift); qp 457 drivers/infiniband/hw/mlx4/qp.c static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) qp 461 drivers/infiniband/hw/mlx4/qp.c qp->sqp_proxy_rcv = qp 462 drivers/infiniband/hw/mlx4/qp.c kmalloc_array(qp->rq.wqe_cnt, sizeof(struct mlx4_ib_buf), qp 464 drivers/infiniband/hw/mlx4/qp.c if (!qp->sqp_proxy_rcv) qp 466 drivers/infiniband/hw/mlx4/qp.c for (i = 0; i < qp->rq.wqe_cnt; i++) { qp 467 drivers/infiniband/hw/mlx4/qp.c qp->sqp_proxy_rcv[i].addr = qp 470 drivers/infiniband/hw/mlx4/qp.c if (!qp->sqp_proxy_rcv[i].addr) qp 472 drivers/infiniband/hw/mlx4/qp.c qp->sqp_proxy_rcv[i].map = qp 473 drivers/infiniband/hw/mlx4/qp.c ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr, qp 476 drivers/infiniband/hw/mlx4/qp.c if (ib_dma_mapping_error(dev, qp->sqp_proxy_rcv[i].map)) { qp 477 drivers/infiniband/hw/mlx4/qp.c kfree(qp->sqp_proxy_rcv[i].addr); qp 486 drivers/infiniband/hw/mlx4/qp.c ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, qp 489 drivers/infiniband/hw/mlx4/qp.c kfree(qp->sqp_proxy_rcv[i].addr); qp 491 drivers/infiniband/hw/mlx4/qp.c kfree(qp->sqp_proxy_rcv); qp 492 drivers/infiniband/hw/mlx4/qp.c qp->sqp_proxy_rcv = NULL; qp 496 drivers/infiniband/hw/mlx4/qp.c static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) qp 500 drivers/infiniband/hw/mlx4/qp.c for (i = 0; i < qp->rq.wqe_cnt; i++) { qp 501 drivers/infiniband/hw/mlx4/qp.c ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, qp 504 drivers/infiniband/hw/mlx4/qp.c kfree(qp->sqp_proxy_rcv[i].addr); qp 506 drivers/infiniband/hw/mlx4/qp.c kfree(qp->sqp_proxy_rcv); qp 528 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp) qp 530 drivers/infiniband/hw/mlx4/qp.c mutex_lock(&dev->counters_table[qp->port - 1].mutex); qp 531 drivers/infiniband/hw/mlx4/qp.c mlx4_counter_free(dev->dev, qp->counter_index->index); qp 532 drivers/infiniband/hw/mlx4/qp.c list_del(&qp->counter_index->list); qp 533 drivers/infiniband/hw/mlx4/qp.c mutex_unlock(&dev->counters_table[qp->port - 1].mutex); qp 535 drivers/infiniband/hw/mlx4/qp.c kfree(qp->counter_index); qp 536 drivers/infiniband/hw/mlx4/qp.c qp->counter_index = NULL; qp 644 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp) qp 649 drivers/infiniband/hw/mlx4/qp.c qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS; qp 651 drivers/infiniband/hw/mlx4/qp.c err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn, 0, qp->mqp.usage); qp 655 drivers/infiniband/hw/mlx4/qp.c err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp); qp 659 drivers/infiniband/hw/mlx4/qp.c mutex_init(&qp->mutex); qp 661 drivers/infiniband/hw/mlx4/qp.c INIT_LIST_HEAD(&qp->gid_list); qp 662 drivers/infiniband/hw/mlx4/qp.c INIT_LIST_HEAD(&qp->steering_rules); qp 664 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET; qp 665 drivers/infiniband/hw/mlx4/qp.c qp->state = IB_QPS_RESET; qp 668 drivers/infiniband/hw/mlx4/qp.c qp->sq_no_prefetch = 1; qp 669 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_cnt = 1; qp 670 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE; qp 671 drivers/infiniband/hw/mlx4/qp.c qp->buf_size = qp->sq.wqe_cnt << MLX4_IB_MIN_SQ_STRIDE; qp 672 drivers/infiniband/hw/mlx4/qp.c qp->mtt = (to_mqp( qp 675 drivers/infiniband/hw/mlx4/qp.c qp->rss_ctx = kzalloc(sizeof(*qp->rss_ctx), GFP_KERNEL); qp 676 drivers/infiniband/hw/mlx4/qp.c if (!qp->rss_ctx) { qp 681 drivers/infiniband/hw/mlx4/qp.c err = set_qp_rss(dev, qp->rss_ctx, init_attr, ucmd); qp 688 drivers/infiniband/hw/mlx4/qp.c kfree(qp->rss_ctx); qp 691 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_remove(dev->dev, &qp->mqp); qp 692 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_free(dev->dev, &qp->mqp); qp 703 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp; qp 757 drivers/infiniband/hw/mlx4/qp.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 758 drivers/infiniband/hw/mlx4/qp.c if (!qp) qp 761 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid = 0xFFFF; qp 762 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid = 0xFFFF; qp 764 drivers/infiniband/hw/mlx4/qp.c err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp); qp 766 drivers/infiniband/hw/mlx4/qp.c kfree(qp); qp 770 drivers/infiniband/hw/mlx4/qp.c qp->ibqp.qp_num = qp->mqp.qpn; qp 772 drivers/infiniband/hw/mlx4/qp.c return &qp->ibqp; qp 781 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp, int range_size, int *wqn) qp 801 drivers/infiniband/hw/mlx4/qp.c qp->mqp.usage); qp 818 drivers/infiniband/hw/mlx4/qp.c qp->wqn_range = range; qp 831 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp, bool dirty_release) qp 838 drivers/infiniband/hw/mlx4/qp.c range = qp->wqn_range; qp 859 drivers/infiniband/hw/mlx4/qp.c struct ib_udata *udata, struct mlx4_ib_qp *qp) qp 874 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET; qp 876 drivers/infiniband/hw/mlx4/qp.c mutex_init(&qp->mutex); qp 877 drivers/infiniband/hw/mlx4/qp.c spin_lock_init(&qp->sq.lock); qp 878 drivers/infiniband/hw/mlx4/qp.c spin_lock_init(&qp->rq.lock); qp 879 drivers/infiniband/hw/mlx4/qp.c INIT_LIST_HEAD(&qp->gid_list); qp 880 drivers/infiniband/hw/mlx4/qp.c INIT_LIST_HEAD(&qp->steering_rules); qp 882 drivers/infiniband/hw/mlx4/qp.c qp->state = IB_QPS_RESET; qp 907 drivers/infiniband/hw/mlx4/qp.c qp->flags |= MLX4_IB_QP_SCATTER_FCS; qp 909 drivers/infiniband/hw/mlx4/qp.c err = set_rq_size(dev, &init_attr->cap, true, true, qp, qp->inl_recv_sz); qp 913 drivers/infiniband/hw/mlx4/qp.c qp->sq_no_prefetch = 1; qp 914 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_cnt = 1; qp 915 drivers/infiniband/hw/mlx4/qp.c qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE; qp 916 drivers/infiniband/hw/mlx4/qp.c qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + qp 917 drivers/infiniband/hw/mlx4/qp.c (qp->sq.wqe_cnt << qp->sq.wqe_shift); qp 919 drivers/infiniband/hw/mlx4/qp.c qp->umem = ib_umem_get(udata, wq.buf_addr, qp->buf_size, 0, 0); qp 920 drivers/infiniband/hw/mlx4/qp.c if (IS_ERR(qp->umem)) { qp 921 drivers/infiniband/hw/mlx4/qp.c err = PTR_ERR(qp->umem); qp 925 drivers/infiniband/hw/mlx4/qp.c n = ib_umem_page_count(qp->umem); qp 926 drivers/infiniband/hw/mlx4/qp.c shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n); qp 927 drivers/infiniband/hw/mlx4/qp.c err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt); qp 932 drivers/infiniband/hw/mlx4/qp.c err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem); qp 936 drivers/infiniband/hw/mlx4/qp.c err = mlx4_ib_db_map_user(udata, wq.db_addr, &qp->db); qp 939 drivers/infiniband/hw/mlx4/qp.c qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS; qp 941 drivers/infiniband/hw/mlx4/qp.c err = mlx4_ib_alloc_wqn(context, qp, range_size, &qpn); qp 945 drivers/infiniband/hw/mlx4/qp.c err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp); qp 954 drivers/infiniband/hw/mlx4/qp.c qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); qp 956 drivers/infiniband/hw/mlx4/qp.c qp->mqp.event = mlx4_ib_wq_event; qp 964 drivers/infiniband/hw/mlx4/qp.c list_add_tail(&qp->qps_list, &dev->qp_list); qp 969 drivers/infiniband/hw/mlx4/qp.c list_add_tail(&qp->cq_send_list, &mcq->send_qp_list); qp 971 drivers/infiniband/hw/mlx4/qp.c list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list); qp 978 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_release_wqn(context, qp, 0); qp 980 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_db_unmap_user(context, &qp->db); qp 983 drivers/infiniband/hw/mlx4/qp.c mlx4_mtt_cleanup(dev->dev, &qp->mtt); qp 985 drivers/infiniband/hw/mlx4/qp.c ib_umem_release(qp->umem); qp 998 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp; qp 1053 drivers/infiniband/hw/mlx4/qp.c qp = &sqp->qp; qp 1055 drivers/infiniband/hw/mlx4/qp.c qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL); qp 1056 drivers/infiniband/hw/mlx4/qp.c if (!qp) qp 1059 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid = 0xFFFF; qp 1060 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid = 0xFFFF; qp 1062 drivers/infiniband/hw/mlx4/qp.c qp = *caller_qp; qp 1064 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type = qp_type; qp 1066 drivers/infiniband/hw/mlx4/qp.c mutex_init(&qp->mutex); qp 1067 drivers/infiniband/hw/mlx4/qp.c spin_lock_init(&qp->sq.lock); qp 1068 drivers/infiniband/hw/mlx4/qp.c spin_lock_init(&qp->rq.lock); qp 1069 drivers/infiniband/hw/mlx4/qp.c INIT_LIST_HEAD(&qp->gid_list); qp 1070 drivers/infiniband/hw/mlx4/qp.c INIT_LIST_HEAD(&qp->steering_rules); qp 1072 drivers/infiniband/hw/mlx4/qp.c qp->state = IB_QPS_RESET; qp 1074 drivers/infiniband/hw/mlx4/qp.c qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); qp 1089 drivers/infiniband/hw/mlx4/qp.c qp->inl_recv_sz = ucmd.inl_recv_sz; qp 1099 drivers/infiniband/hw/mlx4/qp.c qp->flags |= MLX4_IB_QP_SCATTER_FCS; qp 1103 drivers/infiniband/hw/mlx4/qp.c qp_has_rq(init_attr), qp, qp->inl_recv_sz); qp 1107 drivers/infiniband/hw/mlx4/qp.c qp->sq_no_prefetch = ucmd.sq_no_prefetch; qp 1109 drivers/infiniband/hw/mlx4/qp.c err = set_user_sq_size(dev, qp, &ucmd); qp 1113 drivers/infiniband/hw/mlx4/qp.c qp->umem = qp 1114 drivers/infiniband/hw/mlx4/qp.c ib_umem_get(udata, ucmd.buf_addr, qp->buf_size, 0, 0); qp 1115 drivers/infiniband/hw/mlx4/qp.c if (IS_ERR(qp->umem)) { qp 1116 drivers/infiniband/hw/mlx4/qp.c err = PTR_ERR(qp->umem); qp 1120 drivers/infiniband/hw/mlx4/qp.c n = ib_umem_page_count(qp->umem); qp 1121 drivers/infiniband/hw/mlx4/qp.c shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n); qp 1122 drivers/infiniband/hw/mlx4/qp.c err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt); qp 1127 drivers/infiniband/hw/mlx4/qp.c err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem); qp 1132 drivers/infiniband/hw/mlx4/qp.c err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &qp->db); qp 1136 drivers/infiniband/hw/mlx4/qp.c qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS; qp 1139 drivers/infiniband/hw/mlx4/qp.c qp_has_rq(init_attr), qp, 0); qp 1143 drivers/infiniband/hw/mlx4/qp.c qp->sq_no_prefetch = 0; qp 1146 drivers/infiniband/hw/mlx4/qp.c qp->flags |= MLX4_IB_QP_LSO; qp 1151 drivers/infiniband/hw/mlx4/qp.c qp->flags |= MLX4_IB_QP_NETIF; qp 1156 drivers/infiniband/hw/mlx4/qp.c err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp); qp 1161 drivers/infiniband/hw/mlx4/qp.c err = mlx4_db_alloc(dev->dev, &qp->db, 0); qp 1165 drivers/infiniband/hw/mlx4/qp.c *qp->db.db = 0; qp 1168 drivers/infiniband/hw/mlx4/qp.c if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, qp 1169 drivers/infiniband/hw/mlx4/qp.c &qp->buf)) { qp 1174 drivers/infiniband/hw/mlx4/qp.c err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift, qp 1175 drivers/infiniband/hw/mlx4/qp.c &qp->mtt); qp 1179 drivers/infiniband/hw/mlx4/qp.c err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf); qp 1183 drivers/infiniband/hw/mlx4/qp.c qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt, qp 1185 drivers/infiniband/hw/mlx4/qp.c qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt, qp 1187 drivers/infiniband/hw/mlx4/qp.c if (!qp->sq.wrid || !qp->rq.wrid) { qp 1191 drivers/infiniband/hw/mlx4/qp.c qp->mqp.usage = MLX4_RES_USAGE_DRIVER; qp 1195 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | qp 1197 drivers/infiniband/hw/mlx4/qp.c if (alloc_proxy_bufs(pd->device, qp)) { qp 1212 drivers/infiniband/hw/mlx4/qp.c qp->mqp.usage); qp 1214 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_NETIF) qp 1218 drivers/infiniband/hw/mlx4/qp.c &qpn, 0, qp->mqp.usage); qp 1224 drivers/infiniband/hw/mlx4/qp.c qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; qp 1226 drivers/infiniband/hw/mlx4/qp.c err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp); qp 1231 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn |= (1 << 23); qp 1238 drivers/infiniband/hw/mlx4/qp.c qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); qp 1240 drivers/infiniband/hw/mlx4/qp.c qp->mqp.event = mlx4_ib_qp_event; qp 1243 drivers/infiniband/hw/mlx4/qp.c *caller_qp = qp; qp 1251 drivers/infiniband/hw/mlx4/qp.c list_add_tail(&qp->qps_list, &dev->qp_list); qp 1256 drivers/infiniband/hw/mlx4/qp.c list_add_tail(&qp->cq_send_list, &mcq->send_qp_list); qp 1258 drivers/infiniband/hw/mlx4/qp.c list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list); qp 1266 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_NETIF) qp 1272 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI) qp 1273 drivers/infiniband/hw/mlx4/qp.c free_proxy_bufs(pd->device, qp); qp 1277 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_db_unmap_user(context, &qp->db); qp 1279 drivers/infiniband/hw/mlx4/qp.c kvfree(qp->sq.wrid); qp 1280 drivers/infiniband/hw/mlx4/qp.c kvfree(qp->rq.wrid); qp 1284 drivers/infiniband/hw/mlx4/qp.c mlx4_mtt_cleanup(dev->dev, &qp->mtt); qp 1287 drivers/infiniband/hw/mlx4/qp.c if (!qp->umem) qp 1288 drivers/infiniband/hw/mlx4/qp.c mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); qp 1289 drivers/infiniband/hw/mlx4/qp.c ib_umem_release(qp->umem); qp 1293 drivers/infiniband/hw/mlx4/qp.c mlx4_db_free(dev->dev, &qp->db); qp 1297 drivers/infiniband/hw/mlx4/qp.c kfree(qp); qp 1347 drivers/infiniband/hw/mlx4/qp.c static void del_gid_entries(struct mlx4_ib_qp *qp) qp 1351 drivers/infiniband/hw/mlx4/qp.c list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { qp 1357 drivers/infiniband/hw/mlx4/qp.c static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp) qp 1359 drivers/infiniband/hw/mlx4/qp.c if (qp->ibqp.qp_type == IB_QPT_XRC_TGT) qp 1360 drivers/infiniband/hw/mlx4/qp.c return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd); qp 1362 drivers/infiniband/hw/mlx4/qp.c return to_mpd(qp->ibqp.pd); qp 1365 drivers/infiniband/hw/mlx4/qp.c static void get_cqs(struct mlx4_ib_qp *qp, enum mlx4_ib_source_type src, qp 1368 drivers/infiniband/hw/mlx4/qp.c switch (qp->ibqp.qp_type) { qp 1370 drivers/infiniband/hw/mlx4/qp.c *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq); qp 1374 drivers/infiniband/hw/mlx4/qp.c *send_cq = to_mcq(qp->ibqp.send_cq); qp 1378 drivers/infiniband/hw/mlx4/qp.c *recv_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.recv_cq) : qp 1379 drivers/infiniband/hw/mlx4/qp.c to_mcq(qp->ibwq.cq); qp 1380 drivers/infiniband/hw/mlx4/qp.c *send_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.send_cq) : qp 1386 drivers/infiniband/hw/mlx4/qp.c static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) qp 1388 drivers/infiniband/hw/mlx4/qp.c if (qp->state != IB_QPS_RESET) { qp 1391 drivers/infiniband/hw/mlx4/qp.c for (i = 0; i < (1 << qp->ibqp.rwq_ind_tbl->log_ind_tbl_size); qp 1393 drivers/infiniband/hw/mlx4/qp.c struct ib_wq *ibwq = qp->ibqp.rwq_ind_tbl->ind_tbl[i]; qp 1403 drivers/infiniband/hw/mlx4/qp.c if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state), qp 1404 drivers/infiniband/hw/mlx4/qp.c MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp)) qp 1406 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn); qp 1409 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_remove(dev->dev, &qp->mqp); qp 1410 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_free(dev->dev, &qp->mqp); qp 1411 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1); qp 1412 drivers/infiniband/hw/mlx4/qp.c del_gid_entries(qp); qp 1413 drivers/infiniband/hw/mlx4/qp.c kfree(qp->rss_ctx); qp 1416 drivers/infiniband/hw/mlx4/qp.c static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, qp 1423 drivers/infiniband/hw/mlx4/qp.c if (qp->state != IB_QPS_RESET) { qp 1424 drivers/infiniband/hw/mlx4/qp.c if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state), qp 1425 drivers/infiniband/hw/mlx4/qp.c MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp)) qp 1427 drivers/infiniband/hw/mlx4/qp.c qp->mqp.qpn); qp 1428 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) { qp 1429 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); qp 1430 drivers/infiniband/hw/mlx4/qp.c qp->pri.smac = 0; qp 1431 drivers/infiniband/hw/mlx4/qp.c qp->pri.smac_port = 0; qp 1433 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.smac) { qp 1434 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); qp 1435 drivers/infiniband/hw/mlx4/qp.c qp->alt.smac = 0; qp 1437 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.vid < 0x1000) { qp 1438 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); qp 1439 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid = 0xFFFF; qp 1440 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_vid = 0xFFFF; qp 1441 drivers/infiniband/hw/mlx4/qp.c qp->pri.update_vid = 0; qp 1443 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.vid < 0x1000) { qp 1444 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); qp 1445 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid = 0xFFFF; qp 1446 drivers/infiniband/hw/mlx4/qp.c qp->alt.candidate_vid = 0xFFFF; qp 1447 drivers/infiniband/hw/mlx4/qp.c qp->alt.update_vid = 0; qp 1451 drivers/infiniband/hw/mlx4/qp.c get_cqs(qp, src, &send_cq, &recv_cq); qp 1457 drivers/infiniband/hw/mlx4/qp.c list_del(&qp->qps_list); qp 1458 drivers/infiniband/hw/mlx4/qp.c list_del(&qp->cq_send_list); qp 1459 drivers/infiniband/hw/mlx4/qp.c list_del(&qp->cq_recv_list); qp 1461 drivers/infiniband/hw/mlx4/qp.c __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, qp 1462 drivers/infiniband/hw/mlx4/qp.c qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL); qp 1464 drivers/infiniband/hw/mlx4/qp.c __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); qp 1467 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_remove(dev->dev, &qp->mqp); qp 1472 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_free(dev->dev, &qp->mqp); qp 1474 drivers/infiniband/hw/mlx4/qp.c if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) { qp 1475 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_NETIF) qp 1476 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1); qp 1483 drivers/infiniband/hw/mlx4/qp.c qp, 1); qp 1485 drivers/infiniband/hw/mlx4/qp.c mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1); qp 1488 drivers/infiniband/hw/mlx4/qp.c mlx4_mtt_cleanup(dev->dev, &qp->mtt); qp 1491 drivers/infiniband/hw/mlx4/qp.c if (qp->rq.wqe_cnt) { qp 1498 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_db_unmap_user(mcontext, &qp->db); qp 1501 drivers/infiniband/hw/mlx4/qp.c kvfree(qp->sq.wrid); qp 1502 drivers/infiniband/hw/mlx4/qp.c kvfree(qp->rq.wrid); qp 1503 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | qp 1505 drivers/infiniband/hw/mlx4/qp.c free_proxy_bufs(&dev->ib_dev, qp); qp 1506 drivers/infiniband/hw/mlx4/qp.c mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); qp 1507 drivers/infiniband/hw/mlx4/qp.c if (qp->rq.wqe_cnt) qp 1508 drivers/infiniband/hw/mlx4/qp.c mlx4_db_free(dev->dev, &qp->db); qp 1510 drivers/infiniband/hw/mlx4/qp.c ib_umem_release(qp->umem); qp 1512 drivers/infiniband/hw/mlx4/qp.c del_gid_entries(qp); qp 1536 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = NULL; qp 1590 drivers/infiniband/hw/mlx4/qp.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 1591 drivers/infiniband/hw/mlx4/qp.c if (!qp) qp 1593 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid = 0xFFFF; qp 1594 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid = 0xFFFF; qp 1598 drivers/infiniband/hw/mlx4/qp.c err = create_qp_common(pd, init_attr, udata, 0, &qp); qp 1600 drivers/infiniband/hw/mlx4/qp.c kfree(qp); qp 1604 drivers/infiniband/hw/mlx4/qp.c qp->ibqp.qp_num = qp->mqp.qpn; qp 1605 drivers/infiniband/hw/mlx4/qp.c qp->xrcdn = xrcdn; qp 1628 drivers/infiniband/hw/mlx4/qp.c err = create_qp_common(pd, init_attr, udata, sqpn, &qp); qp 1632 drivers/infiniband/hw/mlx4/qp.c qp->port = init_attr->port_num; qp 1633 drivers/infiniband/hw/mlx4/qp.c qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : qp 1642 drivers/infiniband/hw/mlx4/qp.c return &qp->ibqp; qp 1670 drivers/infiniband/hw/mlx4/qp.c sqp->qp.flags |= MLX4_IB_ROCE_V2_GSI_QP; qp 1679 drivers/infiniband/hw/mlx4/qp.c static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) qp 1681 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_dev *dev = to_mdev(qp->device); qp 1682 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *mqp = to_mqp(qp); qp 1697 drivers/infiniband/hw/mlx4/qp.c if (qp->rwq_ind_tbl) { qp 1711 drivers/infiniband/hw/mlx4/qp.c int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) qp 1713 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *mqp = to_mqp(qp); qp 1722 drivers/infiniband/hw/mlx4/qp.c return _mlx4_ib_destroy_qp(qp, udata); qp 1749 drivers/infiniband/hw/mlx4/qp.c static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr, qp 1759 drivers/infiniband/hw/mlx4/qp.c dest_rd_atomic = qp->resp_depth; qp 1764 drivers/infiniband/hw/mlx4/qp.c access_flags = qp->atomic_rd_en; qp 1913 drivers/infiniband/hw/mlx4/qp.c static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp, qp 1919 drivers/infiniband/hw/mlx4/qp.c return _mlx4_set_path(dev, &qp->ah_attr, qp 1926 drivers/infiniband/hw/mlx4/qp.c const struct ib_qp_attr *qp, qp 1931 drivers/infiniband/hw/mlx4/qp.c return _mlx4_set_path(dev, &qp->alt_ah_attr, qp 1937 drivers/infiniband/hw/mlx4/qp.c static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) qp 1941 drivers/infiniband/hw/mlx4/qp.c list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { qp 1942 drivers/infiniband/hw/mlx4/qp.c if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) { qp 1944 drivers/infiniband/hw/mlx4/qp.c ge->port = qp->port; qp 1950 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp, qp 1956 drivers/infiniband/hw/mlx4/qp.c u64_mac = atomic64_read(&dev->iboe.mac[qp->port - 1]); qp 1958 drivers/infiniband/hw/mlx4/qp.c context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6); qp 1959 drivers/infiniband/hw/mlx4/qp.c if (!qp->pri.smac && !qp->pri.smac_port) { qp 1960 drivers/infiniband/hw/mlx4/qp.c smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac); qp 1962 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_smac_index = smac_index; qp 1963 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_smac = u64_mac; qp 1964 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_smac_port = qp->port; qp 1973 drivers/infiniband/hw/mlx4/qp.c static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) qp 1979 drivers/infiniband/hw/mlx4/qp.c if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) != qp 1981 drivers/infiniband/hw/mlx4/qp.c !(qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) || qp 1997 drivers/infiniband/hw/mlx4/qp.c qp->counter_index = new_counter_index; qp 1999 drivers/infiniband/hw/mlx4/qp.c mutex_lock(&dev->counters_table[qp->port - 1].mutex); qp 2001 drivers/infiniband/hw/mlx4/qp.c &dev->counters_table[qp->port - 1].counters_list); qp 2002 drivers/infiniband/hw/mlx4/qp.c mutex_unlock(&dev->counters_table[qp->port - 1].mutex); qp 2111 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp) qp 2118 drivers/infiniband/hw/mlx4/qp.c rss_context->base_qpn = cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz); qp 2120 drivers/infiniband/hw/mlx4/qp.c cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz & 0xffffff); qp 2121 drivers/infiniband/hw/mlx4/qp.c if (qp->rss_ctx->flags & (MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6)) qp 2123 drivers/infiniband/hw/mlx4/qp.c rss_context->flags = qp->rss_ctx->flags; qp 2127 drivers/infiniband/hw/mlx4/qp.c memcpy(rss_context->rss_key, qp->rss_ctx->rss_key, qp 2142 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp; qp 2161 drivers/infiniband/hw/mlx4/qp.c qp = to_mqp((struct ib_qp *)ibwq); qp 2171 drivers/infiniband/hw/mlx4/qp.c qp = to_mqp(ibqp); qp 2173 drivers/infiniband/hw/mlx4/qp.c pd = get_pd(qp); qp 2178 drivers/infiniband/hw/mlx4/qp.c rdma_port_get_link_layer(&dev->ib_dev, qp->port) == qp 2187 drivers/infiniband/hw/mlx4/qp.c (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16)); qp 2206 drivers/infiniband/hw/mlx4/qp.c if (qp->inl_recv_sz) qp 2209 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_SCATTER_FCS) qp 2217 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_LSO) qp 2233 drivers/infiniband/hw/mlx4/qp.c if (qp->rq.wqe_cnt) qp 2234 drivers/infiniband/hw/mlx4/qp.c context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3; qp 2235 drivers/infiniband/hw/mlx4/qp.c context->rq_size_stride |= qp->rq.wqe_shift - 4; qp 2238 drivers/infiniband/hw/mlx4/qp.c if (qp->sq.wqe_cnt) qp 2239 drivers/infiniband/hw/mlx4/qp.c context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3; qp 2240 drivers/infiniband/hw/mlx4/qp.c context->sq_size_stride |= qp->sq.wqe_shift - 4; qp 2242 drivers/infiniband/hw/mlx4/qp.c if (new_state == IB_QPS_RESET && qp->counter_index) qp 2243 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_free_qp_counter(dev, qp); qp 2246 drivers/infiniband/hw/mlx4/qp.c context->sq_size_stride |= !!qp->sq_no_prefetch << 7; qp 2247 drivers/infiniband/hw/mlx4/qp.c context->xrcd = cpu_to_be32((u32) qp->xrcdn); qp 2271 drivers/infiniband/hw/mlx4/qp.c err = create_qp_lb_counter(dev, qp); qp 2276 drivers/infiniband/hw/mlx4/qp.c dev->counters_table[qp->port - 1].default_counter; qp 2277 drivers/infiniband/hw/mlx4/qp.c if (qp->counter_index) qp 2278 drivers/infiniband/hw/mlx4/qp.c counter_index = qp->counter_index->index; qp 2283 drivers/infiniband/hw/mlx4/qp.c if (qp->counter_index) { qp 2293 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_NETIF) { qp 2294 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_steer_qp_reg(dev, qp, 1); qp 2299 drivers/infiniband/hw/mlx4/qp.c enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ? qp 2308 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) qp 2316 drivers/infiniband/hw/mlx4/qp.c attr_mask & IB_QP_PORT ? attr->port_num : qp->port; qp 2331 drivers/infiniband/hw/mlx4/qp.c if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path, qp 2365 drivers/infiniband/hw/mlx4/qp.c if (mlx4_set_alt_path(dev, attr, attr_mask, qp, qp 2379 drivers/infiniband/hw/mlx4/qp.c get_cqs(qp, src_type, &send_cq, &recv_cq); qp 2419 drivers/infiniband/hw/mlx4/qp.c context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask); qp 2435 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type & qp 2440 drivers/infiniband/hw/mlx4/qp.c !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) && qp 2458 drivers/infiniband/hw/mlx4/qp.c if (qp->rq.wqe_cnt && qp 2461 drivers/infiniband/hw/mlx4/qp.c context->db_rec_addr = cpu_to_be64(qp->db.dma); qp 2467 drivers/infiniband/hw/mlx4/qp.c context->pri_path.sched_queue = (qp->port - 1) << 6; qp 2468 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || qp 2469 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type & qp 2472 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI) qp 2475 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) qp 2479 drivers/infiniband/hw/mlx4/qp.c if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) == qp 2481 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI || qp 2482 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) qp 2485 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD || qp 2486 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI || qp 2487 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) { qp 2488 drivers/infiniband/hw/mlx4/qp.c err = handle_eth_ud_smac_index(dev, qp, context); qp 2493 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI) qp 2494 drivers/infiniband/hw/mlx4/qp.c dev->qp1_proxy[qp->port - 1] = qp; qp 2511 drivers/infiniband/hw/mlx4/qp.c &dev->ib_dev, qp->port) == qp 2542 drivers/infiniband/hw/mlx4/qp.c for (i = 0; i < qp->sq.wqe_cnt; ++i) { qp 2543 drivers/infiniband/hw/mlx4/qp.c ctrl = get_send_wqe(qp, i); qp 2546 drivers/infiniband/hw/mlx4/qp.c 1 << (qp->sq.wqe_shift - 4); qp 2547 drivers/infiniband/hw/mlx4/qp.c stamp_send_wqe(qp, i); qp 2554 drivers/infiniband/hw/mlx4/qp.c fill_qp_rss_context(context, qp); qp 2558 drivers/infiniband/hw/mlx4/qp.c err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state), qp 2560 drivers/infiniband/hw/mlx4/qp.c sqd_event, &qp->mqp); qp 2564 drivers/infiniband/hw/mlx4/qp.c qp->state = new_state; qp 2567 drivers/infiniband/hw/mlx4/qp.c qp->atomic_rd_en = attr->qp_access_flags; qp 2569 drivers/infiniband/hw/mlx4/qp.c qp->resp_depth = attr->max_dest_rd_atomic; qp 2571 drivers/infiniband/hw/mlx4/qp.c qp->port = attr->port_num; qp 2572 drivers/infiniband/hw/mlx4/qp.c update_mcg_macs(dev, qp); qp 2575 drivers/infiniband/hw/mlx4/qp.c qp->alt_port = attr->alt_port_num; qp 2577 drivers/infiniband/hw/mlx4/qp.c if (is_sqp(dev, qp)) qp 2578 drivers/infiniband/hw/mlx4/qp.c store_sqp_attrs(to_msqp(qp), attr, attr_mask); qp 2584 drivers/infiniband/hw/mlx4/qp.c if (is_qp0(dev, qp)) { qp 2586 drivers/infiniband/hw/mlx4/qp.c if (mlx4_INIT_PORT(dev->dev, qp->port)) qp 2588 drivers/infiniband/hw/mlx4/qp.c qp->port); qp 2592 drivers/infiniband/hw/mlx4/qp.c mlx4_CLOSE_PORT(dev->dev, qp->port); qp 2601 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, qp 2604 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); qp 2606 drivers/infiniband/hw/mlx4/qp.c qp->rq.head = 0; qp 2607 drivers/infiniband/hw/mlx4/qp.c qp->rq.tail = 0; qp 2608 drivers/infiniband/hw/mlx4/qp.c qp->sq.head = 0; qp 2609 drivers/infiniband/hw/mlx4/qp.c qp->sq.tail = 0; qp 2610 drivers/infiniband/hw/mlx4/qp.c qp->sq_next_wqe = 0; qp 2611 drivers/infiniband/hw/mlx4/qp.c if (qp->rq.wqe_cnt) qp 2612 drivers/infiniband/hw/mlx4/qp.c *qp->db.db = 0; qp 2614 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_NETIF) qp 2615 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_steer_qp_reg(dev, qp, 0); qp 2617 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) { qp 2618 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); qp 2619 drivers/infiniband/hw/mlx4/qp.c qp->pri.smac = 0; qp 2620 drivers/infiniband/hw/mlx4/qp.c qp->pri.smac_port = 0; qp 2622 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.smac) { qp 2623 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); qp 2624 drivers/infiniband/hw/mlx4/qp.c qp->alt.smac = 0; qp 2626 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.vid < 0x1000) { qp 2627 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); qp 2628 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid = 0xFFFF; qp 2629 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_vid = 0xFFFF; qp 2630 drivers/infiniband/hw/mlx4/qp.c qp->pri.update_vid = 0; qp 2633 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.vid < 0x1000) { qp 2634 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); qp 2635 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid = 0xFFFF; qp 2636 drivers/infiniband/hw/mlx4/qp.c qp->alt.candidate_vid = 0xFFFF; qp 2637 drivers/infiniband/hw/mlx4/qp.c qp->alt.update_vid = 0; qp 2641 drivers/infiniband/hw/mlx4/qp.c if (err && qp->counter_index) qp 2642 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_free_qp_counter(dev, qp); qp 2644 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_steer_qp_reg(dev, qp, 0); qp 2646 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.candidate_smac || qp 2647 drivers/infiniband/hw/mlx4/qp.c (!qp->pri.candidate_smac && qp->pri.candidate_smac_port)) { qp 2649 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac); qp 2651 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) qp 2652 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); qp 2653 drivers/infiniband/hw/mlx4/qp.c qp->pri.smac = qp->pri.candidate_smac; qp 2654 drivers/infiniband/hw/mlx4/qp.c qp->pri.smac_index = qp->pri.candidate_smac_index; qp 2655 drivers/infiniband/hw/mlx4/qp.c qp->pri.smac_port = qp->pri.candidate_smac_port; qp 2657 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_smac = 0; qp 2658 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_smac_index = 0; qp 2659 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_smac_port = 0; qp 2661 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.candidate_smac) { qp 2663 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->alt.candidate_smac); qp 2665 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.smac) qp 2666 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); qp 2667 drivers/infiniband/hw/mlx4/qp.c qp->alt.smac = qp->alt.candidate_smac; qp 2668 drivers/infiniband/hw/mlx4/qp.c qp->alt.smac_index = qp->alt.candidate_smac_index; qp 2669 drivers/infiniband/hw/mlx4/qp.c qp->alt.smac_port = qp->alt.candidate_smac_port; qp 2671 drivers/infiniband/hw/mlx4/qp.c qp->alt.candidate_smac = 0; qp 2672 drivers/infiniband/hw/mlx4/qp.c qp->alt.candidate_smac_index = 0; qp 2673 drivers/infiniband/hw/mlx4/qp.c qp->alt.candidate_smac_port = 0; qp 2676 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.update_vid) { qp 2678 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.candidate_vid < 0x1000) qp 2679 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port, qp 2680 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_vid); qp 2682 drivers/infiniband/hw/mlx4/qp.c if (qp->pri.vid < 0x1000) qp 2683 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp 2684 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid); qp 2685 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid = qp->pri.candidate_vid; qp 2686 drivers/infiniband/hw/mlx4/qp.c qp->pri.vlan_port = qp->pri.candidate_vlan_port; qp 2687 drivers/infiniband/hw/mlx4/qp.c qp->pri.vlan_index = qp->pri.candidate_vlan_index; qp 2689 drivers/infiniband/hw/mlx4/qp.c qp->pri.candidate_vid = 0xFFFF; qp 2690 drivers/infiniband/hw/mlx4/qp.c qp->pri.update_vid = 0; qp 2693 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.update_vid) { qp 2695 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.candidate_vid < 0x1000) qp 2696 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port, qp 2697 drivers/infiniband/hw/mlx4/qp.c qp->alt.candidate_vid); qp 2699 drivers/infiniband/hw/mlx4/qp.c if (qp->alt.vid < 0x1000) qp 2700 drivers/infiniband/hw/mlx4/qp.c mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp 2701 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid); qp 2702 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid = qp->alt.candidate_vid; qp 2703 drivers/infiniband/hw/mlx4/qp.c qp->alt.vlan_port = qp->alt.candidate_vlan_port; qp 2704 drivers/infiniband/hw/mlx4/qp.c qp->alt.vlan_index = qp->alt.candidate_vlan_index; qp 2706 drivers/infiniband/hw/mlx4/qp.c qp->alt.candidate_vid = 0xFFFF; qp 2707 drivers/infiniband/hw/mlx4/qp.c qp->alt.update_vid = 0; qp 2722 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = to_mqp(ibqp); qp 2725 drivers/infiniband/hw/mlx4/qp.c mutex_lock(&qp->mutex); qp 2727 drivers/infiniband/hw/mlx4/qp.c cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; qp 2792 drivers/infiniband/hw/mlx4/qp.c int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; qp 2842 drivers/infiniband/hw/mlx4/qp.c mutex_unlock(&qp->mutex); qp 2884 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device); qp 2907 drivers/infiniband/hw/mlx4/qp.c if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) qp 2912 drivers/infiniband/hw/mlx4/qp.c if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) { qp 2929 drivers/infiniband/hw/mlx4/qp.c err = ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey); qp 2933 drivers/infiniband/hw/mlx4/qp.c if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER) qp 2937 drivers/infiniband/hw/mlx4/qp.c cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel); qp 2941 drivers/infiniband/hw/mlx4/qp.c if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey)) qp 2944 drivers/infiniband/hw/mlx4/qp.c if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey)) qp 2948 drivers/infiniband/hw/mlx4/qp.c sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn); qp 3035 drivers/infiniband/hw/mlx4/qp.c struct ib_device *ib_dev = sqp->qp.ibqp.device; qp 3059 drivers/infiniband/hw/mlx4/qp.c is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET; qp 3073 drivers/infiniband/hw/mlx4/qp.c err = fill_gid_by_hw_index(ibdev, sqp->qp.port, qp 3122 drivers/infiniband/hw/mlx4/qp.c demux[sqp->qp.port - 1]. qp 3125 drivers/infiniband/hw/mlx4/qp.c to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. qp 3158 drivers/infiniband/hw/mlx4/qp.c mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | qp 3207 drivers/infiniband/hw/mlx4/qp.c sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : qp 3210 drivers/infiniband/hw/mlx4/qp.c sqp->qp.port); qp 3211 drivers/infiniband/hw/mlx4/qp.c if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15) qp 3217 drivers/infiniband/hw/mlx4/qp.c if (!sqp->qp.ibqp.qp_num) qp 3218 drivers/infiniband/hw/mlx4/qp.c err = ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, qp 3221 drivers/infiniband/hw/mlx4/qp.c err = ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index, qp 3231 drivers/infiniband/hw/mlx4/qp.c sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); qp 3492 drivers/infiniband/hw/mlx4/qp.c const struct ib_ud_wr *wr, struct mlx4_ib_qp *qp, qp 3500 drivers/infiniband/hw/mlx4/qp.c if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) && qp 3501 drivers/infiniband/hw/mlx4/qp.c wr->wr.num_sge > qp->sq.max_gs - (halign >> 4))) qp 3536 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = to_mqp(ibqp); qp 3553 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) { qp 3554 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_sqp *sqp = to_msqp(qp); qp 3561 drivers/infiniband/hw/mlx4/qp.c if (!fill_gid_by_hw_index(mdev, sqp->qp.port, qp 3564 drivers/infiniband/hw/mlx4/qp.c qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? qp 3565 drivers/infiniband/hw/mlx4/qp.c to_mqp(sqp->roce_v2_gsi) : qp; qp 3572 drivers/infiniband/hw/mlx4/qp.c spin_lock_irqsave(&qp->sq.lock, flags); qp 3581 drivers/infiniband/hw/mlx4/qp.c ind = qp->sq_next_wqe; qp 3587 drivers/infiniband/hw/mlx4/qp.c if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { qp 3593 drivers/infiniband/hw/mlx4/qp.c if (unlikely(wr->num_sge > qp->sq.max_gs)) { qp 3599 drivers/infiniband/hw/mlx4/qp.c ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); qp 3600 drivers/infiniband/hw/mlx4/qp.c qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id; qp 3610 drivers/infiniband/hw/mlx4/qp.c qp->sq_signal_bits; qp 3617 drivers/infiniband/hw/mlx4/qp.c switch (qp->mlx4_ib_qp_type) { qp 3681 drivers/infiniband/hw/mlx4/qp.c err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr), qp 3705 drivers/infiniband/hw/mlx4/qp.c err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen, qp 3718 drivers/infiniband/hw/mlx4/qp.c err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr), qp 3742 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type); qp 3752 drivers/infiniband/hw/mlx4/qp.c err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl, qp 3778 drivers/infiniband/hw/mlx4/qp.c if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || qp 3779 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI || qp 3780 drivers/infiniband/hw/mlx4/qp.c qp->mlx4_ib_qp_type & qp 3814 drivers/infiniband/hw/mlx4/qp.c (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh; qp 3822 drivers/infiniband/hw/mlx4/qp.c stamp_send_wqe(qp, ind + qp->sq_spare_wqes); qp 3828 drivers/infiniband/hw/mlx4/qp.c qp->sq.head += nreq; qp 3836 drivers/infiniband/hw/mlx4/qp.c writel_relaxed(qp->doorbell_qpn, qp 3839 drivers/infiniband/hw/mlx4/qp.c stamp_send_wqe(qp, ind + qp->sq_spare_wqes - 1); qp 3841 drivers/infiniband/hw/mlx4/qp.c qp->sq_next_wqe = ind; qp 3844 drivers/infiniband/hw/mlx4/qp.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 3858 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = to_mqp(ibqp); qp 3868 drivers/infiniband/hw/mlx4/qp.c max_gs = qp->rq.max_gs; qp 3869 drivers/infiniband/hw/mlx4/qp.c spin_lock_irqsave(&qp->rq.lock, flags); qp 3879 drivers/infiniband/hw/mlx4/qp.c ind = qp->rq.head & (qp->rq.wqe_cnt - 1); qp 3882 drivers/infiniband/hw/mlx4/qp.c if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { qp 3888 drivers/infiniband/hw/mlx4/qp.c if (unlikely(wr->num_sge > qp->rq.max_gs)) { qp 3894 drivers/infiniband/hw/mlx4/qp.c scat = get_recv_wqe(qp, ind); qp 3896 drivers/infiniband/hw/mlx4/qp.c if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | qp 3899 drivers/infiniband/hw/mlx4/qp.c qp->sqp_proxy_rcv[ind].map, qp 3906 drivers/infiniband/hw/mlx4/qp.c scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map); qp 3920 drivers/infiniband/hw/mlx4/qp.c qp->rq.wrid[ind] = wr->wr_id; qp 3922 drivers/infiniband/hw/mlx4/qp.c ind = (ind + 1) & (qp->rq.wqe_cnt - 1); qp 3927 drivers/infiniband/hw/mlx4/qp.c qp->rq.head += nreq; qp 3935 drivers/infiniband/hw/mlx4/qp.c *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); qp 3938 drivers/infiniband/hw/mlx4/qp.c spin_unlock_irqrestore(&qp->rq.lock, flags); qp 4026 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = to_mqp(ibqp); qp 4034 drivers/infiniband/hw/mlx4/qp.c mutex_lock(&qp->mutex); qp 4036 drivers/infiniband/hw/mlx4/qp.c if (qp->state == IB_QPS_RESET) { qp 4041 drivers/infiniband/hw/mlx4/qp.c err = mlx4_qp_query(dev->dev, &qp->mqp, &context); qp 4049 drivers/infiniband/hw/mlx4/qp.c qp->state = to_ib_qp_state(mlx4_state); qp 4050 drivers/infiniband/hw/mlx4/qp.c qp_attr->qp_state = qp->state; qp 4061 drivers/infiniband/hw/mlx4/qp.c if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { qp 4071 drivers/infiniband/hw/mlx4/qp.c qp_attr->port_num = qp->port; qp 4091 drivers/infiniband/hw/mlx4/qp.c qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; qp 4092 drivers/infiniband/hw/mlx4/qp.c qp_attr->cap.max_recv_sge = qp->rq.max_gs; qp 4095 drivers/infiniband/hw/mlx4/qp.c qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; qp 4096 drivers/infiniband/hw/mlx4/qp.c qp_attr->cap.max_send_sge = qp->sq.max_gs; qp 4111 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) qp 4114 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_LSO) qp 4117 drivers/infiniband/hw/mlx4/qp.c if (qp->flags & MLX4_IB_QP_NETIF) qp 4121 drivers/infiniband/hw/mlx4/qp.c qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ? qp 4125 drivers/infiniband/hw/mlx4/qp.c mutex_unlock(&qp->mutex); qp 4135 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp; qp 4171 drivers/infiniband/hw/mlx4/qp.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 4172 drivers/infiniband/hw/mlx4/qp.c if (!qp) qp 4175 drivers/infiniband/hw/mlx4/qp.c qp->pri.vid = 0xFFFF; qp 4176 drivers/infiniband/hw/mlx4/qp.c qp->alt.vid = 0xFFFF; qp 4188 drivers/infiniband/hw/mlx4/qp.c err = create_rq(pd, &ib_qp_init_attr, udata, qp); qp 4190 drivers/infiniband/hw/mlx4/qp.c kfree(qp); qp 4194 drivers/infiniband/hw/mlx4/qp.c qp->ibwq.event_handler = init_attr->event_handler; qp 4195 drivers/infiniband/hw/mlx4/qp.c qp->ibwq.wq_num = qp->mqp.qpn; qp 4196 drivers/infiniband/hw/mlx4/qp.c qp->ibwq.state = IB_WQS_RESET; qp 4198 drivers/infiniband/hw/mlx4/qp.c return &qp->ibwq; qp 4216 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq); qp 4225 drivers/infiniband/hw/mlx4/qp.c qp_cur_state = qp->state; qp 4234 drivers/infiniband/hw/mlx4/qp.c attr.port_num = qp->port; qp 4264 drivers/infiniband/hw/mlx4/qp.c qp->state = qp_new_state; qp 4272 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq); qp 4314 drivers/infiniband/hw/mlx4/qp.c mutex_lock(&qp->mutex); qp 4319 drivers/infiniband/hw/mlx4/qp.c if (qp->rss_usecnt) qp 4325 drivers/infiniband/hw/mlx4/qp.c mutex_unlock(&qp->mutex); qp 4333 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq); qp 4335 drivers/infiniband/hw/mlx4/qp.c if (qp->counter_index) qp 4336 drivers/infiniband/hw/mlx4/qp.c mlx4_ib_free_qp_counter(dev, qp); qp 4338 drivers/infiniband/hw/mlx4/qp.c destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata); qp 4340 drivers/infiniband/hw/mlx4/qp.c kfree(qp); qp 4477 drivers/infiniband/hw/mlx4/qp.c void mlx4_ib_drain_sq(struct ib_qp *qp) qp 4479 drivers/infiniband/hw/mlx4/qp.c struct ib_cq *cq = qp->send_cq; qp 4491 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_dev *dev = to_mdev(qp->device); qp 4494 drivers/infiniband/hw/mlx4/qp.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE); qp 4503 drivers/infiniband/hw/mlx4/qp.c ret = _mlx4_ib_post_send(qp, &swr.wr, &bad_swr, true); qp 4512 drivers/infiniband/hw/mlx4/qp.c void mlx4_ib_drain_rq(struct ib_qp *qp) qp 4514 drivers/infiniband/hw/mlx4/qp.c struct ib_cq *cq = qp->recv_cq; qp 4520 drivers/infiniband/hw/mlx4/qp.c struct mlx4_ib_dev *dev = to_mdev(qp->device); qp 4523 drivers/infiniband/hw/mlx4/qp.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE); qp 4533 drivers/infiniband/hw/mlx4/qp.c ret = _mlx4_ib_post_recv(qp, &rwr, &bad_rwr, true); qp 166 drivers/infiniband/hw/mlx5/cq.c struct mlx5_ib_qp *qp) qp 168 drivers/infiniband/hw/mlx5/cq.c enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1); qp 169 drivers/infiniband/hw/mlx5/cq.c struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); qp 177 drivers/infiniband/hw/mlx5/cq.c if (qp->ibqp.srq || qp->ibqp.xrcd) { qp 180 drivers/infiniband/hw/mlx5/cq.c if (qp->ibqp.xrcd) { qp 184 drivers/infiniband/hw/mlx5/cq.c srq = to_msrq(qp->ibqp.srq); qp 194 drivers/infiniband/hw/mlx5/cq.c wq = &qp->rq; qp 228 drivers/infiniband/hw/mlx5/cq.c if (unlikely(is_qp1(qp->ibqp.qp_type))) { qp 231 drivers/infiniband/hw/mlx5/cq.c ib_find_cached_pkey(&dev->ib_dev, qp->port, pkey, qp 333 drivers/infiniband/hw/mlx5/cq.c static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, qp 339 drivers/infiniband/hw/mlx5/cq.c idx = tail & (qp->sq.wqe_cnt - 1); qp 343 drivers/infiniband/hw/mlx5/cq.c tail = qp->sq.w_list[idx].next; qp 345 drivers/infiniband/hw/mlx5/cq.c tail = qp->sq.w_list[idx].next; qp 346 drivers/infiniband/hw/mlx5/cq.c qp->sq.last_poll = tail; qp 386 drivers/infiniband/hw/mlx5/cq.c static void sw_comp(struct mlx5_ib_qp *qp, int num_entries, struct ib_wc *wc, qp 394 drivers/infiniband/hw/mlx5/cq.c wq = (is_send) ? &qp->sq : &qp->rq; qp 413 drivers/infiniband/hw/mlx5/cq.c wc->qp = &qp->ibqp; qp 422 drivers/infiniband/hw/mlx5/cq.c struct mlx5_ib_qp *qp; qp 426 drivers/infiniband/hw/mlx5/cq.c list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { qp 427 drivers/infiniband/hw/mlx5/cq.c sw_comp(qp, num_entries, wc + *npolled, npolled, true); qp 432 drivers/infiniband/hw/mlx5/cq.c list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { qp 433 drivers/infiniband/hw/mlx5/cq.c sw_comp(qp, num_entries, wc + *npolled, npolled, false); qp 494 drivers/infiniband/hw/mlx5/cq.c wc->qp = &(*cur_qp)->ibqp; qp 608 drivers/infiniband/hw/mlx5/devx.c struct mlx5_ib_qp *qp = to_mqp(uobj->object); qp 609 drivers/infiniband/hw/mlx5/devx.c enum ib_qp_type qp_type = qp->ibqp.qp_type; qp 612 drivers/infiniband/hw/mlx5/devx.c (qp->flags & MLX5_IB_QP_UNDERLAY)) { qp 614 drivers/infiniband/hw/mlx5/devx.c &qp->raw_packet_qp; qp 630 drivers/infiniband/hw/mlx5/devx.c qp->dct.mdct.mqp.qpn) == obj_id; qp 633 drivers/infiniband/hw/mlx5/devx.c qp->ibqp.qp_num) == obj_id; qp 82 drivers/infiniband/hw/mlx5/flow.c struct ib_qp *qp = NULL; qp 131 drivers/infiniband/hw/mlx5/flow.c qp = uverbs_attr_get_obj(attrs, qp 133 drivers/infiniband/hw/mlx5/flow.c if (IS_ERR(qp)) qp 134 drivers/infiniband/hw/mlx5/flow.c return PTR_ERR(qp); qp 136 drivers/infiniband/hw/mlx5/flow.c if (qp->qp_type != IB_QPT_RAW_PACKET) qp 139 drivers/infiniband/hw/mlx5/flow.c mqp = to_mqp(qp); qp 206 drivers/infiniband/hw/mlx5/flow.c ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); qp 62 drivers/infiniband/hw/mlx5/gsi.c static struct mlx5_ib_gsi_qp *gsi_qp(struct ib_qp *qp) qp 64 drivers/infiniband/hw/mlx5/gsi.c return container_of(qp, struct mlx5_ib_gsi_qp, ibqp); qp 109 drivers/infiniband/hw/mlx5/gsi.c wr->wc.qp = &gsi->ibqp; qp 213 drivers/infiniband/hw/mlx5/gsi.c int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp) qp 215 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 216 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); qp 272 drivers/infiniband/hw/mlx5/gsi.c static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp, qp 275 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 285 drivers/infiniband/hw/mlx5/gsi.c ret = ib_modify_qp(qp, &attr, mask); qp 288 drivers/infiniband/hw/mlx5/gsi.c qp->qp_num, ret); qp 293 drivers/infiniband/hw/mlx5/gsi.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE); qp 296 drivers/infiniband/hw/mlx5/gsi.c qp->qp_num, ret); qp 302 drivers/infiniband/hw/mlx5/gsi.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN); qp 305 drivers/infiniband/hw/mlx5/gsi.c qp->qp_num, ret); qp 316 drivers/infiniband/hw/mlx5/gsi.c struct ib_qp *qp; qp 335 drivers/infiniband/hw/mlx5/gsi.c qp = gsi->tx_qps[qp_index]; qp 337 drivers/infiniband/hw/mlx5/gsi.c if (qp) { qp 343 drivers/infiniband/hw/mlx5/gsi.c qp = create_gsi_ud_qp(gsi); qp 344 drivers/infiniband/hw/mlx5/gsi.c if (IS_ERR(qp)) { qp 346 drivers/infiniband/hw/mlx5/gsi.c PTR_ERR(qp)); qp 350 drivers/infiniband/hw/mlx5/gsi.c ret = modify_to_rts(gsi, qp, qp_index); qp 356 drivers/infiniband/hw/mlx5/gsi.c gsi->tx_qps[qp_index] = qp; qp 362 drivers/infiniband/hw/mlx5/gsi.c WARN_ON_ONCE(qp); qp 373 drivers/infiniband/hw/mlx5/gsi.c int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, qp 376 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 377 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); qp 398 drivers/infiniband/hw/mlx5/gsi.c int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 402 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); qp 452 drivers/infiniband/hw/mlx5/gsi.c .qp = &gsi->ibqp, qp 480 drivers/infiniband/hw/mlx5/gsi.c int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr, qp 483 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); qp 524 drivers/infiniband/hw/mlx5/gsi.c int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr, qp 527 drivers/infiniband/hw/mlx5/gsi.c struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); qp 1687 drivers/infiniband/hw/mlx5/main.c int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) qp 1694 drivers/infiniband/hw/mlx5/main.c if (qp) qp 1710 drivers/infiniband/hw/mlx5/main.c void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) qp 1715 drivers/infiniband/hw/mlx5/main.c if (qp) qp 3782 drivers/infiniband/hw/mlx5/main.c static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, qp 3787 drivers/infiniband/hw/mlx5/main.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 3788 drivers/infiniband/hw/mlx5/main.c struct mlx5_ib_qp *mqp = to_mqp(qp); qp 4792 drivers/infiniband/hw/mlx5/main.c if (dev->umrc.qp) qp 4793 drivers/infiniband/hw/mlx5/main.c mlx5_ib_destroy_qp(dev->umrc.qp, NULL); qp 4810 drivers/infiniband/hw/mlx5/main.c struct ib_qp *qp; qp 4841 drivers/infiniband/hw/mlx5/main.c qp = mlx5_ib_create_qp(pd, init_attr, NULL); qp 4842 drivers/infiniband/hw/mlx5/main.c if (IS_ERR(qp)) { qp 4844 drivers/infiniband/hw/mlx5/main.c ret = PTR_ERR(qp); qp 4847 drivers/infiniband/hw/mlx5/main.c qp->device = &dev->ib_dev; qp 4848 drivers/infiniband/hw/mlx5/main.c qp->real_qp = qp; qp 4849 drivers/infiniband/hw/mlx5/main.c qp->uobject = NULL; qp 4850 drivers/infiniband/hw/mlx5/main.c qp->qp_type = MLX5_IB_QPT_REG_UMR; qp 4851 drivers/infiniband/hw/mlx5/main.c qp->send_cq = init_attr->send_cq; qp 4852 drivers/infiniband/hw/mlx5/main.c qp->recv_cq = init_attr->recv_cq; qp 4856 drivers/infiniband/hw/mlx5/main.c ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX | qp 4867 drivers/infiniband/hw/mlx5/main.c ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); qp 4875 drivers/infiniband/hw/mlx5/main.c ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); qp 4881 drivers/infiniband/hw/mlx5/main.c dev->umrc.qp = qp; qp 4898 drivers/infiniband/hw/mlx5/main.c mlx5_ib_destroy_qp(qp, NULL); qp 4899 drivers/infiniband/hw/mlx5/main.c dev->umrc.qp = NULL; qp 5657 drivers/infiniband/hw/mlx5/main.c struct ib_qp *qp) qp 5659 drivers/infiniband/hw/mlx5/main.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 5672 drivers/infiniband/hw/mlx5/main.c err = mlx5_ib_qp_set_counter(qp, counter); qp 5685 drivers/infiniband/hw/mlx5/main.c static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) qp 5687 drivers/infiniband/hw/mlx5/main.c return mlx5_ib_qp_set_counter(qp, NULL); qp 654 drivers/infiniband/hw/mlx5/mlx5_ib.h struct ib_qp *qp; qp 1118 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp); qp 1119 drivers/infiniband/hw/mlx5/mlx5_ib.h void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp); qp 1127 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); qp 1128 drivers/infiniband/hw/mlx5/mlx5_ib.h void mlx5_ib_drain_sq(struct ib_qp *qp); qp 1129 drivers/infiniband/hw/mlx5/mlx5_ib.h void mlx5_ib_drain_rq(struct ib_qp *qp); qp 1134 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer, qp 1136 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer, qp 1316 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp); qp 1317 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, qp 1319 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 1322 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr, qp 1324 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr, qp 1479 drivers/infiniband/hw/mlx5/mlx5_ib.h int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter); qp 834 drivers/infiniband/hw/mlx5/mr.c err = ib_post_send(umrc->qp, &umrwr->wr, &bad); qp 1027 drivers/infiniband/hw/mlx5/odp.c struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) qp 1033 drivers/infiniband/hw/mlx5/odp.c u32 qpn = qp->trans_qp.base.mqp.qpn; qp 1054 drivers/infiniband/hw/mlx5/odp.c if (qp->ibqp.qp_type == IB_QPT_XRC_INI) qp 1057 drivers/infiniband/hw/mlx5/odp.c if (qp->ibqp.qp_type == IB_QPT_UD || qp 1058 drivers/infiniband/hw/mlx5/odp.c qp->qp_sub_type == MLX5_IB_QPT_DCI) { qp 1104 drivers/infiniband/hw/mlx5/odp.c struct mlx5_ib_qp *qp, qp 1108 drivers/infiniband/hw/mlx5/odp.c struct mlx5_ib_wq *wq = &qp->rq; qp 1111 drivers/infiniband/hw/mlx5/odp.c if (qp->wq_sig) { qp 1174 drivers/infiniband/hw/mlx5/odp.c struct mlx5_ib_qp *qp; qp 1198 drivers/infiniband/hw/mlx5/odp.c qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL; qp 1199 drivers/infiniband/hw/mlx5/odp.c if (qp && sq) { qp 1200 drivers/infiniband/hw/mlx5/odp.c ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, qp 1205 drivers/infiniband/hw/mlx5/odp.c dev, pfault, qp, &wqe, &wqe_end, bytes_copied); qp 1206 drivers/infiniband/hw/mlx5/odp.c } else if (qp && !sq) { qp 1207 drivers/infiniband/hw/mlx5/odp.c ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, qp 1212 drivers/infiniband/hw/mlx5/odp.c dev, qp, wqe, &wqe_end, bytes_copied); qp 1213 drivers/infiniband/hw/mlx5/odp.c } else if (!qp) { qp 163 drivers/infiniband/hw/mlx5/qp.c int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, qp 169 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp_base *base = &qp->trans_qp.base; qp 171 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_wq *wq = &qp->sq; qp 229 drivers/infiniband/hw/mlx5/qp.c int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, qp 235 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp_base *base = &qp->trans_qp.base; qp 237 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_wq *wq = &qp->rq; qp 283 drivers/infiniband/hw/mlx5/qp.c static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type) qp 285 drivers/infiniband/hw/mlx5/qp.c struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; qp 290 drivers/infiniband/hw/mlx5/qp.c to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port; qp 295 drivers/infiniband/hw/mlx5/qp.c event.element.qp = ibqp; qp 322 drivers/infiniband/hw/mlx5/qp.c pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn); qp 331 drivers/infiniband/hw/mlx5/qp.c int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd) qp 341 drivers/infiniband/hw/mlx5/qp.c qp->rq.max_gs = 0; qp 342 drivers/infiniband/hw/mlx5/qp.c qp->rq.wqe_cnt = 0; qp 343 drivers/infiniband/hw/mlx5/qp.c qp->rq.wqe_shift = 0; qp 348 drivers/infiniband/hw/mlx5/qp.c qp->rq.wqe_cnt = ucmd->rq_wqe_count; qp 351 drivers/infiniband/hw/mlx5/qp.c qp->rq.wqe_shift = ucmd->rq_wqe_shift; qp 352 drivers/infiniband/hw/mlx5/qp.c if ((1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) < qp->wq_sig) qp 354 drivers/infiniband/hw/mlx5/qp.c qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig; qp 355 drivers/infiniband/hw/mlx5/qp.c qp->rq.max_post = qp->rq.wqe_cnt; qp 357 drivers/infiniband/hw/mlx5/qp.c wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0; qp 362 drivers/infiniband/hw/mlx5/qp.c qp->rq.wqe_cnt = wq_size / wqe_size; qp 370 drivers/infiniband/hw/mlx5/qp.c qp->rq.wqe_shift = ilog2(wqe_size); qp 371 drivers/infiniband/hw/mlx5/qp.c qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig; qp 372 drivers/infiniband/hw/mlx5/qp.c qp->rq.max_post = qp->rq.wqe_cnt; qp 477 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp) qp 496 drivers/infiniband/hw/mlx5/qp.c qp->max_inline_data = wqe_size - sq_overhead(attr) - qp 498 drivers/infiniband/hw/mlx5/qp.c attr->cap.max_inline_data = qp->max_inline_data; qp 501 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; qp 502 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { qp 505 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_cnt, qp 509 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); qp 510 drivers/infiniband/hw/mlx5/qp.c qp->sq.max_gs = get_send_sge(attr, wqe_size); qp 511 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.max_gs < attr->cap.max_send_sge) qp 514 drivers/infiniband/hw/mlx5/qp.c attr->cap.max_send_sge = qp->sq.max_gs; qp 515 drivers/infiniband/hw/mlx5/qp.c qp->sq.max_post = wq_size / wqe_size; qp 516 drivers/infiniband/hw/mlx5/qp.c attr->cap.max_send_wr = qp->sq.max_post; qp 522 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, qp 527 drivers/infiniband/hw/mlx5/qp.c int desc_sz = 1 << qp->sq.wqe_shift; qp 541 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_cnt = ucmd->sq_wqe_count; qp 543 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { qp 545 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_cnt, qp 551 drivers/infiniband/hw/mlx5/qp.c qp->flags & MLX5_IB_QP_UNDERLAY) { qp 552 drivers/infiniband/hw/mlx5/qp.c base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift; qp 553 drivers/infiniband/hw/mlx5/qp.c qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6; qp 555 drivers/infiniband/hw/mlx5/qp.c base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + qp 556 drivers/infiniband/hw/mlx5/qp.c (qp->sq.wqe_cnt << 6); qp 856 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, struct ib_udata *udata, qp 891 drivers/infiniband/hw/mlx5/qp.c } else if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) { qp 909 drivers/infiniband/hw/mlx5/qp.c qp->rq.offset = 0; qp 910 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); qp 911 drivers/infiniband/hw/mlx5/qp.c qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; qp 913 drivers/infiniband/hw/mlx5/qp.c err = set_user_buf_size(dev, qp, &ucmd, base, attr); qp 953 drivers/infiniband/hw/mlx5/qp.c qp->bfregn = bfregn; qp 955 drivers/infiniband/hw/mlx5/qp.c err = mlx5_ib_db_map_user(context, udata, ucmd.db_addr, &qp->db); qp 966 drivers/infiniband/hw/mlx5/qp.c qp->create_type = MLX5_QP_USER; qp 971 drivers/infiniband/hw/mlx5/qp.c mlx5_ib_db_unmap_user(context, &qp->db); qp 986 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base, qp 995 drivers/infiniband/hw/mlx5/qp.c mlx5_ib_db_unmap_user(context, &qp->db); qp 1002 drivers/infiniband/hw/mlx5/qp.c if (qp->bfregn != MLX5_IB_INVALID_BFREG) qp 1003 drivers/infiniband/hw/mlx5/qp.c mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn); qp 1032 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, qp 1048 drivers/infiniband/hw/mlx5/qp.c qp->bf.bfreg = &dev->fp_bfreg; qp 1050 drivers/infiniband/hw/mlx5/qp.c qp->bf.bfreg = &dev->bfreg; qp 1055 drivers/infiniband/hw/mlx5/qp.c qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2; qp 1056 drivers/infiniband/hw/mlx5/qp.c uar_index = qp->bf.bfreg->index; qp 1058 drivers/infiniband/hw/mlx5/qp.c err = calc_sq_size(dev, init_attr, qp); qp 1064 drivers/infiniband/hw/mlx5/qp.c qp->rq.offset = 0; qp 1065 drivers/infiniband/hw/mlx5/qp.c qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; qp 1066 drivers/infiniband/hw/mlx5/qp.c base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift); qp 1069 drivers/infiniband/hw/mlx5/qp.c &qp->buf, dev->mdev->priv.numa_node); qp 1075 drivers/infiniband/hw/mlx5/qp.c if (qp->rq.wqe_cnt) qp 1076 drivers/infiniband/hw/mlx5/qp.c mlx5_init_fbc(qp->buf.frags, qp->rq.wqe_shift, qp 1077 drivers/infiniband/hw/mlx5/qp.c ilog2(qp->rq.wqe_cnt), &qp->rq.fbc); qp 1079 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt) { qp 1080 drivers/infiniband/hw/mlx5/qp.c int sq_strides_offset = (qp->sq.offset & (PAGE_SIZE - 1)) / qp 1082 drivers/infiniband/hw/mlx5/qp.c mlx5_init_fbc_offset(qp->buf.frags + qp 1083 drivers/infiniband/hw/mlx5/qp.c (qp->sq.offset / PAGE_SIZE), qp 1085 drivers/infiniband/hw/mlx5/qp.c ilog2(qp->sq.wqe_cnt), qp 1086 drivers/infiniband/hw/mlx5/qp.c sq_strides_offset, &qp->sq.fbc); qp 1088 drivers/infiniband/hw/mlx5/qp.c qp->sq.cur_edge = get_sq_edge(&qp->sq, 0); qp 1092 drivers/infiniband/hw/mlx5/qp.c MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages; qp 1101 drivers/infiniband/hw/mlx5/qp.c MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); qp 1109 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_SQPN_QP1; qp 1112 drivers/infiniband/hw/mlx5/qp.c mlx5_fill_page_frag_array(&qp->buf, qp 1116 drivers/infiniband/hw/mlx5/qp.c err = mlx5_db_alloc(dev->mdev, &qp->db); qp 1122 drivers/infiniband/hw/mlx5/qp.c qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt, qp 1123 drivers/infiniband/hw/mlx5/qp.c sizeof(*qp->sq.wrid), GFP_KERNEL); qp 1124 drivers/infiniband/hw/mlx5/qp.c qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt, qp 1125 drivers/infiniband/hw/mlx5/qp.c sizeof(*qp->sq.wr_data), GFP_KERNEL); qp 1126 drivers/infiniband/hw/mlx5/qp.c qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt, qp 1127 drivers/infiniband/hw/mlx5/qp.c sizeof(*qp->rq.wrid), GFP_KERNEL); qp 1128 drivers/infiniband/hw/mlx5/qp.c qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt, qp 1129 drivers/infiniband/hw/mlx5/qp.c sizeof(*qp->sq.w_list), GFP_KERNEL); qp 1130 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt, qp 1131 drivers/infiniband/hw/mlx5/qp.c sizeof(*qp->sq.wqe_head), GFP_KERNEL); qp 1133 drivers/infiniband/hw/mlx5/qp.c if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid || qp 1134 drivers/infiniband/hw/mlx5/qp.c !qp->sq.w_list || !qp->sq.wqe_head) { qp 1138 drivers/infiniband/hw/mlx5/qp.c qp->create_type = MLX5_QP_KERNEL; qp 1143 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.wqe_head); qp 1144 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.w_list); qp 1145 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.wrid); qp 1146 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.wr_data); qp 1147 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->rq.wrid); qp 1148 drivers/infiniband/hw/mlx5/qp.c mlx5_db_free(dev->mdev, &qp->db); qp 1154 drivers/infiniband/hw/mlx5/qp.c mlx5_frag_buf_free(dev->mdev, &qp->buf); qp 1158 drivers/infiniband/hw/mlx5/qp.c static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) qp 1160 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.wqe_head); qp 1161 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.w_list); qp 1162 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.wrid); qp 1163 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->sq.wr_data); qp 1164 drivers/infiniband/hw/mlx5/qp.c kvfree(qp->rq.wrid); qp 1165 drivers/infiniband/hw/mlx5/qp.c mlx5_db_free(dev->mdev, &qp->db); qp 1166 drivers/infiniband/hw/mlx5/qp.c mlx5_frag_buf_free(dev->mdev, &qp->buf); qp 1169 drivers/infiniband/hw/mlx5/qp.c static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr) qp 1175 drivers/infiniband/hw/mlx5/qp.c else if (!qp->has_rq) qp 1191 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, qp 1200 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_UNDERLAY) qp 1201 drivers/infiniband/hw/mlx5/qp.c MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn); qp 1450 drivers/infiniband/hw/mlx5/qp.c static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, qp 1456 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; qp 1466 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt) { qp 1467 drivers/infiniband/hw/mlx5/qp.c err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd); qp 1482 drivers/infiniband/hw/mlx5/qp.c sq->base.container_mibqp = qp; qp 1486 drivers/infiniband/hw/mlx5/qp.c if (qp->rq.wqe_cnt) { qp 1487 drivers/infiniband/hw/mlx5/qp.c rq->base.container_mibqp = qp; qp 1489 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_CVLAN_STRIPPING) qp 1491 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_PCI_WRITE_END_PADDING) qp 1498 drivers/infiniband/hw/mlx5/qp.c dev, rq, tdn, &qp->flags_en, pd, out, qp 1525 drivers/infiniband/hw/mlx5/qp.c qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn : qp 1534 drivers/infiniband/hw/mlx5/qp.c destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, pd); qp 1538 drivers/infiniband/hw/mlx5/qp.c if (!qp->sq.wqe_cnt) qp 1548 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp) qp 1550 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; qp 1554 drivers/infiniband/hw/mlx5/qp.c if (qp->rq.wqe_cnt) { qp 1555 drivers/infiniband/hw/mlx5/qp.c destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, qp->ibqp.pd); qp 1559 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt) { qp 1561 drivers/infiniband/hw/mlx5/qp.c destroy_raw_packet_qp_tis(dev, sq, qp->ibqp.pd); qp 1565 drivers/infiniband/hw/mlx5/qp.c static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp, qp 1571 drivers/infiniband/hw/mlx5/qp.c sq->sq = &qp->sq; qp 1572 drivers/infiniband/hw/mlx5/qp.c rq->rq = &qp->rq; qp 1573 drivers/infiniband/hw/mlx5/qp.c sq->doorbell = &qp->db; qp 1574 drivers/infiniband/hw/mlx5/qp.c rq->doorbell = &qp->db; qp 1577 drivers/infiniband/hw/mlx5/qp.c static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) qp 1579 drivers/infiniband/hw/mlx5/qp.c if (qp->flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC | qp 1582 drivers/infiniband/hw/mlx5/qp.c mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn, qp 1583 drivers/infiniband/hw/mlx5/qp.c to_mpd(qp->ibqp.pd)->uid); qp 1586 drivers/infiniband/hw/mlx5/qp.c static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, qp 1663 drivers/infiniband/hw/mlx5/qp.c qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC; qp 1668 drivers/infiniband/hw/mlx5/qp.c qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC; qp 1796 drivers/infiniband/hw/mlx5/qp.c qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn); qp 1801 drivers/infiniband/hw/mlx5/qp.c mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn, qp 1810 drivers/infiniband/hw/mlx5/qp.c resp.tirn = qp->rss_qp.tirn; qp 1831 drivers/infiniband/hw/mlx5/qp.c qp->trans_qp.base.mqp.qpn = 0; qp 1832 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_RSS; qp 1836 drivers/infiniband/hw/mlx5/qp.c mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn, mucontext->devx_uid); qp 1947 drivers/infiniband/hw/mlx5/qp.c struct ib_udata *udata, struct mlx5_ib_qp *qp) qp 1966 drivers/infiniband/hw/mlx5/qp.c mutex_init(&qp->mutex); qp 1967 drivers/infiniband/hw/mlx5/qp.c spin_lock_init(&qp->sq.lock); qp 1968 drivers/infiniband/hw/mlx5/qp.c spin_lock_init(&qp->rq.lock); qp 1978 drivers/infiniband/hw/mlx5/qp.c err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata); qp 1987 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK; qp 2000 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_CROSS_CHANNEL; qp 2002 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_MANAGED_SEND; qp 2004 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_MANAGED_RECV; qp 2024 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_CAP_SCATTER_FCS; qp 2028 drivers/infiniband/hw/mlx5/qp.c qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; qp 2035 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_CVLAN_STRIPPING; qp 2061 drivers/infiniband/hw/mlx5/qp.c qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE); qp 2063 drivers/infiniband/hw/mlx5/qp.c qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE); qp 2070 drivers/infiniband/hw/mlx5/qp.c qp->flags_en |= MLX5_QP_FLAG_TUNNEL_OFFLOADS; qp 2078 drivers/infiniband/hw/mlx5/qp.c qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC; qp 2086 drivers/infiniband/hw/mlx5/qp.c qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC; qp 2095 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_PACKET_BASED_CREDIT; qp 2107 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_UNDERLAY; qp 2108 drivers/infiniband/hw/mlx5/qp.c qp->underlay_qpn = init_attr->source_qpn; qp 2111 drivers/infiniband/hw/mlx5/qp.c qp->wq_sig = !!wq_signature; qp 2115 drivers/infiniband/hw/mlx5/qp.c qp->flags & MLX5_IB_QP_UNDERLAY) ? qp 2116 drivers/infiniband/hw/mlx5/qp.c &qp->raw_packet_qp.rq.base : qp 2117 drivers/infiniband/hw/mlx5/qp.c &qp->trans_qp.base; qp 2119 drivers/infiniband/hw/mlx5/qp.c qp->has_rq = qp_has_rq(init_attr); qp 2120 drivers/infiniband/hw/mlx5/qp.c err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp 2121 drivers/infiniband/hw/mlx5/qp.c qp, udata ? &ucmd : NULL); qp 2132 drivers/infiniband/hw/mlx5/qp.c if (ucmd.rq_wqe_shift != qp->rq.wqe_shift || qp 2133 drivers/infiniband/hw/mlx5/qp.c ucmd.rq_wqe_count != qp->rq.wqe_cnt) { qp 2147 drivers/infiniband/hw/mlx5/qp.c err = create_user_qp(dev, pd, qp, udata, init_attr, &in, qp 2152 drivers/infiniband/hw/mlx5/qp.c err = create_kernel_qp(dev, init_attr, qp, &in, &inlen, qp 2165 drivers/infiniband/hw/mlx5/qp.c qp->create_type = MLX5_QP_EMPTY; qp 2169 drivers/infiniband/hw/mlx5/qp.c qp->port = init_attr->port_num; qp 2182 drivers/infiniband/hw/mlx5/qp.c if (qp->wq_sig) qp 2185 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK) qp 2188 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) qp 2190 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_MANAGED_SEND) qp 2192 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_MANAGED_RECV) qp 2194 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_PACKET_BASED_CREDIT) qp 2196 drivers/infiniband/hw/mlx5/qp.c if (qp->scat_cqe && is_connected(init_attr->qp_type)) { qp 2203 drivers/infiniband/hw/mlx5/qp.c if (qp->rq.wqe_cnt) { qp 2204 drivers/infiniband/hw/mlx5/qp.c MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4); qp 2205 drivers/infiniband/hw/mlx5/qp.c MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt)); qp 2208 drivers/infiniband/hw/mlx5/qp.c MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr)); qp 2210 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt) { qp 2211 drivers/infiniband/hw/mlx5/qp.c MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt)); qp 2249 drivers/infiniband/hw/mlx5/qp.c MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma); qp 2259 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_LSO; qp 2271 drivers/infiniband/hw/mlx5/qp.c qp->flags |= MLX5_IB_QP_PCI_WRITE_END_PADDING; qp 2281 drivers/infiniband/hw/mlx5/qp.c qp->flags & MLX5_IB_QP_UNDERLAY) { qp 2282 drivers/infiniband/hw/mlx5/qp.c qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr; qp 2283 drivers/infiniband/hw/mlx5/qp.c raw_packet_qp_copy_info(qp, &qp->raw_packet_qp); qp 2284 drivers/infiniband/hw/mlx5/qp.c err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata, qp 2297 drivers/infiniband/hw/mlx5/qp.c base->container_mibqp = qp; qp 2307 drivers/infiniband/hw/mlx5/qp.c list_add_tail(&qp->qps_list, &dev->qp_list); qp 2311 drivers/infiniband/hw/mlx5/qp.c list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); qp 2313 drivers/infiniband/hw/mlx5/qp.c list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); qp 2320 drivers/infiniband/hw/mlx5/qp.c if (qp->create_type == MLX5_QP_USER) qp 2321 drivers/infiniband/hw/mlx5/qp.c destroy_qp_user(dev, pd, qp, base, udata); qp 2322 drivers/infiniband/hw/mlx5/qp.c else if (qp->create_type == MLX5_QP_KERNEL) qp 2323 drivers/infiniband/hw/mlx5/qp.c destroy_qp_kernel(dev, qp); qp 2388 drivers/infiniband/hw/mlx5/qp.c static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp) qp 2390 drivers/infiniband/hw/mlx5/qp.c return to_mpd(qp->ibqp.pd); qp 2428 drivers/infiniband/hw/mlx5/qp.c static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, qp 2432 drivers/infiniband/hw/mlx5/qp.c static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, qp 2440 drivers/infiniband/hw/mlx5/qp.c if (qp->ibqp.rwq_ind_tbl) { qp 2441 drivers/infiniband/hw/mlx5/qp.c destroy_rss_raw_qp_tir(dev, qp); qp 2445 drivers/infiniband/hw/mlx5/qp.c base = (qp->ibqp.qp_type == IB_QPT_RAW_PACKET || qp 2446 drivers/infiniband/hw/mlx5/qp.c qp->flags & MLX5_IB_QP_UNDERLAY) ? qp 2447 drivers/infiniband/hw/mlx5/qp.c &qp->raw_packet_qp.rq.base : qp 2448 drivers/infiniband/hw/mlx5/qp.c &qp->trans_qp.base; qp 2450 drivers/infiniband/hw/mlx5/qp.c if (qp->state != IB_QPS_RESET) { qp 2451 drivers/infiniband/hw/mlx5/qp.c if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET && qp 2452 drivers/infiniband/hw/mlx5/qp.c !(qp->flags & MLX5_IB_QP_UNDERLAY)) { qp 2461 drivers/infiniband/hw/mlx5/qp.c err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0); qp 2468 drivers/infiniband/hw/mlx5/qp.c get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq, qp 2474 drivers/infiniband/hw/mlx5/qp.c list_del(&qp->qps_list); qp 2476 drivers/infiniband/hw/mlx5/qp.c list_del(&qp->cq_send_list); qp 2479 drivers/infiniband/hw/mlx5/qp.c list_del(&qp->cq_recv_list); qp 2481 drivers/infiniband/hw/mlx5/qp.c if (qp->create_type == MLX5_QP_KERNEL) { qp 2483 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); qp 2491 drivers/infiniband/hw/mlx5/qp.c if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET || qp 2492 drivers/infiniband/hw/mlx5/qp.c qp->flags & MLX5_IB_QP_UNDERLAY) { qp 2493 drivers/infiniband/hw/mlx5/qp.c destroy_raw_packet_qp(dev, qp); qp 2501 drivers/infiniband/hw/mlx5/qp.c if (qp->create_type == MLX5_QP_KERNEL) qp 2502 drivers/infiniband/hw/mlx5/qp.c destroy_qp_kernel(dev, qp); qp 2503 drivers/infiniband/hw/mlx5/qp.c else if (qp->create_type == MLX5_QP_USER) qp 2504 drivers/infiniband/hw/mlx5/qp.c destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base, udata); qp 2547 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp; qp 2559 drivers/infiniband/hw/mlx5/qp.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 2560 drivers/infiniband/hw/mlx5/qp.c if (!qp) qp 2563 drivers/infiniband/hw/mlx5/qp.c qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL); qp 2564 drivers/infiniband/hw/mlx5/qp.c if (!qp->dct.in) { qp 2569 drivers/infiniband/hw/mlx5/qp.c MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid); qp 2570 drivers/infiniband/hw/mlx5/qp.c dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry); qp 2571 drivers/infiniband/hw/mlx5/qp.c qp->qp_sub_type = MLX5_IB_QPT_DCT; qp 2581 drivers/infiniband/hw/mlx5/qp.c qp->state = IB_QPS_RESET; qp 2583 drivers/infiniband/hw/mlx5/qp.c return &qp->ibqp; qp 2585 drivers/infiniband/hw/mlx5/qp.c kfree(qp); qp 2632 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp; qp 2705 drivers/infiniband/hw/mlx5/qp.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 2706 drivers/infiniband/hw/mlx5/qp.c if (!qp) qp 2709 drivers/infiniband/hw/mlx5/qp.c err = create_qp_common(dev, pd, init_attr, udata, qp); qp 2712 drivers/infiniband/hw/mlx5/qp.c kfree(qp); qp 2717 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.qp_num = 0; qp 2719 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.qp_num = 1; qp 2721 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn; qp 2724 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn, qp 2728 drivers/infiniband/hw/mlx5/qp.c qp->trans_qp.xrcdn = xrcdn; qp 2746 drivers/infiniband/hw/mlx5/qp.c qp->qp_sub_type = init_attr->qp_type; qp 2748 drivers/infiniband/hw/mlx5/qp.c return &qp->ibqp; qp 2770 drivers/infiniband/hw/mlx5/qp.c int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) qp 2772 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 2773 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *mqp = to_mqp(qp); qp 2775 drivers/infiniband/hw/mlx5/qp.c if (unlikely(qp->qp_type == IB_QPT_GSI)) qp 2776 drivers/infiniband/hw/mlx5/qp.c return mlx5_ib_gsi_destroy_qp(qp); qp 2788 drivers/infiniband/hw/mlx5/qp.c static int to_mlx5_access_flags(struct mlx5_ib_qp *qp, qp 2795 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); qp 2800 drivers/infiniband/hw/mlx5/qp.c dest_rd_atomic = qp->trans_qp.resp_depth; qp 2805 drivers/infiniband/hw/mlx5/qp.c access_flags = qp->trans_qp.atomic_rd_en; qp 2815 drivers/infiniband/hw/mlx5/qp.c atomic_mode = get_atomic_mode(dev, qp->ibqp.qp_type); qp 2907 drivers/infiniband/hw/mlx5/qp.c static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, qp 2938 drivers/infiniband/hw/mlx5/qp.c if (qp->ibqp.qp_type == IB_QPT_RC || qp 2939 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.qp_type == IB_QPT_UC || qp 2940 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.qp_type == IB_QPT_XRC_INI || qp 2941 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.qp_type == IB_QPT_XRC_TGT) qp 2977 drivers/infiniband/hw/mlx5/qp.c if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt) qp 2979 drivers/infiniband/hw/mlx5/qp.c &qp->raw_packet_qp.sq, qp 2980 drivers/infiniband/hw/mlx5/qp.c sl & 0xf, qp->ibqp.pd); qp 3267 drivers/infiniband/hw/mlx5/qp.c static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, qp 3271 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; qp 3274 drivers/infiniband/hw/mlx5/qp.c int modify_rq = !!qp->rq.wqe_cnt; qp 3275 drivers/infiniband/hw/mlx5/qp.c int modify_sq = !!qp->sq.wqe_cnt; qp 3316 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.pd); qp 3327 drivers/infiniband/hw/mlx5/qp.c qp->ibqp.pd); qp 3338 drivers/infiniband/hw/mlx5/qp.c raw_qp_param, qp->ibqp.pd); qp 3385 drivers/infiniband/hw/mlx5/qp.c static int __mlx5_ib_qp_set_counter(struct ib_qp *qp, qp 3388 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 3389 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *mqp = to_mqp(qp); qp 3453 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp = to_mqp(ibqp); qp 3454 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp_base *base = &qp->trans_qp.base; qp 3467 drivers/infiniband/hw/mlx5/qp.c qp->qp_sub_type : ibqp->qp_type); qp 3475 drivers/infiniband/hw/mlx5/qp.c pd = get_pd(qp); qp 3497 drivers/infiniband/hw/mlx5/qp.c !(qp->flags & MLX5_IB_QP_SQPN_QP1)) || qp 3514 drivers/infiniband/hw/mlx5/qp.c !(qp->flags & MLX5_IB_QP_UNDERLAY)) || qp 3537 drivers/infiniband/hw/mlx5/qp.c context->pri_path.port = qp->port; qp 3543 drivers/infiniband/hw/mlx5/qp.c err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path, qp 3544 drivers/infiniband/hw/mlx5/qp.c attr_mask & IB_QP_PORT ? attr->port_num : qp->port, qp 3554 drivers/infiniband/hw/mlx5/qp.c err = mlx5_set_path(dev, qp, &attr->alt_ah_attr, qp 3563 drivers/infiniband/hw/mlx5/qp.c get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq, qp 3595 drivers/infiniband/hw/mlx5/qp.c err = to_mlx5_access_flags(qp, attr, attr_mask, &access_flags); qp 3611 drivers/infiniband/hw/mlx5/qp.c if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) qp 3612 drivers/infiniband/hw/mlx5/qp.c context->db_rec_addr = cpu_to_be64(qp->db.dma); qp 3616 drivers/infiniband/hw/mlx5/qp.c qp->port) - 1; qp 3619 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_UNDERLAY) qp 3633 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_SQPN_QP1) qp 3649 drivers/infiniband/hw/mlx5/qp.c if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET || qp 3650 drivers/infiniband/hw/mlx5/qp.c qp->flags & MLX5_IB_QP_UNDERLAY) { qp 3690 drivers/infiniband/hw/mlx5/qp.c err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity); qp 3699 drivers/infiniband/hw/mlx5/qp.c qp->state = new_state; qp 3702 drivers/infiniband/hw/mlx5/qp.c qp->trans_qp.atomic_rd_en = attr->qp_access_flags; qp 3704 drivers/infiniband/hw/mlx5/qp.c qp->trans_qp.resp_depth = attr->max_dest_rd_atomic; qp 3706 drivers/infiniband/hw/mlx5/qp.c qp->port = attr->port_num; qp 3708 drivers/infiniband/hw/mlx5/qp.c qp->trans_qp.alt_port = attr->alt_port_num; qp 3721 drivers/infiniband/hw/mlx5/qp.c qp->rq.head = 0; qp 3722 drivers/infiniband/hw/mlx5/qp.c qp->rq.tail = 0; qp 3723 drivers/infiniband/hw/mlx5/qp.c qp->sq.head = 0; qp 3724 drivers/infiniband/hw/mlx5/qp.c qp->sq.tail = 0; qp 3725 drivers/infiniband/hw/mlx5/qp.c qp->sq.cur_post = 0; qp 3726 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt) qp 3727 drivers/infiniband/hw/mlx5/qp.c qp->sq.cur_edge = get_sq_edge(&qp->sq, 0); qp 3728 drivers/infiniband/hw/mlx5/qp.c qp->sq.last_poll = 0; qp 3729 drivers/infiniband/hw/mlx5/qp.c qp->db.db[MLX5_RCV_DBR] = 0; qp 3730 drivers/infiniband/hw/mlx5/qp.c qp->db.db[MLX5_SND_DBR] = 0; qp 3733 drivers/infiniband/hw/mlx5/qp.c if ((new_state == IB_QPS_RTS) && qp->counter_pending) { qp 3736 drivers/infiniband/hw/mlx5/qp.c qp->counter_pending = 0; qp 3800 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp = to_mqp(ibqp); qp 3810 drivers/infiniband/hw/mlx5/qp.c cur_state = qp->state; qp 3813 drivers/infiniband/hw/mlx5/qp.c dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry); qp 3867 drivers/infiniband/hw/mlx5/qp.c err = mlx5_core_create_dct(dev->mdev, &qp->dct.mdct, qp->dct.in, qp 3872 drivers/infiniband/hw/mlx5/qp.c resp.dctn = qp->dct.mdct.mqp.qpn; qp 3875 drivers/infiniband/hw/mlx5/qp.c mlx5_core_destroy_dct(dev->mdev, &qp->dct.mdct); qp 3883 drivers/infiniband/hw/mlx5/qp.c qp->state = IB_QPS_ERR; qp 3885 drivers/infiniband/hw/mlx5/qp.c qp->state = new_state; qp 3893 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp = to_mqp(ibqp); qp 3930 drivers/infiniband/hw/mlx5/qp.c qp_type = qp->qp_sub_type; qp 3938 drivers/infiniband/hw/mlx5/qp.c mutex_lock(&qp->mutex); qp 3940 drivers/infiniband/hw/mlx5/qp.c cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; qp 3944 drivers/infiniband/hw/mlx5/qp.c port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; qp 3947 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_UNDERLAY) { qp 3976 drivers/infiniband/hw/mlx5/qp.c port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; qp 4010 drivers/infiniband/hw/mlx5/qp.c mutex_unlock(&qp->mutex); qp 4095 drivers/infiniband/hw/mlx5/qp.c static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp, qp 4129 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4132 drivers/infiniband/hw/mlx5/qp.c memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata, qp 4435 drivers/infiniband/hw/mlx5/qp.c static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, qp 4453 drivers/infiniband/hw/mlx5/qp.c if (unlikely(inl > qp->max_inline_data)) qp 4460 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, wqe, qp 4592 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, void **seg, int *size, qp 4687 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4696 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4731 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, void **seg, int *size, qp 4738 drivers/infiniband/hw/mlx5/qp.c u32 pdn = get_pd(qp)->pdn; qp 4744 drivers/infiniband/hw/mlx5/qp.c unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) || qp 4764 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4770 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4772 drivers/infiniband/hw/mlx5/qp.c ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size, qp 4808 drivers/infiniband/hw/mlx5/qp.c static int set_reg_wr(struct mlx5_ib_qp *qp, qp 4814 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd); qp 4822 drivers/infiniband/hw/mlx5/qp.c mlx5_ib_warn(to_mdev(qp->ibqp.device), qp 4832 drivers/infiniband/hw/mlx5/qp.c mlx5_ib_warn(to_mdev(qp->ibqp.device), qp 4845 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4850 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4853 drivers/infiniband/hw/mlx5/qp.c memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs, qp 4864 drivers/infiniband/hw/mlx5/qp.c static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size, qp 4870 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4874 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, seg, *size, cur_edge); qp 4877 drivers/infiniband/hw/mlx5/qp.c static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16) qp 4885 drivers/infiniband/hw/mlx5/qp.c p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx); qp 4888 drivers/infiniband/hw/mlx5/qp.c idx = (idx + 1) & (qp->sq.wqe_cnt - 1); qp 4896 drivers/infiniband/hw/mlx5/qp.c static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg, qp 4902 drivers/infiniband/hw/mlx5/qp.c if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) qp 4905 drivers/infiniband/hw/mlx5/qp.c *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); qp 4906 drivers/infiniband/hw/mlx5/qp.c *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx); qp 4910 drivers/infiniband/hw/mlx5/qp.c (*ctrl)->fm_ce_se = qp->sq_signal_bits | qp 4916 drivers/infiniband/hw/mlx5/qp.c *cur_edge = qp->sq.cur_edge; qp 4921 drivers/infiniband/hw/mlx5/qp.c static int begin_wqe(struct mlx5_ib_qp *qp, void **seg, qp 4926 drivers/infiniband/hw/mlx5/qp.c return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq, qp 4931 drivers/infiniband/hw/mlx5/qp.c static void finish_wqe(struct mlx5_ib_qp *qp, qp 4939 drivers/infiniband/hw/mlx5/qp.c ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) | qp 4941 drivers/infiniband/hw/mlx5/qp.c ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8)); qp 4943 drivers/infiniband/hw/mlx5/qp.c if (unlikely(qp->wq_sig)) qp 4946 drivers/infiniband/hw/mlx5/qp.c qp->sq.wrid[idx] = wr_id; qp 4947 drivers/infiniband/hw/mlx5/qp.c qp->sq.w_list[idx].opcode = mlx5_opcode; qp 4948 drivers/infiniband/hw/mlx5/qp.c qp->sq.wqe_head[idx] = qp->sq.head + nreq; qp 4949 drivers/infiniband/hw/mlx5/qp.c qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB); qp 4950 drivers/infiniband/hw/mlx5/qp.c qp->sq.w_list[idx].next = qp->sq.cur_post; qp 4956 drivers/infiniband/hw/mlx5/qp.c qp->sq.cur_edge = (unlikely(seg == cur_edge)) ? qp 4957 drivers/infiniband/hw/mlx5/qp.c get_sq_edge(&qp->sq, qp->sq.cur_post & qp 4958 drivers/infiniband/hw/mlx5/qp.c (qp->sq.wqe_cnt - 1)) : qp 4969 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp; qp 4997 drivers/infiniband/hw/mlx5/qp.c qp = to_mqp(ibqp); qp 4998 drivers/infiniband/hw/mlx5/qp.c bf = &qp->bf; qp 5000 drivers/infiniband/hw/mlx5/qp.c spin_lock_irqsave(&qp->sq.lock, flags); qp 5011 drivers/infiniband/hw/mlx5/qp.c if (unlikely(num_sge > qp->sq.max_gs)) { qp 5018 drivers/infiniband/hw/mlx5/qp.c err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge, qp 5033 drivers/infiniband/hw/mlx5/qp.c if (qp->next_fence) qp 5038 drivers/infiniband/hw/mlx5/qp.c fence = qp->next_fence; qp 5068 drivers/infiniband/hw/mlx5/qp.c qp->sq.wr_data[idx] = IB_WR_LOCAL_INV; qp 5070 drivers/infiniband/hw/mlx5/qp.c set_linv_wr(qp, &seg, &size, &cur_edge); qp 5075 drivers/infiniband/hw/mlx5/qp.c qp->sq.wr_data[idx] = IB_WR_REG_MR; qp 5077 drivers/infiniband/hw/mlx5/qp.c err = set_reg_wr(qp, reg_wr(wr), &seg, &size, qp 5087 drivers/infiniband/hw/mlx5/qp.c qp->sq.wr_data[idx] = IB_WR_REG_MR_INTEGRITY; qp 5102 drivers/infiniband/hw/mlx5/qp.c err = set_reg_wr(qp, ®_pi_wr, &seg, qp 5109 drivers/infiniband/hw/mlx5/qp.c finish_wqe(qp, ctrl, seg, size, qp 5114 drivers/infiniband/hw/mlx5/qp.c err = begin_wqe(qp, &seg, &ctrl, wr, qp 5146 drivers/infiniband/hw/mlx5/qp.c err = set_pi_umr_wr(wr, qp, &seg, &size, qp 5153 drivers/infiniband/hw/mlx5/qp.c finish_wqe(qp, ctrl, seg, size, cur_edge, idx, qp 5162 drivers/infiniband/hw/mlx5/qp.c err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, qp 5179 drivers/infiniband/hw/mlx5/qp.c finish_wqe(qp, ctrl, seg, size, cur_edge, idx, qp 5183 drivers/infiniband/hw/mlx5/qp.c err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, qp 5200 drivers/infiniband/hw/mlx5/qp.c finish_wqe(qp, ctrl, seg, size, cur_edge, idx, qp 5204 drivers/infiniband/hw/mlx5/qp.c qp->next_fence = qp 5230 drivers/infiniband/hw/mlx5/qp.c if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) { qp 5241 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, &seg, size, &cur_edge); qp 5248 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, &seg, size, &cur_edge); qp 5251 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) { qp 5258 drivers/infiniband/hw/mlx5/qp.c set_eth_seg(wr, qp, &seg, &size, &cur_edge); qp 5259 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, &seg, size, qp 5269 drivers/infiniband/hw/mlx5/qp.c qp->sq.wr_data[idx] = MLX5_IB_WR_UMR; qp 5276 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, &seg, size, &cur_edge); qp 5280 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, &seg, size, &cur_edge); qp 5288 drivers/infiniband/hw/mlx5/qp.c err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge); qp 5296 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, &seg, size, qp 5308 drivers/infiniband/hw/mlx5/qp.c qp->next_fence = next_fence; qp 5309 drivers/infiniband/hw/mlx5/qp.c finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq, qp 5313 drivers/infiniband/hw/mlx5/qp.c dump_wqe(qp, idx, size); qp 5318 drivers/infiniband/hw/mlx5/qp.c qp->sq.head += nreq; qp 5325 drivers/infiniband/hw/mlx5/qp.c qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post); qp 5339 drivers/infiniband/hw/mlx5/qp.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 5358 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp = to_mqp(ibqp); qp 5378 drivers/infiniband/hw/mlx5/qp.c spin_lock_irqsave(&qp->rq.lock, flags); qp 5380 drivers/infiniband/hw/mlx5/qp.c ind = qp->rq.head & (qp->rq.wqe_cnt - 1); qp 5383 drivers/infiniband/hw/mlx5/qp.c if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { qp 5389 drivers/infiniband/hw/mlx5/qp.c if (unlikely(wr->num_sge > qp->rq.max_gs)) { qp 5395 drivers/infiniband/hw/mlx5/qp.c scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind); qp 5396 drivers/infiniband/hw/mlx5/qp.c if (qp->wq_sig) qp 5402 drivers/infiniband/hw/mlx5/qp.c if (i < qp->rq.max_gs) { qp 5408 drivers/infiniband/hw/mlx5/qp.c if (qp->wq_sig) { qp 5410 drivers/infiniband/hw/mlx5/qp.c set_sig_seg(sig, (qp->rq.max_gs + 1) << 2); qp 5413 drivers/infiniband/hw/mlx5/qp.c qp->rq.wrid[ind] = wr->wr_id; qp 5415 drivers/infiniband/hw/mlx5/qp.c ind = (ind + 1) & (qp->rq.wqe_cnt - 1); qp 5420 drivers/infiniband/hw/mlx5/qp.c qp->rq.head += nreq; qp 5427 drivers/infiniband/hw/mlx5/qp.c *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); qp 5430 drivers/infiniband/hw/mlx5/qp.c spin_unlock_irqrestore(&qp->rq.lock, flags); qp 5556 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, u8 *qp_state) qp 5589 drivers/infiniband/hw/mlx5/qp.c qp->raw_packet_qp.sq.base.mqp.qpn, sq_state, qp 5590 drivers/infiniband/hw/mlx5/qp.c qp->raw_packet_qp.rq.base.mqp.qpn, rq_state); qp 5595 drivers/infiniband/hw/mlx5/qp.c *qp_state = qp->state; qp 5601 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp, qp 5604 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp; qp 5611 drivers/infiniband/hw/mlx5/qp.c if (qp->sq.wqe_cnt) { qp 5617 drivers/infiniband/hw/mlx5/qp.c if (qp->rq.wqe_cnt) { qp 5623 drivers/infiniband/hw/mlx5/qp.c return sqrq_state_to_qp_state(sq_state, rq_state, qp, qp 5627 drivers/infiniband/hw/mlx5/qp.c static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, qp 5640 drivers/infiniband/hw/mlx5/qp.c err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb, qp 5650 drivers/infiniband/hw/mlx5/qp.c qp->state = to_ib_qp_state(mlx5_state); qp 5661 drivers/infiniband/hw/mlx5/qp.c if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { qp 5761 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *qp = to_mqp(ibqp); qp 5776 drivers/infiniband/hw/mlx5/qp.c if (unlikely(qp->qp_sub_type == MLX5_IB_QPT_DCT)) qp 5777 drivers/infiniband/hw/mlx5/qp.c return mlx5_ib_dct_query_qp(dev, qp, qp_attr, qp 5780 drivers/infiniband/hw/mlx5/qp.c mutex_lock(&qp->mutex); qp 5782 drivers/infiniband/hw/mlx5/qp.c if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET || qp 5783 drivers/infiniband/hw/mlx5/qp.c qp->flags & MLX5_IB_QP_UNDERLAY) { qp 5784 drivers/infiniband/hw/mlx5/qp.c err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state); qp 5787 drivers/infiniband/hw/mlx5/qp.c qp->state = raw_packet_qp_state; qp 5790 drivers/infiniband/hw/mlx5/qp.c err = query_qp_attr(dev, qp, qp_attr); qp 5795 drivers/infiniband/hw/mlx5/qp.c qp_attr->qp_state = qp->state; qp 5797 drivers/infiniband/hw/mlx5/qp.c qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; qp 5798 drivers/infiniband/hw/mlx5/qp.c qp_attr->cap.max_recv_sge = qp->rq.max_gs; qp 5801 drivers/infiniband/hw/mlx5/qp.c qp_attr->cap.max_send_wr = qp->sq.max_post; qp 5802 drivers/infiniband/hw/mlx5/qp.c qp_attr->cap.max_send_sge = qp->sq.max_gs; qp 5813 drivers/infiniband/hw/mlx5/qp.c qp_attr->cap.max_inline_data = qp->max_inline_data; qp 5818 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK) qp 5821 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) qp 5823 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_MANAGED_SEND) qp 5825 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_MANAGED_RECV) qp 5827 drivers/infiniband/hw/mlx5/qp.c if (qp->flags & MLX5_IB_QP_SQPN_QP1) qp 5830 drivers/infiniband/hw/mlx5/qp.c qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ? qp 5834 drivers/infiniband/hw/mlx5/qp.c mutex_unlock(&qp->mutex); qp 6439 drivers/infiniband/hw/mlx5/qp.c void mlx5_ib_drain_sq(struct ib_qp *qp) qp 6441 drivers/infiniband/hw/mlx5/qp.c struct ib_cq *cq = qp->send_cq; qp 6453 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 6456 drivers/infiniband/hw/mlx5/qp.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE); qp 6465 drivers/infiniband/hw/mlx5/qp.c ret = _mlx5_ib_post_send(qp, &swr.wr, &bad_swr, true); qp 6474 drivers/infiniband/hw/mlx5/qp.c void mlx5_ib_drain_rq(struct ib_qp *qp) qp 6476 drivers/infiniband/hw/mlx5/qp.c struct ib_cq *cq = qp->recv_cq; qp 6482 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 6485 drivers/infiniband/hw/mlx5/qp.c ret = ib_modify_qp(qp, &attr, IB_QP_STATE); qp 6495 drivers/infiniband/hw/mlx5/qp.c ret = _mlx5_ib_post_recv(qp, &rwr, &bad_rwr, true); qp 6508 drivers/infiniband/hw/mlx5/qp.c int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter) qp 6510 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_dev *dev = to_mdev(qp->device); qp 6511 drivers/infiniband/hw/mlx5/qp.c struct mlx5_ib_qp *mqp = to_mqp(qp); qp 6516 drivers/infiniband/hw/mlx5/qp.c qp->counter = counter; qp 6526 drivers/infiniband/hw/mlx5/qp.c err = __mlx5_ib_qp_set_counter(qp, counter); qp 6528 drivers/infiniband/hw/mlx5/qp.c qp->counter = counter; qp 6534 drivers/infiniband/hw/mlx5/qp.c qp->counter = counter; qp 1908 drivers/infiniband/hw/mthca/mthca_cmd.c MTHCA_PUT(inbox, in_wc->qp->qp_num, MAD_IFC_MY_QPN_OFFSET); qp 373 drivers/infiniband/hw/mthca/mthca_cq.c struct mthca_qp *qp, int wqe_index, int is_send, qp 461 drivers/infiniband/hw/mthca/mthca_cq.c mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe); qp 520 drivers/infiniband/hw/mthca/mthca_cq.c *cur_qp = mthca_array_get(&dev->qp_table.qp, qp 531 drivers/infiniband/hw/mthca/mthca_cq.c entry->qp = &(*cur_qp)->ibqp; qp 659 drivers/infiniband/hw/mthca/mthca_cq.c struct mthca_qp *qp = NULL; qp 670 drivers/infiniband/hw/mthca/mthca_cq.c err = mthca_poll_one(dev, cq, &qp, qp 259 drivers/infiniband/hw/mthca/mthca_dev.h struct mthca_array qp; qp 542 drivers/infiniband/hw/mthca/mthca_dev.h void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send, qp 551 drivers/infiniband/hw/mthca/mthca_dev.h struct mthca_qp *qp, qp 563 drivers/infiniband/hw/mthca/mthca_dev.h void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp); qp 144 drivers/infiniband/hw/mthca/mthca_eq.c } __packed qp; qp 282 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 287 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 292 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 297 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 307 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 312 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 317 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 322 drivers/infiniband/hw/mthca/mthca_eq.c mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, qp 91 drivers/infiniband/hw/mthca/mthca_mad.c new_ah = rdma_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, qp 43 drivers/infiniband/hw/mthca/mthca_mcg.c __be32 qp[MTHCA_QP_PER_MGM]; qp 165 drivers/infiniband/hw/mthca/mthca_mcg.c if (mgm->qp[i] == cpu_to_be32(ibqp->qp_num | (1 << 31))) { qp 170 drivers/infiniband/hw/mthca/mthca_mcg.c } else if (!(mgm->qp[i] & cpu_to_be32(1 << 31))) { qp 171 drivers/infiniband/hw/mthca/mthca_mcg.c mgm->qp[i] = cpu_to_be32(ibqp->qp_num | (1 << 31)); qp 242 drivers/infiniband/hw/mthca/mthca_mcg.c if (mgm->qp[i] == cpu_to_be32(ibqp->qp_num | (1 << 31))) qp 244 drivers/infiniband/hw/mthca/mthca_mcg.c if (!(mgm->qp[i] & cpu_to_be32(1 << 31))) qp 254 drivers/infiniband/hw/mthca/mthca_mcg.c mgm->qp[loc] = mgm->qp[i - 1]; qp 255 drivers/infiniband/hw/mthca/mthca_mcg.c mgm->qp[i - 1] = 0; qp 475 drivers/infiniband/hw/mthca/mthca_provider.c struct mthca_qp *qp; qp 486 drivers/infiniband/hw/mthca/mthca_provider.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 487 drivers/infiniband/hw/mthca/mthca_provider.c if (!qp) qp 492 drivers/infiniband/hw/mthca/mthca_provider.c kfree(qp); qp 500 drivers/infiniband/hw/mthca/mthca_provider.c kfree(qp); qp 512 drivers/infiniband/hw/mthca/mthca_provider.c kfree(qp); qp 516 drivers/infiniband/hw/mthca/mthca_provider.c qp->mr.ibmr.lkey = ucmd.lkey; qp 517 drivers/infiniband/hw/mthca/mthca_provider.c qp->sq.db_index = ucmd.sq_db_index; qp 518 drivers/infiniband/hw/mthca/mthca_provider.c qp->rq.db_index = ucmd.rq_db_index; qp 525 drivers/infiniband/hw/mthca/mthca_provider.c &init_attr->cap, qp, udata); qp 538 drivers/infiniband/hw/mthca/mthca_provider.c qp->ibqp.qp_num = qp->qpn; qp 548 drivers/infiniband/hw/mthca/mthca_provider.c qp = kzalloc(sizeof(struct mthca_sqp), GFP_KERNEL); qp 549 drivers/infiniband/hw/mthca/mthca_provider.c if (!qp) qp 552 drivers/infiniband/hw/mthca/mthca_provider.c qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; qp 558 drivers/infiniband/hw/mthca/mthca_provider.c qp->ibqp.qp_num, init_attr->port_num, qp 559 drivers/infiniband/hw/mthca/mthca_provider.c to_msqp(qp), udata); qp 568 drivers/infiniband/hw/mthca/mthca_provider.c kfree(qp); qp 572 drivers/infiniband/hw/mthca/mthca_provider.c init_attr->cap.max_send_wr = qp->sq.max; qp 573 drivers/infiniband/hw/mthca/mthca_provider.c init_attr->cap.max_recv_wr = qp->rq.max; qp 574 drivers/infiniband/hw/mthca/mthca_provider.c init_attr->cap.max_send_sge = qp->sq.max_gs; qp 575 drivers/infiniband/hw/mthca/mthca_provider.c init_attr->cap.max_recv_sge = qp->rq.max_gs; qp 576 drivers/infiniband/hw/mthca/mthca_provider.c init_attr->cap.max_inline_data = qp->max_inline_data; qp 578 drivers/infiniband/hw/mthca/mthca_provider.c return &qp->ibqp; qp 581 drivers/infiniband/hw/mthca/mthca_provider.c static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) qp 590 drivers/infiniband/hw/mthca/mthca_provider.c mthca_unmap_user_db(to_mdev(qp->device), qp 593 drivers/infiniband/hw/mthca/mthca_provider.c to_mqp(qp)->sq.db_index); qp 594 drivers/infiniband/hw/mthca/mthca_provider.c mthca_unmap_user_db(to_mdev(qp->device), qp 597 drivers/infiniband/hw/mthca/mthca_provider.c to_mqp(qp)->rq.db_index); qp 599 drivers/infiniband/hw/mthca/mthca_provider.c mthca_free_qp(to_mdev(qp->device), to_mqp(qp)); qp 600 drivers/infiniband/hw/mthca/mthca_provider.c kfree(qp); qp 289 drivers/infiniband/hw/mthca/mthca_provider.h struct mthca_qp qp; qp 339 drivers/infiniband/hw/mthca/mthca_provider.h static inline struct mthca_sqp *to_msqp(struct mthca_qp *qp) qp 341 drivers/infiniband/hw/mthca/mthca_provider.h return container_of(qp, struct mthca_sqp, qp); qp 196 drivers/infiniband/hw/mthca/mthca_qp.c static int is_sqp(struct mthca_dev *dev, struct mthca_qp *qp) qp 198 drivers/infiniband/hw/mthca/mthca_qp.c return qp->qpn >= dev->qp_table.sqp_start && qp 199 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn <= dev->qp_table.sqp_start + 3; qp 202 drivers/infiniband/hw/mthca/mthca_qp.c static int is_qp0(struct mthca_dev *dev, struct mthca_qp *qp) qp 204 drivers/infiniband/hw/mthca/mthca_qp.c return qp->qpn >= dev->qp_table.sqp_start && qp 205 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn <= dev->qp_table.sqp_start + 1; qp 208 drivers/infiniband/hw/mthca/mthca_qp.c static void *get_recv_wqe(struct mthca_qp *qp, int n) qp 210 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->is_direct) qp 211 drivers/infiniband/hw/mthca/mthca_qp.c return qp->queue.direct.buf + (n << qp->rq.wqe_shift); qp 213 drivers/infiniband/hw/mthca/mthca_qp.c return qp->queue.page_list[(n << qp->rq.wqe_shift) >> PAGE_SHIFT].buf + qp 214 drivers/infiniband/hw/mthca/mthca_qp.c ((n << qp->rq.wqe_shift) & (PAGE_SIZE - 1)); qp 217 drivers/infiniband/hw/mthca/mthca_qp.c static void *get_send_wqe(struct mthca_qp *qp, int n) qp 219 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->is_direct) qp 220 drivers/infiniband/hw/mthca/mthca_qp.c return qp->queue.direct.buf + qp->send_wqe_offset + qp 221 drivers/infiniband/hw/mthca/mthca_qp.c (n << qp->sq.wqe_shift); qp 223 drivers/infiniband/hw/mthca/mthca_qp.c return qp->queue.page_list[(qp->send_wqe_offset + qp 224 drivers/infiniband/hw/mthca/mthca_qp.c (n << qp->sq.wqe_shift)) >> qp 226 drivers/infiniband/hw/mthca/mthca_qp.c ((qp->send_wqe_offset + (n << qp->sq.wqe_shift)) & qp 241 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp; qp 245 drivers/infiniband/hw/mthca/mthca_qp.c qp = mthca_array_get(&dev->qp_table.qp, qpn & (dev->limits.num_qps - 1)); qp 246 drivers/infiniband/hw/mthca/mthca_qp.c if (qp) qp 247 drivers/infiniband/hw/mthca/mthca_qp.c ++qp->refcount; qp 250 drivers/infiniband/hw/mthca/mthca_qp.c if (!qp) { qp 257 drivers/infiniband/hw/mthca/mthca_qp.c qp->port = qp->alt_port; qp 261 drivers/infiniband/hw/mthca/mthca_qp.c event.element.qp = &qp->ibqp; qp 262 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->ibqp.event_handler) qp 263 drivers/infiniband/hw/mthca/mthca_qp.c qp->ibqp.event_handler(&event, qp->ibqp.qp_context); qp 266 drivers/infiniband/hw/mthca/mthca_qp.c if (!--qp->refcount) qp 267 drivers/infiniband/hw/mthca/mthca_qp.c wake_up(&qp->wait); qp 328 drivers/infiniband/hw/mthca/mthca_qp.c static __be32 get_hw_access_flags(struct mthca_qp *qp, const struct ib_qp_attr *attr, qp 338 drivers/infiniband/hw/mthca/mthca_qp.c dest_rd_atomic = qp->resp_depth; qp 343 drivers/infiniband/hw/mthca/mthca_qp.c access_flags = qp->atomic_rd_en; qp 434 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp = to_mqp(ibqp); qp 441 drivers/infiniband/hw/mthca/mthca_qp.c mutex_lock(&qp->mutex); qp 443 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->state == IB_QPS_RESET) { qp 454 drivers/infiniband/hw/mthca/mthca_qp.c err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox); qp 464 drivers/infiniband/hw/mthca/mthca_qp.c qp->state = to_ib_qp_state(mthca_state); qp 465 drivers/infiniband/hw/mthca/mthca_qp.c qp_attr->qp_state = qp->state; qp 476 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->transport == RC || qp->transport == UC) { qp 505 drivers/infiniband/hw/mthca/mthca_qp.c qp_attr->cap.max_send_wr = qp->sq.max; qp 506 drivers/infiniband/hw/mthca/mthca_qp.c qp_attr->cap.max_recv_wr = qp->rq.max; qp 507 drivers/infiniband/hw/mthca/mthca_qp.c qp_attr->cap.max_send_sge = qp->sq.max_gs; qp 508 drivers/infiniband/hw/mthca/mthca_qp.c qp_attr->cap.max_recv_sge = qp->rq.max_gs; qp 509 drivers/infiniband/hw/mthca/mthca_qp.c qp_attr->cap.max_inline_data = qp->max_inline_data; qp 512 drivers/infiniband/hw/mthca/mthca_qp.c qp_init_attr->sq_sig_type = qp->sq_policy; qp 518 drivers/infiniband/hw/mthca/mthca_qp.c mutex_unlock(&qp->mutex); qp 563 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp = to_mqp(ibqp); qp 582 drivers/infiniband/hw/mthca/mthca_qp.c (to_mthca_st(qp->transport) << 16)); qp 603 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->transport == MLX || qp->transport == UD) qp 615 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->rq.max) qp 616 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->rq_size_stride = ilog2(qp->rq.max) << 3; qp 617 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->rq_size_stride |= qp->rq.wqe_shift - 4; qp 619 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->sq.max) qp 620 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->sq_size_stride = ilog2(qp->sq.max) << 3; qp 621 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->sq_size_stride |= qp->sq.wqe_shift - 4; qp 626 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->ibqp.uobject) qp 630 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->local_qpn = cpu_to_be32(qp->qpn); qp 635 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->transport == MLX) qp 637 drivers/infiniband/hw/mthca/mthca_qp.c cpu_to_be32(qp->port << 24); qp 661 drivers/infiniband/hw/mthca/mthca_qp.c attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) qp 711 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->wqe_lkey = cpu_to_be32(qp->mr.ibmr.lkey); qp 715 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->sq_policy == IB_SIGNAL_ALL_WR) qp 738 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->snd_wqe_base_l = cpu_to_be32(qp->send_wqe_offset); qp 739 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->snd_db_index = cpu_to_be32(qp->sq.db_index); qp 751 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->params2 |= get_hw_access_flags(qp, attr, attr_mask); qp 771 drivers/infiniband/hw/mthca/mthca_qp.c ((qp->qpn & (dev->limits.num_qps - 1)) * MTHCA_RDB_ENTRY_SIZE << qp 777 drivers/infiniband/hw/mthca/mthca_qp.c qp_context->rcv_db_index = cpu_to_be32(qp->rq.db_index); qp 793 drivers/infiniband/hw/mthca/mthca_qp.c err = mthca_MODIFY_QP(dev, cur_state, new_state, qp->qpn, 0, qp 801 drivers/infiniband/hw/mthca/mthca_qp.c qp->state = new_state; qp 803 drivers/infiniband/hw/mthca/mthca_qp.c qp->atomic_rd_en = attr->qp_access_flags; qp 805 drivers/infiniband/hw/mthca/mthca_qp.c qp->resp_depth = attr->max_dest_rd_atomic; qp 807 drivers/infiniband/hw/mthca/mthca_qp.c qp->port = attr->port_num; qp 809 drivers/infiniband/hw/mthca/mthca_qp.c qp->alt_port = attr->alt_port_num; qp 811 drivers/infiniband/hw/mthca/mthca_qp.c if (is_sqp(dev, qp)) qp 812 drivers/infiniband/hw/mthca/mthca_qp.c store_attrs(to_msqp(qp), attr, attr_mask); qp 818 drivers/infiniband/hw/mthca/mthca_qp.c if (is_qp0(dev, qp)) { qp 821 drivers/infiniband/hw/mthca/mthca_qp.c init_port(dev, qp->port); qp 827 drivers/infiniband/hw/mthca/mthca_qp.c mthca_CLOSE_IB(dev, qp->port); qp 834 drivers/infiniband/hw/mthca/mthca_qp.c if (new_state == IB_QPS_RESET && !qp->ibqp.uobject) { qp 835 drivers/infiniband/hw/mthca/mthca_qp.c mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq), qp->qpn, qp 836 drivers/infiniband/hw/mthca/mthca_qp.c qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); qp 837 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->ibqp.send_cq != qp->ibqp.recv_cq) qp 838 drivers/infiniband/hw/mthca/mthca_qp.c mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq), qp->qpn, NULL); qp 840 drivers/infiniband/hw/mthca/mthca_qp.c mthca_wq_reset(&qp->sq); qp 841 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.last = get_send_wqe(qp, qp->sq.max - 1); qp 843 drivers/infiniband/hw/mthca/mthca_qp.c mthca_wq_reset(&qp->rq); qp 844 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1); qp 847 drivers/infiniband/hw/mthca/mthca_qp.c *qp->sq.db = 0; qp 848 drivers/infiniband/hw/mthca/mthca_qp.c *qp->rq.db = 0; qp 862 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp = to_mqp(ibqp); qp 866 drivers/infiniband/hw/mthca/mthca_qp.c mutex_lock(&qp->mutex); qp 870 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock_irq(&qp->sq.lock); qp 871 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock(&qp->rq.lock); qp 872 drivers/infiniband/hw/mthca/mthca_qp.c cur_state = qp->state; qp 873 drivers/infiniband/hw/mthca/mthca_qp.c spin_unlock(&qp->rq.lock); qp 874 drivers/infiniband/hw/mthca/mthca_qp.c spin_unlock_irq(&qp->sq.lock); qp 883 drivers/infiniband/hw/mthca/mthca_qp.c qp->transport, cur_state, new_state, qp 924 drivers/infiniband/hw/mthca/mthca_qp.c mutex_unlock(&qp->mutex); qp 928 drivers/infiniband/hw/mthca/mthca_qp.c static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int desc_sz) qp 936 drivers/infiniband/hw/mthca/mthca_qp.c switch (qp->transport) { qp 964 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp) qp 966 drivers/infiniband/hw/mthca/mthca_qp.c int max_data_size = mthca_max_data_size(dev, qp, qp 968 drivers/infiniband/hw/mthca/mthca_qp.c 1 << qp->sq.wqe_shift)); qp 970 drivers/infiniband/hw/mthca/mthca_qp.c qp->max_inline_data = mthca_max_inline_data(pd, max_data_size); qp 972 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.max_gs = min_t(int, dev->limits.max_sg, qp 974 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max_gs = min_t(int, dev->limits.max_sg, qp 975 drivers/infiniband/hw/mthca/mthca_qp.c (min(dev->limits.max_desc_sz, 1 << qp->rq.wqe_shift) - qp 989 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp, qp 996 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max_gs * sizeof (struct mthca_data_seg); qp 1001 drivers/infiniband/hw/mthca/mthca_qp.c for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size; qp 1002 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.wqe_shift++) qp 1005 drivers/infiniband/hw/mthca/mthca_qp.c size = qp->sq.max_gs * sizeof (struct mthca_data_seg); qp 1006 drivers/infiniband/hw/mthca/mthca_qp.c switch (qp->transport) { qp 1045 drivers/infiniband/hw/mthca/mthca_qp.c for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size; qp 1046 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.wqe_shift++) qp 1049 drivers/infiniband/hw/mthca/mthca_qp.c qp->send_wqe_offset = ALIGN(qp->rq.max << qp->rq.wqe_shift, qp 1050 drivers/infiniband/hw/mthca/mthca_qp.c 1 << qp->sq.wqe_shift); qp 1060 drivers/infiniband/hw/mthca/mthca_qp.c size = PAGE_ALIGN(qp->send_wqe_offset + qp 1061 drivers/infiniband/hw/mthca/mthca_qp.c (qp->sq.max << qp->sq.wqe_shift)); qp 1063 drivers/infiniband/hw/mthca/mthca_qp.c qp->wrid = kmalloc_array(qp->rq.max + qp->sq.max, sizeof(u64), qp 1065 drivers/infiniband/hw/mthca/mthca_qp.c if (!qp->wrid) qp 1069 drivers/infiniband/hw/mthca/mthca_qp.c &qp->queue, &qp->is_direct, pd, 0, &qp->mr); qp 1076 drivers/infiniband/hw/mthca/mthca_qp.c kfree(qp->wrid); qp 1081 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp) qp 1083 drivers/infiniband/hw/mthca/mthca_qp.c mthca_buf_free(dev, PAGE_ALIGN(qp->send_wqe_offset + qp 1084 drivers/infiniband/hw/mthca/mthca_qp.c (qp->sq.max << qp->sq.wqe_shift)), qp 1085 drivers/infiniband/hw/mthca/mthca_qp.c &qp->queue, qp->is_direct, &qp->mr); qp 1086 drivers/infiniband/hw/mthca/mthca_qp.c kfree(qp->wrid); qp 1090 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp) qp 1095 drivers/infiniband/hw/mthca/mthca_qp.c ret = mthca_table_get(dev, dev->qp_table.qp_table, qp->qpn); qp 1099 drivers/infiniband/hw/mthca/mthca_qp.c ret = mthca_table_get(dev, dev->qp_table.eqp_table, qp->qpn); qp 1104 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn << dev->qp_table.rdb_shift); qp 1113 drivers/infiniband/hw/mthca/mthca_qp.c mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn); qp 1116 drivers/infiniband/hw/mthca/mthca_qp.c mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn); qp 1122 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp) qp 1125 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn << dev->qp_table.rdb_shift); qp 1126 drivers/infiniband/hw/mthca/mthca_qp.c mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn); qp 1127 drivers/infiniband/hw/mthca/mthca_qp.c mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn); qp 1131 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp) qp 1134 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ, qp 1135 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn, &qp->rq.db); qp 1136 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->rq.db_index < 0) qp 1139 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ, qp 1140 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn, &qp->sq.db); qp 1141 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->sq.db_index < 0) { qp 1142 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index); qp 1151 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp) qp 1154 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free_db(dev, MTHCA_DB_TYPE_SQ, qp->sq.db_index); qp 1155 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index); qp 1164 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp, qp 1171 drivers/infiniband/hw/mthca/mthca_qp.c qp->refcount = 1; qp 1172 drivers/infiniband/hw/mthca/mthca_qp.c init_waitqueue_head(&qp->wait); qp 1173 drivers/infiniband/hw/mthca/mthca_qp.c mutex_init(&qp->mutex); qp 1174 drivers/infiniband/hw/mthca/mthca_qp.c qp->state = IB_QPS_RESET; qp 1175 drivers/infiniband/hw/mthca/mthca_qp.c qp->atomic_rd_en = 0; qp 1176 drivers/infiniband/hw/mthca/mthca_qp.c qp->resp_depth = 0; qp 1177 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq_policy = send_policy; qp 1178 drivers/infiniband/hw/mthca/mthca_qp.c mthca_wq_reset(&qp->sq); qp 1179 drivers/infiniband/hw/mthca/mthca_qp.c mthca_wq_reset(&qp->rq); qp 1181 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock_init(&qp->sq.lock); qp 1182 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock_init(&qp->rq.lock); qp 1184 drivers/infiniband/hw/mthca/mthca_qp.c ret = mthca_map_memfree(dev, qp); qp 1188 drivers/infiniband/hw/mthca/mthca_qp.c ret = mthca_alloc_wqe_buf(dev, pd, qp, udata); qp 1190 drivers/infiniband/hw/mthca/mthca_qp.c mthca_unmap_memfree(dev, qp); qp 1194 drivers/infiniband/hw/mthca/mthca_qp.c mthca_adjust_qp_caps(dev, pd, qp); qp 1204 drivers/infiniband/hw/mthca/mthca_qp.c ret = mthca_alloc_memfree(dev, qp); qp 1206 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free_wqe_buf(dev, qp); qp 1207 drivers/infiniband/hw/mthca/mthca_qp.c mthca_unmap_memfree(dev, qp); qp 1214 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max_gs * sizeof (struct mthca_data_seg)) / 16; qp 1216 drivers/infiniband/hw/mthca/mthca_qp.c for (i = 0; i < qp->rq.max; ++i) { qp 1217 drivers/infiniband/hw/mthca/mthca_qp.c next = get_recv_wqe(qp, i); qp 1218 drivers/infiniband/hw/mthca/mthca_qp.c next->nda_op = cpu_to_be32(((i + 1) & (qp->rq.max - 1)) << qp 1219 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.wqe_shift); qp 1223 drivers/infiniband/hw/mthca/mthca_qp.c (void *) scatter < (void *) next + (1 << qp->rq.wqe_shift); qp 1228 drivers/infiniband/hw/mthca/mthca_qp.c for (i = 0; i < qp->sq.max; ++i) { qp 1229 drivers/infiniband/hw/mthca/mthca_qp.c next = get_send_wqe(qp, i); qp 1230 drivers/infiniband/hw/mthca/mthca_qp.c next->nda_op = cpu_to_be32((((i + 1) & (qp->sq.max - 1)) << qp 1231 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.wqe_shift) + qp 1232 drivers/infiniband/hw/mthca/mthca_qp.c qp->send_wqe_offset); qp 1235 drivers/infiniband/hw/mthca/mthca_qp.c for (i = 0; i < qp->rq.max; ++i) { qp 1236 drivers/infiniband/hw/mthca/mthca_qp.c next = get_recv_wqe(qp, i); qp 1237 drivers/infiniband/hw/mthca/mthca_qp.c next->nda_op = htonl((((i + 1) % qp->rq.max) << qp 1238 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.wqe_shift) | 1); qp 1243 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.last = get_send_wqe(qp, qp->sq.max - 1); qp 1244 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1); qp 1250 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_pd *pd, struct mthca_qp *qp) qp 1252 drivers/infiniband/hw/mthca/mthca_qp.c int max_data_size = mthca_max_data_size(dev, qp, dev->limits.max_desc_sz); qp 1266 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->transport == MLX && cap->max_send_sge + 2 > dev->limits.max_sg) qp 1270 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max = cap->max_recv_wr ? qp 1272 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.max = cap->max_send_wr ? qp 1275 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max = cap->max_recv_wr; qp 1276 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.max = cap->max_send_wr; qp 1279 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max_gs = cap->max_recv_sge; qp 1280 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.max_gs = max_t(int, cap->max_send_sge, qp 1295 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp, qp 1301 drivers/infiniband/hw/mthca/mthca_qp.c case IB_QPT_RC: qp->transport = RC; break; qp 1302 drivers/infiniband/hw/mthca/mthca_qp.c case IB_QPT_UC: qp->transport = UC; break; qp 1303 drivers/infiniband/hw/mthca/mthca_qp.c case IB_QPT_UD: qp->transport = UD; break; qp 1307 drivers/infiniband/hw/mthca/mthca_qp.c err = mthca_set_qp_size(dev, cap, pd, qp); qp 1311 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn = mthca_alloc(&dev->qp_table.alloc); qp 1312 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->qpn == -1) qp 1316 drivers/infiniband/hw/mthca/mthca_qp.c qp->port = 0; qp 1319 drivers/infiniband/hw/mthca/mthca_qp.c send_policy, qp, udata); qp 1321 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free(&dev->qp_table.alloc, qp->qpn); qp 1326 drivers/infiniband/hw/mthca/mthca_qp.c mthca_array_set(&dev->qp_table.qp, qp 1327 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn & (dev->limits.num_qps - 1), qp); qp 1377 drivers/infiniband/hw/mthca/mthca_qp.c sqp->qp.transport = MLX; qp 1378 drivers/infiniband/hw/mthca/mthca_qp.c err = mthca_set_qp_size(dev, cap, pd, &sqp->qp); qp 1382 drivers/infiniband/hw/mthca/mthca_qp.c sqp->header_buf_size = sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE; qp 1389 drivers/infiniband/hw/mthca/mthca_qp.c if (mthca_array_get(&dev->qp_table.qp, mqpn)) qp 1392 drivers/infiniband/hw/mthca/mthca_qp.c mthca_array_set(&dev->qp_table.qp, mqpn, sqp); qp 1398 drivers/infiniband/hw/mthca/mthca_qp.c sqp->qp.port = port; qp 1399 drivers/infiniband/hw/mthca/mthca_qp.c sqp->qp.qpn = mqpn; qp 1400 drivers/infiniband/hw/mthca/mthca_qp.c sqp->qp.transport = MLX; qp 1403 drivers/infiniband/hw/mthca/mthca_qp.c send_policy, &sqp->qp, udata); qp 1419 drivers/infiniband/hw/mthca/mthca_qp.c mthca_array_clear(&dev->qp_table.qp, mqpn); qp 1431 drivers/infiniband/hw/mthca/mthca_qp.c static inline int get_qp_refcount(struct mthca_dev *dev, struct mthca_qp *qp) qp 1436 drivers/infiniband/hw/mthca/mthca_qp.c c = qp->refcount; qp 1443 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp) qp 1448 drivers/infiniband/hw/mthca/mthca_qp.c send_cq = to_mcq(qp->ibqp.send_cq); qp 1449 drivers/infiniband/hw/mthca/mthca_qp.c recv_cq = to_mcq(qp->ibqp.recv_cq); qp 1458 drivers/infiniband/hw/mthca/mthca_qp.c mthca_array_clear(&dev->qp_table.qp, qp 1459 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn & (dev->limits.num_qps - 1)); qp 1460 drivers/infiniband/hw/mthca/mthca_qp.c --qp->refcount; qp 1465 drivers/infiniband/hw/mthca/mthca_qp.c wait_event(qp->wait, !get_qp_refcount(dev, qp)); qp 1467 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->state != IB_QPS_RESET) qp 1468 drivers/infiniband/hw/mthca/mthca_qp.c mthca_MODIFY_QP(dev, qp->state, IB_QPS_RESET, qp->qpn, 0, qp 1476 drivers/infiniband/hw/mthca/mthca_qp.c if (!qp->ibqp.uobject) { qp 1477 drivers/infiniband/hw/mthca/mthca_qp.c mthca_cq_clean(dev, recv_cq, qp->qpn, qp 1478 drivers/infiniband/hw/mthca/mthca_qp.c qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); qp 1480 drivers/infiniband/hw/mthca/mthca_qp.c mthca_cq_clean(dev, send_cq, qp->qpn, NULL); qp 1482 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free_memfree(dev, qp); qp 1483 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free_wqe_buf(dev, qp); qp 1486 drivers/infiniband/hw/mthca/mthca_qp.c mthca_unmap_memfree(dev, qp); qp 1488 drivers/infiniband/hw/mthca/mthca_qp.c if (is_sqp(dev, qp)) { qp 1489 drivers/infiniband/hw/mthca/mthca_qp.c atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count)); qp 1491 drivers/infiniband/hw/mthca/mthca_qp.c to_msqp(qp)->header_buf_size, qp 1492 drivers/infiniband/hw/mthca/mthca_qp.c to_msqp(qp)->header_buf, qp 1493 drivers/infiniband/hw/mthca/mthca_qp.c to_msqp(qp)->header_dma); qp 1495 drivers/infiniband/hw/mthca/mthca_qp.c mthca_free(&dev->qp_table.alloc, qp->qpn); qp 1516 drivers/infiniband/hw/mthca/mthca_qp.c mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MTHCA_MLX_VL15 : 0) | qp 1537 drivers/infiniband/hw/mthca/mthca_qp.c sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; qp 1541 drivers/infiniband/hw/mthca/mthca_qp.c if (!sqp->qp.ibqp.qp_num) qp 1542 drivers/infiniband/hw/mthca/mthca_qp.c ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port, qp 1545 drivers/infiniband/hw/mthca/mthca_qp.c ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port, qp 1552 drivers/infiniband/hw/mthca/mthca_qp.c sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); qp 1559 drivers/infiniband/hw/mthca/mthca_qp.c data->lkey = cpu_to_be32(to_mpd(sqp->qp.ibqp.pd)->ntmr.ibmr.lkey); qp 1627 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp = to_mqp(ibqp); qp 1647 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock_irqsave(&qp->sq.lock, flags); qp 1651 drivers/infiniband/hw/mthca/mthca_qp.c ind = qp->sq.next_ind; qp 1654 drivers/infiniband/hw/mthca/mthca_qp.c if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { qp 1656 drivers/infiniband/hw/mthca/mthca_qp.c " %d max, %d nreq)\n", qp->qpn, qp 1657 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.head, qp->sq.tail, qp 1658 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.max, nreq); qp 1664 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_send_wqe(qp, ind); qp 1665 drivers/infiniband/hw/mthca/mthca_qp.c prev_wqe = qp->sq.last; qp 1666 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.last = wqe; qp 1683 drivers/infiniband/hw/mthca/mthca_qp.c switch (qp->transport) { qp 1738 drivers/infiniband/hw/mthca/mthca_qp.c err = build_mlx_header(dev, to_msqp(qp), ind, ud_wr(wr), qp 1750 drivers/infiniband/hw/mthca/mthca_qp.c if (wr->num_sge > qp->sq.max_gs) { qp 1764 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->transport == MLX) { qp 1772 drivers/infiniband/hw/mthca/mthca_qp.c qp->wrid[ind + qp->rq.max] = wr->wr_id; qp 1782 drivers/infiniband/hw/mthca/mthca_qp.c cpu_to_be32(((ind << qp->sq.wqe_shift) + qp 1783 drivers/infiniband/hw/mthca/mthca_qp.c qp->send_wqe_offset) | qp 1799 drivers/infiniband/hw/mthca/mthca_qp.c if (unlikely(ind >= qp->sq.max)) qp 1800 drivers/infiniband/hw/mthca/mthca_qp.c ind -= qp->sq.max; qp 1807 drivers/infiniband/hw/mthca/mthca_qp.c mthca_write64(((qp->sq.next_ind << qp->sq.wqe_shift) + qp 1808 drivers/infiniband/hw/mthca/mthca_qp.c qp->send_wqe_offset) | f0 | op0, qp 1809 drivers/infiniband/hw/mthca/mthca_qp.c (qp->qpn << 8) | size0, qp 1814 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.next_ind = ind; qp 1815 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.head += nreq; qp 1817 drivers/infiniband/hw/mthca/mthca_qp.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 1825 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp = to_mqp(ibqp); qp 1843 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock_irqsave(&qp->rq.lock, flags); qp 1847 drivers/infiniband/hw/mthca/mthca_qp.c ind = qp->rq.next_ind; qp 1850 drivers/infiniband/hw/mthca/mthca_qp.c if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { qp 1852 drivers/infiniband/hw/mthca/mthca_qp.c " %d max, %d nreq)\n", qp->qpn, qp 1853 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.head, qp->rq.tail, qp 1854 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max, nreq); qp 1860 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_recv_wqe(qp, ind); qp 1861 drivers/infiniband/hw/mthca/mthca_qp.c prev_wqe = qp->rq.last; qp 1862 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.last = wqe; qp 1871 drivers/infiniband/hw/mthca/mthca_qp.c if (unlikely(wr->num_sge > qp->rq.max_gs)) { qp 1883 drivers/infiniband/hw/mthca/mthca_qp.c qp->wrid[ind] = wr->wr_id; qp 1892 drivers/infiniband/hw/mthca/mthca_qp.c if (unlikely(ind >= qp->rq.max)) qp 1893 drivers/infiniband/hw/mthca/mthca_qp.c ind -= qp->rq.max; qp 1901 drivers/infiniband/hw/mthca/mthca_qp.c mthca_write64((qp->rq.next_ind << qp->rq.wqe_shift) | size0, qp 1902 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn << 8, dev->kar + MTHCA_RECEIVE_DOORBELL, qp 1905 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.next_ind = ind; qp 1906 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB; qp 1914 drivers/infiniband/hw/mthca/mthca_qp.c mthca_write64((qp->rq.next_ind << qp->rq.wqe_shift) | size0, qp 1915 drivers/infiniband/hw/mthca/mthca_qp.c qp->qpn << 8 | nreq, dev->kar + MTHCA_RECEIVE_DOORBELL, qp 1919 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.next_ind = ind; qp 1920 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.head += nreq; qp 1922 drivers/infiniband/hw/mthca/mthca_qp.c spin_unlock_irqrestore(&qp->rq.lock, flags); qp 1930 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp = to_mqp(ibqp); qp 1951 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock_irqsave(&qp->sq.lock, flags); qp 1955 drivers/infiniband/hw/mthca/mthca_qp.c ind = qp->sq.head & (qp->sq.max - 1); qp 1962 drivers/infiniband/hw/mthca/mthca_qp.c ((qp->sq.head & 0xffff) << 8) | f0 | op0; qp 1964 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB; qp 1971 drivers/infiniband/hw/mthca/mthca_qp.c *qp->sq.db = cpu_to_be32(qp->sq.head & 0xffff); qp 1979 drivers/infiniband/hw/mthca/mthca_qp.c mthca_write64(dbhi, (qp->qpn << 8) | size0, qp 1984 drivers/infiniband/hw/mthca/mthca_qp.c if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { qp 1986 drivers/infiniband/hw/mthca/mthca_qp.c " %d max, %d nreq)\n", qp->qpn, qp 1987 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.head, qp->sq.tail, qp 1988 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.max, nreq); qp 1994 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_send_wqe(qp, ind); qp 1995 drivers/infiniband/hw/mthca/mthca_qp.c prev_wqe = qp->sq.last; qp 1996 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.last = wqe; qp 2013 drivers/infiniband/hw/mthca/mthca_qp.c switch (qp->transport) { qp 2068 drivers/infiniband/hw/mthca/mthca_qp.c err = build_mlx_header(dev, to_msqp(qp), ind, ud_wr(wr), qp 2080 drivers/infiniband/hw/mthca/mthca_qp.c if (wr->num_sge > qp->sq.max_gs) { qp 2094 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->transport == MLX) { qp 2102 drivers/infiniband/hw/mthca/mthca_qp.c qp->wrid[ind + qp->rq.max] = wr->wr_id; qp 2112 drivers/infiniband/hw/mthca/mthca_qp.c cpu_to_be32(((ind << qp->sq.wqe_shift) + qp 2113 drivers/infiniband/hw/mthca/mthca_qp.c qp->send_wqe_offset) | qp 2129 drivers/infiniband/hw/mthca/mthca_qp.c if (unlikely(ind >= qp->sq.max)) qp 2130 drivers/infiniband/hw/mthca/mthca_qp.c ind -= qp->sq.max; qp 2135 drivers/infiniband/hw/mthca/mthca_qp.c dbhi = (nreq << 24) | ((qp->sq.head & 0xffff) << 8) | f0 | op0; qp 2137 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.head += nreq; qp 2144 drivers/infiniband/hw/mthca/mthca_qp.c *qp->sq.db = cpu_to_be32(qp->sq.head & 0xffff); qp 2152 drivers/infiniband/hw/mthca/mthca_qp.c mthca_write64(dbhi, (qp->qpn << 8) | size0, dev->kar + MTHCA_SEND_DOORBELL, qp 2156 drivers/infiniband/hw/mthca/mthca_qp.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 2164 drivers/infiniband/hw/mthca/mthca_qp.c struct mthca_qp *qp = to_mqp(ibqp); qp 2172 drivers/infiniband/hw/mthca/mthca_qp.c spin_lock_irqsave(&qp->rq.lock, flags); qp 2176 drivers/infiniband/hw/mthca/mthca_qp.c ind = qp->rq.head & (qp->rq.max - 1); qp 2179 drivers/infiniband/hw/mthca/mthca_qp.c if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { qp 2181 drivers/infiniband/hw/mthca/mthca_qp.c " %d max, %d nreq)\n", qp->qpn, qp 2182 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.head, qp->rq.tail, qp 2183 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.max, nreq); qp 2189 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_recv_wqe(qp, ind); qp 2195 drivers/infiniband/hw/mthca/mthca_qp.c if (unlikely(wr->num_sge > qp->rq.max_gs)) { qp 2206 drivers/infiniband/hw/mthca/mthca_qp.c if (i < qp->rq.max_gs) qp 2209 drivers/infiniband/hw/mthca/mthca_qp.c qp->wrid[ind] = wr->wr_id; qp 2212 drivers/infiniband/hw/mthca/mthca_qp.c if (unlikely(ind >= qp->rq.max)) qp 2213 drivers/infiniband/hw/mthca/mthca_qp.c ind -= qp->rq.max; qp 2217 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.head += nreq; qp 2224 drivers/infiniband/hw/mthca/mthca_qp.c *qp->rq.db = cpu_to_be32(qp->rq.head & 0xffff); qp 2227 drivers/infiniband/hw/mthca/mthca_qp.c spin_unlock_irqrestore(&qp->rq.lock, flags); qp 2231 drivers/infiniband/hw/mthca/mthca_qp.c void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send, qp 2240 drivers/infiniband/hw/mthca/mthca_qp.c if (qp->ibqp.srq && !is_send) { qp 2246 drivers/infiniband/hw/mthca/mthca_qp.c next = get_send_wqe(qp, index); qp 2248 drivers/infiniband/hw/mthca/mthca_qp.c next = get_recv_wqe(qp, index); qp 2278 drivers/infiniband/hw/mthca/mthca_qp.c err = mthca_array_init(&dev->qp_table.qp, qp 2300 drivers/infiniband/hw/mthca/mthca_qp.c mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps); qp 2313 drivers/infiniband/hw/mthca/mthca_qp.c mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps); qp 665 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp) qp 670 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp == NULL) qp 672 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ocrdma_qp_state_change(qp, new_ib_qps, &old_ib_qps); qp 678 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp = NULL; qp 697 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp = dev->qp_tbl[qpid]; qp 698 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp == NULL) { qp 733 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ib_evt.element.qp = &qp->ibqp; qp 735 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ocrdma_process_qpcat_error(dev, qp); qp 738 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ib_evt.element.qp = &qp->ibqp; qp 742 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ib_evt.element.qp = &qp->ibqp; qp 746 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ib_evt.element.qp = &qp->ibqp; qp 756 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ib_evt.element.srq = &qp->srq->ibsrq; qp 762 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ib_evt.element.srq = &qp->srq->ibsrq; qp 768 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ib_evt.element.qp = &qp->ibqp; qp 784 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->ibqp.event_handler) qp 785 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->ibqp.event_handler(&ib_evt, qp->ibqp.qp_context); qp 790 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->srq->ibsrq.event_handler) qp 791 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->srq->ibsrq.event_handler(&ib_evt, qp 792 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->srq->ibsrq. qp 905 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp; qp 912 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp = list_entry(cur, struct ocrdma_qp, sq_entry); qp 914 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp = list_entry(cur, struct ocrdma_qp, rq_entry); qp 916 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->srq) qp 921 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->sq_cq == qp->rq_cq) qp 926 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->sq_cq == cq) qp 927 drivers/infiniband/hw/ocrdma/ocrdma_hw.c bcq = qp->rq_cq; qp 929 drivers/infiniband/hw/ocrdma/ocrdma_hw.c bcq = qp->sq_cq; qp 2082 drivers/infiniband/hw/ocrdma/ocrdma_hw.c bool ocrdma_is_qp_in_sq_flushlist(struct ocrdma_cq *cq, struct ocrdma_qp *qp) qp 2087 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp == tmp) { qp 2095 drivers/infiniband/hw/ocrdma/ocrdma_hw.c bool ocrdma_is_qp_in_rq_flushlist(struct ocrdma_cq *cq, struct ocrdma_qp *qp) qp 2100 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp == tmp) { qp 2108 drivers/infiniband/hw/ocrdma/ocrdma_hw.c void ocrdma_flush_qp(struct ocrdma_qp *qp) qp 2112 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); qp 2115 drivers/infiniband/hw/ocrdma/ocrdma_hw.c found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); qp 2117 drivers/infiniband/hw/ocrdma/ocrdma_hw.c list_add_tail(&qp->sq_entry, &qp->sq_cq->sq_head); qp 2118 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (!qp->srq) { qp 2119 drivers/infiniband/hw/ocrdma/ocrdma_hw.c found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); qp 2121 drivers/infiniband/hw/ocrdma/ocrdma_hw.c list_add_tail(&qp->rq_entry, &qp->rq_cq->rq_head); qp 2126 drivers/infiniband/hw/ocrdma/ocrdma_hw.c static void ocrdma_init_hwq_ptr(struct ocrdma_qp *qp) qp 2128 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.head = 0; qp 2129 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.tail = 0; qp 2130 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.head = 0; qp 2131 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.tail = 0; qp 2134 drivers/infiniband/hw/ocrdma/ocrdma_hw.c int ocrdma_qp_state_change(struct ocrdma_qp *qp, enum ib_qp_state new_ib_state, qp 2142 drivers/infiniband/hw/ocrdma/ocrdma_hw.c spin_lock_irqsave(&qp->q_lock, flags); qp 2145 drivers/infiniband/hw/ocrdma/ocrdma_hw.c *old_ib_state = get_ibqp_state(qp->state); qp 2146 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (new_state == qp->state) { qp 2147 drivers/infiniband/hw/ocrdma/ocrdma_hw.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 2153 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ocrdma_init_hwq_ptr(qp); qp 2154 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ocrdma_del_flush_qp(qp); qp 2156 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ocrdma_flush_qp(qp); qp 2159 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->state = new_state; qp 2161 drivers/infiniband/hw/ocrdma/ocrdma_hw.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 2165 drivers/infiniband/hw/ocrdma/ocrdma_hw.c static u32 ocrdma_set_create_qp_mbx_access_flags(struct ocrdma_qp *qp) qp 2168 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->cap_flags & OCRDMA_QP_INB_RD) qp 2170 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->cap_flags & OCRDMA_QP_INB_WR) qp 2172 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->cap_flags & OCRDMA_QP_MW_BIND) qp 2174 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->cap_flags & OCRDMA_QP_LKEY0) qp 2176 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->cap_flags & OCRDMA_QP_FAST_REG) qp 2183 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp) qp 2188 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_pd *pd = qp->pd; qp 2205 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.max_cnt = max_wqe_allocated; qp 2208 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.va = dma_alloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL); qp 2209 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (!qp->sq.va) qp 2211 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.len = len; qp 2212 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.pa = pa; qp 2213 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.entry_size = dev->attr.wqe_size; qp 2227 drivers/infiniband/hw/ocrdma/ocrdma_hw.c cmd->max_wqe_rqe |= (ilog2(qp->sq.max_cnt) << qp 2238 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp) qp 2243 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_pd *pd = qp->pd; qp 2255 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.max_cnt = max_rqe_allocated; qp 2258 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.va = dma_alloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL); qp 2259 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (!qp->rq.va) qp 2261 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.pa = pa; qp 2262 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.len = len; qp 2263 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.entry_size = dev->attr.rqe_size; qp 2274 drivers/infiniband/hw/ocrdma/ocrdma_hw.c cmd->max_wqe_rqe |= (ilog2(qp->rq.max_cnt) << qp 2285 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp, qp 2289 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->dpp_enabled = true; qp 2300 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp) qp 2302 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_pd *pd = qp->pd; qp 2314 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->ird_q_va = dma_alloc_coherent(&pdev->dev, ird_q_len, &pa, qp 2316 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (!qp->ird_q_va) qp 2321 drivers/infiniband/hw/ocrdma/ocrdma_hw.c rqe = (struct ocrdma_hdr_wqe *)(qp->ird_q_va + qp 2333 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_qp *qp, qp 2338 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->id = rsp->qp_id & OCRDMA_CREATE_QP_RSP_QP_ID_MASK; qp 2339 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.dbid = rsp->sq_rq_id & OCRDMA_CREATE_QP_RSP_RQ_ID_MASK; qp 2340 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.dbid = rsp->sq_rq_id >> OCRDMA_CREATE_QP_RSP_SQ_ID_SHIFT; qp 2341 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->max_ird = rsp->max_ord_ird & OCRDMA_CREATE_QP_RSP_MAX_IRD_MASK; qp 2342 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->max_ord = (rsp->max_ord_ird >> OCRDMA_CREATE_QP_RSP_MAX_ORD_SHIFT); qp 2343 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->dpp_enabled = false; qp 2345 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->dpp_enabled = true; qp 2358 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.max_cnt = max_wqe_allocated; qp 2359 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq.max_wqe_idx = max_wqe_allocated - 1; qp 2362 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.max_cnt = max_rqe_allocated; qp 2363 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq.max_wqe_idx = max_rqe_allocated - 1; qp 2367 drivers/infiniband/hw/ocrdma/ocrdma_hw.c int ocrdma_mbx_create_qp(struct ocrdma_qp *qp, struct ib_qp_init_attr *attrs, qp 2373 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_pd *pd = qp->pd; qp 2400 drivers/infiniband/hw/ocrdma/ocrdma_hw.c status = ocrdma_set_create_qp_sq_cmd(cmd, attrs, qp); qp 2408 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->srq = srq; qp 2410 drivers/infiniband/hw/ocrdma/ocrdma_hw.c status = ocrdma_set_create_qp_rq_cmd(cmd, attrs, qp); qp 2415 drivers/infiniband/hw/ocrdma/ocrdma_hw.c status = ocrdma_set_create_qp_ird_cmd(cmd, qp); qp 2422 drivers/infiniband/hw/ocrdma/ocrdma_hw.c flags = ocrdma_set_create_qp_mbx_access_flags(qp); qp 2434 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sq_cq = cq; qp 2438 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->rq_cq = cq; qp 2442 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ocrdma_set_create_qp_dpp_cmd(cmd, pd, qp, enable_dpp_cq, qp 2450 drivers/infiniband/hw/ocrdma/ocrdma_hw.c ocrdma_get_create_qp_rsp(rsp, qp, attrs, dpp_offset, dpp_credit_lmt); qp 2451 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->state = OCRDMA_QPS_RST; qp 2455 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->rq.va) qp 2456 drivers/infiniband/hw/ocrdma/ocrdma_hw.c dma_free_coherent(&pdev->dev, qp->rq.len, qp->rq.va, qp->rq.pa); qp 2459 drivers/infiniband/hw/ocrdma/ocrdma_hw.c dma_free_coherent(&pdev->dev, qp->sq.len, qp->sq.va, qp->sq.pa); qp 2466 drivers/infiniband/hw/ocrdma/ocrdma_hw.c int ocrdma_mbx_query_qp(struct ocrdma_dev *dev, struct ocrdma_qp *qp, qp 2476 drivers/infiniband/hw/ocrdma/ocrdma_hw.c cmd->qp_id = qp->id; qp 2487 drivers/infiniband/hw/ocrdma/ocrdma_hw.c static int ocrdma_set_av_params(struct ocrdma_qp *qp, qp 2501 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); qp 2528 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->sgid_idx = grh->sgid_index; qp 2573 drivers/infiniband/hw/ocrdma/ocrdma_hw.c static int ocrdma_set_qp_params(struct ocrdma_qp *qp, qp 2578 drivers/infiniband/hw/ocrdma/ocrdma_hw.c struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); qp 2586 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->qkey = attrs->qkey; qp 2591 drivers/infiniband/hw/ocrdma/ocrdma_hw.c status = ocrdma_set_av_params(qp, cmd, attrs, attr_mask); qp 2594 drivers/infiniband/hw/ocrdma/ocrdma_hw.c } else if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_UD) { qp 2664 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->max_ord = attrs->max_rd_atomic; qp 2672 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->max_ird = attrs->max_dest_rd_atomic; qp 2675 drivers/infiniband/hw/ocrdma/ocrdma_hw.c cmd->params.max_ord_ird = (qp->max_ord << qp 2677 drivers/infiniband/hw/ocrdma/ocrdma_hw.c (qp->max_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK); qp 2682 drivers/infiniband/hw/ocrdma/ocrdma_hw.c int ocrdma_mbx_modify_qp(struct ocrdma_dev *dev, struct ocrdma_qp *qp, qp 2692 drivers/infiniband/hw/ocrdma/ocrdma_hw.c cmd->params.id = qp->id; qp 2702 drivers/infiniband/hw/ocrdma/ocrdma_hw.c (qp->state << OCRDMA_QP_PARAMS_STATE_SHIFT) & qp 2706 drivers/infiniband/hw/ocrdma/ocrdma_hw.c status = ocrdma_set_qp_params(qp, cmd, attrs, attr_mask); qp 2718 drivers/infiniband/hw/ocrdma/ocrdma_hw.c int ocrdma_mbx_destroy_qp(struct ocrdma_dev *dev, struct ocrdma_qp *qp) qp 2727 drivers/infiniband/hw/ocrdma/ocrdma_hw.c cmd->qp_id = qp->id; qp 2734 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->sq.va) qp 2735 drivers/infiniband/hw/ocrdma/ocrdma_hw.c dma_free_coherent(&pdev->dev, qp->sq.len, qp->sq.va, qp->sq.pa); qp 2736 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (!qp->srq && qp->rq.va) qp 2737 drivers/infiniband/hw/ocrdma/ocrdma_hw.c dma_free_coherent(&pdev->dev, qp->rq.len, qp->rq.va, qp->rq.pa); qp 2738 drivers/infiniband/hw/ocrdma/ocrdma_hw.c if (qp->dpp_enabled) qp 2739 drivers/infiniband/hw/ocrdma/ocrdma_hw.c qp->pd->num_dpp_qp++; qp 1094 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) qp 1098 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { qp 1099 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c dev->qp_tbl[qp->id] = qp; qp 1105 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) qp 1107 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c dev->qp_tbl[qp->id] = NULL; qp 1184 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, qp 1191 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_pd *pd = qp->pd; qp 1197 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.qp_id = qp->id; qp 1198 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.sq_dbid = qp->sq.dbid; qp 1200 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); qp 1201 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); qp 1202 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.num_wqe_allocated = qp->sq.max_cnt; qp 1204 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.rq_dbid = qp->rq.dbid; qp 1206 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); qp 1207 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); qp 1208 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c uresp.num_rqe_allocated = qp->rq.max_cnt; qp 1216 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->dpp_enabled) { qp 1243 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, qp 1247 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->sq_db = dev->nic_info.db + qp 1250 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->rq_db = dev->nic_info.db + qp 1254 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->sq_db = dev->nic_info.db + qp 1257 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->rq_db = dev->nic_info.db + qp 1263 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) qp 1265 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->wqe_wr_id_tbl = qp 1266 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)), qp 1268 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->wqe_wr_id_tbl == NULL) qp 1270 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->rqe_wr_id_tbl = qp 1271 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL); qp 1272 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->rqe_wr_id_tbl == NULL) qp 1278 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, qp 1282 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->pd = pd; qp 1283 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_lock_init(&qp->q_lock); qp 1284 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c INIT_LIST_HEAD(&qp->sq_entry); qp 1285 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c INIT_LIST_HEAD(&qp->rq_entry); qp 1287 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->qp_type = attrs->qp_type; qp 1288 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; qp 1289 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->max_inline_data = attrs->cap.max_inline_data; qp 1290 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->sq.max_sges = attrs->cap.max_send_sge; qp 1291 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->rq.max_sges = attrs->cap.max_recv_sge; qp 1292 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->state = OCRDMA_QPS_RST; qp 1293 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; qp 1312 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp; qp 1326 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 1327 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!qp) { qp 1331 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_set_qp_init_params(qp, pd, attrs); qp 1333 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | qp 1337 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, qp 1345 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_alloc_wr_id_tbl(qp); qp 1350 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_add_qpn_map(dev, qp); qp 1353 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_set_qp_db(dev, qp, pd); qp 1355 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, qp 1362 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->ibqp.qp_num = qp->id; qp 1364 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c return &qp->ibqp; qp 1367 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_del_qpn_map(dev, qp); qp 1369 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_mbx_destroy_qp(dev, qp); qp 1372 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kfree(qp->wqe_wr_id_tbl); qp 1373 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kfree(qp->rqe_wr_id_tbl); qp 1374 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kfree(qp); qp 1384 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp; qp 1388 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp = get_ocrdma_qp(ibqp); qp 1391 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); qp 1397 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); qp 1405 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp; qp 1409 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp = get_ocrdma_qp(ibqp); qp 1415 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_lock_irqsave(&qp->q_lock, flags); qp 1416 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c old_qps = get_ibqp_state(qp->state); qp 1421 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 1426 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, qp 1475 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); qp 1480 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_mbx_query_qp(dev, qp, ¶ms); qp 1484 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->qp_type == IB_QPT_UD) qp 1496 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); qp 1497 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; qp 1498 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; qp 1499 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp_attr->cap.max_send_sge = qp->sq.max_sges; qp 1500 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp_attr->cap.max_recv_sge = qp->rq.max_sges; qp 1501 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp_attr->cap.max_inline_data = qp->max_inline_data; qp 1508 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->sgid_idx, qp 1551 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); qp 1569 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int is_hw_sq_empty(struct ocrdma_qp *qp) qp 1571 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c return (qp->sq.tail == qp->sq.head); qp 1574 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int is_hw_rq_empty(struct ocrdma_qp *qp) qp 1576 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c return (qp->rq.tail == qp->rq.head); qp 1601 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) qp 1623 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) qp 1634 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qpn == 0 || qpn != qp->id) qp 1638 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->sq); qp 1640 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->srq) { qp 1643 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->srq->rq.max_wqe_idx; qp 1645 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_lock_irqsave(&qp->srq->q_lock, flags); qp 1646 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->srq->rq); qp 1647 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); qp 1648 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock_irqrestore(&qp->srq->q_lock, flags); qp 1651 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->rq); qp 1665 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c void ocrdma_del_flush_qp(struct ocrdma_qp *qp) qp 1669 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); qp 1673 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); qp 1675 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c list_del(&qp->sq_entry); qp 1676 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!qp->srq) { qp 1677 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); qp 1679 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c list_del(&qp->rq_entry); qp 1687 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp; qp 1693 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp = get_ocrdma_qp(ibqp); qp 1696 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c pd = qp->pd; qp 1699 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->state != OCRDMA_QPS_RST) { qp 1709 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c (void) ocrdma_mbx_destroy_qp(dev, qp); qp 1715 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); qp 1716 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) { qp 1717 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_lock(&qp->rq_cq->cq_lock); qp 1718 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_del_qpn_map(dev, qp); qp 1719 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock(&qp->rq_cq->cq_lock); qp 1721 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_del_qpn_map(dev, qp); qp 1723 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); qp 1726 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_discard_cqes(qp, qp->sq_cq); qp 1727 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_discard_cqes(qp, qp->rq_cq); qp 1732 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, qp 1733 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c PAGE_ALIGN(qp->sq.len)); qp 1734 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!qp->srq) qp 1735 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, qp 1736 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c PAGE_ALIGN(qp->rq.len)); qp 1739 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_del_flush_qp(qp); qp 1741 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kfree(qp->wqe_wr_id_tbl); qp 1742 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kfree(qp->rqe_wr_id_tbl); qp 1743 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c kfree(qp); qp 1886 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, qp 1895 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->qp_type == IB_QPT_GSI) qp 1896 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ud_hdr->qkey = qp->qkey; qp 1932 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, qp 1940 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { qp 1942 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (unlikely(hdr->total_len > qp->max_inline_data)) { qp 1945 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->max_inline_data, hdr->total_len); qp 1972 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, qp 1979 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { qp 1980 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_build_ud_hdr(qp, hdr, wr); qp 1987 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); qp 1991 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, qp 1999 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); qp 2009 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, qp 2038 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static int ocrdma_build_reg(struct ocrdma_qp *qp, qp 2094 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) qp 2096 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); qp 2098 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c iowrite32(val, qp->sq_db); qp 2105 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); qp 2109 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_lock_irqsave(&qp->q_lock, flags); qp 2110 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { qp 2111 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 2117 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->qp_type == IB_QPT_UD && qp 2124 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || qp 2125 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c wr->num_sge > qp->sq.max_sges) { qp 2130 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c hdr = ocrdma_hwq_head(&qp->sq); qp 2132 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) qp 2148 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_build_send(qp, hdr, wr); qp 2154 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_build_send(qp, hdr, wr); qp 2162 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_build_write(qp, hdr, wr); qp 2165 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_build_read(qp, hdr, wr); qp 2176 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); qp 2186 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) qp 2187 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; qp 2189 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; qp 2190 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; qp 2196 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_ring_sq_db(qp); qp 2199 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_head(&qp->sq); qp 2202 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 2206 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) qp 2208 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); qp 2210 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c iowrite32(val, qp->rq_db); qp 2239 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); qp 2242 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_lock_irqsave(&qp->q_lock, flags); qp 2243 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { qp 2244 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 2249 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || qp 2250 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c wr->num_sge > qp->rq.max_sges) { qp 2255 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c rqe = ocrdma_hwq_head(&qp->rq); qp 2258 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; qp 2263 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_ring_rq_db(qp); qp 2266 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_head(&qp->rq); qp 2269 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 2415 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, qp 2422 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); qp 2424 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; qp 2453 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, qp 2465 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { qp 2486 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp, int status) qp 2491 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ibwc->qp = &qp->ibqp; qp 2494 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_flush_qp(qp); qp 2495 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); qp 2500 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { qp 2502 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_set_cqe_status_flushed(qp, cqe); qp 2508 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp, int status) qp 2511 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; qp 2512 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->rq); qp 2514 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c return ocrdma_update_err_cqe(ibwc, cqe, qp, status); qp 2518 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp, int status) qp 2520 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_update_wc(qp, ibwc, qp->sq.tail); qp 2521 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->sq); qp 2523 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c return ocrdma_update_err_cqe(ibwc, cqe, qp, status); qp 2527 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, qp 2532 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); qp 2541 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { qp 2545 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { qp 2548 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); qp 2559 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c } else if (is_hw_sq_empty(qp)) { qp 2566 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); qp 2571 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, qp 2576 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c int tail = qp->sq.tail; qp 2579 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!qp->wqe_wr_id_tbl[tail].signaled) { qp 2584 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ibwc->qp = &qp->ibqp; qp 2585 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_update_wc(qp, ibwc, tail); qp 2589 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; qp 2593 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->sq); qp 2597 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, qp 2607 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); qp 2609 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); qp 2642 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp) qp 2648 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c srq = get_ocrdma_srq(qp->ibqp.srq); qp 2660 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, qp 2665 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); qp 2673 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { qp 2674 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { qp 2677 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); qp 2683 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c } else if (is_hw_rq_empty(qp)) { qp 2690 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); qp 2695 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, qp 2700 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c dev = get_ocrdma_dev(qp->ibqp.device); qp 2702 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ibwc->qp = &qp->ibqp; qp 2705 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) qp 2721 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->ibqp.srq) { qp 2722 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_update_free_srq_cqe(ibwc, cqe, qp); qp 2724 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; qp 2725 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->rq); qp 2729 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, qp 2736 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { qp 2747 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_poll_success_rcqe(qp, cqe, ibwc); qp 2749 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, qp 2774 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp = NULL; qp 2789 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c qp = dev->qp_tbl[qpn]; qp 2790 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c BUG_ON(qp == NULL); qp 2793 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, qp 2796 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, qp 2828 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp, struct ib_wc *ibwc) qp 2833 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) qp 2835 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { qp 2836 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_update_wc(qp, ibwc, qp->sq.tail); qp 2837 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->sq); qp 2838 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { qp 2839 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; qp 2840 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c ocrdma_hwq_inc_tail(&qp->rq); qp 2859 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c struct ocrdma_qp *qp; qp 2876 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c list_for_each_entry(qp, &cq->sq_head, sq_entry) { qp 2879 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); qp 90 drivers/infiniband/hw/ocrdma/ocrdma_verbs.h void ocrdma_del_flush_qp(struct ocrdma_qp *qp); qp 675 drivers/infiniband/hw/qedr/main.c struct qedr_qp *qp; qp 745 drivers/infiniband/hw/qedr/main.c qp = (struct qedr_qp *)(uintptr_t)roce_handle64; qp 746 drivers/infiniband/hw/qedr/main.c if (qp) { qp 747 drivers/infiniband/hw/qedr/main.c ibqp = &qp->ibqp; qp 750 drivers/infiniband/hw/qedr/main.c event.element.qp = ibqp; qp 758 drivers/infiniband/hw/qedr/main.c DP_ERR(dev, "QP event %d on handle %p\n", e_code, qp); qp 547 drivers/infiniband/hw/qedr/qedr.h struct qedr_qp *qp; qp 84 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct qedr_qp *qp = container_of(ref, struct qedr_qp, refcnt); qp 86 drivers/infiniband/hw/qedr/qedr_iw_cm.c kfree(qp); qp 94 drivers/infiniband/hw/qedr/qedr_iw_cm.c if (ep->qp) qp 95 drivers/infiniband/hw/qedr/qedr_iw_cm.c kref_put(&ep->qp->refcnt, qedr_iw_free_qp); qp 179 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct ib_qp *ibqp = &ep->qp->ibqp; qp 187 drivers/infiniband/hw/qedr/qedr_iw_cm.c event.element.qp = ibqp; qp 206 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct qedr_qp *qp = ep->qp; qp 214 drivers/infiniband/hw/qedr/qedr_iw_cm.c &qp->iwarp_cm_flags)) qp 236 drivers/infiniband/hw/qedr/qedr_iw_cm.c dev->ops->rdma_modify_qp(dev->rdma_ctx, qp->qed_qp, &qp_params); qp 238 drivers/infiniband/hw/qedr/qedr_iw_cm.c complete(&ep->qp->iwarp_cm_comp); qp 280 drivers/infiniband/hw/qedr/qedr_iw_cm.c if ((params->status == -ECONNREFUSED) && (!ep->qp)) { qp 287 drivers/infiniband/hw/qedr/qedr_iw_cm.c complete(&ep->qp->iwarp_cm_comp); qp 300 drivers/infiniband/hw/qedr/qedr_iw_cm.c complete(&ep->qp->iwarp_cm_comp); qp 511 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct qedr_qp *qp; qp 514 drivers/infiniband/hw/qedr/qedr_iw_cm.c qp = xa_load(&dev->qps, qpn); qp 515 drivers/infiniband/hw/qedr/qedr_iw_cm.c if (qp) qp 516 drivers/infiniband/hw/qedr/qedr_iw_cm.c kref_get(&qp->refcnt); qp 519 drivers/infiniband/hw/qedr/qedr_iw_cm.c return qp; qp 533 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct qedr_qp *qp; qp 560 drivers/infiniband/hw/qedr/qedr_iw_cm.c qp = qedr_iw_load_qp(dev, conn_param->qpn); qp 561 drivers/infiniband/hw/qedr/qedr_iw_cm.c if (!qp) { qp 566 drivers/infiniband/hw/qedr/qedr_iw_cm.c ep->qp = qp; qp 620 drivers/infiniband/hw/qedr/qedr_iw_cm.c conn_param->private_data_len, qp->rq_psn); qp 626 drivers/infiniband/hw/qedr/qedr_iw_cm.c in_params.qp = qp->qed_qp; qp 630 drivers/infiniband/hw/qedr/qedr_iw_cm.c &qp->iwarp_cm_flags)) qp 635 drivers/infiniband/hw/qedr/qedr_iw_cm.c complete(&qp->iwarp_cm_comp); qp 728 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct qedr_qp *qp; qp 734 drivers/infiniband/hw/qedr/qedr_iw_cm.c qp = qedr_iw_load_qp(dev, conn_param->qpn); qp 735 drivers/infiniband/hw/qedr/qedr_iw_cm.c if (!qp) { qp 740 drivers/infiniband/hw/qedr/qedr_iw_cm.c ep->qp = qp; qp 746 drivers/infiniband/hw/qedr/qedr_iw_cm.c params.qp = ep->qp->qed_qp; qp 753 drivers/infiniband/hw/qedr/qedr_iw_cm.c &qp->iwarp_cm_flags)) qp 758 drivers/infiniband/hw/qedr/qedr_iw_cm.c complete(&qp->iwarp_cm_comp); qp 780 drivers/infiniband/hw/qedr/qedr_iw_cm.c ep->qp = NULL; qp 787 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 789 drivers/infiniband/hw/qedr/qedr_iw_cm.c kref_get(&qp->refcnt); qp 794 drivers/infiniband/hw/qedr/qedr_iw_cm.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 796 drivers/infiniband/hw/qedr/qedr_iw_cm.c kref_put(&qp->refcnt, qedr_iw_free_qp); qp 45 drivers/infiniband/hw/qedr/qedr_iw_cm.h void qedr_iw_qp_add_ref(struct ib_qp *qp); qp 47 drivers/infiniband/hw/qedr/qedr_iw_cm.h void qedr_iw_qp_rem_ref(struct ib_qp *qp); qp 58 drivers/infiniband/hw/qedr/qedr_roce_cm.c void qedr_store_gsi_qp_cq(struct qedr_dev *dev, struct qedr_qp *qp, qp 64 drivers/infiniband/hw/qedr/qedr_roce_cm.c dev->gsi_qp = qp; qp 76 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp = dev->gsi_qp; qp 81 drivers/infiniband/hw/qedr/qedr_roce_cm.c dev->gsi_sqcq, dev->gsi_rqcq, qp->sq.gsi_cons, qp 88 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_lock_irqsave(&qp->q_lock, flags); qp 89 drivers/infiniband/hw/qedr/qedr_roce_cm.c qedr_inc_sw_gsi_cons(&qp->sq); qp 90 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 101 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp = dev->gsi_qp; qp 104 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_lock_irqsave(&qp->q_lock, flags); qp 106 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->rqe_wr_id[qp->rq.gsi_cons].rc = data->u.data_length_error ? qp 108 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->rqe_wr_id[qp->rq.gsi_cons].vlan = data->vlan; qp 110 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = qp 112 drivers/infiniband/hw/qedr/qedr_roce_cm.c *((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) = qp 114 drivers/infiniband/hw/qedr/qedr_roce_cm.c *((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) = qp 117 drivers/infiniband/hw/qedr/qedr_roce_cm.c qedr_inc_sw_gsi_cons(&qp->rq); qp 119 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 263 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct ib_qp_init_attr *attrs, struct qedr_qp *qp) qp 324 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp) qp 332 drivers/infiniband/hw/qedr/qedr_roce_cm.c rc = qedr_ll2_start(dev, attrs, qp); qp 339 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->ibqp.qp_num = 1; qp 340 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->rq.max_wr = attrs->cap.max_recv_wr; qp 341 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->sq.max_wr = attrs->cap.max_send_wr; qp 343 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), qp 345 drivers/infiniband/hw/qedr/qedr_roce_cm.c if (!qp->rqe_wr_id) qp 347 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), qp 349 drivers/infiniband/hw/qedr/qedr_roce_cm.c if (!qp->wqe_wr_id) qp 352 drivers/infiniband/hw/qedr/qedr_roce_cm.c qedr_store_gsi_qp_cq(dev, qp, attrs); qp 360 drivers/infiniband/hw/qedr/qedr_roce_cm.c DP_DEBUG(dev, QEDR_MSG_GSI, "created GSI QP %p\n", qp); qp 362 drivers/infiniband/hw/qedr/qedr_roce_cm.c return &qp->ibqp; qp 365 drivers/infiniband/hw/qedr/qedr_roce_cm.c kfree(qp->rqe_wr_id); qp 382 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp, qp 451 drivers/infiniband/hw/qedr/qedr_roce_cm.c udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); qp 493 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp, qp 506 drivers/infiniband/hw/qedr/qedr_roce_cm.c rc = qedr_gsi_build_header(dev, qp, swr, &udh, &roce_mode); qp 547 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 548 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_dev *dev = qp->dev; qp 552 drivers/infiniband/hw/qedr/qedr_roce_cm.c if (qp->state != QED_ROCE_QP_STATE_RTS) { qp 556 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->state); qp 575 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_lock_irqsave(&qp->q_lock, flags); qp 577 drivers/infiniband/hw/qedr/qedr_roce_cm.c rc = qedr_gsi_build_packet(dev, qp, wr, &pkt); qp 579 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 586 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; qp 587 drivers/infiniband/hw/qedr/qedr_roce_cm.c qedr_inc_sw_prod(&qp->sq); qp 588 drivers/infiniband/hw/qedr/qedr_roce_cm.c DP_DEBUG(qp->dev, QEDR_MSG_GSI, qp 597 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 617 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 621 drivers/infiniband/hw/qedr/qedr_roce_cm.c if ((qp->state != QED_ROCE_QP_STATE_RTR) && qp 622 drivers/infiniband/hw/qedr/qedr_roce_cm.c (qp->state != QED_ROCE_QP_STATE_RTS)) { qp 626 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->state); qp 630 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_lock_irqsave(&qp->q_lock, flags); qp 653 drivers/infiniband/hw/qedr/qedr_roce_cm.c memset(&qp->rqe_wr_id[qp->rq.prod], 0, qp 654 drivers/infiniband/hw/qedr/qedr_roce_cm.c sizeof(qp->rqe_wr_id[qp->rq.prod])); qp 655 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0]; qp 656 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; qp 658 drivers/infiniband/hw/qedr/qedr_roce_cm.c qedr_inc_sw_prod(&qp->rq); qp 663 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 667 drivers/infiniband/hw/qedr/qedr_roce_cm.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 676 drivers/infiniband/hw/qedr/qedr_roce_cm.c struct qedr_qp *qp = dev->gsi_qp; qp 683 drivers/infiniband/hw/qedr/qedr_roce_cm.c while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) { qp 686 drivers/infiniband/hw/qedr/qedr_roce_cm.c wc[i].qp = &qp->ibqp; qp 687 drivers/infiniband/hw/qedr/qedr_roce_cm.c wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; qp 690 drivers/infiniband/hw/qedr/qedr_roce_cm.c wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc) ? qp 693 drivers/infiniband/hw/qedr/qedr_roce_cm.c wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length; qp 695 drivers/infiniband/hw/qedr/qedr_roce_cm.c ether_addr_copy(wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac); qp 698 drivers/infiniband/hw/qedr/qedr_roce_cm.c vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan & qp 703 drivers/infiniband/hw/qedr/qedr_roce_cm.c wc[i].sl = (qp->rqe_wr_id[qp->rq.cons].vlan & qp 707 drivers/infiniband/hw/qedr/qedr_roce_cm.c qedr_inc_sw_cons(&qp->rq); qp 711 drivers/infiniband/hw/qedr/qedr_roce_cm.c while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) { qp 714 drivers/infiniband/hw/qedr/qedr_roce_cm.c wc[i].qp = &qp->ibqp; qp 715 drivers/infiniband/hw/qedr/qedr_roce_cm.c wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; qp 719 drivers/infiniband/hw/qedr/qedr_roce_cm.c qedr_inc_sw_cons(&qp->sq); qp 727 drivers/infiniband/hw/qedr/qedr_roce_cm.c num_entries, i, qp->rq.cons, qp->rq.gsi_cons, qp->sq.cons, qp 728 drivers/infiniband/hw/qedr/qedr_roce_cm.c qp->sq.gsi_cons, qp->ibqp.qp_num); qp 55 drivers/infiniband/hw/qedr/qedr_roce_cm.h struct qedr_qp *qp); qp 57 drivers/infiniband/hw/qedr/qedr_roce_cm.h struct qedr_qp *qp, struct ib_qp_init_attr *attrs); qp 1141 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp) qp 1153 drivers/infiniband/hw/qedr/verbs.c uresp->rq_icid = qp->icid; qp 1158 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp) qp 1164 drivers/infiniband/hw/qedr/verbs.c uresp->sq_icid = qp->icid; qp 1166 drivers/infiniband/hw/qedr/verbs.c uresp->sq_icid = qp->icid + 1; qp 1170 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, struct ib_udata *udata) qp 1176 drivers/infiniband/hw/qedr/verbs.c qedr_copy_sq_uresp(dev, &uresp, qp); qp 1177 drivers/infiniband/hw/qedr/verbs.c qedr_copy_rq_uresp(dev, &uresp, qp); qp 1180 drivers/infiniband/hw/qedr/verbs.c uresp.qp_id = qp->qp_id; qp 1186 drivers/infiniband/hw/qedr/verbs.c qp->icid); qp 1192 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 1196 drivers/infiniband/hw/qedr/verbs.c spin_lock_init(&qp->q_lock); qp 1198 drivers/infiniband/hw/qedr/verbs.c kref_init(&qp->refcnt); qp 1199 drivers/infiniband/hw/qedr/verbs.c init_completion(&qp->iwarp_cm_comp); qp 1201 drivers/infiniband/hw/qedr/verbs.c qp->pd = pd; qp 1202 drivers/infiniband/hw/qedr/verbs.c qp->qp_type = attrs->qp_type; qp 1203 drivers/infiniband/hw/qedr/verbs.c qp->max_inline_data = attrs->cap.max_inline_data; qp 1204 drivers/infiniband/hw/qedr/verbs.c qp->sq.max_sges = attrs->cap.max_send_sge; qp 1205 drivers/infiniband/hw/qedr/verbs.c qp->state = QED_ROCE_QP_STATE_RESET; qp 1206 drivers/infiniband/hw/qedr/verbs.c qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; qp 1207 drivers/infiniband/hw/qedr/verbs.c qp->sq_cq = get_qedr_cq(attrs->send_cq); qp 1208 drivers/infiniband/hw/qedr/verbs.c qp->dev = dev; qp 1211 drivers/infiniband/hw/qedr/verbs.c qp->srq = get_qedr_srq(attrs->srq); qp 1213 drivers/infiniband/hw/qedr/verbs.c qp->rq_cq = get_qedr_cq(attrs->recv_cq); qp 1214 drivers/infiniband/hw/qedr/verbs.c qp->rq.max_sges = attrs->cap.max_recv_sge; qp 1217 drivers/infiniband/hw/qedr/verbs.c qp->rq.max_sges, qp->rq_cq->icid); qp 1222 drivers/infiniband/hw/qedr/verbs.c pd->pd_id, qp->qp_type, qp->max_inline_data, qp 1223 drivers/infiniband/hw/qedr/verbs.c qp->state, qp->signaled, (attrs->srq) ? 1 : 0); qp 1226 drivers/infiniband/hw/qedr/verbs.c qp->sq.max_sges, qp->sq_cq->icid); qp 1229 drivers/infiniband/hw/qedr/verbs.c static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) qp 1231 drivers/infiniband/hw/qedr/verbs.c qp->sq.db = dev->db_addr + qp 1233 drivers/infiniband/hw/qedr/verbs.c qp->sq.db_data.data.icid = qp->icid + 1; qp 1234 drivers/infiniband/hw/qedr/verbs.c if (!qp->srq) { qp 1235 drivers/infiniband/hw/qedr/verbs.c qp->rq.db = dev->db_addr + qp 1237 drivers/infiniband/hw/qedr/verbs.c qp->rq.db_data.data.icid = qp->icid; qp 1516 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 1522 drivers/infiniband/hw/qedr/verbs.c params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); qp 1523 drivers/infiniband/hw/qedr/verbs.c params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); qp 1534 drivers/infiniband/hw/qedr/verbs.c if (!qp->srq) { qp 1539 drivers/infiniband/hw/qedr/verbs.c params->srq_id = qp->srq->srq_id; qp 1544 drivers/infiniband/hw/qedr/verbs.c static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) qp 1553 drivers/infiniband/hw/qedr/verbs.c qp, qp 1554 drivers/infiniband/hw/qedr/verbs.c qp->usq.buf_addr, qp 1555 drivers/infiniband/hw/qedr/verbs.c qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); qp 1560 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 1563 drivers/infiniband/hw/qedr/verbs.c qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; qp 1564 drivers/infiniband/hw/qedr/verbs.c qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; qp 1566 drivers/infiniband/hw/qedr/verbs.c qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, qp 1567 drivers/infiniband/hw/qedr/verbs.c &qp->usq.pbl_info, FW_PAGE_SHIFT); qp 1568 drivers/infiniband/hw/qedr/verbs.c if (!qp->srq) { qp 1569 drivers/infiniband/hw/qedr/verbs.c qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; qp 1570 drivers/infiniband/hw/qedr/verbs.c qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; qp 1573 drivers/infiniband/hw/qedr/verbs.c qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, qp 1574 drivers/infiniband/hw/qedr/verbs.c &qp->urq.pbl_info, FW_PAGE_SHIFT); qp 1577 drivers/infiniband/hw/qedr/verbs.c static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) qp 1579 drivers/infiniband/hw/qedr/verbs.c ib_umem_release(qp->usq.umem); qp 1580 drivers/infiniband/hw/qedr/verbs.c qp->usq.umem = NULL; qp 1582 drivers/infiniband/hw/qedr/verbs.c ib_umem_release(qp->urq.umem); qp 1583 drivers/infiniband/hw/qedr/verbs.c qp->urq.umem = NULL; qp 1586 drivers/infiniband/hw/qedr/verbs.c qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); qp 1587 drivers/infiniband/hw/qedr/verbs.c qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl); qp 1589 drivers/infiniband/hw/qedr/verbs.c kfree(qp->usq.pbl_tbl); qp 1590 drivers/infiniband/hw/qedr/verbs.c kfree(qp->urq.pbl_tbl); qp 1595 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 1607 drivers/infiniband/hw/qedr/verbs.c qp->create_type = QEDR_QP_CREATE_USER; qp 1616 drivers/infiniband/hw/qedr/verbs.c rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, qp 1621 drivers/infiniband/hw/qedr/verbs.c if (!qp->srq) { qp 1623 drivers/infiniband/hw/qedr/verbs.c rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, qp 1630 drivers/infiniband/hw/qedr/verbs.c qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); qp 1633 drivers/infiniband/hw/qedr/verbs.c in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; qp 1634 drivers/infiniband/hw/qedr/verbs.c in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; qp 1635 drivers/infiniband/hw/qedr/verbs.c if (!qp->srq) { qp 1636 drivers/infiniband/hw/qedr/verbs.c in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; qp 1637 drivers/infiniband/hw/qedr/verbs.c in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; qp 1640 drivers/infiniband/hw/qedr/verbs.c qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, qp 1643 drivers/infiniband/hw/qedr/verbs.c if (!qp->qed_qp) { qp 1649 drivers/infiniband/hw/qedr/verbs.c qedr_iwarp_populate_user_qp(dev, qp, &out_params); qp 1651 drivers/infiniband/hw/qedr/verbs.c qp->qp_id = out_params.qp_id; qp 1652 drivers/infiniband/hw/qedr/verbs.c qp->icid = out_params.icid; qp 1654 drivers/infiniband/hw/qedr/verbs.c rc = qedr_copy_qp_uresp(dev, qp, udata); qp 1658 drivers/infiniband/hw/qedr/verbs.c qedr_qp_user_print(dev, qp); qp 1662 drivers/infiniband/hw/qedr/verbs.c rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); qp 1667 drivers/infiniband/hw/qedr/verbs.c qedr_cleanup_user(dev, qp); qp 1671 drivers/infiniband/hw/qedr/verbs.c static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) qp 1673 drivers/infiniband/hw/qedr/verbs.c qp->sq.db = dev->db_addr + qp 1675 drivers/infiniband/hw/qedr/verbs.c qp->sq.db_data.data.icid = qp->icid; qp 1677 drivers/infiniband/hw/qedr/verbs.c qp->rq.db = dev->db_addr + qp 1679 drivers/infiniband/hw/qedr/verbs.c qp->rq.db_data.data.icid = qp->icid; qp 1680 drivers/infiniband/hw/qedr/verbs.c qp->rq.iwarp_db2 = dev->db_addr + qp 1682 drivers/infiniband/hw/qedr/verbs.c qp->rq.iwarp_db2_data.data.icid = qp->icid; qp 1683 drivers/infiniband/hw/qedr/verbs.c qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; qp 1688 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 1701 drivers/infiniband/hw/qedr/verbs.c &qp->sq.pbl, NULL); qp 1706 drivers/infiniband/hw/qedr/verbs.c in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); qp 1707 drivers/infiniband/hw/qedr/verbs.c in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); qp 1715 drivers/infiniband/hw/qedr/verbs.c &qp->rq.pbl, NULL); qp 1719 drivers/infiniband/hw/qedr/verbs.c in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); qp 1720 drivers/infiniband/hw/qedr/verbs.c in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); qp 1722 drivers/infiniband/hw/qedr/verbs.c qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, qp 1725 drivers/infiniband/hw/qedr/verbs.c if (!qp->qed_qp) qp 1728 drivers/infiniband/hw/qedr/verbs.c qp->qp_id = out_params.qp_id; qp 1729 drivers/infiniband/hw/qedr/verbs.c qp->icid = out_params.icid; qp 1731 drivers/infiniband/hw/qedr/verbs.c qedr_set_roce_db_info(dev, qp); qp 1737 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 1752 drivers/infiniband/hw/qedr/verbs.c qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, qp 1755 drivers/infiniband/hw/qedr/verbs.c if (!qp->qed_qp) qp 1768 drivers/infiniband/hw/qedr/verbs.c &qp->sq.pbl, &ext_pbl); qp 1782 drivers/infiniband/hw/qedr/verbs.c &qp->rq.pbl, &ext_pbl); qp 1787 drivers/infiniband/hw/qedr/verbs.c qp->qp_id = out_params.qp_id; qp 1788 drivers/infiniband/hw/qedr/verbs.c qp->icid = out_params.icid; qp 1790 drivers/infiniband/hw/qedr/verbs.c qedr_set_iwarp_db_info(dev, qp); qp 1794 drivers/infiniband/hw/qedr/verbs.c dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); qp 1799 drivers/infiniband/hw/qedr/verbs.c static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) qp 1801 drivers/infiniband/hw/qedr/verbs.c dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); qp 1802 drivers/infiniband/hw/qedr/verbs.c kfree(qp->wqe_wr_id); qp 1804 drivers/infiniband/hw/qedr/verbs.c dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); qp 1805 drivers/infiniband/hw/qedr/verbs.c kfree(qp->rqe_wr_id); qp 1809 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 1821 drivers/infiniband/hw/qedr/verbs.c qp->create_type = QEDR_QP_CREATE_KERNEL; qp 1833 drivers/infiniband/hw/qedr/verbs.c qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, qp 1836 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), qp 1838 drivers/infiniband/hw/qedr/verbs.c if (!qp->wqe_wr_id) { qp 1844 drivers/infiniband/hw/qedr/verbs.c in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); qp 1845 drivers/infiniband/hw/qedr/verbs.c in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); qp 1851 drivers/infiniband/hw/qedr/verbs.c qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); qp 1854 drivers/infiniband/hw/qedr/verbs.c qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), qp 1856 drivers/infiniband/hw/qedr/verbs.c if (!qp->rqe_wr_id) { qp 1859 drivers/infiniband/hw/qedr/verbs.c kfree(qp->wqe_wr_id); qp 1863 drivers/infiniband/hw/qedr/verbs.c qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); qp 1870 drivers/infiniband/hw/qedr/verbs.c n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; qp 1873 drivers/infiniband/hw/qedr/verbs.c rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, qp 1876 drivers/infiniband/hw/qedr/verbs.c rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, qp 1879 drivers/infiniband/hw/qedr/verbs.c qedr_cleanup_kernel(dev, qp); qp 1890 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp; qp 1909 drivers/infiniband/hw/qedr/verbs.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 1910 drivers/infiniband/hw/qedr/verbs.c if (!qp) { qp 1915 drivers/infiniband/hw/qedr/verbs.c qedr_set_common_qp_params(dev, qp, pd, attrs); qp 1918 drivers/infiniband/hw/qedr/verbs.c ibqp = qedr_create_gsi_qp(dev, attrs, qp); qp 1920 drivers/infiniband/hw/qedr/verbs.c kfree(qp); qp 1925 drivers/infiniband/hw/qedr/verbs.c rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); qp 1927 drivers/infiniband/hw/qedr/verbs.c rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); qp 1932 drivers/infiniband/hw/qedr/verbs.c qp->ibqp.qp_num = qp->qp_id; qp 1935 drivers/infiniband/hw/qedr/verbs.c rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); qp 1940 drivers/infiniband/hw/qedr/verbs.c return &qp->ibqp; qp 1943 drivers/infiniband/hw/qedr/verbs.c kfree(qp); qp 2000 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 2013 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = 0; qp 2014 drivers/infiniband/hw/qedr/verbs.c qedr_reset_qp_hwq_info(&qp->sq); qp 2015 drivers/infiniband/hw/qedr/verbs.c qedr_reset_qp_hwq_info(&qp->rq); qp 2030 drivers/infiniband/hw/qedr/verbs.c writel(qp->rq.db_data.raw, qp->rq.db); qp 2083 drivers/infiniband/hw/qedr/verbs.c if ((qp->rq.prod != qp->rq.cons) || qp 2084 drivers/infiniband/hw/qedr/verbs.c (qp->sq.prod != qp->sq.cons)) { qp 2087 drivers/infiniband/hw/qedr/verbs.c qp->rq.prod, qp->rq.cons, qp->sq.prod, qp 2088 drivers/infiniband/hw/qedr/verbs.c qp->sq.cons); qp 2108 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 2110 drivers/infiniband/hw/qedr/verbs.c struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); qp 2117 drivers/infiniband/hw/qedr/verbs.c "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, qp 2120 drivers/infiniband/hw/qedr/verbs.c old_qp_state = qedr_get_ibqp_state(qp->state); qp 2132 drivers/infiniband/hw/qedr/verbs.c attr_mask, qp->qp_id, ibqp->qp_type, qp 2161 drivers/infiniband/hw/qedr/verbs.c qp->qkey = attr->qkey; qp 2185 drivers/infiniband/hw/qedr/verbs.c qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), qp 2190 drivers/infiniband/hw/qedr/verbs.c if (!qp->mtu) { qp 2191 drivers/infiniband/hw/qedr/verbs.c qp->mtu = qp 2193 drivers/infiniband/hw/qedr/verbs.c pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); qp 2203 drivers/infiniband/hw/qedr/verbs.c qp->sgid_idx = grh->sgid_index; qp 2230 drivers/infiniband/hw/qedr/verbs.c qp_params.mtu = qp->mtu; qp 2236 drivers/infiniband/hw/qedr/verbs.c if (qp->mtu) qp 2237 drivers/infiniband/hw/qedr/verbs.c qp_params.mtu = qp->mtu; qp 2280 drivers/infiniband/hw/qedr/verbs.c qp->rq_psn = attr->rq_psn; qp 2308 drivers/infiniband/hw/qedr/verbs.c qp->sq_psn = attr->sq_psn; qp 2333 drivers/infiniband/hw/qedr/verbs.c qp->dest_qp_num = attr->dest_qp_num; qp 2336 drivers/infiniband/hw/qedr/verbs.c cur_state = qp->state; qp 2343 drivers/infiniband/hw/qedr/verbs.c if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && qp 2345 drivers/infiniband/hw/qedr/verbs.c qp->state = QED_ROCE_QP_STATE_ERR; qp 2347 drivers/infiniband/hw/qedr/verbs.c if (qp->qp_type != IB_QPT_GSI) qp 2349 drivers/infiniband/hw/qedr/verbs.c qp->qed_qp, &qp_params); qp 2352 drivers/infiniband/hw/qedr/verbs.c if ((qp->qp_type != IB_QPT_GSI) && (!udata)) qp 2353 drivers/infiniband/hw/qedr/verbs.c rc = qedr_update_qp_state(dev, qp, cur_state, qp 2355 drivers/infiniband/hw/qedr/verbs.c qp->state = qp_params.new_state; qp 2381 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 2382 drivers/infiniband/hw/qedr/verbs.c struct qedr_dev *dev = qp->dev; qp 2387 drivers/infiniband/hw/qedr/verbs.c rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); qp 2404 drivers/infiniband/hw/qedr/verbs.c qp_attr->cap.max_send_wr = qp->sq.max_wr; qp 2405 drivers/infiniband/hw/qedr/verbs.c qp_attr->cap.max_recv_wr = qp->rq.max_wr; qp 2406 drivers/infiniband/hw/qedr/verbs.c qp_attr->cap.max_send_sge = qp->sq.max_sges; qp 2407 drivers/infiniband/hw/qedr/verbs.c qp_attr->cap.max_recv_sge = qp->rq.max_sges; qp 2413 drivers/infiniband/hw/qedr/verbs.c params.flow_label, qp->sgid_idx, qp 2443 drivers/infiniband/hw/qedr/verbs.c static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, qp 2448 drivers/infiniband/hw/qedr/verbs.c if (qp->qp_type != IB_QPT_GSI) { qp 2449 drivers/infiniband/hw/qedr/verbs.c rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); qp 2454 drivers/infiniband/hw/qedr/verbs.c if (qp->create_type == QEDR_QP_CREATE_USER) qp 2455 drivers/infiniband/hw/qedr/verbs.c qedr_cleanup_user(dev, qp); qp 2457 drivers/infiniband/hw/qedr/verbs.c qedr_cleanup_kernel(dev, qp); qp 2464 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 2465 drivers/infiniband/hw/qedr/verbs.c struct qedr_dev *dev = qp->dev; qp 2470 drivers/infiniband/hw/qedr/verbs.c qp, qp->qp_type); qp 2473 drivers/infiniband/hw/qedr/verbs.c if ((qp->state != QED_ROCE_QP_STATE_RESET) && qp 2474 drivers/infiniband/hw/qedr/verbs.c (qp->state != QED_ROCE_QP_STATE_ERR) && qp 2475 drivers/infiniband/hw/qedr/verbs.c (qp->state != QED_ROCE_QP_STATE_INIT)) { qp 2489 drivers/infiniband/hw/qedr/verbs.c &qp->iwarp_cm_flags)) qp 2490 drivers/infiniband/hw/qedr/verbs.c wait_for_completion(&qp->iwarp_cm_comp); qp 2503 drivers/infiniband/hw/qedr/verbs.c &qp->iwarp_cm_flags)) qp 2504 drivers/infiniband/hw/qedr/verbs.c wait_for_completion(&qp->iwarp_cm_comp); qp 2507 drivers/infiniband/hw/qedr/verbs.c if (qp->qp_type == IB_QPT_GSI) qp 2515 drivers/infiniband/hw/qedr/verbs.c xa_erase(&dev->qps, qp->qp_id); qp 2517 drivers/infiniband/hw/qedr/verbs.c qedr_free_qp_resources(dev, qp, udata); qp 2520 drivers/infiniband/hw/qedr/verbs.c qedr_iw_qp_rem_ref(&qp->ibqp); qp 2927 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, u8 *wqe_size, qp 2961 drivers/infiniband/hw/qedr/verbs.c wqe = (char *)qed_chain_produce(&qp->sq.pbl); qp 3012 drivers/infiniband/hw/qedr/verbs.c static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, qp 3019 drivers/infiniband/hw/qedr/verbs.c struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); qp 3034 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 3049 drivers/infiniband/hw/qedr/verbs.c return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, qp 3053 drivers/infiniband/hw/qedr/verbs.c return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); qp 3057 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, qp 3068 drivers/infiniband/hw/qedr/verbs.c return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, qp 3072 drivers/infiniband/hw/qedr/verbs.c return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); qp 3075 drivers/infiniband/hw/qedr/verbs.c static int qedr_prepare_reg(struct qedr_qp *qp, qp 3082 drivers/infiniband/hw/qedr/verbs.c fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); qp 3108 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].mr = mr; qp 3139 drivers/infiniband/hw/qedr/verbs.c static inline bool qedr_can_post_send(struct qedr_qp *qp, qp 3143 drivers/infiniband/hw/qedr/verbs.c struct qedr_dev *dev = qp->dev; qp 3146 drivers/infiniband/hw/qedr/verbs.c err_wr = wr->num_sge > qp->sq.max_sges; qp 3147 drivers/infiniband/hw/qedr/verbs.c wq_is_full = qedr_wq_is_full(&qp->sq); qp 3148 drivers/infiniband/hw/qedr/verbs.c pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < qp 3151 drivers/infiniband/hw/qedr/verbs.c if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { qp 3154 drivers/infiniband/hw/qedr/verbs.c qp); qp 3155 drivers/infiniband/hw/qedr/verbs.c qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; qp 3158 drivers/infiniband/hw/qedr/verbs.c if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { qp 3161 drivers/infiniband/hw/qedr/verbs.c qp); qp 3162 drivers/infiniband/hw/qedr/verbs.c qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; qp 3166 drivers/infiniband/hw/qedr/verbs.c !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { qp 3169 drivers/infiniband/hw/qedr/verbs.c qp); qp 3170 drivers/infiniband/hw/qedr/verbs.c qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; qp 3181 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 3196 drivers/infiniband/hw/qedr/verbs.c if (!qedr_can_post_send(qp, wr)) { qp 3201 drivers/infiniband/hw/qedr/verbs.c wqe = qed_chain_produce(&qp->sq.pbl); qp 3202 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].signaled = qp 3203 drivers/infiniband/hw/qedr/verbs.c !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; qp 3208 drivers/infiniband/hw/qedr/verbs.c comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; qp 3212 drivers/infiniband/hw/qedr/verbs.c wqe->prev_wqe_size = qp->prev_wqe_size; qp 3214 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); qp 3226 drivers/infiniband/hw/qedr/verbs.c swqe2 = qed_chain_produce(&qp->sq.pbl); qp 3229 drivers/infiniband/hw/qedr/verbs.c length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, qp 3232 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; qp 3233 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = swqe->wqe_size; qp 3234 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; qp 3241 drivers/infiniband/hw/qedr/verbs.c swqe2 = qed_chain_produce(&qp->sq.pbl); qp 3242 drivers/infiniband/hw/qedr/verbs.c length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, qp 3245 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; qp 3246 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = swqe->wqe_size; qp 3247 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; qp 3252 drivers/infiniband/hw/qedr/verbs.c swqe2 = qed_chain_produce(&qp->sq.pbl); qp 3255 drivers/infiniband/hw/qedr/verbs.c length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, qp 3258 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; qp 3259 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = swqe->wqe_size; qp 3260 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; qp 3274 drivers/infiniband/hw/qedr/verbs.c rwqe2 = qed_chain_produce(&qp->sq.pbl); qp 3275 drivers/infiniband/hw/qedr/verbs.c length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, qp 3278 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; qp 3279 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = rwqe->wqe_size; qp 3280 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; qp 3287 drivers/infiniband/hw/qedr/verbs.c rwqe2 = qed_chain_produce(&qp->sq.pbl); qp 3288 drivers/infiniband/hw/qedr/verbs.c length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, qp 3291 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; qp 3292 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = rwqe->wqe_size; qp 3293 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; qp 3304 drivers/infiniband/hw/qedr/verbs.c rwqe2 = qed_chain_produce(&qp->sq.pbl); qp 3305 drivers/infiniband/hw/qedr/verbs.c length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, qp 3308 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; qp 3309 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = rwqe->wqe_size; qp 3310 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; qp 3318 drivers/infiniband/hw/qedr/verbs.c awqe2 = qed_chain_produce(&qp->sq.pbl); qp 3322 drivers/infiniband/hw/qedr/verbs.c awqe3 = qed_chain_produce(&qp->sq.pbl); qp 3336 drivers/infiniband/hw/qedr/verbs.c qedr_prepare_sq_sges(qp, NULL, wr); qp 3338 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; qp 3339 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = awqe1->wqe_size; qp 3348 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; qp 3349 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = iwqe->wqe_size; qp 3357 drivers/infiniband/hw/qedr/verbs.c rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); qp 3364 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; qp 3365 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = fwqe1->wqe_size; qp 3380 drivers/infiniband/hw/qedr/verbs.c value = le16_to_cpu(qp->sq.db_data.data.value); qp 3381 drivers/infiniband/hw/qedr/verbs.c qed_chain_set_prod(&qp->sq.pbl, value, wqe); qp 3384 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = wqe->prev_wqe_size; qp 3396 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 3402 drivers/infiniband/hw/qedr/verbs.c if (qp->qp_type == IB_QPT_GSI) qp 3405 drivers/infiniband/hw/qedr/verbs.c spin_lock_irqsave(&qp->q_lock, flags); qp 3408 drivers/infiniband/hw/qedr/verbs.c if ((qp->state != QED_ROCE_QP_STATE_RTS) && qp 3409 drivers/infiniband/hw/qedr/verbs.c (qp->state != QED_ROCE_QP_STATE_ERR) && qp 3410 drivers/infiniband/hw/qedr/verbs.c (qp->state != QED_ROCE_QP_STATE_SQD)) { qp 3411 drivers/infiniband/hw/qedr/verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 3415 drivers/infiniband/hw/qedr/verbs.c qp->icid, qp->state); qp 3425 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; qp 3427 drivers/infiniband/hw/qedr/verbs.c qedr_inc_sw_prod(&qp->sq); qp 3429 drivers/infiniband/hw/qedr/verbs.c qp->sq.db_data.data.value++; qp 3448 drivers/infiniband/hw/qedr/verbs.c writel(qp->sq.db_data.raw, qp->sq.db); qp 3450 drivers/infiniband/hw/qedr/verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 3554 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp = get_qedr_qp(ibqp); qp 3555 drivers/infiniband/hw/qedr/verbs.c struct qedr_dev *dev = qp->dev; qp 3559 drivers/infiniband/hw/qedr/verbs.c if (qp->qp_type == IB_QPT_GSI) qp 3562 drivers/infiniband/hw/qedr/verbs.c spin_lock_irqsave(&qp->q_lock, flags); qp 3564 drivers/infiniband/hw/qedr/verbs.c if (qp->state == QED_ROCE_QP_STATE_RESET) { qp 3565 drivers/infiniband/hw/qedr/verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 3573 drivers/infiniband/hw/qedr/verbs.c if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < qp 3575 drivers/infiniband/hw/qedr/verbs.c wr->num_sge > qp->rq.max_sges) { qp 3577 drivers/infiniband/hw/qedr/verbs.c qed_chain_get_elem_left_u32(&qp->rq.pbl), qp 3579 drivers/infiniband/hw/qedr/verbs.c qp->rq.max_sges); qp 3587 drivers/infiniband/hw/qedr/verbs.c qed_chain_produce(&qp->rq.pbl); qp 3610 drivers/infiniband/hw/qedr/verbs.c qed_chain_produce(&qp->rq.pbl); qp 3622 drivers/infiniband/hw/qedr/verbs.c qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; qp 3623 drivers/infiniband/hw/qedr/verbs.c qp->rqe_wr_id[qp->rq.prod].wqe_size = i; qp 3625 drivers/infiniband/hw/qedr/verbs.c qedr_inc_sw_prod(&qp->rq); qp 3636 drivers/infiniband/hw/qedr/verbs.c qp->rq.db_data.data.value++; qp 3638 drivers/infiniband/hw/qedr/verbs.c writel(qp->rq.db_data.raw, qp->rq.db); qp 3641 drivers/infiniband/hw/qedr/verbs.c writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); qp 3647 drivers/infiniband/hw/qedr/verbs.c spin_unlock_irqrestore(&qp->q_lock, flags); qp 3663 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp; qp 3665 drivers/infiniband/hw/qedr/verbs.c qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, qp 3668 drivers/infiniband/hw/qedr/verbs.c return qp; qp 3691 drivers/infiniband/hw/qedr/verbs.c static inline void qedr_chk_if_fmr(struct qedr_qp *qp) qp 3693 drivers/infiniband/hw/qedr/verbs.c if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) qp 3694 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; qp 3697 drivers/infiniband/hw/qedr/verbs.c static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, qp 3704 drivers/infiniband/hw/qedr/verbs.c while (num_entries && qp->sq.wqe_cons != hw_cons) { qp 3705 drivers/infiniband/hw/qedr/verbs.c if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { qp 3706 drivers/infiniband/hw/qedr/verbs.c qedr_chk_if_fmr(qp); qp 3715 drivers/infiniband/hw/qedr/verbs.c wc->src_qp = qp->id; qp 3716 drivers/infiniband/hw/qedr/verbs.c wc->qp = &qp->ibqp; qp 3718 drivers/infiniband/hw/qedr/verbs.c wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; qp 3719 drivers/infiniband/hw/qedr/verbs.c wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; qp 3723 drivers/infiniband/hw/qedr/verbs.c wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; qp 3730 drivers/infiniband/hw/qedr/verbs.c qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; qp 3734 drivers/infiniband/hw/qedr/verbs.c wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; qp 3744 drivers/infiniband/hw/qedr/verbs.c while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) qp 3745 drivers/infiniband/hw/qedr/verbs.c qed_chain_consume(&qp->sq.pbl); qp 3746 drivers/infiniband/hw/qedr/verbs.c qedr_inc_sw_cons(&qp->sq); qp 3753 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp, struct qedr_cq *cq, qp 3761 drivers/infiniband/hw/qedr/verbs.c cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, qp 3765 drivers/infiniband/hw/qedr/verbs.c if (qp->state != QED_ROCE_QP_STATE_ERR) qp 3768 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3769 drivers/infiniband/hw/qedr/verbs.c cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, qp 3774 drivers/infiniband/hw/qedr/verbs.c qp->state = QED_ROCE_QP_STATE_ERR; qp 3775 drivers/infiniband/hw/qedr/verbs.c cnt = process_req(dev, qp, cq, num_entries, wc, qp 3786 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3792 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3798 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3804 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3810 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3816 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3822 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3828 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3834 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3840 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3846 drivers/infiniband/hw/qedr/verbs.c cq->icid, qp->icid); qp 3849 drivers/infiniband/hw/qedr/verbs.c cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, qp 3909 drivers/infiniband/hw/qedr/verbs.c static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, qp 3933 drivers/infiniband/hw/qedr/verbs.c wc->src_qp = qp->id; qp 3934 drivers/infiniband/hw/qedr/verbs.c wc->qp = &qp->ibqp; qp 3938 drivers/infiniband/hw/qedr/verbs.c static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, qp 3942 drivers/infiniband/hw/qedr/verbs.c struct qedr_srq *srq = qp->srq; qp 3953 drivers/infiniband/hw/qedr/verbs.c wc->src_qp = qp->id; qp 3954 drivers/infiniband/hw/qedr/verbs.c wc->qp = &qp->ibqp; qp 3957 drivers/infiniband/hw/qedr/verbs.c __process_resp_one(dev, qp, cq, wc, resp, wr_id); qp 3963 drivers/infiniband/hw/qedr/verbs.c static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, qp 3967 drivers/infiniband/hw/qedr/verbs.c u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; qp 3969 drivers/infiniband/hw/qedr/verbs.c __process_resp_one(dev, qp, cq, wc, resp, wr_id); qp 3971 drivers/infiniband/hw/qedr/verbs.c while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) qp 3972 drivers/infiniband/hw/qedr/verbs.c qed_chain_consume(&qp->rq.pbl); qp 3973 drivers/infiniband/hw/qedr/verbs.c qedr_inc_sw_cons(&qp->rq); qp 3978 drivers/infiniband/hw/qedr/verbs.c static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, qp 3983 drivers/infiniband/hw/qedr/verbs.c while (num_entries && qp->rq.wqe_cons != hw_cons) { qp 3988 drivers/infiniband/hw/qedr/verbs.c wc->src_qp = qp->id; qp 3990 drivers/infiniband/hw/qedr/verbs.c wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; qp 3991 drivers/infiniband/hw/qedr/verbs.c wc->qp = &qp->ibqp; qp 3995 drivers/infiniband/hw/qedr/verbs.c while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) qp 3996 drivers/infiniband/hw/qedr/verbs.c qed_chain_consume(&qp->rq.pbl); qp 3997 drivers/infiniband/hw/qedr/verbs.c qedr_inc_sw_cons(&qp->rq); qp 4003 drivers/infiniband/hw/qedr/verbs.c static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, qp 4006 drivers/infiniband/hw/qedr/verbs.c if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { qp 4012 drivers/infiniband/hw/qedr/verbs.c static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, qp 4019 drivers/infiniband/hw/qedr/verbs.c cnt = process_resp_one_srq(dev, qp, cq, wc, resp); qp 4025 drivers/infiniband/hw/qedr/verbs.c static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, qp 4033 drivers/infiniband/hw/qedr/verbs.c cnt = process_resp_flush(qp, cq, num_entries, wc, qp 4035 drivers/infiniband/hw/qedr/verbs.c try_consume_resp_cqe(cq, qp, resp, update); qp 4037 drivers/infiniband/hw/qedr/verbs.c cnt = process_resp_one(dev, qp, cq, wc, resp); qp 4045 drivers/infiniband/hw/qedr/verbs.c static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, qp 4048 drivers/infiniband/hw/qedr/verbs.c if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { qp 4078 drivers/infiniband/hw/qedr/verbs.c struct qedr_qp *qp; qp 4084 drivers/infiniband/hw/qedr/verbs.c qp = cqe_get_qp(cqe); qp 4085 drivers/infiniband/hw/qedr/verbs.c if (!qp) { qp 4090 drivers/infiniband/hw/qedr/verbs.c wc->qp = &qp->ibqp; qp 4094 drivers/infiniband/hw/qedr/verbs.c cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, qp 4096 drivers/infiniband/hw/qedr/verbs.c try_consume_req_cqe(cq, qp, &cqe->req, &update); qp 4099 drivers/infiniband/hw/qedr/verbs.c cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, qp 4103 drivers/infiniband/hw/qedr/verbs.c cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, qp 249 drivers/infiniband/hw/qib/qib.h struct rvt_qp *qp; qp 1516 drivers/infiniband/hw/qib/qib.h void qib_stop_send_queue(struct rvt_qp *qp); qp 1517 drivers/infiniband/hw/qib/qib.h void qib_quiesce_qp(struct rvt_qp *qp); qp 1518 drivers/infiniband/hw/qib/qib.h void qib_flush_qp_waiters(struct rvt_qp *qp); qp 1520 drivers/infiniband/hw/qib/qib.h u32 qib_mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu); qp 1521 drivers/infiniband/hw/qib/qib.h void qib_notify_error_qp(struct rvt_qp *qp); qp 1522 drivers/infiniband/hw/qib/qib.h int qib_get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 306 drivers/infiniband/hw/qib/qib_driver.c struct rvt_qp *qp = NULL; qp 351 drivers/infiniband/hw/qib/qib_driver.c qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); qp 352 drivers/infiniband/hw/qib/qib_driver.c if (!qp) { qp 361 drivers/infiniband/hw/qib/qib_driver.c spin_lock(&qp->r_lock); qp 364 drivers/infiniband/hw/qib/qib_driver.c if (!(ib_rvt_state_ops[qp->state] & qp 370 drivers/infiniband/hw/qib/qib_driver.c switch (qp->ibqp.qp_type) { qp 376 drivers/infiniband/hw/qib/qib_driver.c qp, qp 384 drivers/infiniband/hw/qib/qib_driver.c diff = qib_cmp24(psn, qp->r_psn); qp 385 drivers/infiniband/hw/qib/qib_driver.c if (!qp->r_nak_state && diff >= 0) { qp 387 drivers/infiniband/hw/qib/qib_driver.c qp->r_nak_state = qp 390 drivers/infiniband/hw/qib/qib_driver.c qp->r_ack_psn = qp->r_psn; qp 399 drivers/infiniband/hw/qib/qib_driver.c if (list_empty(&qp->rspwait)) { qp 400 drivers/infiniband/hw/qib/qib_driver.c qp->r_flags |= qp 402 drivers/infiniband/hw/qib/qib_driver.c rvt_get_qp(qp); qp 404 drivers/infiniband/hw/qib/qib_driver.c &qp->rspwait, qp 420 drivers/infiniband/hw/qib/qib_driver.c spin_unlock(&qp->r_lock); qp 453 drivers/infiniband/hw/qib/qib_driver.c struct rvt_qp *qp, *nqp; qp 553 drivers/infiniband/hw/qib/qib_driver.c list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { qp 554 drivers/infiniband/hw/qib/qib_driver.c list_del_init(&qp->rspwait); qp 555 drivers/infiniband/hw/qib/qib_driver.c if (qp->r_flags & RVT_R_RSP_NAK) { qp 556 drivers/infiniband/hw/qib/qib_driver.c qp->r_flags &= ~RVT_R_RSP_NAK; qp 557 drivers/infiniband/hw/qib/qib_driver.c qib_send_rc_ack(qp); qp 559 drivers/infiniband/hw/qib/qib_driver.c if (qp->r_flags & RVT_R_RSP_SEND) { qp 562 drivers/infiniband/hw/qib/qib_driver.c qp->r_flags &= ~RVT_R_RSP_SEND; qp 563 drivers/infiniband/hw/qib/qib_driver.c spin_lock_irqsave(&qp->s_lock, flags); qp 564 drivers/infiniband/hw/qib/qib_driver.c if (ib_rvt_state_ops[qp->state] & qp 566 drivers/infiniband/hw/qib/qib_driver.c qib_schedule_send(qp); qp 567 drivers/infiniband/hw/qib/qib_driver.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 569 drivers/infiniband/hw/qib/qib_driver.c rvt_put_qp(qp); qp 175 drivers/infiniband/hw/qib/qib_mad.h __be32 qp; qp 224 drivers/infiniband/hw/qib/qib_qp.c if (rcu_dereference(ibp->rvp.qp[0])) qp 226 drivers/infiniband/hw/qib/qib_qp.c if (rcu_dereference(ibp->rvp.qp[1])) qp 233 drivers/infiniband/hw/qib/qib_qp.c void qib_notify_qp_reset(struct rvt_qp *qp) qp 235 drivers/infiniband/hw/qib/qib_qp.c struct qib_qp_priv *priv = qp->priv; qp 240 drivers/infiniband/hw/qib/qib_qp.c void qib_notify_error_qp(struct rvt_qp *qp) qp 242 drivers/infiniband/hw/qib/qib_qp.c struct qib_qp_priv *priv = qp->priv; qp 243 drivers/infiniband/hw/qib/qib_qp.c struct qib_ibdev *dev = to_idev(qp->ibqp.device); qp 246 drivers/infiniband/hw/qib/qib_qp.c if (!list_empty(&priv->iowait) && !(qp->s_flags & RVT_S_BUSY)) { qp 247 drivers/infiniband/hw/qib/qib_qp.c qp->s_flags &= ~RVT_S_ANY_WAIT_IO; qp 252 drivers/infiniband/hw/qib/qib_qp.c if (!(qp->s_flags & RVT_S_BUSY)) { qp 253 drivers/infiniband/hw/qib/qib_qp.c qp->s_hdrwords = 0; qp 254 drivers/infiniband/hw/qib/qib_qp.c if (qp->s_rdma_mr) { qp 255 drivers/infiniband/hw/qib/qib_qp.c rvt_put_mr(qp->s_rdma_mr); qp 256 drivers/infiniband/hw/qib/qib_qp.c qp->s_rdma_mr = NULL; qp 291 drivers/infiniband/hw/qib/qib_qp.c int qib_get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 294 drivers/infiniband/hw/qib/qib_qp.c int mtu, pmtu, pidx = qp->port_num - 1; qp 314 drivers/infiniband/hw/qib/qib_qp.c u32 qib_mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu) qp 319 drivers/infiniband/hw/qib/qib_qp.c void *qib_qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp) qp 326 drivers/infiniband/hw/qib/qib_qp.c priv->owner = qp; qp 340 drivers/infiniband/hw/qib/qib_qp.c void qib_qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) qp 342 drivers/infiniband/hw/qib/qib_qp.c struct qib_qp_priv *priv = qp->priv; qp 348 drivers/infiniband/hw/qib/qib_qp.c void qib_stop_send_queue(struct rvt_qp *qp) qp 350 drivers/infiniband/hw/qib/qib_qp.c struct qib_qp_priv *priv = qp->priv; qp 355 drivers/infiniband/hw/qib/qib_qp.c void qib_quiesce_qp(struct rvt_qp *qp) qp 357 drivers/infiniband/hw/qib/qib_qp.c struct qib_qp_priv *priv = qp->priv; qp 366 drivers/infiniband/hw/qib/qib_qp.c void qib_flush_qp_waiters(struct rvt_qp *qp) qp 368 drivers/infiniband/hw/qib/qib_qp.c struct qib_qp_priv *priv = qp->priv; qp 369 drivers/infiniband/hw/qib/qib_qp.c struct qib_ibdev *dev = to_idev(qp->ibqp.device); qp 385 drivers/infiniband/hw/qib/qib_qp.c int qib_check_send_wqe(struct rvt_qp *qp, qp 390 drivers/infiniband/hw/qib/qib_qp.c switch (qp->ibqp.qp_type) { qp 395 drivers/infiniband/hw/qib/qib_qp.c if (wqe->length > qp->pmtu) qp 427 drivers/infiniband/hw/qib/qib_qp.c struct rvt_qp *qp = iter->qp; qp 428 drivers/infiniband/hw/qib/qib_qp.c struct qib_qp_priv *priv = qp->priv; qp 430 drivers/infiniband/hw/qib/qib_qp.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 434 drivers/infiniband/hw/qib/qib_qp.c qp->ibqp.qp_num, qp 435 drivers/infiniband/hw/qib/qib_qp.c qp_type_str[qp->ibqp.qp_type], qp 436 drivers/infiniband/hw/qib/qib_qp.c qp->state, qp 438 drivers/infiniband/hw/qib/qib_qp.c qp->s_hdrwords, qp 439 drivers/infiniband/hw/qib/qib_qp.c qp->s_flags, qp 442 drivers/infiniband/hw/qib/qib_qp.c qp->timeout, qp 444 drivers/infiniband/hw/qib/qib_qp.c qp->s_lsn, qp 445 drivers/infiniband/hw/qib/qib_qp.c qp->s_last_psn, qp 446 drivers/infiniband/hw/qib/qib_qp.c qp->s_psn, qp->s_next_psn, qp 447 drivers/infiniband/hw/qib/qib_qp.c qp->s_sending_psn, qp->s_sending_hpsn, qp 448 drivers/infiniband/hw/qib/qib_qp.c qp->s_last, qp->s_acked, qp->s_cur, qp 449 drivers/infiniband/hw/qib/qib_qp.c qp->s_tail, qp->s_head, qp->s_size, qp 450 drivers/infiniband/hw/qib/qib_qp.c qp->remote_qpn, qp 451 drivers/infiniband/hw/qib/qib_qp.c rdma_ah_get_dlid(&qp->remote_ah_attr)); qp 62 drivers/infiniband/hw/qib/qib_rc.c static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp, qp 72 drivers/infiniband/hw/qib/qib_rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) qp 78 drivers/infiniband/hw/qib/qib_rc.c switch (qp->s_ack_state) { qp 81 drivers/infiniband/hw/qib/qib_rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 93 drivers/infiniband/hw/qib/qib_rc.c if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) qp 94 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail_ack_queue = 0; qp 99 drivers/infiniband/hw/qib/qib_rc.c if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { qp 100 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_flags & RVT_S_ACK_PENDING) qp 105 drivers/infiniband/hw/qib/qib_rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 115 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail_ack_queue = qp->r_head_ack_queue; qp 119 drivers/infiniband/hw/qib/qib_rc.c qp->s_rdma_mr = e->rdma_sge.mr; qp 120 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_rdma_mr) qp 121 drivers/infiniband/hw/qib/qib_rc.c rvt_get_mr(qp->s_rdma_mr); qp 122 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_rdma_sge.sge = e->rdma_sge; qp 123 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_rdma_sge.num_sge = 1; qp 124 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur_sge = &qp->s_ack_rdma_sge; qp 127 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); qp 129 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); qp 132 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 134 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_rdma_psn = e->psn; qp 135 drivers/infiniband/hw/qib/qib_rc.c bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; qp 138 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur_sge = NULL; qp 140 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); qp 141 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.at.aeth = rvt_compute_aeth(qp); qp 147 drivers/infiniband/hw/qib/qib_rc.c bth0 = qp->s_ack_state << 24; qp 151 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); qp 154 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur_sge = &qp->s_ack_rdma_sge; qp 155 drivers/infiniband/hw/qib/qib_rc.c qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; qp 156 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_rdma_mr) qp 157 drivers/infiniband/hw/qib/qib_rc.c rvt_get_mr(qp->s_rdma_mr); qp 158 drivers/infiniband/hw/qib/qib_rc.c len = qp->s_ack_rdma_sge.sge.sge_length; qp 162 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 164 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); qp 165 drivers/infiniband/hw/qib/qib_rc.c e = &qp->s_ack_queue[qp->s_tail_ack_queue]; qp 168 drivers/infiniband/hw/qib/qib_rc.c bth0 = qp->s_ack_state << 24; qp 169 drivers/infiniband/hw/qib/qib_rc.c bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; qp 180 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(SEND_ONLY); qp 181 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~RVT_S_ACK_PENDING; qp 182 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur_sge = NULL; qp 183 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_nak_state) qp 185 drivers/infiniband/hw/qib/qib_rc.c cpu_to_be32((qp->r_msn & IB_MSN_MASK) | qp 186 drivers/infiniband/hw/qib/qib_rc.c (qp->s_nak_state << qp 189 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 193 drivers/infiniband/hw/qib/qib_rc.c bth2 = qp->s_ack_psn & QIB_PSN_MASK; qp 195 drivers/infiniband/hw/qib/qib_rc.c qp->s_rdma_ack_cnt++; qp 196 drivers/infiniband/hw/qib/qib_rc.c qp->s_hdrwords = hwords; qp 197 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur_size = len; qp 198 drivers/infiniband/hw/qib/qib_rc.c qib_make_ruc_header(qp, ohdr, bth0, bth2); qp 202 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(ACKNOWLEDGE); qp 203 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING); qp 215 drivers/infiniband/hw/qib/qib_rc.c int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags) qp 217 drivers/infiniband/hw/qib/qib_rc.c struct qib_qp_priv *priv = qp->priv; qp 218 drivers/infiniband/hw/qib/qib_rc.c struct qib_ibdev *dev = to_idev(qp->ibqp.device); qp 226 drivers/infiniband/hw/qib/qib_rc.c u32 pmtu = qp->pmtu; qp 232 drivers/infiniband/hw/qib/qib_rc.c if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) qp 236 drivers/infiniband/hw/qib/qib_rc.c if ((qp->s_flags & RVT_S_RESP_PENDING) && qp 237 drivers/infiniband/hw/qib/qib_rc.c qib_make_rc_ack(dev, qp, ohdr, pmtu)) qp 240 drivers/infiniband/hw/qib/qib_rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { qp 241 drivers/infiniband/hw/qib/qib_rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) qp 244 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_last == READ_ONCE(qp->s_head)) qp 248 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_WAIT_DMA; qp 251 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 252 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? qp 258 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK)) qp 261 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { qp 262 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { qp 263 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_WAIT_PSN; qp 266 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_psn = qp->s_psn; qp 267 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_hpsn = qp->s_psn - 1; qp 275 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 276 drivers/infiniband/hw/qib/qib_rc.c switch (qp->s_state) { qp 278 drivers/infiniband/hw/qib/qib_rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) qp 288 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_cur == qp->s_tail) { qp 290 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_tail == READ_ONCE(qp->s_head)) qp 297 drivers/infiniband/hw/qib/qib_rc.c qp->s_num_rd_atomic) { qp 298 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_WAIT_FENCE; qp 302 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->psn; qp 310 drivers/infiniband/hw/qib/qib_rc.c ss = &qp->s_sge; qp 311 drivers/infiniband/hw/qib/qib_rc.c bth2 = qp->s_psn & QIB_PSN_MASK; qp 316 drivers/infiniband/hw/qib/qib_rc.c if (!rvt_rc_credit_avail(qp, wqe)) qp 319 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_FIRST); qp 324 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_ONLY); qp 326 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); qp 334 drivers/infiniband/hw/qib/qib_rc.c if (++qp->s_cur == qp->s_size) qp 335 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 339 drivers/infiniband/hw/qib/qib_rc.c if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 340 drivers/infiniband/hw/qib/qib_rc.c qp->s_lsn++; qp 344 drivers/infiniband/hw/qib/qib_rc.c if (!rvt_rc_credit_avail(qp, wqe)) qp 354 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_WRITE_FIRST); qp 359 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_WRITE_ONLY); qp 361 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); qp 370 drivers/infiniband/hw/qib/qib_rc.c if (++qp->s_cur == qp->s_size) qp 371 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 380 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_num_rd_atomic >= qp 381 drivers/infiniband/hw/qib/qib_rc.c qp->s_max_rd_atomic) { qp 382 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_WAIT_RDMAR; qp 385 drivers/infiniband/hw/qib/qib_rc.c qp->s_num_rd_atomic++; qp 386 drivers/infiniband/hw/qib/qib_rc.c if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 387 drivers/infiniband/hw/qib/qib_rc.c qp->s_lsn++; qp 395 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_READ_REQUEST); qp 400 drivers/infiniband/hw/qib/qib_rc.c if (++qp->s_cur == qp->s_size) qp 401 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 411 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_num_rd_atomic >= qp 412 drivers/infiniband/hw/qib/qib_rc.c qp->s_max_rd_atomic) { qp 413 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_WAIT_RDMAR; qp 416 drivers/infiniband/hw/qib/qib_rc.c qp->s_num_rd_atomic++; qp 417 drivers/infiniband/hw/qib/qib_rc.c if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) qp 418 drivers/infiniband/hw/qib/qib_rc.c qp->s_lsn++; qp 421 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(COMPARE_SWAP); qp 427 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(FETCH_ADD); qp 440 drivers/infiniband/hw/qib/qib_rc.c if (++qp->s_cur == qp->s_size) qp 441 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 447 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.sge = wqe->sg_list[0]; qp 448 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.sg_list = wqe->sg_list + 1; qp 449 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.num_sge = wqe->wr.num_sge; qp 450 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.total_len = wqe->length; qp 451 drivers/infiniband/hw/qib/qib_rc.c qp->s_len = wqe->length; qp 453 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail++; qp 454 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_tail >= qp->s_size) qp 455 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail = 0; qp 458 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->lpsn + 1; qp 460 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn++; qp 473 drivers/infiniband/hw/qib/qib_rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); qp 476 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_MIDDLE); qp 479 drivers/infiniband/hw/qib/qib_rc.c bth2 = qp->s_psn++ & QIB_PSN_MASK; qp 480 drivers/infiniband/hw/qib/qib_rc.c ss = &qp->s_sge; qp 481 drivers/infiniband/hw/qib/qib_rc.c len = qp->s_len; qp 487 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_LAST); qp 489 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); qp 497 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur++; qp 498 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_cur >= qp->s_size) qp 499 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 512 drivers/infiniband/hw/qib/qib_rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); qp 515 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_WRITE_MIDDLE); qp 518 drivers/infiniband/hw/qib/qib_rc.c bth2 = qp->s_psn++ & QIB_PSN_MASK; qp 519 drivers/infiniband/hw/qib/qib_rc.c ss = &qp->s_sge; qp 520 drivers/infiniband/hw/qib/qib_rc.c len = qp->s_len; qp 526 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_WRITE_LAST); qp 528 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); qp 536 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur++; qp 537 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_cur >= qp->s_size) qp 538 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 551 drivers/infiniband/hw/qib/qib_rc.c len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; qp 557 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_READ_REQUEST); qp 559 drivers/infiniband/hw/qib/qib_rc.c bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; qp 560 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->lpsn + 1; qp 563 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur++; qp 564 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_cur == qp->s_size) qp 565 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 568 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_hpsn = bth2; qp 572 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_flags & RVT_S_SEND_ONE) { qp 573 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~RVT_S_SEND_ONE; qp 574 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_WAIT_ACK; qp 577 drivers/infiniband/hw/qib/qib_rc.c qp->s_len -= len; qp 578 drivers/infiniband/hw/qib/qib_rc.c qp->s_hdrwords = hwords; qp 579 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur_sge = ss; qp 580 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur_size = len; qp 581 drivers/infiniband/hw/qib/qib_rc.c qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); qp 585 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~RVT_S_BUSY; qp 597 drivers/infiniband/hw/qib/qib_rc.c void qib_send_rc_ack(struct rvt_qp *qp) qp 599 drivers/infiniband/hw/qib/qib_rc.c struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 600 drivers/infiniband/hw/qib/qib_rc.c struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 613 drivers/infiniband/hw/qib/qib_rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 615 drivers/infiniband/hw/qib/qib_rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) qp 619 drivers/infiniband/hw/qib/qib_rc.c if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt) qp 627 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & qp 630 drivers/infiniband/hw/qib/qib_rc.c rdma_ah_read_grh(&qp->remote_ah_attr), qp 636 drivers/infiniband/hw/qib/qib_rc.c bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); qp 637 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_mig_state == IB_MIG_MIGRATED) qp 639 drivers/infiniband/hw/qib/qib_rc.c if (qp->r_nak_state) qp 640 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | qp 641 drivers/infiniband/hw/qib/qib_rc.c (qp->r_nak_state << qp 644 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.aeth = rvt_compute_aeth(qp); qp 645 drivers/infiniband/hw/qib/qib_rc.c lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 | qp 646 drivers/infiniband/hw/qib/qib_rc.c rdma_ah_get_sl(&qp->remote_ah_attr) << 4; qp 648 drivers/infiniband/hw/qib/qib_rc.c hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); qp 651 drivers/infiniband/hw/qib/qib_rc.c rdma_ah_get_path_bits(&qp->remote_ah_attr)); qp 653 drivers/infiniband/hw/qib/qib_rc.c ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); qp 654 drivers/infiniband/hw/qib/qib_rc.c ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); qp 656 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 663 drivers/infiniband/hw/qib/qib_rc.c qp->s_srate, lrh0 >> 12); qp 676 drivers/infiniband/hw/qib/qib_rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 711 drivers/infiniband/hw/qib/qib_rc.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 713 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; qp 714 drivers/infiniband/hw/qib/qib_rc.c qp->s_nak_state = qp->r_nak_state; qp 715 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_psn = qp->r_ack_psn; qp 718 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 721 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 735 drivers/infiniband/hw/qib/qib_rc.c static void reset_psn(struct rvt_qp *qp, u32 psn) qp 737 drivers/infiniband/hw/qib/qib_rc.c u32 n = qp->s_acked; qp 738 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); qp 741 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = n; qp 748 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_LAST); qp 757 drivers/infiniband/hw/qib/qib_rc.c if (++n == qp->s_size) qp 759 drivers/infiniband/hw/qib/qib_rc.c if (n == qp->s_tail) qp 761 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, n); qp 765 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = n; qp 771 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_LAST); qp 785 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); qp 790 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_READ_RESPONSE_LAST); qp 794 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); qp 802 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_LAST); qp 805 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = psn; qp 811 drivers/infiniband/hw/qib/qib_rc.c if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && qp 812 drivers/infiniband/hw/qib/qib_rc.c (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) qp 813 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_WAIT_PSN; qp 820 drivers/infiniband/hw/qib/qib_rc.c void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait) qp 822 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 825 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_retry == 0) { qp 826 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_mig_state == IB_MIG_ARMED) { qp 827 drivers/infiniband/hw/qib/qib_rc.c qib_migrate_qp(qp); qp 828 drivers/infiniband/hw/qib/qib_rc.c qp->s_retry = qp->s_retry_cnt; qp 829 drivers/infiniband/hw/qib/qib_rc.c } else if (qp->s_last == qp->s_acked) { qp 830 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); qp 831 drivers/infiniband/hw/qib/qib_rc.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 836 drivers/infiniband/hw/qib/qib_rc.c qp->s_retry--; qp 838 drivers/infiniband/hw/qib/qib_rc.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 842 drivers/infiniband/hw/qib/qib_rc.c ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; qp 844 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | qp 848 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_SEND_ONE; qp 849 drivers/infiniband/hw/qib/qib_rc.c reset_psn(qp, psn); qp 856 drivers/infiniband/hw/qib/qib_rc.c static void reset_sending_psn(struct rvt_qp *qp, u32 psn) qp 859 drivers/infiniband/hw/qib/qib_rc.c u32 n = qp->s_last; qp 863 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, n); qp 866 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_psn = wqe->lpsn + 1; qp 868 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_psn = psn + 1; qp 871 drivers/infiniband/hw/qib/qib_rc.c if (++n == qp->s_size) qp 873 drivers/infiniband/hw/qib/qib_rc.c if (n == qp->s_tail) qp 881 drivers/infiniband/hw/qib/qib_rc.c void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr) qp 888 drivers/infiniband/hw/qib/qib_rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) qp 900 drivers/infiniband/hw/qib/qib_rc.c WARN_ON(!qp->s_rdma_ack_cnt); qp 901 drivers/infiniband/hw/qib/qib_rc.c qp->s_rdma_ack_cnt--; qp 906 drivers/infiniband/hw/qib/qib_rc.c reset_sending_psn(qp, psn); qp 912 drivers/infiniband/hw/qib/qib_rc.c if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && qp 913 drivers/infiniband/hw/qib/qib_rc.c !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) && qp 914 drivers/infiniband/hw/qib/qib_rc.c (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) qp 915 drivers/infiniband/hw/qib/qib_rc.c rvt_add_retry_timer(qp); qp 917 drivers/infiniband/hw/qib/qib_rc.c while (qp->s_last != qp->s_acked) { qp 918 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 919 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && qp 920 drivers/infiniband/hw/qib/qib_rc.c qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) qp 922 drivers/infiniband/hw/qib/qib_rc.c rvt_qp_complete_swqe(qp, qp 931 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_flags & RVT_S_WAIT_PSN && qp 932 drivers/infiniband/hw/qib/qib_rc.c qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { qp 933 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~RVT_S_WAIT_PSN; qp 934 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_psn = qp->s_psn; qp 935 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_hpsn = qp->s_psn - 1; qp 936 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 940 drivers/infiniband/hw/qib/qib_rc.c static inline void update_last_psn(struct rvt_qp *qp, u32 psn) qp 942 drivers/infiniband/hw/qib/qib_rc.c qp->s_last_psn = psn; qp 950 drivers/infiniband/hw/qib/qib_rc.c static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, qp 959 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || qp 960 drivers/infiniband/hw/qib/qib_rc.c qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) qp 961 drivers/infiniband/hw/qib/qib_rc.c rvt_qp_complete_swqe(qp, qp 968 drivers/infiniband/hw/qib/qib_rc.c qp->s_retry = qp->s_retry_cnt; qp 969 drivers/infiniband/hw/qib/qib_rc.c update_last_psn(qp, wqe->lpsn); qp 976 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_acked == qp->s_cur) { qp 977 drivers/infiniband/hw/qib/qib_rc.c if (++qp->s_cur >= qp->s_size) qp 978 drivers/infiniband/hw/qib/qib_rc.c qp->s_cur = 0; qp 979 drivers/infiniband/hw/qib/qib_rc.c qp->s_acked = qp->s_cur; qp 980 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 981 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_acked != qp->s_tail) { qp 982 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_LAST); qp 983 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->psn; qp 986 drivers/infiniband/hw/qib/qib_rc.c if (++qp->s_acked >= qp->s_size) qp 987 drivers/infiniband/hw/qib/qib_rc.c qp->s_acked = 0; qp 988 drivers/infiniband/hw/qib/qib_rc.c if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) qp 989 drivers/infiniband/hw/qib/qib_rc.c qp->s_draining = 0; qp 990 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1006 drivers/infiniband/hw/qib/qib_rc.c static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, qp 1025 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1026 drivers/infiniband/hw/qib/qib_rc.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 1060 drivers/infiniband/hw/qib/qib_rc.c if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { qp 1061 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RDMAR_SEQ; qp 1062 drivers/infiniband/hw/qib/qib_rc.c qib_restart_rc(qp, qp->s_last_psn + 1, 0); qp 1063 drivers/infiniband/hw/qib/qib_rc.c if (list_empty(&qp->rspwait)) { qp 1064 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RSP_SEND; qp 1065 drivers/infiniband/hw/qib/qib_rc.c rvt_get_qp(qp); qp 1066 drivers/infiniband/hw/qib/qib_rc.c list_add_tail(&qp->rspwait, qp 1081 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_num_rd_atomic && qp 1085 drivers/infiniband/hw/qib/qib_rc.c qp->s_num_rd_atomic--; qp 1087 drivers/infiniband/hw/qib/qib_rc.c if ((qp->s_flags & RVT_S_WAIT_FENCE) && qp 1088 drivers/infiniband/hw/qib/qib_rc.c !qp->s_num_rd_atomic) { qp 1089 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~(RVT_S_WAIT_FENCE | qp 1091 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 1092 drivers/infiniband/hw/qib/qib_rc.c } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { qp 1093 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~(RVT_S_WAIT_RDMAR | qp 1095 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 1098 drivers/infiniband/hw/qib/qib_rc.c wqe = do_rc_completion(qp, wqe, ibp); qp 1099 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_acked == qp->s_tail) qp 1106 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_acked != qp->s_tail) { qp 1111 drivers/infiniband/hw/qib/qib_rc.c rvt_mod_retry_timer(qp); qp 1116 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(qp->s_psn, psn) <= 0) qp 1117 drivers/infiniband/hw/qib/qib_rc.c reset_psn(qp, psn + 1); qp 1120 drivers/infiniband/hw/qib/qib_rc.c rvt_stop_rc_timers(qp); qp 1121 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(qp->s_psn, psn) <= 0) { qp 1122 drivers/infiniband/hw/qib/qib_rc.c qp->s_state = OP(SEND_LAST); qp 1123 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = psn + 1; qp 1126 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_flags & RVT_S_WAIT_ACK) { qp 1127 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~RVT_S_WAIT_ACK; qp 1128 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 1130 drivers/infiniband/hw/qib/qib_rc.c rvt_get_credit(qp, aeth); qp 1131 drivers/infiniband/hw/qib/qib_rc.c qp->s_rnr_retry = qp->s_rnr_retry_cnt; qp 1132 drivers/infiniband/hw/qib/qib_rc.c qp->s_retry = qp->s_retry_cnt; qp 1133 drivers/infiniband/hw/qib/qib_rc.c update_last_psn(qp, psn); qp 1138 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_acked == qp->s_tail) qp 1140 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_flags & RVT_S_WAIT_RNR) qp 1142 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_rnr_retry == 0) { qp 1146 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_rnr_retry_cnt < 7) qp 1147 drivers/infiniband/hw/qib/qib_rc.c qp->s_rnr_retry--; qp 1150 drivers/infiniband/hw/qib/qib_rc.c update_last_psn(qp, psn - 1); qp 1152 drivers/infiniband/hw/qib/qib_rc.c ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; qp 1154 drivers/infiniband/hw/qib/qib_rc.c reset_psn(qp, psn); qp 1156 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); qp 1157 drivers/infiniband/hw/qib/qib_rc.c rvt_stop_rc_timers(qp); qp 1158 drivers/infiniband/hw/qib/qib_rc.c rvt_add_rnr_timer(qp, aeth); qp 1162 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_acked == qp->s_tail) qp 1165 drivers/infiniband/hw/qib/qib_rc.c update_last_psn(qp, psn - 1); qp 1176 drivers/infiniband/hw/qib/qib_rc.c qib_restart_rc(qp, psn, 0); qp 1177 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 1194 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_last == qp->s_acked) { qp 1195 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, status); qp 1196 drivers/infiniband/hw/qib/qib_rc.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 1204 drivers/infiniband/hw/qib/qib_rc.c qp->s_retry = qp->s_retry_cnt; qp 1205 drivers/infiniband/hw/qib/qib_rc.c qp->s_rnr_retry = qp->s_rnr_retry_cnt; qp 1215 drivers/infiniband/hw/qib/qib_rc.c rvt_stop_rc_timers(qp); qp 1223 drivers/infiniband/hw/qib/qib_rc.c static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn, qp 1229 drivers/infiniband/hw/qib/qib_rc.c rvt_stop_rc_timers(qp); qp 1231 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1238 drivers/infiniband/hw/qib/qib_rc.c wqe = do_rc_completion(qp, wqe, ibp); qp 1242 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RDMAR_SEQ; qp 1243 drivers/infiniband/hw/qib/qib_rc.c qib_restart_rc(qp, qp->s_last_psn + 1, 0); qp 1244 drivers/infiniband/hw/qib/qib_rc.c if (list_empty(&qp->rspwait)) { qp 1245 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RSP_SEND; qp 1246 drivers/infiniband/hw/qib/qib_rc.c rvt_get_qp(qp); qp 1247 drivers/infiniband/hw/qib/qib_rc.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 1270 drivers/infiniband/hw/qib/qib_rc.c struct rvt_qp *qp, qp 1289 drivers/infiniband/hw/qib/qib_rc.c if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && qp 1290 drivers/infiniband/hw/qib/qib_rc.c (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { qp 1296 drivers/infiniband/hw/qib/qib_rc.c if (!(qp->s_flags & RVT_S_BUSY)) { qp 1307 drivers/infiniband/hw/qib/qib_rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 1308 drivers/infiniband/hw/qib/qib_rc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) qp 1312 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0) qp 1316 drivers/infiniband/hw/qib/qib_rc.c diff = qib_cmp24(psn, qp->s_last_psn); qp 1322 drivers/infiniband/hw/qib/qib_rc.c rvt_get_credit(qp, aeth); qp 1331 drivers/infiniband/hw/qib/qib_rc.c if (qp->r_flags & RVT_R_RDMAR_SEQ) { qp 1332 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) qp 1334 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags &= ~RVT_R_RDMAR_SEQ; qp 1337 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(qp->s_acked == qp->s_tail)) qp 1339 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1351 drivers/infiniband/hw/qib/qib_rc.c if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || qp 1355 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1363 drivers/infiniband/hw/qib/qib_rc.c qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, qp 1369 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) qp 1376 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(pmtu >= qp->s_rdma_read_len)) qp 1383 drivers/infiniband/hw/qib/qib_rc.c rvt_mod_retry_timer(qp); qp 1384 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_flags & RVT_S_WAIT_ACK) { qp 1385 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags &= ~RVT_S_WAIT_ACK; qp 1386 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 1390 drivers/infiniband/hw/qib/qib_rc.c qp->s_retry = qp->s_retry_cnt; qp 1396 drivers/infiniband/hw/qib/qib_rc.c qp->s_rdma_read_len -= pmtu; qp 1397 drivers/infiniband/hw/qib/qib_rc.c update_last_psn(qp, psn); qp 1398 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1399 drivers/infiniband/hw/qib/qib_rc.c rvt_copy_sge(qp, &qp->s_rdma_read_sge, qp 1405 drivers/infiniband/hw/qib/qib_rc.c if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) qp 1421 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); qp 1422 drivers/infiniband/hw/qib/qib_rc.c qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, qp 1428 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) qp 1443 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(tlen != qp->s_rdma_read_len)) qp 1446 drivers/infiniband/hw/qib/qib_rc.c rvt_copy_sge(qp, &qp->s_rdma_read_sge, qp 1448 drivers/infiniband/hw/qib/qib_rc.c WARN_ON(qp->s_rdma_read_sge.num_sge); qp 1449 drivers/infiniband/hw/qib/qib_rc.c (void) do_rc_ack(qp, aeth, psn, qp 1459 drivers/infiniband/hw/qib/qib_rc.c rdma_seq_err(qp, ibp, psn, rcd); qp 1465 drivers/infiniband/hw/qib/qib_rc.c if (qp->s_last == qp->s_acked) { qp 1466 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, status); qp 1467 drivers/infiniband/hw/qib/qib_rc.c rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 1470 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1492 drivers/infiniband/hw/qib/qib_rc.c struct rvt_qp *qp, qp 1498 drivers/infiniband/hw/qib/qib_rc.c struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 1510 drivers/infiniband/hw/qib/qib_rc.c if (!qp->r_nak_state) { qp 1512 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = IB_NAK_PSN_ERROR; qp 1514 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = qp->r_psn; qp 1520 drivers/infiniband/hw/qib/qib_rc.c if (list_empty(&qp->rspwait)) { qp 1521 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RSP_NAK; qp 1522 drivers/infiniband/hw/qib/qib_rc.c rvt_get_qp(qp); qp 1523 drivers/infiniband/hw/qib/qib_rc.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 1549 drivers/infiniband/hw/qib/qib_rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 1551 drivers/infiniband/hw/qib/qib_rc.c for (i = qp->r_head_ack_queue; ; i = prev) { qp 1552 drivers/infiniband/hw/qib/qib_rc.c if (i == qp->s_tail_ack_queue) qp 1558 drivers/infiniband/hw/qib/qib_rc.c if (prev == qp->r_head_ack_queue) { qp 1562 drivers/infiniband/hw/qib/qib_rc.c e = &qp->s_ack_queue[prev]; qp 1568 drivers/infiniband/hw/qib/qib_rc.c if (prev == qp->s_tail_ack_queue && qp 1596 drivers/infiniband/hw/qib/qib_rc.c qp->pmtu; qp 1609 drivers/infiniband/hw/qib/qib_rc.c ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, qp 1621 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail_ack_queue = prev; qp 1634 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail_ack_queue = prev; qp 1649 drivers/infiniband/hw/qib/qib_rc.c if (i == qp->r_head_ack_queue) { qp 1650 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1651 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = 0; qp 1652 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = qp->r_psn - 1; qp 1660 drivers/infiniband/hw/qib/qib_rc.c if (!(qp->s_flags & RVT_S_RESP_PENDING)) { qp 1661 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1662 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = 0; qp 1663 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; qp 1670 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail_ack_queue = i; qp 1673 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(ACKNOWLEDGE); qp 1674 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_RESP_PENDING; qp 1675 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = 0; qp 1676 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 1679 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1687 drivers/infiniband/hw/qib/qib_rc.c static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n) qp 1694 drivers/infiniband/hw/qib/qib_rc.c qp->s_tail_ack_queue = next; qp 1695 drivers/infiniband/hw/qib/qib_rc.c qp->s_ack_state = OP(ACKNOWLEDGE); qp 1712 drivers/infiniband/hw/qib/qib_rc.c int has_grh, void *data, u32 tlen, struct rvt_qp *qp) qp 1721 drivers/infiniband/hw/qib/qib_rc.c u32 pmtu = qp->pmtu; qp 1737 drivers/infiniband/hw/qib/qib_rc.c if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) qp 1751 drivers/infiniband/hw/qib/qib_rc.c qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, qp 1757 drivers/infiniband/hw/qib/qib_rc.c diff = qib_cmp24(psn, qp->r_psn); qp 1759 drivers/infiniband/hw/qib/qib_rc.c if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) qp 1765 drivers/infiniband/hw/qib/qib_rc.c switch (qp->r_state) { qp 1798 drivers/infiniband/hw/qib/qib_rc.c if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) qp 1799 drivers/infiniband/hw/qib/qib_rc.c rvt_comm_est(qp); qp 1804 drivers/infiniband/hw/qib/qib_rc.c ret = rvt_get_rwqe(qp, false); qp 1809 drivers/infiniband/hw/qib/qib_rc.c qp->r_rcv_len = 0; qp 1817 drivers/infiniband/hw/qib/qib_rc.c qp->r_rcv_len += pmtu; qp 1818 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(qp->r_rcv_len > qp->r_len)) qp 1820 drivers/infiniband/hw/qib/qib_rc.c rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); qp 1825 drivers/infiniband/hw/qib/qib_rc.c ret = rvt_get_rwqe(qp, true); qp 1834 drivers/infiniband/hw/qib/qib_rc.c ret = rvt_get_rwqe(qp, false); qp 1839 drivers/infiniband/hw/qib/qib_rc.c qp->r_rcv_len = 0; qp 1863 drivers/infiniband/hw/qib/qib_rc.c wc.byte_len = tlen + qp->r_rcv_len; qp 1864 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(wc.byte_len > qp->r_len)) qp 1866 drivers/infiniband/hw/qib/qib_rc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); qp 1867 drivers/infiniband/hw/qib/qib_rc.c rvt_put_ss(&qp->r_sge); qp 1868 drivers/infiniband/hw/qib/qib_rc.c qp->r_msn++; qp 1869 drivers/infiniband/hw/qib/qib_rc.c if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) qp 1871 drivers/infiniband/hw/qib/qib_rc.c wc.wr_id = qp->r_wr_id; qp 1878 drivers/infiniband/hw/qib/qib_rc.c wc.qp = &qp->ibqp; qp 1879 drivers/infiniband/hw/qib/qib_rc.c wc.src_qp = qp->remote_qpn; qp 1880 drivers/infiniband/hw/qib/qib_rc.c wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr); qp 1881 drivers/infiniband/hw/qib/qib_rc.c wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); qp 1888 drivers/infiniband/hw/qib/qib_rc.c rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); qp 1894 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) qp 1899 drivers/infiniband/hw/qib/qib_rc.c qp->r_len = be32_to_cpu(reth->length); qp 1900 drivers/infiniband/hw/qib/qib_rc.c qp->r_rcv_len = 0; qp 1901 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.sg_list = NULL; qp 1902 drivers/infiniband/hw/qib/qib_rc.c if (qp->r_len != 0) { qp 1908 drivers/infiniband/hw/qib/qib_rc.c ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, qp 1912 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.num_sge = 1; qp 1914 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.num_sge = 0; qp 1915 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.sge.mr = NULL; qp 1916 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.sge.vaddr = NULL; qp 1917 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.sge.length = 0; qp 1918 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.sge.sge_length = 0; qp 1924 drivers/infiniband/hw/qib/qib_rc.c ret = rvt_get_rwqe(qp, true); qp 1928 drivers/infiniband/hw/qib/qib_rc.c rvt_put_ss(&qp->r_sge); qp 1941 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) qp 1943 drivers/infiniband/hw/qib/qib_rc.c next = qp->r_head_ack_queue + 1; qp 1947 drivers/infiniband/hw/qib/qib_rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 1948 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(next == qp->s_tail_ack_queue)) { qp 1949 drivers/infiniband/hw/qib/qib_rc.c if (!qp->s_ack_queue[next].sent) qp 1951 drivers/infiniband/hw/qib/qib_rc.c qib_update_ack_queue(qp, next); qp 1953 drivers/infiniband/hw/qib/qib_rc.c e = &qp->s_ack_queue[qp->r_head_ack_queue]; qp 1966 drivers/infiniband/hw/qib/qib_rc.c ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, qp 1974 drivers/infiniband/hw/qib/qib_rc.c qp->r_psn += rvt_div_mtu(qp, len - 1); qp 1984 drivers/infiniband/hw/qib/qib_rc.c e->lpsn = qp->r_psn; qp 1990 drivers/infiniband/hw/qib/qib_rc.c qp->r_msn++; qp 1991 drivers/infiniband/hw/qib/qib_rc.c qp->r_psn++; qp 1992 drivers/infiniband/hw/qib/qib_rc.c qp->r_state = opcode; qp 1993 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = 0; qp 1994 drivers/infiniband/hw/qib/qib_rc.c qp->r_head_ack_queue = next; qp 1997 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_RESP_PENDING; qp 1998 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 2013 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) qp 2015 drivers/infiniband/hw/qib/qib_rc.c next = qp->r_head_ack_queue + 1; qp 2018 drivers/infiniband/hw/qib/qib_rc.c spin_lock_irqsave(&qp->s_lock, flags); qp 2019 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(next == qp->s_tail_ack_queue)) { qp 2020 drivers/infiniband/hw/qib/qib_rc.c if (!qp->s_ack_queue[next].sent) qp 2022 drivers/infiniband/hw/qib/qib_rc.c qib_update_ack_queue(qp, next); qp 2024 drivers/infiniband/hw/qib/qib_rc.c e = &qp->s_ack_queue[qp->r_head_ack_queue]; qp 2035 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), qp 2040 drivers/infiniband/hw/qib/qib_rc.c maddr = (atomic64_t *) qp->r_sge.sge.vaddr; qp 2044 drivers/infiniband/hw/qib/qib_rc.c (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, qp 2047 drivers/infiniband/hw/qib/qib_rc.c rvt_put_mr(qp->r_sge.sge.mr); qp 2048 drivers/infiniband/hw/qib/qib_rc.c qp->r_sge.num_sge = 0; qp 2053 drivers/infiniband/hw/qib/qib_rc.c qp->r_msn++; qp 2054 drivers/infiniband/hw/qib/qib_rc.c qp->r_psn++; qp 2055 drivers/infiniband/hw/qib/qib_rc.c qp->r_state = opcode; qp 2056 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = 0; qp 2057 drivers/infiniband/hw/qib/qib_rc.c qp->r_head_ack_queue = next; qp 2060 drivers/infiniband/hw/qib/qib_rc.c qp->s_flags |= RVT_S_RESP_PENDING; qp 2061 drivers/infiniband/hw/qib/qib_rc.c qib_schedule_send(qp); qp 2070 drivers/infiniband/hw/qib/qib_rc.c qp->r_psn++; qp 2071 drivers/infiniband/hw/qib/qib_rc.c qp->r_state = opcode; qp 2072 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = psn; qp 2073 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = 0; qp 2080 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; qp 2081 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = qp->r_psn; qp 2083 drivers/infiniband/hw/qib/qib_rc.c if (list_empty(&qp->rspwait)) { qp 2084 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RSP_NAK; qp 2085 drivers/infiniband/hw/qib/qib_rc.c rvt_get_qp(qp); qp 2086 drivers/infiniband/hw/qib/qib_rc.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 2091 drivers/infiniband/hw/qib/qib_rc.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 2092 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; qp 2093 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = qp->r_psn; qp 2095 drivers/infiniband/hw/qib/qib_rc.c if (list_empty(&qp->rspwait)) { qp 2096 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RSP_NAK; qp 2097 drivers/infiniband/hw/qib/qib_rc.c rvt_get_qp(qp); qp 2098 drivers/infiniband/hw/qib/qib_rc.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 2103 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2105 drivers/infiniband/hw/qib/qib_rc.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 2106 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = IB_NAK_INVALID_REQUEST; qp 2107 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = qp->r_psn; qp 2109 drivers/infiniband/hw/qib/qib_rc.c if (list_empty(&qp->rspwait)) { qp 2110 drivers/infiniband/hw/qib/qib_rc.c qp->r_flags |= RVT_R_RSP_NAK; qp 2111 drivers/infiniband/hw/qib/qib_rc.c rvt_get_qp(qp); qp 2112 drivers/infiniband/hw/qib/qib_rc.c list_add_tail(&qp->rspwait, &rcd->qp_wait_list); qp 2117 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2119 drivers/infiniband/hw/qib/qib_rc.c rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); qp 2120 drivers/infiniband/hw/qib/qib_rc.c qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; qp 2121 drivers/infiniband/hw/qib/qib_rc.c qp->r_ack_psn = qp->r_psn; qp 2123 drivers/infiniband/hw/qib/qib_rc.c qib_send_rc_ack(qp); qp 2127 drivers/infiniband/hw/qib/qib_rc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 44 drivers/infiniband/hw/qib/qib_ruc.c void qib_migrate_qp(struct rvt_qp *qp) qp 48 drivers/infiniband/hw/qib/qib_ruc.c qp->s_mig_state = IB_MIG_MIGRATED; qp 49 drivers/infiniband/hw/qib/qib_ruc.c qp->remote_ah_attr = qp->alt_ah_attr; qp 50 drivers/infiniband/hw/qib/qib_ruc.c qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); qp 51 drivers/infiniband/hw/qib/qib_ruc.c qp->s_pkey_index = qp->s_alt_pkey_index; qp 53 drivers/infiniband/hw/qib/qib_ruc.c ev.device = qp->ibqp.device; qp 54 drivers/infiniband/hw/qib/qib_ruc.c ev.element.qp = &qp->ibqp; qp 56 drivers/infiniband/hw/qib/qib_ruc.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 83 drivers/infiniband/hw/qib/qib_ruc.c int has_grh, struct rvt_qp *qp, u32 bth0) qp 88 drivers/infiniband/hw/qib/qib_ruc.c if (qp->s_mig_state == IB_MIG_ARMED && (bth0 & IB_BTH_MIG_REQ)) { qp 90 drivers/infiniband/hw/qib/qib_ruc.c if (rdma_ah_get_ah_flags(&qp->alt_ah_attr) & qp 96 drivers/infiniband/hw/qib/qib_ruc.c if (!(rdma_ah_get_ah_flags(&qp->alt_ah_attr) & qp 99 drivers/infiniband/hw/qib/qib_ruc.c grh = rdma_ah_read_grh(&qp->alt_ah_attr); qp 110 drivers/infiniband/hw/qib/qib_ruc.c qib_get_pkey(ibp, qp->s_alt_pkey_index))) { qp 114 drivers/infiniband/hw/qib/qib_ruc.c 0, qp->ibqp.qp_num, qp 120 drivers/infiniband/hw/qib/qib_ruc.c rdma_ah_get_dlid(&qp->alt_ah_attr)) || qp 122 drivers/infiniband/hw/qib/qib_ruc.c rdma_ah_get_port_num(&qp->alt_ah_attr)) qp 124 drivers/infiniband/hw/qib/qib_ruc.c spin_lock_irqsave(&qp->s_lock, flags); qp 125 drivers/infiniband/hw/qib/qib_ruc.c qib_migrate_qp(qp); qp 126 drivers/infiniband/hw/qib/qib_ruc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 129 drivers/infiniband/hw/qib/qib_ruc.c if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & qp 135 drivers/infiniband/hw/qib/qib_ruc.c if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & qp 138 drivers/infiniband/hw/qib/qib_ruc.c grh = rdma_ah_read_grh(&qp->remote_ah_attr); qp 149 drivers/infiniband/hw/qib/qib_ruc.c qib_get_pkey(ibp, qp->s_pkey_index))) { qp 153 drivers/infiniband/hw/qib/qib_ruc.c 0, qp->ibqp.qp_num, qp 159 drivers/infiniband/hw/qib/qib_ruc.c rdma_ah_get_dlid(&qp->remote_ah_attr) || qp 160 drivers/infiniband/hw/qib/qib_ruc.c ppd_from_ibp(ibp)->port != qp->port_num) qp 162 drivers/infiniband/hw/qib/qib_ruc.c if (qp->s_mig_state == IB_MIG_REARM && qp 164 drivers/infiniband/hw/qib/qib_ruc.c qp->s_mig_state = IB_MIG_ARMED; qp 206 drivers/infiniband/hw/qib/qib_ruc.c void qib_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, qp 209 drivers/infiniband/hw/qib/qib_ruc.c struct qib_qp_priv *priv = qp->priv; qp 210 drivers/infiniband/hw/qib/qib_ruc.c struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 216 drivers/infiniband/hw/qib/qib_ruc.c extra_bytes = -qp->s_cur_size & 3; qp 217 drivers/infiniband/hw/qib/qib_ruc.c nwords = (qp->s_cur_size + extra_bytes) >> 2; qp 219 drivers/infiniband/hw/qib/qib_ruc.c if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { qp 220 drivers/infiniband/hw/qib/qib_ruc.c qp->s_hdrwords += qp 222 drivers/infiniband/hw/qib/qib_ruc.c rdma_ah_read_grh(&qp->remote_ah_attr), qp 223 drivers/infiniband/hw/qib/qib_ruc.c qp->s_hdrwords, nwords); qp 226 drivers/infiniband/hw/qib/qib_ruc.c lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 | qp 227 drivers/infiniband/hw/qib/qib_ruc.c rdma_ah_get_sl(&qp->remote_ah_attr) << 4; qp 230 drivers/infiniband/hw/qib/qib_ruc.c cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); qp 232 drivers/infiniband/hw/qib/qib_ruc.c cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC); qp 235 drivers/infiniband/hw/qib/qib_ruc.c rdma_ah_get_path_bits(&qp->remote_ah_attr)); qp 236 drivers/infiniband/hw/qib/qib_ruc.c bth0 |= qib_get_pkey(ibp, qp->s_pkey_index); qp 238 drivers/infiniband/hw/qib/qib_ruc.c if (qp->s_mig_state == IB_MIG_MIGRATED) qp 241 drivers/infiniband/hw/qib/qib_ruc.c ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); qp 250 drivers/infiniband/hw/qib/qib_ruc.c struct rvt_qp *qp = priv->owner; qp 252 drivers/infiniband/hw/qib/qib_ruc.c qib_do_send(qp); qp 263 drivers/infiniband/hw/qib/qib_ruc.c void qib_do_send(struct rvt_qp *qp) qp 265 drivers/infiniband/hw/qib/qib_ruc.c struct qib_qp_priv *priv = qp->priv; qp 266 drivers/infiniband/hw/qib/qib_ruc.c struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 268 drivers/infiniband/hw/qib/qib_ruc.c int (*make_req)(struct rvt_qp *qp, unsigned long *flags); qp 271 drivers/infiniband/hw/qib/qib_ruc.c if ((qp->ibqp.qp_type == IB_QPT_RC || qp 272 drivers/infiniband/hw/qib/qib_ruc.c qp->ibqp.qp_type == IB_QPT_UC) && qp 273 drivers/infiniband/hw/qib/qib_ruc.c (rdma_ah_get_dlid(&qp->remote_ah_attr) & qp 275 drivers/infiniband/hw/qib/qib_ruc.c rvt_ruc_loopback(qp); qp 279 drivers/infiniband/hw/qib/qib_ruc.c if (qp->ibqp.qp_type == IB_QPT_RC) qp 281 drivers/infiniband/hw/qib/qib_ruc.c else if (qp->ibqp.qp_type == IB_QPT_UC) qp 286 drivers/infiniband/hw/qib/qib_ruc.c spin_lock_irqsave(&qp->s_lock, flags); qp 289 drivers/infiniband/hw/qib/qib_ruc.c if (!qib_send_ok(qp)) { qp 290 drivers/infiniband/hw/qib/qib_ruc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 294 drivers/infiniband/hw/qib/qib_ruc.c qp->s_flags |= RVT_S_BUSY; qp 298 drivers/infiniband/hw/qib/qib_ruc.c if (qp->s_hdrwords != 0) { qp 299 drivers/infiniband/hw/qib/qib_ruc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 304 drivers/infiniband/hw/qib/qib_ruc.c if (qib_verbs_send(qp, priv->s_hdr, qp->s_hdrwords, qp 305 drivers/infiniband/hw/qib/qib_ruc.c qp->s_cur_sge, qp->s_cur_size)) qp 308 drivers/infiniband/hw/qib/qib_ruc.c qp->s_hdrwords = 0; qp 309 drivers/infiniband/hw/qib/qib_ruc.c spin_lock_irqsave(&qp->s_lock, flags); qp 311 drivers/infiniband/hw/qib/qib_ruc.c } while (make_req(qp, &flags)); qp 313 drivers/infiniband/hw/qib/qib_ruc.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 492 drivers/infiniband/hw/qib/qib_sdma.c struct qib_qp_priv *priv = tx->qp->priv; qp 517 drivers/infiniband/hw/qib/qib_sdma.c struct rvt_qp *qp; qp 605 drivers/infiniband/hw/qib/qib_sdma.c priv = tx->qp->priv; qp 623 drivers/infiniband/hw/qib/qib_sdma.c qp = tx->qp; qp 624 drivers/infiniband/hw/qib/qib_sdma.c priv = qp->priv; qp 626 drivers/infiniband/hw/qib/qib_sdma.c spin_lock(&qp->r_lock); qp 627 drivers/infiniband/hw/qib/qib_sdma.c spin_lock(&qp->s_lock); qp 628 drivers/infiniband/hw/qib/qib_sdma.c if (qp->ibqp.qp_type == IB_QPT_RC) { qp 630 drivers/infiniband/hw/qib/qib_sdma.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) qp 631 drivers/infiniband/hw/qib/qib_sdma.c rvt_error_qp(qp, IB_WC_GENERAL_ERR); qp 632 drivers/infiniband/hw/qib/qib_sdma.c } else if (qp->s_wqe) qp 633 drivers/infiniband/hw/qib/qib_sdma.c rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR); qp 634 drivers/infiniband/hw/qib/qib_sdma.c spin_unlock(&qp->s_lock); qp 635 drivers/infiniband/hw/qib/qib_sdma.c spin_unlock(&qp->r_lock); qp 640 drivers/infiniband/hw/qib/qib_sdma.c qp = tx->qp; qp 641 drivers/infiniband/hw/qib/qib_sdma.c priv = qp->priv; qp 642 drivers/infiniband/hw/qib/qib_sdma.c spin_lock(&qp->s_lock); qp 643 drivers/infiniband/hw/qib/qib_sdma.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 661 drivers/infiniband/hw/qib/qib_sdma.c qp->s_flags |= RVT_S_WAIT_DMA_DESC; qp 665 drivers/infiniband/hw/qib/qib_sdma.c qp->s_flags &= ~RVT_S_BUSY; qp 666 drivers/infiniband/hw/qib/qib_sdma.c spin_unlock(&qp->s_lock); qp 669 drivers/infiniband/hw/qib/qib_sdma.c spin_unlock(&qp->s_lock); qp 48 drivers/infiniband/hw/qib/qib_uc.c int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags) qp 50 drivers/infiniband/hw/qib/qib_uc.c struct qib_qp_priv *priv = qp->priv; qp 56 drivers/infiniband/hw/qib/qib_uc.c u32 pmtu = qp->pmtu; qp 59 drivers/infiniband/hw/qib/qib_uc.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { qp 60 drivers/infiniband/hw/qib/qib_uc.c if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) qp 63 drivers/infiniband/hw/qib/qib_uc.c if (qp->s_last == READ_ONCE(qp->s_head)) qp 67 drivers/infiniband/hw/qib/qib_uc.c qp->s_flags |= RVT_S_WAIT_DMA; qp 70 drivers/infiniband/hw/qib/qib_uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 71 drivers/infiniband/hw/qib/qib_uc.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); qp 76 drivers/infiniband/hw/qib/qib_uc.c if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) qp 84 drivers/infiniband/hw/qib/qib_uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 85 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = NULL; qp 86 drivers/infiniband/hw/qib/qib_uc.c switch (qp->s_state) { qp 88 drivers/infiniband/hw/qib/qib_uc.c if (!(ib_rvt_state_ops[qp->state] & qp 92 drivers/infiniband/hw/qib/qib_uc.c if (qp->s_cur == READ_ONCE(qp->s_head)) qp 97 drivers/infiniband/hw/qib/qib_uc.c qp->s_psn = wqe->psn; qp 98 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.sge = wqe->sg_list[0]; qp 99 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.sg_list = wqe->sg_list + 1; qp 100 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.num_sge = wqe->wr.num_sge; qp 101 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.total_len = wqe->length; qp 103 drivers/infiniband/hw/qib/qib_uc.c qp->s_len = len; qp 108 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(SEND_FIRST); qp 113 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(SEND_ONLY); qp 115 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = qp 123 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; qp 124 drivers/infiniband/hw/qib/qib_uc.c if (++qp->s_cur >= qp->s_size) qp 125 drivers/infiniband/hw/qib/qib_uc.c qp->s_cur = 0; qp 137 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(RDMA_WRITE_FIRST); qp 142 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(RDMA_WRITE_ONLY); qp 144 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = qp 152 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; qp 153 drivers/infiniband/hw/qib/qib_uc.c if (++qp->s_cur >= qp->s_size) qp 154 drivers/infiniband/hw/qib/qib_uc.c qp->s_cur = 0; qp 163 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(SEND_MIDDLE); qp 166 drivers/infiniband/hw/qib/qib_uc.c len = qp->s_len; qp 172 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(SEND_LAST); qp 174 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); qp 181 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; qp 182 drivers/infiniband/hw/qib/qib_uc.c if (++qp->s_cur >= qp->s_size) qp 183 drivers/infiniband/hw/qib/qib_uc.c qp->s_cur = 0; qp 187 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(RDMA_WRITE_MIDDLE); qp 190 drivers/infiniband/hw/qib/qib_uc.c len = qp->s_len; qp 196 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = OP(RDMA_WRITE_LAST); qp 198 drivers/infiniband/hw/qib/qib_uc.c qp->s_state = qp 206 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; qp 207 drivers/infiniband/hw/qib/qib_uc.c if (++qp->s_cur >= qp->s_size) qp 208 drivers/infiniband/hw/qib/qib_uc.c qp->s_cur = 0; qp 211 drivers/infiniband/hw/qib/qib_uc.c qp->s_len -= len; qp 212 drivers/infiniband/hw/qib/qib_uc.c qp->s_hdrwords = hwords; qp 213 drivers/infiniband/hw/qib/qib_uc.c qp->s_cur_sge = &qp->s_sge; qp 214 drivers/infiniband/hw/qib/qib_uc.c qp->s_cur_size = len; qp 215 drivers/infiniband/hw/qib/qib_uc.c qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), qp 216 drivers/infiniband/hw/qib/qib_uc.c qp->s_psn++ & QIB_PSN_MASK); qp 220 drivers/infiniband/hw/qib/qib_uc.c qp->s_flags &= ~RVT_S_BUSY; qp 238 drivers/infiniband/hw/qib/qib_uc.c int has_grh, void *data, u32 tlen, struct rvt_qp *qp) qp 246 drivers/infiniband/hw/qib/qib_uc.c u32 pmtu = qp->pmtu; qp 260 drivers/infiniband/hw/qib/qib_uc.c if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) qp 267 drivers/infiniband/hw/qib/qib_uc.c if (unlikely(qib_cmp24(psn, qp->r_psn) != 0)) { qp 272 drivers/infiniband/hw/qib/qib_uc.c qp->r_psn = psn; qp 274 drivers/infiniband/hw/qib/qib_uc.c if (qp->r_state == OP(SEND_FIRST) || qp 275 drivers/infiniband/hw/qib/qib_uc.c qp->r_state == OP(SEND_MIDDLE)) { qp 276 drivers/infiniband/hw/qib/qib_uc.c set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); qp 277 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.num_sge = 0; qp 279 drivers/infiniband/hw/qib/qib_uc.c rvt_put_ss(&qp->r_sge); qp 280 drivers/infiniband/hw/qib/qib_uc.c qp->r_state = OP(SEND_LAST); qp 298 drivers/infiniband/hw/qib/qib_uc.c switch (qp->r_state) { qp 326 drivers/infiniband/hw/qib/qib_uc.c if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) qp 327 drivers/infiniband/hw/qib/qib_uc.c rvt_comm_est(qp); qp 335 drivers/infiniband/hw/qib/qib_uc.c if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) qp 336 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge = qp->s_rdma_read_sge; qp 338 drivers/infiniband/hw/qib/qib_uc.c ret = rvt_get_rwqe(qp, false); qp 347 drivers/infiniband/hw/qib/qib_uc.c qp->s_rdma_read_sge = qp->r_sge; qp 349 drivers/infiniband/hw/qib/qib_uc.c qp->r_rcv_len = 0; qp 359 drivers/infiniband/hw/qib/qib_uc.c qp->r_rcv_len += pmtu; qp 360 drivers/infiniband/hw/qib/qib_uc.c if (unlikely(qp->r_rcv_len > qp->r_len)) qp 362 drivers/infiniband/hw/qib/qib_uc.c rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false); qp 384 drivers/infiniband/hw/qib/qib_uc.c wc.byte_len = tlen + qp->r_rcv_len; qp 385 drivers/infiniband/hw/qib/qib_uc.c if (unlikely(wc.byte_len > qp->r_len)) qp 388 drivers/infiniband/hw/qib/qib_uc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false); qp 389 drivers/infiniband/hw/qib/qib_uc.c rvt_put_ss(&qp->s_rdma_read_sge); qp 391 drivers/infiniband/hw/qib/qib_uc.c wc.wr_id = qp->r_wr_id; qp 393 drivers/infiniband/hw/qib/qib_uc.c wc.qp = &qp->ibqp; qp 394 drivers/infiniband/hw/qib/qib_uc.c wc.src_qp = qp->remote_qpn; qp 395 drivers/infiniband/hw/qib/qib_uc.c wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr); qp 396 drivers/infiniband/hw/qib/qib_uc.c wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); qp 403 drivers/infiniband/hw/qib/qib_uc.c rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); qp 410 drivers/infiniband/hw/qib/qib_uc.c if (unlikely(!(qp->qp_access_flags & qp 416 drivers/infiniband/hw/qib/qib_uc.c qp->r_len = be32_to_cpu(reth->length); qp 417 drivers/infiniband/hw/qib/qib_uc.c qp->r_rcv_len = 0; qp 418 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.sg_list = NULL; qp 419 drivers/infiniband/hw/qib/qib_uc.c if (qp->r_len != 0) { qp 425 drivers/infiniband/hw/qib/qib_uc.c ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, qp 429 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.num_sge = 1; qp 431 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.num_sge = 0; qp 432 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.sge.mr = NULL; qp 433 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.sge.vaddr = NULL; qp 434 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.sge.length = 0; qp 435 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.sge.sge_length = 0; qp 448 drivers/infiniband/hw/qib/qib_uc.c qp->r_rcv_len += pmtu; qp 449 drivers/infiniband/hw/qib/qib_uc.c if (unlikely(qp->r_rcv_len > qp->r_len)) qp 451 drivers/infiniband/hw/qib/qib_uc.c rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); qp 468 drivers/infiniband/hw/qib/qib_uc.c if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) qp 470 drivers/infiniband/hw/qib/qib_uc.c if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) qp 471 drivers/infiniband/hw/qib/qib_uc.c rvt_put_ss(&qp->s_rdma_read_sge); qp 473 drivers/infiniband/hw/qib/qib_uc.c ret = rvt_get_rwqe(qp, true); qp 479 drivers/infiniband/hw/qib/qib_uc.c wc.byte_len = qp->r_len; qp 481 drivers/infiniband/hw/qib/qib_uc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); qp 482 drivers/infiniband/hw/qib/qib_uc.c rvt_put_ss(&qp->r_sge); qp 495 drivers/infiniband/hw/qib/qib_uc.c if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) qp 497 drivers/infiniband/hw/qib/qib_uc.c rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); qp 498 drivers/infiniband/hw/qib/qib_uc.c rvt_put_ss(&qp->r_sge); qp 505 drivers/infiniband/hw/qib/qib_uc.c qp->r_psn++; qp 506 drivers/infiniband/hw/qib/qib_uc.c qp->r_state = opcode; qp 510 drivers/infiniband/hw/qib/qib_uc.c set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); qp 511 drivers/infiniband/hw/qib/qib_uc.c qp->r_sge.num_sge = 0; qp 517 drivers/infiniband/hw/qib/qib_uc.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 57 drivers/infiniband/hw/qib/qib_ud.c struct rvt_qp *qp; qp 67 drivers/infiniband/hw/qib/qib_ud.c qp = rvt_lookup_qpn(rdi, &ibp->rvp, rvt_get_swqe_remote_qpn(swqe)); qp 68 drivers/infiniband/hw/qib/qib_ud.c if (!qp) { qp 75 drivers/infiniband/hw/qib/qib_ud.c dqptype = qp->ibqp.qp_type == IB_QPT_GSI ? qp 76 drivers/infiniband/hw/qib/qib_ud.c IB_QPT_UD : qp->ibqp.qp_type; qp 79 drivers/infiniband/hw/qib/qib_ud.c !(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { qp 87 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_num > 1) { qp 93 drivers/infiniband/hw/qib/qib_ud.c pkey2 = qib_get_pkey(ibp, qp->s_pkey_index); qp 99 drivers/infiniband/hw/qib/qib_ud.c sqp->ibqp.qp_num, qp->ibqp.qp_num, qp 111 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_num) { qp 116 drivers/infiniband/hw/qib/qib_ud.c if (unlikely(qkey != qp->qkey)) qp 133 drivers/infiniband/hw/qib/qib_ud.c spin_lock_irqsave(&qp->r_lock, flags); qp 138 drivers/infiniband/hw/qib/qib_ud.c if (qp->r_flags & RVT_R_REUSE_SGE) qp 139 drivers/infiniband/hw/qib/qib_ud.c qp->r_flags &= ~RVT_R_REUSE_SGE; qp 143 drivers/infiniband/hw/qib/qib_ud.c ret = rvt_get_rwqe(qp, false); qp 145 drivers/infiniband/hw/qib/qib_ud.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 149 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_num == 0) qp 155 drivers/infiniband/hw/qib/qib_ud.c if (unlikely(wc.byte_len > qp->r_len)) { qp 156 drivers/infiniband/hw/qib/qib_ud.c qp->r_flags |= RVT_R_REUSE_SGE; qp 166 drivers/infiniband/hw/qib/qib_ud.c rvt_copy_sge(qp, &qp->r_sge, &grh, qp 170 drivers/infiniband/hw/qib/qib_ud.c rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); qp 178 drivers/infiniband/hw/qib/qib_ud.c rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false); qp 198 drivers/infiniband/hw/qib/qib_ud.c rvt_put_ss(&qp->r_sge); qp 199 drivers/infiniband/hw/qib/qib_ud.c if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) qp 201 drivers/infiniband/hw/qib/qib_ud.c wc.wr_id = qp->r_wr_id; qp 204 drivers/infiniband/hw/qib/qib_ud.c wc.qp = &qp->ibqp; qp 206 drivers/infiniband/hw/qib/qib_ud.c wc.pkey_index = qp->ibqp.qp_type == IB_QPT_GSI ? qp 212 drivers/infiniband/hw/qib/qib_ud.c wc.port_num = qp->port_num; qp 214 drivers/infiniband/hw/qib/qib_ud.c rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); qp 217 drivers/infiniband/hw/qib/qib_ud.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 230 drivers/infiniband/hw/qib/qib_ud.c int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) qp 232 drivers/infiniband/hw/qib/qib_ud.c struct qib_qp_priv *priv = qp->priv; qp 246 drivers/infiniband/hw/qib/qib_ud.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) { qp 247 drivers/infiniband/hw/qib/qib_ud.c if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) qp 250 drivers/infiniband/hw/qib/qib_ud.c if (qp->s_last == READ_ONCE(qp->s_head)) qp 254 drivers/infiniband/hw/qib/qib_ud.c qp->s_flags |= RVT_S_WAIT_DMA; qp 257 drivers/infiniband/hw/qib/qib_ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 258 drivers/infiniband/hw/qib/qib_ud.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); qp 263 drivers/infiniband/hw/qib/qib_ud.c if (qp->s_cur == READ_ONCE(qp->s_head)) qp 266 drivers/infiniband/hw/qib/qib_ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); qp 267 drivers/infiniband/hw/qib/qib_ud.c next_cur = qp->s_cur + 1; qp 268 drivers/infiniband/hw/qib/qib_ud.c if (next_cur >= qp->s_size) qp 272 drivers/infiniband/hw/qib/qib_ud.c ibp = to_iport(qp->ibqp.device, qp->port_num); qp 294 drivers/infiniband/hw/qib/qib_ud.c qp->s_flags |= RVT_S_WAIT_DMA; qp 297 drivers/infiniband/hw/qib/qib_ud.c qp->s_cur = next_cur; qp 298 drivers/infiniband/hw/qib/qib_ud.c spin_unlock_irqrestore(&qp->s_lock, tflags); qp 299 drivers/infiniband/hw/qib/qib_ud.c qib_ud_loopback(qp, wqe); qp 300 drivers/infiniband/hw/qib/qib_ud.c spin_lock_irqsave(&qp->s_lock, tflags); qp 302 drivers/infiniband/hw/qib/qib_ud.c rvt_send_complete(qp, wqe, IB_WC_SUCCESS); qp 307 drivers/infiniband/hw/qib/qib_ud.c qp->s_cur = next_cur; qp 312 drivers/infiniband/hw/qib/qib_ud.c qp->s_hdrwords = 7; qp 313 drivers/infiniband/hw/qib/qib_ud.c qp->s_cur_size = wqe->length; qp 314 drivers/infiniband/hw/qib/qib_ud.c qp->s_cur_sge = &qp->s_sge; qp 315 drivers/infiniband/hw/qib/qib_ud.c qp->s_srate = rdma_ah_get_static_rate(ah_attr); qp 316 drivers/infiniband/hw/qib/qib_ud.c qp->s_wqe = wqe; qp 317 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.sge = wqe->sg_list[0]; qp 318 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.sg_list = wqe->sg_list + 1; qp 319 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.num_sge = wqe->wr.num_sge; qp 320 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.total_len = wqe->length; qp 324 drivers/infiniband/hw/qib/qib_ud.c qp->s_hdrwords += qib_make_grh(ibp, &priv->s_hdr->u.l.grh, qp 326 drivers/infiniband/hw/qib/qib_ud.c qp->s_hdrwords, nwords); qp 339 drivers/infiniband/hw/qib/qib_ud.c qp->s_hdrwords++; qp 345 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_type == IB_QPT_SMI) qp 353 drivers/infiniband/hw/qib/qib_ud.c cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC); qp 364 drivers/infiniband/hw/qib/qib_ud.c bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? QIB_DEFAULT_P_KEY : qp 365 drivers/infiniband/hw/qib/qib_ud.c qib_get_pkey(ibp, qp->ibqp.qp_type == IB_QPT_GSI ? qp 366 drivers/infiniband/hw/qib/qib_ud.c rvt_get_swqe_pkey_index(wqe) : qp->s_pkey_index); qp 382 drivers/infiniband/hw/qib/qib_ud.c cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : qp 384 drivers/infiniband/hw/qib/qib_ud.c ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); qp 389 drivers/infiniband/hw/qib/qib_ud.c qp->s_flags &= ~RVT_S_BUSY; qp 427 drivers/infiniband/hw/qib/qib_ud.c int has_grh, void *data, u32 tlen, struct rvt_qp *qp) qp 463 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_num) { qp 467 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_num > 1) { qp 471 drivers/infiniband/hw/qib/qib_ud.c pkey2 = qib_get_pkey(ibp, qp->s_pkey_index); qp 477 drivers/infiniband/hw/qib/qib_ud.c src_qp, qp->ibqp.qp_num, qp 482 drivers/infiniband/hw/qib/qib_ud.c if (unlikely(qkey != qp->qkey)) qp 486 drivers/infiniband/hw/qib/qib_ud.c if (unlikely(qp->ibqp.qp_num == 1 && qp 508 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_num > 1 && qp 527 drivers/infiniband/hw/qib/qib_ud.c if (qp->r_flags & RVT_R_REUSE_SGE) qp 528 drivers/infiniband/hw/qib/qib_ud.c qp->r_flags &= ~RVT_R_REUSE_SGE; qp 532 drivers/infiniband/hw/qib/qib_ud.c ret = rvt_get_rwqe(qp, false); qp 534 drivers/infiniband/hw/qib/qib_ud.c rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); qp 538 drivers/infiniband/hw/qib/qib_ud.c if (qp->ibqp.qp_num == 0) qp 544 drivers/infiniband/hw/qib/qib_ud.c if (unlikely(wc.byte_len > qp->r_len)) { qp 545 drivers/infiniband/hw/qib/qib_ud.c qp->r_flags |= RVT_R_REUSE_SGE; qp 549 drivers/infiniband/hw/qib/qib_ud.c rvt_copy_sge(qp, &qp->r_sge, &hdr->u.l.grh, qp 553 drivers/infiniband/hw/qib/qib_ud.c rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); qp 554 drivers/infiniband/hw/qib/qib_ud.c rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), qp 556 drivers/infiniband/hw/qib/qib_ud.c rvt_put_ss(&qp->r_sge); qp 557 drivers/infiniband/hw/qib/qib_ud.c if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) qp 559 drivers/infiniband/hw/qib/qib_ud.c wc.wr_id = qp->r_wr_id; qp 563 drivers/infiniband/hw/qib/qib_ud.c wc.qp = &qp->ibqp; qp 565 drivers/infiniband/hw/qib/qib_ud.c wc.pkey_index = qp->ibqp.qp_type == IB_QPT_GSI ? qp 575 drivers/infiniband/hw/qib/qib_ud.c wc.port_num = qp->port_num; qp 577 drivers/infiniband/hw/qib/qib_ud.c rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); qp 224 drivers/infiniband/hw/qib/qib_verbs.c int has_grh, void *data, u32 tlen, struct rvt_qp *qp) qp 228 drivers/infiniband/hw/qib/qib_verbs.c spin_lock(&qp->r_lock); qp 231 drivers/infiniband/hw/qib/qib_verbs.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { qp 236 drivers/infiniband/hw/qib/qib_verbs.c switch (qp->ibqp.qp_type) { qp 243 drivers/infiniband/hw/qib/qib_verbs.c qib_ud_rcv(ibp, hdr, has_grh, data, tlen, qp); qp 247 drivers/infiniband/hw/qib/qib_verbs.c qib_rc_rcv(rcd, hdr, has_grh, data, tlen, qp); qp 251 drivers/infiniband/hw/qib/qib_verbs.c qib_uc_rcv(ibp, hdr, has_grh, data, tlen, qp); qp 259 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock(&qp->r_lock); qp 280 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp; qp 334 drivers/infiniband/hw/qib/qib_verbs.c qib_qp_rcv(rcd, hdr, 1, data, tlen, p->qp); qp 344 drivers/infiniband/hw/qib/qib_verbs.c qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); qp 345 drivers/infiniband/hw/qib/qib_verbs.c if (!qp) { qp 350 drivers/infiniband/hw/qib/qib_verbs.c qib_qp_rcv(rcd, hdr, lnh == QIB_LRH_GRH, data, tlen, qp); qp 367 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp = NULL; qp 374 drivers/infiniband/hw/qib/qib_verbs.c qp = priv->owner; qp 376 drivers/infiniband/hw/qib/qib_verbs.c rvt_get_qp(qp); qp 382 drivers/infiniband/hw/qib/qib_verbs.c if (qp) { qp 383 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 384 drivers/infiniband/hw/qib/qib_verbs.c if (qp->s_flags & RVT_S_WAIT_KMEM) { qp 385 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_WAIT_KMEM; qp 386 drivers/infiniband/hw/qib/qib_verbs.c qib_schedule_send(qp); qp 388 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 389 drivers/infiniband/hw/qib/qib_verbs.c rvt_put_qp(qp); qp 561 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp) qp 563 drivers/infiniband/hw/qib/qib_verbs.c struct qib_qp_priv *priv = qp->priv; qp 567 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 575 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 578 drivers/infiniband/hw/qib/qib_verbs.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK && qp 581 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags |= RVT_S_WAIT_TX; qp 584 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_BUSY; qp 586 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 593 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp) qp 609 drivers/infiniband/hw/qib/qib_verbs.c tx = __get_txreq(dev, qp); qp 617 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp; qp 621 drivers/infiniband/hw/qib/qib_verbs.c qp = tx->qp; qp 622 drivers/infiniband/hw/qib/qib_verbs.c dev = to_idev(qp->ibqp.device); qp 645 drivers/infiniband/hw/qib/qib_verbs.c qp = priv->owner; qp 647 drivers/infiniband/hw/qib/qib_verbs.c rvt_get_qp(qp); qp 650 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 651 drivers/infiniband/hw/qib/qib_verbs.c if (qp->s_flags & RVT_S_WAIT_TX) { qp 652 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_WAIT_TX; qp 653 drivers/infiniband/hw/qib/qib_verbs.c qib_schedule_send(qp); qp 655 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 657 drivers/infiniband/hw/qib/qib_verbs.c rvt_put_qp(qp); qp 670 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp; qp 682 drivers/infiniband/hw/qib/qib_verbs.c qp = qpp->owner; qp 683 drivers/infiniband/hw/qib/qib_verbs.c if (qp->port_num != ppd->port) qp 691 drivers/infiniband/hw/qib/qib_verbs.c rvt_get_qp(qp); qp 692 drivers/infiniband/hw/qib/qib_verbs.c qps[n++] = qp; qp 698 drivers/infiniband/hw/qib/qib_verbs.c qp = qps[i]; qp 699 drivers/infiniband/hw/qib/qib_verbs.c spin_lock(&qp->s_lock); qp 700 drivers/infiniband/hw/qib/qib_verbs.c if (qp->s_flags & RVT_S_WAIT_DMA_DESC) { qp 701 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_WAIT_DMA_DESC; qp 702 drivers/infiniband/hw/qib/qib_verbs.c qib_schedule_send(qp); qp 704 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock(&qp->s_lock); qp 705 drivers/infiniband/hw/qib/qib_verbs.c rvt_put_qp(qp); qp 716 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp = tx->qp; qp 717 drivers/infiniband/hw/qib/qib_verbs.c struct qib_qp_priv *priv = qp->priv; qp 719 drivers/infiniband/hw/qib/qib_verbs.c spin_lock(&qp->s_lock); qp 721 drivers/infiniband/hw/qib/qib_verbs.c rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS); qp 722 drivers/infiniband/hw/qib/qib_verbs.c else if (qp->ibqp.qp_type == IB_QPT_RC) { qp 728 drivers/infiniband/hw/qib/qib_verbs.c struct qib_ibdev *dev = to_idev(qp->ibqp.device); qp 732 drivers/infiniband/hw/qib/qib_verbs.c qib_rc_send_complete(qp, hdr); qp 735 drivers/infiniband/hw/qib/qib_verbs.c if (qp->state == IB_QPS_RESET) qp 737 drivers/infiniband/hw/qib/qib_verbs.c else if (qp->s_flags & RVT_S_WAIT_DMA) { qp 738 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_WAIT_DMA; qp 739 drivers/infiniband/hw/qib/qib_verbs.c qib_schedule_send(qp); qp 742 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock(&qp->s_lock); qp 747 drivers/infiniband/hw/qib/qib_verbs.c static int wait_kmem(struct qib_ibdev *dev, struct rvt_qp *qp) qp 749 drivers/infiniband/hw/qib/qib_verbs.c struct qib_qp_priv *priv = qp->priv; qp 753 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 754 drivers/infiniband/hw/qib/qib_verbs.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 759 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags |= RVT_S_WAIT_KMEM; qp 763 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_BUSY; qp 766 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 771 drivers/infiniband/hw/qib/qib_verbs.c static int qib_verbs_send_dma(struct rvt_qp *qp, struct ib_header *hdr, qp 775 drivers/infiniband/hw/qib/qib_verbs.c struct qib_qp_priv *priv = qp->priv; qp 776 drivers/infiniband/hw/qib/qib_verbs.c struct qib_ibdev *dev = to_idev(qp->ibqp.device); qp 778 drivers/infiniband/hw/qib/qib_verbs.c struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); qp 794 drivers/infiniband/hw/qib/qib_verbs.c tx = get_txreq(dev, qp); qp 798 drivers/infiniband/hw/qib/qib_verbs.c control = dd->f_setpbc_control(ppd, plen, qp->s_srate, qp 800 drivers/infiniband/hw/qib/qib_verbs.c tx->qp = qp; qp 801 drivers/infiniband/hw/qib/qib_verbs.c tx->wqe = qp->s_wqe; qp 802 drivers/infiniband/hw/qib/qib_verbs.c tx->mr = qp->s_rdma_mr; qp 803 drivers/infiniband/hw/qib/qib_verbs.c if (qp->s_rdma_mr) qp 804 drivers/infiniband/hw/qib/qib_verbs.c qp->s_rdma_mr = NULL; qp 861 drivers/infiniband/hw/qib/qib_verbs.c ret = wait_kmem(dev, qp); qp 875 drivers/infiniband/hw/qib/qib_verbs.c static int no_bufs_available(struct rvt_qp *qp) qp 877 drivers/infiniband/hw/qib/qib_verbs.c struct qib_qp_priv *priv = qp->priv; qp 878 drivers/infiniband/hw/qib/qib_verbs.c struct qib_ibdev *dev = to_idev(qp->ibqp.device); qp 889 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 890 drivers/infiniband/hw/qib/qib_verbs.c if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { qp 894 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags |= RVT_S_WAIT_PIO; qp 900 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_BUSY; qp 903 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 907 drivers/infiniband/hw/qib/qib_verbs.c static int qib_verbs_send_pio(struct rvt_qp *qp, struct ib_header *ibhdr, qp 911 drivers/infiniband/hw/qib/qib_verbs.c struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 912 drivers/infiniband/hw/qib/qib_verbs.c struct qib_pportdata *ppd = dd->pport + qp->port_num - 1; qp 922 drivers/infiniband/hw/qib/qib_verbs.c control = dd->f_setpbc_control(ppd, plen, qp->s_srate, qp 927 drivers/infiniband/hw/qib/qib_verbs.c return no_bufs_available(qp); qp 988 drivers/infiniband/hw/qib/qib_verbs.c if (qp->s_rdma_mr) { qp 989 drivers/infiniband/hw/qib/qib_verbs.c rvt_put_mr(qp->s_rdma_mr); qp 990 drivers/infiniband/hw/qib/qib_verbs.c qp->s_rdma_mr = NULL; qp 992 drivers/infiniband/hw/qib/qib_verbs.c if (qp->s_wqe) { qp 993 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 994 drivers/infiniband/hw/qib/qib_verbs.c rvt_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS); qp 995 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 996 drivers/infiniband/hw/qib/qib_verbs.c } else if (qp->ibqp.qp_type == IB_QPT_RC) { qp 997 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 998 drivers/infiniband/hw/qib/qib_verbs.c qib_rc_send_complete(qp, ibhdr); qp 999 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1015 drivers/infiniband/hw/qib/qib_verbs.c int qib_verbs_send(struct rvt_qp *qp, struct ib_header *hdr, qp 1018 drivers/infiniband/hw/qib/qib_verbs.c struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); qp 1034 drivers/infiniband/hw/qib/qib_verbs.c if (qp->ibqp.qp_type == IB_QPT_SMI || qp 1036 drivers/infiniband/hw/qib/qib_verbs.c ret = qib_verbs_send_pio(qp, hdr, hdrwords, ss, len, qp 1039 drivers/infiniband/hw/qib/qib_verbs.c ret = qib_verbs_send_dma(qp, hdr, hdrwords, ss, len, qp 1149 drivers/infiniband/hw/qib/qib_verbs.c struct rvt_qp *qp; qp 1168 drivers/infiniband/hw/qib/qib_verbs.c qp = priv->owner; qp 1170 drivers/infiniband/hw/qib/qib_verbs.c rvt_get_qp(qp); qp 1171 drivers/infiniband/hw/qib/qib_verbs.c qps[n++] = qp; qp 1178 drivers/infiniband/hw/qib/qib_verbs.c qp = qps[i]; qp 1180 drivers/infiniband/hw/qib/qib_verbs.c spin_lock_irqsave(&qp->s_lock, flags); qp 1181 drivers/infiniband/hw/qib/qib_verbs.c if (qp->s_flags & RVT_S_WAIT_PIO) { qp 1182 drivers/infiniband/hw/qib/qib_verbs.c qp->s_flags &= ~RVT_S_WAIT_PIO; qp 1183 drivers/infiniband/hw/qib/qib_verbs.c qib_schedule_send(qp); qp 1185 drivers/infiniband/hw/qib/qib_verbs.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 1188 drivers/infiniband/hw/qib/qib_verbs.c rvt_put_qp(qp); qp 1353 drivers/infiniband/hw/qib/qib_verbs.c qp0 = rcu_dereference(ibp->rvp.qp[0]); qp 1429 drivers/infiniband/hw/qib/qib_verbs.c RCU_INIT_POINTER(ibp->rvp.qp[0], NULL); qp 1430 drivers/infiniband/hw/qib/qib_verbs.c RCU_INIT_POINTER(ibp->rvp.qp[1], NULL); qp 1688 drivers/infiniband/hw/qib/qib_verbs.c bool _qib_schedule_send(struct rvt_qp *qp) qp 1691 drivers/infiniband/hw/qib/qib_verbs.c to_iport(qp->ibqp.device, qp->port_num); qp 1693 drivers/infiniband/hw/qib/qib_verbs.c struct qib_qp_priv *priv = qp->priv; qp 1705 drivers/infiniband/hw/qib/qib_verbs.c bool qib_schedule_send(struct rvt_qp *qp) qp 1707 drivers/infiniband/hw/qib/qib_verbs.c if (qib_send_ok(qp)) qp 1708 drivers/infiniband/hw/qib/qib_verbs.c return _qib_schedule_send(qp); qp 219 drivers/infiniband/hw/qib/qib_verbs.h static inline int qib_send_ok(struct rvt_qp *qp) qp 221 drivers/infiniband/hw/qib/qib_verbs.h return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) && qp 222 drivers/infiniband/hw/qib/qib_verbs.h (qp->s_hdrwords || (qp->s_flags & RVT_S_RESP_PENDING) || qp 223 drivers/infiniband/hw/qib/qib_verbs.h !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); qp 226 drivers/infiniband/hw/qib/qib_verbs.h bool _qib_schedule_send(struct rvt_qp *qp); qp 227 drivers/infiniband/hw/qib/qib_verbs.h bool qib_schedule_send(struct rvt_qp *qp); qp 274 drivers/infiniband/hw/qib/qib_verbs.h void *qib_qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp); qp 275 drivers/infiniband/hw/qib/qib_verbs.h void qib_qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); qp 276 drivers/infiniband/hw/qib/qib_verbs.h void qib_notify_qp_reset(struct rvt_qp *qp); qp 279 drivers/infiniband/hw/qib/qib_verbs.h void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait); qp 292 drivers/infiniband/hw/qib/qib_verbs.h int qib_verbs_send(struct rvt_qp *qp, struct ib_header *hdr, qp 296 drivers/infiniband/hw/qib/qib_verbs.h int has_grh, void *data, u32 tlen, struct rvt_qp *qp); qp 299 drivers/infiniband/hw/qib/qib_verbs.h int has_grh, void *data, u32 tlen, struct rvt_qp *qp); qp 303 drivers/infiniband/hw/qib/qib_verbs.h int qib_check_send_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 310 drivers/infiniband/hw/qib/qib_verbs.h void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr); qp 312 drivers/infiniband/hw/qib/qib_verbs.h int qib_post_ud_send(struct rvt_qp *qp, const struct ib_send_wr *wr); qp 315 drivers/infiniband/hw/qib/qib_verbs.h int has_grh, void *data, u32 tlen, struct rvt_qp *qp); qp 319 drivers/infiniband/hw/qib/qib_verbs.h void qib_migrate_qp(struct rvt_qp *qp); qp 322 drivers/infiniband/hw/qib/qib_verbs.h int has_grh, struct rvt_qp *qp, u32 bth0); qp 327 drivers/infiniband/hw/qib/qib_verbs.h void qib_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, qp 332 drivers/infiniband/hw/qib/qib_verbs.h void qib_do_send(struct rvt_qp *qp); qp 334 drivers/infiniband/hw/qib/qib_verbs.h void qib_send_rc_ack(struct rvt_qp *qp); qp 336 drivers/infiniband/hw/qib/qib_verbs.h int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags); qp 338 drivers/infiniband/hw/qib/qib_verbs.h int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags); qp 340 drivers/infiniband/hw/qib/qib_verbs.h int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags); qp 487 drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c ib_event.element.qp = &qp_grp->ibqp; qp 384 drivers/infiniband/hw/usnic/usnic_ib_verbs.c int usnic_ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 397 drivers/infiniband/hw/usnic/usnic_ib_verbs.c qp_grp = to_uqp_grp(qp); qp 542 drivers/infiniband/hw/usnic/usnic_ib_verbs.c int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) qp 549 drivers/infiniband/hw/usnic/usnic_ib_verbs.c qp_grp = to_uqp_grp(qp); qp 46 drivers/infiniband/hw/usnic/usnic_ib_verbs.h int usnic_ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 58 drivers/infiniband/hw/usnic/usnic_ib_verbs.h int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); qp 534 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h void _pvrdma_flush_cqe(struct pvrdma_qp *qp, struct pvrdma_cq *cq); qp 274 drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c void _pvrdma_flush_cqe(struct pvrdma_qp *qp, struct pvrdma_cq *cq) qp 302 drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c if ((curr_cqe->qp & 0xFFFF) != qp->qp_handle) { qp 347 drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c if (dev->qp_tbl[cqe->qp & 0xffff]) qp 348 drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c *cur_qp = (struct pvrdma_qp *)dev->qp_tbl[cqe->qp & 0xffff]; qp 355 drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c wc->qp = &(*cur_qp)->ibqp; qp 313 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c struct pvrdma_qp *qp; qp 317 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c qp = dev->qp_tbl[qpn % dev->dsr->caps.max_qp]; qp 318 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c if (qp) qp 319 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c refcount_inc(&qp->refcnt); qp 322 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c if (qp && qp->ibqp.event_handler) { qp 323 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c struct ib_qp *ibqp = &qp->ibqp; qp 327 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c e.element.qp = ibqp; qp 331 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c if (qp) { qp 332 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c if (refcount_dec_and_test(&qp->refcnt)) qp 333 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c complete(&qp->free); qp 55 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c static inline void get_cqs(struct pvrdma_qp *qp, struct pvrdma_cq **send_cq, qp 58 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c *send_cq = to_vcq(qp->ibqp.send_cq); qp 59 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c *recv_cq = to_vcq(qp->ibqp.recv_cq); qp 98 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c static void pvrdma_reset_qp(struct pvrdma_qp *qp) qp 104 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c get_cqs(qp, &scq, &rcq); qp 107 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c _pvrdma_flush_cqe(qp, scq); qp 109 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c _pvrdma_flush_cqe(qp, rcq); qp 117 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->rq.ring) { qp 118 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c atomic_set(&qp->rq.ring->cons_head, 0); qp 119 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c atomic_set(&qp->rq.ring->prod_tail, 0); qp 121 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->sq.ring) { qp 122 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c atomic_set(&qp->sq.ring->cons_head, 0); qp 123 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c atomic_set(&qp->sq.ring->prod_tail, 0); qp 129 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *qp) qp 137 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, req_cap->max_recv_wr)); qp 138 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.max_sg = roundup_pow_of_two(max(1U, req_cap->max_recv_sge)); qp 141 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c req_cap->max_recv_wr = qp->rq.wqe_cnt; qp 142 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c req_cap->max_recv_sge = qp->rq.max_sg; qp 144 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.wqe_size = roundup_pow_of_two(sizeof(struct pvrdma_rq_wqe_hdr) + qp 146 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.max_sg); qp 147 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages_recv = (qp->rq.wqe_cnt * qp->rq.wqe_size + PAGE_SIZE - 1) / qp 154 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *qp) qp 162 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.wqe_cnt = roundup_pow_of_two(max(1U, req_cap->max_send_wr)); qp 163 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.max_sg = roundup_pow_of_two(max(1U, req_cap->max_send_sge)); qp 166 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c req_cap->max_send_wr = qp->sq.wqe_cnt; qp 167 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c req_cap->max_send_sge = qp->sq.max_sg; qp 169 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.wqe_size = roundup_pow_of_two(sizeof(struct pvrdma_sq_wqe_hdr) + qp 171 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.max_sg); qp 173 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages_send = PVRDMA_QP_NUM_HEADER_PAGES + qp 174 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c (qp->sq.wqe_cnt * qp->sq.wqe_size + PAGE_SIZE - 1) / qp 192 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *qp = NULL; qp 239 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 240 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (!qp) { qp 245 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c spin_lock_init(&qp->sq.lock); qp 246 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c spin_lock_init(&qp->rq.lock); qp 247 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c mutex_init(&qp->mutex); qp 248 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c refcount_set(&qp->refcnt, 1); qp 249 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_completion(&qp->free); qp 251 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->state = IB_QPS_RESET; qp 252 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->is_kernel = !udata; qp 254 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (!qp->is_kernel) { qp 265 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rumem = ib_umem_get(udata, ucmd.rbuf_addr, qp 267 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (IS_ERR(qp->rumem)) { qp 268 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ret = PTR_ERR(qp->rumem); qp 271 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->srq = NULL; qp 273 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rumem = NULL; qp 274 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->srq = to_vsrq(init_attr->srq); qp 277 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sumem = ib_umem_get(udata, ucmd.sbuf_addr, qp 279 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (IS_ERR(qp->sumem)) { qp 281 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ib_umem_release(qp->rumem); qp 282 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ret = PTR_ERR(qp->sumem); qp 286 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages_send = ib_umem_page_count(qp->sumem); qp 288 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages_recv = ib_umem_page_count(qp->rumem); qp 290 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages_recv = 0; qp 291 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages = qp->npages_send + qp->npages_recv; qp 294 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c &init_attr->cap, qp); qp 299 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c &init_attr->cap, qp); qp 303 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages = qp->npages_send + qp->npages_recv; qp 306 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.offset = PVRDMA_QP_NUM_HEADER_PAGES * PAGE_SIZE; qp 309 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.offset = qp->npages_send * PAGE_SIZE; qp 312 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->npages < 0 || qp->npages > PVRDMA_PAGE_DIR_MAX_PAGES) { qp 319 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ret = pvrdma_page_dir_init(dev, &qp->pdir, qp->npages, qp 320 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->is_kernel); qp 327 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (!qp->is_kernel) { qp 328 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_page_dir_insert_umem(&qp->pdir, qp->sumem, 0); qp 330 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_page_dir_insert_umem(&qp->pdir, qp 331 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rumem, qp 332 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->npages_send); qp 335 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.ring = qp->pdir.pages[0]; qp 336 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.ring = is_srq ? NULL : &qp->sq.ring[1]; qp 366 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c cmd->total_chunks = qp->npages; qp 367 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c cmd->send_chunks = qp->npages_send - PVRDMA_QP_NUM_HEADER_PAGES; qp 368 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c cmd->pdir_dma = qp->pdir.dir_dma; qp 382 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->qp_handle = resp->qpn; qp 383 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->port = init_attr->port_num; qp 384 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->ibqp.qp_num = resp->qpn; qp 386 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c dev->qp_tbl[qp->qp_handle % dev->dsr->caps.max_qp] = qp; qp 389 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c return &qp->ibqp; qp 392 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_page_dir_cleanup(dev, &qp->pdir); qp 394 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ib_umem_release(qp->rumem); qp 395 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ib_umem_release(qp->sumem); qp 397 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c kfree(qp); qp 403 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c static void pvrdma_free_qp(struct pvrdma_qp *qp) qp 405 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_dev *dev = to_vdev(qp->ibqp.device); qp 411 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c get_cqs(qp, &scq, &rcq); qp 414 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c _pvrdma_flush_cqe(qp, scq); qp 416 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c _pvrdma_flush_cqe(qp, rcq); qp 419 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c dev->qp_tbl[qp->qp_handle] = NULL; qp 424 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (refcount_dec_and_test(&qp->refcnt)) qp 425 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c complete(&qp->free); qp 426 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c wait_for_completion(&qp->free); qp 428 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ib_umem_release(qp->rumem); qp 429 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ib_umem_release(qp->sumem); qp 431 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_page_dir_cleanup(dev, &qp->pdir); qp 433 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c kfree(qp); qp 445 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c int pvrdma_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) qp 447 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *vqp = to_vqp(qp); qp 456 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c ret = pvrdma_cmd_post(to_vdev(qp->device), &req, NULL, 0); qp 458 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c dev_warn(&to_vdev(qp->device)->pdev->dev, qp 479 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *qp = to_vqp(ibqp); qp 487 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c mutex_lock(&qp->mutex); qp 489 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->state; qp 521 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->qkey = attr->qkey; qp 528 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->state = next_state; qp 531 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c cmd->qp_handle = qp->qp_handle; qp 573 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_reset_qp(qp); qp 576 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c mutex_unlock(&qp->mutex); qp 581 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c static inline void *get_sq_wqe(struct pvrdma_qp *qp, unsigned int n) qp 583 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c return pvrdma_page_dir_get_ptr(&qp->pdir, qp 584 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.offset + n * qp->sq.wqe_size); qp 587 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c static inline void *get_rq_wqe(struct pvrdma_qp *qp, unsigned int n) qp 589 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c return pvrdma_page_dir_get_ptr(&qp->pdir, qp 590 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.offset + n * qp->rq.wqe_size); qp 621 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *qp = to_vqp(ibqp); qp 632 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->state < IB_QPS_RTS) { qp 637 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c spin_lock_irqsave(&qp->sq.lock, flags); qp 643 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.ring, qp->sq.wqe_cnt, &tail))) { qp 651 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (unlikely(wr->num_sge > qp->sq.max_sg || wr->num_sge < 0)) { qp 685 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->ibqp.qp_type != IB_QPT_UD && qp 686 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->ibqp.qp_type != IB_QPT_RC && qp 693 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c } else if (qp->ibqp.qp_type == IB_QPT_UD || qp 694 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->ibqp.qp_type == IB_QPT_GSI) { qp 705 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c wqe_hdr = (struct pvrdma_sq_wqe_hdr *)get_sq_wqe(qp, tail); qp 721 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c switch (qp->ibqp.qp_type) { qp 739 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->qkey : ud_wr(wr)->remote_qkey; qp 803 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_idx_ring_inc(&qp->sq.ring->prod_tail, qp 804 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->sq.wqe_cnt); qp 812 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c spin_unlock_irqrestore(&qp->sq.lock, flags); qp 815 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_write_uar_qp(dev, PVRDMA_UAR_QP_SEND | qp->qp_handle); qp 833 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *qp = to_vqp(ibqp); qp 843 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->state == IB_QPS_RESET) { qp 848 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->srq) { qp 854 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c spin_lock_irqsave(&qp->rq.lock, flags); qp 859 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (unlikely(wr->num_sge > qp->rq.max_sg || qp 869 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.ring, qp->rq.wqe_cnt, &tail))) { qp 877 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c wqe_hdr = (struct pvrdma_rq_wqe_hdr *)get_rq_wqe(qp, tail); qp 894 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_idx_ring_inc(&qp->rq.ring->prod_tail, qp 895 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->rq.wqe_cnt); qp 900 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c spin_unlock_irqrestore(&qp->rq.lock, flags); qp 902 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c pvrdma_write_uar_qp(dev, PVRDMA_UAR_QP_RECV | qp->qp_handle); qp 907 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c spin_unlock_irqrestore(&qp->rq.lock, flags); qp 925 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c struct pvrdma_qp *qp = to_vqp(ibqp); qp 932 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c mutex_lock(&qp->mutex); qp 934 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c if (qp->state == IB_QPS_RESET) { qp 941 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c cmd->qp_handle = qp->qp_handle; qp 980 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c qp->state = attr->qp_state; qp 987 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_attr->event_handler = qp->ibqp.event_handler; qp 988 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_attr->qp_context = qp->ibqp.qp_context; qp 989 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_attr->send_cq = qp->ibqp.send_cq; qp 990 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_attr->recv_cq = qp->ibqp.recv_cq; qp 991 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_attr->srq = qp->ibqp.srq; qp 995 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_attr->qp_type = qp->ibqp.qp_type; qp 997 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c init_attr->port_num = qp->port; qp 999 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c mutex_unlock(&qp->mutex); qp 435 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h int pvrdma_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); qp 128 drivers/infiniband/sw/rdmavt/cq.c uqueue[head].qp_num = entry->qp->qp_num; qp 75 drivers/infiniband/sw/rdmavt/mcast.c static struct rvt_mcast_qp *rvt_mcast_qp_alloc(struct rvt_qp *qp) qp 83 drivers/infiniband/sw/rdmavt/mcast.c mqp->qp = qp; qp 84 drivers/infiniband/sw/rdmavt/mcast.c rvt_get_qp(qp); qp 92 drivers/infiniband/sw/rdmavt/mcast.c struct rvt_qp *qp = mqp->qp; qp 95 drivers/infiniband/sw/rdmavt/mcast.c rvt_put_qp(qp); qp 228 drivers/infiniband/sw/rdmavt/mcast.c if (p->qp == mqp->qp) { qp 282 drivers/infiniband/sw/rdmavt/mcast.c struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); qp 284 drivers/infiniband/sw/rdmavt/mcast.c struct rvt_ibport *ibp = rdi->ports[qp->port_num - 1]; qp 289 drivers/infiniband/sw/rdmavt/mcast.c if (ibqp->qp_num <= 1 || qp->state == IB_QPS_RESET) qp 300 drivers/infiniband/sw/rdmavt/mcast.c mqp = rvt_mcast_qp_alloc(qp); qp 345 drivers/infiniband/sw/rdmavt/mcast.c struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); qp 347 drivers/infiniband/sw/rdmavt/mcast.c struct rvt_ibport *ibp = rdi->ports[qp->port_num - 1]; qp 386 drivers/infiniband/sw/rdmavt/mcast.c if (p->qp != qp) qp 451 drivers/infiniband/sw/rdmavt/mr.c static void rvt_dereg_clean_qp_cb(struct rvt_qp *qp, u64 v) qp 456 drivers/infiniband/sw/rdmavt/mr.c if (mr->pd != qp->ibqp.pd) qp 458 drivers/infiniband/sw/rdmavt/mr.c rvt_qp_mr_clean(qp, mr->lkey); qp 658 drivers/infiniband/sw/rdmavt/mr.c int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key, qp 663 drivers/infiniband/sw/rdmavt/mr.c if (qp->ibqp.pd != mr->mr.pd) qp 691 drivers/infiniband/sw/rdmavt/mr.c int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey) qp 693 drivers/infiniband/sw/rdmavt/mr.c struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device); qp 703 drivers/infiniband/sw/rdmavt/mr.c if (unlikely(!mr || mr->lkey != rkey || qp->ibqp.pd != mr->pd)) qp 1024 drivers/infiniband/sw/rdmavt/mr.c int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, qp 1027 drivers/infiniband/sw/rdmavt/mr.c struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device); qp 1039 drivers/infiniband/sw/rdmavt/mr.c struct rvt_pd *pd = ibpd_to_rvtpd(qp->ibqp.pd); qp 1067 drivers/infiniband/sw/rdmavt/mr.c mr->lkey != rkey || qp->ibqp.pd != mr->pd)) qp 64 drivers/infiniband/sw/rdmavt/qp.c static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 464 drivers/infiniband/sw/rdmavt/qp.c static void rvt_free_qp_cb(struct rvt_qp *qp, u64 v) qp 467 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 470 drivers/infiniband/sw/rdmavt/qp.c rvt_reset_qp(rdi, qp, qp->ibqp.qp_type); qp 618 drivers/infiniband/sw/rdmavt/qp.c static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends) qp 621 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 623 drivers/infiniband/sw/rdmavt/qp.c if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) qp 624 drivers/infiniband/sw/rdmavt/qp.c rvt_put_ss(&qp->s_rdma_read_sge); qp 626 drivers/infiniband/sw/rdmavt/qp.c rvt_put_ss(&qp->r_sge); qp 629 drivers/infiniband/sw/rdmavt/qp.c while (qp->s_last != qp->s_head) { qp 630 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last); qp 632 drivers/infiniband/sw/rdmavt/qp.c rvt_put_qp_swqe(qp, wqe); qp 633 drivers/infiniband/sw/rdmavt/qp.c if (++qp->s_last >= qp->s_size) qp 634 drivers/infiniband/sw/rdmavt/qp.c qp->s_last = 0; qp 637 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_rdma_mr) { qp 638 drivers/infiniband/sw/rdmavt/qp.c rvt_put_mr(qp->s_rdma_mr); qp 639 drivers/infiniband/sw/rdmavt/qp.c qp->s_rdma_mr = NULL; qp 643 drivers/infiniband/sw/rdmavt/qp.c for (n = 0; qp->s_ack_queue && n < rvt_max_atomic(rdi); n++) { qp 644 drivers/infiniband/sw/rdmavt/qp.c struct rvt_ack_entry *e = &qp->s_ack_queue[n]; qp 678 drivers/infiniband/sw/rdmavt/qp.c static bool rvt_qp_sends_has_lkey(struct rvt_qp *qp, u32 lkey) qp 680 drivers/infiniband/sw/rdmavt/qp.c u32 s_last = qp->s_last; qp 682 drivers/infiniband/sw/rdmavt/qp.c while (s_last != qp->s_head) { qp 683 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last); qp 688 drivers/infiniband/sw/rdmavt/qp.c if (++s_last >= qp->s_size) qp 691 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_rdma_mr) qp 692 drivers/infiniband/sw/rdmavt/qp.c if (rvt_mr_has_lkey(qp->s_rdma_mr, lkey)) qp 702 drivers/infiniband/sw/rdmavt/qp.c static bool rvt_qp_acks_has_lkey(struct rvt_qp *qp, u32 lkey) qp 705 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 707 drivers/infiniband/sw/rdmavt/qp.c for (i = 0; qp->s_ack_queue && i < rvt_max_atomic(rdi); i++) { qp 708 drivers/infiniband/sw/rdmavt/qp.c struct rvt_ack_entry *e = &qp->s_ack_queue[i]; qp 727 drivers/infiniband/sw/rdmavt/qp.c void rvt_qp_mr_clean(struct rvt_qp *qp, u32 lkey) qp 731 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.qp_type == IB_QPT_SMI || qp 732 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.qp_type == IB_QPT_GSI) qp 735 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irq(&qp->r_lock); qp 736 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_hlock); qp 737 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_lock); qp 739 drivers/infiniband/sw/rdmavt/qp.c if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET) qp 742 drivers/infiniband/sw/rdmavt/qp.c if (rvt_ss_has_lkey(&qp->r_sge, lkey) || qp 743 drivers/infiniband/sw/rdmavt/qp.c rvt_qp_sends_has_lkey(qp, lkey) || qp 744 drivers/infiniband/sw/rdmavt/qp.c rvt_qp_acks_has_lkey(qp, lkey)) qp 745 drivers/infiniband/sw/rdmavt/qp.c lastwqe = rvt_error_qp(qp, IB_WC_LOC_PROT_ERR); qp 747 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_lock); qp 748 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_hlock); qp 749 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irq(&qp->r_lock); qp 753 drivers/infiniband/sw/rdmavt/qp.c ev.device = qp->ibqp.device; qp 754 drivers/infiniband/sw/rdmavt/qp.c ev.element.qp = &qp->ibqp; qp 756 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 768 drivers/infiniband/sw/rdmavt/qp.c static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) qp 770 drivers/infiniband/sw/rdmavt/qp.c struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; qp 771 drivers/infiniband/sw/rdmavt/qp.c u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); qp 777 drivers/infiniband/sw/rdmavt/qp.c if (rcu_dereference_protected(rvp->qp[0], qp 778 drivers/infiniband/sw/rdmavt/qp.c lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { qp 779 drivers/infiniband/sw/rdmavt/qp.c RCU_INIT_POINTER(rvp->qp[0], NULL); qp 780 drivers/infiniband/sw/rdmavt/qp.c } else if (rcu_dereference_protected(rvp->qp[1], qp 781 drivers/infiniband/sw/rdmavt/qp.c lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { qp 782 drivers/infiniband/sw/rdmavt/qp.c RCU_INIT_POINTER(rvp->qp[1], NULL); qp 792 drivers/infiniband/sw/rdmavt/qp.c if (q == qp) { qp 794 drivers/infiniband/sw/rdmavt/qp.c rcu_dereference_protected(qp->next, qp 797 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_qpremove(qp, n); qp 806 drivers/infiniband/sw/rdmavt/qp.c rvt_put_qp(qp); qp 861 drivers/infiniband/sw/rdmavt/qp.c static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 864 drivers/infiniband/sw/rdmavt/qp.c qp->remote_qpn = 0; qp 865 drivers/infiniband/sw/rdmavt/qp.c qp->qkey = 0; qp 866 drivers/infiniband/sw/rdmavt/qp.c qp->qp_access_flags = 0; qp 867 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags &= RVT_S_SIGNAL_REQ_WR; qp 868 drivers/infiniband/sw/rdmavt/qp.c qp->s_hdrwords = 0; qp 869 drivers/infiniband/sw/rdmavt/qp.c qp->s_wqe = NULL; qp 870 drivers/infiniband/sw/rdmavt/qp.c qp->s_draining = 0; qp 871 drivers/infiniband/sw/rdmavt/qp.c qp->s_next_psn = 0; qp 872 drivers/infiniband/sw/rdmavt/qp.c qp->s_last_psn = 0; qp 873 drivers/infiniband/sw/rdmavt/qp.c qp->s_sending_psn = 0; qp 874 drivers/infiniband/sw/rdmavt/qp.c qp->s_sending_hpsn = 0; qp 875 drivers/infiniband/sw/rdmavt/qp.c qp->s_psn = 0; qp 876 drivers/infiniband/sw/rdmavt/qp.c qp->r_psn = 0; qp 877 drivers/infiniband/sw/rdmavt/qp.c qp->r_msn = 0; qp 879 drivers/infiniband/sw/rdmavt/qp.c qp->s_state = IB_OPCODE_RC_SEND_LAST; qp 880 drivers/infiniband/sw/rdmavt/qp.c qp->r_state = IB_OPCODE_RC_SEND_LAST; qp 882 drivers/infiniband/sw/rdmavt/qp.c qp->s_state = IB_OPCODE_UC_SEND_LAST; qp 883 drivers/infiniband/sw/rdmavt/qp.c qp->r_state = IB_OPCODE_UC_SEND_LAST; qp 885 drivers/infiniband/sw/rdmavt/qp.c qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE; qp 886 drivers/infiniband/sw/rdmavt/qp.c qp->r_nak_state = 0; qp 887 drivers/infiniband/sw/rdmavt/qp.c qp->r_aflags = 0; qp 888 drivers/infiniband/sw/rdmavt/qp.c qp->r_flags = 0; qp 889 drivers/infiniband/sw/rdmavt/qp.c qp->s_head = 0; qp 890 drivers/infiniband/sw/rdmavt/qp.c qp->s_tail = 0; qp 891 drivers/infiniband/sw/rdmavt/qp.c qp->s_cur = 0; qp 892 drivers/infiniband/sw/rdmavt/qp.c qp->s_acked = 0; qp 893 drivers/infiniband/sw/rdmavt/qp.c qp->s_last = 0; qp 894 drivers/infiniband/sw/rdmavt/qp.c qp->s_ssn = 1; qp 895 drivers/infiniband/sw/rdmavt/qp.c qp->s_lsn = 0; qp 896 drivers/infiniband/sw/rdmavt/qp.c qp->s_mig_state = IB_MIG_MIGRATED; qp 897 drivers/infiniband/sw/rdmavt/qp.c qp->r_head_ack_queue = 0; qp 898 drivers/infiniband/sw/rdmavt/qp.c qp->s_tail_ack_queue = 0; qp 899 drivers/infiniband/sw/rdmavt/qp.c qp->s_acked_ack_queue = 0; qp 900 drivers/infiniband/sw/rdmavt/qp.c qp->s_num_rd_atomic = 0; qp 901 drivers/infiniband/sw/rdmavt/qp.c if (qp->r_rq.kwq) qp 902 drivers/infiniband/sw/rdmavt/qp.c qp->r_rq.kwq->count = qp->r_rq.size; qp 903 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.num_sge = 0; qp 904 drivers/infiniband/sw/rdmavt/qp.c atomic_set(&qp->s_reserved_used, 0); qp 914 drivers/infiniband/sw/rdmavt/qp.c static void _rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 916 drivers/infiniband/sw/rdmavt/qp.c __must_hold(&qp->s_lock) qp 917 drivers/infiniband/sw/rdmavt/qp.c __must_hold(&qp->s_hlock) qp 918 drivers/infiniband/sw/rdmavt/qp.c __must_hold(&qp->r_lock) qp 920 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->r_lock); qp 921 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_hlock); qp 922 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_lock); qp 923 drivers/infiniband/sw/rdmavt/qp.c if (qp->state != IB_QPS_RESET) { qp 924 drivers/infiniband/sw/rdmavt/qp.c qp->state = IB_QPS_RESET; qp 927 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.flush_qp_waiters(qp); qp 928 drivers/infiniband/sw/rdmavt/qp.c rvt_stop_rc_timers(qp); qp 929 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags &= ~(RVT_S_TIMER | RVT_S_ANY_WAIT); qp 930 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_lock); qp 931 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_hlock); qp 932 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irq(&qp->r_lock); qp 935 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.stop_send_queue(qp); qp 936 drivers/infiniband/sw/rdmavt/qp.c rvt_del_timers_sync(qp); qp 938 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.quiesce_qp(qp); qp 941 drivers/infiniband/sw/rdmavt/qp.c rvt_remove_qp(rdi, qp); qp 944 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irq(&qp->r_lock); qp 945 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_hlock); qp 946 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_lock); qp 948 drivers/infiniband/sw/rdmavt/qp.c rvt_clear_mr_refs(qp, 1); qp 953 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.notify_qp_reset(qp); qp 955 drivers/infiniband/sw/rdmavt/qp.c rvt_init_qp(rdi, qp, type); qp 956 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->r_lock); qp 957 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_hlock); qp 958 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_lock); qp 970 drivers/infiniband/sw/rdmavt/qp.c static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 973 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irq(&qp->r_lock); qp 974 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_hlock); qp 975 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_lock); qp 976 drivers/infiniband/sw/rdmavt/qp.c _rvt_reset_qp(rdi, qp, type); qp 977 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_lock); qp 978 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_hlock); qp 979 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irq(&qp->r_lock); qp 1012 drivers/infiniband/sw/rdmavt/qp.c static void free_ud_wq_attr(struct rvt_qp *qp) qp 1017 drivers/infiniband/sw/rdmavt/qp.c for (i = 0; qp->allowed_ops == IB_OPCODE_UD && i < qp->s_size; i++) { qp 1018 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_swqe_ptr(qp, i); qp 1032 drivers/infiniband/sw/rdmavt/qp.c static int alloc_ud_wq_attr(struct rvt_qp *qp, int node) qp 1037 drivers/infiniband/sw/rdmavt/qp.c for (i = 0; qp->allowed_ops == IB_OPCODE_UD && i < qp->s_size; i++) { qp 1038 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_swqe_ptr(qp, i); qp 1042 drivers/infiniband/sw/rdmavt/qp.c free_ud_wq_attr(qp); qp 1068 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp; qp 1117 drivers/infiniband/sw/rdmavt/qp.c sz = sizeof(*qp); qp 1123 drivers/infiniband/sw/rdmavt/qp.c sg_list_sz = sizeof(*qp->r_sg_list) * qp 1126 drivers/infiniband/sw/rdmavt/qp.c sg_list_sz = sizeof(*qp->r_sg_list) * qp 1128 drivers/infiniband/sw/rdmavt/qp.c qp = kzalloc_node(sz + sg_list_sz, GFP_KERNEL, qp 1130 drivers/infiniband/sw/rdmavt/qp.c if (!qp) qp 1132 drivers/infiniband/sw/rdmavt/qp.c qp->allowed_ops = get_allowed_ops(init_attr->qp_type); qp 1134 drivers/infiniband/sw/rdmavt/qp.c RCU_INIT_POINTER(qp->next, NULL); qp 1136 drivers/infiniband/sw/rdmavt/qp.c qp->s_ack_queue = qp 1138 drivers/infiniband/sw/rdmavt/qp.c sizeof(*qp->s_ack_queue), qp 1141 drivers/infiniband/sw/rdmavt/qp.c if (!qp->s_ack_queue) qp 1145 drivers/infiniband/sw/rdmavt/qp.c timer_setup(&qp->s_timer, rvt_rc_timeout, 0); qp 1146 drivers/infiniband/sw/rdmavt/qp.c hrtimer_init(&qp->s_rnr_timer, CLOCK_MONOTONIC, qp 1148 drivers/infiniband/sw/rdmavt/qp.c qp->s_rnr_timer.function = rvt_rc_rnr_retry; qp 1154 drivers/infiniband/sw/rdmavt/qp.c priv = rdi->driver_f.qp_priv_alloc(rdi, qp); qp 1159 drivers/infiniband/sw/rdmavt/qp.c qp->priv = priv; qp 1160 drivers/infiniband/sw/rdmavt/qp.c qp->timeout_jiffies = qp 1161 drivers/infiniband/sw/rdmavt/qp.c usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / qp 1166 drivers/infiniband/sw/rdmavt/qp.c qp->r_rq.size = init_attr->cap.max_recv_wr + 1; qp 1167 drivers/infiniband/sw/rdmavt/qp.c qp->r_rq.max_sge = init_attr->cap.max_recv_sge; qp 1168 drivers/infiniband/sw/rdmavt/qp.c sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + qp 1170 drivers/infiniband/sw/rdmavt/qp.c err = rvt_alloc_rq(&qp->r_rq, qp->r_rq.size * sz, qp 1182 drivers/infiniband/sw/rdmavt/qp.c spin_lock_init(&qp->r_lock); qp 1183 drivers/infiniband/sw/rdmavt/qp.c spin_lock_init(&qp->s_hlock); qp 1184 drivers/infiniband/sw/rdmavt/qp.c spin_lock_init(&qp->s_lock); qp 1185 drivers/infiniband/sw/rdmavt/qp.c atomic_set(&qp->refcount, 0); qp 1186 drivers/infiniband/sw/rdmavt/qp.c atomic_set(&qp->local_ops_pending, 0); qp 1187 drivers/infiniband/sw/rdmavt/qp.c init_waitqueue_head(&qp->wait); qp 1188 drivers/infiniband/sw/rdmavt/qp.c INIT_LIST_HEAD(&qp->rspwait); qp 1189 drivers/infiniband/sw/rdmavt/qp.c qp->state = IB_QPS_RESET; qp 1190 drivers/infiniband/sw/rdmavt/qp.c qp->s_wq = swq; qp 1191 drivers/infiniband/sw/rdmavt/qp.c qp->s_size = sqsize; qp 1192 drivers/infiniband/sw/rdmavt/qp.c qp->s_avail = init_attr->cap.max_send_wr; qp 1193 drivers/infiniband/sw/rdmavt/qp.c qp->s_max_sge = init_attr->cap.max_send_sge; qp 1195 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags = RVT_S_SIGNAL_REQ_WR; qp 1196 drivers/infiniband/sw/rdmavt/qp.c err = alloc_ud_wq_attr(qp, rdi->dparms.node); qp 1209 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.qp_num = err; qp 1210 drivers/infiniband/sw/rdmavt/qp.c qp->port_num = init_attr->port_num; qp 1211 drivers/infiniband/sw/rdmavt/qp.c rvt_init_qp(rdi, qp, init_attr->qp_type); qp 1213 drivers/infiniband/sw/rdmavt/qp.c err = rdi->driver_f.qp_priv_init(rdi, qp, init_attr); qp 1233 drivers/infiniband/sw/rdmavt/qp.c if (!qp->r_rq.wq) { qp 1243 drivers/infiniband/sw/rdmavt/qp.c u32 s = sizeof(struct rvt_rwq) + qp->r_rq.size * sz; qp 1245 drivers/infiniband/sw/rdmavt/qp.c qp->ip = rvt_create_mmap_info(rdi, s, udata, qp 1246 drivers/infiniband/sw/rdmavt/qp.c qp->r_rq.wq); qp 1247 drivers/infiniband/sw/rdmavt/qp.c if (IS_ERR(qp->ip)) { qp 1248 drivers/infiniband/sw/rdmavt/qp.c ret = ERR_CAST(qp->ip); qp 1252 drivers/infiniband/sw/rdmavt/qp.c err = ib_copy_to_udata(udata, &qp->ip->offset, qp 1253 drivers/infiniband/sw/rdmavt/qp.c sizeof(qp->ip->offset)); qp 1259 drivers/infiniband/sw/rdmavt/qp.c qp->pid = current->pid; qp 1285 drivers/infiniband/sw/rdmavt/qp.c if (qp->ip) { qp 1287 drivers/infiniband/sw/rdmavt/qp.c list_add(&qp->ip->pending_mmaps, &rdi->pending_mmaps); qp 1291 drivers/infiniband/sw/rdmavt/qp.c ret = &qp->ibqp; qp 1296 drivers/infiniband/sw/rdmavt/qp.c if (qp->ip) qp 1297 drivers/infiniband/sw/rdmavt/qp.c kref_put(&qp->ip->ref, rvt_release_mmap_info); qp 1300 drivers/infiniband/sw/rdmavt/qp.c rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); qp 1303 drivers/infiniband/sw/rdmavt/qp.c rvt_free_rq(&qp->r_rq); qp 1304 drivers/infiniband/sw/rdmavt/qp.c free_ud_wq_attr(qp); qp 1307 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.qp_priv_free(rdi, qp); qp 1310 drivers/infiniband/sw/rdmavt/qp.c kfree(qp->s_ack_queue); qp 1311 drivers/infiniband/sw/rdmavt/qp.c kfree(qp); qp 1330 drivers/infiniband/sw/rdmavt/qp.c int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err) qp 1334 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 1336 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->r_lock); qp 1337 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_lock); qp 1338 drivers/infiniband/sw/rdmavt/qp.c if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET) qp 1341 drivers/infiniband/sw/rdmavt/qp.c qp->state = IB_QPS_ERR; qp 1343 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) { qp 1344 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR); qp 1345 drivers/infiniband/sw/rdmavt/qp.c del_timer(&qp->s_timer); qp 1348 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_flags & RVT_S_ANY_WAIT_SEND) qp 1349 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags &= ~RVT_S_ANY_WAIT_SEND; qp 1351 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.notify_error_qp(qp); qp 1354 drivers/infiniband/sw/rdmavt/qp.c if (READ_ONCE(qp->s_last) != qp->s_head) qp 1355 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.schedule_send(qp); qp 1357 drivers/infiniband/sw/rdmavt/qp.c rvt_clear_mr_refs(qp, 0); qp 1360 drivers/infiniband/sw/rdmavt/qp.c wc.qp = &qp->ibqp; qp 1363 drivers/infiniband/sw/rdmavt/qp.c if (test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) { qp 1364 drivers/infiniband/sw/rdmavt/qp.c wc.wr_id = qp->r_wr_id; qp 1366 drivers/infiniband/sw/rdmavt/qp.c rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); qp 1370 drivers/infiniband/sw/rdmavt/qp.c if (qp->r_rq.kwq) { qp 1376 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->r_rq.kwq->c_lock); qp 1378 drivers/infiniband/sw/rdmavt/qp.c if (qp->ip) { qp 1379 drivers/infiniband/sw/rdmavt/qp.c wq = qp->r_rq.wq; qp 1383 drivers/infiniband/sw/rdmavt/qp.c kwq = qp->r_rq.kwq; qp 1388 drivers/infiniband/sw/rdmavt/qp.c if (head >= qp->r_rq.size) qp 1390 drivers/infiniband/sw/rdmavt/qp.c if (tail >= qp->r_rq.size) qp 1393 drivers/infiniband/sw/rdmavt/qp.c wc.wr_id = rvt_get_rwqe_ptr(&qp->r_rq, tail)->wr_id; qp 1394 drivers/infiniband/sw/rdmavt/qp.c if (++tail >= qp->r_rq.size) qp 1396 drivers/infiniband/sw/rdmavt/qp.c rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); qp 1398 drivers/infiniband/sw/rdmavt/qp.c if (qp->ip) qp 1402 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->r_rq.kwq->c_lock); qp 1403 drivers/infiniband/sw/rdmavt/qp.c } else if (qp->ibqp.event_handler) { qp 1416 drivers/infiniband/sw/rdmavt/qp.c static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) qp 1418 drivers/infiniband/sw/rdmavt/qp.c struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; qp 1421 drivers/infiniband/sw/rdmavt/qp.c rvt_get_qp(qp); qp 1424 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.qp_num <= 1) { qp 1425 drivers/infiniband/sw/rdmavt/qp.c rcu_assign_pointer(rvp->qp[qp->ibqp.qp_num], qp); qp 1427 drivers/infiniband/sw/rdmavt/qp.c u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); qp 1429 drivers/infiniband/sw/rdmavt/qp.c qp->next = rdi->qp_dev->qp_table[n]; qp 1430 drivers/infiniband/sw/rdmavt/qp.c rcu_assign_pointer(rdi->qp_dev->qp_table[n], qp); qp 1431 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_qpinsert(qp, n); qp 1450 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); qp 1458 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irq(&qp->r_lock); qp 1459 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_hlock); qp 1460 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_lock); qp 1463 drivers/infiniband/sw/rdmavt/qp.c attr->cur_qp_state : qp->state; qp 1465 drivers/infiniband/sw/rdmavt/qp.c opa_ah = rdma_cap_opa_ah(ibqp->device, qp->port_num); qp 1472 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.check_modify_qp(qp, attr, attr_mask, udata)) qp 1486 drivers/infiniband/sw/rdmavt/qp.c if (rvt_check_ah(qp->ibqp.device, &attr->ah_attr)) qp 1501 drivers/infiniband/sw/rdmavt/qp.c if (rvt_check_ah(qp->ibqp.device, &attr->alt_ah_attr)) qp 1516 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.qp_type == IB_QPT_SMI || qp 1517 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.qp_type == IB_QPT_GSI || qp 1543 drivers/infiniband/sw/rdmavt/qp.c pmtu = rdi->driver_f.get_pmtu_from_attr(rdi, qp, attr); qp 1550 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_mig_state == IB_MIG_ARMED) qp 1555 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_mig_state == IB_MIG_REARM) qp 1559 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_mig_state == IB_MIG_ARMED) qp 1572 drivers/infiniband/sw/rdmavt/qp.c if (qp->state != IB_QPS_RESET) qp 1573 drivers/infiniband/sw/rdmavt/qp.c _rvt_reset_qp(rdi, qp, ibqp->qp_type); qp 1578 drivers/infiniband/sw/rdmavt/qp.c qp->r_flags &= ~RVT_R_COMM_EST; qp 1579 drivers/infiniband/sw/rdmavt/qp.c qp->state = new_state; qp 1583 drivers/infiniband/sw/rdmavt/qp.c qp->s_draining = qp->s_last != qp->s_cur; qp 1584 drivers/infiniband/sw/rdmavt/qp.c qp->state = new_state; qp 1588 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.qp_type == IB_QPT_RC) qp 1590 drivers/infiniband/sw/rdmavt/qp.c qp->state = new_state; qp 1594 drivers/infiniband/sw/rdmavt/qp.c lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); qp 1598 drivers/infiniband/sw/rdmavt/qp.c qp->state = new_state; qp 1603 drivers/infiniband/sw/rdmavt/qp.c qp->s_pkey_index = attr->pkey_index; qp 1606 drivers/infiniband/sw/rdmavt/qp.c qp->port_num = attr->port_num; qp 1609 drivers/infiniband/sw/rdmavt/qp.c qp->remote_qpn = attr->dest_qp_num; qp 1612 drivers/infiniband/sw/rdmavt/qp.c qp->s_next_psn = attr->sq_psn & rdi->dparms.psn_modify_mask; qp 1613 drivers/infiniband/sw/rdmavt/qp.c qp->s_psn = qp->s_next_psn; qp 1614 drivers/infiniband/sw/rdmavt/qp.c qp->s_sending_psn = qp->s_next_psn; qp 1615 drivers/infiniband/sw/rdmavt/qp.c qp->s_last_psn = qp->s_next_psn - 1; qp 1616 drivers/infiniband/sw/rdmavt/qp.c qp->s_sending_hpsn = qp->s_last_psn; qp 1620 drivers/infiniband/sw/rdmavt/qp.c qp->r_psn = attr->rq_psn & rdi->dparms.psn_modify_mask; qp 1623 drivers/infiniband/sw/rdmavt/qp.c qp->qp_access_flags = attr->qp_access_flags; qp 1626 drivers/infiniband/sw/rdmavt/qp.c rdma_replace_ah_attr(&qp->remote_ah_attr, &attr->ah_attr); qp 1627 drivers/infiniband/sw/rdmavt/qp.c qp->s_srate = rdma_ah_get_static_rate(&attr->ah_attr); qp 1628 drivers/infiniband/sw/rdmavt/qp.c qp->srate_mbps = ib_rate_to_mbps(qp->s_srate); qp 1632 drivers/infiniband/sw/rdmavt/qp.c rdma_replace_ah_attr(&qp->alt_ah_attr, &attr->alt_ah_attr); qp 1633 drivers/infiniband/sw/rdmavt/qp.c qp->s_alt_pkey_index = attr->alt_pkey_index; qp 1637 drivers/infiniband/sw/rdmavt/qp.c qp->s_mig_state = attr->path_mig_state; qp 1639 drivers/infiniband/sw/rdmavt/qp.c qp->remote_ah_attr = qp->alt_ah_attr; qp 1640 drivers/infiniband/sw/rdmavt/qp.c qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); qp 1641 drivers/infiniband/sw/rdmavt/qp.c qp->s_pkey_index = qp->s_alt_pkey_index; qp 1646 drivers/infiniband/sw/rdmavt/qp.c qp->pmtu = rdi->driver_f.mtu_from_qp(rdi, qp, pmtu); qp 1647 drivers/infiniband/sw/rdmavt/qp.c qp->log_pmtu = ilog2(qp->pmtu); qp 1651 drivers/infiniband/sw/rdmavt/qp.c qp->s_retry_cnt = attr->retry_cnt; qp 1652 drivers/infiniband/sw/rdmavt/qp.c qp->s_retry = attr->retry_cnt; qp 1656 drivers/infiniband/sw/rdmavt/qp.c qp->s_rnr_retry_cnt = attr->rnr_retry; qp 1657 drivers/infiniband/sw/rdmavt/qp.c qp->s_rnr_retry = attr->rnr_retry; qp 1661 drivers/infiniband/sw/rdmavt/qp.c qp->r_min_rnr_timer = attr->min_rnr_timer; qp 1664 drivers/infiniband/sw/rdmavt/qp.c qp->timeout = attr->timeout; qp 1665 drivers/infiniband/sw/rdmavt/qp.c qp->timeout_jiffies = rvt_timeout_to_jiffies(qp->timeout); qp 1669 drivers/infiniband/sw/rdmavt/qp.c qp->qkey = attr->qkey; qp 1672 drivers/infiniband/sw/rdmavt/qp.c qp->r_max_rd_atomic = attr->max_dest_rd_atomic; qp 1675 drivers/infiniband/sw/rdmavt/qp.c qp->s_max_rd_atomic = attr->max_rd_atomic; qp 1678 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.modify_qp(qp, attr, attr_mask, udata); qp 1680 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_lock); qp 1681 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_hlock); qp 1682 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irq(&qp->r_lock); qp 1685 drivers/infiniband/sw/rdmavt/qp.c rvt_insert_qp(rdi, qp); qp 1688 drivers/infiniband/sw/rdmavt/qp.c ev.device = qp->ibqp.device; qp 1689 drivers/infiniband/sw/rdmavt/qp.c ev.element.qp = &qp->ibqp; qp 1691 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 1694 drivers/infiniband/sw/rdmavt/qp.c ev.device = qp->ibqp.device; qp 1695 drivers/infiniband/sw/rdmavt/qp.c ev.element.qp = &qp->ibqp; qp 1697 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 1702 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_lock); qp 1703 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_hlock); qp 1704 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irq(&qp->r_lock); qp 1719 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); qp 1722 drivers/infiniband/sw/rdmavt/qp.c rvt_reset_qp(rdi, qp, ibqp->qp_type); qp 1724 drivers/infiniband/sw/rdmavt/qp.c wait_event(qp->wait, !atomic_read(&qp->refcount)); qp 1726 drivers/infiniband/sw/rdmavt/qp.c rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); qp 1730 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.qp_type == IB_QPT_RC) { qp 1736 drivers/infiniband/sw/rdmavt/qp.c if (qp->ip) qp 1737 drivers/infiniband/sw/rdmavt/qp.c kref_put(&qp->ip->ref, rvt_release_mmap_info); qp 1738 drivers/infiniband/sw/rdmavt/qp.c kvfree(qp->r_rq.kwq); qp 1739 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.qp_priv_free(rdi, qp); qp 1740 drivers/infiniband/sw/rdmavt/qp.c kfree(qp->s_ack_queue); qp 1741 drivers/infiniband/sw/rdmavt/qp.c rdma_destroy_ah_attr(&qp->remote_ah_attr); qp 1742 drivers/infiniband/sw/rdmavt/qp.c rdma_destroy_ah_attr(&qp->alt_ah_attr); qp 1743 drivers/infiniband/sw/rdmavt/qp.c free_ud_wq_attr(qp); qp 1744 drivers/infiniband/sw/rdmavt/qp.c vfree(qp->s_wq); qp 1745 drivers/infiniband/sw/rdmavt/qp.c kfree(qp); qp 1761 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); qp 1764 drivers/infiniband/sw/rdmavt/qp.c attr->qp_state = qp->state; qp 1766 drivers/infiniband/sw/rdmavt/qp.c attr->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu); qp 1767 drivers/infiniband/sw/rdmavt/qp.c attr->path_mig_state = qp->s_mig_state; qp 1768 drivers/infiniband/sw/rdmavt/qp.c attr->qkey = qp->qkey; qp 1769 drivers/infiniband/sw/rdmavt/qp.c attr->rq_psn = qp->r_psn & rdi->dparms.psn_mask; qp 1770 drivers/infiniband/sw/rdmavt/qp.c attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask; qp 1771 drivers/infiniband/sw/rdmavt/qp.c attr->dest_qp_num = qp->remote_qpn; qp 1772 drivers/infiniband/sw/rdmavt/qp.c attr->qp_access_flags = qp->qp_access_flags; qp 1773 drivers/infiniband/sw/rdmavt/qp.c attr->cap.max_send_wr = qp->s_size - 1 - qp 1775 drivers/infiniband/sw/rdmavt/qp.c attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1; qp 1776 drivers/infiniband/sw/rdmavt/qp.c attr->cap.max_send_sge = qp->s_max_sge; qp 1777 drivers/infiniband/sw/rdmavt/qp.c attr->cap.max_recv_sge = qp->r_rq.max_sge; qp 1779 drivers/infiniband/sw/rdmavt/qp.c attr->ah_attr = qp->remote_ah_attr; qp 1780 drivers/infiniband/sw/rdmavt/qp.c attr->alt_ah_attr = qp->alt_ah_attr; qp 1781 drivers/infiniband/sw/rdmavt/qp.c attr->pkey_index = qp->s_pkey_index; qp 1782 drivers/infiniband/sw/rdmavt/qp.c attr->alt_pkey_index = qp->s_alt_pkey_index; qp 1784 drivers/infiniband/sw/rdmavt/qp.c attr->sq_draining = qp->s_draining; qp 1785 drivers/infiniband/sw/rdmavt/qp.c attr->max_rd_atomic = qp->s_max_rd_atomic; qp 1786 drivers/infiniband/sw/rdmavt/qp.c attr->max_dest_rd_atomic = qp->r_max_rd_atomic; qp 1787 drivers/infiniband/sw/rdmavt/qp.c attr->min_rnr_timer = qp->r_min_rnr_timer; qp 1788 drivers/infiniband/sw/rdmavt/qp.c attr->port_num = qp->port_num; qp 1789 drivers/infiniband/sw/rdmavt/qp.c attr->timeout = qp->timeout; qp 1790 drivers/infiniband/sw/rdmavt/qp.c attr->retry_cnt = qp->s_retry_cnt; qp 1791 drivers/infiniband/sw/rdmavt/qp.c attr->rnr_retry = qp->s_rnr_retry_cnt; qp 1793 drivers/infiniband/sw/rdmavt/qp.c rdma_ah_get_port_num(&qp->alt_ah_attr); qp 1794 drivers/infiniband/sw/rdmavt/qp.c attr->alt_timeout = qp->alt_timeout; qp 1796 drivers/infiniband/sw/rdmavt/qp.c init_attr->event_handler = qp->ibqp.event_handler; qp 1797 drivers/infiniband/sw/rdmavt/qp.c init_attr->qp_context = qp->ibqp.qp_context; qp 1798 drivers/infiniband/sw/rdmavt/qp.c init_attr->send_cq = qp->ibqp.send_cq; qp 1799 drivers/infiniband/sw/rdmavt/qp.c init_attr->recv_cq = qp->ibqp.recv_cq; qp 1800 drivers/infiniband/sw/rdmavt/qp.c init_attr->srq = qp->ibqp.srq; qp 1802 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_flags & RVT_S_SIGNAL_REQ_WR) qp 1806 drivers/infiniband/sw/rdmavt/qp.c init_attr->qp_type = qp->ibqp.qp_type; qp 1807 drivers/infiniband/sw/rdmavt/qp.c init_attr->port_num = qp->port_num; qp 1824 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); qp 1825 drivers/infiniband/sw/rdmavt/qp.c struct rvt_krwq *wq = qp->r_rq.kwq; qp 1827 drivers/infiniband/sw/rdmavt/qp.c int qp_err_flush = (ib_rvt_state_ops[qp->state] & RVT_FLUSH_RECV) && qp 1828 drivers/infiniband/sw/rdmavt/qp.c !qp->ibqp.srq; qp 1831 drivers/infiniband/sw/rdmavt/qp.c if (!(ib_rvt_state_ops[qp->state] & RVT_POST_RECV_OK) || !wq) { qp 1841 drivers/infiniband/sw/rdmavt/qp.c if ((unsigned)wr->num_sge > qp->r_rq.max_sge) { qp 1846 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irqsave(&qp->r_rq.kwq->p_lock, flags); qp 1848 drivers/infiniband/sw/rdmavt/qp.c if (next >= qp->r_rq.size) qp 1851 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->r_rq.kwq->p_lock, flags); qp 1859 drivers/infiniband/sw/rdmavt/qp.c wc.qp = &qp->ibqp; qp 1863 drivers/infiniband/sw/rdmavt/qp.c rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); qp 1865 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_rwqe_ptr(&qp->r_rq, wq->head); qp 1879 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->r_rq.kwq->p_lock, flags); qp 1903 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp, qp 1911 drivers/infiniband/sw/rdmavt/qp.c if (!(post_parms[wr->opcode].qpt_support & BIT(qp->ibqp.qp_type))) qp 1914 drivers/infiniband/sw/rdmavt/qp.c ibpd_to_rvtpd(qp->ibqp.pd)->user) qp 1922 drivers/infiniband/sw/rdmavt/qp.c !qp->s_max_rd_atomic) qp 1926 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.qp_type != IB_QPT_UC && qp 1927 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.qp_type != IB_QPT_RC) { qp 1928 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.pd != ud_wr(wr)->ah->pd) qp 1950 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp, qp 1962 drivers/infiniband/sw/rdmavt/qp.c reserved_used = atomic_read(&qp->s_reserved_used); qp 1968 drivers/infiniband/sw/rdmavt/qp.c if (likely(qp->s_avail)) qp 1971 drivers/infiniband/sw/rdmavt/qp.c slast = smp_load_acquire(&qp->s_last); qp 1972 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_head >= slast) qp 1973 drivers/infiniband/sw/rdmavt/qp.c avail = qp->s_size - (qp->s_head - slast); qp 1975 drivers/infiniband/sw/rdmavt/qp.c avail = slast - qp->s_head; qp 1977 drivers/infiniband/sw/rdmavt/qp.c reserved_used = atomic_read(&qp->s_reserved_used); qp 1983 drivers/infiniband/sw/rdmavt/qp.c qp->s_avail = avail; qp 1984 drivers/infiniband/sw/rdmavt/qp.c if (WARN_ON(qp->s_avail > qp 1985 drivers/infiniband/sw/rdmavt/qp.c (qp->s_size - 1 - rdi->dparms.reserved_operations))) qp 1988 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.qp_num, qp->s_size, qp->s_avail, qp 1989 drivers/infiniband/sw/rdmavt/qp.c qp->s_head, qp->s_tail, qp->s_cur, qp 1990 drivers/infiniband/sw/rdmavt/qp.c qp->s_acked, qp->s_last); qp 1999 drivers/infiniband/sw/rdmavt/qp.c static int rvt_post_one_wr(struct rvt_qp *qp, qp 2010 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 2020 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(wr->num_sge > qp->s_max_sge)) qp 2023 drivers/infiniband/sw/rdmavt/qp.c ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr); qp 2042 drivers/infiniband/sw/rdmavt/qp.c ret = rvt_fast_reg_mr(qp, qp 2051 drivers/infiniband/sw/rdmavt/qp.c atomic_read(&qp->local_ops_pending)) { qp 2055 drivers/infiniband/sw/rdmavt/qp.c qp, wr->ex.invalidate_rkey); qp 2068 drivers/infiniband/sw/rdmavt/qp.c ret = rvt_qp_is_avail(qp, rdi, reserved_op); qp 2071 drivers/infiniband/sw/rdmavt/qp.c next = qp->s_head + 1; qp 2072 drivers/infiniband/sw/rdmavt/qp.c if (next >= qp->s_size) qp 2076 drivers/infiniband/sw/rdmavt/qp.c pd = ibpd_to_rvtpd(qp->ibqp.pd); qp 2077 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_swqe_ptr(qp, qp->s_head); qp 2111 drivers/infiniband/sw/rdmavt/qp.c log_pmtu = qp->log_pmtu; qp 2112 drivers/infiniband/sw/rdmavt/qp.c if (qp->allowed_ops == IB_OPCODE_UD) { qp 2121 drivers/infiniband/sw/rdmavt/qp.c atomic_inc(&qp->local_ops_pending); qp 2128 drivers/infiniband/sw/rdmavt/qp.c wqe->ssn = qp->s_ssn++; qp 2129 drivers/infiniband/sw/rdmavt/qp.c wqe->psn = qp->s_next_psn; qp 2138 drivers/infiniband/sw/rdmavt/qp.c ret = rdi->driver_f.setup_wqe(qp, wqe, call_send); qp 2144 drivers/infiniband/sw/rdmavt/qp.c qp->s_next_psn = wqe->lpsn + 1; qp 2148 drivers/infiniband/sw/rdmavt/qp.c rvt_qp_wqe_reserve(qp, wqe); qp 2151 drivers/infiniband/sw/rdmavt/qp.c qp->s_avail--; qp 2153 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_post_one_wr(qp, wqe, wr->num_sge); qp 2155 drivers/infiniband/sw/rdmavt/qp.c qp->s_head = next; qp 2160 drivers/infiniband/sw/rdmavt/qp.c if (qp->allowed_ops == IB_OPCODE_UD) qp 2185 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); qp 2192 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irqsave(&qp->s_hlock, flags); qp 2198 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK))) { qp 2199 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->s_hlock, flags); qp 2208 drivers/infiniband/sw/rdmavt/qp.c call_send = qp->s_head == READ_ONCE(qp->s_last) && !wr->next; qp 2211 drivers/infiniband/sw/rdmavt/qp.c err = rvt_post_one_wr(qp, wr, &call_send); qp 2219 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->s_hlock, flags); qp 2226 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.do_send(qp); qp 2228 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.schedule_send_no_lock(qp); qp 2306 drivers/infiniband/sw/rdmavt/qp.c static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe) qp 2313 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 2316 drivers/infiniband/sw/rdmavt/qp.c pd = ibpd_to_rvtpd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd); qp 2317 drivers/infiniband/sw/rdmavt/qp.c ss = &qp->r_sge; qp 2318 drivers/infiniband/sw/rdmavt/qp.c ss->sg_list = qp->r_sg_list; qp 2319 drivers/infiniband/sw/rdmavt/qp.c qp->r_len = 0; qp 2329 drivers/infiniband/sw/rdmavt/qp.c qp->r_len += wqe->sg_list[i].length; qp 2333 drivers/infiniband/sw/rdmavt/qp.c ss->total_len = qp->r_len; qp 2347 drivers/infiniband/sw/rdmavt/qp.c wc.qp = &qp->ibqp; qp 2349 drivers/infiniband/sw/rdmavt/qp.c rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); qp 2407 drivers/infiniband/sw/rdmavt/qp.c int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only) qp 2421 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.srq) { qp 2422 drivers/infiniband/sw/rdmavt/qp.c srq = ibsrq_to_rvtsrq(qp->ibqp.srq); qp 2429 drivers/infiniband/sw/rdmavt/qp.c rq = &qp->r_rq; qp 2430 drivers/infiniband/sw/rdmavt/qp.c ip = qp->ip; qp 2434 drivers/infiniband/sw/rdmavt/qp.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { qp 2472 drivers/infiniband/sw/rdmavt/qp.c if (!wr_id_only && !init_sge(qp, wqe)) { qp 2476 drivers/infiniband/sw/rdmavt/qp.c qp->r_wr_id = wqe->wr_id; qp 2480 drivers/infiniband/sw/rdmavt/qp.c set_bit(RVT_R_WRID_VALID, &qp->r_aflags); qp 2493 drivers/infiniband/sw/rdmavt/qp.c ev.device = qp->ibqp.device; qp 2494 drivers/infiniband/sw/rdmavt/qp.c ev.element.srq = qp->ibqp.srq; qp 2512 drivers/infiniband/sw/rdmavt/qp.c void rvt_comm_est(struct rvt_qp *qp) qp 2514 drivers/infiniband/sw/rdmavt/qp.c qp->r_flags |= RVT_R_COMM_EST; qp 2515 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.event_handler) { qp 2518 drivers/infiniband/sw/rdmavt/qp.c ev.device = qp->ibqp.device; qp 2519 drivers/infiniband/sw/rdmavt/qp.c ev.element.qp = &qp->ibqp; qp 2521 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 2526 drivers/infiniband/sw/rdmavt/qp.c void rvt_rc_error(struct rvt_qp *qp, enum ib_wc_status err) qp 2531 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irqsave(&qp->s_lock, flags); qp 2532 drivers/infiniband/sw/rdmavt/qp.c lastwqe = rvt_error_qp(qp, err); qp 2533 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2538 drivers/infiniband/sw/rdmavt/qp.c ev.device = qp->ibqp.device; qp 2539 drivers/infiniband/sw/rdmavt/qp.c ev.element.qp = &qp->ibqp; qp 2541 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); qp 2569 drivers/infiniband/sw/rdmavt/qp.c void rvt_add_retry_timer_ext(struct rvt_qp *qp, u8 shift) qp 2571 drivers/infiniband/sw/rdmavt/qp.c struct ib_qp *ibqp = &qp->ibqp; qp 2574 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_lock); qp 2575 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags |= RVT_S_TIMER; qp 2577 drivers/infiniband/sw/rdmavt/qp.c qp->s_timer.expires = jiffies + rdi->busy_jiffies + qp 2578 drivers/infiniband/sw/rdmavt/qp.c (qp->timeout_jiffies << shift); qp 2579 drivers/infiniband/sw/rdmavt/qp.c add_timer(&qp->s_timer); qp 2589 drivers/infiniband/sw/rdmavt/qp.c void rvt_add_rnr_timer(struct rvt_qp *qp, u32 aeth) qp 2593 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_lock); qp 2594 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags |= RVT_S_WAIT_RNR; qp 2596 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_rnrnak_add(qp, to); qp 2597 drivers/infiniband/sw/rdmavt/qp.c hrtimer_start(&qp->s_rnr_timer, qp 2607 drivers/infiniband/sw/rdmavt/qp.c void rvt_stop_rc_timers(struct rvt_qp *qp) qp 2609 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_lock); qp 2611 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) { qp 2612 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR); qp 2613 drivers/infiniband/sw/rdmavt/qp.c del_timer(&qp->s_timer); qp 2614 drivers/infiniband/sw/rdmavt/qp.c hrtimer_try_to_cancel(&qp->s_rnr_timer); qp 2626 drivers/infiniband/sw/rdmavt/qp.c static void rvt_stop_rnr_timer(struct rvt_qp *qp) qp 2628 drivers/infiniband/sw/rdmavt/qp.c lockdep_assert_held(&qp->s_lock); qp 2630 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_flags & RVT_S_WAIT_RNR) { qp 2631 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags &= ~RVT_S_WAIT_RNR; qp 2632 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_rnrnak_stop(qp, 0); qp 2640 drivers/infiniband/sw/rdmavt/qp.c void rvt_del_timers_sync(struct rvt_qp *qp) qp 2642 drivers/infiniband/sw/rdmavt/qp.c del_timer_sync(&qp->s_timer); qp 2643 drivers/infiniband/sw/rdmavt/qp.c hrtimer_cancel(&qp->s_rnr_timer); qp 2652 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp = from_timer(qp, t, s_timer); qp 2653 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 2656 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irqsave(&qp->r_lock, flags); qp 2657 drivers/infiniband/sw/rdmavt/qp.c spin_lock(&qp->s_lock); qp 2658 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_flags & RVT_S_TIMER) { qp 2659 drivers/infiniband/sw/rdmavt/qp.c struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; qp 2661 drivers/infiniband/sw/rdmavt/qp.c qp->s_flags &= ~RVT_S_TIMER; qp 2663 drivers/infiniband/sw/rdmavt/qp.c del_timer(&qp->s_timer); qp 2664 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_rc_timeout(qp, qp->s_last_psn + 1); qp 2666 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.notify_restart_rc(qp, qp 2667 drivers/infiniband/sw/rdmavt/qp.c qp->s_last_psn + 1, qp 2669 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.schedule_send(qp); qp 2671 drivers/infiniband/sw/rdmavt/qp.c spin_unlock(&qp->s_lock); qp 2672 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 2680 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp = container_of(t, struct rvt_qp, s_rnr_timer); qp 2681 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 2684 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irqsave(&qp->s_lock, flags); qp 2685 drivers/infiniband/sw/rdmavt/qp.c rvt_stop_rnr_timer(qp); qp 2686 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_rnrnak_timeout(qp, 0); qp 2687 drivers/infiniband/sw/rdmavt/qp.c rdi->driver_f.schedule_send(qp); qp 2688 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->s_lock, flags); qp 2713 drivers/infiniband/sw/rdmavt/qp.c void (*cb)(struct rvt_qp *qp, u64 v)) qp 2748 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *pqp = iter->qp; qp 2749 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp; qp 2768 drivers/infiniband/sw/rdmavt/qp.c qp = rcu_dereference(pqp->next); qp 2776 drivers/infiniband/sw/rdmavt/qp.c qp = rcu_dereference(rvp->qp[n & 1]); qp 2778 drivers/infiniband/sw/rdmavt/qp.c qp = rcu_dereference( qp 2783 drivers/infiniband/sw/rdmavt/qp.c pqp = qp; qp 2784 drivers/infiniband/sw/rdmavt/qp.c if (qp) { qp 2785 drivers/infiniband/sw/rdmavt/qp.c iter->qp = qp; qp 2812 drivers/infiniband/sw/rdmavt/qp.c void (*cb)(struct rvt_qp *qp, u64 v)) qp 2826 drivers/infiniband/sw/rdmavt/qp.c rvt_get_qp(i.qp); qp 2828 drivers/infiniband/sw/rdmavt/qp.c i.cb(i.qp, i.v); qp 2830 drivers/infiniband/sw/rdmavt/qp.c rvt_put_qp(i.qp); qp 2840 drivers/infiniband/sw/rdmavt/qp.c void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 2846 drivers/infiniband/sw/rdmavt/qp.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND)) qp 2848 drivers/infiniband/sw/rdmavt/qp.c rdi = ib_to_rvt(qp->ibqp.device); qp 2850 drivers/infiniband/sw/rdmavt/qp.c old_last = qp->s_last; qp 2851 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_qp_send_completion(qp, wqe, old_last); qp 2852 drivers/infiniband/sw/rdmavt/qp.c last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode], qp 2854 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_acked == old_last) qp 2855 drivers/infiniband/sw/rdmavt/qp.c qp->s_acked = last; qp 2856 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_cur == old_last) qp 2857 drivers/infiniband/sw/rdmavt/qp.c qp->s_cur = last; qp 2858 drivers/infiniband/sw/rdmavt/qp.c if (qp->s_tail == old_last) qp 2859 drivers/infiniband/sw/rdmavt/qp.c qp->s_tail = last; qp 2860 drivers/infiniband/sw/rdmavt/qp.c if (qp->state == IB_QPS_SQD && last == qp->s_cur) qp 2861 drivers/infiniband/sw/rdmavt/qp.c qp->s_draining = 0; qp 2874 drivers/infiniband/sw/rdmavt/qp.c void rvt_copy_sge(struct rvt_qp *qp, struct rvt_sge_state *ss, qp 2882 drivers/infiniband/sw/rdmavt/qp.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 2968 drivers/infiniband/sw/rdmavt/qp.c struct rvt_qp *qp; qp 2989 drivers/infiniband/sw/rdmavt/qp.c qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), rvp, qp 3026 drivers/infiniband/sw/rdmavt/qp.c if (!qp) { qp 3030 drivers/infiniband/sw/rdmavt/qp.c spin_lock_irqsave(&qp->r_lock, flags); qp 3031 drivers/infiniband/sw/rdmavt/qp.c if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) || qp 3032 drivers/infiniband/sw/rdmavt/qp.c qp->ibqp.qp_type != sqp->ibqp.qp_type) { qp 3061 drivers/infiniband/sw/rdmavt/qp.c ret = rvt_get_rwqe(qp, false); qp 3066 drivers/infiniband/sw/rdmavt/qp.c if (wqe->length > qp->r_len) qp 3070 drivers/infiniband/sw/rdmavt/qp.c if (!rvt_invalidate_rkey(qp, qp 3087 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) qp 3091 drivers/infiniband/sw/rdmavt/qp.c ret = rvt_get_rwqe(qp, true); qp 3099 drivers/infiniband/sw/rdmavt/qp.c copy_last = rvt_is_user_qp(qp); qp 3100 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) qp 3105 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length, qp 3110 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.sg_list = NULL; qp 3111 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.num_sge = 1; qp 3112 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.total_len = wqe->length; qp 3116 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) qp 3118 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length, qp 3126 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.sge = wqe->sg_list[0]; qp 3127 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.sg_list = wqe->sg_list + 1; qp 3128 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.num_sge = wqe->wr.num_sge; qp 3129 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.total_len = wqe->length; qp 3134 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) qp 3136 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), qp 3142 drivers/infiniband/sw/rdmavt/qp.c maddr = (atomic64_t *)qp->r_sge.sge.vaddr; qp 3147 drivers/infiniband/sw/rdmavt/qp.c (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr, qp 3149 drivers/infiniband/sw/rdmavt/qp.c rvt_put_mr(qp->r_sge.sge.mr); qp 3150 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.num_sge = 0; qp 3163 drivers/infiniband/sw/rdmavt/qp.c rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, qp 3169 drivers/infiniband/sw/rdmavt/qp.c rvt_put_ss(&qp->r_sge); qp 3171 drivers/infiniband/sw/rdmavt/qp.c if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) qp 3178 drivers/infiniband/sw/rdmavt/qp.c wc.wr_id = qp->r_wr_id; qp 3181 drivers/infiniband/sw/rdmavt/qp.c wc.qp = &qp->ibqp; qp 3182 drivers/infiniband/sw/rdmavt/qp.c wc.src_qp = qp->remote_qpn; qp 3183 drivers/infiniband/sw/rdmavt/qp.c wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; qp 3184 drivers/infiniband/sw/rdmavt/qp.c wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); qp 3187 drivers/infiniband/sw/rdmavt/qp.c rvt_recv_cq(qp, &wc, wqe->wr.send_flags & IB_SEND_SOLICITED); qp 3190 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 3204 drivers/infiniband/sw/rdmavt/qp.c if (qp->ibqp.qp_type == IB_QPT_UC) qp 3217 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 3221 drivers/infiniband/sw/rdmavt/qp.c rvt_add_rnr_timer(sqp, qp->r_min_rnr_timer << qp 3243 drivers/infiniband/sw/rdmavt/qp.c rvt_rc_error(qp, wc.status); qp 3246 drivers/infiniband/sw/rdmavt/qp.c spin_unlock_irqrestore(&qp->r_lock, flags); qp 3259 drivers/infiniband/sw/rdmavt/qp.c ev.element.qp = &sqp->ibqp; qp 94 drivers/infiniband/sw/rdmavt/rc.c __be32 rvt_compute_aeth(struct rvt_qp *qp) qp 96 drivers/infiniband/sw/rdmavt/rc.c u32 aeth = qp->r_msn & IB_MSN_MASK; qp 98 drivers/infiniband/sw/rdmavt/rc.c if (qp->ibqp.srq) { qp 110 drivers/infiniband/sw/rdmavt/rc.c credits = READ_ONCE(qp->r_rq.kwq->count); qp 113 drivers/infiniband/sw/rdmavt/rc.c if (qp->ip) { qp 114 drivers/infiniband/sw/rdmavt/rc.c head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head); qp 115 drivers/infiniband/sw/rdmavt/rc.c tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail); qp 117 drivers/infiniband/sw/rdmavt/rc.c head = READ_ONCE(qp->r_rq.kwq->head); qp 118 drivers/infiniband/sw/rdmavt/rc.c tail = READ_ONCE(qp->r_rq.kwq->tail); qp 120 drivers/infiniband/sw/rdmavt/rc.c if (head >= qp->r_rq.size) qp 122 drivers/infiniband/sw/rdmavt/rc.c if (tail >= qp->r_rq.size) qp 132 drivers/infiniband/sw/rdmavt/rc.c credits += qp->r_rq.size; qp 165 drivers/infiniband/sw/rdmavt/rc.c void rvt_get_credit(struct rvt_qp *qp, u32 aeth) qp 167 drivers/infiniband/sw/rdmavt/rc.c struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 170 drivers/infiniband/sw/rdmavt/rc.c lockdep_assert_held(&qp->s_lock); qp 177 drivers/infiniband/sw/rdmavt/rc.c if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { qp 178 drivers/infiniband/sw/rdmavt/rc.c qp->s_flags |= RVT_S_UNLIMITED_CREDIT; qp 179 drivers/infiniband/sw/rdmavt/rc.c if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { qp 180 drivers/infiniband/sw/rdmavt/rc.c qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; qp 181 drivers/infiniband/sw/rdmavt/rc.c rdi->driver_f.schedule_send(qp); qp 184 drivers/infiniband/sw/rdmavt/rc.c } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { qp 187 drivers/infiniband/sw/rdmavt/rc.c if (rvt_cmp_msn(credit, qp->s_lsn) > 0) { qp 188 drivers/infiniband/sw/rdmavt/rc.c qp->s_lsn = credit; qp 189 drivers/infiniband/sw/rdmavt/rc.c if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { qp 190 drivers/infiniband/sw/rdmavt/rc.c qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; qp 191 drivers/infiniband/sw/rdmavt/rc.c rdi->driver_f.schedule_send(qp); qp 131 drivers/infiniband/sw/rdmavt/trace_cq.h __entry->qpn = wc->qp->qp_num; qp 60 drivers/infiniband/sw/rdmavt/trace_qp.h TP_PROTO(struct rvt_qp *qp, u32 bucket), qp 61 drivers/infiniband/sw/rdmavt/trace_qp.h TP_ARGS(qp, bucket), qp 63 drivers/infiniband/sw/rdmavt/trace_qp.h RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device)) qp 68 drivers/infiniband/sw/rdmavt/trace_qp.h RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device)) qp 69 drivers/infiniband/sw/rdmavt/trace_qp.h __entry->qpn = qp->ibqp.qp_num; qp 81 drivers/infiniband/sw/rdmavt/trace_qp.h TP_PROTO(struct rvt_qp *qp, u32 bucket), qp 82 drivers/infiniband/sw/rdmavt/trace_qp.h TP_ARGS(qp, bucket)); qp 85 drivers/infiniband/sw/rdmavt/trace_qp.h TP_PROTO(struct rvt_qp *qp, u32 bucket), qp 86 drivers/infiniband/sw/rdmavt/trace_qp.h TP_ARGS(qp, bucket)); qp 90 drivers/infiniband/sw/rdmavt/trace_qp.h TP_PROTO(struct rvt_qp *qp, u32 to), qp 91 drivers/infiniband/sw/rdmavt/trace_qp.h TP_ARGS(qp, to), qp 93 drivers/infiniband/sw/rdmavt/trace_qp.h RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device)) qp 100 drivers/infiniband/sw/rdmavt/trace_qp.h RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device)) qp 101 drivers/infiniband/sw/rdmavt/trace_qp.h __entry->qpn = qp->ibqp.qp_num; qp 102 drivers/infiniband/sw/rdmavt/trace_qp.h __entry->hrtimer = &qp->s_rnr_timer; qp 103 drivers/infiniband/sw/rdmavt/trace_qp.h __entry->s_flags = qp->s_flags; qp 118 drivers/infiniband/sw/rdmavt/trace_qp.h TP_PROTO(struct rvt_qp *qp, u32 to), qp 119 drivers/infiniband/sw/rdmavt/trace_qp.h TP_ARGS(qp, to)); qp 123 drivers/infiniband/sw/rdmavt/trace_qp.h TP_PROTO(struct rvt_qp *qp, u32 to), qp 124 drivers/infiniband/sw/rdmavt/trace_qp.h TP_ARGS(qp, to)); qp 128 drivers/infiniband/sw/rdmavt/trace_qp.h TP_PROTO(struct rvt_qp *qp, u32 to), qp 129 drivers/infiniband/sw/rdmavt/trace_qp.h TP_ARGS(qp, to)); qp 60 drivers/infiniband/sw/rdmavt/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 61 drivers/infiniband/sw/rdmavt/trace_rc.h TP_ARGS(qp, psn), qp 63 drivers/infiniband/sw/rdmavt/trace_rc.h RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device)) qp 74 drivers/infiniband/sw/rdmavt/trace_rc.h RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device)) qp 75 drivers/infiniband/sw/rdmavt/trace_rc.h __entry->qpn = qp->ibqp.qp_num; qp 76 drivers/infiniband/sw/rdmavt/trace_rc.h __entry->s_flags = qp->s_flags; qp 78 drivers/infiniband/sw/rdmavt/trace_rc.h __entry->s_psn = qp->s_psn; qp 79 drivers/infiniband/sw/rdmavt/trace_rc.h __entry->s_next_psn = qp->s_next_psn; qp 80 drivers/infiniband/sw/rdmavt/trace_rc.h __entry->s_sending_psn = qp->s_sending_psn; qp 81 drivers/infiniband/sw/rdmavt/trace_rc.h __entry->s_sending_hpsn = qp->s_sending_hpsn; qp 82 drivers/infiniband/sw/rdmavt/trace_rc.h __entry->r_psn = qp->r_psn; qp 99 drivers/infiniband/sw/rdmavt/trace_rc.h TP_PROTO(struct rvt_qp *qp, u32 psn), qp 100 drivers/infiniband/sw/rdmavt/trace_rc.h TP_ARGS(qp, psn) qp 91 drivers/infiniband/sw/rdmavt/trace_tx.h TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, int wr_num_sge), qp 92 drivers/infiniband/sw/rdmavt/trace_tx.h TP_ARGS(qp, wqe, wr_num_sge), qp 94 drivers/infiniband/sw/rdmavt/trace_tx.h RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device)) qp 114 drivers/infiniband/sw/rdmavt/trace_tx.h RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device)) qp 117 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->qpn = qp->ibqp.qp_num; qp 118 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->qpt = qp->ibqp.qp_type; qp 123 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->size = qp->s_size; qp 124 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->avail = qp->s_avail; qp 125 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->head = qp->s_head; qp 126 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->last = qp->s_last; qp 127 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->pid = qp->pid; qp 158 drivers/infiniband/sw/rdmavt/trace_tx.h TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, u32 idx), qp 159 drivers/infiniband/sw/rdmavt/trace_tx.h TP_ARGS(qp, wqe, idx), qp 161 drivers/infiniband/sw/rdmavt/trace_tx.h RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device)) qp 173 drivers/infiniband/sw/rdmavt/trace_tx.h RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device)) qp 176 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->qpn = qp->ibqp.qp_num; qp 177 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->qpt = qp->ibqp.qp_type; qp 104 drivers/infiniband/sw/rxe/rxe_av.c if (!pkt || !pkt->qp) qp 107 drivers/infiniband/sw/rxe/rxe_av.c if (qp_type(pkt->qp) == IB_QPT_RC || qp_type(pkt->qp) == IB_QPT_UC) qp 108 drivers/infiniband/sw/rxe/rxe_av.c return &pkt->qp->pri_av; qp 141 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_qp *qp = from_timer(qp, t, retrans_timer); qp 143 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->valid) { qp 144 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.timeout = 1; qp 145 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->comp.task, 1); qp 149 drivers/infiniband/sw/rxe/rxe_comp.c void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) qp 153 drivers/infiniband/sw/rxe/rxe_comp.c skb_queue_tail(&qp->resp_pkts, skb); qp 155 drivers/infiniband/sw/rxe/rxe_comp.c must_sched = skb_queue_len(&qp->resp_pkts) > 1; qp 159 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->comp.task, must_sched); qp 162 drivers/infiniband/sw/rxe/rxe_comp.c static inline enum comp_state get_wqe(struct rxe_qp *qp, qp 171 drivers/infiniband/sw/rxe/rxe_comp.c wqe = queue_head(qp->sq.queue); qp 190 drivers/infiniband/sw/rxe/rxe_comp.c static inline void reset_retry_counters(struct rxe_qp *qp) qp 192 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.retry_cnt = qp->attr.retry_cnt; qp 193 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.rnr_retry = qp->attr.rnr_retry; qp 194 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.started_retry = 0; qp 197 drivers/infiniband/sw/rxe/rxe_comp.c static inline enum comp_state check_psn(struct rxe_qp *qp, qp 212 drivers/infiniband/sw/rxe/rxe_comp.c reset_retry_counters(qp); qp 220 drivers/infiniband/sw/rxe/rxe_comp.c diff = psn_compare(pkt->psn, qp->comp.psn); qp 236 drivers/infiniband/sw/rxe/rxe_comp.c static inline enum comp_state check_ack(struct rxe_qp *qp, qp 242 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 245 drivers/infiniband/sw/rxe/rxe_comp.c switch (qp->comp.opcode) { qp 294 drivers/infiniband/sw/rxe/rxe_comp.c reset_retry_counters(qp); qp 306 drivers/infiniband/sw/rxe/rxe_comp.c reset_retry_counters(qp); qp 313 drivers/infiniband/sw/rxe/rxe_comp.c reset_retry_counters(qp); qp 326 drivers/infiniband/sw/rxe/rxe_comp.c if (psn_compare(pkt->psn, qp->comp.psn) > 0) { qp 329 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.psn = pkt->psn; qp 330 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->req.wait_psn) { qp 331 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.wait_psn = 0; qp 332 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->req.task, 0); qp 367 drivers/infiniband/sw/rxe/rxe_comp.c static inline enum comp_state do_read(struct rxe_qp *qp, qp 373 drivers/infiniband/sw/rxe/rxe_comp.c ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, qp 385 drivers/infiniband/sw/rxe/rxe_comp.c static inline enum comp_state do_atomic(struct rxe_qp *qp, qp 393 drivers/infiniband/sw/rxe/rxe_comp.c ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, qp 402 drivers/infiniband/sw/rxe/rxe_comp.c static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe, qp 407 drivers/infiniband/sw/rxe/rxe_comp.c if (!qp->is_user) { qp 417 drivers/infiniband/sw/rxe/rxe_comp.c wc->qp = &qp->ibqp; qp 428 drivers/infiniband/sw/rxe/rxe_comp.c uwc->qp_num = qp->ibqp.qp_num; qp 440 drivers/infiniband/sw/rxe/rxe_comp.c static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) qp 442 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 445 drivers/infiniband/sw/rxe/rxe_comp.c if ((qp->sq_sig_type == IB_SIGNAL_ALL_WR) || qp 448 drivers/infiniband/sw/rxe/rxe_comp.c make_send_cqe(qp, wqe, &cqe); qp 449 drivers/infiniband/sw/rxe/rxe_comp.c advance_consumer(qp->sq.queue); qp 450 drivers/infiniband/sw/rxe/rxe_comp.c rxe_cq_post(qp->scq, &cqe, 0); qp 452 drivers/infiniband/sw/rxe/rxe_comp.c advance_consumer(qp->sq.queue); qp 464 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->req.wait_fence) { qp 465 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.wait_fence = 0; qp 466 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->req.task, 0); qp 470 drivers/infiniband/sw/rxe/rxe_comp.c static inline enum comp_state complete_ack(struct rxe_qp *qp, qp 478 drivers/infiniband/sw/rxe/rxe_comp.c atomic_inc(&qp->req.rd_atomic); qp 479 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->req.need_rd_atomic) { qp 480 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.timeout_retry = 0; qp 481 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.need_rd_atomic = 0; qp 482 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->req.task, 0); qp 486 drivers/infiniband/sw/rxe/rxe_comp.c if (unlikely(qp->req.state == QP_STATE_DRAIN)) { qp 488 drivers/infiniband/sw/rxe/rxe_comp.c spin_lock_irqsave(&qp->state_lock, flags); qp 489 drivers/infiniband/sw/rxe/rxe_comp.c if ((qp->req.state == QP_STATE_DRAIN) && qp 490 drivers/infiniband/sw/rxe/rxe_comp.c (qp->comp.psn == qp->req.psn)) { qp 491 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.state = QP_STATE_DRAINED; qp 492 drivers/infiniband/sw/rxe/rxe_comp.c spin_unlock_irqrestore(&qp->state_lock, flags); qp 494 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->ibqp.event_handler) { qp 497 drivers/infiniband/sw/rxe/rxe_comp.c ev.device = qp->ibqp.device; qp 498 drivers/infiniband/sw/rxe/rxe_comp.c ev.element.qp = &qp->ibqp; qp 500 drivers/infiniband/sw/rxe/rxe_comp.c qp->ibqp.event_handler(&ev, qp 501 drivers/infiniband/sw/rxe/rxe_comp.c qp->ibqp.qp_context); qp 504 drivers/infiniband/sw/rxe/rxe_comp.c spin_unlock_irqrestore(&qp->state_lock, flags); qp 508 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); qp 510 drivers/infiniband/sw/rxe/rxe_comp.c if (psn_compare(pkt->psn, qp->comp.psn) >= 0) qp 516 drivers/infiniband/sw/rxe/rxe_comp.c static inline enum comp_state complete_wqe(struct rxe_qp *qp, qp 521 drivers/infiniband/sw/rxe/rxe_comp.c if (psn_compare(wqe->last_psn, qp->comp.psn) >= 0) { qp 522 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.psn = (wqe->last_psn + 1) & BTH_PSN_MASK; qp 523 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.opcode = -1; qp 526 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->req.wait_psn) { qp 527 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.wait_psn = 0; qp 528 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->req.task, 1); qp 532 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); qp 537 drivers/infiniband/sw/rxe/rxe_comp.c static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify) qp 542 drivers/infiniband/sw/rxe/rxe_comp.c while ((skb = skb_dequeue(&qp->resp_pkts))) { qp 543 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(qp); qp 547 drivers/infiniband/sw/rxe/rxe_comp.c while ((wqe = queue_head(qp->sq.queue))) { qp 550 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); qp 552 drivers/infiniband/sw/rxe/rxe_comp.c advance_consumer(qp->sq.queue); qp 559 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_qp *qp = (struct rxe_qp *)arg; qp 560 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 566 drivers/infiniband/sw/rxe/rxe_comp.c rxe_add_ref(qp); qp 568 drivers/infiniband/sw/rxe/rxe_comp.c if (!qp->valid || qp->req.state == QP_STATE_ERROR || qp 569 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.state == QP_STATE_RESET) { qp 570 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drain_resp_pkts(qp, qp->valid && qp 571 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.state == QP_STATE_ERROR); qp 575 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.timeout) { qp 576 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.timeout_retry = 1; qp 577 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.timeout = 0; qp 579 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.timeout_retry = 0; qp 582 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->req.need_retry) qp 588 drivers/infiniband/sw/rxe/rxe_comp.c pr_debug("qp#%d state = %s\n", qp_num(qp), qp 592 drivers/infiniband/sw/rxe/rxe_comp.c skb = skb_dequeue(&qp->resp_pkts); qp 595 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.timeout_retry = 0; qp 601 drivers/infiniband/sw/rxe/rxe_comp.c state = get_wqe(qp, pkt, &wqe); qp 605 drivers/infiniband/sw/rxe/rxe_comp.c state = check_psn(qp, pkt, wqe); qp 609 drivers/infiniband/sw/rxe/rxe_comp.c state = check_ack(qp, pkt, wqe); qp 613 drivers/infiniband/sw/rxe/rxe_comp.c state = do_read(qp, pkt, wqe); qp 617 drivers/infiniband/sw/rxe/rxe_comp.c state = do_atomic(qp, pkt, wqe); qp 629 drivers/infiniband/sw/rxe/rxe_comp.c state = complete_ack(qp, pkt, wqe); qp 633 drivers/infiniband/sw/rxe/rxe_comp.c state = complete_wqe(qp, pkt, wqe); qp 638 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.opcode = -1; qp 640 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.opcode = pkt->opcode; qp 642 drivers/infiniband/sw/rxe/rxe_comp.c if (psn_compare(pkt->psn, qp->comp.psn) >= 0) qp 643 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.psn = (pkt->psn + 1) & BTH_PSN_MASK; qp 645 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->req.wait_psn) { qp 646 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.wait_psn = 0; qp 647 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->req.task, 1); qp 655 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(pkt->qp); qp 662 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.timeout_retry && wqe) { qp 675 drivers/infiniband/sw/rxe/rxe_comp.c if ((qp_type(qp) == IB_QPT_RC) && qp 676 drivers/infiniband/sw/rxe/rxe_comp.c (qp->req.state == QP_STATE_READY) && qp 677 drivers/infiniband/sw/rxe/rxe_comp.c (psn_compare(qp->req.psn, qp->comp.psn) > 0) && qp 678 drivers/infiniband/sw/rxe/rxe_comp.c qp->qp_timeout_jiffies) qp 679 drivers/infiniband/sw/rxe/rxe_comp.c mod_timer(&qp->retrans_timer, qp 680 drivers/infiniband/sw/rxe/rxe_comp.c jiffies + qp->qp_timeout_jiffies); qp 700 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.started_retry && qp 701 drivers/infiniband/sw/rxe/rxe_comp.c !qp->comp.timeout_retry) { qp 703 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(pkt->qp); qp 711 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.retry_cnt > 0) { qp 712 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.retry_cnt != 7) qp 713 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.retry_cnt--; qp 719 drivers/infiniband/sw/rxe/rxe_comp.c if (psn_compare(qp->req.psn, qp 720 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.psn) > 0) { qp 726 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.need_retry = 1; qp 727 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.started_retry = 1; qp 728 drivers/infiniband/sw/rxe/rxe_comp.c rxe_run_task(&qp->req.task, 0); qp 732 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(pkt->qp); qp 747 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.rnr_retry > 0) { qp 748 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.rnr_retry != 7) qp 749 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.rnr_retry--; qp 751 drivers/infiniband/sw/rxe/rxe_comp.c qp->req.need_retry = 1; qp 753 drivers/infiniband/sw/rxe/rxe_comp.c qp_num(qp)); qp 754 drivers/infiniband/sw/rxe/rxe_comp.c mod_timer(&qp->rnr_nak_timer, qp 757 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(pkt->qp); qp 771 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); qp 772 drivers/infiniband/sw/rxe/rxe_comp.c rxe_qp_error(qp); qp 775 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(pkt->qp); qp 789 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(qp); qp 797 drivers/infiniband/sw/rxe/rxe_comp.c rxe_drop_ref(qp); qp 43 drivers/infiniband/sw/rxe/rxe_hdr.h struct rxe_qp *qp; /* qp that owns packet */ qp 73 drivers/infiniband/sw/rxe/rxe_loc.h int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, qp 76 drivers/infiniband/sw/rxe/rxe_loc.h int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, qp 79 drivers/infiniband/sw/rxe/rxe_loc.h void rxe_drop_all_mcast_groups(struct rxe_qp *qp); qp 157 drivers/infiniband/sw/rxe/rxe_loc.h int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd, qp 162 drivers/infiniband/sw/rxe/rxe_loc.h int rxe_qp_to_init(struct rxe_qp *qp, struct ib_qp_init_attr *init); qp 164 drivers/infiniband/sw/rxe/rxe_loc.h int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, qp 167 drivers/infiniband/sw/rxe/rxe_loc.h int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, qp 170 drivers/infiniband/sw/rxe/rxe_loc.h int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask); qp 172 drivers/infiniband/sw/rxe/rxe_loc.h void rxe_qp_error(struct rxe_qp *qp); qp 174 drivers/infiniband/sw/rxe/rxe_loc.h void rxe_qp_destroy(struct rxe_qp *qp); qp 178 drivers/infiniband/sw/rxe/rxe_loc.h static inline int qp_num(struct rxe_qp *qp) qp 180 drivers/infiniband/sw/rxe/rxe_loc.h return qp->ibqp.qp_num; qp 183 drivers/infiniband/sw/rxe/rxe_loc.h static inline enum ib_qp_type qp_type(struct rxe_qp *qp) qp 185 drivers/infiniband/sw/rxe/rxe_loc.h return qp->ibqp.qp_type; qp 188 drivers/infiniband/sw/rxe/rxe_loc.h static inline enum ib_qp_state qp_state(struct rxe_qp *qp) qp 190 drivers/infiniband/sw/rxe/rxe_loc.h return qp->attr.qp_state; qp 193 drivers/infiniband/sw/rxe/rxe_loc.h static inline int qp_mtu(struct rxe_qp *qp) qp 195 drivers/infiniband/sw/rxe/rxe_loc.h if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) qp 196 drivers/infiniband/sw/rxe/rxe_loc.h return qp->attr.path_mtu; qp 207 drivers/infiniband/sw/rxe/rxe_loc.h void free_rd_atomic_resource(struct rxe_qp *qp, struct resp_res *res); qp 209 drivers/infiniband/sw/rxe/rxe_loc.h static inline void rxe_advance_resp_resource(struct rxe_qp *qp) qp 211 drivers/infiniband/sw/rxe/rxe_loc.h qp->resp.res_head++; qp 212 drivers/infiniband/sw/rxe/rxe_loc.h if (unlikely(qp->resp.res_head == qp->attr.max_dest_rd_atomic)) qp 213 drivers/infiniband/sw/rxe/rxe_loc.h qp->resp.res_head = 0; qp 241 drivers/infiniband/sw/rxe/rxe_loc.h void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb); qp 243 drivers/infiniband/sw/rxe/rxe_loc.h void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb); qp 245 drivers/infiniband/sw/rxe/rxe_loc.h static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp) qp 247 drivers/infiniband/sw/rxe/rxe_loc.h return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type]; qp 250 drivers/infiniband/sw/rxe/rxe_loc.h static inline int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, qp 255 drivers/infiniband/sw/rxe/rxe_loc.h struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 257 drivers/infiniband/sw/rxe/rxe_loc.h if ((is_request && (qp->req.state != QP_STATE_READY)) || qp 258 drivers/infiniband/sw/rxe/rxe_loc.h (!is_request && (qp->resp.state != QP_STATE_READY))) { qp 277 drivers/infiniband/sw/rxe/rxe_loc.h if ((qp_type(qp) != IB_QPT_RC) && qp 280 drivers/infiniband/sw/rxe/rxe_loc.h rxe_run_task(&qp->comp.task, 1); qp 78 drivers/infiniband/sw/rxe/rxe_mcast.c int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, qp 85 drivers/infiniband/sw/rxe/rxe_mcast.c spin_lock_bh(&qp->grp_lock); qp 88 drivers/infiniband/sw/rxe/rxe_mcast.c if (elem->qp == qp) { qp 109 drivers/infiniband/sw/rxe/rxe_mcast.c elem->qp = qp; qp 113 drivers/infiniband/sw/rxe/rxe_mcast.c list_add(&elem->grp_list, &qp->grp_list); qp 118 drivers/infiniband/sw/rxe/rxe_mcast.c spin_unlock_bh(&qp->grp_lock); qp 122 drivers/infiniband/sw/rxe/rxe_mcast.c int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, qp 132 drivers/infiniband/sw/rxe/rxe_mcast.c spin_lock_bh(&qp->grp_lock); qp 136 drivers/infiniband/sw/rxe/rxe_mcast.c if (elem->qp == qp) { qp 142 drivers/infiniband/sw/rxe/rxe_mcast.c spin_unlock_bh(&qp->grp_lock); qp 151 drivers/infiniband/sw/rxe/rxe_mcast.c spin_unlock_bh(&qp->grp_lock); qp 157 drivers/infiniband/sw/rxe/rxe_mcast.c void rxe_drop_all_mcast_groups(struct rxe_qp *qp) qp 163 drivers/infiniband/sw/rxe/rxe_mcast.c spin_lock_bh(&qp->grp_lock); qp 164 drivers/infiniband/sw/rxe/rxe_mcast.c if (list_empty(&qp->grp_list)) { qp 165 drivers/infiniband/sw/rxe/rxe_mcast.c spin_unlock_bh(&qp->grp_lock); qp 168 drivers/infiniband/sw/rxe/rxe_mcast.c elem = list_first_entry(&qp->grp_list, struct rxe_mc_elem, qp 171 drivers/infiniband/sw/rxe/rxe_mcast.c spin_unlock_bh(&qp->grp_lock); qp 151 drivers/infiniband/sw/rxe/rxe_net.c struct rxe_qp *qp, qp 156 drivers/infiniband/sw/rxe/rxe_net.c if (qp_type(qp) == IB_QPT_RC) qp 157 drivers/infiniband/sw/rxe/rxe_net.c dst = sk_dst_get(qp->sk->sk); qp 159 drivers/infiniband/sw/rxe/rxe_net.c if (!dst || !dst_check(dst, qp->dst_cookie)) { qp 179 drivers/infiniband/sw/rxe/rxe_net.c qp->dst_cookie = qp 184 drivers/infiniband/sw/rxe/rxe_net.c if (dst && (qp_type(qp) == IB_QPT_RC)) { qp 186 drivers/infiniband/sw/rxe/rxe_net.c sk_dst_set(qp->sk->sk, dst); qp 345 drivers/infiniband/sw/rxe/rxe_net.c struct rxe_qp *qp = pkt->qp; qp 353 drivers/infiniband/sw/rxe/rxe_net.c dst = rxe_find_route(skb->dev, qp, av); qp 359 drivers/infiniband/sw/rxe/rxe_net.c prepare_udp_hdr(skb, cpu_to_be16(qp->src_port), qp 371 drivers/infiniband/sw/rxe/rxe_net.c struct rxe_qp *qp = pkt->qp; qp 377 drivers/infiniband/sw/rxe/rxe_net.c dst = rxe_find_route(skb->dev, qp, av); qp 383 drivers/infiniband/sw/rxe/rxe_net.c prepare_udp_hdr(skb, cpu_to_be16(qp->src_port), qp 414 drivers/infiniband/sw/rxe/rxe_net.c struct rxe_qp *qp = sk->sk_user_data; qp 415 drivers/infiniband/sw/rxe/rxe_net.c int skb_out = atomic_dec_return(&qp->skb_out); qp 417 drivers/infiniband/sw/rxe/rxe_net.c if (unlikely(qp->need_req_skb && qp 419 drivers/infiniband/sw/rxe/rxe_net.c rxe_run_task(&qp->req.task, 1); qp 421 drivers/infiniband/sw/rxe/rxe_net.c rxe_drop_ref(qp); qp 429 drivers/infiniband/sw/rxe/rxe_net.c skb->sk = pkt->qp->sk->sk; qp 431 drivers/infiniband/sw/rxe/rxe_net.c rxe_add_ref(pkt->qp); qp 432 drivers/infiniband/sw/rxe/rxe_net.c atomic_inc(&pkt->qp->skb_out); qp 440 drivers/infiniband/sw/rxe/rxe_net.c atomic_dec(&pkt->qp->skb_out); qp 441 drivers/infiniband/sw/rxe/rxe_net.c rxe_drop_ref(pkt->qp); qp 125 drivers/infiniband/sw/rxe/rxe_qp.c static int alloc_rd_atomic_resources(struct rxe_qp *qp, unsigned int n) qp 127 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.res_head = 0; qp 128 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.res_tail = 0; qp 129 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.resources = kcalloc(n, sizeof(struct resp_res), GFP_KERNEL); qp 131 drivers/infiniband/sw/rxe/rxe_qp.c if (!qp->resp.resources) qp 137 drivers/infiniband/sw/rxe/rxe_qp.c static void free_rd_atomic_resources(struct rxe_qp *qp) qp 139 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->resp.resources) { qp 142 drivers/infiniband/sw/rxe/rxe_qp.c for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) { qp 143 drivers/infiniband/sw/rxe/rxe_qp.c struct resp_res *res = &qp->resp.resources[i]; qp 145 drivers/infiniband/sw/rxe/rxe_qp.c free_rd_atomic_resource(qp, res); qp 147 drivers/infiniband/sw/rxe/rxe_qp.c kfree(qp->resp.resources); qp 148 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.resources = NULL; qp 152 drivers/infiniband/sw/rxe/rxe_qp.c void free_rd_atomic_resource(struct rxe_qp *qp, struct resp_res *res) qp 155 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_ref(qp); qp 164 drivers/infiniband/sw/rxe/rxe_qp.c static void cleanup_rd_atomic_resources(struct rxe_qp *qp) qp 169 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->resp.resources) { qp 170 drivers/infiniband/sw/rxe/rxe_qp.c for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) { qp 171 drivers/infiniband/sw/rxe/rxe_qp.c res = &qp->resp.resources[i]; qp 172 drivers/infiniband/sw/rxe/rxe_qp.c free_rd_atomic_resource(qp, res); qp 177 drivers/infiniband/sw/rxe/rxe_qp.c static void rxe_qp_init_misc(struct rxe_dev *rxe, struct rxe_qp *qp, qp 183 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq_sig_type = init->sq_sig_type; qp 184 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.path_mtu = 1; qp 185 drivers/infiniband/sw/rxe/rxe_qp.c qp->mtu = ib_mtu_enum_to_int(qp->attr.path_mtu); qp 187 drivers/infiniband/sw/rxe/rxe_qp.c qpn = qp->pelem.index; qp 192 drivers/infiniband/sw/rxe/rxe_qp.c qp->ibqp.qp_num = 0; qp 194 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.port_num = init->port_num; qp 198 drivers/infiniband/sw/rxe/rxe_qp.c qp->ibqp.qp_num = 1; qp 200 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.port_num = init->port_num; qp 204 drivers/infiniband/sw/rxe/rxe_qp.c qp->ibqp.qp_num = qpn; qp 208 drivers/infiniband/sw/rxe/rxe_qp.c INIT_LIST_HEAD(&qp->grp_list); qp 210 drivers/infiniband/sw/rxe/rxe_qp.c skb_queue_head_init(&qp->send_pkts); qp 212 drivers/infiniband/sw/rxe/rxe_qp.c spin_lock_init(&qp->grp_lock); qp 213 drivers/infiniband/sw/rxe/rxe_qp.c spin_lock_init(&qp->state_lock); qp 215 drivers/infiniband/sw/rxe/rxe_qp.c atomic_set(&qp->ssn, 0); qp 216 drivers/infiniband/sw/rxe/rxe_qp.c atomic_set(&qp->skb_out, 0); qp 219 drivers/infiniband/sw/rxe/rxe_qp.c static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, qp 226 drivers/infiniband/sw/rxe/rxe_qp.c err = sock_create_kern(&init_net, AF_INET, SOCK_DGRAM, 0, &qp->sk); qp 229 drivers/infiniband/sw/rxe/rxe_qp.c qp->sk->sk->sk_user_data = qp; qp 238 drivers/infiniband/sw/rxe/rxe_qp.c qp->src_port = RXE_ROCE_V2_SPORT + qp 239 drivers/infiniband/sw/rxe/rxe_qp.c (hash_32_generic(qp_num(qp), 14) & 0x3fff); qp 241 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq.max_wr = init->cap.max_send_wr; qp 242 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq.max_sge = init->cap.max_send_sge; qp 243 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq.max_inline = init->cap.max_inline_data; qp 246 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq.max_sge * sizeof(struct ib_sge), qp 248 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq.max_inline); qp 250 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq.queue = rxe_queue_init(rxe, qp 251 drivers/infiniband/sw/rxe/rxe_qp.c &qp->sq.max_wr, qp 253 drivers/infiniband/sw/rxe/rxe_qp.c if (!qp->sq.queue) qp 257 drivers/infiniband/sw/rxe/rxe_qp.c qp->sq.queue->buf, qp->sq.queue->buf_size, qp 258 drivers/infiniband/sw/rxe/rxe_qp.c &qp->sq.queue->ip); qp 261 drivers/infiniband/sw/rxe/rxe_qp.c vfree(qp->sq.queue->buf); qp 262 drivers/infiniband/sw/rxe/rxe_qp.c kfree(qp->sq.queue); qp 266 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.wqe_index = producer_index(qp->sq.queue); qp 267 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.state = QP_STATE_RESET; qp 268 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.opcode = -1; qp 269 drivers/infiniband/sw/rxe/rxe_qp.c qp->comp.opcode = -1; qp 271 drivers/infiniband/sw/rxe/rxe_qp.c spin_lock_init(&qp->sq.sq_lock); qp 272 drivers/infiniband/sw/rxe/rxe_qp.c skb_queue_head_init(&qp->req_pkts); qp 274 drivers/infiniband/sw/rxe/rxe_qp.c rxe_init_task(rxe, &qp->req.task, qp, qp 276 drivers/infiniband/sw/rxe/rxe_qp.c rxe_init_task(rxe, &qp->comp.task, qp, qp 279 drivers/infiniband/sw/rxe/rxe_qp.c qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */ qp 281 drivers/infiniband/sw/rxe/rxe_qp.c timer_setup(&qp->rnr_nak_timer, rnr_nak_timer, 0); qp 282 drivers/infiniband/sw/rxe/rxe_qp.c timer_setup(&qp->retrans_timer, retransmit_timer, 0); qp 287 drivers/infiniband/sw/rxe/rxe_qp.c static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, qp 295 drivers/infiniband/sw/rxe/rxe_qp.c if (!qp->srq) { qp 296 drivers/infiniband/sw/rxe/rxe_qp.c qp->rq.max_wr = init->cap.max_recv_wr; qp 297 drivers/infiniband/sw/rxe/rxe_qp.c qp->rq.max_sge = init->cap.max_recv_sge; qp 299 drivers/infiniband/sw/rxe/rxe_qp.c wqe_size = rcv_wqe_size(qp->rq.max_sge); qp 302 drivers/infiniband/sw/rxe/rxe_qp.c qp_num(qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size); qp 304 drivers/infiniband/sw/rxe/rxe_qp.c qp->rq.queue = rxe_queue_init(rxe, qp 305 drivers/infiniband/sw/rxe/rxe_qp.c &qp->rq.max_wr, qp 307 drivers/infiniband/sw/rxe/rxe_qp.c if (!qp->rq.queue) qp 311 drivers/infiniband/sw/rxe/rxe_qp.c qp->rq.queue->buf, qp->rq.queue->buf_size, qp 312 drivers/infiniband/sw/rxe/rxe_qp.c &qp->rq.queue->ip); qp 314 drivers/infiniband/sw/rxe/rxe_qp.c vfree(qp->rq.queue->buf); qp 315 drivers/infiniband/sw/rxe/rxe_qp.c kfree(qp->rq.queue); qp 320 drivers/infiniband/sw/rxe/rxe_qp.c spin_lock_init(&qp->rq.producer_lock); qp 321 drivers/infiniband/sw/rxe/rxe_qp.c spin_lock_init(&qp->rq.consumer_lock); qp 323 drivers/infiniband/sw/rxe/rxe_qp.c skb_queue_head_init(&qp->resp_pkts); qp 325 drivers/infiniband/sw/rxe/rxe_qp.c rxe_init_task(rxe, &qp->resp.task, qp, qp 328 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.opcode = OPCODE_NONE; qp 329 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.msn = 0; qp 330 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.state = QP_STATE_RESET; qp 336 drivers/infiniband/sw/rxe/rxe_qp.c int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd, qp 353 drivers/infiniband/sw/rxe/rxe_qp.c qp->pd = pd; qp 354 drivers/infiniband/sw/rxe/rxe_qp.c qp->rcq = rcq; qp 355 drivers/infiniband/sw/rxe/rxe_qp.c qp->scq = scq; qp 356 drivers/infiniband/sw/rxe/rxe_qp.c qp->srq = srq; qp 358 drivers/infiniband/sw/rxe/rxe_qp.c rxe_qp_init_misc(rxe, qp, init); qp 360 drivers/infiniband/sw/rxe/rxe_qp.c err = rxe_qp_init_req(rxe, qp, init, udata, uresp); qp 364 drivers/infiniband/sw/rxe/rxe_qp.c err = rxe_qp_init_resp(rxe, qp, init, udata, uresp); qp 368 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.qp_state = IB_QPS_RESET; qp 369 drivers/infiniband/sw/rxe/rxe_qp.c qp->valid = 1; qp 374 drivers/infiniband/sw/rxe/rxe_qp.c rxe_queue_cleanup(qp->sq.queue); qp 386 drivers/infiniband/sw/rxe/rxe_qp.c int rxe_qp_to_init(struct rxe_qp *qp, struct ib_qp_init_attr *init) qp 388 drivers/infiniband/sw/rxe/rxe_qp.c init->event_handler = qp->ibqp.event_handler; qp 389 drivers/infiniband/sw/rxe/rxe_qp.c init->qp_context = qp->ibqp.qp_context; qp 390 drivers/infiniband/sw/rxe/rxe_qp.c init->send_cq = qp->ibqp.send_cq; qp 391 drivers/infiniband/sw/rxe/rxe_qp.c init->recv_cq = qp->ibqp.recv_cq; qp 392 drivers/infiniband/sw/rxe/rxe_qp.c init->srq = qp->ibqp.srq; qp 394 drivers/infiniband/sw/rxe/rxe_qp.c init->cap.max_send_wr = qp->sq.max_wr; qp 395 drivers/infiniband/sw/rxe/rxe_qp.c init->cap.max_send_sge = qp->sq.max_sge; qp 396 drivers/infiniband/sw/rxe/rxe_qp.c init->cap.max_inline_data = qp->sq.max_inline; qp 398 drivers/infiniband/sw/rxe/rxe_qp.c if (!qp->srq) { qp 399 drivers/infiniband/sw/rxe/rxe_qp.c init->cap.max_recv_wr = qp->rq.max_wr; qp 400 drivers/infiniband/sw/rxe/rxe_qp.c init->cap.max_recv_sge = qp->rq.max_sge; qp 403 drivers/infiniband/sw/rxe/rxe_qp.c init->sq_sig_type = qp->sq_sig_type; qp 405 drivers/infiniband/sw/rxe/rxe_qp.c init->qp_type = qp->ibqp.qp_type; qp 414 drivers/infiniband/sw/rxe/rxe_qp.c int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, qp 418 drivers/infiniband/sw/rxe/rxe_qp.c attr->cur_qp_state : qp->attr.qp_state; qp 422 drivers/infiniband/sw/rxe/rxe_qp.c if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask)) { qp 429 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->req.state == QP_STATE_DRAIN && qp 442 drivers/infiniband/sw/rxe/rxe_qp.c if (mask & IB_QP_CAP && rxe_qp_chk_cap(rxe, &attr->cap, !!qp->srq)) qp 500 drivers/infiniband/sw/rxe/rxe_qp.c static void rxe_qp_reset(struct rxe_qp *qp) qp 503 drivers/infiniband/sw/rxe/rxe_qp.c rxe_disable_task(&qp->resp.task); qp 506 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->sq.queue) { qp 507 drivers/infiniband/sw/rxe/rxe_qp.c if (qp_type(qp) == IB_QPT_RC) qp 508 drivers/infiniband/sw/rxe/rxe_qp.c rxe_disable_task(&qp->comp.task); qp 509 drivers/infiniband/sw/rxe/rxe_qp.c rxe_disable_task(&qp->req.task); qp 513 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.state = QP_STATE_RESET; qp 514 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.state = QP_STATE_RESET; qp 519 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->resp.task); qp 521 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->sq.queue) { qp 522 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->comp.task); qp 523 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->req.task); qp 524 drivers/infiniband/sw/rxe/rxe_qp.c rxe_queue_reset(qp->sq.queue); qp 528 drivers/infiniband/sw/rxe/rxe_qp.c atomic_set(&qp->ssn, 0); qp 529 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.opcode = -1; qp 530 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.need_retry = 0; qp 531 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.noack_pkts = 0; qp 532 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.msn = 0; qp 533 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.opcode = -1; qp 534 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.drop_msg = 0; qp 535 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.goto_error = 0; qp 536 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.sent_psn_nak = 0; qp 538 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->resp.mr) { qp 539 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_ref(qp->resp.mr); qp 540 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.mr = NULL; qp 543 drivers/infiniband/sw/rxe/rxe_qp.c cleanup_rd_atomic_resources(qp); qp 546 drivers/infiniband/sw/rxe/rxe_qp.c rxe_enable_task(&qp->resp.task); qp 548 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->sq.queue) { qp 549 drivers/infiniband/sw/rxe/rxe_qp.c if (qp_type(qp) == IB_QPT_RC) qp 550 drivers/infiniband/sw/rxe/rxe_qp.c rxe_enable_task(&qp->comp.task); qp 552 drivers/infiniband/sw/rxe/rxe_qp.c rxe_enable_task(&qp->req.task); qp 557 drivers/infiniband/sw/rxe/rxe_qp.c static void rxe_qp_drain(struct rxe_qp *qp) qp 559 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->sq.queue) { qp 560 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->req.state != QP_STATE_DRAINED) { qp 561 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.state = QP_STATE_DRAIN; qp 562 drivers/infiniband/sw/rxe/rxe_qp.c if (qp_type(qp) == IB_QPT_RC) qp 563 drivers/infiniband/sw/rxe/rxe_qp.c rxe_run_task(&qp->comp.task, 1); qp 565 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->comp.task); qp 566 drivers/infiniband/sw/rxe/rxe_qp.c rxe_run_task(&qp->req.task, 1); qp 572 drivers/infiniband/sw/rxe/rxe_qp.c void rxe_qp_error(struct rxe_qp *qp) qp 574 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.state = QP_STATE_ERROR; qp 575 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.state = QP_STATE_ERROR; qp 576 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.qp_state = IB_QPS_ERR; qp 579 drivers/infiniband/sw/rxe/rxe_qp.c rxe_run_task(&qp->resp.task, 1); qp 581 drivers/infiniband/sw/rxe/rxe_qp.c if (qp_type(qp) == IB_QPT_RC) qp 582 drivers/infiniband/sw/rxe/rxe_qp.c rxe_run_task(&qp->comp.task, 1); qp 584 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->comp.task); qp 585 drivers/infiniband/sw/rxe/rxe_qp.c rxe_run_task(&qp->req.task, 1); qp 589 drivers/infiniband/sw/rxe/rxe_qp.c int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, qp 597 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.max_rd_atomic = max_rd_atomic; qp 598 drivers/infiniband/sw/rxe/rxe_qp.c atomic_set(&qp->req.rd_atomic, max_rd_atomic); qp 605 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.max_dest_rd_atomic = max_dest_rd_atomic; qp 607 drivers/infiniband/sw/rxe/rxe_qp.c free_rd_atomic_resources(qp); qp 609 drivers/infiniband/sw/rxe/rxe_qp.c err = alloc_rd_atomic_resources(qp, max_dest_rd_atomic); qp 615 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.cur_qp_state = attr->qp_state; qp 618 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.en_sqd_async_notify = attr->en_sqd_async_notify; qp 621 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.qp_access_flags = attr->qp_access_flags; qp 624 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.pkey_index = attr->pkey_index; qp 627 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.port_num = attr->port_num; qp 630 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.qkey = attr->qkey; qp 633 drivers/infiniband/sw/rxe/rxe_qp.c rxe_init_av(&attr->ah_attr, &qp->pri_av); qp 637 drivers/infiniband/sw/rxe/rxe_qp.c rxe_init_av(&attr->alt_ah_attr, &qp->alt_av); qp 638 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.alt_port_num = attr->alt_port_num; qp 639 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.alt_pkey_index = attr->alt_pkey_index; qp 640 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.alt_timeout = attr->alt_timeout; qp 644 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.path_mtu = attr->path_mtu; qp 645 drivers/infiniband/sw/rxe/rxe_qp.c qp->mtu = ib_mtu_enum_to_int(attr->path_mtu); qp 649 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.timeout = attr->timeout; qp 651 drivers/infiniband/sw/rxe/rxe_qp.c qp->qp_timeout_jiffies = 0; qp 656 drivers/infiniband/sw/rxe/rxe_qp.c qp->qp_timeout_jiffies = j ? j : 1; qp 661 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.retry_cnt = attr->retry_cnt; qp 662 drivers/infiniband/sw/rxe/rxe_qp.c qp->comp.retry_cnt = attr->retry_cnt; qp 663 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d set retry count = %d\n", qp_num(qp), qp 668 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.rnr_retry = attr->rnr_retry; qp 669 drivers/infiniband/sw/rxe/rxe_qp.c qp->comp.rnr_retry = attr->rnr_retry; qp 670 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d set rnr retry count = %d\n", qp_num(qp), qp 675 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.rq_psn = (attr->rq_psn & BTH_PSN_MASK); qp 676 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.psn = qp->attr.rq_psn; qp 677 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d set resp psn = 0x%x\n", qp_num(qp), qp 678 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.psn); qp 682 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.min_rnr_timer = attr->min_rnr_timer; qp 683 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d set min rnr timer = 0x%x\n", qp_num(qp), qp 688 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.sq_psn = (attr->sq_psn & BTH_PSN_MASK); qp 689 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.psn = qp->attr.sq_psn; qp 690 drivers/infiniband/sw/rxe/rxe_qp.c qp->comp.psn = qp->attr.sq_psn; qp 691 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d set req psn = 0x%x\n", qp_num(qp), qp->req.psn); qp 695 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.path_mig_state = attr->path_mig_state; qp 698 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.dest_qp_num = attr->dest_qp_num; qp 701 drivers/infiniband/sw/rxe/rxe_qp.c qp->attr.qp_state = attr->qp_state; qp 705 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d state -> RESET\n", qp_num(qp)); qp 706 drivers/infiniband/sw/rxe/rxe_qp.c rxe_qp_reset(qp); qp 710 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d state -> INIT\n", qp_num(qp)); qp 711 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.state = QP_STATE_INIT; qp 712 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.state = QP_STATE_INIT; qp 716 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d state -> RTR\n", qp_num(qp)); qp 717 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.state = QP_STATE_READY; qp 721 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d state -> RTS\n", qp_num(qp)); qp 722 drivers/infiniband/sw/rxe/rxe_qp.c qp->req.state = QP_STATE_READY; qp 726 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d state -> SQD\n", qp_num(qp)); qp 727 drivers/infiniband/sw/rxe/rxe_qp.c rxe_qp_drain(qp); qp 731 drivers/infiniband/sw/rxe/rxe_qp.c pr_warn("qp#%d state -> SQE !!?\n", qp_num(qp)); qp 736 drivers/infiniband/sw/rxe/rxe_qp.c pr_debug("qp#%d state -> ERR\n", qp_num(qp)); qp 737 drivers/infiniband/sw/rxe/rxe_qp.c rxe_qp_error(qp); qp 746 drivers/infiniband/sw/rxe/rxe_qp.c int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask) qp 748 drivers/infiniband/sw/rxe/rxe_qp.c *attr = qp->attr; qp 750 drivers/infiniband/sw/rxe/rxe_qp.c attr->rq_psn = qp->resp.psn; qp 751 drivers/infiniband/sw/rxe/rxe_qp.c attr->sq_psn = qp->req.psn; qp 753 drivers/infiniband/sw/rxe/rxe_qp.c attr->cap.max_send_wr = qp->sq.max_wr; qp 754 drivers/infiniband/sw/rxe/rxe_qp.c attr->cap.max_send_sge = qp->sq.max_sge; qp 755 drivers/infiniband/sw/rxe/rxe_qp.c attr->cap.max_inline_data = qp->sq.max_inline; qp 757 drivers/infiniband/sw/rxe/rxe_qp.c if (!qp->srq) { qp 758 drivers/infiniband/sw/rxe/rxe_qp.c attr->cap.max_recv_wr = qp->rq.max_wr; qp 759 drivers/infiniband/sw/rxe/rxe_qp.c attr->cap.max_recv_sge = qp->rq.max_sge; qp 762 drivers/infiniband/sw/rxe/rxe_qp.c rxe_av_to_attr(&qp->pri_av, &attr->ah_attr); qp 763 drivers/infiniband/sw/rxe/rxe_qp.c rxe_av_to_attr(&qp->alt_av, &attr->alt_ah_attr); qp 765 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->req.state == QP_STATE_DRAIN) { qp 782 drivers/infiniband/sw/rxe/rxe_qp.c void rxe_qp_destroy(struct rxe_qp *qp) qp 784 drivers/infiniband/sw/rxe/rxe_qp.c qp->valid = 0; qp 785 drivers/infiniband/sw/rxe/rxe_qp.c qp->qp_timeout_jiffies = 0; qp 786 drivers/infiniband/sw/rxe/rxe_qp.c rxe_cleanup_task(&qp->resp.task); qp 788 drivers/infiniband/sw/rxe/rxe_qp.c if (qp_type(qp) == IB_QPT_RC) { qp 789 drivers/infiniband/sw/rxe/rxe_qp.c del_timer_sync(&qp->retrans_timer); qp 790 drivers/infiniband/sw/rxe/rxe_qp.c del_timer_sync(&qp->rnr_nak_timer); qp 793 drivers/infiniband/sw/rxe/rxe_qp.c rxe_cleanup_task(&qp->req.task); qp 794 drivers/infiniband/sw/rxe/rxe_qp.c rxe_cleanup_task(&qp->comp.task); qp 797 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->req.task); qp 798 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->sq.queue) { qp 799 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->comp.task); qp 800 drivers/infiniband/sw/rxe/rxe_qp.c __rxe_do_task(&qp->req.task); qp 807 drivers/infiniband/sw/rxe/rxe_qp.c struct rxe_qp *qp = container_of(work, typeof(*qp), cleanup_work.work); qp 809 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_all_mcast_groups(qp); qp 811 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->sq.queue) qp 812 drivers/infiniband/sw/rxe/rxe_qp.c rxe_queue_cleanup(qp->sq.queue); qp 814 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->srq) qp 815 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_ref(qp->srq); qp 817 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->rq.queue) qp 818 drivers/infiniband/sw/rxe/rxe_qp.c rxe_queue_cleanup(qp->rq.queue); qp 820 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->scq) qp 821 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_ref(qp->scq); qp 822 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->rcq) qp 823 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_ref(qp->rcq); qp 824 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->pd) qp 825 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_ref(qp->pd); qp 827 drivers/infiniband/sw/rxe/rxe_qp.c if (qp->resp.mr) { qp 828 drivers/infiniband/sw/rxe/rxe_qp.c rxe_drop_ref(qp->resp.mr); qp 829 drivers/infiniband/sw/rxe/rxe_qp.c qp->resp.mr = NULL; qp 832 drivers/infiniband/sw/rxe/rxe_qp.c if (qp_type(qp) == IB_QPT_RC) qp 833 drivers/infiniband/sw/rxe/rxe_qp.c sk_dst_reset(qp->sk->sk); qp 835 drivers/infiniband/sw/rxe/rxe_qp.c free_rd_atomic_resources(qp); qp 837 drivers/infiniband/sw/rxe/rxe_qp.c kernel_sock_shutdown(qp->sk, SHUT_RDWR); qp 838 drivers/infiniband/sw/rxe/rxe_qp.c sock_release(qp->sk); qp 844 drivers/infiniband/sw/rxe/rxe_qp.c struct rxe_qp *qp = container_of(arg, typeof(*qp), pelem); qp 846 drivers/infiniband/sw/rxe/rxe_qp.c execute_in_process_context(rxe_qp_do_cleanup, &qp->cleanup_work); qp 40 drivers/infiniband/sw/rxe/rxe_recv.c struct rxe_qp *qp) qp 42 drivers/infiniband/sw/rxe/rxe_recv.c if (unlikely(!qp->valid)) qp 45 drivers/infiniband/sw/rxe/rxe_recv.c switch (qp_type(qp)) { qp 72 drivers/infiniband/sw/rxe/rxe_recv.c if (unlikely(qp->resp.state != QP_STATE_READY)) qp 74 drivers/infiniband/sw/rxe/rxe_recv.c } else if (unlikely(qp->req.state < QP_STATE_READY || qp 75 drivers/infiniband/sw/rxe/rxe_recv.c qp->req.state > QP_STATE_DRAINED)) { qp 102 drivers/infiniband/sw/rxe/rxe_recv.c u32 qpn, struct rxe_qp *qp) qp 127 drivers/infiniband/sw/rxe/rxe_recv.c port->pkey_tbl[qp->attr.pkey_index] qp 133 drivers/infiniband/sw/rxe/rxe_recv.c pkt->pkey_index = qp->attr.pkey_index; qp 136 drivers/infiniband/sw/rxe/rxe_recv.c if ((qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_GSI) && qp 138 drivers/infiniband/sw/rxe/rxe_recv.c u32 qkey = (qpn == 1) ? GSI_QKEY : qp->attr.qkey; qp 155 drivers/infiniband/sw/rxe/rxe_recv.c struct rxe_qp *qp) qp 159 drivers/infiniband/sw/rxe/rxe_recv.c if (qp_type(qp) != IB_QPT_RC && qp_type(qp) != IB_QPT_UC) qp 162 drivers/infiniband/sw/rxe/rxe_recv.c if (unlikely(pkt->port_num != qp->attr.port_num)) { qp 164 drivers/infiniband/sw/rxe/rxe_recv.c pkt->port_num, qp->attr.port_num); qp 170 drivers/infiniband/sw/rxe/rxe_recv.c &qp->pri_av.sgid_addr._sockaddr_in.sin_addr; qp 172 drivers/infiniband/sw/rxe/rxe_recv.c &qp->pri_av.dgid_addr._sockaddr_in.sin_addr; qp 190 drivers/infiniband/sw/rxe/rxe_recv.c &qp->pri_av.sgid_addr._sockaddr_in6.sin6_addr; qp 192 drivers/infiniband/sw/rxe/rxe_recv.c &qp->pri_av.dgid_addr._sockaddr_in6.sin6_addr; qp 218 drivers/infiniband/sw/rxe/rxe_recv.c struct rxe_qp *qp = NULL; qp 236 drivers/infiniband/sw/rxe/rxe_recv.c qp = rxe_pool_get_index(&rxe->qp_pool, index); qp 237 drivers/infiniband/sw/rxe/rxe_recv.c if (unlikely(!qp)) { qp 242 drivers/infiniband/sw/rxe/rxe_recv.c err = check_type_state(rxe, pkt, qp); qp 246 drivers/infiniband/sw/rxe/rxe_recv.c err = check_addr(rxe, pkt, qp); qp 250 drivers/infiniband/sw/rxe/rxe_recv.c err = check_keys(rxe, pkt, qpn, qp); qp 260 drivers/infiniband/sw/rxe/rxe_recv.c pkt->qp = qp; qp 264 drivers/infiniband/sw/rxe/rxe_recv.c rxe_drop_ref(qp); qp 272 drivers/infiniband/sw/rxe/rxe_recv.c rxe_resp_queue_pkt(pkt->qp, skb); qp 274 drivers/infiniband/sw/rxe/rxe_recv.c rxe_comp_queue_pkt(pkt->qp, skb); qp 282 drivers/infiniband/sw/rxe/rxe_recv.c struct rxe_qp *qp; qp 300 drivers/infiniband/sw/rxe/rxe_recv.c qp = mce->qp; qp 304 drivers/infiniband/sw/rxe/rxe_recv.c err = check_type_state(rxe, pkt, qp); qp 308 drivers/infiniband/sw/rxe/rxe_recv.c err = check_keys(rxe, pkt, bth_qpn(pkt), qp); qp 318 drivers/infiniband/sw/rxe/rxe_recv.c pkt->qp = qp; qp 319 drivers/infiniband/sw/rxe/rxe_recv.c rxe_add_ref(qp); qp 376 drivers/infiniband/sw/rxe/rxe_recv.c pkt->qp = NULL; qp 417 drivers/infiniband/sw/rxe/rxe_recv.c if (pkt->qp) qp 418 drivers/infiniband/sw/rxe/rxe_recv.c rxe_drop_ref(pkt->qp); qp 41 drivers/infiniband/sw/rxe/rxe_req.c static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe, qp 44 drivers/infiniband/sw/rxe/rxe_req.c static inline void retry_first_write_send(struct rxe_qp *qp, qp 51 drivers/infiniband/sw/rxe/rxe_req.c int to_send = (wqe->dma.resid > qp->mtu) ? qp 52 drivers/infiniband/sw/rxe/rxe_req.c qp->mtu : wqe->dma.resid; qp 54 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode = next_opcode(qp, wqe, qp 64 drivers/infiniband/sw/rxe/rxe_req.c wqe->iova += qp->mtu; qp 68 drivers/infiniband/sw/rxe/rxe_req.c static void req_retry(struct rxe_qp *qp) qp 76 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wqe_index = consumer_index(qp->sq.queue); qp 77 drivers/infiniband/sw/rxe/rxe_req.c qp->req.psn = qp->comp.psn; qp 78 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode = -1; qp 80 drivers/infiniband/sw/rxe/rxe_req.c for (wqe_index = consumer_index(qp->sq.queue); qp 81 drivers/infiniband/sw/rxe/rxe_req.c wqe_index != producer_index(qp->sq.queue); qp 82 drivers/infiniband/sw/rxe/rxe_req.c wqe_index = next_index(qp->sq.queue, wqe_index)) { qp 83 drivers/infiniband/sw/rxe/rxe_req.c wqe = addr_from_index(qp->sq.queue, wqe_index); qp 84 drivers/infiniband/sw/rxe/rxe_req.c mask = wr_opcode_mask(wqe->wr.opcode, qp); qp 108 drivers/infiniband/sw/rxe/rxe_req.c npsn = (qp->comp.psn - wqe->first_psn) & qp 110 drivers/infiniband/sw/rxe/rxe_req.c retry_first_write_send(qp, wqe, mask, npsn); qp 115 drivers/infiniband/sw/rxe/rxe_req.c qp->mtu; qp 116 drivers/infiniband/sw/rxe/rxe_req.c wqe->iova += npsn * qp->mtu; qp 126 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer); qp 128 drivers/infiniband/sw/rxe/rxe_req.c pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp)); qp 129 drivers/infiniband/sw/rxe/rxe_req.c rxe_run_task(&qp->req.task, 1); qp 132 drivers/infiniband/sw/rxe/rxe_req.c static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) qp 134 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe = queue_head(qp->sq.queue); qp 137 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely(qp->req.state == QP_STATE_DRAIN)) { qp 141 drivers/infiniband/sw/rxe/rxe_req.c spin_lock_irqsave(&qp->state_lock, flags); qp 143 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.state != QP_STATE_DRAIN) { qp 145 drivers/infiniband/sw/rxe/rxe_req.c spin_unlock_irqrestore(&qp->state_lock, qp 150 drivers/infiniband/sw/rxe/rxe_req.c if (wqe && ((qp->req.wqe_index != qp 151 drivers/infiniband/sw/rxe/rxe_req.c consumer_index(qp->sq.queue)) || qp 154 drivers/infiniband/sw/rxe/rxe_req.c spin_unlock_irqrestore(&qp->state_lock, qp 159 drivers/infiniband/sw/rxe/rxe_req.c qp->req.state = QP_STATE_DRAINED; qp 160 drivers/infiniband/sw/rxe/rxe_req.c spin_unlock_irqrestore(&qp->state_lock, flags); qp 162 drivers/infiniband/sw/rxe/rxe_req.c if (qp->ibqp.event_handler) { qp 165 drivers/infiniband/sw/rxe/rxe_req.c ev.device = qp->ibqp.device; qp 166 drivers/infiniband/sw/rxe/rxe_req.c ev.element.qp = &qp->ibqp; qp 168 drivers/infiniband/sw/rxe/rxe_req.c qp->ibqp.event_handler(&ev, qp 169 drivers/infiniband/sw/rxe/rxe_req.c qp->ibqp.qp_context); qp 174 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.wqe_index == producer_index(qp->sq.queue)) qp 177 drivers/infiniband/sw/rxe/rxe_req.c wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index); qp 179 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely((qp->req.state == QP_STATE_DRAIN || qp 180 drivers/infiniband/sw/rxe/rxe_req.c qp->req.state == QP_STATE_DRAINED) && qp 185 drivers/infiniband/sw/rxe/rxe_req.c (qp->req.wqe_index != consumer_index(qp->sq.queue)))) { qp 186 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wait_fence = 1; qp 190 drivers/infiniband/sw/rxe/rxe_req.c wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp); qp 194 drivers/infiniband/sw/rxe/rxe_req.c static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits) qp 198 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || qp 199 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) qp 209 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || qp 210 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) qp 220 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || qp 221 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) qp 231 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || qp 232 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) qp 251 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || qp 252 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) qp 266 drivers/infiniband/sw/rxe/rxe_req.c static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits) qp 270 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || qp 271 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) qp 281 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || qp 282 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) qp 292 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || qp 293 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) qp 303 drivers/infiniband/sw/rxe/rxe_req.c if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || qp 304 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) qp 317 drivers/infiniband/sw/rxe/rxe_req.c static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe, qp 320 drivers/infiniband/sw/rxe/rxe_req.c int fits = (wqe->dma.resid <= qp->mtu); qp 322 drivers/infiniband/sw/rxe/rxe_req.c switch (qp_type(qp)) { qp 324 drivers/infiniband/sw/rxe/rxe_req.c return next_opcode_rc(qp, opcode, fits); qp 327 drivers/infiniband/sw/rxe/rxe_req.c return next_opcode_uc(qp, opcode, fits); qp 348 drivers/infiniband/sw/rxe/rxe_req.c static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe) qp 355 drivers/infiniband/sw/rxe/rxe_req.c qp->req.need_rd_atomic = 1; qp 356 drivers/infiniband/sw/rxe/rxe_req.c depth = atomic_dec_return(&qp->req.rd_atomic); qp 359 drivers/infiniband/sw/rxe/rxe_req.c qp->req.need_rd_atomic = 0; qp 364 drivers/infiniband/sw/rxe/rxe_req.c atomic_inc(&qp->req.rd_atomic); qp 368 drivers/infiniband/sw/rxe/rxe_req.c static inline int get_mtu(struct rxe_qp *qp) qp 370 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 372 drivers/infiniband/sw/rxe/rxe_req.c if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC)) qp 373 drivers/infiniband/sw/rxe/rxe_req.c return qp->mtu; qp 378 drivers/infiniband/sw/rxe/rxe_req.c static struct sk_buff *init_req_packet(struct rxe_qp *qp, qp 383 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 402 drivers/infiniband/sw/rxe/rxe_req.c pkt->qp = qp; qp 403 drivers/infiniband/sw/rxe/rxe_req.c pkt->psn = qp->req.psn; qp 422 drivers/infiniband/sw/rxe/rxe_req.c pkey = (qp_type(qp) == IB_QPT_GSI) ? qp 424 drivers/infiniband/sw/rxe/rxe_req.c port->pkey_tbl[qp->attr.pkey_index]; qp 427 drivers/infiniband/sw/rxe/rxe_req.c qp->attr.dest_qp_num; qp 430 drivers/infiniband/sw/rxe/rxe_req.c (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK)); qp 432 drivers/infiniband/sw/rxe/rxe_req.c qp->req.noack_pkts = 0; qp 463 drivers/infiniband/sw/rxe/rxe_req.c if (qp->ibqp.qp_num == 1) qp 467 drivers/infiniband/sw/rxe/rxe_req.c deth_set_sqp(pkt, qp->ibqp.qp_num); qp 473 drivers/infiniband/sw/rxe/rxe_req.c static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe, qp 477 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 496 drivers/infiniband/sw/rxe/rxe_req.c err = copy_data(qp->pd, 0, &wqe->dma, qp 517 drivers/infiniband/sw/rxe/rxe_req.c static void update_wqe_state(struct rxe_qp *qp, qp 522 drivers/infiniband/sw/rxe/rxe_req.c if (qp_type(qp) == IB_QPT_RC) qp 529 drivers/infiniband/sw/rxe/rxe_req.c static void update_wqe_psn(struct rxe_qp *qp, qp 535 drivers/infiniband/sw/rxe/rxe_req.c int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu; qp 542 drivers/infiniband/sw/rxe/rxe_req.c wqe->first_psn = qp->req.psn; qp 543 drivers/infiniband/sw/rxe/rxe_req.c wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK; qp 547 drivers/infiniband/sw/rxe/rxe_req.c qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK; qp 549 drivers/infiniband/sw/rxe/rxe_req.c qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; qp 553 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_qp *qp, qp 560 drivers/infiniband/sw/rxe/rxe_req.c *rollback_psn = qp->req.psn; qp 564 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_qp *qp, qp 571 drivers/infiniband/sw/rxe/rxe_req.c qp->req.psn = rollback_psn; qp 574 drivers/infiniband/sw/rxe/rxe_req.c static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe, qp 577 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode = pkt->opcode; qp 580 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wqe_index = next_index(qp->sq.queue, qp->req.wqe_index); qp 582 drivers/infiniband/sw/rxe/rxe_req.c qp->need_req_skb = 0; qp 584 drivers/infiniband/sw/rxe/rxe_req.c if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer)) qp 585 drivers/infiniband/sw/rxe/rxe_req.c mod_timer(&qp->retrans_timer, qp 586 drivers/infiniband/sw/rxe/rxe_req.c jiffies + qp->qp_timeout_jiffies); qp 591 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_qp *qp = (struct rxe_qp *)arg; qp 603 drivers/infiniband/sw/rxe/rxe_req.c rxe_add_ref(qp); qp 606 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR)) qp 609 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely(qp->req.state == QP_STATE_RESET)) { qp 610 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wqe_index = consumer_index(qp->sq.queue); qp 611 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode = -1; qp 612 drivers/infiniband/sw/rxe/rxe_req.c qp->req.need_rd_atomic = 0; qp 613 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wait_psn = 0; qp 614 drivers/infiniband/sw/rxe/rxe_req.c qp->req.need_retry = 0; qp 618 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely(qp->req.need_retry)) { qp 619 drivers/infiniband/sw/rxe/rxe_req.c req_retry(qp); qp 620 drivers/infiniband/sw/rxe/rxe_req.c qp->req.need_retry = 0; qp 623 drivers/infiniband/sw/rxe/rxe_req.c wqe = req_next_wqe(qp); qp 629 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 659 drivers/infiniband/sw/rxe/rxe_req.c qp->sq_sig_type == IB_SIGNAL_ALL_WR) qp 660 drivers/infiniband/sw/rxe/rxe_req.c rxe_run_task(&qp->comp.task, 1); qp 661 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wqe_index = next_index(qp->sq.queue, qp 662 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wqe_index); qp 666 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely(qp_type(qp) == IB_QPT_RC && qp 667 drivers/infiniband/sw/rxe/rxe_req.c qp->req.psn > (qp->comp.psn + RXE_MAX_UNACKED_PSNS))) { qp 668 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wait_psn = 1; qp 673 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely(atomic_read(&qp->skb_out) > qp 675 drivers/infiniband/sw/rxe/rxe_req.c qp->need_req_skb = 1; qp 679 drivers/infiniband/sw/rxe/rxe_req.c opcode = next_opcode(qp, wqe, wqe->wr.opcode); qp 687 drivers/infiniband/sw/rxe/rxe_req.c if (check_init_depth(qp, wqe)) qp 691 drivers/infiniband/sw/rxe/rxe_req.c mtu = get_mtu(qp); qp 694 drivers/infiniband/sw/rxe/rxe_req.c if (qp_type(qp) == IB_QPT_UD) { qp 702 drivers/infiniband/sw/rxe/rxe_req.c wqe->first_psn = qp->req.psn; qp 703 drivers/infiniband/sw/rxe/rxe_req.c wqe->last_psn = qp->req.psn; qp 704 drivers/infiniband/sw/rxe/rxe_req.c qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; qp 705 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode = IB_OPCODE_UD_SEND_ONLY; qp 706 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wqe_index = next_index(qp->sq.queue, qp 707 drivers/infiniband/sw/rxe/rxe_req.c qp->req.wqe_index); qp 710 drivers/infiniband/sw/rxe/rxe_req.c __rxe_do_task(&qp->comp.task); qp 711 drivers/infiniband/sw/rxe/rxe_req.c rxe_drop_ref(qp); qp 717 drivers/infiniband/sw/rxe/rxe_req.c skb = init_req_packet(qp, wqe, opcode, payload, &pkt); qp 719 drivers/infiniband/sw/rxe/rxe_req.c pr_err("qp#%d Failed allocating skb\n", qp_num(qp)); qp 723 drivers/infiniband/sw/rxe/rxe_req.c if (fill_packet(qp, wqe, &pkt, skb, payload)) { qp 724 drivers/infiniband/sw/rxe/rxe_req.c pr_debug("qp#%d Error during fill packet\n", qp_num(qp)); qp 735 drivers/infiniband/sw/rxe/rxe_req.c save_state(wqe, qp, &rollback_wqe, &rollback_psn); qp 736 drivers/infiniband/sw/rxe/rxe_req.c update_wqe_state(qp, wqe, &pkt); qp 737 drivers/infiniband/sw/rxe/rxe_req.c update_wqe_psn(qp, wqe, &pkt, payload); qp 738 drivers/infiniband/sw/rxe/rxe_req.c ret = rxe_xmit_packet(qp, &pkt, skb); qp 740 drivers/infiniband/sw/rxe/rxe_req.c qp->need_req_skb = 1; qp 742 drivers/infiniband/sw/rxe/rxe_req.c rollback_state(wqe, qp, &rollback_wqe, rollback_psn); qp 745 drivers/infiniband/sw/rxe/rxe_req.c rxe_run_task(&qp->req.task, 1); qp 752 drivers/infiniband/sw/rxe/rxe_req.c update_state(qp, wqe, &pkt, payload); qp 759 drivers/infiniband/sw/rxe/rxe_req.c __rxe_do_task(&qp->comp.task); qp 762 drivers/infiniband/sw/rxe/rxe_req.c rxe_drop_ref(qp); qp 107 drivers/infiniband/sw/rxe/rxe_resp.c void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) qp 112 drivers/infiniband/sw/rxe/rxe_resp.c skb_queue_tail(&qp->req_pkts, skb); qp 115 drivers/infiniband/sw/rxe/rxe_resp.c (skb_queue_len(&qp->req_pkts) > 1); qp 117 drivers/infiniband/sw/rxe/rxe_resp.c rxe_run_task(&qp->resp.task, must_sched); qp 120 drivers/infiniband/sw/rxe/rxe_resp.c static inline enum resp_states get_req(struct rxe_qp *qp, qp 125 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.state == QP_STATE_ERROR) { qp 126 drivers/infiniband/sw/rxe/rxe_resp.c while ((skb = skb_dequeue(&qp->req_pkts))) { qp 127 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drop_ref(qp); qp 135 drivers/infiniband/sw/rxe/rxe_resp.c skb = skb_peek(&qp->req_pkts); qp 141 drivers/infiniband/sw/rxe/rxe_resp.c return (qp->resp.res) ? RESPST_READ_REPLY : RESPST_CHK_PSN; qp 144 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states check_psn(struct rxe_qp *qp, qp 147 drivers/infiniband/sw/rxe/rxe_resp.c int diff = psn_compare(pkt->psn, qp->resp.psn); qp 148 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 150 drivers/infiniband/sw/rxe/rxe_resp.c switch (qp_type(qp)) { qp 153 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.sent_psn_nak) qp 156 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.sent_psn_nak = 1; qp 165 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.sent_psn_nak) qp 166 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.sent_psn_nak = 0; qp 171 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.drop_msg || diff != 0) { qp 173 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 0; qp 177 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 1; qp 188 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states check_op_seq(struct rxe_qp *qp, qp 191 drivers/infiniband/sw/rxe/rxe_resp.c switch (qp_type(qp)) { qp 193 drivers/infiniband/sw/rxe/rxe_resp.c switch (qp->resp.opcode) { qp 234 drivers/infiniband/sw/rxe/rxe_resp.c switch (qp->resp.opcode) { qp 265 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 1; qp 278 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states check_op_valid(struct rxe_qp *qp, qp 281 drivers/infiniband/sw/rxe/rxe_resp.c switch (qp_type(qp)) { qp 284 drivers/infiniband/sw/rxe/rxe_resp.c !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) || qp 286 drivers/infiniband/sw/rxe/rxe_resp.c !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) || qp 288 drivers/infiniband/sw/rxe/rxe_resp.c !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) { qp 296 drivers/infiniband/sw/rxe/rxe_resp.c !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) { qp 297 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 1; qp 316 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states get_srq_wqe(struct rxe_qp *qp) qp 318 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_srq *srq = qp->srq; qp 335 drivers/infiniband/sw/rxe/rxe_resp.c memcpy(&qp->resp.srq_wqe, wqe, sizeof(qp->resp.srq_wqe)); qp 337 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = &qp->resp.srq_wqe.wqe; qp 351 drivers/infiniband/sw/rxe/rxe_resp.c ev.device = qp->ibqp.device; qp 352 drivers/infiniband/sw/rxe/rxe_resp.c ev.element.srq = qp->ibqp.srq; qp 358 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states check_resource(struct rxe_qp *qp, qp 361 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_srq *srq = qp->srq; qp 363 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.state == QP_STATE_ERROR) { qp 364 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { qp 365 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.status = IB_WC_WR_FLUSH_ERR; qp 368 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = queue_head(qp->rq.queue); qp 369 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { qp 370 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.status = IB_WC_WR_FLUSH_ERR; qp 385 drivers/infiniband/sw/rxe/rxe_resp.c if (likely(qp->attr.max_dest_rd_atomic > 0)) qp 393 drivers/infiniband/sw/rxe/rxe_resp.c return get_srq_wqe(qp); qp 395 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = queue_head(qp->rq.queue); qp 396 drivers/infiniband/sw/rxe/rxe_resp.c return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR; qp 402 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states check_length(struct rxe_qp *qp, qp 405 drivers/infiniband/sw/rxe/rxe_resp.c switch (qp_type(qp)) { qp 417 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states check_rkey(struct rxe_qp *qp, qp 425 drivers/infiniband/sw/rxe/rxe_resp.c int mtu = qp->mtu; qp 431 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.va = reth_va(pkt); qp 432 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.rkey = reth_rkey(pkt); qp 433 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.resid = reth_len(pkt); qp 434 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.length = reth_len(pkt); qp 439 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.va = atmeth_va(pkt); qp 440 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.rkey = atmeth_rkey(pkt); qp 441 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.resid = sizeof(u64); qp 454 drivers/infiniband/sw/rxe/rxe_resp.c va = qp->resp.va; qp 455 drivers/infiniband/sw/rxe/rxe_resp.c rkey = qp->resp.rkey; qp 456 drivers/infiniband/sw/rxe/rxe_resp.c resid = qp->resp.resid; qp 459 drivers/infiniband/sw/rxe/rxe_resp.c mem = lookup_mem(qp->pd, access, rkey, lookup_remote); qp 496 drivers/infiniband/sw/rxe/rxe_resp.c WARN_ON_ONCE(qp->resp.mr); qp 498 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.mr = mem; qp 507 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr, qp 512 drivers/infiniband/sw/rxe/rxe_resp.c err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma, qp 521 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states write_data_in(struct rxe_qp *qp, qp 528 drivers/infiniband/sw/rxe/rxe_resp.c err = rxe_mem_copy(qp->resp.mr, qp->resp.va, payload_addr(pkt), qp 535 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.va += data_len; qp 536 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.resid -= data_len; qp 545 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states process_atomic(struct rxe_qp *qp, qp 551 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_mem *mr = qp->resp.mr; qp 568 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.atomic_orig = *vaddr; qp 585 drivers/infiniband/sw/rxe/rxe_resp.c static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, qp 594 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 608 drivers/infiniband/sw/rxe/rxe_resp.c skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack); qp 612 drivers/infiniband/sw/rxe/rxe_resp.c ack->qp = qp; qp 622 drivers/infiniband/sw/rxe/rxe_resp.c bth_set_qpn(ack, qp->attr.dest_qp_num); qp 631 drivers/infiniband/sw/rxe/rxe_resp.c aeth_set_msn(ack, qp->resp.msn); qp 635 drivers/infiniband/sw/rxe/rxe_resp.c atmack_set_orig(ack, qp->resp.atomic_orig); qp 657 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states read_reply(struct rxe_qp *qp, qp 662 drivers/infiniband/sw/rxe/rxe_resp.c int mtu = qp->mtu; qp 667 drivers/infiniband/sw/rxe/rxe_resp.c struct resp_res *res = qp->resp.res; qp 675 drivers/infiniband/sw/rxe/rxe_resp.c res = &qp->resp.resources[qp->resp.res_head]; qp 677 drivers/infiniband/sw/rxe/rxe_resp.c free_rd_atomic_resource(qp, res); qp 678 drivers/infiniband/sw/rxe/rxe_resp.c rxe_advance_resp_resource(qp); qp 683 drivers/infiniband/sw/rxe/rxe_resp.c res->read.va = qp->resp.va; qp 684 drivers/infiniband/sw/rxe/rxe_resp.c res->read.va_org = qp->resp.va; qp 697 drivers/infiniband/sw/rxe/rxe_resp.c res->read.resid = qp->resp.resid; qp 698 drivers/infiniband/sw/rxe/rxe_resp.c res->read.length = qp->resp.resid; qp 699 drivers/infiniband/sw/rxe/rxe_resp.c res->read.rkey = qp->resp.rkey; qp 702 drivers/infiniband/sw/rxe/rxe_resp.c res->read.mr = qp->resp.mr; qp 703 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.mr = NULL; qp 705 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.res = res; qp 725 drivers/infiniband/sw/rxe/rxe_resp.c skb = prepare_ack_packet(qp, req_pkt, &ack_pkt, opcode, payload, qp 736 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 745 drivers/infiniband/sw/rxe/rxe_resp.c err = rxe_xmit_packet(qp, &ack_pkt, skb); qp 758 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.res = NULL; qp 760 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.opcode = -1; qp 761 drivers/infiniband/sw/rxe/rxe_resp.c if (psn_compare(res->cur_psn, qp->resp.psn) >= 0) qp 762 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.psn = res->cur_psn; qp 784 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt) qp 789 drivers/infiniband/sw/rxe/rxe_resp.c if (qp_type(qp) == IB_QPT_UD || qp 790 drivers/infiniband/sw/rxe/rxe_resp.c qp_type(qp) == IB_QPT_SMI || qp 791 drivers/infiniband/sw/rxe/rxe_resp.c qp_type(qp) == IB_QPT_GSI) { qp 796 drivers/infiniband/sw/rxe/rxe_resp.c err = send_data_in(qp, &hdr, sizeof(hdr)); qp 800 drivers/infiniband/sw/rxe/rxe_resp.c err = send_data_in(qp, payload_addr(pkt), payload_size(pkt)); qp 804 drivers/infiniband/sw/rxe/rxe_resp.c err = write_data_in(qp, pkt); qp 809 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.msn++; qp 812 drivers/infiniband/sw/rxe/rxe_resp.c err = process_atomic(qp, pkt); qp 821 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK; qp 822 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.ack_psn = qp->resp.psn; qp 824 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.opcode = pkt->opcode; qp 825 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.status = IB_WC_SUCCESS; qp 829 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.msn++; qp 831 drivers/infiniband/sw/rxe/rxe_resp.c } else if (qp_type(qp) == IB_QPT_RC) qp 837 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states do_complete(struct rxe_qp *qp, qp 843 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_recv_wqe *wqe = qp->resp.wqe; qp 844 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 851 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->rcq->is_user) { qp 852 drivers/infiniband/sw/rxe/rxe_resp.c uwc->status = qp->resp.status; qp 853 drivers/infiniband/sw/rxe/rxe_resp.c uwc->qp_num = qp->ibqp.qp_num; qp 856 drivers/infiniband/sw/rxe/rxe_resp.c wc->status = qp->resp.status; qp 857 drivers/infiniband/sw/rxe/rxe_resp.c wc->qp = &qp->ibqp; qp 869 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.length : wqe->dma.length - wqe->dma.resid; qp 874 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->rcq->is_user) { qp 887 drivers/infiniband/sw/rxe/rxe_resp.c uwc->qp_num = qp->ibqp.qp_num; qp 892 drivers/infiniband/sw/rxe/rxe_resp.c uwc->port_num = qp->attr.port_num; qp 929 drivers/infiniband/sw/rxe/rxe_resp.c wc->qp = &qp->ibqp; qp 934 drivers/infiniband/sw/rxe/rxe_resp.c wc->port_num = qp->attr.port_num; qp 939 drivers/infiniband/sw/rxe/rxe_resp.c if (!qp->srq) qp 940 drivers/infiniband/sw/rxe/rxe_resp.c advance_consumer(qp->rq.queue); qp 942 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = NULL; qp 944 drivers/infiniband/sw/rxe/rxe_resp.c if (rxe_cq_post(qp->rcq, &cqe, pkt ? bth_se(pkt) : 1)) qp 947 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.state == QP_STATE_ERROR) qp 952 drivers/infiniband/sw/rxe/rxe_resp.c else if (qp_type(qp) == IB_QPT_RC) qp 958 drivers/infiniband/sw/rxe/rxe_resp.c static int send_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt, qp 965 drivers/infiniband/sw/rxe/rxe_resp.c skb = prepare_ack_packet(qp, pkt, &ack_pkt, IB_OPCODE_RC_ACKNOWLEDGE, qp 972 drivers/infiniband/sw/rxe/rxe_resp.c err = rxe_xmit_packet(qp, &ack_pkt, skb); qp 980 drivers/infiniband/sw/rxe/rxe_resp.c static int send_atomic_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt, qp 988 drivers/infiniband/sw/rxe/rxe_resp.c skb = prepare_ack_packet(qp, pkt, &ack_pkt, qp 996 drivers/infiniband/sw/rxe/rxe_resp.c rxe_add_ref(qp); qp 998 drivers/infiniband/sw/rxe/rxe_resp.c res = &qp->resp.resources[qp->resp.res_head]; qp 999 drivers/infiniband/sw/rxe/rxe_resp.c free_rd_atomic_resource(qp, res); qp 1000 drivers/infiniband/sw/rxe/rxe_resp.c rxe_advance_resp_resource(qp); qp 1013 drivers/infiniband/sw/rxe/rxe_resp.c rc = rxe_xmit_packet(qp, &ack_pkt, skb); qp 1016 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drop_ref(qp); qp 1022 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states acknowledge(struct rxe_qp *qp, qp 1025 drivers/infiniband/sw/rxe/rxe_resp.c if (qp_type(qp) != IB_QPT_RC) qp 1028 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.aeth_syndrome != AETH_ACK_UNLIMITED) qp 1029 drivers/infiniband/sw/rxe/rxe_resp.c send_ack(qp, pkt, qp->resp.aeth_syndrome, pkt->psn); qp 1031 drivers/infiniband/sw/rxe/rxe_resp.c send_atomic_ack(qp, pkt, AETH_ACK_UNLIMITED); qp 1033 drivers/infiniband/sw/rxe/rxe_resp.c send_ack(qp, pkt, AETH_ACK_UNLIMITED, pkt->psn); qp 1038 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states cleanup(struct rxe_qp *qp, qp 1044 drivers/infiniband/sw/rxe/rxe_resp.c skb = skb_dequeue(&qp->req_pkts); qp 1045 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drop_ref(qp); qp 1049 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.mr) { qp 1050 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drop_ref(qp->resp.mr); qp 1051 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.mr = NULL; qp 1057 drivers/infiniband/sw/rxe/rxe_resp.c static struct resp_res *find_resource(struct rxe_qp *qp, u32 psn) qp 1061 drivers/infiniband/sw/rxe/rxe_resp.c for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) { qp 1062 drivers/infiniband/sw/rxe/rxe_resp.c struct resp_res *res = &qp->resp.resources[i]; qp 1076 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states duplicate_request(struct rxe_qp *qp, qp 1080 drivers/infiniband/sw/rxe/rxe_resp.c u32 prev_psn = (qp->resp.ack_psn - 1) & BTH_PSN_MASK; qp 1086 drivers/infiniband/sw/rxe/rxe_resp.c send_ack(qp, pkt, AETH_ACK_UNLIMITED, prev_psn); qp 1092 drivers/infiniband/sw/rxe/rxe_resp.c res = find_resource(qp, pkt->psn); qp 1131 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.res = res; qp 1139 drivers/infiniband/sw/rxe/rxe_resp.c res = find_resource(qp, pkt->psn); qp 1143 drivers/infiniband/sw/rxe/rxe_resp.c rc = rxe_xmit_packet(qp, pkt, res->atomic.skb); qp 1160 drivers/infiniband/sw/rxe/rxe_resp.c static void do_class_ac_error(struct rxe_qp *qp, u8 syndrome, qp 1163 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.aeth_syndrome = syndrome; qp 1164 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.status = status; qp 1167 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.goto_error = 1; qp 1170 drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states do_class_d1e_error(struct rxe_qp *qp) qp 1173 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->srq) { qp 1175 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 1; qp 1176 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { qp 1177 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.status = IB_WC_REM_INV_REQ_ERR; qp 1188 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { qp 1189 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe->dma.resid = qp->resp.wqe->dma.length; qp 1190 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe->dma.cur_sge = 0; qp 1191 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe->dma.sge_offset = 0; qp 1192 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.opcode = -1; qp 1195 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.mr) { qp 1196 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drop_ref(qp->resp.mr); qp 1197 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.mr = NULL; qp 1204 drivers/infiniband/sw/rxe/rxe_resp.c static void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify) qp 1208 drivers/infiniband/sw/rxe/rxe_resp.c while ((skb = skb_dequeue(&qp->req_pkts))) { qp 1209 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drop_ref(qp); qp 1216 drivers/infiniband/sw/rxe/rxe_resp.c while (!qp->srq && qp->rq.queue && queue_head(qp->rq.queue)) qp 1217 drivers/infiniband/sw/rxe/rxe_resp.c advance_consumer(qp->rq.queue); qp 1222 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_qp *qp = (struct rxe_qp *)arg; qp 1223 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_dev *rxe = to_rdev(qp->ibqp.device); qp 1228 drivers/infiniband/sw/rxe/rxe_resp.c rxe_add_ref(qp); qp 1230 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED; qp 1232 drivers/infiniband/sw/rxe/rxe_resp.c if (!qp->valid) { qp 1237 drivers/infiniband/sw/rxe/rxe_resp.c switch (qp->resp.state) { qp 1248 drivers/infiniband/sw/rxe/rxe_resp.c pr_debug("qp#%d state = %s\n", qp_num(qp), qp 1252 drivers/infiniband/sw/rxe/rxe_resp.c state = get_req(qp, &pkt); qp 1255 drivers/infiniband/sw/rxe/rxe_resp.c state = check_psn(qp, pkt); qp 1258 drivers/infiniband/sw/rxe/rxe_resp.c state = check_op_seq(qp, pkt); qp 1261 drivers/infiniband/sw/rxe/rxe_resp.c state = check_op_valid(qp, pkt); qp 1264 drivers/infiniband/sw/rxe/rxe_resp.c state = check_resource(qp, pkt); qp 1267 drivers/infiniband/sw/rxe/rxe_resp.c state = check_length(qp, pkt); qp 1270 drivers/infiniband/sw/rxe/rxe_resp.c state = check_rkey(qp, pkt); qp 1273 drivers/infiniband/sw/rxe/rxe_resp.c state = execute(qp, pkt); qp 1276 drivers/infiniband/sw/rxe/rxe_resp.c state = do_complete(qp, pkt); qp 1279 drivers/infiniband/sw/rxe/rxe_resp.c state = read_reply(qp, pkt); qp 1282 drivers/infiniband/sw/rxe/rxe_resp.c state = acknowledge(qp, pkt); qp 1285 drivers/infiniband/sw/rxe/rxe_resp.c state = cleanup(qp, pkt); qp 1288 drivers/infiniband/sw/rxe/rxe_resp.c state = duplicate_request(qp, pkt); qp 1292 drivers/infiniband/sw/rxe/rxe_resp.c send_ack(qp, pkt, AETH_NAK_PSN_SEQ_ERROR, qp->resp.psn); qp 1302 drivers/infiniband/sw/rxe/rxe_resp.c do_class_ac_error(qp, AETH_NAK_INVALID_REQ, qp 1308 drivers/infiniband/sw/rxe/rxe_resp.c state = do_class_d1e_error(qp); qp 1311 drivers/infiniband/sw/rxe/rxe_resp.c if (qp_type(qp) == IB_QPT_RC) { qp 1314 drivers/infiniband/sw/rxe/rxe_resp.c send_ack(qp, pkt, AETH_RNR_NAK | qp 1316 drivers/infiniband/sw/rxe/rxe_resp.c qp->attr.min_rnr_timer), qp 1320 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 1; qp 1326 drivers/infiniband/sw/rxe/rxe_resp.c if (qp_type(qp) == IB_QPT_RC) { qp 1328 drivers/infiniband/sw/rxe/rxe_resp.c do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR, qp 1332 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 1; qp 1333 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->srq) { qp 1335 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.status = IB_WC_REM_ACCESS_ERR; qp 1345 drivers/infiniband/sw/rxe/rxe_resp.c if (qp_type(qp) == IB_QPT_RC) { qp 1347 drivers/infiniband/sw/rxe/rxe_resp.c do_class_ac_error(qp, AETH_NAK_INVALID_REQ, qp 1350 drivers/infiniband/sw/rxe/rxe_resp.c } else if (qp->srq) { qp 1352 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.status = IB_WC_REM_INV_REQ_ERR; qp 1356 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.drop_msg = 1; qp 1363 drivers/infiniband/sw/rxe/rxe_resp.c do_class_ac_error(qp, AETH_NAK_REM_OP_ERR, qp 1374 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.goto_error) { qp 1382 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.goto_error) { qp 1390 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drain_req_pkts(qp, false); qp 1391 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = NULL; qp 1395 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.goto_error = 0; qp 1396 drivers/infiniband/sw/rxe/rxe_resp.c pr_warn("qp#%d moved to error state\n", qp_num(qp)); qp 1397 drivers/infiniband/sw/rxe/rxe_resp.c rxe_qp_error(qp); qp 1408 drivers/infiniband/sw/rxe/rxe_resp.c rxe_drop_ref(qp); qp 418 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp; qp 431 drivers/infiniband/sw/rxe/rxe_verbs.c qp = rxe_alloc(&rxe->qp_pool); qp 432 drivers/infiniband/sw/rxe/rxe_verbs.c if (!qp) { qp 442 drivers/infiniband/sw/rxe/rxe_verbs.c qp->is_user = 1; qp 445 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_add_index(qp); qp 447 drivers/infiniband/sw/rxe/rxe_verbs.c err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibpd, udata); qp 451 drivers/infiniband/sw/rxe/rxe_verbs.c return &qp->ibqp; qp 454 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_drop_index(qp); qp 456 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_drop_ref(qp); qp 466 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp = to_rqp(ibqp); qp 468 drivers/infiniband/sw/rxe/rxe_verbs.c err = rxe_qp_chk_attr(rxe, qp, attr, mask); qp 472 drivers/infiniband/sw/rxe/rxe_verbs.c err = rxe_qp_from_attr(qp, attr, mask, udata); qp 485 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp = to_rqp(ibqp); qp 487 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_qp_to_init(qp, init); qp 488 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_qp_to_attr(qp, attr, mask); qp 495 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp = to_rqp(ibqp); qp 497 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_qp_destroy(qp); qp 498 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_drop_index(qp); qp 499 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_drop_ref(qp); qp 503 drivers/infiniband/sw/rxe/rxe_verbs.c static int validate_send_wr(struct rxe_qp *qp, const struct ib_send_wr *ibwr, qp 507 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_sq *sq = &qp->sq; qp 530 drivers/infiniband/sw/rxe/rxe_verbs.c static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr, qp 538 drivers/infiniband/sw/rxe/rxe_verbs.c if (qp_type(qp) == IB_QPT_UD || qp 539 drivers/infiniband/sw/rxe/rxe_verbs.c qp_type(qp) == IB_QPT_SMI || qp 540 drivers/infiniband/sw/rxe/rxe_verbs.c qp_type(qp) == IB_QPT_GSI) { qp 543 drivers/infiniband/sw/rxe/rxe_verbs.c if (qp_type(qp) == IB_QPT_GSI) qp 586 drivers/infiniband/sw/rxe/rxe_verbs.c static int init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr, qp 595 drivers/infiniband/sw/rxe/rxe_verbs.c init_send_wr(qp, &wqe->wr, ibwr); qp 597 drivers/infiniband/sw/rxe/rxe_verbs.c if (qp_type(qp) == IB_QPT_UD || qp 598 drivers/infiniband/sw/rxe/rxe_verbs.c qp_type(qp) == IB_QPT_SMI || qp 599 drivers/infiniband/sw/rxe/rxe_verbs.c qp_type(qp) == IB_QPT_GSI) qp 629 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->ssn = atomic_add_return(1, &qp->ssn); qp 634 drivers/infiniband/sw/rxe/rxe_verbs.c static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr, qp 638 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_sq *sq = &qp->sq; qp 642 drivers/infiniband/sw/rxe/rxe_verbs.c err = validate_send_wr(qp, ibwr, mask, length); qp 646 drivers/infiniband/sw/rxe/rxe_verbs.c spin_lock_irqsave(&qp->sq.sq_lock, flags); qp 655 drivers/infiniband/sw/rxe/rxe_verbs.c err = init_send_wqe(qp, ibwr, mask, length, send_wqe); qp 666 drivers/infiniband/sw/rxe/rxe_verbs.c spin_unlock_irqrestore(&qp->sq.sq_lock, flags); qp 671 drivers/infiniband/sw/rxe/rxe_verbs.c spin_unlock_irqrestore(&qp->sq.sq_lock, flags); qp 675 drivers/infiniband/sw/rxe/rxe_verbs.c static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr, qp 684 drivers/infiniband/sw/rxe/rxe_verbs.c mask = wr_opcode_mask(wr->opcode, qp); qp 702 drivers/infiniband/sw/rxe/rxe_verbs.c err = post_one_send(qp, wr, mask, length); qp 711 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_run_task(&qp->req.task, 1); qp 712 drivers/infiniband/sw/rxe/rxe_verbs.c if (unlikely(qp->req.state == QP_STATE_ERROR)) qp 713 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_run_task(&qp->comp.task, 1); qp 721 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp = to_rqp(ibqp); qp 723 drivers/infiniband/sw/rxe/rxe_verbs.c if (unlikely(!qp->valid)) { qp 728 drivers/infiniband/sw/rxe/rxe_verbs.c if (unlikely(qp->req.state < QP_STATE_READY)) { qp 733 drivers/infiniband/sw/rxe/rxe_verbs.c if (qp->is_user) { qp 735 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_run_task(&qp->req.task, 0); qp 738 drivers/infiniband/sw/rxe/rxe_verbs.c return rxe_post_send_kernel(qp, wr, bad_wr); qp 745 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp = to_rqp(ibqp); qp 746 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_rq *rq = &qp->rq; qp 749 drivers/infiniband/sw/rxe/rxe_verbs.c if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) { qp 755 drivers/infiniband/sw/rxe/rxe_verbs.c if (unlikely(qp->srq)) { qp 774 drivers/infiniband/sw/rxe/rxe_verbs.c if (qp->resp.state == QP_STATE_ERROR) qp 775 drivers/infiniband/sw/rxe/rxe_verbs.c rxe_run_task(&qp->resp.task, 1); qp 1050 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp = to_rqp(ibqp); qp 1058 drivers/infiniband/sw/rxe/rxe_verbs.c err = rxe_mcast_add_grp_elem(rxe, qp, grp); qp 1067 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_qp *qp = to_rqp(ibqp); qp 1069 drivers/infiniband/sw/rxe/rxe_verbs.c return rxe_mcast_drop_grp_elem(rxe, qp, mgid); qp 368 drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_qp *qp; qp 448 drivers/infiniband/sw/rxe/rxe_verbs.h static inline struct rxe_qp *to_rqp(struct ib_qp *qp) qp 450 drivers/infiniband/sw/rxe/rxe_verbs.h return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL; qp 487 drivers/infiniband/sw/siw/siw.h struct siw_qp *qp; qp 493 drivers/infiniband/sw/siw/siw.h #define tx_wqe(qp) (&(qp)->tx_ctx.wqe_active) qp 503 drivers/infiniband/sw/siw/siw.h int (*rx_data)(struct siw_qp *qp); qp 521 drivers/infiniband/sw/siw/siw.h int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attr, qp 523 drivers/infiniband/sw/siw/siw.h int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl); qp 524 drivers/infiniband/sw/siw/siw.h void siw_qp_llp_close(struct siw_qp *qp); qp 525 drivers/infiniband/sw/siw/siw.h void siw_qp_cm_drop(struct siw_qp *qp, int schedule); qp 526 drivers/infiniband/sw/siw/siw.h void siw_send_terminate(struct siw_qp *qp); qp 528 drivers/infiniband/sw/siw/siw.h void siw_qp_get_ref(struct ib_qp *qp); qp 529 drivers/infiniband/sw/siw/siw.h void siw_qp_put_ref(struct ib_qp *qp); qp 530 drivers/infiniband/sw/siw/siw.h int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp); qp 533 drivers/infiniband/sw/siw/siw.h void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, qp 539 drivers/infiniband/sw/siw/siw.h int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, qp 541 drivers/infiniband/sw/siw/siw.h int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, qp 548 drivers/infiniband/sw/siw/siw.h int siw_qp_sq_process(struct siw_qp *qp); qp 549 drivers/infiniband/sw/siw/siw.h int siw_sq_start(struct siw_qp *qp); qp 550 drivers/infiniband/sw/siw/siw.h int siw_activate_tx(struct siw_qp *qp); qp 556 drivers/infiniband/sw/siw/siw.h int siw_proc_send(struct siw_qp *qp); qp 557 drivers/infiniband/sw/siw/siw.h int siw_proc_rreq(struct siw_qp *qp); qp 558 drivers/infiniband/sw/siw/siw.h int siw_proc_rresp(struct siw_qp *qp); qp 559 drivers/infiniband/sw/siw/siw.h int siw_proc_write(struct siw_qp *qp); qp 560 drivers/infiniband/sw/siw/siw.h int siw_proc_terminate(struct siw_qp *qp); qp 565 drivers/infiniband/sw/siw/siw.h static inline void set_rx_fpdu_context(struct siw_qp *qp, u8 opcode) qp 568 drivers/infiniband/sw/siw/siw.h qp->rx_fpdu = &qp->rx_tagged; qp 570 drivers/infiniband/sw/siw/siw.h qp->rx_fpdu = &qp->rx_untagged; qp 572 drivers/infiniband/sw/siw/siw.h qp->rx_stream.rdmap_op = opcode; qp 587 drivers/infiniband/sw/siw/siw.h return to_siw_base_qp(base_qp)->qp; qp 612 drivers/infiniband/sw/siw/siw.h struct siw_qp *qp; qp 615 drivers/infiniband/sw/siw/siw.h qp = xa_load(&sdev->qp_xa, id); qp 616 drivers/infiniband/sw/siw/siw.h if (likely(qp && kref_get_unless_zero(&qp->ref))) { qp 618 drivers/infiniband/sw/siw/siw.h return qp; qp 624 drivers/infiniband/sw/siw/siw.h static inline u32 qp_id(struct siw_qp *qp) qp 626 drivers/infiniband/sw/siw/siw.h return qp->qp_num; qp 629 drivers/infiniband/sw/siw/siw.h static inline void siw_qp_get(struct siw_qp *qp) qp 631 drivers/infiniband/sw/siw/siw.h kref_get(&qp->ref); qp 634 drivers/infiniband/sw/siw/siw.h static inline void siw_qp_put(struct siw_qp *qp) qp 636 drivers/infiniband/sw/siw/siw.h kref_put(&qp->ref, siw_free_qp); qp 639 drivers/infiniband/sw/siw/siw.h static inline int siw_sq_empty(struct siw_qp *qp) qp 641 drivers/infiniband/sw/siw/siw.h struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; qp 646 drivers/infiniband/sw/siw/siw.h static inline struct siw_sqe *sq_get_next(struct siw_qp *qp) qp 648 drivers/infiniband/sw/siw/siw.h struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; qp 656 drivers/infiniband/sw/siw/siw.h static inline struct siw_sqe *orq_get_current(struct siw_qp *qp) qp 658 drivers/infiniband/sw/siw/siw.h return &qp->orq[qp->orq_get % qp->attrs.orq_size]; qp 661 drivers/infiniband/sw/siw/siw.h static inline struct siw_sqe *orq_get_tail(struct siw_qp *qp) qp 663 drivers/infiniband/sw/siw/siw.h return &qp->orq[qp->orq_put % qp->attrs.orq_size]; qp 666 drivers/infiniband/sw/siw/siw.h static inline struct siw_sqe *orq_get_free(struct siw_qp *qp) qp 668 drivers/infiniband/sw/siw/siw.h struct siw_sqe *orq_e = orq_get_tail(qp); qp 676 drivers/infiniband/sw/siw/siw.h static inline int siw_orq_empty(struct siw_qp *qp) qp 678 drivers/infiniband/sw/siw/siw.h return qp->orq[qp->orq_get % qp->attrs.orq_size].flags == 0 ? 1 : 0; qp 681 drivers/infiniband/sw/siw/siw.h static inline struct siw_sqe *irq_alloc_free(struct siw_qp *qp) qp 683 drivers/infiniband/sw/siw/siw.h struct siw_sqe *irq_e = &qp->irq[qp->irq_put % qp->attrs.irq_size]; qp 686 drivers/infiniband/sw/siw/siw.h qp->irq_put++; qp 720 drivers/infiniband/sw/siw/siw.h #define siw_dbg_qp(qp, fmt, ...) \ qp 721 drivers/infiniband/sw/siw/siw.h ibdev_dbg(&qp->sdev->base_dev, "QP[%u] %s: " fmt, qp_id(qp), __func__, \ qp 741 drivers/infiniband/sw/siw/siw.h void siw_sq_flush(struct siw_qp *qp); qp 742 drivers/infiniband/sw/siw/siw.h void siw_rq_flush(struct siw_qp *qp); qp 72 drivers/infiniband/sw/siw/siw_cm.c static void siw_qp_socket_assoc(struct siw_cep *cep, struct siw_qp *qp) qp 79 drivers/infiniband/sw/siw/siw_cm.c qp->attrs.sk = s; qp 109 drivers/infiniband/sw/siw/siw_cm.c struct siw_qp *qp = NULL; qp 119 drivers/infiniband/sw/siw/siw_cm.c qp = sk_to_qp(sk); qp 122 drivers/infiniband/sw/siw/siw_cm.c rd_desc.arg.data = qp; qp 132 drivers/infiniband/sw/siw/siw_cm.c if (!qp->rx_stream.rx_suspend) qp 136 drivers/infiniband/sw/siw/siw_cm.c if (qp) qp 137 drivers/infiniband/sw/siw/siw_cm.c siw_qp_socket_assoc(cep, qp); qp 359 drivers/infiniband/sw/siw/siw_cm.c cep->qp ? qp_id(cep->qp) : UINT_MAX, reason, status); qp 372 drivers/infiniband/sw/siw/siw_cm.c void siw_qp_cm_drop(struct siw_qp *qp, int schedule) qp 374 drivers/infiniband/sw/siw/siw_cm.c struct siw_cep *cep = qp->cep; qp 376 drivers/infiniband/sw/siw/siw_cm.c qp->rx_stream.rx_suspend = 1; qp 377 drivers/infiniband/sw/siw/siw_cm.c qp->tx_ctx.tx_suspend = 1; qp 379 drivers/infiniband/sw/siw/siw_cm.c if (!qp->cep) qp 393 drivers/infiniband/sw/siw/siw_cm.c if (qp->term_info.valid) qp 394 drivers/infiniband/sw/siw/siw_cm.c siw_send_terminate(qp); qp 430 drivers/infiniband/sw/siw/siw_cm.c if (cep->qp) { qp 431 drivers/infiniband/sw/siw/siw_cm.c cep->qp = NULL; qp 432 drivers/infiniband/sw/siw/siw_cm.c siw_qp_put(qp); qp 719 drivers/infiniband/sw/siw/siw_cm.c struct siw_qp *qp = cep->qp; qp 741 drivers/infiniband/sw/siw/siw_cm.c siw_init_terminate(qp, TERM_ERROR_LAYER_LLP, LLP_ETYPE_MPA, qp 743 drivers/infiniband/sw/siw/siw_cm.c siw_send_terminate(qp); qp 755 drivers/infiniband/sw/siw/siw_cm.c qp->tx_ctx.gso_seg_limit = 0; qp 818 drivers/infiniband/sw/siw/siw_cm.c siw_init_terminate(qp, TERM_ERROR_LAYER_LLP, qp 821 drivers/infiniband/sw/siw/siw_cm.c siw_send_terminate(qp); qp 845 drivers/infiniband/sw/siw/siw_cm.c siw_init_terminate(qp, TERM_ERROR_LAYER_LLP, qp 849 drivers/infiniband/sw/siw/siw_cm.c siw_send_terminate(qp); qp 871 drivers/infiniband/sw/siw/siw_cm.c down_write(&qp->state_lock); qp 872 drivers/infiniband/sw/siw/siw_cm.c if (qp->attrs.state > SIW_QP_STATE_RTR) { qp 874 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 877 drivers/infiniband/sw/siw/siw_cm.c rv = siw_qp_modify(qp, &qp_attrs, qp_attr_mask); qp 879 drivers/infiniband/sw/siw/siw_cm.c siw_qp_socket_assoc(cep, qp); qp 881 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 885 drivers/infiniband/sw/siw/siw_cm.c rv = siw_qp_mpa_rts(qp, mpa_p2p_mode); qp 1012 drivers/infiniband/sw/siw/siw_cm.c cep->qp ? qp_id(cep->qp) : UINT_MAX, qp 1066 drivers/infiniband/sw/siw/siw_cm.c if (cep->qp && cep->qp->term_info.valid) qp 1067 drivers/infiniband/sw/siw/siw_cm.c siw_send_terminate(cep->qp); qp 1149 drivers/infiniband/sw/siw/siw_cm.c cep->qp ? qp_id(cep->qp) : UINT_MAX); qp 1155 drivers/infiniband/sw/siw/siw_cm.c if (cep->qp) { qp 1156 drivers/infiniband/sw/siw/siw_cm.c struct siw_qp *qp = cep->qp; qp 1161 drivers/infiniband/sw/siw/siw_cm.c siw_qp_get(qp); qp 1164 drivers/infiniband/sw/siw/siw_cm.c siw_qp_llp_close(qp); qp 1165 drivers/infiniband/sw/siw/siw_cm.c siw_qp_put(qp); qp 1168 drivers/infiniband/sw/siw/siw_cm.c cep->qp = NULL; qp 1169 drivers/infiniband/sw/siw/siw_cm.c siw_qp_put(qp); qp 1214 drivers/infiniband/sw/siw/siw_cm.c cep->qp ? qp_id(cep->qp) : -1, type, delay); qp 1300 drivers/infiniband/sw/siw/siw_cm.c if (cep->qp) qp 1301 drivers/infiniband/sw/siw/siw_cm.c cep->qp->tx_ctx.tx_suspend = 1; qp 1339 drivers/infiniband/sw/siw/siw_cm.c struct siw_qp *qp; qp 1369 drivers/infiniband/sw/siw/siw_cm.c qp = siw_qp_id2obj(sdev, params->qpn); qp 1370 drivers/infiniband/sw/siw/siw_cm.c if (!qp) { qp 1376 drivers/infiniband/sw/siw/siw_cm.c siw_dbg_qp(qp, qp 1384 drivers/infiniband/sw/siw/siw_cm.c siw_dbg_qp(qp, qp 1403 drivers/infiniband/sw/siw/siw_cm.c siw_dbg_qp(qp, "kernel_bindconnect: error %d\n", rv); qp 1412 drivers/infiniband/sw/siw/siw_cm.c siw_dbg_qp(qp, "setsockopt NODELAY error: %d\n", rv); qp 1425 drivers/infiniband/sw/siw/siw_cm.c qp->cep = cep; qp 1428 drivers/infiniband/sw/siw/siw_cm.c cep->qp = qp; qp 1509 drivers/infiniband/sw/siw/siw_cm.c siw_dbg_cep(cep, "[QP %u]: exit\n", qp_id(qp)); qp 1522 drivers/infiniband/sw/siw/siw_cm.c cep->qp = NULL; qp 1528 drivers/infiniband/sw/siw/siw_cm.c qp->cep = NULL; qp 1540 drivers/infiniband/sw/siw/siw_cm.c if (qp) qp 1541 drivers/infiniband/sw/siw/siw_cm.c siw_qp_put(qp); qp 1564 drivers/infiniband/sw/siw/siw_cm.c struct siw_qp *qp; qp 1588 drivers/infiniband/sw/siw/siw_cm.c qp = siw_qp_id2obj(sdev, params->qpn); qp 1589 drivers/infiniband/sw/siw/siw_cm.c if (!qp) { qp 1596 drivers/infiniband/sw/siw/siw_cm.c down_write(&qp->state_lock); qp 1597 drivers/infiniband/sw/siw/siw_cm.c if (qp->attrs.state > SIW_QP_STATE_RTR) { qp 1599 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 1606 drivers/infiniband/sw/siw/siw_cm.c qp->tx_ctx.gso_seg_limit = 0; qp 1613 drivers/infiniband/sw/siw/siw_cm.c qp_id(qp), params->ord, sdev->attrs.max_ord, qp 1616 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 1626 drivers/infiniband/sw/siw/siw_cm.c qp_id(qp), params->private_data_len, max_priv_data); qp 1628 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 1639 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 1649 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 1680 drivers/infiniband/sw/siw/siw_cm.c siw_dbg_cep(cep, "[QP%u]: moving to rts\n", qp_id(qp)); qp 1684 drivers/infiniband/sw/siw/siw_cm.c qp->cep = cep; qp 1687 drivers/infiniband/sw/siw/siw_cm.c cep->qp = qp; qp 1692 drivers/infiniband/sw/siw/siw_cm.c rv = siw_qp_modify(qp, &qp_attrs, qp 1696 drivers/infiniband/sw/siw/siw_cm.c up_write(&qp->state_lock); qp 1702 drivers/infiniband/sw/siw/siw_cm.c qp_id(qp), params->private_data_len); qp 1712 drivers/infiniband/sw/siw/siw_cm.c siw_qp_socket_assoc(cep, qp); qp 1731 drivers/infiniband/sw/siw/siw_cm.c if (qp->cep) { qp 1733 drivers/infiniband/sw/siw/siw_cm.c qp->cep = NULL; qp 1735 drivers/infiniband/sw/siw/siw_cm.c cep->qp = NULL; qp 1736 drivers/infiniband/sw/siw/siw_cm.c siw_qp_put(qp); qp 50 drivers/infiniband/sw/siw/siw_cm.h struct siw_qp *qp; qp 130 drivers/infiniband/sw/siw/siw_cm.h #define sk_to_qp(sk) (((struct siw_cep *)((sk)->sk_user_data))->qp) qp 73 drivers/infiniband/sw/siw/siw_cq.c wc->qp = cqe->base_qp; qp 239 drivers/infiniband/sw/siw/siw_main.c struct siw_qp *qp = siw_qp_id2obj(to_siw_dev(base_dev), id); qp 241 drivers/infiniband/sw/siw/siw_main.c if (qp) { qp 245 drivers/infiniband/sw/siw/siw_main.c siw_qp_put(qp); qp 246 drivers/infiniband/sw/siw/siw_main.c return qp->ib_qp; qp 448 drivers/infiniband/sw/siw/siw_main.c struct siw_qp *qp = list_entry(pos, struct siw_qp, devq); qp 450 drivers/infiniband/sw/siw/siw_main.c down_write(&qp->state_lock); qp 451 drivers/infiniband/sw/siw/siw_main.c WARN_ON(siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE)); qp 452 drivers/infiniband/sw/siw/siw_main.c up_write(&qp->state_lock); qp 95 drivers/infiniband/sw/siw/siw_qp.c struct siw_qp *qp; qp 102 drivers/infiniband/sw/siw/siw_qp.c qp = sk_to_qp(sk); qp 104 drivers/infiniband/sw/siw/siw_qp.c if (likely(!qp->rx_stream.rx_suspend && qp 105 drivers/infiniband/sw/siw/siw_qp.c down_read_trylock(&qp->state_lock))) { qp 106 drivers/infiniband/sw/siw/siw_qp.c read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 }; qp 108 drivers/infiniband/sw/siw/siw_qp.c if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) qp 117 drivers/infiniband/sw/siw/siw_qp.c up_read(&qp->state_lock); qp 119 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "unable to process RX, suspend: %d\n", qp 120 drivers/infiniband/sw/siw/siw_qp.c qp->rx_stream.rx_suspend); qp 126 drivers/infiniband/sw/siw/siw_qp.c void siw_qp_llp_close(struct siw_qp *qp) qp 128 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "enter llp close, state = %s\n", qp 129 drivers/infiniband/sw/siw/siw_qp.c siw_qp_state_to_string[qp->attrs.state]); qp 131 drivers/infiniband/sw/siw/siw_qp.c down_write(&qp->state_lock); qp 133 drivers/infiniband/sw/siw/siw_qp.c qp->rx_stream.rx_suspend = 1; qp 134 drivers/infiniband/sw/siw/siw_qp.c qp->tx_ctx.tx_suspend = 1; qp 135 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.sk = NULL; qp 137 drivers/infiniband/sw/siw/siw_qp.c switch (qp->attrs.state) { qp 142 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_ERROR; qp 151 drivers/infiniband/sw/siw/siw_qp.c if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) qp 152 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_ERROR; qp 154 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_IDLE; qp 158 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "llp close: no state transition needed: %s\n", qp 159 drivers/infiniband/sw/siw/siw_qp.c siw_qp_state_to_string[qp->attrs.state]); qp 162 drivers/infiniband/sw/siw/siw_qp.c siw_sq_flush(qp); qp 163 drivers/infiniband/sw/siw/siw_qp.c siw_rq_flush(qp); qp 168 drivers/infiniband/sw/siw/siw_qp.c if (qp->cep) { qp 169 drivers/infiniband/sw/siw/siw_qp.c siw_cep_put(qp->cep); qp 170 drivers/infiniband/sw/siw/siw_qp.c qp->cep = NULL; qp 173 drivers/infiniband/sw/siw/siw_qp.c up_write(&qp->state_lock); qp 175 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "llp close exit: state %s\n", qp 176 drivers/infiniband/sw/siw/siw_qp.c siw_qp_state_to_string[qp->attrs.state]); qp 194 drivers/infiniband/sw/siw/siw_qp.c (void)siw_sq_start(cep->qp); qp 200 drivers/infiniband/sw/siw/siw_qp.c static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size) qp 205 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.irq_size = irq_size; qp 206 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.orq_size = orq_size; qp 208 drivers/infiniband/sw/siw/siw_qp.c qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe)); qp 209 drivers/infiniband/sw/siw/siw_qp.c if (!qp->irq) { qp 210 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "irq malloc for %d failed\n", irq_size); qp 211 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.irq_size = 0; qp 214 drivers/infiniband/sw/siw/siw_qp.c qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe)); qp 215 drivers/infiniband/sw/siw/siw_qp.c if (!qp->orq) { qp 216 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "orq malloc for %d failed\n", orq_size); qp 217 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.orq_size = 0; qp 218 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.irq_size = 0; qp 219 drivers/infiniband/sw/siw/siw_qp.c vfree(qp->irq); qp 222 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size); qp 226 drivers/infiniband/sw/siw/siw_qp.c static int siw_qp_enable_crc(struct siw_qp *qp) qp 228 drivers/infiniband/sw/siw/siw_qp.c struct siw_rx_stream *c_rx = &qp->rx_stream; qp 229 drivers/infiniband/sw/siw/siw_qp.c struct siw_iwarp_tx *c_tx = &qp->tx_ctx; qp 260 drivers/infiniband/sw/siw/siw_qp.c int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl) qp 262 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); qp 266 drivers/infiniband/sw/siw/siw_qp.c spin_lock_irqsave(&qp->sq_lock, flags); qp 269 drivers/infiniband/sw/siw/siw_qp.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 295 drivers/infiniband/sw/siw/siw_qp.c spin_lock(&qp->orq_lock); qp 297 drivers/infiniband/sw/siw/siw_qp.c rreq = orq_get_free(qp); qp 300 drivers/infiniband/sw/siw/siw_qp.c qp->orq_put++; qp 304 drivers/infiniband/sw/siw/siw_qp.c spin_unlock(&qp->orq_lock); qp 311 drivers/infiniband/sw/siw/siw_qp.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 314 drivers/infiniband/sw/siw/siw_qp.c rv = siw_sq_start(qp); qp 363 drivers/infiniband/sw/siw/siw_qp.c void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype, qp 366 drivers/infiniband/sw/siw/siw_qp.c if (!qp->term_info.valid) { qp 367 drivers/infiniband/sw/siw/siw_qp.c memset(&qp->term_info, 0, sizeof(qp->term_info)); qp 368 drivers/infiniband/sw/siw/siw_qp.c qp->term_info.layer = layer; qp 369 drivers/infiniband/sw/siw/siw_qp.c qp->term_info.etype = etype; qp 370 drivers/infiniband/sw/siw/siw_qp.c qp->term_info.ecode = ecode; qp 371 drivers/infiniband/sw/siw/siw_qp.c qp->term_info.in_tx = in_tx; qp 372 drivers/infiniband/sw/siw/siw_qp.c qp->term_info.valid = 1; qp 374 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", qp 386 drivers/infiniband/sw/siw/siw_qp.c void siw_send_terminate(struct siw_qp *qp) qp 392 drivers/infiniband/sw/siw/siw_qp.c struct socket *s = qp->attrs.sk; qp 393 drivers/infiniband/sw/siw/siw_qp.c struct siw_rx_stream *srx = &qp->rx_stream; qp 398 drivers/infiniband/sw/siw/siw_qp.c if (!qp->term_info.valid) qp 401 drivers/infiniband/sw/siw/siw_qp.c qp->term_info.valid = 0; qp 403 drivers/infiniband/sw/siw/siw_qp.c if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { qp 404 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n", qp 405 drivers/infiniband/sw/siw/siw_qp.c tx_type(tx_wqe(qp))); qp 408 drivers/infiniband/sw/siw/siw_qp.c if (!s && qp->cep) qp 410 drivers/infiniband/sw/siw/siw_qp.c s = qp->cep->sock; qp 413 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n"); qp 428 drivers/infiniband/sw/siw/siw_qp.c if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || qp 429 drivers/infiniband/sw/siw/siw_qp.c ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && qp 430 drivers/infiniband/sw/siw/siw_qp.c (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { qp 440 drivers/infiniband/sw/siw/siw_qp.c __rdmap_term_set_layer(term, qp->term_info.layer); qp 441 drivers/infiniband/sw/siw/siw_qp.c __rdmap_term_set_etype(term, qp->term_info.etype); qp 442 drivers/infiniband/sw/siw/siw_qp.c __rdmap_term_set_ecode(term, qp->term_info.ecode); qp 444 drivers/infiniband/sw/siw/siw_qp.c switch (qp->term_info.layer) { qp 446 drivers/infiniband/sw/siw/siw_qp.c if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) qp 450 drivers/infiniband/sw/siw/siw_qp.c if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { qp 459 drivers/infiniband/sw/siw/siw_qp.c if (qp->term_info.in_tx) { qp 461 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); qp 511 drivers/infiniband/sw/siw/siw_qp.c if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || qp 512 drivers/infiniband/sw/siw/siw_qp.c (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) qp 539 drivers/infiniband/sw/siw/siw_qp.c if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && qp 540 drivers/infiniband/sw/siw/siw_qp.c (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || qp 541 drivers/infiniband/sw/siw/siw_qp.c ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && qp 542 drivers/infiniband/sw/siw/siw_qp.c (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) qp 582 drivers/infiniband/sw/siw/siw_qp.c if (qp->tx_ctx.mpa_crc_hd) { qp 583 drivers/infiniband/sw/siw/siw_qp.c crypto_shash_init(qp->tx_ctx.mpa_crc_hd); qp 584 drivers/infiniband/sw/siw/siw_qp.c if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, qp 590 drivers/infiniband/sw/siw/siw_qp.c if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, qp 595 drivers/infiniband/sw/siw/siw_qp.c crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); qp 599 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", qp 611 drivers/infiniband/sw/siw/siw_qp.c static void siw_qp_modify_nonstate(struct siw_qp *qp, qp 617 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; qp 619 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; qp 622 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; qp 624 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; qp 627 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.flags |= SIW_RDMA_READ_ENABLED; qp 629 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; qp 633 drivers/infiniband/sw/siw/siw_qp.c static int siw_qp_nextstate_from_idle(struct siw_qp *qp, qp 642 drivers/infiniband/sw/siw/siw_qp.c rv = siw_qp_enable_crc(qp); qp 647 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "no socket\n"); qp 652 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "no MPA\n"); qp 659 drivers/infiniband/sw/siw/siw_qp.c qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; qp 660 drivers/infiniband/sw/siw/siw_qp.c qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; qp 661 drivers/infiniband/sw/siw/siw_qp.c qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; qp 666 drivers/infiniband/sw/siw/siw_qp.c qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; qp 667 drivers/infiniband/sw/siw/siw_qp.c qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; qp 668 drivers/infiniband/sw/siw/siw_qp.c qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; qp 674 drivers/infiniband/sw/siw/siw_qp.c rv = siw_qp_readq_init(qp, attrs->irq_size, qp 679 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.sk = attrs->sk; qp 680 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_RTS; qp 682 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n", qp 684 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.orq_size, qp->attrs.irq_size); qp 688 drivers/infiniband/sw/siw/siw_qp.c siw_rq_flush(qp); qp 689 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_ERROR; qp 690 drivers/infiniband/sw/siw/siw_qp.c if (qp->cep) { qp 691 drivers/infiniband/sw/siw/siw_qp.c siw_cep_put(qp->cep); qp 692 drivers/infiniband/sw/siw/siw_qp.c qp->cep = NULL; qp 702 drivers/infiniband/sw/siw/siw_qp.c static int siw_qp_nextstate_from_rts(struct siw_qp *qp, qp 717 drivers/infiniband/sw/siw/siw_qp.c if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { qp 718 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_CLOSING; qp 720 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_ERROR; qp 721 drivers/infiniband/sw/siw/siw_qp.c siw_sq_flush(qp); qp 723 drivers/infiniband/sw/siw/siw_qp.c siw_rq_flush(qp); qp 729 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_TERMINATE; qp 731 drivers/infiniband/sw/siw/siw_qp.c siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, qp 751 drivers/infiniband/sw/siw/siw_qp.c siw_sq_flush(qp); qp 752 drivers/infiniband/sw/siw/siw_qp.c siw_rq_flush(qp); qp 753 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_ERROR; qp 763 drivers/infiniband/sw/siw/siw_qp.c static void siw_qp_nextstate_from_term(struct siw_qp *qp, qp 768 drivers/infiniband/sw/siw/siw_qp.c siw_rq_flush(qp); qp 769 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_ERROR; qp 771 drivers/infiniband/sw/siw/siw_qp.c if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) qp 772 drivers/infiniband/sw/siw/siw_qp.c siw_sq_flush(qp); qp 780 drivers/infiniband/sw/siw/siw_qp.c static int siw_qp_nextstate_from_close(struct siw_qp *qp, qp 787 drivers/infiniband/sw/siw/siw_qp.c WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); qp 788 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_IDLE; qp 803 drivers/infiniband/sw/siw/siw_qp.c qp->attrs.state = SIW_QP_STATE_ERROR; qp 805 drivers/infiniband/sw/siw/siw_qp.c if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) qp 806 drivers/infiniband/sw/siw/siw_qp.c siw_sq_flush(qp); qp 808 drivers/infiniband/sw/siw/siw_qp.c siw_rq_flush(qp); qp 812 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "state transition undefined: %s => %s\n", qp 813 drivers/infiniband/sw/siw/siw_qp.c siw_qp_state_to_string[qp->attrs.state], qp 824 drivers/infiniband/sw/siw/siw_qp.c int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs, qp 832 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "state: %s => %s\n", qp 833 drivers/infiniband/sw/siw/siw_qp.c siw_qp_state_to_string[qp->attrs.state], qp 837 drivers/infiniband/sw/siw/siw_qp.c siw_qp_modify_nonstate(qp, attrs, mask); qp 842 drivers/infiniband/sw/siw/siw_qp.c switch (qp->attrs.state) { qp 845 drivers/infiniband/sw/siw/siw_qp.c rv = siw_qp_nextstate_from_idle(qp, attrs, mask); qp 849 drivers/infiniband/sw/siw/siw_qp.c drop_conn = siw_qp_nextstate_from_rts(qp, attrs); qp 853 drivers/infiniband/sw/siw/siw_qp.c siw_qp_nextstate_from_term(qp, attrs); qp 857 drivers/infiniband/sw/siw/siw_qp.c siw_qp_nextstate_from_close(qp, attrs); qp 863 drivers/infiniband/sw/siw/siw_qp.c siw_qp_cm_drop(qp, 0); qp 886 drivers/infiniband/sw/siw/siw_qp.c int siw_activate_tx(struct siw_qp *qp) qp 889 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); qp 892 drivers/infiniband/sw/siw/siw_qp.c irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; qp 895 drivers/infiniband/sw/siw/siw_qp.c sqe = sq_get_next(qp); qp 901 drivers/infiniband/sw/siw/siw_qp.c if (sqe && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { qp 902 drivers/infiniband/sw/siw/siw_qp.c qp->irq_burst = 0; qp 929 drivers/infiniband/sw/siw/siw_qp.c qp->irq_get++; qp 936 drivers/infiniband/sw/siw/siw_qp.c sqe = sq_get_next(qp); qp 968 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "cannot fence read\n"); qp 972 drivers/infiniband/sw/siw/siw_qp.c spin_lock(&qp->orq_lock); qp 974 drivers/infiniband/sw/siw/siw_qp.c if (!siw_orq_empty(qp)) { qp 975 drivers/infiniband/sw/siw/siw_qp.c qp->tx_ctx.orq_fence = 1; qp 978 drivers/infiniband/sw/siw/siw_qp.c spin_unlock(&qp->orq_lock); qp 986 drivers/infiniband/sw/siw/siw_qp.c spin_lock(&qp->orq_lock); qp 988 drivers/infiniband/sw/siw/siw_qp.c rreq = orq_get_free(qp); qp 995 drivers/infiniband/sw/siw/siw_qp.c qp->orq_put++; qp 997 drivers/infiniband/sw/siw/siw_qp.c qp->tx_ctx.orq_fence = 1; qp 1000 drivers/infiniband/sw/siw/siw_qp.c spin_unlock(&qp->orq_lock); qp 1005 drivers/infiniband/sw/siw/siw_qp.c qp->sq_get++; qp 1011 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "error %d\n", rv); qp 1047 drivers/infiniband/sw/siw/siw_qp.c int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, qp 1050 drivers/infiniband/sw/siw/siw_qp.c struct siw_cq *cq = qp->scq; qp 1074 drivers/infiniband/sw/siw/siw_qp.c cqe->base_qp = qp->ib_qp; qp 1076 drivers/infiniband/sw/siw/siw_qp.c cqe->qp_id = qp_id(qp); qp 1105 drivers/infiniband/sw/siw/siw_qp.c int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, qp 1108 drivers/infiniband/sw/siw/siw_qp.c struct siw_cq *cq = qp->rcq; qp 1132 drivers/infiniband/sw/siw/siw_qp.c cqe->base_qp = qp->ib_qp; qp 1138 drivers/infiniband/sw/siw/siw_qp.c cqe->qp_id = qp_id(qp); qp 1175 drivers/infiniband/sw/siw/siw_qp.c void siw_sq_flush(struct siw_qp *qp) qp 1178 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); qp 1184 drivers/infiniband/sw/siw/siw_qp.c while (qp->attrs.orq_size) { qp 1185 drivers/infiniband/sw/siw/siw_qp.c sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; qp 1189 drivers/infiniband/sw/siw/siw_qp.c if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) qp 1193 drivers/infiniband/sw/siw/siw_qp.c qp->orq_get++; qp 1199 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n", qp 1212 drivers/infiniband/sw/siw/siw_qp.c siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, qp 1220 drivers/infiniband/sw/siw/siw_qp.c while (qp->attrs.sq_size) { qp 1221 drivers/infiniband/sw/siw/siw_qp.c sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; qp 1226 drivers/infiniband/sw/siw/siw_qp.c if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) qp 1234 drivers/infiniband/sw/siw/siw_qp.c qp->sq_get++; qp 1237 drivers/infiniband/sw/siw/siw_qp.c siw_qp_event(qp, IB_EVENT_SQ_DRAINED); qp 1251 drivers/infiniband/sw/siw/siw_qp.c void siw_rq_flush(struct siw_qp *qp) qp 1253 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; qp 1259 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n", qp 1265 drivers/infiniband/sw/siw/siw_qp.c siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, qp 1270 drivers/infiniband/sw/siw/siw_qp.c siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); qp 1274 drivers/infiniband/sw/siw/siw_qp.c wqe = &qp->rx_tagged.wqe_active; qp 1283 drivers/infiniband/sw/siw/siw_qp.c while (qp->attrs.rq_size) { qp 1285 drivers/infiniband/sw/siw/siw_qp.c &qp->recvq[qp->rq_get % qp->attrs.rq_size]; qp 1290 drivers/infiniband/sw/siw/siw_qp.c if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0) qp 1294 drivers/infiniband/sw/siw/siw_qp.c qp->rq_get++; qp 1298 drivers/infiniband/sw/siw/siw_qp.c int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) qp 1300 drivers/infiniband/sw/siw/siw_qp.c int rv = xa_alloc(&sdev->qp_xa, &qp->ib_qp->qp_num, qp, xa_limit_32b, qp 1304 drivers/infiniband/sw/siw/siw_qp.c kref_init(&qp->ref); qp 1305 drivers/infiniband/sw/siw/siw_qp.c qp->sdev = sdev; qp 1306 drivers/infiniband/sw/siw/siw_qp.c qp->qp_num = qp->ib_qp->qp_num; qp 1307 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "new QP\n"); qp 1314 drivers/infiniband/sw/siw/siw_qp.c struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref); qp 1315 drivers/infiniband/sw/siw/siw_qp.c struct siw_base_qp *siw_base_qp = to_siw_base_qp(qp->ib_qp); qp 1316 drivers/infiniband/sw/siw/siw_qp.c struct siw_device *sdev = qp->sdev; qp 1319 drivers/infiniband/sw/siw/siw_qp.c if (qp->cep) qp 1320 drivers/infiniband/sw/siw/siw_qp.c siw_cep_put(qp->cep); qp 1322 drivers/infiniband/sw/siw/siw_qp.c found = xa_erase(&sdev->qp_xa, qp_id(qp)); qp 1323 drivers/infiniband/sw/siw/siw_qp.c WARN_ON(found != qp); qp 1325 drivers/infiniband/sw/siw/siw_qp.c list_del(&qp->devq); qp 1328 drivers/infiniband/sw/siw/siw_qp.c vfree(qp->sendq); qp 1329 drivers/infiniband/sw/siw/siw_qp.c vfree(qp->recvq); qp 1330 drivers/infiniband/sw/siw/siw_qp.c vfree(qp->irq); qp 1331 drivers/infiniband/sw/siw/siw_qp.c vfree(qp->orq); qp 1333 drivers/infiniband/sw/siw/siw_qp.c siw_put_tx_cpu(qp->tx_cpu); qp 1336 drivers/infiniband/sw/siw/siw_qp.c siw_dbg_qp(qp, "free QP\n"); qp 1337 drivers/infiniband/sw/siw/siw_qp.c kfree_rcu(qp, rcu); qp 329 drivers/infiniband/sw/siw/siw_qp_rx.c static struct siw_wqe *siw_rqe_get(struct siw_qp *qp) qp 337 drivers/infiniband/sw/siw/siw_qp_rx.c srq = qp->srq; qp 345 drivers/infiniband/sw/siw/siw_qp_rx.c if (unlikely(!qp->recvq)) qp 348 drivers/infiniband/sw/siw/siw_qp_rx.c rqe = &qp->recvq[qp->rq_get % qp->attrs.rq_size]; qp 356 drivers/infiniband/sw/siw/siw_qp_rx.c wqe = rx_wqe(&qp->rx_untagged); qp 376 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "too many sge's: %d\n", rqe->num_sge); qp 382 drivers/infiniband/sw/siw/siw_qp_rx.c qp->rq_get++; qp 420 drivers/infiniband/sw/siw/siw_qp_rx.c int siw_proc_send(struct siw_qp *qp) qp 422 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_stream *srx = &qp->rx_stream; qp 423 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_fpdu *frx = &qp->rx_untagged; qp 430 drivers/infiniband/sw/siw/siw_qp_rx.c wqe = siw_rqe_get(qp); qp 432 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 443 drivers/infiniband/sw/siw/siw_qp_rx.c siw_qp_event(qp, IB_EVENT_QP_FATAL); qp 474 drivers/infiniband/sw/siw/siw_qp_rx.c pd = qp->srq == NULL ? qp->pd : qp->srq->base_srq.pd; qp 479 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 483 drivers/infiniband/sw/siw/siw_qp_rx.c siw_qp_event(qp, IB_EVENT_QP_ACCESS_ERR); qp 501 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 539 drivers/infiniband/sw/siw/siw_qp_rx.c int siw_proc_write(struct siw_qp *qp) qp 541 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_stream *srx = &qp->rx_stream; qp 542 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_fpdu *frx = &qp->rx_tagged; qp 552 drivers/infiniband/sw/siw/siw_qp_rx.c siw_qp_event(qp, IB_EVENT_QP_FATAL); qp 561 drivers/infiniband/sw/siw/siw_qp_rx.c rx_mem(frx) = siw_mem_id2obj(qp->sdev, srx->ddp_stag >> 8); qp 563 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, qp 567 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 583 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 588 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_check_mem(qp->pd, mem, srx->ddp_to + srx->fpdu_part_rcvd, qp 591 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 595 drivers/infiniband/sw/siw/siw_qp_rx.c siw_qp_event(qp, IB_EVENT_QP_ACCESS_ERR); qp 612 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 630 drivers/infiniband/sw/siw/siw_qp_rx.c int siw_proc_rreq(struct siw_qp *qp) qp 632 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_stream *srx = &qp->rx_stream; qp 637 drivers/infiniband/sw/siw/siw_qp_rx.c pr_warn("siw: [QP %u]: rreq with mpa len %d\n", qp_id(qp), qp 659 drivers/infiniband/sw/siw/siw_qp_rx.c static int siw_init_rresp(struct siw_qp *qp, struct siw_rx_stream *srx) qp 661 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *tx_work = tx_wqe(qp); qp 675 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 680 drivers/infiniband/sw/siw/siw_qp_rx.c spin_lock_irqsave(&qp->sq_lock, flags); qp 692 drivers/infiniband/sw/siw/siw_qp_rx.c resp = irq_alloc_free(qp); qp 714 drivers/infiniband/sw/siw/siw_qp_rx.c pr_warn("siw: [QP %u]: irq %d exceeded %d\n", qp_id(qp), qp 715 drivers/infiniband/sw/siw/siw_qp_rx.c qp->irq_put % qp->attrs.irq_size, qp->attrs.irq_size); qp 717 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, qp 723 drivers/infiniband/sw/siw/siw_qp_rx.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 726 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_sq_start(qp); qp 737 drivers/infiniband/sw/siw/siw_qp_rx.c static int siw_orqe_start_rx(struct siw_qp *qp) qp 745 drivers/infiniband/sw/siw/siw_qp_rx.c orqe = orq_get_current(qp); qp 748 drivers/infiniband/sw/siw/siw_qp_rx.c wqe = rx_wqe(&qp->rx_tagged); qp 777 drivers/infiniband/sw/siw/siw_qp_rx.c int siw_proc_rresp(struct siw_qp *qp) qp 779 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_stream *srx = &qp->rx_stream; qp 780 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_fpdu *frx = &qp->rx_tagged; qp 789 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(qp), wqe->wr_status, wqe->sqe.opcode); qp 796 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_orqe_start_rx(qp); qp 799 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(qp), qp->orq_get % qp->attrs.orq_size); qp 804 drivers/infiniband/sw/siw/siw_qp_rx.c siw_qp_event(qp, IB_EVENT_QP_FATAL); qp 810 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(qp), wqe->wr_status); qp 825 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_check_sge(qp->pd, sge, mem, IB_ACCESS_LOCAL_WRITE, 0, qp 828 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "target mem check: %d\n", rv); qp 831 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, qp 835 drivers/infiniband/sw/siw/siw_qp_rx.c siw_qp_event(qp, IB_EVENT_QP_ACCESS_ERR); qp 870 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, DDP_ETYPE_CATASTROPHIC, qp 875 drivers/infiniband/sw/siw/siw_qp_rx.c int siw_proc_terminate(struct siw_qp *qp) qp 877 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_stream *srx = &qp->rx_stream; qp 891 drivers/infiniband/sw/siw/siw_qp_rx.c qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] || qp 934 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "TERM reports RDMAP hdr type %u, len %u (%s)\n", qp 938 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "TERM reports DDP hdr type %u, len %u (%s)\n", qp 952 drivers/infiniband/sw/siw/siw_qp_rx.c static int siw_get_trailer(struct siw_qp *qp, struct siw_rx_stream *srx) qp 958 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "expected %d, available %d, pad %u\n", qp 985 drivers/infiniband/sw/siw/siw_qp_rx.c crc_in, crc_own, qp->rx_stream.rdmap_op); qp 987 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_LLP, qp 1000 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_qp *qp = rx_qp(srx); qp 1067 drivers/infiniband/sw/siw/siw_qp_rx.c set_rx_fpdu_context(qp, opcode); qp 1068 drivers/infiniband/sw/siw/siw_qp_rx.c frx = qp->rx_fpdu; qp 1125 drivers/infiniband/sw/siw/siw_qp_rx.c set_rx_fpdu_context(qp, frx->prev_rdmap_op); qp 1138 drivers/infiniband/sw/siw/siw_qp_rx.c static int siw_check_tx_fence(struct siw_qp *qp) qp 1140 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *tx_waiting = tx_wqe(qp); qp 1145 drivers/infiniband/sw/siw/siw_qp_rx.c spin_lock_irqsave(&qp->orq_lock, flags); qp 1147 drivers/infiniband/sw/siw/siw_qp_rx.c rreq = orq_get_current(qp); qp 1152 drivers/infiniband/sw/siw/siw_qp_rx.c if (qp->tx_ctx.orq_fence) { qp 1155 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(qp), tx_waiting->wr_status); qp 1162 drivers/infiniband/sw/siw/siw_qp_rx.c rreq = orq_get_tail(qp); qp 1164 drivers/infiniband/sw/siw/siw_qp_rx.c pr_warn("siw: [QP %u]: no ORQE\n", qp_id(qp)); qp 1170 drivers/infiniband/sw/siw/siw_qp_rx.c qp->orq_put++; qp 1171 drivers/infiniband/sw/siw/siw_qp_rx.c qp->tx_ctx.orq_fence = 0; qp 1174 drivers/infiniband/sw/siw/siw_qp_rx.c } else if (siw_orq_empty(qp)) { qp 1175 drivers/infiniband/sw/siw/siw_qp_rx.c qp->tx_ctx.orq_fence = 0; qp 1179 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(qp), qp->orq_get, qp->orq_put); qp 1183 drivers/infiniband/sw/siw/siw_qp_rx.c qp->orq_get++; qp 1185 drivers/infiniband/sw/siw/siw_qp_rx.c spin_unlock_irqrestore(&qp->orq_lock, flags); qp 1188 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_sq_start(qp); qp 1205 drivers/infiniband/sw/siw/siw_qp_rx.c static int siw_rdmap_complete(struct siw_qp *qp, int error) qp 1207 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_stream *srx = &qp->rx_stream; qp 1208 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = rx_wqe(qp->rx_fpdu); qp 1234 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_invalidate_stag(qp->pd, srx->inval_stag); qp 1237 drivers/infiniband/sw/siw/siw_qp_rx.c qp, TERM_ERROR_LAYER_RDMAP, qp 1245 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_rqe_complete(qp, &wqe->rqe, wqe->processed, qp 1249 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_rqe_complete(qp, &wqe->rqe, wqe->processed, qp 1261 drivers/infiniband/sw/siw/siw_qp_rx.c qp->rx_fpdu->first_ddp_seg) || error == -ENODATA) qp 1267 drivers/infiniband/sw/siw/siw_qp_rx.c } else if (qp->kernel_verbs && qp 1272 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_invalidate_stag(qp->pd, wqe->sqe.sge[0].lkey); qp 1274 drivers/infiniband/sw/siw/siw_qp_rx.c siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, qp 1288 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_sqe_complete(qp, &wqe->sqe, wqe->processed, qp 1293 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_check_tx_fence(qp); qp 1296 drivers/infiniband/sw/siw/siw_qp_rx.c WRITE_ONCE(orq_get_current(qp)->flags, 0); qp 1301 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_init_rresp(qp, srx); qp 1316 drivers/infiniband/sw/siw/siw_qp_rx.c if (rx_mem(&qp->rx_tagged)) { qp 1317 drivers/infiniband/sw/siw/siw_qp_rx.c siw_mem_put(rx_mem(&qp->rx_tagged)); qp 1318 drivers/infiniband/sw/siw/siw_qp_rx.c rx_mem(&qp->rx_tagged) = NULL; qp 1343 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_qp *qp = rd_desc->arg.data; qp 1344 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_rx_stream *srx = &qp->rx_stream; qp 1352 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "new data, len %d\n", srx->skb_new); qp 1387 drivers/infiniband/sw/siw/siw_qp_rx.c qp->rx_fpdu->first_ddp_seg = 0; qp 1395 drivers/infiniband/sw/siw/siw_qp_rx.c rv = iwarp_pktinfo[qp->rx_stream.rdmap_op].rx_data(qp); qp 1417 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_get_trailer(qp, srx); qp 1431 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_rdmap_complete(qp, 0); qp 1437 drivers/infiniband/sw/siw/siw_qp_rx.c pr_warn("QP[%u]: RX out of state\n", qp_id(qp)); qp 1443 drivers/infiniband/sw/siw/siw_qp_rx.c qp->rx_fpdu->more_ddp_segs) && run_completion) qp 1444 drivers/infiniband/sw/siw/siw_qp_rx.c siw_rdmap_complete(qp, rv); qp 1446 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "rx error %d, rx state %d\n", rv, qp 1449 drivers/infiniband/sw/siw/siw_qp_rx.c siw_qp_cm_drop(qp, 1); qp 1454 drivers/infiniband/sw/siw/siw_qp_rx.c siw_dbg_qp(qp, "fpdu fragment, state %d, missing %d\n", qp 693 drivers/infiniband/sw/siw/siw_qp_tx.c static void siw_prepare_fpdu(struct siw_qp *qp, struct siw_wqe *wqe) qp 695 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_iwarp_tx *c_tx = &qp->tx_ctx; qp 781 drivers/infiniband/sw/siw/siw_qp_tx.c static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe) qp 783 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_iwarp_tx *c_tx = &qp->tx_ctx; qp 784 drivers/infiniband/sw/siw/siw_qp_tx.c struct socket *s = qp->attrs.sk; qp 785 drivers/infiniband/sw/siw/siw_qp_tx.c int rv = 0, burst_len = qp->tx_ctx.burst; qp 806 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_check_sgl_tx(qp->pd, wqe, 0); qp 820 drivers/infiniband/sw/siw/siw_qp_tx.c if (!qp->kernel_verbs) { qp 837 drivers/infiniband/sw/siw/siw_qp_tx.c siw_prepare_fpdu(qp, wqe); qp 846 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "wr type %d, state %d, data %u, sent %u, id %llx\n", qp 858 drivers/infiniband/sw/siw/siw_qp_tx.c if (siw_sq_empty(qp) || !siw_tcp_nagle || burst_len == 1) qp 894 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "WQE completed\n"); qp 901 drivers/infiniband/sw/siw/siw_qp_tx.c siw_prepare_fpdu(qp, wqe); qp 905 drivers/infiniband/sw/siw/siw_qp_tx.c qp->tx_ctx.burst = burst_len; qp 910 drivers/infiniband/sw/siw/siw_qp_tx.c siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, qp 913 drivers/infiniband/sw/siw/siw_qp_tx.c siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, qp 966 drivers/infiniband/sw/siw/siw_qp_tx.c static int siw_qp_sq_proc_local(struct siw_qp *qp, struct siw_wqe *wqe) qp 972 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_fastreg_mr(qp->pd, &wqe->sqe); qp 976 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_invalidate_stag(qp->pd, wqe->sqe.rkey); qp 1012 drivers/infiniband/sw/siw/siw_qp_tx.c int siw_qp_sq_process(struct siw_qp *qp) qp 1014 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_wqe *wqe = tx_wqe(qp); qp 1019 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "enter for type %d\n", tx_type(wqe)); qp 1025 drivers/infiniband/sw/siw/siw_qp_tx.c if (unlikely(qp->tx_ctx.tx_suspend)) { qp 1026 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "tx suspended\n"); qp 1032 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_qp_sq_proc_tx(qp, wqe); qp 1034 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_qp_sq_proc_local(qp, wqe); qp 1050 drivers/infiniband/sw/siw/siw_qp_tx.c siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, qp 1071 drivers/infiniband/sw/siw/siw_qp_tx.c spin_lock_irqsave(&qp->sq_lock, flags); qp 1073 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_activate_tx(qp); qp 1074 drivers/infiniband/sw/siw/siw_qp_tx.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 1082 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "sq paused: hd/tr %d of %d, data %d\n", qp 1083 drivers/infiniband/sw/siw/siw_qp_tx.c qp->tx_ctx.ctrl_sent, qp->tx_ctx.ctrl_len, qp 1084 drivers/infiniband/sw/siw/siw_qp_tx.c qp->tx_ctx.bytes_unsent); qp 1088 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_sq_start(qp); qp 1103 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "wqe type %d processing failed: %d\n", qp 1106 drivers/infiniband/sw/siw/siw_qp_tx.c spin_lock_irqsave(&qp->sq_lock, flags); qp 1113 drivers/infiniband/sw/siw/siw_qp_tx.c qp->orq_put--; qp 1114 drivers/infiniband/sw/siw/siw_qp_tx.c qp->orq[qp->orq_put % qp->attrs.orq_size].flags = 0; qp 1116 drivers/infiniband/sw/siw/siw_qp_tx.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 1120 drivers/infiniband/sw/siw/siw_qp_tx.c if (!qp->tx_ctx.tx_suspend) qp 1121 drivers/infiniband/sw/siw/siw_qp_tx.c siw_qp_cm_drop(qp, 0); qp 1135 drivers/infiniband/sw/siw/siw_qp_tx.c siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, qp 1138 drivers/infiniband/sw/siw/siw_qp_tx.c siw_qp_event(qp, IB_EVENT_QP_FATAL); qp 1143 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "proc. read.response failed: %d\n", rv); qp 1145 drivers/infiniband/sw/siw/siw_qp_tx.c siw_qp_event(qp, IB_EVENT_QP_REQ_ERR); qp 1161 drivers/infiniband/sw/siw/siw_qp_tx.c static void siw_sq_resume(struct siw_qp *qp) qp 1163 drivers/infiniband/sw/siw/siw_qp_tx.c if (down_read_trylock(&qp->state_lock)) { qp 1164 drivers/infiniband/sw/siw/siw_qp_tx.c if (likely(qp->attrs.state == SIW_QP_STATE_RTS && qp 1165 drivers/infiniband/sw/siw/siw_qp_tx.c !qp->tx_ctx.tx_suspend)) { qp 1166 drivers/infiniband/sw/siw/siw_qp_tx.c int rv = siw_qp_sq_process(qp); qp 1168 drivers/infiniband/sw/siw/siw_qp_tx.c up_read(&qp->state_lock); qp 1171 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "SQ task failed: err %d\n", rv); qp 1173 drivers/infiniband/sw/siw/siw_qp_tx.c if (!qp->tx_ctx.tx_suspend) qp 1174 drivers/infiniband/sw/siw/siw_qp_tx.c siw_qp_cm_drop(qp, 0); qp 1177 drivers/infiniband/sw/siw/siw_qp_tx.c up_read(&qp->state_lock); qp 1180 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "Resume SQ while QP locked\n"); qp 1182 drivers/infiniband/sw/siw/siw_qp_tx.c siw_qp_put(qp); qp 1202 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_qp *qp; qp 1231 drivers/infiniband/sw/siw/siw_qp_tx.c qp = container_of(fifo_list, struct siw_qp, tx_list); qp 1233 drivers/infiniband/sw/siw/siw_qp_tx.c qp->tx_list.next = NULL; qp 1235 drivers/infiniband/sw/siw/siw_qp_tx.c siw_sq_resume(qp); qp 1240 drivers/infiniband/sw/siw/siw_qp_tx.c llist_for_each_entry(qp, active, tx_list) { qp 1241 drivers/infiniband/sw/siw/siw_qp_tx.c qp->tx_list.next = NULL; qp 1242 drivers/infiniband/sw/siw/siw_qp_tx.c siw_sq_resume(qp); qp 1248 drivers/infiniband/sw/siw/siw_qp_tx.c int siw_sq_start(struct siw_qp *qp) qp 1250 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) qp 1253 drivers/infiniband/sw/siw/siw_qp_tx.c if (unlikely(!cpu_online(qp->tx_cpu))) { qp 1254 drivers/infiniband/sw/siw/siw_qp_tx.c siw_put_tx_cpu(qp->tx_cpu); qp 1255 drivers/infiniband/sw/siw/siw_qp_tx.c qp->tx_cpu = siw_get_tx_cpu(qp->sdev); qp 1256 drivers/infiniband/sw/siw/siw_qp_tx.c if (qp->tx_cpu < 0) { qp 1262 drivers/infiniband/sw/siw/siw_qp_tx.c siw_qp_get(qp); qp 1264 drivers/infiniband/sw/siw/siw_qp_tx.c llist_add(&qp->tx_list, &per_cpu(siw_tx_task_g, qp->tx_cpu).active); qp 1266 drivers/infiniband/sw/siw/siw_qp_tx.c wake_up(&per_cpu(siw_tx_task_g, qp->tx_cpu).waiting); qp 310 drivers/infiniband/sw/siw/siw_verbs.c struct siw_qp *qp = NULL; qp 369 drivers/infiniband/sw/siw/siw_verbs.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 370 drivers/infiniband/sw/siw/siw_verbs.c if (!qp) { qp 374 drivers/infiniband/sw/siw/siw_verbs.c siw_base_qp->qp = qp; qp 375 drivers/infiniband/sw/siw/siw_verbs.c qp->ib_qp = &siw_base_qp->base_qp; qp 377 drivers/infiniband/sw/siw/siw_verbs.c init_rwsem(&qp->state_lock); qp 378 drivers/infiniband/sw/siw/siw_verbs.c spin_lock_init(&qp->sq_lock); qp 379 drivers/infiniband/sw/siw/siw_verbs.c spin_lock_init(&qp->rq_lock); qp 380 drivers/infiniband/sw/siw/siw_verbs.c spin_lock_init(&qp->orq_lock); qp 382 drivers/infiniband/sw/siw/siw_verbs.c qp->kernel_verbs = !udata; qp 383 drivers/infiniband/sw/siw/siw_verbs.c qp->xa_sq_index = SIW_INVAL_UOBJ_KEY; qp 384 drivers/infiniband/sw/siw/siw_verbs.c qp->xa_rq_index = SIW_INVAL_UOBJ_KEY; qp 386 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_qp_add(sdev, qp); qp 398 drivers/infiniband/sw/siw/siw_verbs.c if (qp->kernel_verbs) qp 399 drivers/infiniband/sw/siw/siw_verbs.c qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe)); qp 401 drivers/infiniband/sw/siw/siw_verbs.c qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe)); qp 403 drivers/infiniband/sw/siw/siw_verbs.c if (qp->sendq == NULL) { qp 410 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.flags |= SIW_SIGNAL_ALL_WR; qp 416 drivers/infiniband/sw/siw/siw_verbs.c qp->pd = pd; qp 417 drivers/infiniband/sw/siw/siw_verbs.c qp->scq = scq; qp 418 drivers/infiniband/sw/siw/siw_verbs.c qp->rcq = rcq; qp 426 drivers/infiniband/sw/siw/siw_verbs.c qp->srq = to_siw_srq(attrs->srq); qp 427 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.rq_size = 0; qp 428 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg(base_dev, "QP [%u]: SRQ attached\n", qp->qp_num); qp 430 drivers/infiniband/sw/siw/siw_verbs.c if (qp->kernel_verbs) qp 431 drivers/infiniband/sw/siw/siw_verbs.c qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe)); qp 433 drivers/infiniband/sw/siw/siw_verbs.c qp->recvq = qp 436 drivers/infiniband/sw/siw/siw_verbs.c if (qp->recvq == NULL) { qp 441 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.rq_size = num_rqe; qp 443 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.sq_size = num_sqe; qp 444 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.sq_max_sges = attrs->cap.max_send_sge; qp 445 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.rq_max_sges = attrs->cap.max_recv_sge; qp 448 drivers/infiniband/sw/siw/siw_verbs.c qp->tx_ctx.gso_seg_limit = 1; qp 449 drivers/infiniband/sw/siw/siw_verbs.c qp->tx_ctx.zcopy_tx = zcopy_tx; qp 451 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.state = SIW_QP_STATE_IDLE; qp 458 drivers/infiniband/sw/siw/siw_verbs.c uresp.qp_id = qp_id(qp); qp 460 drivers/infiniband/sw/siw/siw_verbs.c if (qp->sendq) { qp 461 drivers/infiniband/sw/siw/siw_verbs.c qp->xa_sq_index = qp 462 drivers/infiniband/sw/siw/siw_verbs.c siw_create_uobj(uctx, qp->sendq, qp 465 drivers/infiniband/sw/siw/siw_verbs.c if (qp->recvq) { qp 466 drivers/infiniband/sw/siw/siw_verbs.c qp->xa_rq_index = qp 467 drivers/infiniband/sw/siw/siw_verbs.c siw_create_uobj(uctx, qp->recvq, qp 470 drivers/infiniband/sw/siw/siw_verbs.c if (qp->xa_sq_index == SIW_INVAL_UOBJ_KEY || qp 471 drivers/infiniband/sw/siw/siw_verbs.c qp->xa_rq_index == SIW_INVAL_UOBJ_KEY) { qp 475 drivers/infiniband/sw/siw/siw_verbs.c uresp.sq_key = qp->xa_sq_index << PAGE_SHIFT; qp 476 drivers/infiniband/sw/siw/siw_verbs.c uresp.rq_key = qp->xa_rq_index << PAGE_SHIFT; qp 486 drivers/infiniband/sw/siw/siw_verbs.c qp->tx_cpu = siw_get_tx_cpu(sdev); qp 487 drivers/infiniband/sw/siw/siw_verbs.c if (qp->tx_cpu < 0) { qp 491 drivers/infiniband/sw/siw/siw_verbs.c INIT_LIST_HEAD(&qp->devq); qp 493 drivers/infiniband/sw/siw/siw_verbs.c list_add_tail(&qp->devq, &sdev->qp_list); qp 496 drivers/infiniband/sw/siw/siw_verbs.c return qp->ib_qp; qp 499 drivers/infiniband/sw/siw/siw_verbs.c xa_erase(&sdev->qp_xa, qp_id(qp)); qp 503 drivers/infiniband/sw/siw/siw_verbs.c if (qp) { qp 504 drivers/infiniband/sw/siw/siw_verbs.c if (qp->xa_sq_index != SIW_INVAL_UOBJ_KEY) qp 505 drivers/infiniband/sw/siw/siw_verbs.c kfree(xa_erase(&uctx->xa, qp->xa_sq_index)); qp 506 drivers/infiniband/sw/siw/siw_verbs.c if (qp->xa_rq_index != SIW_INVAL_UOBJ_KEY) qp 507 drivers/infiniband/sw/siw/siw_verbs.c kfree(xa_erase(&uctx->xa, qp->xa_rq_index)); qp 509 drivers/infiniband/sw/siw/siw_verbs.c vfree(qp->sendq); qp 510 drivers/infiniband/sw/siw/siw_verbs.c vfree(qp->recvq); qp 511 drivers/infiniband/sw/siw/siw_verbs.c kfree(qp); qp 526 drivers/infiniband/sw/siw/siw_verbs.c struct siw_qp *qp; qp 530 drivers/infiniband/sw/siw/siw_verbs.c qp = to_siw_qp(base_qp); qp 536 drivers/infiniband/sw/siw/siw_verbs.c qp_attr->cap.max_send_wr = qp->attrs.sq_size; qp 537 drivers/infiniband/sw/siw/siw_verbs.c qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges; qp 538 drivers/infiniband/sw/siw/siw_verbs.c qp_attr->cap.max_recv_wr = qp->attrs.rq_size; qp 539 drivers/infiniband/sw/siw/siw_verbs.c qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges; qp 541 drivers/infiniband/sw/siw/siw_verbs.c qp_attr->max_rd_atomic = qp->attrs.irq_size; qp 542 drivers/infiniband/sw/siw/siw_verbs.c qp_attr->max_dest_rd_atomic = qp->attrs.orq_size; qp 563 drivers/infiniband/sw/siw/siw_verbs.c struct siw_qp *qp = to_siw_qp(base_qp); qp 582 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "desired IB QP state: %s\n", qp 588 drivers/infiniband/sw/siw/siw_verbs.c qp->tx_ctx.tx_suspend = 1; qp 595 drivers/infiniband/sw/siw/siw_verbs.c down_write(&qp->state_lock); qp 597 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask); qp 599 drivers/infiniband/sw/siw/siw_verbs.c up_write(&qp->state_lock); qp 606 drivers/infiniband/sw/siw/siw_verbs.c struct siw_qp *qp = to_siw_qp(base_qp); qp 612 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "state %d\n", qp->attrs.state); qp 618 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.flags |= SIW_QP_IN_DESTROY; qp 619 drivers/infiniband/sw/siw/siw_verbs.c qp->rx_stream.rx_suspend = 1; qp 621 drivers/infiniband/sw/siw/siw_verbs.c if (uctx && qp->xa_sq_index != SIW_INVAL_UOBJ_KEY) qp 622 drivers/infiniband/sw/siw/siw_verbs.c kfree(xa_erase(&uctx->xa, qp->xa_sq_index)); qp 623 drivers/infiniband/sw/siw/siw_verbs.c if (uctx && qp->xa_rq_index != SIW_INVAL_UOBJ_KEY) qp 624 drivers/infiniband/sw/siw/siw_verbs.c kfree(xa_erase(&uctx->xa, qp->xa_rq_index)); qp 626 drivers/infiniband/sw/siw/siw_verbs.c down_write(&qp->state_lock); qp 629 drivers/infiniband/sw/siw/siw_verbs.c siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE); qp 631 drivers/infiniband/sw/siw/siw_verbs.c if (qp->cep) { qp 632 drivers/infiniband/sw/siw/siw_verbs.c siw_cep_put(qp->cep); qp 633 drivers/infiniband/sw/siw/siw_verbs.c qp->cep = NULL; qp 635 drivers/infiniband/sw/siw/siw_verbs.c up_write(&qp->state_lock); qp 637 drivers/infiniband/sw/siw/siw_verbs.c kfree(qp->tx_ctx.mpa_crc_hd); qp 638 drivers/infiniband/sw/siw/siw_verbs.c kfree(qp->rx_stream.mpa_crc_hd); qp 640 drivers/infiniband/sw/siw/siw_verbs.c qp->scq = qp->rcq = NULL; qp 642 drivers/infiniband/sw/siw/siw_verbs.c siw_qp_put(qp); qp 689 drivers/infiniband/sw/siw/siw_verbs.c static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr, qp 698 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR); qp 710 drivers/infiniband/sw/siw/siw_verbs.c static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr, qp 718 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR); qp 741 drivers/infiniband/sw/siw/siw_verbs.c struct siw_qp *qp = to_siw_qp(base_qp); qp 742 drivers/infiniband/sw/siw/siw_verbs.c struct siw_wqe *wqe = tx_wqe(qp); qp 747 drivers/infiniband/sw/siw/siw_verbs.c if (wr && !qp->kernel_verbs) { qp 748 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "wr must be empty for user mapped sq\n"); qp 757 drivers/infiniband/sw/siw/siw_verbs.c if (!down_read_trylock(&qp->state_lock)) { qp 758 drivers/infiniband/sw/siw/siw_verbs.c if (qp->attrs.state == SIW_QP_STATE_ERROR) { qp 768 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_sq_flush_wr(qp, wr, bad_wr); qp 770 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "QP locked, state %d\n", qp 771 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.state); qp 777 drivers/infiniband/sw/siw/siw_verbs.c if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) { qp 778 drivers/infiniband/sw/siw/siw_verbs.c if (qp->attrs.state == SIW_QP_STATE_ERROR) { qp 786 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_sq_flush_wr(qp, wr, bad_wr); qp 788 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "QP out of state %d\n", qp 789 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.state); qp 793 drivers/infiniband/sw/siw/siw_verbs.c up_read(&qp->state_lock); qp 796 drivers/infiniband/sw/siw/siw_verbs.c spin_lock_irqsave(&qp->sq_lock, flags); qp 799 drivers/infiniband/sw/siw/siw_verbs.c u32 idx = qp->sq_put % qp->attrs.sq_size; qp 800 drivers/infiniband/sw/siw/siw_verbs.c struct siw_sqe *sqe = &qp->sendq[idx]; qp 803 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "sq full\n"); qp 807 drivers/infiniband/sw/siw/siw_verbs.c if (wr->num_sge > qp->attrs.sq_max_sges) { qp 808 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge); qp 815 drivers/infiniband/sw/siw/siw_verbs.c (qp->attrs.flags & SIW_SIGNAL_ALL_WR)) qp 907 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "ib wr type %d unsupported\n", qp 912 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n", qp 923 drivers/infiniband/sw/siw/siw_verbs.c qp->sq_put++; qp 935 drivers/infiniband/sw/siw/siw_verbs.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 938 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_activate_tx(qp); qp 939 drivers/infiniband/sw/siw/siw_verbs.c spin_unlock_irqrestore(&qp->sq_lock, flags); qp 944 drivers/infiniband/sw/siw/siw_verbs.c if (qp->kernel_verbs) { qp 945 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_sq_start(qp); qp 947 drivers/infiniband/sw/siw/siw_verbs.c qp->tx_ctx.in_syscall = 1; qp 949 drivers/infiniband/sw/siw/siw_verbs.c if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend)) qp 950 drivers/infiniband/sw/siw/siw_verbs.c siw_qp_cm_drop(qp, 0); qp 952 drivers/infiniband/sw/siw/siw_verbs.c qp->tx_ctx.in_syscall = 0; qp 956 drivers/infiniband/sw/siw/siw_verbs.c up_read(&qp->state_lock); qp 963 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "error %d\n", rv); qp 981 drivers/infiniband/sw/siw/siw_verbs.c struct siw_qp *qp = to_siw_qp(base_qp); qp 985 drivers/infiniband/sw/siw/siw_verbs.c if (qp->srq) { qp 989 drivers/infiniband/sw/siw/siw_verbs.c if (!qp->kernel_verbs) { qp 990 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "no kernel post_recv for user mapped sq\n"); qp 999 drivers/infiniband/sw/siw/siw_verbs.c if (!down_read_trylock(&qp->state_lock)) { qp 1000 drivers/infiniband/sw/siw/siw_verbs.c if (qp->attrs.state == SIW_QP_STATE_ERROR) { qp 1010 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_rq_flush_wr(qp, wr, bad_wr); qp 1012 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "QP locked, state %d\n", qp 1013 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.state); qp 1019 drivers/infiniband/sw/siw/siw_verbs.c if (qp->attrs.state > SIW_QP_STATE_RTS) { qp 1020 drivers/infiniband/sw/siw/siw_verbs.c if (qp->attrs.state == SIW_QP_STATE_ERROR) { qp 1028 drivers/infiniband/sw/siw/siw_verbs.c rv = siw_rq_flush_wr(qp, wr, bad_wr); qp 1030 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "QP out of state %d\n", qp 1031 drivers/infiniband/sw/siw/siw_verbs.c qp->attrs.state); qp 1035 drivers/infiniband/sw/siw/siw_verbs.c up_read(&qp->state_lock); qp 1042 drivers/infiniband/sw/siw/siw_verbs.c spin_lock_irqsave(&qp->rq_lock, flags); qp 1045 drivers/infiniband/sw/siw/siw_verbs.c u32 idx = qp->rq_put % qp->attrs.rq_size; qp 1046 drivers/infiniband/sw/siw/siw_verbs.c struct siw_rqe *rqe = &qp->recvq[idx]; qp 1049 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "RQ full\n"); qp 1053 drivers/infiniband/sw/siw/siw_verbs.c if (wr->num_sge > qp->attrs.rq_max_sges) { qp 1054 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge); qp 1067 drivers/infiniband/sw/siw/siw_verbs.c qp->rq_put++; qp 1070 drivers/infiniband/sw/siw/siw_verbs.c spin_unlock_irqrestore(&qp->rq_lock, flags); qp 1072 drivers/infiniband/sw/siw/siw_verbs.c up_read(&qp->state_lock); qp 1075 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "error %d\n", rv); qp 1800 drivers/infiniband/sw/siw/siw_verbs.c void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype) qp 1803 drivers/infiniband/sw/siw/siw_verbs.c struct ib_qp *base_qp = qp->ib_qp; qp 1809 drivers/infiniband/sw/siw/siw_verbs.c if (qp->attrs.flags & SIW_QP_IN_DESTROY) qp 1814 drivers/infiniband/sw/siw/siw_verbs.c event.element.qp = base_qp; qp 1817 drivers/infiniband/sw/siw/siw_verbs.c siw_dbg_qp(qp, "reporting event %d\n", etype); qp 86 drivers/infiniband/sw/siw/siw_verbs.h void siw_qp_event(struct siw_qp *qp, enum ib_event_type type); qp 236 drivers/infiniband/ulp/ipoib/ipoib.h struct ib_qp *qp; qp 247 drivers/infiniband/ulp/ipoib/ipoib.h struct ib_qp *qp; qp 367 drivers/infiniband/ulp/ipoib/ipoib.h struct ib_qp *qp; qp 129 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_post_recv(rx->qp, wr, NULL); qp 227 drivers/infiniband/ulp/ipoib/ipoib_cm.c if (ib_post_send(p->qp, &ipoib_cm_rx_drain_wr, NULL)) qp 274 drivers/infiniband/ulp/ipoib/ipoib_cm.c struct ib_cm_id *cm_id, struct ib_qp *qp, qp 287 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); qp 299 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); qp 319 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); qp 421 drivers/infiniband/ulp/ipoib/ipoib_cm.c struct ib_qp *qp, qp 429 drivers/infiniband/ulp/ipoib/ipoib_cm.c data.qpn = cpu_to_be32(priv->qp->qp_num); qp 437 drivers/infiniband/ulp/ipoib/ipoib_cm.c rep.qp_num = qp->qp_num; qp 462 drivers/infiniband/ulp/ipoib/ipoib_cm.c p->qp = ipoib_cm_create_rx_qp(dev, p); qp 463 drivers/infiniband/ulp/ipoib/ipoib_cm.c if (IS_ERR(p->qp)) { qp 464 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = PTR_ERR(p->qp); qp 469 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ipoib_cm_modify_rx_qp(dev, cm_id, p->qp, psn); qp 489 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ipoib_cm_send_rep(dev, cm_id, p->qp, &event->param.req_rcvd, psn); qp 492 drivers/infiniband/ulp/ipoib/ipoib_cm.c if (ib_modify_qp(p->qp, &ipoib_cm_err_attr, IB_QP_STATE)) qp 498 drivers/infiniband/ulp/ipoib/ipoib_cm.c ib_destroy_qp(p->qp); qp 519 drivers/infiniband/ulp/ipoib/ipoib_cm.c if (ib_modify_qp(p->qp, &ipoib_cm_err_attr, IB_QP_STATE)) qp 589 drivers/infiniband/ulp/ipoib/ipoib_cm.c p = wc->qp->qp_context; qp 705 drivers/infiniband/ulp/ipoib/ipoib_cm.c return ib_post_send(tx->qp, &priv->tx_wr.wr, NULL); qp 741 drivers/infiniband/ulp/ipoib/ipoib_cm.c tx->tx_head, skb->len, tx->qp->qp_num); qp 762 drivers/infiniband/ulp/ipoib/ipoib_cm.c tx->qp->qp_num); qp 797 drivers/infiniband/ulp/ipoib/ipoib_cm.c struct ipoib_cm_tx *tx = wc->qp->qp_context; qp 887 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_cm_listen(priv->cm.id, cpu_to_be64(IPOIB_CM_IETF_ID | priv->qp->qp_num), qp 891 drivers/infiniband/ulp/ipoib/ipoib_cm.c IPOIB_CM_IETF_ID | priv->qp->qp_num); qp 916 drivers/infiniband/ulp/ipoib/ipoib_cm.c ib_destroy_qp(rx->qp); qp 946 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(p->qp, &ipoib_cm_err_attr, IB_QP_STATE); qp 1012 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(p->qp, &qp_attr, qp_attr_mask); qp 1024 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(p->qp, &qp_attr, qp_attr_mask); qp 1083 drivers/infiniband/ulp/ipoib/ipoib_cm.c struct ib_cm_id *id, struct ib_qp *qp, qp 1091 drivers/infiniband/ulp/ipoib/ipoib_cm.c data.qpn = cpu_to_be32(priv->qp->qp_num); qp 1097 drivers/infiniband/ulp/ipoib/ipoib_cm.c req.qp_num = qp->qp_num; qp 1098 drivers/infiniband/ulp/ipoib/ipoib_cm.c req.qp_type = qp->qp_type; qp 1120 drivers/infiniband/ulp/ipoib/ipoib_cm.c struct ib_cm_id *cm_id, struct ib_qp *qp) qp 1136 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); qp 1159 drivers/infiniband/ulp/ipoib/ipoib_cm.c p->qp = ipoib_cm_create_tx_qp(p->dev, p); qp 1161 drivers/infiniband/ulp/ipoib/ipoib_cm.c if (IS_ERR(p->qp)) { qp 1162 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = PTR_ERR(p->qp); qp 1174 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ipoib_cm_modify_tx_init(p->dev, p->id, p->qp); qp 1180 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ipoib_cm_send_req(p->dev, p->id, p->qp, qpn, pathrec); qp 1187 drivers/infiniband/ulp/ipoib/ipoib_cm.c p->qp->qp_num, pathrec->dgid.raw, qpn); qp 1195 drivers/infiniband/ulp/ipoib/ipoib_cm.c ib_destroy_qp(p->qp); qp 1197 drivers/infiniband/ulp/ipoib/ipoib_cm.c p->qp = NULL; qp 1210 drivers/infiniband/ulp/ipoib/ipoib_cm.c p->qp ? p->qp->qp_num : 0, p->tx_head, p->tx_tail); qp 1246 drivers/infiniband/ulp/ipoib/ipoib_cm.c if (p->qp) qp 1247 drivers/infiniband/ulp/ipoib/ipoib_cm.c ib_destroy_qp(p->qp); qp 1498 drivers/infiniband/ulp/ipoib/ipoib_cm.c ret = ib_modify_qp(p->qp, &ipoib_cm_err_attr, IB_QP_STATE); qp 111 drivers/infiniband/ulp/ipoib/ipoib_ib.c ret = ib_post_recv(priv->qp, &priv->rx_wr, NULL); qp 239 drivers/infiniband/ulp/ipoib/ipoib_ib.c if (wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num) { qp 355 drivers/infiniband/ulp/ipoib/ipoib_ib.c ret = ib_query_qp(priv->qp, &qp_attr, IB_QP_STATE, &query_init_attr); qp 362 drivers/infiniband/ulp/ipoib/ipoib_ib.c __func__, priv->qp->qp_num, qp_attr.qp_state); qp 368 drivers/infiniband/ulp/ipoib/ipoib_ib.c ret = ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE); qp 371 drivers/infiniband/ulp/ipoib/ipoib_ib.c ret, priv->qp->qp_num); qp 375 drivers/infiniband/ulp/ipoib/ipoib_ib.c __func__, priv->qp->qp_num); qp 378 drivers/infiniband/ulp/ipoib/ipoib_ib.c priv->qp->qp_num, qp_attr.qp_state); qp 563 drivers/infiniband/ulp/ipoib/ipoib_ib.c return ib_post_send(priv->qp, &priv->tx_wr.wr, NULL); qp 737 drivers/infiniband/ulp/ipoib/ipoib_ib.c struct ib_qp *qp, qp 744 drivers/infiniband/ulp/ipoib/ipoib_ib.c ret = ib_query_qp(qp, &qp_attr, IB_QP_STATE, &query_init_attr); qp 791 drivers/infiniband/ulp/ipoib/ipoib_ib.c if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE)) qp 792 drivers/infiniband/ulp/ipoib/ipoib_ib.c check_qp_movement_and_print(priv, priv->qp, IB_QPS_ERR); qp 841 drivers/infiniband/ulp/ipoib/ipoib_ib.c if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE)) qp 1719 drivers/infiniband/ulp/ipoib/ipoib_main.c priv->dev->dev_addr[1] = (priv->qp->qp_num >> 16) & 0xff; qp 1720 drivers/infiniband/ulp/ipoib/ipoib_main.c priv->dev->dev_addr[2] = (priv->qp->qp_num >> 8) & 0xff; qp 1721 drivers/infiniband/ulp/ipoib/ipoib_main.c priv->dev->dev_addr[3] = (priv->qp->qp_num) & 0xff; qp 1752 drivers/infiniband/ulp/ipoib/ipoib_main.c priv->qp = NULL; qp 61 drivers/infiniband/ulp/ipoib/ipoib_verbs.c ret = ib_modify_qp(priv->qp, qp_attr, IB_QP_QKEY); qp 69 drivers/infiniband/ulp/ipoib/ipoib_verbs.c ret = ib_attach_mcast(priv->qp, mgid, mlid); qp 84 drivers/infiniband/ulp/ipoib/ipoib_verbs.c ret = ib_detach_mcast(priv->qp, mgid, mlid); qp 108 drivers/infiniband/ulp/ipoib/ipoib_verbs.c ret = ib_modify_qp(priv->qp, &qp_attr, attr_mask); qp 117 drivers/infiniband/ulp/ipoib/ipoib_verbs.c ret = ib_modify_qp(priv->qp, &qp_attr, attr_mask); qp 127 drivers/infiniband/ulp/ipoib/ipoib_verbs.c ret = ib_modify_qp(priv->qp, &qp_attr, attr_mask); qp 137 drivers/infiniband/ulp/ipoib/ipoib_verbs.c if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE)) qp 209 drivers/infiniband/ulp/ipoib/ipoib_verbs.c priv->qp = ib_create_qp(priv->pd, &init_attr); qp 210 drivers/infiniband/ulp/ipoib/ipoib_verbs.c if (IS_ERR(priv->qp)) { qp 256 drivers/infiniband/ulp/ipoib/ipoib_verbs.c if (priv->qp) { qp 257 drivers/infiniband/ulp/ipoib/ipoib_verbs.c if (ib_destroy_qp(priv->qp)) qp 260 drivers/infiniband/ulp/ipoib/ipoib_verbs.c priv->qp = NULL; qp 433 drivers/infiniband/ulp/iser/iscsi_iser.h struct ib_qp *qp; qp 560 drivers/infiniband/ulp/iser/iser_initiator.c struct ib_conn *ib_conn = wc->qp->qp_context; qp 652 drivers/infiniband/ulp/iser/iser_initiator.c struct ib_conn *ib_conn = wc->qp->qp_context; qp 727 drivers/infiniband/ulp/iser/iser_initiator.c struct ib_conn *ib_conn = wc->qp->qp_context; qp 424 drivers/infiniband/ulp/iser/iser_verbs.c ib_conn->qp = ib_conn->cma_id->qp; qp 427 drivers/infiniband/ulp/iser/iser_verbs.c ib_conn->cma_id->qp); qp 542 drivers/infiniband/ulp/iser/iser_verbs.c iser_conn, ib_conn->cma_id, ib_conn->qp); qp 544 drivers/infiniband/ulp/iser/iser_verbs.c if (ib_conn->qp != NULL) { qp 549 drivers/infiniband/ulp/iser/iser_verbs.c ib_conn->qp = NULL; qp 629 drivers/infiniband/ulp/iser/iser_verbs.c ib_drain_sq(ib_conn->qp); qp 791 drivers/infiniband/ulp/iser/iser_verbs.c (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr); qp 792 drivers/infiniband/ulp/iser/iser_verbs.c iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num); qp 990 drivers/infiniband/ulp/iser/iser_verbs.c ib_ret = ib_post_recv(ib_conn->qp, &wr, NULL); qp 1021 drivers/infiniband/ulp/iser/iser_verbs.c ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, NULL); qp 1062 drivers/infiniband/ulp/iser/iser_verbs.c ib_ret = ib_post_send(ib_conn->qp, first_wr, NULL); qp 1120 drivers/infiniband/ulp/iser/iser_verbs.c struct iser_conn *iser_conn = to_iser_conn(wc->qp->qp_context); qp 144 drivers/infiniband/ulp/isert/ib_isert.c return cma_id->qp; qp 154 drivers/infiniband/ulp/isert/ib_isert.c isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); qp 155 drivers/infiniband/ulp/isert/ib_isert.c if (IS_ERR(isert_conn->qp)) { qp 156 drivers/infiniband/ulp/isert/ib_isert.c ret = PTR_ERR(isert_conn->qp); qp 574 drivers/infiniband/ulp/isert/ib_isert.c if (isert_conn->qp) { qp 575 drivers/infiniband/ulp/isert/ib_isert.c struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; qp 578 drivers/infiniband/ulp/isert/ib_isert.c ib_destroy_qp(isert_conn->qp); qp 595 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = cma_id->qp->qp_context; qp 709 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = cma_id->qp->qp_context; qp 717 drivers/infiniband/ulp/isert/ib_isert.c ib_drain_qp(isert_conn->qp); qp 736 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = cma_id->qp->qp_context; qp 738 drivers/infiniband/ulp/isert/ib_isert.c ib_drain_qp(isert_conn->qp); qp 775 drivers/infiniband/ulp/isert/ib_isert.c isert_conn = cma_id->qp->qp_context; qp 821 drivers/infiniband/ulp/isert/ib_isert.c ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); qp 848 drivers/infiniband/ulp/isert/ib_isert.c ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); qp 874 drivers/infiniband/ulp/isert/ib_isert.c ret = ib_post_send(isert_conn->qp, &send_wr, NULL); qp 981 drivers/infiniband/ulp/isert/ib_isert.c ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); qp 1398 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = wc->qp->qp_context; qp 1457 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = wc->qp->qp_context; qp 1495 drivers/infiniband/ulp/isert/ib_isert.c rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, qp 1500 drivers/infiniband/ulp/isert/ib_isert.c rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, qp 1655 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = wc->qp->qp_context; qp 1697 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = wc->qp->qp_context; qp 1773 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = wc->qp->qp_context; qp 1789 drivers/infiniband/ulp/isert/ib_isert.c struct isert_conn *isert_conn = wc->qp->qp_context; qp 1832 drivers/infiniband/ulp/isert/ib_isert.c ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); qp 2149 drivers/infiniband/ulp/isert/ib_isert.c ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, qp 2154 drivers/infiniband/ulp/isert/ib_isert.c ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, qp 2167 drivers/infiniband/ulp/isert/ib_isert.c ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); qp 2634 drivers/infiniband/ulp/isert/ib_isert.c ib_drain_qp(isert_conn->qp); qp 2646 drivers/infiniband/ulp/isert/ib_isert.c ib_drain_qp(isert_conn->qp); qp 157 drivers/infiniband/ulp/isert/ib_isert.h struct ib_qp *qp; qp 636 drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc, qp 780 drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c ah = rdma_create_ah(port->mad_agent->qp->pd, &ah_attr, 0); qp 281 drivers/infiniband/ulp/srp/ib_srp.c struct ib_qp *qp) qp 302 drivers/infiniband/ulp/srp/ib_srp.c ret = ib_modify_qp(qp, attr, qp 547 drivers/infiniband/ulp/srp/ib_srp.c ib_drain_qp(ch->qp); qp 548 drivers/infiniband/ulp/srp/ib_srp.c ib_destroy_qp(ch->qp); qp 557 drivers/infiniband/ulp/srp/ib_srp.c struct ib_qp *qp; qp 594 drivers/infiniband/ulp/srp/ib_srp.c qp = ch->rdma_cm.cm_id->qp; qp 596 drivers/infiniband/ulp/srp/ib_srp.c qp = ib_create_qp(dev->pd, init_attr); qp 597 drivers/infiniband/ulp/srp/ib_srp.c if (!IS_ERR(qp)) { qp 598 drivers/infiniband/ulp/srp/ib_srp.c ret = srp_init_ib_qp(target, qp); qp 600 drivers/infiniband/ulp/srp/ib_srp.c ib_destroy_qp(qp); qp 602 drivers/infiniband/ulp/srp/ib_srp.c ret = PTR_ERR(qp); qp 629 drivers/infiniband/ulp/srp/ib_srp.c if (ch->qp) qp 636 drivers/infiniband/ulp/srp/ib_srp.c ch->qp = qp; qp 657 drivers/infiniband/ulp/srp/ib_srp.c ib_destroy_qp(qp); qp 696 drivers/infiniband/ulp/srp/ib_srp.c if (!ch->qp) qp 719 drivers/infiniband/ulp/srp/ib_srp.c ch->qp = NULL; qp 905 drivers/infiniband/ulp/srp/ib_srp.c req->ib_param.qp_num = ch->qp->qp_num; qp 906 drivers/infiniband/ulp/srp/ib_srp.c req->ib_param.qp_type = ch->qp->qp_type; qp 1239 drivers/infiniband/ulp/srp/ib_srp.c return ib_post_send(ch->qp, &wr, NULL); qp 1590 drivers/infiniband/ulp/srp/ib_srp.c err = ib_post_send(ch->qp, &wr.wr, NULL); qp 2092 drivers/infiniband/ulp/srp/ib_srp.c return ib_post_send(ch->qp, &wr, NULL); qp 2112 drivers/infiniband/ulp/srp/ib_srp.c return ib_post_recv(ch->qp, &wr, NULL); qp 2147 drivers/infiniband/ulp/srp/ib_srp.c rsp->tag, ch - target->ch, ch->qp->qp_num); qp 2592 drivers/infiniband/ulp/srp/ib_srp.c ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); qp 2603 drivers/infiniband/ulp/srp/ib_srp.c ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); qp 157 drivers/infiniband/ulp/srp/ib_srp.h struct ib_qp *qp; qp 232 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num, qp 483 drivers/infiniband/ulp/srpt/ib_srpt.c ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc, qp 841 drivers/infiniband/ulp/srpt/ib_srpt.c return ib_post_recv(ch->qp, &wr, NULL); qp 865 drivers/infiniband/ulp/srpt/ib_srpt.c ch->qp->qp_num); qp 867 drivers/infiniband/ulp/srpt/ib_srpt.c return ib_post_send(ch->qp, &wr.wr, NULL); qp 874 drivers/infiniband/ulp/srpt/ib_srpt.c pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num, qp 884 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num); qp 918 drivers/infiniband/ulp/srpt/ib_srpt.c ret = rdma_rw_ctx_init(&ctx->rw, ch->qp, ch->sport->port, qp 947 drivers/infiniband/ulp/srpt/ib_srpt.c rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port, qp 965 drivers/infiniband/ulp/srpt/ib_srpt.c rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port, qp 1110 drivers/infiniband/ulp/srpt/ib_srpt.c static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp) qp 1131 drivers/infiniband/ulp/srpt/ib_srpt.c ret = ib_modify_qp(qp, attr, qp 1150 drivers/infiniband/ulp/srpt/ib_srpt.c static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp) qp 1165 drivers/infiniband/ulp/srpt/ib_srpt.c ret = ib_modify_qp(qp, &qp_attr, attr_mask); qp 1182 drivers/infiniband/ulp/srpt/ib_srpt.c static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp) qp 1195 drivers/infiniband/ulp/srpt/ib_srpt.c ret = ib_modify_qp(qp, &qp_attr, attr_mask); qp 1210 drivers/infiniband/ulp/srpt/ib_srpt.c return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE); qp 1834 drivers/infiniband/ulp/srpt/ib_srpt.c ch->qp = ch->rdma_cm.cm_id->qp; qp 1836 drivers/infiniband/ulp/srpt/ib_srpt.c ch->qp = ib_create_qp(sdev->pd, qp_init); qp 1837 drivers/infiniband/ulp/srpt/ib_srpt.c if (!IS_ERR(ch->qp)) { qp 1838 drivers/infiniband/ulp/srpt/ib_srpt.c ret = srpt_init_ch_qp(ch, ch->qp); qp 1840 drivers/infiniband/ulp/srpt/ib_srpt.c ib_destroy_qp(ch->qp); qp 1842 drivers/infiniband/ulp/srpt/ib_srpt.c ret = PTR_ERR(ch->qp); qp 1876 drivers/infiniband/ulp/srpt/ib_srpt.c ch->qp = NULL; qp 1883 drivers/infiniband/ulp/srpt/ib_srpt.c ib_destroy_qp(ch->qp); qp 1911 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num, ret); qp 1916 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num, ret); qp 1984 drivers/infiniband/ulp/srpt/ib_srpt.c pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num, qp 1994 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num, ch->state); qp 2095 drivers/infiniband/ulp/srpt/ib_srpt.c pr_debug("%s-%d\n", ch->sess_name, ch->qp->qp_num); qp 2381 drivers/infiniband/ulp/srpt/ib_srpt.c ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rtr(ch, ch->qp); qp 2413 drivers/infiniband/ulp/srpt/ib_srpt.c rep_param->ib_cm.qp_num = ch->qp->qp_num; qp 2574 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num, reason, private_data_len ? qp 2590 drivers/infiniband/ulp/srpt/ib_srpt.c ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rts(ch, ch->qp); qp 2593 drivers/infiniband/ulp/srpt/ib_srpt.c ch->qp->qp_num); qp 2605 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num); qp 2652 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num); qp 2657 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num); qp 2662 drivers/infiniband/ulp/srpt/ib_srpt.c ch->qp->qp_num); qp 2707 drivers/infiniband/ulp/srpt/ib_srpt.c ch->qp->qp_num); qp 2754 drivers/infiniband/ulp/srpt/ib_srpt.c first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, ch->sport->port, qp 2759 drivers/infiniband/ulp/srpt/ib_srpt.c ret = ib_post_send(ch->qp, first_wr, NULL); qp 2830 drivers/infiniband/ulp/srpt/ib_srpt.c first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, qp 2870 drivers/infiniband/ulp/srpt/ib_srpt.c ret = ib_post_send(ch->qp, first_wr, NULL); qp 2970 drivers/infiniband/ulp/srpt/ib_srpt.c ch->sess_name, ch->qp->qp_num, qp 296 drivers/infiniband/ulp/srpt/ib_srpt.h struct ib_qp *qp; qp 341 drivers/interconnect/qcom/qcs404.c struct qcom_icc_provider *qp; qp 354 drivers/interconnect/qcom/qcs404.c qp = to_qcom_provider(provider); qp 395 drivers/interconnect/qcom/qcs404.c for (i = 0; i < qp->num_clks; i++) { qp 396 drivers/interconnect/qcom/qcs404.c ret = clk_set_rate(qp->bus_clks[i].clk, rate); qp 399 drivers/interconnect/qcom/qcs404.c qp->bus_clks[i].id, ret); qp 416 drivers/interconnect/qcom/qcs404.c struct qcom_icc_provider *qp; qp 432 drivers/interconnect/qcom/qcs404.c qp = devm_kzalloc(dev, sizeof(*qp), GFP_KERNEL); qp 433 drivers/interconnect/qcom/qcs404.c if (!qp) qp 441 drivers/interconnect/qcom/qcs404.c qp->bus_clks = devm_kmemdup(dev, bus_clocks, sizeof(bus_clocks), qp 443 drivers/interconnect/qcom/qcs404.c if (!qp->bus_clks) qp 446 drivers/interconnect/qcom/qcs404.c qp->num_clks = ARRAY_SIZE(bus_clocks); qp 447 drivers/interconnect/qcom/qcs404.c ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); qp 451 drivers/interconnect/qcom/qcs404.c ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); qp 455 drivers/interconnect/qcom/qcs404.c provider = &qp->provider; qp 466 drivers/interconnect/qcom/qcs404.c clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); qp 493 drivers/interconnect/qcom/qcs404.c platform_set_drvdata(pdev, qp); qp 501 drivers/interconnect/qcom/qcs404.c clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); qp 509 drivers/interconnect/qcom/qcs404.c struct qcom_icc_provider *qp = platform_get_drvdata(pdev); qp 510 drivers/interconnect/qcom/qcs404.c struct icc_provider *provider = &qp->provider; qp 517 drivers/interconnect/qcom/qcs404.c clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); qp 675 drivers/interconnect/qcom/sdm845.c struct qcom_icc_provider *qp; qp 687 drivers/interconnect/qcom/sdm845.c qp = to_qcom_provider(node->provider); qp 691 drivers/interconnect/qcom/sdm845.c for (i = 0; i < qp->num_bcms; i++) { qp 692 drivers/interconnect/qcom/sdm845.c if (qp->bcms[i]->dirty) { qp 693 drivers/interconnect/qcom/sdm845.c bcm_aggregate(qp->bcms[i]); qp 694 drivers/interconnect/qcom/sdm845.c list_add_tail(&qp->bcms[i]->list, &commit_list); qp 707 drivers/interconnect/qcom/sdm845.c ret = rpmh_invalidate(qp->dev); qp 713 drivers/interconnect/qcom/sdm845.c ret = rpmh_write_batch(qp->dev, RPMH_ACTIVE_ONLY_STATE, qp 722 drivers/interconnect/qcom/sdm845.c for (i = 0; i < qp->num_bcms; i++) { qp 728 drivers/interconnect/qcom/sdm845.c if (qp->bcms[i]->vote_x[QCOM_ICC_BUCKET_WAKE] != qp 729 drivers/interconnect/qcom/sdm845.c qp->bcms[i]->vote_x[QCOM_ICC_BUCKET_SLEEP] || qp 730 drivers/interconnect/qcom/sdm845.c qp->bcms[i]->vote_y[QCOM_ICC_BUCKET_WAKE] != qp 731 drivers/interconnect/qcom/sdm845.c qp->bcms[i]->vote_y[QCOM_ICC_BUCKET_SLEEP]) { qp 732 drivers/interconnect/qcom/sdm845.c list_add_tail(&qp->bcms[i]->list, &commit_list); qp 741 drivers/interconnect/qcom/sdm845.c ret = rpmh_write_batch(qp->dev, RPMH_WAKE_ONLY_STATE, cmds, commit_idx); qp 749 drivers/interconnect/qcom/sdm845.c ret = rpmh_write_batch(qp->dev, RPMH_SLEEP_STATE, cmds, commit_idx); qp 777 drivers/interconnect/qcom/sdm845.c struct qcom_icc_provider *qp; qp 789 drivers/interconnect/qcom/sdm845.c qp = devm_kzalloc(&pdev->dev, sizeof(*qp), GFP_KERNEL); qp 790 drivers/interconnect/qcom/sdm845.c if (!qp) qp 798 drivers/interconnect/qcom/sdm845.c provider = &qp->provider; qp 807 drivers/interconnect/qcom/sdm845.c qp->dev = &pdev->dev; qp 808 drivers/interconnect/qcom/sdm845.c qp->bcms = desc->bcms; qp 809 drivers/interconnect/qcom/sdm845.c qp->num_bcms = desc->num_bcms; qp 841 drivers/interconnect/qcom/sdm845.c for (i = 0; i < qp->num_bcms; i++) qp 842 drivers/interconnect/qcom/sdm845.c qcom_icc_bcm_init(qp->bcms[i], &pdev->dev); qp 850 drivers/interconnect/qcom/sdm845.c sort(qp->bcms, qp->num_bcms, sizeof(*qp->bcms), cmp_vcd, NULL); qp 852 drivers/interconnect/qcom/sdm845.c platform_set_drvdata(pdev, qp); qp 869 drivers/interconnect/qcom/sdm845.c struct qcom_icc_provider *qp = platform_get_drvdata(pdev); qp 870 drivers/interconnect/qcom/sdm845.c struct icc_provider *provider = &qp->provider; qp 782 drivers/iommu/arm-smmu-v3.c struct arm_smmu_queue_poll *qp) qp 784 drivers/iommu/arm-smmu-v3.c qp->delay = 1; qp 785 drivers/iommu/arm-smmu-v3.c qp->spin_cnt = 0; qp 786 drivers/iommu/arm-smmu-v3.c qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); qp 787 drivers/iommu/arm-smmu-v3.c qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); qp 790 drivers/iommu/arm-smmu-v3.c static int queue_poll(struct arm_smmu_queue_poll *qp) qp 792 drivers/iommu/arm-smmu-v3.c if (ktime_compare(ktime_get(), qp->timeout) > 0) qp 795 drivers/iommu/arm-smmu-v3.c if (qp->wfe) { qp 797 drivers/iommu/arm-smmu-v3.c } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) { qp 800 drivers/iommu/arm-smmu-v3.c udelay(qp->delay); qp 801 drivers/iommu/arm-smmu-v3.c qp->delay *= 2; qp 802 drivers/iommu/arm-smmu-v3.c qp->spin_cnt = 0; qp 1159 drivers/iommu/arm-smmu-v3.c struct arm_smmu_queue_poll qp; qp 1174 drivers/iommu/arm-smmu-v3.c queue_poll_init(smmu, &qp); qp 1180 drivers/iommu/arm-smmu-v3.c ret = queue_poll(&qp); qp 1194 drivers/iommu/arm-smmu-v3.c struct arm_smmu_queue_poll qp; qp 1198 drivers/iommu/arm-smmu-v3.c queue_poll_init(smmu, &qp); qp 1204 drivers/iommu/arm-smmu-v3.c qp.wfe = false; qp 1205 drivers/iommu/arm-smmu-v3.c smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp))); qp 1217 drivers/iommu/arm-smmu-v3.c struct arm_smmu_queue_poll qp; qp 1222 drivers/iommu/arm-smmu-v3.c queue_poll_init(smmu, &qp); qp 1228 drivers/iommu/arm-smmu-v3.c ret = queue_poll(&qp); qp 173 drivers/media/pci/solo6x10/solo6x10-enc.c unsigned int qp) qp 178 drivers/media/pci/solo6x10/solo6x10-enc.c if ((ch > 31) || (qp > 3)) qp 197 drivers/media/pci/solo6x10/solo6x10-enc.c solo_dev->jpeg_qp[idx] |= (qp & 3) << ch; qp 252 drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c solo_reg_write(solo_dev, SOLO_VE_CH_QP(ch), solo_enc->qp); qp 257 drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c solo_reg_write(solo_dev, SOLO_VE_CH_QP_E(ch), solo_enc->qp); qp 1078 drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c solo_enc->qp = ctrl->val; qp 1079 drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c solo_reg_write(solo_dev, SOLO_VE_CH_QP(solo_enc->ch), solo_enc->qp); qp 1080 drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c solo_reg_write(solo_dev, SOLO_VE_CH_QP_E(solo_enc->ch), solo_enc->qp); qp 1261 drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c solo_enc->qp = SOLO_DEFAULT_QP; qp 149 drivers/media/pci/solo6x10/solo6x10.h u8 mode, gop, qp, interlaced, interval; qp 375 drivers/media/pci/solo6x10/solo6x10.h unsigned int qp); qp 130 drivers/media/pci/tw5864/tw5864-h264.c static int tw5864_h264_gen_pps_rbsp(u8 *buf, size_t size, int qp) qp 145 drivers/media/pci/tw5864/tw5864-h264.c bs_write_se(s, qp - 26); /* pic_init_qp_minus26 */ qp 146 drivers/media/pci/tw5864/tw5864-h264.c bs_write_se(s, qp - 26); /* pic_init_qs_minus26 */ qp 197 drivers/media/pci/tw5864/tw5864-h264.c void tw5864_h264_put_stream_header(u8 **buf, size_t *space_left, int qp, qp 224 drivers/media/pci/tw5864/tw5864-h264.c nal_len = tw5864_h264_gen_pps_rbsp(*buf, *space_left, qp); qp 244 drivers/media/pci/tw5864/tw5864-video.c input->reg_dsp_qp = input->qp; qp 245 drivers/media/pci/tw5864/tw5864-video.c input->reg_dsp_ref_mvp_lambda = lambda_lookup_table[input->qp]; qp 246 drivers/media/pci/tw5864/tw5864-video.c input->reg_dsp_i4x4_weight = intra4x4_lambda3[input->qp]; qp 508 drivers/media/pci/tw5864/tw5864-video.c input->qp = ctrl->val; qp 509 drivers/media/pci/tw5864/tw5864-video.c input->reg_dsp_qp = input->qp; qp 510 drivers/media/pci/tw5864/tw5864-video.c input->reg_dsp_ref_mvp_lambda = lambda_lookup_table[input->qp]; qp 511 drivers/media/pci/tw5864/tw5864-video.c input->reg_dsp_i4x4_weight = intra4x4_lambda3[input->qp]; qp 1155 drivers/media/pci/tw5864/tw5864-video.c input->qp = QP_VALUE; qp 1252 drivers/media/pci/tw5864/tw5864-video.c tw5864_h264_put_stream_header(&dst, &dst_space, input->qp, qp 120 drivers/media/pci/tw5864/tw5864.h int qp; qp 190 drivers/media/pci/tw5864/tw5864.h void tw5864_h264_put_stream_header(u8 **buf, size_t *space_left, int qp, qp 195 drivers/media/platform/vicodec/codec-fwht.c static void quantize_intra(s16 *coeff, s16 *de_coeff, u16 qp) qp 203 drivers/media/platform/vicodec/codec-fwht.c if (*coeff >= -qp && *coeff <= qp) qp 221 drivers/media/platform/vicodec/codec-fwht.c static void quantize_inter(s16 *coeff, s16 *de_coeff, u16 qp) qp 229 drivers/media/platform/vicodec/codec-fwht.c if (*coeff >= -qp && *coeff <= qp) qp 52 drivers/misc/mic/scif/scif_api.c ep->qp_info.qp = kzalloc(sizeof(*ep->qp_info.qp), GFP_KERNEL); qp 53 drivers/misc/mic/scif/scif_api.c if (!ep->qp_info.qp) qp 71 drivers/misc/mic/scif/scif_api.c kfree(ep->qp_info.qp); qp 425 drivers/misc/mic/scif/scif_api.c ep->qp_info.qp = NULL; qp 476 drivers/misc/mic/scif/scif_api.c err = scif_setup_qp_connect(ep->qp_info.qp, &ep->qp_info.qp_offset, qp 519 drivers/misc/mic/scif/scif_api.c ep->qp_info.qp, qp 684 drivers/misc/mic/scif/scif_api.c ep->qp_info.qp->magic = SCIFEP_MAGIC; qp 818 drivers/misc/mic/scif/scif_api.c cep->qp_info.qp = kzalloc(sizeof(*cep->qp_info.qp), GFP_KERNEL); qp 819 drivers/misc/mic/scif/scif_api.c if (!cep->qp_info.qp) { qp 828 drivers/misc/mic/scif/scif_api.c cep->qp_info.qp->magic = SCIFEP_MAGIC; qp 834 drivers/misc/mic/scif/scif_api.c err = scif_setup_qp_accept(cep->qp_info.qp, &cep->qp_info.qp_offset, qp 942 drivers/misc/mic/scif/scif_api.c struct scif_qp *qp = ep->qp_info.qp; qp 949 drivers/misc/mic/scif/scif_api.c write_count = scif_rb_space(&qp->outbound_q); qp 953 drivers/misc/mic/scif/scif_api.c ret = scif_rb_write(&qp->outbound_q, msg, qp 958 drivers/misc/mic/scif/scif_api.c scif_rb_commit(&qp->outbound_q); qp 983 drivers/misc/mic/scif/scif_api.c (scif_rb_space(&qp->outbound_q) >= qp 1005 drivers/misc/mic/scif/scif_api.c struct scif_qp *qp = ep->qp_info.qp; qp 1012 drivers/misc/mic/scif/scif_api.c read_count = scif_rb_count(&qp->inbound_q, remaining_len); qp 1020 drivers/misc/mic/scif/scif_api.c read_size = scif_rb_get_next(&qp->inbound_q, qp 1029 drivers/misc/mic/scif/scif_api.c scif_rb_update_read_ptr(&qp->inbound_q); qp 1068 drivers/misc/mic/scif/scif_api.c scif_rb_count(&qp->inbound_q, qp 1345 drivers/misc/mic/scif/scif_api.c if (scif_rb_count(&ep->qp_info.qp->inbound_q, 1)) qp 1348 drivers/misc/mic/scif/scif_api.c if (scif_rb_space(&ep->qp_info.qp->outbound_q)) qp 14 drivers/misc/mic/scif/scif_epd.c struct scif_qp *qp = ep->qp_info.qp; qp 16 drivers/misc/mic/scif/scif_epd.c if (qp->outbound_q.rb_base) { qp 17 drivers/misc/mic/scif/scif_epd.c scif_iounmap((void *)qp->outbound_q.rb_base, qp 18 drivers/misc/mic/scif/scif_epd.c qp->outbound_q.size, ep->remote_dev); qp 19 drivers/misc/mic/scif/scif_epd.c qp->outbound_q.rb_base = NULL; qp 21 drivers/misc/mic/scif/scif_epd.c if (qp->remote_qp) { qp 22 drivers/misc/mic/scif/scif_epd.c scif_iounmap((void *)qp->remote_qp, qp 24 drivers/misc/mic/scif/scif_epd.c qp->remote_qp = NULL; qp 26 drivers/misc/mic/scif/scif_epd.c if (qp->local_qp) { qp 27 drivers/misc/mic/scif/scif_epd.c scif_unmap_single(qp->local_qp, ep->remote_dev, qp 29 drivers/misc/mic/scif/scif_epd.c qp->local_qp = 0x0; qp 31 drivers/misc/mic/scif/scif_epd.c if (qp->local_buf) { qp 32 drivers/misc/mic/scif/scif_epd.c scif_unmap_single(qp->local_buf, ep->remote_dev, qp 34 drivers/misc/mic/scif/scif_epd.c qp->local_buf = 0; qp 41 drivers/misc/mic/scif/scif_epd.c struct scif_qp *qp = ep->qp_info.qp; qp 43 drivers/misc/mic/scif/scif_epd.c if (qp) { qp 47 drivers/misc/mic/scif/scif_epd.c kfree(qp->inbound_q.rb_base); qp 48 drivers/misc/mic/scif/scif_epd.c kfree(qp); qp 54 drivers/misc/mic/scif/scif_epd.h struct scif_qp *qp; qp 53 drivers/misc/mic/scif/scif_nm.c struct scif_qp *qp = scifdev->qpairs; qp 55 drivers/misc/mic/scif/scif_nm.c if (!qp) qp 57 drivers/misc/mic/scif/scif_nm.c scif_unmap_single(qp->local_buf, scifdev, qp->inbound_q.size); qp 58 drivers/misc/mic/scif/scif_nm.c kfree(qp->inbound_q.rb_base); qp 59 drivers/misc/mic/scif/scif_nm.c scif_unmap_single(qp->local_qp, scifdev, sizeof(struct scif_qp)); qp 66 drivers/misc/mic/scif/scif_nm.c struct scif_qp *qp = &dev->qpairs[0]; qp 68 drivers/misc/mic/scif/scif_nm.c if (!qp) qp 70 drivers/misc/mic/scif/scif_nm.c scif_iounmap((void *)qp->remote_qp, sizeof(struct scif_qp), dev); qp 71 drivers/misc/mic/scif/scif_nm.c scif_iounmap((void *)qp->outbound_q.rb_base, qp 73 drivers/misc/mic/scif/scif_nm.c qp->remote_qp = NULL; qp 74 drivers/misc/mic/scif/scif_nm.c qp->local_write = 0; qp 75 drivers/misc/mic/scif/scif_nm.c qp->inbound_q.current_write_offset = 0; qp 76 drivers/misc/mic/scif/scif_nm.c qp->inbound_q.current_read_offset = 0; qp 96 drivers/misc/mic/scif/scif_nodeqp.c int scif_setup_qp_connect(struct scif_qp *qp, dma_addr_t *qp_offset, qp 99 drivers/misc/mic/scif/scif_nodeqp.c void *local_q = qp->inbound_q.rb_base; qp 103 drivers/misc/mic/scif/scif_nodeqp.c spin_lock_init(&qp->send_lock); qp 104 drivers/misc/mic/scif/scif_nodeqp.c spin_lock_init(&qp->recv_lock); qp 115 drivers/misc/mic/scif/scif_nodeqp.c err = scif_map_single(&qp->local_buf, local_q, scifdev, local_size); qp 122 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_init(&qp->inbound_q, qp 124 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_write, qp 130 drivers/misc/mic/scif/scif_nodeqp.c qp->inbound_q.read_ptr = NULL; qp 131 drivers/misc/mic/scif/scif_nodeqp.c err = scif_map_single(qp_offset, qp, qp 135 drivers/misc/mic/scif/scif_nodeqp.c qp->local_qp = *qp_offset; qp 138 drivers/misc/mic/scif/scif_nodeqp.c scif_unmap_single(qp->local_buf, scifdev, local_size); qp 139 drivers/misc/mic/scif/scif_nodeqp.c qp->local_buf = 0; qp 146 drivers/misc/mic/scif/scif_nodeqp.c int scif_setup_qp_accept(struct scif_qp *qp, dma_addr_t *qp_offset, qp 156 drivers/misc/mic/scif/scif_nodeqp.c spin_lock_init(&qp->send_lock); qp 157 drivers/misc/mic/scif/scif_nodeqp.c spin_lock_init(&qp->recv_lock); qp 162 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp = remote_qp; qp 163 drivers/misc/mic/scif/scif_nodeqp.c if (qp->remote_qp->magic != SCIFEP_MAGIC) { qp 167 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_buf = remote_qp->local_buf; qp 168 drivers/misc/mic/scif/scif_nodeqp.c remote_size = qp->remote_qp->inbound_q.size; qp 169 drivers/misc/mic/scif/scif_nodeqp.c remote_q = scif_ioremap(qp->remote_buf, remote_size, scifdev); qp 174 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp->local_write = 0; qp 179 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_init(&qp->outbound_q, qp 180 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_read, qp 181 drivers/misc/mic/scif/scif_nodeqp.c &qp->remote_qp->local_write, qp 189 drivers/misc/mic/scif/scif_nodeqp.c err = scif_map_single(&qp->local_buf, local_q, scifdev, local_size); qp 192 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp->local_read = 0; qp 197 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_init(&qp->inbound_q, qp 198 drivers/misc/mic/scif/scif_nodeqp.c &qp->remote_qp->local_read, qp 199 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_write, qp 201 drivers/misc/mic/scif/scif_nodeqp.c err = scif_map_single(qp_offset, qp, scifdev, qp 205 drivers/misc/mic/scif/scif_nodeqp.c qp->local_qp = *qp_offset; qp 208 drivers/misc/mic/scif/scif_nodeqp.c scif_unmap_single(qp->local_buf, scifdev, local_size); qp 209 drivers/misc/mic/scif/scif_nodeqp.c qp->local_buf = 0; qp 214 drivers/misc/mic/scif/scif_nodeqp.c qp->outbound_q.rb_base = NULL; qp 216 drivers/misc/mic/scif/scif_nodeqp.c scif_iounmap(qp->remote_qp, sizeof(struct scif_qp), scifdev); qp 217 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp = NULL; qp 222 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp, u64 payload) qp 229 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp = scif_ioremap(payload, sizeof(struct scif_qp), scifdev); qp 231 drivers/misc/mic/scif/scif_nodeqp.c if (!qp->remote_qp) { qp 236 drivers/misc/mic/scif/scif_nodeqp.c if (qp->remote_qp->magic != SCIFEP_MAGIC) { qp 244 drivers/misc/mic/scif/scif_nodeqp.c tmp_phys = qp->remote_qp->local_buf; qp 245 drivers/misc/mic/scif/scif_nodeqp.c remote_size = qp->remote_qp->inbound_q.size; qp 251 drivers/misc/mic/scif/scif_nodeqp.c qp->local_read = 0; qp 252 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_init(&qp->outbound_q, qp 253 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_read, qp 254 drivers/misc/mic/scif/scif_nodeqp.c &qp->remote_qp->local_write, qp 261 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp->local_read = qp->inbound_q.current_read_offset; qp 266 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_init(&qp->inbound_q, qp 267 drivers/misc/mic/scif/scif_nodeqp.c &qp->remote_qp->local_read, qp 268 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_write, qp 269 drivers/misc/mic/scif/scif_nodeqp.c qp->inbound_q.rb_base, qp 270 drivers/misc/mic/scif/scif_nodeqp.c get_count_order(qp->inbound_q.size)); qp 327 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp; qp 331 drivers/misc/mic/scif/scif_nodeqp.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 332 drivers/misc/mic/scif/scif_nodeqp.c if (!qp) { qp 336 drivers/misc/mic/scif/scif_nodeqp.c qp->magic = SCIFEP_MAGIC; qp 337 drivers/misc/mic/scif/scif_nodeqp.c scifdev->qpairs = qp; qp 338 drivers/misc/mic/scif/scif_nodeqp.c err = scif_setup_qp_connect(qp, &scifdev->qp_dma_addr, qp 611 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp = scifdev->qpairs; qp 615 drivers/misc/mic/scif/scif_nodeqp.c if (!qp) { qp 619 drivers/misc/mic/scif/scif_nodeqp.c spin_lock(&qp->send_lock); qp 621 drivers/misc/mic/scif/scif_nodeqp.c while ((err = scif_rb_write(&qp->outbound_q, qp 631 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_commit(&qp->outbound_q); qp 632 drivers/misc/mic/scif/scif_nodeqp.c spin_unlock(&qp->send_lock); qp 852 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp = &peerdev->qpairs[0]; qp 854 drivers/misc/mic/scif/scif_nodeqp.c if (qp->qp_state != SCIF_QP_ONLINE || qp 855 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp->qp_state != SCIF_QP_ONLINE) { qp 859 drivers/misc/mic/scif/scif_nodeqp.c qp->qp_state); qp 870 drivers/misc/mic/scif/scif_nodeqp.c __func__, __LINE__, peerdev->node, qp->qp_state); qp 871 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp->qp_state = SCIF_QP_OFFLINE; qp 889 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp; qp 915 drivers/misc/mic/scif/scif_nodeqp.c qp = &peerdev->qpairs[0]; qp 921 drivers/misc/mic/scif/scif_nodeqp.c qp->remote_qp->qp_state = SCIF_QP_ONLINE; qp 1074 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp, struct scifmsg *msg) qp 1098 drivers/misc/mic/scif/scif_nodeqp.c void scif_nodeqp_intrhandler(struct scif_dev *scifdev, struct scif_qp *qp) qp 1104 drivers/misc/mic/scif/scif_nodeqp.c read_size = scif_rb_get_next(&qp->inbound_q, &msg, sizeof(msg)); qp 1107 drivers/misc/mic/scif/scif_nodeqp.c scif_nodeqp_msg_handler(scifdev, qp, &msg); qp 1114 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_update_read_ptr(&qp->inbound_q); qp 1131 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp = scifdev->qpairs; qp 1136 drivers/misc/mic/scif/scif_nodeqp.c spin_lock(&qp->recv_lock); qp 1143 drivers/misc/mic/scif/scif_nodeqp.c spin_unlock(&qp->recv_lock); qp 1146 drivers/misc/mic/scif/scif_nodeqp.c scif_nodeqp_msg_handler(scifdev, qp, &msg->msg); qp 1185 drivers/misc/mic/scif/scif_nodeqp.c scif_loopb_msg_handler(struct scif_dev *scifdev, struct scif_qp *qp) qp 1194 drivers/misc/mic/scif/scif_nodeqp.c read_size = scif_rb_get_next(&qp->inbound_q, &msg->msg, qp 1198 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_update_read_ptr(&qp->inbound_q); qp 1201 drivers/misc/mic/scif/scif_nodeqp.c spin_lock(&qp->recv_lock); qp 1203 drivers/misc/mic/scif/scif_nodeqp.c spin_unlock(&qp->recv_lock); qp 1205 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_update_read_ptr(&qp->inbound_q); qp 1220 drivers/misc/mic/scif/scif_nodeqp.c struct scif_qp *qp; qp 1242 drivers/misc/mic/scif/scif_nodeqp.c qp = scifdev->qpairs; qp 1243 drivers/misc/mic/scif/scif_nodeqp.c qp->magic = SCIFEP_MAGIC; qp 1244 drivers/misc/mic/scif/scif_nodeqp.c spin_lock_init(&qp->send_lock); qp 1245 drivers/misc/mic/scif/scif_nodeqp.c spin_lock_init(&qp->recv_lock); qp 1257 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_init(&qp->outbound_q, qp 1258 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_read, qp 1259 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_write, qp 1262 drivers/misc/mic/scif/scif_nodeqp.c scif_rb_init(&qp->inbound_q, qp 1263 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_read, qp 1264 drivers/misc/mic/scif/scif_nodeqp.c &qp->local_write, qp 186 drivers/misc/mic/scif/scif_nodeqp.h void scif_nodeqp_intrhandler(struct scif_dev *scifdev, struct scif_qp *qp); qp 187 drivers/misc/mic/scif/scif_nodeqp.h int scif_loopb_msg_handler(struct scif_dev *scifdev, struct scif_qp *qp); qp 190 drivers/misc/mic/scif/scif_nodeqp.h int scif_setup_qp_connect(struct scif_qp *qp, dma_addr_t *qp_offset, qp 192 drivers/misc/mic/scif/scif_nodeqp.h int scif_setup_qp_accept(struct scif_qp *qp, dma_addr_t *qp_offset, qp 196 drivers/misc/mic/scif/scif_nodeqp.h struct scif_qp *qp, u64 payload); qp 196 drivers/misc/vmw_vmci/vmci_queue_pair.c struct qp_entry qp; qp 214 drivers/misc/vmw_vmci/vmci_queue_pair.c struct qp_entry qp; qp 820 drivers/misc/vmw_vmci/vmci_queue_pair.c struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle); qp 822 drivers/misc/vmw_vmci/vmci_queue_pair.c entry = qp ? container_of( qp 823 drivers/misc/vmw_vmci/vmci_queue_pair.c qp, struct qp_guest_endpoint, qp) : NULL; qp 834 drivers/misc/vmw_vmci/vmci_queue_pair.c struct qp_entry *qp = qp_list_find(&qp_broker_list, handle); qp 836 drivers/misc/vmw_vmci/vmci_queue_pair.c entry = qp ? container_of( qp 837 drivers/misc/vmw_vmci/vmci_queue_pair.c qp, struct qp_broker_entry, qp) : NULL; qp 892 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.peer = peer; qp 893 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.flags = flags; qp 894 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.produce_size = produce_size; qp 895 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.consume_size = consume_size; qp 896 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.ref_count = 0; qp 900 drivers/misc/vmw_vmci/vmci_queue_pair.c INIT_LIST_HEAD(&entry->qp.list_item); qp 906 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.handle = vmci_resource_handle(&entry->resource); qp 908 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_list_find(&qp_guest_endpoints, entry->qp.handle)) { qp 925 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_free_queue(entry->produce_q, entry->qp.produce_size); qp 926 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_free_queue(entry->consume_q, entry->qp.consume_size); qp 958 drivers/misc/vmw_vmci/vmci_queue_pair.c alloc_msg->handle = entry->qp.handle; qp 959 drivers/misc/vmw_vmci/vmci_queue_pair.c alloc_msg->peer = entry->qp.peer; qp 960 drivers/misc/vmw_vmci/vmci_queue_pair.c alloc_msg->flags = entry->qp.flags; qp 961 drivers/misc/vmw_vmci/vmci_queue_pair.c alloc_msg->produce_size = entry->qp.produce_size; qp 962 drivers/misc/vmw_vmci/vmci_queue_pair.c alloc_msg->consume_size = entry->qp.consume_size; qp 1029 drivers/misc/vmw_vmci/vmci_queue_pair.c if (entry->qp.flags & VMCI_QPFLAG_LOCAL) { qp 1032 drivers/misc/vmw_vmci/vmci_queue_pair.c if (entry->qp.ref_count > 1) { qp 1064 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.ref_count--; qp 1065 drivers/misc/vmw_vmci/vmci_queue_pair.c if (entry->qp.ref_count == 0) qp 1066 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_list_remove_entry(&qp_guest_endpoints, &entry->qp); qp 1070 drivers/misc/vmw_vmci/vmci_queue_pair.c ref_count = entry->qp.ref_count; qp 1110 drivers/misc/vmw_vmci/vmci_queue_pair.c if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { qp 1112 drivers/misc/vmw_vmci/vmci_queue_pair.c if (queue_pair_entry->qp.ref_count > 1) { qp 1118 drivers/misc/vmw_vmci/vmci_queue_pair.c if (queue_pair_entry->qp.produce_size != consume_size || qp 1119 drivers/misc/vmw_vmci/vmci_queue_pair.c queue_pair_entry->qp.consume_size != qp 1121 drivers/misc/vmw_vmci/vmci_queue_pair.c queue_pair_entry->qp.flags != qp 1181 drivers/misc/vmw_vmci/vmci_queue_pair.c if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { qp 1194 drivers/misc/vmw_vmci/vmci_queue_pair.c if (queue_pair_entry->qp.handle.context != context_id || qp 1195 drivers/misc/vmw_vmci/vmci_queue_pair.c (queue_pair_entry->qp.peer != VMCI_INVALID_ID && qp 1196 drivers/misc/vmw_vmci/vmci_queue_pair.c queue_pair_entry->qp.peer != context_id)) { qp 1201 drivers/misc/vmw_vmci/vmci_queue_pair.c if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) { qp 1216 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp); qp 1219 drivers/misc/vmw_vmci/vmci_queue_pair.c queue_pair_entry->qp.ref_count++; qp 1220 drivers/misc/vmw_vmci/vmci_queue_pair.c *handle = queue_pair_entry->qp.handle; qp 1229 drivers/misc/vmw_vmci/vmci_queue_pair.c if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) && qp 1230 drivers/misc/vmw_vmci/vmci_queue_pair.c queue_pair_entry->qp.ref_count == 1) { qp 1333 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.handle = handle; qp 1334 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.peer = peer; qp 1335 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.flags = flags; qp 1336 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.produce_size = guest_produce_size; qp 1337 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.consume_size = guest_consume_size; qp 1338 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.ref_count = 1; qp 1362 drivers/misc/vmw_vmci/vmci_queue_pair.c INIT_LIST_HEAD(&entry->qp.list_item); qp 1367 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp), qp 1376 drivers/misc/vmw_vmci/vmci_queue_pair.c (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1); qp 1401 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_list_add_entry(&qp_broker_list, &entry->qp); qp 1415 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.handle = vmci_resource_handle(&entry->resource); qp 1418 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.handle); qp 1420 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.handle); qp 1423 drivers/misc/vmw_vmci/vmci_queue_pair.c vmci_ctx_qp_create(context, entry->qp.handle); qp 1523 drivers/misc/vmw_vmci/vmci_queue_pair.c if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) || qp 1556 drivers/misc/vmw_vmci/vmci_queue_pair.c if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id) qp 1585 drivers/misc/vmw_vmci/vmci_queue_pair.c if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER)) qp 1595 drivers/misc/vmw_vmci/vmci_queue_pair.c if (entry->qp.produce_size != produce_size || qp 1596 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.consume_size != consume_size) { qp 1599 drivers/misc/vmw_vmci/vmci_queue_pair.c } else if (entry->qp.produce_size != consume_size || qp 1600 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.consume_size != produce_size) { qp 1655 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_notify_peer(true, entry->qp.handle, context_id, qp 1659 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->create_id, entry->qp.handle.context, qp 1660 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.handle.resource); qp 1664 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.ref_count++; qp 1675 drivers/misc/vmw_vmci/vmci_queue_pair.c vmci_ctx_qp_create(context, entry->qp.handle); qp 2030 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->create_id, entry->qp.handle.context, qp 2031 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.handle.resource); qp 2112 drivers/misc/vmw_vmci/vmci_queue_pair.c entry->qp.ref_count--; qp 2114 drivers/misc/vmw_vmci/vmci_queue_pair.c is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL; qp 2160 drivers/misc/vmw_vmci/vmci_queue_pair.c if (entry->qp.ref_count == 0) { qp 2161 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_list_remove_entry(&qp_broker_list, &entry->qp); qp 2167 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_host_free_queue(entry->produce_q, entry->qp.produce_size); qp 2168 drivers/misc/vmw_vmci/vmci_queue_pair.c qp_host_free_queue(entry->consume_q, entry->qp.consume_size); qp 2240 drivers/misc/vmw_vmci/vmci_queue_pair.c page_store.len = QPE_NUM_PAGES(entry->qp); qp 2275 drivers/net/ethernet/broadcom/bcmsysport.c unsigned int q, qp, port; qp 2314 drivers/net/ethernet/broadcom/bcmsysport.c for (q = 0, qp = 0; q < dev->num_tx_queues && qp < num_tx_queues; qp 2324 drivers/net/ethernet/broadcom/bcmsysport.c ring->switch_queue = qp; qp 2327 drivers/net/ethernet/broadcom/bcmsysport.c priv->ring_map[qp + port * num_tx_queues] = ring; qp 2328 drivers/net/ethernet/broadcom/bcmsysport.c qp++; qp 2342 drivers/net/ethernet/broadcom/bcmsysport.c unsigned int q, qp, port; qp 2368 drivers/net/ethernet/broadcom/bcmsysport.c qp = ring->switch_queue; qp 2369 drivers/net/ethernet/broadcom/bcmsysport.c priv->ring_map[qp + port * num_tx_queues] = NULL; qp 4776 drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c adap->vres.qp.start = val[0]; qp 4777 drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c adap->vres.qp.size = val[1] - val[0] + 1; qp 290 drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h struct cxgb4_range qp; qp 5174 drivers/net/ethernet/chelsio/cxgb4/t4_hw.c __be32 *qp = &cmd.iq0_to_iq2; qp 5195 drivers/net/ethernet/chelsio/cxgb4/t4_hw.c *qp++ = cpu_to_be32(v); qp 1272 drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c __be32 *qp = &cmd.iq0_to_iq2; qp 1313 drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c *qp++ = cpu_to_be32(FW_RSS_IND_TBL_CMD_IQ0_V(qbuf[0]) | qp 858 drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c struct hinic_qp *qp = &func_to_io->qps[i]; qp 863 drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c return &qp->sq; qp 876 drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c struct hinic_qp *qp = &func_to_io->qps[i]; qp 881 drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c return &qp->rq; qp 931 drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq); qp 951 drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c hw_ci.sq_id = qp->q_id; qp 110 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c struct hinic_qp *qp; qp 126 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c qp = &func_to_io->qps[i]; qp 128 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c hinic_sq_prepare_ctxt(&sq_ctxt[i], &qp->sq, qp 129 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c base_qpn + qp->q_id); qp 154 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c struct hinic_qp *qp; qp 170 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c qp = &func_to_io->qps[i]; qp 172 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c hinic_rq_prepare_ctxt(&rq_ctxt[i], &qp->rq, qp 173 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c base_qpn + qp->q_id); qp 269 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c struct hinic_qp *qp, int q_id, qp 278 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c qp->q_id = q_id; qp 305 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c err = hinic_init_sq(&qp->sq, hwif, &func_to_io->sq_wq[q_id], qp 314 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c err = hinic_init_rq(&qp->rq, hwif, &func_to_io->rq_wq[q_id], qp 324 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c hinic_clean_sq(&qp->sq); qp 343 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c struct hinic_qp *qp) qp 345 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c int q_id = qp->q_id; qp 347 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c hinic_clean_rq(&qp->rq); qp 348 drivers/net/ethernet/huawei/hinic/hinic_hw_io.c hinic_clean_sq(&qp->sq); qp 616 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq); qp 624 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c HINIC_SQ_DB_INFO_SET(qp->q_id, QID)); qp 328 drivers/net/ethernet/huawei/hinic/hinic_rx.c struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq); qp 387 drivers/net/ethernet/huawei/hinic/hinic_rx.c skb_record_rx_queue(skb, qp->q_id); qp 480 drivers/net/ethernet/huawei/hinic/hinic_rx.c struct hinic_qp *qp; qp 496 drivers/net/ethernet/huawei/hinic/hinic_rx.c qp = container_of(rq, struct hinic_qp, rq); qp 497 drivers/net/ethernet/huawei/hinic/hinic_rx.c cpumask_set_cpu(qp->q_id % num_online_cpus(), &rq->affinity_mask); qp 521 drivers/net/ethernet/huawei/hinic/hinic_rx.c struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq); qp 532 drivers/net/ethernet/huawei/hinic/hinic_rx.c "hinic_rxq%d", qp->q_id); qp 471 drivers/net/ethernet/huawei/hinic/hinic_tx.c struct hinic_qp *qp; qp 474 drivers/net/ethernet/huawei/hinic/hinic_tx.c qp = container_of(txq->sq, struct hinic_qp, sq); qp 505 drivers/net/ethernet/huawei/hinic/hinic_tx.c netif_stop_subqueue(netdev, qp->q_id); qp 512 drivers/net/ethernet/huawei/hinic/hinic_tx.c netif_wake_subqueue(nic_dev->netdev, qp->q_id); qp 610 drivers/net/ethernet/huawei/hinic/hinic_tx.c struct hinic_qp *qp = container_of(txq->sq, struct hinic_qp, sq); qp 654 drivers/net/ethernet/huawei/hinic/hinic_tx.c if (__netif_subqueue_stopped(nic_dev->netdev, qp->q_id) && qp 656 drivers/net/ethernet/huawei/hinic/hinic_tx.c netdev_txq = netdev_get_tx_queue(txq->netdev, qp->q_id); qp 660 drivers/net/ethernet/huawei/hinic/hinic_tx.c netif_wake_subqueue(nic_dev->netdev, qp->q_id); qp 760 drivers/net/ethernet/huawei/hinic/hinic_tx.c struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq); qp 785 drivers/net/ethernet/huawei/hinic/hinic_tx.c irqname_len = snprintf(NULL, 0, "hinic_txq%d", qp->q_id) + 1; qp 792 drivers/net/ethernet/huawei/hinic/hinic_tx.c sprintf(txq->irq_name, "hinic_txq%d", qp->q_id); qp 349 drivers/net/ethernet/ibm/ehea/ehea.h struct ehea_qp *qp; qp 204 drivers/net/ethernet/ibm/ehea/ehea_hw.h static inline void ehea_update_sqa(struct ehea_qp *qp, u16 nr_wqes) qp 206 drivers/net/ethernet/ibm/ehea/ehea_hw.h struct h_epa epa = qp->epas.kernel; qp 211 drivers/net/ethernet/ibm/ehea/ehea_hw.h static inline void ehea_update_rq3a(struct ehea_qp *qp, u16 nr_wqes) qp 213 drivers/net/ethernet/ibm/ehea/ehea_hw.h struct h_epa epa = qp->epas.kernel; qp 218 drivers/net/ethernet/ibm/ehea/ehea_hw.h static inline void ehea_update_rq2a(struct ehea_qp *qp, u16 nr_wqes) qp 220 drivers/net/ethernet/ibm/ehea/ehea_hw.h struct h_epa epa = qp->epas.kernel; qp 225 drivers/net/ethernet/ibm/ehea/ehea_hw.h static inline void ehea_update_rq1a(struct ehea_qp *qp, u16 nr_wqes) qp 227 drivers/net/ethernet/ibm/ehea/ehea_hw.h struct h_epa epa = qp->epas.kernel; qp 193 drivers/net/ethernet/ibm/ehea/ehea_main.c arr[i++].fwh = pr->qp->fw_handle; qp 415 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_update_rq1a(pr->qp, adder); qp 435 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_update_rq1a(pr->qp, i - 1); qp 443 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp *qp = pr->qp; qp 485 drivers/net/ethernet/ibm/ehea/ehea_main.c rwqe = ehea_get_next_rwqe(qp, rq_nr); qp 505 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_update_rq2a(pr->qp, adder); qp 507 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_update_rq3a(pr->qp, adder); qp 639 drivers/net/ethernet/ibm/ehea/ehea_main.c pr->qp->init_attr.qp_nr); qp 654 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp *qp = pr->qp; qp 670 drivers/net/ethernet/ibm/ehea/ehea_main.c cqe = ehea_poll_rq1(qp, &wqe_index); qp 672 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_inc_rq1(qp); qp 737 drivers/net/ethernet/ibm/ehea/ehea_main.c cqe = ehea_poll_rq1(qp, &wqe_index); qp 772 drivers/net/ethernet/ibm/ehea/ehea_main.c swqe = ehea_get_swqe(pr->qp, &swqe_index); qp 782 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_post_swqe(pr->qp, swqe); qp 893 drivers/net/ethernet/ibm/ehea/ehea_main.c cqe = ehea_poll_rq1(pr->qp, &wqe_index); qp 922 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp *qp; qp 934 drivers/net/ethernet/ibm/ehea/ehea_main.c qp = port->port_res[qp_token].qp; qp 936 drivers/net/ethernet/ibm/ehea/ehea_main.c resource_type = ehea_error_data(port->adapter, qp->fw_handle, qp 1250 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp_init_attr *init_attr = &pr->qp->init_attr; qp 1364 drivers/net/ethernet/ibm/ehea/ehea_main.c port->port_res[i].qp->init_attr.qp_nr; qp 1367 drivers/net/ethernet/ibm/ehea/ehea_main.c port->port_res[0].qp->init_attr.qp_nr; qp 1512 drivers/net/ethernet/ibm/ehea/ehea_main.c pr->qp = ehea_create_qp(adapter, adapter->pd, init_attr); qp 1513 drivers/net/ethernet/ibm/ehea/ehea_main.c if (!pr->qp) { qp 1557 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_destroy_qp(pr->qp); qp 1569 drivers/net/ethernet/ibm/ehea/ehea_main.c if (pr->qp) qp 1572 drivers/net/ethernet/ibm/ehea/ehea_main.c ret = ehea_destroy_qp(pr->qp); qp 1797 drivers/net/ethernet/ibm/ehea/ehea_main.c cb7->def_uc_qpn = enable == 1 ? port->port_res[0].qp->fw_handle : 0; qp 2023 drivers/net/ethernet/ibm/ehea/ehea_main.c swqe = ehea_get_swqe(pr->qp, &swqe_index); qp 2066 drivers/net/ethernet/ibm/ehea/ehea_main.c "post swqe on QP %d\n", pr->qp->init_attr.qp_nr); qp 2075 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_post_swqe(pr->qp, swqe); qp 2161 drivers/net/ethernet/ibm/ehea/ehea_main.c static int ehea_activate_qp(struct ehea_adapter *adapter, struct ehea_qp *qp) qp 2175 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2183 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2191 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2199 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2207 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2215 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2223 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2344 drivers/net/ethernet/ibm/ehea/ehea_main.c ret = ehea_activate_qp(port->adapter, port->port_res[i].qp); qp 2473 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp qp = *orig_qp; qp 2474 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp_init_attr *init_attr = &qp.init_attr; qp 2480 drivers/net/ethernet/ibm/ehea/ehea_main.c swqe = ehea_get_swqe(&qp, &wqe_index); qp 2525 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp *qp = pr->qp; qp 2528 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_purge_sq(qp); qp 2531 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2542 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2551 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2576 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp qp = *orig_qp; qp 2577 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp_init_attr *init_attr = &qp.init_attr; qp 2589 drivers/net/ethernet/ibm/ehea/ehea_main.c rwqe = ehea_get_next_rwqe(&qp, 2); qp 2598 drivers/net/ethernet/ibm/ehea/ehea_main.c rwqe = ehea_get_next_rwqe(&qp, 3); qp 2627 drivers/net/ethernet/ibm/ehea/ehea_main.c struct ehea_qp *qp = pr->qp; qp 2635 drivers/net/ethernet/ibm/ehea/ehea_main.c ehea_update_rqs(qp, pr); qp 2638 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2649 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 2658 drivers/net/ethernet/ibm/ehea/ehea_main.c hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle, qp 357 drivers/net/ethernet/ibm/ehea/ehea_qmr.c static int ehea_qp_alloc_register(struct ehea_qp *qp, struct hw_queue *hw_queue, qp 378 drivers/net/ethernet/ibm/ehea/ehea_qmr.c qp->fw_handle, rpage, 1); qp 402 drivers/net/ethernet/ibm/ehea/ehea_qmr.c struct ehea_qp *qp; qp 407 drivers/net/ethernet/ibm/ehea/ehea_qmr.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 408 drivers/net/ethernet/ibm/ehea/ehea_qmr.c if (!qp) qp 411 drivers/net/ethernet/ibm/ehea/ehea_qmr.c qp->adapter = adapter; qp 414 drivers/net/ethernet/ibm/ehea/ehea_qmr.c &qp->fw_handle, &qp->epas); qp 425 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ret = ehea_qp_alloc_register(qp, &qp->hw_squeue, init_attr->nr_sq_pages, qp 434 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue1, qp 445 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue2, qp 457 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue3, qp 468 drivers/net/ethernet/ibm/ehea/ehea_qmr.c qp->init_attr = *init_attr; qp 470 drivers/net/ethernet/ibm/ehea/ehea_qmr.c return qp; qp 473 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hw_queue_dtor(&qp->hw_rqueue2); qp 476 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hw_queue_dtor(&qp->hw_rqueue1); qp 479 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hw_queue_dtor(&qp->hw_squeue); qp 482 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ehea_h_disable_and_get_hea(adapter->handle, qp->fw_handle); qp 483 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ehea_h_free_resource(adapter->handle, qp->fw_handle, FORCE_FREE); qp 486 drivers/net/ethernet/ibm/ehea/ehea_qmr.c kfree(qp); qp 490 drivers/net/ethernet/ibm/ehea/ehea_qmr.c static u64 ehea_destroy_qp_res(struct ehea_qp *qp, u64 force) qp 493 drivers/net/ethernet/ibm/ehea/ehea_qmr.c struct ehea_qp_init_attr *qp_attr = &qp->init_attr; qp 496 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ehea_h_disable_and_get_hea(qp->adapter->handle, qp->fw_handle); qp 497 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hret = ehea_h_free_resource(qp->adapter->handle, qp->fw_handle, force); qp 501 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hw_queue_dtor(&qp->hw_squeue); qp 502 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hw_queue_dtor(&qp->hw_rqueue1); qp 505 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hw_queue_dtor(&qp->hw_rqueue2); qp 507 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hw_queue_dtor(&qp->hw_rqueue3); qp 508 drivers/net/ethernet/ibm/ehea/ehea_qmr.c kfree(qp); qp 513 drivers/net/ethernet/ibm/ehea/ehea_qmr.c int ehea_destroy_qp(struct ehea_qp *qp) qp 516 drivers/net/ethernet/ibm/ehea/ehea_qmr.c if (!qp) qp 519 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hcp_epas_dtor(&qp->epas); qp 521 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hret = ehea_destroy_qp_res(qp, NORMAL_FREE); qp 523 drivers/net/ethernet/ibm/ehea/ehea_qmr.c ehea_error_data(qp->adapter, qp->fw_handle, &aer, &aerr); qp 524 drivers/net/ethernet/ibm/ehea/ehea_qmr.c hret = ehea_destroy_qp_res(qp, FORCE_FREE); qp 292 drivers/net/ethernet/ibm/ehea/ehea_qmr.h static inline struct ehea_rwqe *ehea_get_next_rwqe(struct ehea_qp *qp, qp 298 drivers/net/ethernet/ibm/ehea/ehea_qmr.h queue = &qp->hw_rqueue1; qp 300 drivers/net/ethernet/ibm/ehea/ehea_qmr.h queue = &qp->hw_rqueue2; qp 302 drivers/net/ethernet/ibm/ehea/ehea_qmr.h queue = &qp->hw_rqueue3; qp 325 drivers/net/ethernet/ibm/ehea/ehea_qmr.h static inline struct ehea_cqe *ehea_poll_rq1(struct ehea_qp *qp, int *wqe_index) qp 327 drivers/net/ethernet/ibm/ehea/ehea_qmr.h struct hw_queue *queue = &qp->hw_rqueue1; qp 338 drivers/net/ethernet/ibm/ehea/ehea_qmr.h static inline void ehea_inc_rq1(struct ehea_qp *qp) qp 340 drivers/net/ethernet/ibm/ehea/ehea_qmr.h hw_qeit_inc(&qp->hw_rqueue1); qp 372 drivers/net/ethernet/ibm/ehea/ehea_qmr.h int ehea_destroy_qp(struct ehea_qp *qp); qp 3522 drivers/net/ethernet/intel/i40e/i40e_main.c u32 qp; qp 3528 drivers/net/ethernet/intel/i40e/i40e_main.c qp = vsi->base_queue; qp 3551 drivers/net/ethernet/intel/i40e/i40e_main.c wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); qp 3553 drivers/net/ethernet/intel/i40e/i40e_main.c u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; qp 3563 drivers/net/ethernet/intel/i40e/i40e_main.c wr32(hw, I40E_QINT_RQCTL(qp), val); qp 3569 drivers/net/ethernet/intel/i40e/i40e_main.c (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | qp 3579 drivers/net/ethernet/intel/i40e/i40e_main.c ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | qp 3588 drivers/net/ethernet/intel/i40e/i40e_main.c wr32(hw, I40E_QINT_TQCTL(qp), val); qp 3589 drivers/net/ethernet/intel/i40e/i40e_main.c qp++; qp 4604 drivers/net/ethernet/intel/i40e/i40e_main.c u32 val, qp; qp 4642 drivers/net/ethernet/intel/i40e/i40e_main.c qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) qp 4648 drivers/net/ethernet/intel/i40e/i40e_main.c while (qp != I40E_QUEUE_END_OF_LIST) { qp 4651 drivers/net/ethernet/intel/i40e/i40e_main.c val = rd32(hw, I40E_QINT_RQCTL(qp)); qp 4661 drivers/net/ethernet/intel/i40e/i40e_main.c wr32(hw, I40E_QINT_RQCTL(qp), val); qp 4663 drivers/net/ethernet/intel/i40e/i40e_main.c val = rd32(hw, I40E_QINT_TQCTL(qp)); qp 4676 drivers/net/ethernet/intel/i40e/i40e_main.c wr32(hw, I40E_QINT_TQCTL(qp), val); qp 4677 drivers/net/ethernet/intel/i40e/i40e_main.c qp = next; qp 4684 drivers/net/ethernet/intel/i40e/i40e_main.c qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) qp 4690 drivers/net/ethernet/intel/i40e/i40e_main.c val = rd32(hw, I40E_QINT_RQCTL(qp)); qp 4699 drivers/net/ethernet/intel/i40e/i40e_main.c wr32(hw, I40E_QINT_RQCTL(qp), val); qp 4701 drivers/net/ethernet/intel/i40e/i40e_main.c val = rd32(hw, I40E_QINT_TQCTL(qp)); qp 4711 drivers/net/ethernet/intel/i40e/i40e_main.c wr32(hw, I40E_QINT_TQCTL(qp), val); qp 566 drivers/net/ethernet/mellanox/mlx4/en_netdev.c struct mlx4_qp qp; qp 569 drivers/net/ethernet/mellanox/mlx4/en_netdev.c qp.qpn = *qpn; qp 573 drivers/net/ethernet/mellanox/mlx4/en_netdev.c err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH); qp 617 drivers/net/ethernet/mellanox/mlx4/en_netdev.c struct mlx4_qp qp; qp 620 drivers/net/ethernet/mellanox/mlx4/en_netdev.c qp.qpn = qpn; qp 624 drivers/net/ethernet/mellanox/mlx4/en_netdev.c mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH); qp 94 drivers/net/ethernet/mellanox/mlx4/en_resources.c int mlx4_en_change_mcast_lb(struct mlx4_en_priv *priv, struct mlx4_qp *qp, qp 104 drivers/net/ethernet/mellanox/mlx4/en_resources.c ret = mlx4_update_qp(priv->mdev->dev, qp->qpn, qp 111 drivers/net/ethernet/mellanox/mlx4/en_resources.c void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event) qp 1059 drivers/net/ethernet/mellanox/mlx4/en_rx.c struct mlx4_qp *qp) qp 1069 drivers/net/ethernet/mellanox/mlx4/en_rx.c err = mlx4_qp_alloc(mdev->dev, qpn, qp); qp 1074 drivers/net/ethernet/mellanox/mlx4/en_rx.c qp->event = mlx4_en_sqp_event; qp 1091 drivers/net/ethernet/mellanox/mlx4/en_rx.c err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); qp 1093 drivers/net/ethernet/mellanox/mlx4/en_rx.c mlx4_qp_remove(mdev->dev, qp); qp 1094 drivers/net/ethernet/mellanox/mlx4/en_rx.c mlx4_qp_free(mdev->dev, qp); qp 538 drivers/net/ethernet/mellanox/mlx4/eq.c be32_to_cpu(eqe->event.qp.qpn) qp 553 drivers/net/ethernet/mellanox/mlx4/eq.c mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & qp 2713 drivers/net/ethernet/mellanox/mlx4/fw.c struct mlx4_qp qp; qp 2760 drivers/net/ethernet/mellanox/mlx4/fw.c qp.qpn = be32_to_cpu(mgm->qp[i]); qp 2762 drivers/net/ethernet/mellanox/mlx4/fw.c err = mlx4_multicast_detach(dev, &qp, qp 2766 drivers/net/ethernet/mellanox/mlx4/fw.c err = mlx4_multicast_attach(dev, &qp, qp 997 drivers/net/ethernet/mellanox/mlx4/main.c dev->quotas.qp = func_cap->qp_quota; qp 218 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count++] = cpu_to_be32(pqp->qpn & MGM_QPN_MASK); qp 355 drivers/net/ethernet/mellanox/mlx4/mcg.c u32 qpn = be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK; qp 477 drivers/net/ethernet/mellanox/mlx4/mcg.c if ((be32_to_cpu(mgm->qp[i]) & qp 501 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count++] = qp 525 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); qp 591 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); qp 636 drivers/net/ethernet/mellanox/mlx4/mcg.c if ((be32_to_cpu(mgm->qp[i]) & qp 652 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[loc] = mgm->qp[members_count - 1]; qp 653 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count - 1] = 0; qp 1104 drivers/net/ethernet/mellanox/mlx4/mcg.c int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1159 drivers/net/ethernet/mellanox/mlx4/mcg.c if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { qp 1160 drivers/net/ethernet/mellanox/mlx4/mcg.c mlx4_dbg(dev, "QP %06x already a member of MGM\n", qp->qpn); qp 1166 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count++] = cpu_to_be32((qp->qpn & MGM_QPN_MASK) | qp 1169 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count++] = cpu_to_be32(qp->qpn & MGM_QPN_MASK); qp 1195 drivers/net/ethernet/mellanox/mlx4/mcg.c index, qp->qpn); qp 1198 drivers/net/ethernet/mellanox/mlx4/mcg.c index, qp->qpn); qp 1214 drivers/net/ethernet/mellanox/mlx4/mcg.c int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1249 drivers/net/ethernet/mellanox/mlx4/mcg.c check_duplicate_entry(dev, port, steer, index, qp->qpn) && qp 1250 drivers/net/ethernet/mellanox/mlx4/mcg.c !promisc_steering_entry(dev, port, steer, index, qp->qpn, NULL)) qp 1255 drivers/net/ethernet/mellanox/mlx4/mcg.c if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { qp 1261 drivers/net/ethernet/mellanox/mlx4/mcg.c mlx4_err(dev, "QP %06x not found in MGM\n", qp->qpn); qp 1267 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[loc] = mgm->qp[members_count - 1]; qp 1268 drivers/net/ethernet/mellanox/mlx4/mcg.c mgm->qp[members_count - 1] = 0; qp 1273 drivers/net/ethernet/mellanox/mlx4/mcg.c index, qp->qpn); qp 1335 drivers/net/ethernet/mellanox/mlx4/mcg.c static int mlx4_QP_ATTACH(struct mlx4_dev *dev, struct mlx4_qp *qp, qp 1351 drivers/net/ethernet/mellanox/mlx4/mcg.c qpn = qp->qpn; qp 1367 drivers/net/ethernet/mellanox/mlx4/mcg.c int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, qp 1384 drivers/net/ethernet/mellanox/mlx4/mcg.c rule.qpn = qp->qpn; qp 1407 drivers/net/ethernet/mellanox/mlx4/mcg.c int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1422 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_QP_ATTACH(dev, qp, gid, 1, qp 1424 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_qp_attach_common(dev, qp, gid, qp 1429 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_trans_to_dmfs_attach(dev, qp, gid, port, qp 1438 drivers/net/ethernet/mellanox/mlx4/mcg.c int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1452 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); qp 1454 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_qp_detach_common(dev, qp, gid, prot, qp 1530 drivers/net/ethernet/mellanox/mlx4/mcg.c struct mlx4_qp *qp, u8 gid[16], qp 1537 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_QP_ATTACH(dev, qp, gid, 1, qp 1540 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_qp_attach_common(dev, qp, gid, block_mcast_loopback, qp 1545 drivers/net/ethernet/mellanox/mlx4/mcg.c int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, qp 1552 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); qp 1554 drivers/net/ethernet/mellanox/mlx4/mcg.c return mlx4_qp_detach_common(dev, qp, gid, prot, MLX4_UC_STEER); qp 624 drivers/net/ethernet/mellanox/mlx4/mlx4.h __be32 qp[MLX4_MAX_QP_PER_MGM]; qp 1321 drivers/net/ethernet/mellanox/mlx4/mlx4.h int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1323 drivers/net/ethernet/mellanox/mlx4/mlx4.h int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1326 drivers/net/ethernet/mellanox/mlx4/mlx4.h int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, qp 753 drivers/net/ethernet/mellanox/mlx4/mlx4_en.h void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event); qp 754 drivers/net/ethernet/mellanox/mlx4/mlx4_en.h int mlx4_en_change_mcast_lb(struct mlx4_en_priv *priv, struct mlx4_qp *qp, qp 52 drivers/net/ethernet/mellanox/mlx4/qp.c struct mlx4_qp *qp; qp 56 drivers/net/ethernet/mellanox/mlx4/qp.c qp = __mlx4_qp_lookup(dev, qpn); qp 57 drivers/net/ethernet/mellanox/mlx4/qp.c if (qp) qp 58 drivers/net/ethernet/mellanox/mlx4/qp.c refcount_inc(&qp->refcount); qp 62 drivers/net/ethernet/mellanox/mlx4/qp.c if (!qp) { qp 67 drivers/net/ethernet/mellanox/mlx4/qp.c qp->event(qp, event_type); qp 69 drivers/net/ethernet/mellanox/mlx4/qp.c if (refcount_dec_and_test(&qp->refcount)) qp 70 drivers/net/ethernet/mellanox/mlx4/qp.c complete(&qp->free); qp 74 drivers/net/ethernet/mellanox/mlx4/qp.c static int is_master_qp0(struct mlx4_dev *dev, struct mlx4_qp *qp, int *real_qp0, int *proxy_qp0) qp 79 drivers/net/ethernet/mellanox/mlx4/qp.c *proxy_qp0 = qp->qpn >= pf_proxy_offset && qp->qpn <= pf_proxy_offset + 1; qp 81 drivers/net/ethernet/mellanox/mlx4/qp.c *real_qp0 = qp->qpn >= dev->phys_caps.base_sqpn && qp 82 drivers/net/ethernet/mellanox/mlx4/qp.c qp->qpn <= dev->phys_caps.base_sqpn + 1; qp 91 drivers/net/ethernet/mellanox/mlx4/qp.c int sqd_event, struct mlx4_qp *qp, int native) qp 145 drivers/net/ethernet/mellanox/mlx4/qp.c ret = mlx4_cmd(dev, 0, qp->qpn, 2, qp 149 drivers/net/ethernet/mellanox/mlx4/qp.c is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { qp 150 drivers/net/ethernet/mellanox/mlx4/qp.c port = (qp->qpn & 1) + 1; qp 174 drivers/net/ethernet/mellanox/mlx4/qp.c cpu_to_be16(mlx4_qp_roce_entropy(dev, qp->qpn)); qp 180 drivers/net/ethernet/mellanox/mlx4/qp.c cpu_to_be32(qp->qpn); qp 183 drivers/net/ethernet/mellanox/mlx4/qp.c qp->qpn | (!!sqd_event << 31), qp 187 drivers/net/ethernet/mellanox/mlx4/qp.c if (mlx4_is_master(dev) && is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { qp 188 drivers/net/ethernet/mellanox/mlx4/qp.c port = (qp->qpn & 1) + 1; qp 212 drivers/net/ethernet/mellanox/mlx4/qp.c int sqd_event, struct mlx4_qp *qp) qp 215 drivers/net/ethernet/mellanox/mlx4/qp.c optpar, sqd_event, qp, 0); qp 394 drivers/net/ethernet/mellanox/mlx4/qp.c struct mlx4_qp *qp; qp 398 drivers/net/ethernet/mellanox/mlx4/qp.c qp = __mlx4_qp_lookup(dev, qpn); qp 401 drivers/net/ethernet/mellanox/mlx4/qp.c return qp; qp 404 drivers/net/ethernet/mellanox/mlx4/qp.c int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp) qp 413 drivers/net/ethernet/mellanox/mlx4/qp.c qp->qpn = qpn; qp 420 drivers/net/ethernet/mellanox/mlx4/qp.c err = radix_tree_insert(&dev->qp_table_tree, qp->qpn & qp 421 drivers/net/ethernet/mellanox/mlx4/qp.c (dev->caps.num_qps - 1), qp); qp 426 drivers/net/ethernet/mellanox/mlx4/qp.c refcount_set(&qp->refcount, 1); qp 427 drivers/net/ethernet/mellanox/mlx4/qp.c init_completion(&qp->free); qp 513 drivers/net/ethernet/mellanox/mlx4/qp.c void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp) qp 519 drivers/net/ethernet/mellanox/mlx4/qp.c radix_tree_delete(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1)); qp 524 drivers/net/ethernet/mellanox/mlx4/qp.c void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp) qp 526 drivers/net/ethernet/mellanox/mlx4/qp.c if (refcount_dec_and_test(&qp->refcount)) qp 527 drivers/net/ethernet/mellanox/mlx4/qp.c complete(&qp->free); qp 528 drivers/net/ethernet/mellanox/mlx4/qp.c wait_for_completion(&qp->free); qp 530 drivers/net/ethernet/mellanox/mlx4/qp.c mlx4_qp_free_icm(dev, qp->qpn); qp 893 drivers/net/ethernet/mellanox/mlx4/qp.c int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, qp 903 drivers/net/ethernet/mellanox/mlx4/qp.c err = mlx4_cmd_box(dev, 0, mailbox->dma, qp->qpn, 0, qp 916 drivers/net/ethernet/mellanox/mlx4/qp.c struct mlx4_qp *qp, enum mlx4_qp_state *qp_state) qp 933 drivers/net/ethernet/mellanox/mlx4/qp.c context, 0, 0, qp); qp 950 drivers/net/ethernet/mellanox/mlx4/qp.c struct mlx4_qp qp; qp 953 drivers/net/ethernet/mellanox/mlx4/qp.c qp.qpn = qpn; qp 954 drivers/net/ethernet/mellanox/mlx4/qp.c err = mlx4_qp_query(dev, &qp, &context); qp 452 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c dev->quotas.qp = dev->caps.num_qps - dev->caps.reserved_qps - qp 462 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c dev->quotas.qp = qp 1505 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c enum res_qp_states state, struct res_qp **qp, qp 1559 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp) qp 1560 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c *qp = r; qp 2960 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct res_qp *qp; qp 2977 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c err = qp_res_start_move_to(dev, slave, qpn, RES_QP_HW, &qp, 0); qp 2980 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->local_qpn = local_qpn; qp 2981 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->sched_queue = 0; qp 2982 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->param3 = 0; qp 2983 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->vlan_control = 0; qp 2984 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->fvl_rx = 0; qp 2985 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->pri_path_fl = 0; qp 2986 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->vlan_index = 0; qp 2987 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->feup = 0; qp 2988 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->qpc_flags = be32_to_cpu(qpc->flags); qp 3021 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->mtt = mtt; qp 3023 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->rcq = rcq; qp 3025 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->scq = scq; qp 3033 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->srq = srq; qp 3037 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->param3 = qpc->param3; qp 3739 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct res_qp *qp; qp 3741 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c err = get_res(dev, slave, qpn, RES_QP, &qp); qp 3744 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp->com.from_state != RES_QP_HW) { qp 3828 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct res_qp *qp; qp 3851 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c err = get_res(dev, slave, qpn, RES_QP, &qp); qp 3854 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp->com.from_state != RES_QP_HW) { qp 3870 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->sched_queue = orig_sched_queue; qp 3871 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->vlan_control = orig_vlan_control; qp 3872 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->fvl_rx = orig_fvl_rx; qp 3873 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->pri_path_fl = orig_pri_path_fl; qp 3874 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->vlan_index = orig_vlan_index; qp 3875 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->feup = orig_feup; qp 3992 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct res_qp *qp; qp 3994 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, &qp, 0); qp 4001 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->mtt->ref_count); qp 4002 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->rcq->ref_count); qp 4003 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->scq->ref_count); qp 4004 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp->srq) qp 4005 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->srq->ref_count); qp 4077 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c static int qp_attach(struct mlx4_dev *dev, int slave, struct mlx4_qp *qp, qp 4086 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c return mlx4_trans_to_dmfs_attach(dev, qp, gid, port, qp 4097 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c return mlx4_qp_attach_common(dev, qp, gid, qp 4104 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c static int qp_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, qp 4112 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c return mlx4_qp_detach_common(dev, qp, gid, prot, type); qp 4143 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct mlx4_qp qp; /* dummy for calling attach/detach */ qp 4160 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp.qpn = qpn; qp 4162 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c err = qp_attach(dev, slave, &qp, gid, block_loopback, prot, qp 4180 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c err = qp_detach(dev, &qp, gid, prot, type, reg_id); qp 4189 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp_detach(dev, &qp, gid, prot, type, reg_id); qp 4585 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct mlx4_qp qp; /* dummy for calling attach/detach */ qp 4593 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp.qpn = rqp->local_qpn; qp 4594 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c (void) mlx4_qp_detach_common(dev, &qp, rgid->gid, qp 4665 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct res_qp *qp; qp 4678 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c list_for_each_entry_safe(qp, tmp, qp_list, com.list) { qp 4680 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp->com.owner == slave) { qp 4681 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qpn = qp->com.res_id; qp 4682 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c detach_qp(dev, slave, qp); qp 4683 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c state = qp->com.from_state; qp 4688 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c rb_erase(&qp->com.node, qp 4690 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c list_del(&qp->com.list); qp 4697 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c kfree(qp); qp 4708 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->local_qpn, 2, qp 4714 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c slave, qp->local_qpn); qp 4715 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->rcq->ref_count); qp 4716 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->scq->ref_count); qp 4717 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->mtt->ref_count); qp 4718 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp->srq) qp 4719 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c atomic_dec(&qp->srq->ref_count); qp 5283 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c struct res_qp *qp; qp 5339 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c list_for_each_entry_safe(qp, tmp, qp_list, com.list) { qp 5341 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp->com.owner == work->slave) { qp 5342 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (qp->com.from_state != RES_QP_HW || qp 5343 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c !qp->sched_queue || /* no INIT2RTR trans yet */ qp 5344 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c mlx4_is_qp_reserved(dev, qp->local_qpn) || qp 5345 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET)) { qp 5349 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c port = (qp->sched_queue >> 6 & 1) + 1; qp 5354 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c if (MLX4_QP_ST_RC == ((qp->qpc_flags >> 16) & 0xff)) qp 5360 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c upd_context->qp_context.param3 = qp->param3; qp 5361 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c upd_context->qp_context.pri_path.vlan_control = qp->vlan_control; qp 5362 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c upd_context->qp_context.pri_path.fvl_rx = qp->fvl_rx; qp 5363 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c upd_context->qp_context.pri_path.vlan_index = qp->vlan_index; qp 5364 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c upd_context->qp_context.pri_path.fl = qp->pri_path_fl; qp 5365 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c upd_context->qp_context.pri_path.feup = qp->feup; qp 5367 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->sched_queue; qp 5369 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c upd_context->qp_context.param3 = qp->param3 & ~cpu_to_be32(MLX4_STRIP_VLAN); qp 5373 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->fvl_rx | MLX4_FVL_RX_FORCE_ETH_VLAN; qp 5375 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->pri_path_fl | MLX4_FL_ETH_HIDE_CQE_VLAN; qp 5381 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->feup | MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN; qp 5383 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->sched_queue & 0xC7; qp 5393 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c qp->local_qpn & 0xffffff, qp 5398 drivers/net/ethernet/mellanox/mlx4/resource_tracker.c work->slave, port, qp->local_qpn, err); qp 202 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c static u64 qp_read_field(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp, qp 216 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c err = mlx5_core_qp_query(dev, qp, out, outlen); qp 229 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c param = qp->pid; qp 449 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c int mlx5_debug_qp_add(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp) qp 457 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c &qp->dbg, qp->qpn, qp_fields, qp 458 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c ARRAY_SIZE(qp_fields), qp); qp 460 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c qp->dbg = NULL; qp 465 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c void mlx5_debug_qp_remove(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp) qp 470 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c if (qp->dbg) qp 471 drivers/net/ethernet/mellanox/mlx5/core/debugfs.c rem_res_tree(qp->dbg); qp 110 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) { qp 115 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c ix = conn->qp.rq.pc & (conn->qp.rq.size - 1); qp 116 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix); qp 121 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.pc++; qp 122 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.bufs[ix] = buf; qp 126 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff); qp 135 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc); qp 149 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c ix = conn->qp.sq.pc & (conn->qp.sq.size - 1); qp 151 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix); qp 166 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) | qp 168 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.mqp.qpn << 8)); qp 170 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.pc++; qp 171 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.bufs[ix] = buf; qp 181 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (!conn->qp.active) qp 189 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c spin_lock_irqsave(&conn->qp.sq.lock, flags); qp 191 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) { qp 192 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c list_add_tail(&buf->list, &conn->qp.sq.backlog); qp 199 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c spin_unlock_irqrestore(&conn->qp.sq.lock, flags); qp 256 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1); qp 257 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c buf = conn->qp.rq.bufs[ix]; qp 258 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.bufs[ix] = NULL; qp 259 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.cc++; qp 270 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (unlikely(status || !conn->qp.active)) { qp 271 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.active = false; qp 297 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c spin_lock_irqsave(&conn->qp.sq.lock, flags); qp 299 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1); qp 300 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c buf = conn->qp.sq.bufs[ix]; qp 301 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.bufs[ix] = NULL; qp 302 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.cc++; qp 305 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (unlikely(!list_empty(&conn->qp.sq.backlog))) { qp 306 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (likely(conn->qp.active)) { qp 307 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c nextbuf = list_first_entry(&conn->qp.sq.backlog, qp 314 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c spin_unlock_irqrestore(&conn->qp.sq.lock, flags); qp 329 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.active = false; qp 378 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn = container_of(mqp, struct mlx5_fpga_conn, qp.mqp); qp 412 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (unlikely(!conn->qp.active)) qp 423 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (unlikely(!conn->qp.active)) qp 529 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq, qp 530 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c &conn->qp.wq_ctrl); qp 542 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.pc = 0; qp 543 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.cc = 0; qp 544 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.size = roundup_pow_of_two(rx_size); qp 545 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.pc = 0; qp 546 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.cc = 0; qp 547 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.size = roundup_pow_of_two(tx_size); qp 550 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size)); qp 551 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size)); qp 556 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.bufs = kvcalloc(conn->qp.rq.size, qp 557 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c sizeof(conn->qp.rq.bufs[0]), qp 559 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (!conn->qp.rq.bufs) { qp 564 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.bufs = kvcalloc(conn->qp.sq.size, qp 565 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c sizeof(conn->qp.sq.bufs[0]), qp 567 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (!conn->qp.sq.bufs) { qp 574 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.wq_ctrl.buf.npages; qp 584 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); qp 591 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size)); qp 593 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size)); qp 596 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); qp 600 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_fill_page_frag_array(&conn->qp.wq_ctrl.buf, qp 603 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c err = mlx5_core_create_qp(mdev, &conn->qp.mqp, in, inlen); qp 607 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.mqp.event = mlx5_fpga_conn_event; qp 608 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.mqp.qpn); qp 613 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c kvfree(conn->qp.sq.bufs); qp 615 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c kvfree(conn->qp.rq.bufs); qp 617 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_wq_destroy(&conn->qp.wq_ctrl); qp 627 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c for (ix = 0; ix < conn->qp.rq.size; ix++) { qp 628 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c if (!conn->qp.rq.bufs[ix]) qp 630 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]); qp 631 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c kfree(conn->qp.rq.bufs[ix]); qp 632 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.rq.bufs[ix] = NULL; qp 641 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c for (ix = 0; ix < conn->qp.sq.size; ix++) { qp 642 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c buf = conn->qp.sq.bufs[ix]; qp 645 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sq.bufs[ix] = NULL; qp 651 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) { qp 661 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_core_destroy_qp(conn->fdev->mdev, &conn->qp.mqp); qp 664 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c kvfree(conn->qp.sq.bufs); qp 665 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c kvfree(conn->qp.rq.bufs); qp 666 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_wq_destroy(&conn->qp.wq_ctrl); qp 673 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.mqp.qpn); qp 676 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c &conn->qp.mqp); qp 686 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.mqp.qpn); qp 701 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); qp 704 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c &conn->qp.mqp); qp 742 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.sgid_index); qp 749 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c &conn->qp.mqp); qp 786 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c &conn->qp.mqp); qp 821 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.active = true; qp 866 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c INIT_LIST_HEAD(&conn->qp.sq.backlog); qp 868 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c spin_lock_init(&conn->qp.sq.lock); qp 887 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index); qp 894 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, qp 904 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index); qp 934 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.mqp.qpn); qp 963 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL, qp 966 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index); qp 979 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c conn->qp.active = false; qp 985 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c &conn->qp.mqp); qp 991 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0, qp 993 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index); qp 83 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.h } qp; qp 163 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c struct mlx5_core_qp *qp = &ipriv->qp; qp 177 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c ret = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RST2INIT_QP, 0, context, qp); qp 184 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c ret = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_INIT2RTR_QP, 0, context, qp); qp 190 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c ret = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RTR2RTS_QP, 0, context, qp); qp 200 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2ERR_QP, 0, &context, qp); qp 213 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c &ipriv->qp); qp 220 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c int mlx5i_create_underlay_qp(struct mlx5_core_dev *mdev, struct mlx5_core_qp *qp) qp 243 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c ret = mlx5_core_create_qp(mdev, qp, in, inlen); qp 254 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c void mlx5i_destroy_underlay_qp(struct mlx5_core_dev *mdev, struct mlx5_core_qp *qp) qp 256 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5_core_destroy_qp(mdev, qp); qp 276 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c err = mlx5i_create_underlay_qp(priv->mdev, &ipriv->qp); qp 282 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c err = mlx5i_create_tis(priv->mdev, ipriv->qp.qpn, &priv->tisn[0][0]); qp 291 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5i_destroy_underlay_qp(priv->mdev, &ipriv->qp); qp 300 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5i_destroy_underlay_qp(priv->mdev, &ipriv->qp); qp 479 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c dev->dev_addr[1] = (ipriv->qp.qpn >> 16) & 0xff; qp 480 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c dev->dev_addr[2] = (ipriv->qp.qpn >> 8) & 0xff; qp 481 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c dev->dev_addr[3] = (ipriv->qp.qpn) & 0xff; qp 484 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5i_pkey_add_qpn(dev ,ipriv->qp.qpn); qp 511 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5i_pkey_del_qpn(dev, ipriv->qp.qpn); qp 531 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c err = mlx5_fs_add_rx_underlay_qpn(mdev, ipriv->qp.qpn); qp 548 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qp.qpn); qp 574 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qp.qpn); qp 593 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5_core_dbg(mdev, "attaching QPN 0x%x, MGID %pI6\n", ipriv->qp.qpn, gid->raw); qp 594 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c err = mlx5_core_attach_mcg(mdev, gid, ipriv->qp.qpn); qp 597 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c ipriv->qp.qpn, gid->raw); qp 616 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c mlx5_core_dbg(mdev, "detaching QPN 0x%x, MGID %pI6\n", ipriv->qp.qpn, gid->raw); qp 618 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c err = mlx5_core_detach_mcg(mdev, gid, ipriv->qp.qpn); qp 621 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c ipriv->qp.qpn, gid->raw); qp 54 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h struct mlx5_core_qp qp; qp 65 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h int mlx5i_create_underlay_qp(struct mlx5_core_dev *mdev, struct mlx5_core_qp *qp); qp 66 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h void mlx5i_destroy_underlay_qp(struct mlx5_core_dev *mdev, struct mlx5_core_qp *qp); qp 207 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c err = mlx5_fs_add_rx_underlay_qpn(mdev, ipriv->qp.qpn); qp 213 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c err = mlx5i_create_tis(mdev, ipriv->qp.qpn, &epriv->tisn[0][0]); qp 233 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qp.qpn); qp 256 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qp.qpn); qp 313 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c err = mlx5i_create_underlay_qp(priv->mdev, &ipriv->qp); qp 326 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c mlx5i_destroy_underlay_qp(priv->mdev, &ipriv->qp); qp 130 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp; qp 179 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp = (struct mlx5_core_qp *)common; qp 180 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->event(qp, event_type); qp 197 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp, qp 203 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->common.res = rsc_type; qp 206 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN), qp 207 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp); qp 212 drivers/net/ethernet/mellanox/mlx5/core/qp.c refcount_set(&qp->common.refcount, 1); qp 213 drivers/net/ethernet/mellanox/mlx5/core/qp.c init_completion(&qp->common.free); qp 214 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->pid = current->pid; qp 220 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp) qp 227 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->qpn | (qp->common.res << MLX5_USER_INDEX_LEN)); qp 229 drivers/net/ethernet/mellanox/mlx5/core/qp.c mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp); qp 230 drivers/net/ethernet/mellanox/mlx5/core/qp.c wait_for_completion(&qp->common.free); qp 238 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp = &dct->mqp; qp 248 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->qpn, err); qp 257 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(destroy_dct_in, in, dctn, qp->qpn); qp 258 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(destroy_dct_in, in, uid, qp->uid); qp 269 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp = &dct->mqp; qp 281 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->qpn = MLX5_GET(create_dct_out, out, dctn); qp 282 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->uid = MLX5_GET(create_dct_in, in, uid); qp 283 drivers/net/ethernet/mellanox/mlx5/core/qp.c err = create_resource_common(dev, qp, MLX5_RES_DCT); qp 295 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp, qp 309 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->uid = MLX5_GET(create_qp_in, in, uid); qp 310 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->qpn = MLX5_GET(create_qp_out, out, qpn); qp 311 drivers/net/ethernet/mellanox/mlx5/core/qp.c mlx5_core_dbg(dev, "qpn = 0x%x\n", qp->qpn); qp 313 drivers/net/ethernet/mellanox/mlx5/core/qp.c err = create_resource_common(dev, qp, MLX5_RES_QP); qp 317 drivers/net/ethernet/mellanox/mlx5/core/qp.c err = mlx5_debug_qp_add(dev, qp); qp 320 drivers/net/ethernet/mellanox/mlx5/core/qp.c qp->qpn); qp 330 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(destroy_qp_in, din, qpn, qp->qpn); qp 331 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(destroy_qp_in, din, uid, qp->uid); qp 342 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp = &dct->mqp; qp 345 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(drain_dct_in, in, dctn, qp->qpn); qp 346 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(drain_dct_in, in, uid, qp->uid); qp 359 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp) qp 365 drivers/net/ethernet/mellanox/mlx5/core/qp.c mlx5_debug_qp_remove(dev, qp); qp 367 drivers/net/ethernet/mellanox/mlx5/core/qp.c destroy_resource_common(dev, qp); qp 370 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(destroy_qp_in, in, qpn, qp->qpn); qp 371 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(destroy_qp_in, in, uid, qp->uid); qp 508 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp) qp 513 drivers/net/ethernet/mellanox/mlx5/core/qp.c err = modify_qp_mbox_alloc(dev, opcode, qp->qpn, qp 514 drivers/net/ethernet/mellanox/mlx5/core/qp.c opt_param_mask, qpc, &mbox, qp->uid); qp 545 drivers/net/ethernet/mellanox/mlx5/core/qp.c int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp, qp 551 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(query_qp_in, in, qpn, qp->qpn); qp 560 drivers/net/ethernet/mellanox/mlx5/core/qp.c struct mlx5_core_qp *qp = &dct->mqp; qp 563 drivers/net/ethernet/mellanox/mlx5/core/qp.c MLX5_SET(query_dct_in, in, dctn, qp->qpn); qp 58 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c (dr_cq->qp->sq.wqe_cnt - 1); qp 59 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1; qp 61 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c ++dr_cq->qp->sq.cc; qp 64 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c (dr_cq->qp->sq.wqe_cnt - 1); qp 65 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1; qp 379 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dr_post_send(send_ring->qp, send_info); qp 645 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c struct mlx5dr_qp *dr_qp = dmn->send_ring->qp; qp 886 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dmn->send_ring->qp = dr_create_rc_qp(dmn->mdev, &init_attr); qp 887 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c if (!dmn->send_ring->qp) { qp 892 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dmn->send_ring->cq->qp = dmn->send_ring->qp; qp 895 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dmn->info.max_inline_size = min(dmn->send_ring->qp->max_inline_data, qp 942 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dr_destroy_qp(dmn->mdev, dmn->send_ring->qp); qp 954 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c dr_destroy_qp(dmn->mdev, send_ring->qp); qp 997 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h struct mlx5dr_qp *qp; qp 1013 drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h struct mlx5dr_qp *qp; qp 208 drivers/net/ethernet/qlogic/qed/qed_iwarp.c struct qed_rdma_qp *qp, qp 218 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->shared_queue = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, qp 220 drivers/net/ethernet/qlogic/qed/qed_iwarp.c &qp->shared_queue_phys_addr, qp 222 drivers/net/ethernet/qlogic/qed/qed_iwarp.c if (!qp->shared_queue) qp 225 drivers/net/ethernet/qlogic/qed/qed_iwarp.c out_params->sq_pbl_virt = (u8 *)qp->shared_queue + qp 227 drivers/net/ethernet/qlogic/qed/qed_iwarp.c out_params->sq_pbl_phys = qp->shared_queue_phys_addr + qp 229 drivers/net/ethernet/qlogic/qed/qed_iwarp.c out_params->rq_pbl_virt = (u8 *)qp->shared_queue + qp 231 drivers/net/ethernet/qlogic/qed/qed_iwarp.c out_params->rq_pbl_phys = qp->shared_queue_phys_addr + qp 238 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->icid = (u16)cid; qp 242 drivers/net/ethernet/qlogic/qed/qed_iwarp.c init_data.cid = qp->icid; qp 255 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->fmr_and_reserved_lkey); qp 258 drivers/net/ethernet/qlogic/qed/qed_iwarp.c IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP, qp->signal_all); qp 262 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->incoming_rdma_read_en); qp 266 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->incoming_rdma_write_en); qp 270 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->incoming_atomic_en); qp 273 drivers/net/ethernet/qlogic/qed/qed_iwarp.c IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG, qp->use_srq); qp 275 drivers/net/ethernet/qlogic/qed/qed_iwarp.c p_ramrod->pd = qp->pd; qp 276 drivers/net/ethernet/qlogic/qed/qed_iwarp.c p_ramrod->sq_num_pages = qp->sq_num_pages; qp 277 drivers/net/ethernet/qlogic/qed/qed_iwarp.c p_ramrod->rq_num_pages = qp->rq_num_pages; qp 279 drivers/net/ethernet/qlogic/qed/qed_iwarp.c p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id); qp 281 drivers/net/ethernet/qlogic/qed/qed_iwarp.c p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); qp 282 drivers/net/ethernet/qlogic/qed/qed_iwarp.c p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); qp 285 drivers/net/ethernet/qlogic/qed/qed_iwarp.c cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->sq_cq_id); qp 287 drivers/net/ethernet/qlogic/qed/qed_iwarp.c cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->rq_cq_id); qp 289 drivers/net/ethernet/qlogic/qed/qed_iwarp.c p_ramrod->dpi = cpu_to_le16(qp->dpi); qp 307 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->shared_queue, qp->shared_queue_phys_addr); qp 312 drivers/net/ethernet/qlogic/qed/qed_iwarp.c static int qed_iwarp_modify_fw(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) qp 321 drivers/net/ethernet/qlogic/qed/qed_iwarp.c init_data.cid = qp->icid; qp 334 drivers/net/ethernet/qlogic/qed/qed_iwarp.c if (qp->iwarp_state == QED_IWARP_QP_STATE_CLOSING) qp 341 drivers/net/ethernet/qlogic/qed/qed_iwarp.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x)rc=%d\n", qp->icid, rc); qp 395 drivers/net/ethernet/qlogic/qed/qed_iwarp.c struct qed_rdma_qp *qp, qp 406 drivers/net/ethernet/qlogic/qed/qed_iwarp.c prev_iw_state = qp->iwarp_state; qp 417 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->iwarp_state = QED_IWARP_QP_STATE_RTS; qp 420 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->iwarp_state = QED_IWARP_QP_STATE_ERROR; qp 434 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->iwarp_state = QED_IWARP_QP_STATE_CLOSING; qp 439 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->iwarp_state = QED_IWARP_QP_STATE_ERROR; qp 449 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->iwarp_state = new_state; qp 460 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->iwarp_state = new_state; qp 467 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->icid, qp 469 drivers/net/ethernet/qlogic/qed/qed_iwarp.c iwarp_state_names[qp->iwarp_state], qp 475 drivers/net/ethernet/qlogic/qed/qed_iwarp.c rc = qed_iwarp_modify_fw(p_hwfn, qp); qp 480 drivers/net/ethernet/qlogic/qed/qed_iwarp.c int qed_iwarp_fw_destroy(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) qp 488 drivers/net/ethernet/qlogic/qed/qed_iwarp.c init_data.cid = qp->icid; qp 500 drivers/net/ethernet/qlogic/qed/qed_iwarp.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) rc = %d\n", qp->icid, rc); qp 519 drivers/net/ethernet/qlogic/qed/qed_iwarp.c if (ep->qp) qp 520 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp->ep = NULL; qp 525 drivers/net/ethernet/qlogic/qed/qed_iwarp.c int qed_iwarp_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) qp 527 drivers/net/ethernet/qlogic/qed/qed_iwarp.c struct qed_iwarp_ep *ep = qp->ep; qp 531 drivers/net/ethernet/qlogic/qed/qed_iwarp.c if (qp->iwarp_state != QED_IWARP_QP_STATE_ERROR) { qp 532 drivers/net/ethernet/qlogic/qed/qed_iwarp.c rc = qed_iwarp_modify_qp(p_hwfn, qp, qp 551 drivers/net/ethernet/qlogic/qed/qed_iwarp.c rc = qed_iwarp_fw_destroy(p_hwfn, qp); qp 553 drivers/net/ethernet/qlogic/qed/qed_iwarp.c if (qp->shared_queue) qp 556 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->shared_queue, qp->shared_queue_phys_addr); qp 843 drivers/net/ethernet/qlogic/qed/qed_iwarp.c struct qed_rdma_qp *qp; qp 850 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp = ep->qp; qp 851 drivers/net/ethernet/qlogic/qed/qed_iwarp.c reject = !qp; qp 854 drivers/net/ethernet/qlogic/qed/qed_iwarp.c init_data.cid = reject ? ep->tcp_cid : qp->icid; qp 898 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->shared_queue_phys_addr); qp 900 drivers/net/ethernet/qlogic/qed/qed_iwarp.c RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + qp->stats_queue; qp 914 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->cid = qp->icid; /* Now they're migrated. */ qp 919 drivers/net/ethernet/qlogic/qed/qed_iwarp.c reject ? 0xffff : qp->icid, qp 931 drivers/net/ethernet/qlogic/qed/qed_iwarp.c if (ep->qp) qp 932 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp->ep = NULL; qp 933 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp = NULL; qp 1033 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp->max_rd_atomic_req = ep->cm_info.ord; qp 1034 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp->max_rd_atomic_resp = ep->cm_info.ird; qp 1035 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_RTS, 1); qp 1104 drivers/net/ethernet/qlogic/qed/qed_iwarp.c (!ep->qp)) { /* Rejected */ qp 1176 drivers/net/ethernet/qlogic/qed/qed_iwarp.c iparams->qp->icid, iparams->cm_info.ord, qp 1199 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp = iparams->qp; qp 1200 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp->ep = ep; qp 1239 drivers/net/ethernet/qlogic/qed/qed_iwarp.c iparams->qp->icid, ep->tcp_cid, rc); qp 1472 drivers/net/ethernet/qlogic/qed/qed_iwarp.c iparams->qp->icid, ep->tcp_cid); qp 1479 drivers/net/ethernet/qlogic/qed/qed_iwarp.c iparams->qp->icid, qp 1487 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp = iparams->qp; qp 1488 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp->ep = ep; qp 1518 drivers/net/ethernet/qlogic/qed/qed_iwarp.c iparams->qp, QED_IWARP_QP_STATE_ERROR, 1); qp 1538 drivers/net/ethernet/qlogic/qed/qed_iwarp.c ep->qp = NULL; qp 2846 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_ERROR, true); qp 3201 drivers/net/ethernet/qlogic/qed/qed_iwarp.c struct qed_rdma_qp *qp; qp 3210 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp = ep->qp; qp 3213 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qp->icid, ep->tcp_cid); qp 3216 drivers/net/ethernet/qlogic/qed/qed_iwarp.c init_data.cid = qp->icid; qp 3235 drivers/net/ethernet/qlogic/qed/qed_iwarp.c qed_iwarp_query_qp(struct qed_rdma_qp *qp, qp 3238 drivers/net/ethernet/qlogic/qed/qed_iwarp.c out_params->state = qed_iwarp2roce_state(qp->iwarp_state); qp 139 drivers/net/ethernet/qlogic/qed/qed_iwarp.h struct qed_rdma_qp *qp; qp 201 drivers/net/ethernet/qlogic/qed/qed_iwarp.h struct qed_rdma_qp *qp, qp 204 drivers/net/ethernet/qlogic/qed/qed_iwarp.h int qed_iwarp_modify_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp, qp 207 drivers/net/ethernet/qlogic/qed/qed_iwarp.h int qed_iwarp_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp); qp 209 drivers/net/ethernet/qlogic/qed/qed_iwarp.h int qed_iwarp_fw_destroy(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp); qp 211 drivers/net/ethernet/qlogic/qed/qed_iwarp.h void qed_iwarp_query_qp(struct qed_rdma_qp *qp, qp 1178 drivers/net/ethernet/qlogic/qed/qed_rdma.c struct qed_rdma_qp *qp, qp 1184 drivers/net/ethernet/qlogic/qed/qed_rdma.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 1189 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->mtu = qp->mtu; qp 1190 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->dest_qp = qp->dest_qp; qp 1191 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->incoming_atomic_en = qp->incoming_atomic_en; qp 1192 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->e2e_flow_control_en = qp->e2e_flow_control_en; qp 1193 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en; qp 1194 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en; qp 1195 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->dgid = qp->dgid; qp 1196 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->flow_label = qp->flow_label; qp 1197 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->hop_limit_ttl = qp->hop_limit_ttl; qp 1198 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->traffic_class_tos = qp->traffic_class_tos; qp 1199 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->timeout = qp->ack_timeout; qp 1200 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->rnr_retry = qp->rnr_retry_cnt; qp 1201 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->retry_cnt = qp->retry_cnt; qp 1202 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer; qp 1204 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->max_rd_atomic = qp->max_rd_atomic_req; qp 1205 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp; qp 1206 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->sqd_async = qp->sqd_async; qp 1209 drivers/net/ethernet/qlogic/qed/qed_rdma.c qed_iwarp_query_qp(qp, out_params); qp 1211 drivers/net/ethernet/qlogic/qed/qed_rdma.c rc = qed_roce_query_qp(p_hwfn, qp, out_params); qp 1217 drivers/net/ethernet/qlogic/qed/qed_rdma.c static int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp) qp 1222 drivers/net/ethernet/qlogic/qed/qed_rdma.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 1225 drivers/net/ethernet/qlogic/qed/qed_rdma.c rc = qed_iwarp_destroy_qp(p_hwfn, qp); qp 1227 drivers/net/ethernet/qlogic/qed/qed_rdma.c rc = qed_roce_destroy_qp(p_hwfn, qp); qp 1230 drivers/net/ethernet/qlogic/qed/qed_rdma.c kfree(qp); qp 1242 drivers/net/ethernet/qlogic/qed/qed_rdma.c struct qed_rdma_qp *qp; qp 1284 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp = kzalloc(sizeof(*qp), GFP_KERNEL); qp 1285 drivers/net/ethernet/qlogic/qed/qed_rdma.c if (!qp) qp 1288 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->cur_state = QED_ROCE_QP_STATE_RESET; qp 1289 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->qp_handle.hi = cpu_to_le32(in_params->qp_handle_hi); qp 1290 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->qp_handle.lo = cpu_to_le32(in_params->qp_handle_lo); qp 1291 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->qp_handle_async.hi = cpu_to_le32(in_params->qp_handle_async_hi); qp 1292 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->qp_handle_async.lo = cpu_to_le32(in_params->qp_handle_async_lo); qp 1293 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->use_srq = in_params->use_srq; qp 1294 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->signal_all = in_params->signal_all; qp 1295 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey; qp 1296 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->pd = in_params->pd; qp 1297 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->dpi = in_params->dpi; qp 1298 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->sq_cq_id = in_params->sq_cq_id; qp 1299 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->sq_num_pages = in_params->sq_num_pages; qp 1300 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->sq_pbl_ptr = in_params->sq_pbl_ptr; qp 1301 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->rq_cq_id = in_params->rq_cq_id; qp 1302 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->rq_num_pages = in_params->rq_num_pages; qp 1303 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->rq_pbl_ptr = in_params->rq_pbl_ptr; qp 1304 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->srq_id = in_params->srq_id; qp 1305 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->req_offloaded = false; qp 1306 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->resp_offloaded = false; qp 1307 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->e2e_flow_control_en = qp->use_srq ? false : true; qp 1308 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->stats_queue = in_params->stats_queue; qp 1311 drivers/net/ethernet/qlogic/qed/qed_rdma.c rc = qed_iwarp_create_qp(p_hwfn, qp, out_params); qp 1312 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->qpid = qp->icid; qp 1314 drivers/net/ethernet/qlogic/qed/qed_rdma.c rc = qed_roce_alloc_cid(p_hwfn, &qp->icid); qp 1315 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->qpid = ((0xFF << 16) | qp->icid); qp 1319 drivers/net/ethernet/qlogic/qed/qed_rdma.c kfree(qp); qp 1323 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->icid = qp->icid; qp 1324 drivers/net/ethernet/qlogic/qed/qed_rdma.c out_params->qp_id = qp->qpid; qp 1327 drivers/net/ethernet/qlogic/qed/qed_rdma.c return qp; qp 1331 drivers/net/ethernet/qlogic/qed/qed_rdma.c struct qed_rdma_qp *qp, qp 1339 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->icid, params->new_state); qp 1348 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->incoming_rdma_read_en = params->incoming_rdma_read_en; qp 1349 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->incoming_rdma_write_en = params->incoming_rdma_write_en; qp 1350 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->incoming_atomic_en = params->incoming_atomic_en; qp 1355 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->roce_mode = params->roce_mode; qp 1357 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->pkey = params->pkey; qp 1360 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->e2e_flow_control_en = params->e2e_flow_control_en; qp 1362 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->dest_qp = params->dest_qp; qp 1369 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->traffic_class_tos = params->traffic_class_tos; qp 1370 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->flow_label = params->flow_label; qp 1371 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->hop_limit_ttl = params->hop_limit_ttl; qp 1373 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->sgid = params->sgid; qp 1374 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->dgid = params->dgid; qp 1375 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->udp_src_port = 0; qp 1376 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->vlan_id = params->vlan_id; qp 1377 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->mtu = params->mtu; qp 1378 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->lb_indication = params->lb_indication; qp 1379 drivers/net/ethernet/qlogic/qed/qed_rdma.c memcpy((u8 *)&qp->remote_mac_addr[0], qp 1382 drivers/net/ethernet/qlogic/qed/qed_rdma.c memcpy((u8 *)&qp->local_mac_addr[0], qp 1385 drivers/net/ethernet/qlogic/qed/qed_rdma.c memcpy((u8 *)&qp->local_mac_addr[0], qp 1390 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->rq_psn = params->rq_psn; qp 1392 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->sq_psn = params->sq_psn; qp 1395 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->max_rd_atomic_req = params->max_rd_atomic_req; qp 1398 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->max_rd_atomic_resp = params->max_rd_atomic_resp; qp 1401 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->ack_timeout = params->ack_timeout; qp 1403 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->retry_cnt = params->retry_cnt; qp 1406 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->rnr_retry_cnt = params->rnr_retry_cnt; qp 1409 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->min_rnr_nak_timer = params->min_rnr_nak_timer; qp 1411 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->sqd_async = params->sqd_async; qp 1413 drivers/net/ethernet/qlogic/qed/qed_rdma.c prev_state = qp->cur_state; qp 1416 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->cur_state = params->new_state; qp 1418 drivers/net/ethernet/qlogic/qed/qed_rdma.c qp->cur_state); qp 1423 drivers/net/ethernet/qlogic/qed/qed_rdma.c qed_roce2iwarp_state(qp->cur_state); qp 1425 drivers/net/ethernet/qlogic/qed/qed_rdma.c rc = qed_iwarp_modify_qp(p_hwfn, qp, new_state, 0); qp 1427 drivers/net/ethernet/qlogic/qed/qed_rdma.c rc = qed_roce_modify_qp(p_hwfn, qp, prev_state, params); qp 119 drivers/net/ethernet/qlogic/qed/qed_roce.c static void qed_rdma_copy_gids(struct qed_rdma_qp *qp, __le32 *src_gid, qp 124 drivers/net/ethernet/qlogic/qed/qed_roce.c if (qp->roce_mode == ROCE_V2_IPV4) { qp 130 drivers/net/ethernet/qlogic/qed/qed_roce.c src_gid[3] = cpu_to_le32(qp->sgid.ipv4_addr); qp 131 drivers/net/ethernet/qlogic/qed/qed_roce.c dst_gid[3] = cpu_to_le32(qp->dgid.ipv4_addr); qp 134 drivers/net/ethernet/qlogic/qed/qed_roce.c for (i = 0; i < ARRAY_SIZE(qp->sgid.dwords); i++) { qp 135 drivers/net/ethernet/qlogic/qed/qed_roce.c src_gid[i] = cpu_to_le32(qp->sgid.dwords[i]); qp 136 drivers/net/ethernet/qlogic/qed/qed_roce.c dst_gid[i] = cpu_to_le32(qp->dgid.dwords[i]); qp 229 drivers/net/ethernet/qlogic/qed/qed_roce.c static u8 qed_roce_get_qp_tc(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) qp 233 drivers/net/ethernet/qlogic/qed/qed_roce.c if (qp->vlan_id) { qp 234 drivers/net/ethernet/qlogic/qed/qed_roce.c pri = (qp->vlan_id & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; qp 240 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->icid, tc, qp->vlan_id ? "enabled" : "disabled"); qp 246 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp) qp 257 drivers/net/ethernet/qlogic/qed/qed_roce.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 260 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->irq_num_pages = 1; qp 261 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->irq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, qp 263 drivers/net/ethernet/qlogic/qed/qed_roce.c &qp->irq_phys_addr, GFP_KERNEL); qp 264 drivers/net/ethernet/qlogic/qed/qed_roce.c if (!qp->irq) { qp 274 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid; qp 287 drivers/net/ethernet/qlogic/qed/qed_roce.c roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); qp 293 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->incoming_rdma_read_en); qp 297 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->incoming_rdma_write_en); qp 301 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->incoming_atomic_en); qp 305 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->e2e_flow_control_en); qp 308 drivers/net/ethernet/qlogic/qed/qed_roce.c ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG, qp->use_srq); qp 312 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->fmr_and_reserved_lkey); qp 316 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->min_rnr_nak_timer); qp 318 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->max_ird = qp->max_rd_atomic_resp; qp 319 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->traffic_class = qp->traffic_class_tos; qp 320 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->hop_limit = qp->hop_limit_ttl; qp 321 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->irq_num_pages = qp->irq_num_pages; qp 322 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->p_key = cpu_to_le16(qp->pkey); qp 323 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->flow_label = cpu_to_le32(qp->flow_label); qp 324 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); qp 325 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->mtu = cpu_to_le16(qp->mtu); qp 326 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->initial_psn = cpu_to_le32(qp->rq_psn); qp 327 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->pd = cpu_to_le16(qp->pd); qp 328 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->rq_num_pages = cpu_to_le16(qp->rq_num_pages); qp 329 drivers/net/ethernet/qlogic/qed/qed_roce.c DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr); qp 330 drivers/net/ethernet/qlogic/qed/qed_roce.c DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr); qp 331 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); qp 332 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); qp 333 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); qp 334 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); qp 335 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); qp 337 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->rq_cq_id); qp 339 drivers/net/ethernet/qlogic/qed/qed_roce.c tc = qed_roce_get_qp_tc(p_hwfn, qp); qp 344 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->icid, regular_latency_queue - CM_TX_PQ_BASE, qp 351 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->dpi = cpu_to_le16(qp->dpi); qp 353 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); qp 354 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); qp 356 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->udp_src_port = qp->udp_src_port; qp 357 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); qp 358 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id); qp 362 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->stats_queue; qp 368 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->resp_offloaded = true; qp 369 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->cq_prod = 0; qp 372 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_roce_set_real_cid(p_hwfn, qp->icid - qp 380 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->irq_num_pages * RDMA_RING_PAGE_SIZE, qp 381 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->irq, qp->irq_phys_addr); qp 387 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp) qp 398 drivers/net/ethernet/qlogic/qed/qed_roce.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 401 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->orq_num_pages = 1; qp 402 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->orq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, qp 404 drivers/net/ethernet/qlogic/qed/qed_roce.c &qp->orq_phys_addr, GFP_KERNEL); qp 405 drivers/net/ethernet/qlogic/qed/qed_roce.c if (!qp->orq) { qp 415 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid + 1; qp 429 drivers/net/ethernet/qlogic/qed/qed_roce.c roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); qp 435 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->fmr_and_reserved_lkey); qp 438 drivers/net/ethernet/qlogic/qed/qed_roce.c ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP, qp->signal_all); qp 441 drivers/net/ethernet/qlogic/qed/qed_roce.c ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); qp 445 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->rnr_retry_cnt); qp 447 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->max_ord = qp->max_rd_atomic_req; qp 448 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->traffic_class = qp->traffic_class_tos; qp 449 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->hop_limit = qp->hop_limit_ttl; qp 450 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->orq_num_pages = qp->orq_num_pages; qp 451 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->p_key = cpu_to_le16(qp->pkey); qp 452 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->flow_label = cpu_to_le32(qp->flow_label); qp 453 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); qp 454 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); qp 455 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->mtu = cpu_to_le16(qp->mtu); qp 456 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->initial_psn = cpu_to_le32(qp->sq_psn); qp 457 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->pd = cpu_to_le16(qp->pd); qp 458 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->sq_num_pages = cpu_to_le16(qp->sq_num_pages); qp 459 drivers/net/ethernet/qlogic/qed/qed_roce.c DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr); qp 460 drivers/net/ethernet/qlogic/qed/qed_roce.c DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr); qp 461 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); qp 462 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); qp 463 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); qp 464 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); qp 465 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); qp 467 drivers/net/ethernet/qlogic/qed/qed_roce.c cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->sq_cq_id); qp 469 drivers/net/ethernet/qlogic/qed/qed_roce.c tc = qed_roce_get_qp_tc(p_hwfn, qp); qp 474 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->icid, regular_latency_queue - CM_TX_PQ_BASE, qp 481 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->dpi = cpu_to_le16(qp->dpi); qp 483 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); qp 484 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); qp 486 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->udp_src_port = qp->udp_src_port; qp 487 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); qp 489 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->stats_queue; qp 495 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->req_offloaded = true; qp 498 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->icid + 1 - qp 506 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->orq_num_pages * RDMA_RING_PAGE_SIZE, qp 507 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->orq, qp->orq_phys_addr); qp 512 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp, qp 520 drivers/net/ethernet/qlogic/qed/qed_roce.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 522 drivers/net/ethernet/qlogic/qed/qed_roce.c if (move_to_err && !qp->resp_offloaded) qp 527 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid; qp 548 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->incoming_rdma_read_en); qp 552 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->incoming_rdma_write_en); qp 556 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->incoming_atomic_en); qp 560 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->e2e_flow_control_en); qp 589 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->min_rnr_nak_timer); qp 591 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->max_ird = qp->max_rd_atomic_resp; qp 592 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->traffic_class = qp->traffic_class_tos; qp 593 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->hop_limit = qp->hop_limit_ttl; qp 594 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->p_key = cpu_to_le16(qp->pkey); qp 595 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->flow_label = cpu_to_le32(qp->flow_label); qp 596 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->mtu = cpu_to_le16(qp->mtu); qp 597 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); qp 605 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp, qp 614 drivers/net/ethernet/qlogic/qed/qed_roce.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 616 drivers/net/ethernet/qlogic/qed/qed_roce.c if (move_to_err && !(qp->req_offloaded)) qp 621 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid + 1; qp 645 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->sqd_async); qp 677 drivers/net/ethernet/qlogic/qed/qed_roce.c ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); qp 681 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->rnr_retry_cnt); qp 683 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->max_ord = qp->max_rd_atomic_req; qp 684 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->traffic_class = qp->traffic_class_tos; qp 685 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->hop_limit = qp->hop_limit_ttl; qp 686 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->p_key = cpu_to_le16(qp->pkey); qp 687 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->flow_label = cpu_to_le32(qp->flow_label); qp 688 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); qp 689 drivers/net/ethernet/qlogic/qed/qed_roce.c p_ramrod->mtu = cpu_to_le16(qp->mtu); qp 690 drivers/net/ethernet/qlogic/qed/qed_roce.c qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); qp 698 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp, qp 708 drivers/net/ethernet/qlogic/qed/qed_roce.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 709 drivers/net/ethernet/qlogic/qed/qed_roce.c *cq_prod = qp->cq_prod; qp 711 drivers/net/ethernet/qlogic/qed/qed_roce.c if (!qp->resp_offloaded) { qp 717 drivers/net/ethernet/qlogic/qed/qed_roce.c cid = qp->icid - qp 727 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid; qp 759 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->cq_prod = *cq_prod; qp 763 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->irq_num_pages * RDMA_RING_PAGE_SIZE, qp 764 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->irq, qp->irq_phys_addr); qp 766 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->resp_offloaded = false; qp 779 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp) qp 788 drivers/net/ethernet/qlogic/qed/qed_roce.c DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); qp 790 drivers/net/ethernet/qlogic/qed/qed_roce.c if (!qp->req_offloaded) qp 805 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid + 1; qp 824 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->orq_num_pages * RDMA_RING_PAGE_SIZE, qp 825 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->orq, qp->orq_phys_addr); qp 827 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->req_offloaded = false; qp 839 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp, qp 855 drivers/net/ethernet/qlogic/qed/qed_roce.c if ((!(qp->resp_offloaded)) && (!(qp->req_offloaded))) { qp 860 drivers/net/ethernet/qlogic/qed/qed_roce.c out_params->rq_psn = qp->rq_psn; qp 861 drivers/net/ethernet/qlogic/qed/qed_roce.c out_params->sq_psn = qp->sq_psn; qp 862 drivers/net/ethernet/qlogic/qed/qed_roce.c out_params->state = qp->cur_state; qp 868 drivers/net/ethernet/qlogic/qed/qed_roce.c if (!(qp->resp_offloaded)) { qp 887 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid; qp 909 drivers/net/ethernet/qlogic/qed/qed_roce.c if (!(qp->req_offloaded)) { qp 911 drivers/net/ethernet/qlogic/qed/qed_roce.c out_params->sq_psn = qp->sq_psn; qp 915 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->cur_state = QED_ROCE_QP_STATE_ERR; qp 917 drivers/net/ethernet/qlogic/qed/qed_roce.c out_params->state = qp->cur_state; qp 936 drivers/net/ethernet/qlogic/qed/qed_roce.c init_data.cid = qp->icid + 1; qp 962 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->cur_state = QED_ROCE_QP_STATE_ERR; qp 965 drivers/net/ethernet/qlogic/qed/qed_roce.c out_params->state = qp->cur_state; qp 979 drivers/net/ethernet/qlogic/qed/qed_roce.c int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) qp 985 drivers/net/ethernet/qlogic/qed/qed_roce.c if ((qp->cur_state != QED_ROCE_QP_STATE_RESET) && qp 986 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state != QED_ROCE_QP_STATE_ERR) && qp 987 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state != QED_ROCE_QP_STATE_INIT)) { qp 993 drivers/net/ethernet/qlogic/qed/qed_roce.c if (qp->cur_state != QED_ROCE_QP_STATE_RESET) { qp 994 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp, qp 1000 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp); qp 1009 drivers/net/ethernet/qlogic/qed/qed_roce.c struct qed_rdma_qp *qp, qp 1020 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state == QED_ROCE_QP_STATE_RTR)) { qp 1022 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_create_responder(p_hwfn, qp); qp 1025 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { qp 1027 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_create_requester(p_hwfn, qp); qp 1032 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, qp 1036 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { qp 1038 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, qp 1043 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, qp 1047 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { qp 1049 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_requester(p_hwfn, qp, true, false, qp 1053 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { qp 1055 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, qp 1060 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, qp 1064 drivers/net/ethernet/qlogic/qed/qed_roce.c (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { qp 1066 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, qp 1071 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, qp 1075 drivers/net/ethernet/qlogic/qed/qed_roce.c } else if (qp->cur_state == QED_ROCE_QP_STATE_ERR) { qp 1077 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_responder(p_hwfn, qp, true, qp 1082 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, true, qp 1085 drivers/net/ethernet/qlogic/qed/qed_roce.c } else if (qp->cur_state == QED_ROCE_QP_STATE_RESET) { qp 1091 drivers/net/ethernet/qlogic/qed/qed_roce.c qp, qp 1097 drivers/net/ethernet/qlogic/qed/qed_roce.c qp->cq_prod = cq_prod; qp 1099 drivers/net/ethernet/qlogic/qed/qed_roce.c rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp); qp 48 drivers/net/ethernet/qlogic/qed/qed_roce.h int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp); qp 51 drivers/net/ethernet/qlogic/qed/qed_roce.h struct qed_rdma_qp *qp, qp 55 drivers/net/ethernet/qlogic/qed/qed_roce.h struct qed_rdma_qp *qp, qp 2139 drivers/net/ethernet/sun/sunhme.c struct quattro *qp = (struct quattro *) cookie; qp 2143 drivers/net/ethernet/sun/sunhme.c struct net_device *dev = qp->happy_meals[i]; qp 2563 drivers/net/ethernet/sun/sunhme.c struct quattro *qp; qp 2566 drivers/net/ethernet/sun/sunhme.c qp = platform_get_drvdata(op); qp 2567 drivers/net/ethernet/sun/sunhme.c if (qp) qp 2568 drivers/net/ethernet/sun/sunhme.c return qp; qp 2570 drivers/net/ethernet/sun/sunhme.c qp = kmalloc(sizeof(struct quattro), GFP_KERNEL); qp 2571 drivers/net/ethernet/sun/sunhme.c if (qp != NULL) { qp 2575 drivers/net/ethernet/sun/sunhme.c qp->happy_meals[i] = NULL; qp 2577 drivers/net/ethernet/sun/sunhme.c qp->quattro_dev = child; qp 2578 drivers/net/ethernet/sun/sunhme.c qp->next = qfe_sbus_list; qp 2579 drivers/net/ethernet/sun/sunhme.c qfe_sbus_list = qp; qp 2581 drivers/net/ethernet/sun/sunhme.c platform_set_drvdata(op, qp); qp 2583 drivers/net/ethernet/sun/sunhme.c return qp; qp 2592 drivers/net/ethernet/sun/sunhme.c struct quattro *qp; qp 2594 drivers/net/ethernet/sun/sunhme.c for (qp = qfe_sbus_list; qp != NULL; qp = qp->next) { qp 2595 drivers/net/ethernet/sun/sunhme.c struct platform_device *op = qp->quattro_dev; qp 2599 drivers/net/ethernet/sun/sunhme.c if (!qp->happy_meals[qfe_slot]) qp 2608 drivers/net/ethernet/sun/sunhme.c qp); qp 2621 drivers/net/ethernet/sun/sunhme.c struct quattro *qp; qp 2623 drivers/net/ethernet/sun/sunhme.c for (qp = qfe_sbus_list; qp != NULL; qp = qp->next) { qp 2624 drivers/net/ethernet/sun/sunhme.c struct platform_device *op = qp->quattro_dev; qp 2628 drivers/net/ethernet/sun/sunhme.c if (!qp->happy_meals[qfe_slot]) qp 2634 drivers/net/ethernet/sun/sunhme.c free_irq(op->archdata.irqs[0], qp); qp 2643 drivers/net/ethernet/sun/sunhme.c struct quattro *qp; qp 2646 drivers/net/ethernet/sun/sunhme.c for (qp = qfe_pci_list; qp != NULL; qp = qp->next) { qp 2647 drivers/net/ethernet/sun/sunhme.c struct pci_dev *qpdev = qp->quattro_dev; qp 2650 drivers/net/ethernet/sun/sunhme.c return qp; qp 2652 drivers/net/ethernet/sun/sunhme.c qp = kmalloc(sizeof(struct quattro), GFP_KERNEL); qp 2653 drivers/net/ethernet/sun/sunhme.c if (qp != NULL) { qp 2657 drivers/net/ethernet/sun/sunhme.c qp->happy_meals[i] = NULL; qp 2659 drivers/net/ethernet/sun/sunhme.c qp->quattro_dev = bdev; qp 2660 drivers/net/ethernet/sun/sunhme.c qp->next = qfe_pci_list; qp 2661 drivers/net/ethernet/sun/sunhme.c qfe_pci_list = qp; qp 2664 drivers/net/ethernet/sun/sunhme.c qp->nranges = 0; qp 2666 drivers/net/ethernet/sun/sunhme.c return qp; qp 2685 drivers/net/ethernet/sun/sunhme.c struct quattro *qp = NULL; qp 2698 drivers/net/ethernet/sun/sunhme.c qp = quattro_sbus_find(op); qp 2699 drivers/net/ethernet/sun/sunhme.c if (qp == NULL) qp 2702 drivers/net/ethernet/sun/sunhme.c if (qp->happy_meals[qfe_slot] == NULL) qp 2748 drivers/net/ethernet/sun/sunhme.c if (qp != NULL) { qp 2749 drivers/net/ethernet/sun/sunhme.c hp->qfe_parent = qp; qp 2751 drivers/net/ethernet/sun/sunhme.c qp->happy_meals[qfe_slot] = dev; qp 2799 drivers/net/ethernet/sun/sunhme.c if (qp != NULL) qp 2888 drivers/net/ethernet/sun/sunhme.c if (qp) qp 2889 drivers/net/ethernet/sun/sunhme.c qp->happy_meals[qfe_slot] = NULL; qp 2987 drivers/net/ethernet/sun/sunhme.c struct quattro *qp = NULL; qp 3017 drivers/net/ethernet/sun/sunhme.c qp = quattro_pci_find(pdev); qp 3018 drivers/net/ethernet/sun/sunhme.c if (qp == NULL) qp 3021 drivers/net/ethernet/sun/sunhme.c if (qp->happy_meals[qfe_slot] == NULL) qp 3043 drivers/net/ethernet/sun/sunhme.c if (qp != NULL) { qp 3044 drivers/net/ethernet/sun/sunhme.c hp->qfe_parent = qp; qp 3046 drivers/net/ethernet/sun/sunhme.c qp->happy_meals[qfe_slot] = dev; qp 3114 drivers/net/ethernet/sun/sunhme.c if (qp != NULL) qp 3173 drivers/net/ethernet/sun/sunhme.c struct pci_dev *qpdev = qp->quattro_dev; qp 3207 drivers/net/ethernet/sun/sunhme.c if (qp != NULL) qp 3208 drivers/net/ethernet/sun/sunhme.c qp->happy_meals[qfe_slot] = NULL; qp 938 drivers/net/ethernet/sun/sunqe.c struct sunqe *qp = platform_get_drvdata(op); qp 939 drivers/net/ethernet/sun/sunqe.c struct net_device *net_dev = qp->dev; qp 943 drivers/net/ethernet/sun/sunqe.c of_iounmap(&op->resource[0], qp->qcregs, CREG_REG_SIZE); qp 944 drivers/net/ethernet/sun/sunqe.c of_iounmap(&op->resource[1], qp->mregs, MREGS_REG_SIZE); qp 946 drivers/net/ethernet/sun/sunqe.c qp->qe_block, qp->qblock_dvma); qp 948 drivers/net/ethernet/sun/sunqe.c qp->buffers, qp->buffers_dvma); qp 299 drivers/net/ethernet/sun/sunqe.h #define TX_BUFFS_AVAIL(qp) \ qp 300 drivers/net/ethernet/sun/sunqe.h (((qp)->tx_old <= (qp)->tx_new) ? \ qp 301 drivers/net/ethernet/sun/sunqe.h (qp)->tx_old + (TX_RING_SIZE - 1) - (qp)->tx_new : \ qp 302 drivers/net/ethernet/sun/sunqe.h (qp)->tx_old - (qp)->tx_new - 1) qp 76 drivers/net/ntb_netdev.c struct ntb_transport_qp *qp; qp 89 drivers/net/ntb_netdev.c ntb_transport_link_query(dev->qp)); qp 92 drivers/net/ntb_netdev.c if (ntb_transport_link_query(dev->qp)) qp 99 drivers/net/ntb_netdev.c static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, qp 138 drivers/net/ntb_netdev.c rc = ntb_transport_rx_enqueue(qp, skb, skb->data, ndev->mtu + ETH_HLEN); qp 147 drivers/net/ntb_netdev.c struct ntb_transport_qp *qp, int size) qp 157 drivers/net/ntb_netdev.c if (likely(ntb_transport_tx_free_entry(qp) < size)) { qp 167 drivers/net/ntb_netdev.c struct ntb_transport_qp *qp, int size) qp 170 drivers/net/ntb_netdev.c (ntb_transport_tx_free_entry(qp) >= size)) qp 173 drivers/net/ntb_netdev.c return __ntb_netdev_maybe_stop_tx(ndev, qp, size); qp 176 drivers/net/ntb_netdev.c static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data, qp 197 drivers/net/ntb_netdev.c if (ntb_transport_tx_free_entry(dev->qp) >= tx_start) { qp 213 drivers/net/ntb_netdev.c ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop); qp 215 drivers/net/ntb_netdev.c rc = ntb_transport_tx_enqueue(dev->qp, skb, skb->data, skb->len); qp 220 drivers/net/ntb_netdev.c ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop); qp 235 drivers/net/ntb_netdev.c if (ntb_transport_tx_free_entry(dev->qp) < tx_stop) { qp 261 drivers/net/ntb_netdev.c rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data, qp 272 drivers/net/ntb_netdev.c ntb_transport_link_up(dev->qp); qp 278 drivers/net/ntb_netdev.c while ((skb = ntb_transport_rx_remove(dev->qp, &len))) qp 289 drivers/net/ntb_netdev.c ntb_transport_link_down(dev->qp); qp 291 drivers/net/ntb_netdev.c while ((skb = ntb_transport_rx_remove(dev->qp, &len))) qp 305 drivers/net/ntb_netdev.c if (new_mtu > ntb_transport_max_size(dev->qp) - ETH_HLEN) qp 314 drivers/net/ntb_netdev.c ntb_transport_link_down(dev->qp); qp 319 drivers/net/ntb_netdev.c for (i = 0; (skb = ntb_transport_rx_remove(dev->qp, &len)); i++) qp 329 drivers/net/ntb_netdev.c rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data, qp 340 drivers/net/ntb_netdev.c ntb_transport_link_up(dev->qp); qp 345 drivers/net/ntb_netdev.c ntb_transport_link_down(dev->qp); qp 347 drivers/net/ntb_netdev.c while ((skb = ntb_transport_rx_remove(dev->qp, &len))) qp 439 drivers/net/ntb_netdev.c dev->qp = ntb_transport_create_queue(ndev, client_dev, qp 441 drivers/net/ntb_netdev.c if (!dev->qp) { qp 446 drivers/net/ntb_netdev.c ndev->mtu = ntb_transport_max_size(dev->qp) - ETH_HLEN; qp 457 drivers/net/ntb_netdev.c ntb_transport_free_queue(dev->qp); qp 469 drivers/net/ntb_netdev.c ntb_transport_free_queue(dev->qp); qp 479 drivers/net/virtio_net.c unsigned int qp; qp 481 drivers/net/virtio_net.c qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id(); qp 482 drivers/net/virtio_net.c return &vi->sq[qp]; qp 120 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp; qp 148 drivers/ntb/ntb_transport.c void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, qp 160 drivers/ntb/ntb_transport.c void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, qp 272 drivers/ntb/ntb_transport.c #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) qp 279 drivers/ntb/ntb_transport.c static int ntb_async_tx_submit(struct ntb_transport_qp *qp, qp 468 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp; qp 472 drivers/ntb/ntb_transport.c qp = filp->private_data; qp 474 drivers/ntb/ntb_transport.c if (!qp || !qp->link_is_up) qp 487 drivers/ntb/ntb_transport.c "rx_bytes - \t%llu\n", qp->rx_bytes); qp 489 drivers/ntb/ntb_transport.c "rx_pkts - \t%llu\n", qp->rx_pkts); qp 491 drivers/ntb/ntb_transport.c "rx_memcpy - \t%llu\n", qp->rx_memcpy); qp 493 drivers/ntb/ntb_transport.c "rx_async - \t%llu\n", qp->rx_async); qp 495 drivers/ntb/ntb_transport.c "rx_ring_empty - %llu\n", qp->rx_ring_empty); qp 497 drivers/ntb/ntb_transport.c "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); qp 499 drivers/ntb/ntb_transport.c "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); qp 501 drivers/ntb/ntb_transport.c "rx_err_ver - \t%llu\n", qp->rx_err_ver); qp 503 drivers/ntb/ntb_transport.c "rx_buff - \t0x%p\n", qp->rx_buff); qp 505 drivers/ntb/ntb_transport.c "rx_index - \t%u\n", qp->rx_index); qp 507 drivers/ntb/ntb_transport.c "rx_max_entry - \t%u\n", qp->rx_max_entry); qp 509 drivers/ntb/ntb_transport.c "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry); qp 512 drivers/ntb/ntb_transport.c "tx_bytes - \t%llu\n", qp->tx_bytes); qp 514 drivers/ntb/ntb_transport.c "tx_pkts - \t%llu\n", qp->tx_pkts); qp 516 drivers/ntb/ntb_transport.c "tx_memcpy - \t%llu\n", qp->tx_memcpy); qp 518 drivers/ntb/ntb_transport.c "tx_async - \t%llu\n", qp->tx_async); qp 520 drivers/ntb/ntb_transport.c "tx_ring_full - \t%llu\n", qp->tx_ring_full); qp 522 drivers/ntb/ntb_transport.c "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); qp 524 drivers/ntb/ntb_transport.c "tx_mw - \t0x%p\n", qp->tx_mw); qp 526 drivers/ntb/ntb_transport.c "tx_index (H) - \t%u\n", qp->tx_index); qp 529 drivers/ntb/ntb_transport.c qp->remote_rx_info->entry); qp 531 drivers/ntb/ntb_transport.c "tx_max_entry - \t%u\n", qp->tx_max_entry); qp 534 drivers/ntb/ntb_transport.c ntb_transport_tx_free_entry(qp)); qp 540 drivers/ntb/ntb_transport.c qp->tx_dma_chan ? "Yes" : "No"); qp 543 drivers/ntb/ntb_transport.c qp->rx_dma_chan ? "Yes" : "No"); qp 546 drivers/ntb/ntb_transport.c qp->link_is_up ? "Up" : "Down"); qp 618 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; qp 642 drivers/ntb/ntb_transport.c qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); qp 645 drivers/ntb/ntb_transport.c qp->remote_rx_info = qp->rx_buff + rx_size; qp 648 drivers/ntb/ntb_transport.c qp->rx_max_frame = min(transport_mtu, rx_size / 2); qp 649 drivers/ntb/ntb_transport.c qp->rx_max_entry = rx_size / qp->rx_max_frame; qp 650 drivers/ntb/ntb_transport.c qp->rx_index = 0; qp 658 drivers/ntb/ntb_transport.c for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { qp 663 drivers/ntb/ntb_transport.c entry->qp = qp; qp 664 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, qp 665 drivers/ntb/ntb_transport.c &qp->rx_free_q); qp 666 drivers/ntb/ntb_transport.c qp->rx_alloc_entry++; qp 669 drivers/ntb/ntb_transport.c qp->remote_rx_info->entry = qp->rx_max_entry - 1; qp 672 drivers/ntb/ntb_transport.c for (i = 0; i < qp->rx_max_entry; i++) { qp 673 drivers/ntb/ntb_transport.c void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - qp 678 drivers/ntb/ntb_transport.c qp->rx_pkts = 0; qp 679 drivers/ntb/ntb_transport.c qp->tx_pkts = 0; qp 680 drivers/ntb/ntb_transport.c qp->tx_index = 0; qp 687 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = dev; qp 689 drivers/ntb/ntb_transport.c tasklet_schedule(&qp->rxc_db_work); qp 697 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; qp 706 drivers/ntb/ntb_transport.c qp->peer_msi_desc.addr_offset = qp 707 drivers/ntb/ntb_transport.c ntb_peer_spad_read(qp->ndev, PIDX, spad); qp 708 drivers/ntb/ntb_transport.c qp->peer_msi_desc.data = qp 709 drivers/ntb/ntb_transport.c ntb_peer_spad_read(qp->ndev, PIDX, spad + 1); qp 711 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, "QP%d Peer MSI addr=%x data=%x\n", qp 712 drivers/ntb/ntb_transport.c qp_num, qp->peer_msi_desc.addr_offset, qp->peer_msi_desc.data); qp 714 drivers/ntb/ntb_transport.c if (qp->peer_msi_desc.addr_offset) { qp 715 drivers/ntb/ntb_transport.c qp->use_msi = true; qp 716 drivers/ntb/ntb_transport.c dev_info(&qp->ndev->pdev->dev, qp 724 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; qp 732 drivers/ntb/ntb_transport.c dev_warn_once(&qp->ndev->pdev->dev, qp 737 drivers/ntb/ntb_transport.c ntb_spad_write(qp->ndev, spad, 0); qp 738 drivers/ntb/ntb_transport.c ntb_spad_write(qp->ndev, spad + 1, 0); qp 740 drivers/ntb/ntb_transport.c if (!qp->msi_irq) { qp 741 drivers/ntb/ntb_transport.c qp->msi_irq = ntbm_msi_request_irq(qp->ndev, ntb_transport_isr, qp 742 drivers/ntb/ntb_transport.c KBUILD_MODNAME, qp, qp 743 drivers/ntb/ntb_transport.c &qp->msi_desc); qp 744 drivers/ntb/ntb_transport.c if (qp->msi_irq < 0) { qp 745 drivers/ntb/ntb_transport.c dev_warn(&qp->ndev->pdev->dev, qp 752 drivers/ntb/ntb_transport.c rc = ntb_spad_write(qp->ndev, spad, qp->msi_desc.addr_offset); qp 756 drivers/ntb/ntb_transport.c rc = ntb_spad_write(qp->ndev, spad + 1, qp->msi_desc.data); qp 760 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, "QP%d MSI %d addr=%x data=%x\n", qp 761 drivers/ntb/ntb_transport.c qp_num, qp->msi_irq, qp->msi_desc.addr_offset, qp 762 drivers/ntb/ntb_transport.c qp->msi_desc.data); qp 767 drivers/ntb/ntb_transport.c devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp); qp 914 drivers/ntb/ntb_transport.c static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) qp 916 drivers/ntb/ntb_transport.c qp->link_is_up = false; qp 917 drivers/ntb/ntb_transport.c qp->active = false; qp 919 drivers/ntb/ntb_transport.c qp->tx_index = 0; qp 920 drivers/ntb/ntb_transport.c qp->rx_index = 0; qp 921 drivers/ntb/ntb_transport.c qp->rx_bytes = 0; qp 922 drivers/ntb/ntb_transport.c qp->rx_pkts = 0; qp 923 drivers/ntb/ntb_transport.c qp->rx_ring_empty = 0; qp 924 drivers/ntb/ntb_transport.c qp->rx_err_no_buf = 0; qp 925 drivers/ntb/ntb_transport.c qp->rx_err_oflow = 0; qp 926 drivers/ntb/ntb_transport.c qp->rx_err_ver = 0; qp 927 drivers/ntb/ntb_transport.c qp->rx_memcpy = 0; qp 928 drivers/ntb/ntb_transport.c qp->rx_async = 0; qp 929 drivers/ntb/ntb_transport.c qp->tx_bytes = 0; qp 930 drivers/ntb/ntb_transport.c qp->tx_pkts = 0; qp 931 drivers/ntb/ntb_transport.c qp->tx_ring_full = 0; qp 932 drivers/ntb/ntb_transport.c qp->tx_err_no_buf = 0; qp 933 drivers/ntb/ntb_transport.c qp->tx_memcpy = 0; qp 934 drivers/ntb/ntb_transport.c qp->tx_async = 0; qp 937 drivers/ntb/ntb_transport.c static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) qp 939 drivers/ntb/ntb_transport.c struct ntb_transport_ctx *nt = qp->transport; qp 942 drivers/ntb/ntb_transport.c dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); qp 944 drivers/ntb/ntb_transport.c cancel_delayed_work_sync(&qp->link_work); qp 945 drivers/ntb/ntb_transport.c ntb_qp_link_down_reset(qp); qp 947 drivers/ntb/ntb_transport.c if (qp->event_handler) qp 948 drivers/ntb/ntb_transport.c qp->event_handler(qp->cb_data, qp->link_is_up); qp 953 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = container_of(work, qp 956 drivers/ntb/ntb_transport.c struct ntb_transport_ctx *nt = qp->transport; qp 958 drivers/ntb/ntb_transport.c ntb_qp_link_cleanup(qp); qp 961 drivers/ntb/ntb_transport.c schedule_delayed_work(&qp->link_work, qp 965 drivers/ntb/ntb_transport.c static void ntb_qp_link_down(struct ntb_transport_qp *qp) qp 967 drivers/ntb/ntb_transport.c schedule_work(&qp->link_cleanup); qp 972 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp; qp 981 drivers/ntb/ntb_transport.c qp = &nt->qp_vec[i]; qp 982 drivers/ntb/ntb_transport.c ntb_qp_link_cleanup(qp); qp 983 drivers/ntb/ntb_transport.c cancel_work_sync(&qp->link_cleanup); qp 984 drivers/ntb/ntb_transport.c cancel_delayed_work_sync(&qp->link_work); qp 1099 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = &nt->qp_vec[i]; qp 1104 drivers/ntb/ntb_transport.c if (qp->client_ready) qp 1105 drivers/ntb/ntb_transport.c schedule_delayed_work(&qp->link_work, 0); qp 1126 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = container_of(work, qp 1129 drivers/ntb/ntb_transport.c struct pci_dev *pdev = qp->ndev->pdev; qp 1130 drivers/ntb/ntb_transport.c struct ntb_transport_ctx *nt = qp->transport; qp 1137 drivers/ntb/ntb_transport.c ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num)); qp 1143 drivers/ntb/ntb_transport.c if (val & BIT(qp->qp_num)) { qp 1144 drivers/ntb/ntb_transport.c dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); qp 1145 drivers/ntb/ntb_transport.c qp->link_is_up = true; qp 1146 drivers/ntb/ntb_transport.c qp->active = true; qp 1148 drivers/ntb/ntb_transport.c if (qp->event_handler) qp 1149 drivers/ntb/ntb_transport.c qp->event_handler(qp->cb_data, qp->link_is_up); qp 1151 drivers/ntb/ntb_transport.c if (qp->active) qp 1152 drivers/ntb/ntb_transport.c tasklet_schedule(&qp->rxc_db_work); qp 1154 drivers/ntb/ntb_transport.c schedule_delayed_work(&qp->link_work, qp 1161 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp; qp 1173 drivers/ntb/ntb_transport.c qp = &nt->qp_vec[qp_num]; qp 1174 drivers/ntb/ntb_transport.c qp->qp_num = qp_num; qp 1175 drivers/ntb/ntb_transport.c qp->transport = nt; qp 1176 drivers/ntb/ntb_transport.c qp->ndev = nt->ndev; qp 1177 drivers/ntb/ntb_transport.c qp->client_ready = false; qp 1178 drivers/ntb/ntb_transport.c qp->event_handler = NULL; qp 1179 drivers/ntb/ntb_transport.c ntb_qp_link_down_reset(qp); qp 1195 drivers/ntb/ntb_transport.c qp->tx_mw_size = tx_size; qp 1196 drivers/ntb/ntb_transport.c qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; qp 1197 drivers/ntb/ntb_transport.c if (!qp->tx_mw) qp 1200 drivers/ntb/ntb_transport.c qp->tx_mw_phys = mw_base + qp_offset; qp 1201 drivers/ntb/ntb_transport.c if (!qp->tx_mw_phys) qp 1205 drivers/ntb/ntb_transport.c qp->rx_info = qp->tx_mw + tx_size; qp 1208 drivers/ntb/ntb_transport.c qp->tx_max_frame = min(transport_mtu, tx_size / 2); qp 1209 drivers/ntb/ntb_transport.c qp->tx_max_entry = tx_size / qp->tx_max_frame; qp 1215 drivers/ntb/ntb_transport.c qp->debugfs_dir = debugfs_create_dir(debugfs_name, qp 1218 drivers/ntb/ntb_transport.c qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, qp 1219 drivers/ntb/ntb_transport.c qp->debugfs_dir, qp, qp 1222 drivers/ntb/ntb_transport.c qp->debugfs_dir = NULL; qp 1223 drivers/ntb/ntb_transport.c qp->debugfs_stats = NULL; qp 1226 drivers/ntb/ntb_transport.c INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); qp 1227 drivers/ntb/ntb_transport.c INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); qp 1229 drivers/ntb/ntb_transport.c spin_lock_init(&qp->ntb_rx_q_lock); qp 1230 drivers/ntb/ntb_transport.c spin_lock_init(&qp->ntb_tx_free_q_lock); qp 1232 drivers/ntb/ntb_transport.c INIT_LIST_HEAD(&qp->rx_post_q); qp 1233 drivers/ntb/ntb_transport.c INIT_LIST_HEAD(&qp->rx_pend_q); qp 1234 drivers/ntb/ntb_transport.c INIT_LIST_HEAD(&qp->rx_free_q); qp 1235 drivers/ntb/ntb_transport.c INIT_LIST_HEAD(&qp->tx_free_q); qp 1237 drivers/ntb/ntb_transport.c tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, qp 1238 drivers/ntb/ntb_transport.c (unsigned long)qp); qp 1406 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp; qp 1418 drivers/ntb/ntb_transport.c qp = &nt->qp_vec[i]; qp 1420 drivers/ntb/ntb_transport.c ntb_transport_free_queue(qp); qp 1421 drivers/ntb/ntb_transport.c debugfs_remove_recursive(qp->debugfs_dir); qp 1439 drivers/ntb/ntb_transport.c static void ntb_complete_rxc(struct ntb_transport_qp *qp) qp 1446 drivers/ntb/ntb_transport.c spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); qp 1448 drivers/ntb/ntb_transport.c while (!list_empty(&qp->rx_post_q)) { qp 1449 drivers/ntb/ntb_transport.c entry = list_first_entry(&qp->rx_post_q, qp 1455 drivers/ntb/ntb_transport.c iowrite32(entry->rx_index, &qp->rx_info->entry); qp 1460 drivers/ntb/ntb_transport.c list_move_tail(&entry->entry, &qp->rx_free_q); qp 1462 drivers/ntb/ntb_transport.c spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); qp 1464 drivers/ntb/ntb_transport.c if (qp->rx_handler && qp->client_ready) qp 1465 drivers/ntb/ntb_transport.c qp->rx_handler(qp, qp->cb_data, cb_data, len); qp 1467 drivers/ntb/ntb_transport.c spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); qp 1470 drivers/ntb/ntb_transport.c spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); qp 1489 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = entry->qp; qp 1490 drivers/ntb/ntb_transport.c void *offset = qp->rx_buff + qp->rx_max_frame * qp 1491 drivers/ntb/ntb_transport.c qp->rx_index; qp 1494 drivers/ntb/ntb_transport.c qp->rx_memcpy++; qp 1506 drivers/ntb/ntb_transport.c ntb_complete_rxc(entry->qp); qp 1525 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = entry->qp; qp 1526 drivers/ntb/ntb_transport.c struct dma_chan *chan = qp->rx_dma_chan; qp 1576 drivers/ntb/ntb_transport.c qp->last_cookie = cookie; qp 1578 drivers/ntb/ntb_transport.c qp->rx_async++; qp 1592 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = entry->qp; qp 1593 drivers/ntb/ntb_transport.c struct dma_chan *chan = qp->rx_dma_chan; qp 1607 drivers/ntb/ntb_transport.c qp->rx_async++; qp 1613 drivers/ntb/ntb_transport.c qp->rx_memcpy++; qp 1616 drivers/ntb/ntb_transport.c static int ntb_process_rxc(struct ntb_transport_qp *qp) qp 1622 drivers/ntb/ntb_transport.c offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; qp 1623 drivers/ntb/ntb_transport.c hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); qp 1625 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", qp 1626 drivers/ntb/ntb_transport.c qp->qp_num, hdr->ver, hdr->len, hdr->flags); qp 1629 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); qp 1630 drivers/ntb/ntb_transport.c qp->rx_ring_empty++; qp 1635 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); qp 1636 drivers/ntb/ntb_transport.c ntb_qp_link_down(qp); qp 1641 drivers/ntb/ntb_transport.c if (hdr->ver != (u32)qp->rx_pkts) { qp 1642 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, qp 1644 drivers/ntb/ntb_transport.c qp->rx_pkts, hdr->ver); qp 1645 drivers/ntb/ntb_transport.c qp->rx_err_ver++; qp 1649 drivers/ntb/ntb_transport.c entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); qp 1651 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); qp 1652 drivers/ntb/ntb_transport.c qp->rx_err_no_buf++; qp 1657 drivers/ntb/ntb_transport.c entry->rx_index = qp->rx_index; qp 1660 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, qp 1663 drivers/ntb/ntb_transport.c qp->rx_err_oflow++; qp 1668 drivers/ntb/ntb_transport.c ntb_complete_rxc(qp); qp 1670 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, qp 1672 drivers/ntb/ntb_transport.c qp->rx_index, hdr->ver, hdr->len, entry->len); qp 1674 drivers/ntb/ntb_transport.c qp->rx_bytes += hdr->len; qp 1675 drivers/ntb/ntb_transport.c qp->rx_pkts++; qp 1682 drivers/ntb/ntb_transport.c qp->rx_index++; qp 1683 drivers/ntb/ntb_transport.c qp->rx_index %= qp->rx_max_entry; qp 1690 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = (void *)data; qp 1693 drivers/ntb/ntb_transport.c dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", qp 1694 drivers/ntb/ntb_transport.c __func__, qp->qp_num); qp 1699 drivers/ntb/ntb_transport.c for (i = 0; i < qp->rx_max_entry; i++) { qp 1700 drivers/ntb/ntb_transport.c rc = ntb_process_rxc(qp); qp 1705 drivers/ntb/ntb_transport.c if (i && qp->rx_dma_chan) qp 1706 drivers/ntb/ntb_transport.c dma_async_issue_pending(qp->rx_dma_chan); qp 1708 drivers/ntb/ntb_transport.c if (i == qp->rx_max_entry) { qp 1710 drivers/ntb/ntb_transport.c if (qp->active) qp 1711 drivers/ntb/ntb_transport.c tasklet_schedule(&qp->rxc_db_work); qp 1712 drivers/ntb/ntb_transport.c } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { qp 1714 drivers/ntb/ntb_transport.c ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); qp 1716 drivers/ntb/ntb_transport.c ntb_db_read(qp->ndev); qp 1722 drivers/ntb/ntb_transport.c if (qp->active) qp 1723 drivers/ntb/ntb_transport.c tasklet_schedule(&qp->rxc_db_work); qp 1731 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp = entry->qp; qp 1746 drivers/ntb/ntb_transport.c qp->tx_mw + qp->tx_max_frame * qp 1751 drivers/ntb/ntb_transport.c qp->tx_memcpy++; qp 1763 drivers/ntb/ntb_transport.c if (qp->use_msi) qp 1764 drivers/ntb/ntb_transport.c ntb_msi_peer_trigger(qp->ndev, PIDX, &qp->peer_msi_desc); qp 1766 drivers/ntb/ntb_transport.c ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); qp 1773 drivers/ntb/ntb_transport.c qp->tx_bytes += entry->len; qp 1775 drivers/ntb/ntb_transport.c if (qp->tx_handler) qp 1776 drivers/ntb/ntb_transport.c qp->tx_handler(qp, qp->cb_data, entry->cb_data, qp 1780 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); qp 1801 drivers/ntb/ntb_transport.c static int ntb_async_tx_submit(struct ntb_transport_qp *qp, qp 1805 drivers/ntb/ntb_transport.c struct dma_chan *chan = qp->tx_dma_chan; qp 1815 drivers/ntb/ntb_transport.c dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index; qp 1860 drivers/ntb/ntb_transport.c static void ntb_async_tx(struct ntb_transport_qp *qp, qp 1864 drivers/ntb/ntb_transport.c struct dma_chan *chan = qp->tx_dma_chan; qp 1868 drivers/ntb/ntb_transport.c entry->tx_index = qp->tx_index; qp 1869 drivers/ntb/ntb_transport.c offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; qp 1870 drivers/ntb/ntb_transport.c hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); qp 1874 drivers/ntb/ntb_transport.c iowrite32((u32)qp->tx_pkts, &hdr->ver); qp 1882 drivers/ntb/ntb_transport.c res = ntb_async_tx_submit(qp, entry); qp 1887 drivers/ntb/ntb_transport.c qp->tx_async++; qp 1893 drivers/ntb/ntb_transport.c qp->tx_memcpy++; qp 1896 drivers/ntb/ntb_transport.c static int ntb_process_tx(struct ntb_transport_qp *qp, qp 1899 drivers/ntb/ntb_transport.c if (qp->tx_index == qp->remote_rx_info->entry) { qp 1900 drivers/ntb/ntb_transport.c qp->tx_ring_full++; qp 1904 drivers/ntb/ntb_transport.c if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { qp 1905 drivers/ntb/ntb_transport.c if (qp->tx_handler) qp 1906 drivers/ntb/ntb_transport.c qp->tx_handler(qp, qp->cb_data, NULL, -EIO); qp 1908 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, qp 1909 drivers/ntb/ntb_transport.c &qp->tx_free_q); qp 1913 drivers/ntb/ntb_transport.c ntb_async_tx(qp, entry); qp 1915 drivers/ntb/ntb_transport.c qp->tx_index++; qp 1916 drivers/ntb/ntb_transport.c qp->tx_index %= qp->tx_max_entry; qp 1918 drivers/ntb/ntb_transport.c qp->tx_pkts++; qp 1923 drivers/ntb/ntb_transport.c static void ntb_send_link_down(struct ntb_transport_qp *qp) qp 1925 drivers/ntb/ntb_transport.c struct pci_dev *pdev = qp->ndev->pdev; qp 1929 drivers/ntb/ntb_transport.c if (!qp->link_is_up) qp 1932 drivers/ntb/ntb_transport.c dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); qp 1935 drivers/ntb/ntb_transport.c entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); qp 1949 drivers/ntb/ntb_transport.c rc = ntb_process_tx(qp, entry); qp 1952 drivers/ntb/ntb_transport.c qp->qp_num); qp 1954 drivers/ntb/ntb_transport.c ntb_qp_link_down_reset(qp); qp 1984 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp; qp 2004 drivers/ntb/ntb_transport.c qp = &nt->qp_vec[free_queue]; qp 2005 drivers/ntb/ntb_transport.c qp_bit = BIT_ULL(qp->qp_num); qp 2009 drivers/ntb/ntb_transport.c qp->cb_data = data; qp 2010 drivers/ntb/ntb_transport.c qp->rx_handler = handlers->rx_handler; qp 2011 drivers/ntb/ntb_transport.c qp->tx_handler = handlers->tx_handler; qp 2012 drivers/ntb/ntb_transport.c qp->event_handler = handlers->event_handler; qp 2018 drivers/ntb/ntb_transport.c qp->tx_dma_chan = qp 2021 drivers/ntb/ntb_transport.c if (!qp->tx_dma_chan) qp 2024 drivers/ntb/ntb_transport.c qp->rx_dma_chan = qp 2027 drivers/ntb/ntb_transport.c if (!qp->rx_dma_chan) qp 2030 drivers/ntb/ntb_transport.c qp->tx_dma_chan = NULL; qp 2031 drivers/ntb/ntb_transport.c qp->rx_dma_chan = NULL; qp 2034 drivers/ntb/ntb_transport.c qp->tx_mw_dma_addr = 0; qp 2035 drivers/ntb/ntb_transport.c if (qp->tx_dma_chan) { qp 2036 drivers/ntb/ntb_transport.c qp->tx_mw_dma_addr = qp 2037 drivers/ntb/ntb_transport.c dma_map_resource(qp->tx_dma_chan->device->dev, qp 2038 drivers/ntb/ntb_transport.c qp->tx_mw_phys, qp->tx_mw_size, qp 2040 drivers/ntb/ntb_transport.c if (dma_mapping_error(qp->tx_dma_chan->device->dev, qp 2041 drivers/ntb/ntb_transport.c qp->tx_mw_dma_addr)) { qp 2042 drivers/ntb/ntb_transport.c qp->tx_mw_dma_addr = 0; qp 2048 drivers/ntb/ntb_transport.c qp->tx_dma_chan ? "DMA" : "CPU"); qp 2051 drivers/ntb/ntb_transport.c qp->rx_dma_chan ? "DMA" : "CPU"); qp 2058 drivers/ntb/ntb_transport.c entry->qp = qp; qp 2059 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, qp 2060 drivers/ntb/ntb_transport.c &qp->rx_free_q); qp 2062 drivers/ntb/ntb_transport.c qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; qp 2064 drivers/ntb/ntb_transport.c for (i = 0; i < qp->tx_max_entry; i++) { qp 2069 drivers/ntb/ntb_transport.c entry->qp = qp; qp 2070 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, qp 2071 drivers/ntb/ntb_transport.c &qp->tx_free_q); qp 2074 drivers/ntb/ntb_transport.c ntb_db_clear(qp->ndev, qp_bit); qp 2075 drivers/ntb/ntb_transport.c ntb_db_clear_mask(qp->ndev, qp_bit); qp 2077 drivers/ntb/ntb_transport.c dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); qp 2079 drivers/ntb/ntb_transport.c return qp; qp 2082 drivers/ntb/ntb_transport.c while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) qp 2085 drivers/ntb/ntb_transport.c qp->rx_alloc_entry = 0; qp 2086 drivers/ntb/ntb_transport.c while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) qp 2088 drivers/ntb/ntb_transport.c if (qp->tx_mw_dma_addr) qp 2089 drivers/ntb/ntb_transport.c dma_unmap_resource(qp->tx_dma_chan->device->dev, qp 2090 drivers/ntb/ntb_transport.c qp->tx_mw_dma_addr, qp->tx_mw_size, qp 2092 drivers/ntb/ntb_transport.c if (qp->tx_dma_chan) qp 2093 drivers/ntb/ntb_transport.c dma_release_channel(qp->tx_dma_chan); qp 2094 drivers/ntb/ntb_transport.c if (qp->rx_dma_chan) qp 2095 drivers/ntb/ntb_transport.c dma_release_channel(qp->rx_dma_chan); qp 2108 drivers/ntb/ntb_transport.c void ntb_transport_free_queue(struct ntb_transport_qp *qp) qp 2114 drivers/ntb/ntb_transport.c if (!qp) qp 2117 drivers/ntb/ntb_transport.c pdev = qp->ndev->pdev; qp 2119 drivers/ntb/ntb_transport.c qp->active = false; qp 2121 drivers/ntb/ntb_transport.c if (qp->tx_dma_chan) { qp 2122 drivers/ntb/ntb_transport.c struct dma_chan *chan = qp->tx_dma_chan; qp 2126 drivers/ntb/ntb_transport.c qp->tx_dma_chan = NULL; qp 2131 drivers/ntb/ntb_transport.c dma_sync_wait(chan, qp->last_cookie); qp 2135 drivers/ntb/ntb_transport.c qp->tx_mw_dma_addr, qp->tx_mw_size, qp 2141 drivers/ntb/ntb_transport.c if (qp->rx_dma_chan) { qp 2142 drivers/ntb/ntb_transport.c struct dma_chan *chan = qp->rx_dma_chan; qp 2146 drivers/ntb/ntb_transport.c qp->rx_dma_chan = NULL; qp 2151 drivers/ntb/ntb_transport.c dma_sync_wait(chan, qp->last_cookie); qp 2156 drivers/ntb/ntb_transport.c qp_bit = BIT_ULL(qp->qp_num); qp 2158 drivers/ntb/ntb_transport.c ntb_db_set_mask(qp->ndev, qp_bit); qp 2159 drivers/ntb/ntb_transport.c tasklet_kill(&qp->rxc_db_work); qp 2161 drivers/ntb/ntb_transport.c cancel_delayed_work_sync(&qp->link_work); qp 2163 drivers/ntb/ntb_transport.c qp->cb_data = NULL; qp 2164 drivers/ntb/ntb_transport.c qp->rx_handler = NULL; qp 2165 drivers/ntb/ntb_transport.c qp->tx_handler = NULL; qp 2166 drivers/ntb/ntb_transport.c qp->event_handler = NULL; qp 2168 drivers/ntb/ntb_transport.c while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) qp 2171 drivers/ntb/ntb_transport.c while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { qp 2176 drivers/ntb/ntb_transport.c while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { qp 2181 drivers/ntb/ntb_transport.c while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) qp 2184 drivers/ntb/ntb_transport.c qp->transport->qp_bitmap_free |= qp_bit; qp 2186 drivers/ntb/ntb_transport.c dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); qp 2200 drivers/ntb/ntb_transport.c void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) qp 2205 drivers/ntb/ntb_transport.c if (!qp || qp->client_ready) qp 2208 drivers/ntb/ntb_transport.c entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); qp 2215 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); qp 2233 drivers/ntb/ntb_transport.c int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, qp 2238 drivers/ntb/ntb_transport.c if (!qp) qp 2241 drivers/ntb/ntb_transport.c entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); qp 2253 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); qp 2255 drivers/ntb/ntb_transport.c if (qp->active) qp 2256 drivers/ntb/ntb_transport.c tasklet_schedule(&qp->rxc_db_work); qp 2275 drivers/ntb/ntb_transport.c int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, qp 2281 drivers/ntb/ntb_transport.c if (!qp || !qp->link_is_up || !len) qp 2284 drivers/ntb/ntb_transport.c entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); qp 2286 drivers/ntb/ntb_transport.c qp->tx_err_no_buf++; qp 2298 drivers/ntb/ntb_transport.c rc = ntb_process_tx(qp, entry); qp 2300 drivers/ntb/ntb_transport.c ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, qp 2301 drivers/ntb/ntb_transport.c &qp->tx_free_q); qp 2313 drivers/ntb/ntb_transport.c void ntb_transport_link_up(struct ntb_transport_qp *qp) qp 2315 drivers/ntb/ntb_transport.c if (!qp) qp 2318 drivers/ntb/ntb_transport.c qp->client_ready = true; qp 2320 drivers/ntb/ntb_transport.c if (qp->transport->link_is_up) qp 2321 drivers/ntb/ntb_transport.c schedule_delayed_work(&qp->link_work, 0); qp 2333 drivers/ntb/ntb_transport.c void ntb_transport_link_down(struct ntb_transport_qp *qp) qp 2337 drivers/ntb/ntb_transport.c if (!qp) qp 2340 drivers/ntb/ntb_transport.c qp->client_ready = false; qp 2342 drivers/ntb/ntb_transport.c val = ntb_spad_read(qp->ndev, QP_LINKS); qp 2344 drivers/ntb/ntb_transport.c ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num)); qp 2346 drivers/ntb/ntb_transport.c if (qp->link_is_up) qp 2347 drivers/ntb/ntb_transport.c ntb_send_link_down(qp); qp 2349 drivers/ntb/ntb_transport.c cancel_delayed_work_sync(&qp->link_work); qp 2361 drivers/ntb/ntb_transport.c bool ntb_transport_link_query(struct ntb_transport_qp *qp) qp 2363 drivers/ntb/ntb_transport.c if (!qp) qp 2366 drivers/ntb/ntb_transport.c return qp->link_is_up; qp 2378 drivers/ntb/ntb_transport.c unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) qp 2380 drivers/ntb/ntb_transport.c if (!qp) qp 2383 drivers/ntb/ntb_transport.c return qp->qp_num; qp 2395 drivers/ntb/ntb_transport.c unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) qp 2401 drivers/ntb/ntb_transport.c if (!qp) qp 2404 drivers/ntb/ntb_transport.c rx_chan = qp->rx_dma_chan; qp 2405 drivers/ntb/ntb_transport.c tx_chan = qp->tx_dma_chan; qp 2411 drivers/ntb/ntb_transport.c max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); qp 2418 drivers/ntb/ntb_transport.c unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) qp 2420 drivers/ntb/ntb_transport.c unsigned int head = qp->tx_index; qp 2421 drivers/ntb/ntb_transport.c unsigned int tail = qp->remote_rx_info->entry; qp 2423 drivers/ntb/ntb_transport.c return tail > head ? tail - head : qp->tx_max_entry + tail - head; qp 2430 drivers/ntb/ntb_transport.c struct ntb_transport_qp *qp; qp 2444 drivers/ntb/ntb_transport.c qp = &nt->qp_vec[qp_num]; qp 2446 drivers/ntb/ntb_transport.c if (qp->active) qp 2447 drivers/ntb/ntb_transport.c tasklet_schedule(&qp->rxc_db_work); qp 82 drivers/nvme/host/rdma.c struct ib_qp *qp; qp 275 drivers/nvme/host/rdma.c queue->qp = queue->cm_id->qp; qp 411 drivers/nvme/host/rdma.c ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs); qp 418 drivers/nvme/host/rdma.c ib_destroy_qp(queue->qp); qp 488 drivers/nvme/host/rdma.c ret = ib_mr_pool_init(queue->qp, &queue->qp->rdma_mrs, qp 574 drivers/nvme/host/rdma.c ib_drain_qp(queue->qp); qp 1146 drivers/nvme/host/rdma.c return ib_post_send(queue->qp, &wr, NULL); qp 1160 drivers/nvme/host/rdma.c ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr); qp 1225 drivers/nvme/host/rdma.c req->mr = ib_mr_pool_get(queue->qp, &queue->qp->rdma_mrs); qp 1235 drivers/nvme/host/rdma.c ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr); qp 1365 drivers/nvme/host/rdma.c ret = ib_post_send(queue->qp, first, NULL); qp 1391 drivers/nvme/host/rdma.c ret = ib_post_recv(queue->qp, &wr, NULL); qp 1451 drivers/nvme/host/rdma.c cqe->command_id, queue->qp->qp_num); qp 1592 drivers/nvme/host/rdma.c param.qp_num = queue->qp->qp_num; qp 467 drivers/nvme/target/rdma.c ret = ib_post_recv(cmd->queue->cm_id->qp, &cmd->wr, NULL); qp 506 drivers/nvme/target/rdma.c rdma_rw_ctx_destroy(&rsp->rw, queue->cm_id->qp, qp 565 drivers/nvme/target/rdma.c first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp, qp 576 drivers/nvme/target/rdma.c if (unlikely(ib_post_send(cm_id->qp, first_wr, NULL))) { qp 590 drivers/nvme/target/rdma.c rdma_rw_ctx_destroy(&rsp->rw, queue->cm_id->qp, qp 678 drivers/nvme/target/rdma.c ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num, qp 745 drivers/nvme/target/rdma.c if (rdma_rw_ctx_post(&rsp->rw, queue->cm_id->qp, qp 1056 drivers/nvme/target/rdma.c struct ib_qp *qp = queue->cm_id->qp; qp 1058 drivers/nvme/target/rdma.c ib_drain_qp(qp); qp 1060 drivers/nvme/target/rdma.c ib_destroy_qp(qp); qp 1445 drivers/nvme/target/rdma.c if (cm_id->qp) qp 1446 drivers/nvme/target/rdma.c queue = cm_id->qp->qp_context; qp 760 drivers/scsi/bnx2i/bnx2i.h struct qp_info qp; qp 153 drivers/scsi/bnx2i/bnx2i_hwi.c cq_db = (struct bnx2i_5771x_cq_db *) ep->qp.cq_pgtbl_virt; qp 170 drivers/scsi/bnx2i/bnx2i_hwi.c cq_index = ep->qp.cqe_exp_seq_sn + next_index - 1; qp 171 drivers/scsi/bnx2i/bnx2i_hwi.c if (cq_index > ep->qp.cqe_size * 2) qp 172 drivers/scsi/bnx2i/bnx2i_hwi.c cq_index -= ep->qp.cqe_size * 2; qp 195 drivers/scsi/bnx2i/bnx2i_hwi.c if (!bnx2i_conn->ep->qp.rqe_left) qp 198 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.rqe_left--; qp 199 drivers/scsi/bnx2i/bnx2i_hwi.c memcpy(ptr, (u8 *) bnx2i_conn->ep->qp.rq_cons_qe, len); qp 200 drivers/scsi/bnx2i/bnx2i_hwi.c if (bnx2i_conn->ep->qp.rq_cons_qe == bnx2i_conn->ep->qp.rq_last_qe) { qp 201 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.rq_cons_qe = bnx2i_conn->ep->qp.rq_first_qe; qp 202 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.rq_cons_idx = 0; qp 204 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.rq_cons_qe++; qp 205 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.rq_cons_idx++; qp 220 drivers/scsi/bnx2i/bnx2i_hwi.c writel(cpu_to_le32(msg), conn->ep->qp.ctx_base); qp 234 drivers/scsi/bnx2i/bnx2i_hwi.c u16 hi_bit = (bnx2i_conn->ep->qp.rq_prod_idx & 0x8000); qp 237 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rqe_left += count; qp 238 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_prod_idx &= 0x7FFF; qp 239 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_prod_idx += count; qp 241 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.rq_prod_idx > bnx2i_conn->hba->max_rqes) { qp 242 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_prod_idx %= bnx2i_conn->hba->max_rqes; qp 244 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_prod_idx |= 0x8000; qp 246 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_prod_idx |= hi_bit; qp 249 drivers/scsi/bnx2i/bnx2i_hwi.c rq_db = (struct bnx2i_5771x_sq_rq_db *) ep->qp.rq_pgtbl_virt; qp 250 drivers/scsi/bnx2i/bnx2i_hwi.c rq_db->prod_idx = ep->qp.rq_prod_idx; qp 253 drivers/scsi/bnx2i/bnx2i_hwi.c writew(ep->qp.rq_prod_idx, qp 254 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.ctx_base + CNIC_RECV_DOORBELL); qp 276 drivers/scsi/bnx2i/bnx2i_hwi.c sq_db = (struct bnx2i_5771x_sq_rq_db *) ep->qp.sq_pgtbl_virt; qp 277 drivers/scsi/bnx2i/bnx2i_hwi.c sq_db->prod_idx = ep->qp.sq_prod_idx; qp 280 drivers/scsi/bnx2i/bnx2i_hwi.c writew(count, ep->qp.ctx_base + CNIC_SEND_DOORBELL); qp 297 drivers/scsi/bnx2i/bnx2i_hwi.c if (bnx2i_conn->ep->qp.sq_prod_qe == qp 298 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_last_qe) qp 299 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe = qp 300 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_first_qe; qp 302 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe++; qp 304 drivers/scsi/bnx2i/bnx2i_hwi.c if ((bnx2i_conn->ep->qp.sq_prod_qe + count) <= qp 305 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_last_qe) qp 306 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe += count; qp 308 drivers/scsi/bnx2i/bnx2i_hwi.c tmp_cnt = bnx2i_conn->ep->qp.sq_last_qe - qp 309 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe; qp 310 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe = qp 311 drivers/scsi/bnx2i/bnx2i_hwi.c &bnx2i_conn->ep->qp.sq_first_qe[count - qp 315 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_idx += count; qp 317 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_ring_sq_dbell(bnx2i_conn, bnx2i_conn->ep->qp.sq_prod_idx); qp 338 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe; qp 395 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe; qp 464 drivers/scsi/bnx2i/bnx2i_hwi.c text_wqe = (struct bnx2i_text_request *) bnx2i_conn->ep->qp.sq_prod_qe; qp 510 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe; qp 539 drivers/scsi/bnx2i/bnx2i_hwi.c nopout_wqe = (struct bnx2i_nop_out_request *)ep->qp.sq_prod_qe; qp 597 drivers/scsi/bnx2i/bnx2i_hwi.c bnx2i_conn->ep->qp.sq_prod_qe; qp 731 drivers/scsi/bnx2i/bnx2i_hwi.c (struct bnx2i_cleanup_request *)cmd->conn->ep->qp.sq_prod_qe; qp 801 drivers/scsi/bnx2i/bnx2i_hwi.c dma_addr = ep->qp.sq_pgtbl_phys; qp 805 drivers/scsi/bnx2i/bnx2i_hwi.c dma_addr = ep->qp.cq_pgtbl_phys; qp 813 drivers/scsi/bnx2i/bnx2i_hwi.c dma_addr = ep->qp.rq_pgtbl_phys; qp 817 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *) ep->qp.sq_pgtbl_virt; qp 822 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *) ep->qp.cq_pgtbl_virt; qp 862 drivers/scsi/bnx2i/bnx2i_hwi.c dma_addr = ep->qp.sq_pgtbl_phys + ISCSI_SQ_DB_SIZE; qp 866 drivers/scsi/bnx2i/bnx2i_hwi.c dma_addr = ep->qp.cq_pgtbl_phys + ISCSI_CQ_DB_SIZE; qp 874 drivers/scsi/bnx2i/bnx2i_hwi.c dma_addr = ep->qp.rq_pgtbl_phys + ISCSI_RQ_DB_SIZE; qp 878 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *)((u8 *)ep->qp.sq_pgtbl_virt + ISCSI_SQ_DB_SIZE); qp 882 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *)((u8 *)ep->qp.cq_pgtbl_virt + ISCSI_CQ_DB_SIZE); qp 891 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *)((u8 *)ep->qp.rq_pgtbl_virt + ISCSI_RQ_DB_SIZE); qp 947 drivers/scsi/bnx2i/bnx2i_hwi.c memset(ep->qp.sq_pgtbl_virt, 0, ep->qp.sq_pgtbl_size); qp 948 drivers/scsi/bnx2i/bnx2i_hwi.c num_pages = ep->qp.sq_mem_size / CNIC_PAGE_SIZE; qp 949 drivers/scsi/bnx2i/bnx2i_hwi.c page = ep->qp.sq_phys; qp 952 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *)((u8 *)ep->qp.sq_pgtbl_virt + ISCSI_SQ_DB_SIZE); qp 954 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *) ep->qp.sq_pgtbl_virt; qp 975 drivers/scsi/bnx2i/bnx2i_hwi.c memset(ep->qp.rq_pgtbl_virt, 0, ep->qp.rq_pgtbl_size); qp 976 drivers/scsi/bnx2i/bnx2i_hwi.c num_pages = ep->qp.rq_mem_size / CNIC_PAGE_SIZE; qp 977 drivers/scsi/bnx2i/bnx2i_hwi.c page = ep->qp.rq_phys; qp 980 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *)((u8 *)ep->qp.rq_pgtbl_virt + ISCSI_RQ_DB_SIZE); qp 982 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *) ep->qp.rq_pgtbl_virt; qp 1003 drivers/scsi/bnx2i/bnx2i_hwi.c memset(ep->qp.cq_pgtbl_virt, 0, ep->qp.cq_pgtbl_size); qp 1004 drivers/scsi/bnx2i/bnx2i_hwi.c num_pages = ep->qp.cq_mem_size / CNIC_PAGE_SIZE; qp 1005 drivers/scsi/bnx2i/bnx2i_hwi.c page = ep->qp.cq_phys; qp 1008 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *)((u8 *)ep->qp.cq_pgtbl_virt + ISCSI_CQ_DB_SIZE); qp 1010 drivers/scsi/bnx2i/bnx2i_hwi.c ptbl = (u32 *) ep->qp.cq_pgtbl_virt; qp 1051 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_mem_size = hba->max_sqes * BNX2I_SQ_WQE_SIZE; qp 1052 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_mem_size = qp 1053 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.sq_mem_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK; qp 1054 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_pgtbl_size = qp 1055 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.sq_mem_size / CNIC_PAGE_SIZE) * sizeof(void *); qp 1056 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_pgtbl_size = qp 1057 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.sq_pgtbl_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK; qp 1059 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_pgtbl_virt = qp 1060 drivers/scsi/bnx2i/bnx2i_hwi.c dma_alloc_coherent(&hba->pcidev->dev, ep->qp.sq_pgtbl_size, qp 1061 drivers/scsi/bnx2i/bnx2i_hwi.c &ep->qp.sq_pgtbl_phys, GFP_KERNEL); qp 1062 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.sq_pgtbl_virt) { qp 1064 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_pgtbl_size); qp 1069 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_virt = qp 1070 drivers/scsi/bnx2i/bnx2i_hwi.c dma_alloc_coherent(&hba->pcidev->dev, ep->qp.sq_mem_size, qp 1071 drivers/scsi/bnx2i/bnx2i_hwi.c &ep->qp.sq_phys, GFP_KERNEL); qp 1072 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.sq_virt) { qp 1074 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_mem_size); qp 1078 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_first_qe = ep->qp.sq_virt; qp 1079 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_prod_qe = ep->qp.sq_first_qe; qp 1080 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_cons_qe = ep->qp.sq_first_qe; qp 1081 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_last_qe = &ep->qp.sq_first_qe[hba->max_sqes - 1]; qp 1082 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_prod_idx = 0; qp 1083 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_cons_idx = 0; qp 1084 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sqe_left = hba->max_sqes; qp 1087 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_mem_size = hba->max_cqes * BNX2I_CQE_SIZE; qp 1088 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_mem_size = qp 1089 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.cq_mem_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK; qp 1090 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_pgtbl_size = qp 1091 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.cq_mem_size / CNIC_PAGE_SIZE) * sizeof(void *); qp 1092 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_pgtbl_size = qp 1093 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.cq_pgtbl_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK; qp 1095 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_pgtbl_virt = qp 1096 drivers/scsi/bnx2i/bnx2i_hwi.c dma_alloc_coherent(&hba->pcidev->dev, ep->qp.cq_pgtbl_size, qp 1097 drivers/scsi/bnx2i/bnx2i_hwi.c &ep->qp.cq_pgtbl_phys, GFP_KERNEL); qp 1098 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.cq_pgtbl_virt) { qp 1100 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_pgtbl_size); qp 1105 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_virt = qp 1106 drivers/scsi/bnx2i/bnx2i_hwi.c dma_alloc_coherent(&hba->pcidev->dev, ep->qp.cq_mem_size, qp 1107 drivers/scsi/bnx2i/bnx2i_hwi.c &ep->qp.cq_phys, GFP_KERNEL); qp 1108 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.cq_virt) { qp 1110 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_mem_size); qp 1114 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_first_qe = ep->qp.cq_virt; qp 1115 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_prod_qe = ep->qp.cq_first_qe; qp 1116 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_cons_qe = ep->qp.cq_first_qe; qp 1117 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_last_qe = &ep->qp.cq_first_qe[hba->max_cqes - 1]; qp 1118 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_prod_idx = 0; qp 1119 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_cons_idx = 0; qp 1120 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cqe_left = hba->max_cqes; qp 1121 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cqe_exp_seq_sn = ISCSI_INITIAL_SN; qp 1122 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cqe_size = hba->max_cqes; qp 1125 drivers/scsi/bnx2i/bnx2i_hwi.c cq_db = (struct bnx2i_5771x_cq_db *) ep->qp.cq_pgtbl_virt; qp 1129 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_mem_size = hba->max_rqes * BNX2I_RQ_WQE_SIZE; qp 1130 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_mem_size = qp 1131 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.rq_mem_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK; qp 1132 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_pgtbl_size = qp 1133 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.rq_mem_size / CNIC_PAGE_SIZE) * sizeof(void *); qp 1134 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_pgtbl_size = qp 1135 drivers/scsi/bnx2i/bnx2i_hwi.c (ep->qp.rq_pgtbl_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK; qp 1137 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_pgtbl_virt = qp 1138 drivers/scsi/bnx2i/bnx2i_hwi.c dma_alloc_coherent(&hba->pcidev->dev, ep->qp.rq_pgtbl_size, qp 1139 drivers/scsi/bnx2i/bnx2i_hwi.c &ep->qp.rq_pgtbl_phys, GFP_KERNEL); qp 1140 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.rq_pgtbl_virt) { qp 1142 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_pgtbl_size); qp 1147 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_virt = qp 1148 drivers/scsi/bnx2i/bnx2i_hwi.c dma_alloc_coherent(&hba->pcidev->dev, ep->qp.rq_mem_size, qp 1149 drivers/scsi/bnx2i/bnx2i_hwi.c &ep->qp.rq_phys, GFP_KERNEL); qp 1150 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.rq_virt) { qp 1152 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_mem_size); qp 1156 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_first_qe = ep->qp.rq_virt; qp 1157 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_prod_qe = ep->qp.rq_first_qe; qp 1158 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_cons_qe = ep->qp.rq_first_qe; qp 1159 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_last_qe = &ep->qp.rq_first_qe[hba->max_rqes - 1]; qp 1160 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_prod_idx = 0x8000; qp 1161 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_cons_idx = 0; qp 1162 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rqe_left = hba->max_rqes; qp 1184 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.ctx_base) { qp 1185 drivers/scsi/bnx2i/bnx2i_hwi.c iounmap(ep->qp.ctx_base); qp 1186 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.ctx_base = NULL; qp 1189 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.sq_pgtbl_virt) { qp 1190 drivers/scsi/bnx2i/bnx2i_hwi.c dma_free_coherent(&hba->pcidev->dev, ep->qp.sq_pgtbl_size, qp 1191 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_pgtbl_virt, ep->qp.sq_pgtbl_phys); qp 1192 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_pgtbl_virt = NULL; qp 1193 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_pgtbl_phys = 0; qp 1195 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.sq_virt) { qp 1196 drivers/scsi/bnx2i/bnx2i_hwi.c dma_free_coherent(&hba->pcidev->dev, ep->qp.sq_mem_size, qp 1197 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_virt, ep->qp.sq_phys); qp 1198 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_virt = NULL; qp 1199 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.sq_phys = 0; qp 1203 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.rq_pgtbl_virt) { qp 1204 drivers/scsi/bnx2i/bnx2i_hwi.c dma_free_coherent(&hba->pcidev->dev, ep->qp.rq_pgtbl_size, qp 1205 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_pgtbl_virt, ep->qp.rq_pgtbl_phys); qp 1206 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_pgtbl_virt = NULL; qp 1207 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_pgtbl_phys = 0; qp 1209 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.rq_virt) { qp 1210 drivers/scsi/bnx2i/bnx2i_hwi.c dma_free_coherent(&hba->pcidev->dev, ep->qp.rq_mem_size, qp 1211 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_virt, ep->qp.rq_phys); qp 1212 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_virt = NULL; qp 1213 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.rq_phys = 0; qp 1217 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.cq_pgtbl_virt) { qp 1218 drivers/scsi/bnx2i/bnx2i_hwi.c dma_free_coherent(&hba->pcidev->dev, ep->qp.cq_pgtbl_size, qp 1219 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_pgtbl_virt, ep->qp.cq_pgtbl_phys); qp 1220 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_pgtbl_virt = NULL; qp 1221 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_pgtbl_phys = 0; qp 1223 drivers/scsi/bnx2i/bnx2i_hwi.c if (ep->qp.cq_virt) { qp 1224 drivers/scsi/bnx2i/bnx2i_hwi.c dma_free_coherent(&hba->pcidev->dev, ep->qp.cq_mem_size, qp 1225 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_virt, ep->qp.cq_phys); qp 1226 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_virt = NULL; qp 1227 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.cq_phys = 0; qp 1957 drivers/scsi/bnx2i/bnx2i_hwi.c struct qp_info *qp; qp 1965 drivers/scsi/bnx2i/bnx2i_hwi.c qp = &bnx2i_conn->ep->qp; qp 1967 drivers/scsi/bnx2i/bnx2i_hwi.c if (!qp->cq_virt) { qp 1973 drivers/scsi/bnx2i/bnx2i_hwi.c nopin = (struct bnx2i_nop_in_msg *) qp->cq_cons_qe; qp 1974 drivers/scsi/bnx2i/bnx2i_hwi.c if (nopin->cq_req_sn != qp->cqe_exp_seq_sn) qp 2000 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2004 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2008 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2012 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2016 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe)) qp 2021 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2025 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2030 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2034 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe); qp 2059 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cqe_exp_seq_sn++; qp 2060 drivers/scsi/bnx2i/bnx2i_hwi.c if (qp->cqe_exp_seq_sn == (qp->cqe_size * 2 + 1)) qp 2061 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cqe_exp_seq_sn = ISCSI_INITIAL_SN; qp 2063 drivers/scsi/bnx2i/bnx2i_hwi.c if (qp->cq_cons_qe == qp->cq_last_qe) { qp 2064 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe = qp->cq_first_qe; qp 2065 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_idx = 0; qp 2067 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_qe++; qp 2068 drivers/scsi/bnx2i/bnx2i_hwi.c qp->cq_cons_idx++; qp 2462 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.ctx_base = NULL; qp 2718 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.ctx_base = ioremap_nocache(reg_base + reg_off, 4); qp 2719 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.ctx_base) qp 2739 drivers/scsi/bnx2i/bnx2i_hwi.c ep->qp.ctx_base = ioremap_nocache(ep->hba->reg_base + reg_off, qp 2741 drivers/scsi/bnx2i/bnx2i_hwi.c if (!ep->qp.ctx_base) qp 1216 drivers/scsi/lpfc/lpfc_attr.c struct lpfc_queue *qp = NULL; qp 1261 drivers/scsi/lpfc/lpfc_attr.c list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) { qp 1262 drivers/scsi/lpfc/lpfc_attr.c pring = qp->pring; qp 1707 drivers/scsi/lpfc/lpfc_bsg.c struct lpfc_queue *qp = NULL; qp 1738 drivers/scsi/lpfc/lpfc_bsg.c list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) { qp 1739 drivers/scsi/lpfc/lpfc_bsg.c pring = qp->pring; qp 325 drivers/scsi/lpfc/lpfc_crtn.h int lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, qp 591 drivers/scsi/lpfc/lpfc_crtn.h struct lpfc_sli4_hdw_queue *qp); qp 408 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_sli4_hdw_queue *qp; qp 416 drivers/scsi/lpfc/lpfc_debugfs.c qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_xripool]; qp 419 drivers/scsi/lpfc/lpfc_debugfs.c spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); qp 420 drivers/scsi/lpfc/lpfc_debugfs.c spin_lock(&qp->io_buf_list_get_lock); qp 421 drivers/scsi/lpfc/lpfc_debugfs.c spin_lock(&qp->io_buf_list_put_lock); qp 422 drivers/scsi/lpfc/lpfc_debugfs.c out = qp->total_io_bufs - (qp->get_io_bufs + qp->put_io_bufs + qp 423 drivers/scsi/lpfc/lpfc_debugfs.c qp->abts_scsi_io_bufs + qp->abts_nvme_io_bufs); qp 427 drivers/scsi/lpfc/lpfc_debugfs.c qp->total_io_bufs, qp->get_io_bufs, qp->put_io_bufs, qp 428 drivers/scsi/lpfc/lpfc_debugfs.c qp->empty_io_bufs, qp->abts_scsi_io_bufs, qp 429 drivers/scsi/lpfc/lpfc_debugfs.c qp->abts_nvme_io_bufs, out); qp 430 drivers/scsi/lpfc/lpfc_debugfs.c spin_unlock(&qp->io_buf_list_put_lock); qp 431 drivers/scsi/lpfc/lpfc_debugfs.c spin_unlock(&qp->io_buf_list_get_lock); qp 432 drivers/scsi/lpfc/lpfc_debugfs.c spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); qp 462 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_sli4_hdw_queue *qp; qp 528 drivers/scsi/lpfc/lpfc_debugfs.c qp = &phba->sli4_hba.hdwq[i]; qp 529 drivers/scsi/lpfc/lpfc_debugfs.c multixri_pool = qp->p_multixri_pool; qp 534 drivers/scsi/lpfc/lpfc_debugfs.c txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; qp 540 drivers/scsi/lpfc/lpfc_debugfs.c qp->empty_io_bufs, multixri_pool->pbl_empty_count); qp 598 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_sli4_hdw_queue *qp; qp 611 drivers/scsi/lpfc/lpfc_debugfs.c qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_lock]; qp 619 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.alloc_pvt_pool, qp 620 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.mv_from_pvt_pool, qp 621 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.mv_to_pub_pool, qp 622 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.mv_to_pvt_pool, qp 623 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.free_pvt_pool, qp 624 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.free_pub_pool, qp 625 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.wq_access); qp 629 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.alloc_xri_get, qp 630 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.alloc_xri_put, qp 631 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.free_xri, qp 632 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.wq_access); qp 1621 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_sli4_hdw_queue *qp; qp 1643 drivers/scsi/lpfc/lpfc_debugfs.c qp = &phba->sli4_hba.hdwq[i]; qp 1649 drivers/scsi/lpfc/lpfc_debugfs.c tot_xmt += qp->cpucheck_xmt_io[j]; qp 1650 drivers/scsi/lpfc/lpfc_debugfs.c tot_cmpl += qp->cpucheck_cmpl_io[j]; qp 1652 drivers/scsi/lpfc/lpfc_debugfs.c tot_rcv += qp->cpucheck_rcv_io[j]; qp 1663 drivers/scsi/lpfc/lpfc_debugfs.c if (!qp->cpucheck_xmt_io[j] && qp 1664 drivers/scsi/lpfc/lpfc_debugfs.c !qp->cpucheck_cmpl_io[j] && qp 1665 drivers/scsi/lpfc/lpfc_debugfs.c !qp->cpucheck_rcv_io[j]) qp 1670 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_rcv_io[j], qp 1671 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_xmt_io[j], qp 1672 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_cmpl_io[j]); qp 1676 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_xmt_io[j], qp 1677 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_cmpl_io[j]); qp 2045 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_sli4_hdw_queue *qp; qp 2063 drivers/scsi/lpfc/lpfc_debugfs.c qp = &phba->sli4_hba.hdwq[i]; qp 2064 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.alloc_xri_get = 0; qp 2065 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.alloc_xri_put = 0; qp 2066 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.free_xri = 0; qp 2067 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.wq_access = 0; qp 2068 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.alloc_pvt_pool = 0; qp 2069 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.mv_from_pvt_pool = 0; qp 2070 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.mv_to_pub_pool = 0; qp 2071 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.mv_to_pvt_pool = 0; qp 2072 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.free_pvt_pool = 0; qp 2073 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.free_pub_pool = 0; qp 2074 drivers/scsi/lpfc/lpfc_debugfs.c qp->lock_conflict.wq_access = 0; qp 2402 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_sli4_hdw_queue *qp; qp 2421 drivers/scsi/lpfc/lpfc_debugfs.c qp = &phba->sli4_hba.hdwq[i]; qp 2422 drivers/scsi/lpfc/lpfc_debugfs.c multixri_pool = qp->p_multixri_pool; qp 2426 drivers/scsi/lpfc/lpfc_debugfs.c qp->empty_io_bufs = 0; qp 2861 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_sli4_hdw_queue *qp; qp 2905 drivers/scsi/lpfc/lpfc_debugfs.c qp = &phba->sli4_hba.hdwq[i]; qp 2908 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_rcv_io[j] = 0; qp 2909 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_xmt_io[j] = 0; qp 2910 drivers/scsi/lpfc/lpfc_debugfs.c qp->cpucheck_cmpl_io[j] = 0; qp 3660 drivers/scsi/lpfc/lpfc_debugfs.c __lpfc_idiag_print_wq(struct lpfc_queue *qp, char *wqtype, qp 3663 drivers/scsi/lpfc/lpfc_debugfs.c if (!qp) qp 3670 drivers/scsi/lpfc/lpfc_debugfs.c qp->assoc_qid, qp->q_cnt_1, qp 3671 drivers/scsi/lpfc/lpfc_debugfs.c (unsigned long long)qp->q_cnt_4); qp 3675 drivers/scsi/lpfc/lpfc_debugfs.c qp->queue_id, qp->entry_count, qp 3676 drivers/scsi/lpfc/lpfc_debugfs.c qp->entry_size, qp->host_index, qp 3677 drivers/scsi/lpfc/lpfc_debugfs.c qp->hba_index, qp->notify_interval); qp 3687 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_queue *qp; qp 3691 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.hdwq[qidx].io_wq; qp 3692 drivers/scsi/lpfc/lpfc_debugfs.c if (qp->assoc_qid != cq_id) qp 3694 drivers/scsi/lpfc/lpfc_debugfs.c *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len); qp 3702 drivers/scsi/lpfc/lpfc_debugfs.c __lpfc_idiag_print_cq(struct lpfc_queue *qp, char *cqtype, qp 3705 drivers/scsi/lpfc/lpfc_debugfs.c if (!qp) qp 3713 drivers/scsi/lpfc/lpfc_debugfs.c qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2, qp 3714 drivers/scsi/lpfc/lpfc_debugfs.c qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); qp 3718 drivers/scsi/lpfc/lpfc_debugfs.c qp->queue_id, qp->entry_count, qp 3719 drivers/scsi/lpfc/lpfc_debugfs.c qp->entry_size, qp->host_index, qp 3720 drivers/scsi/lpfc/lpfc_debugfs.c qp->notify_interval, qp->max_proc_limit); qp 3729 drivers/scsi/lpfc/lpfc_debugfs.c __lpfc_idiag_print_rqpair(struct lpfc_queue *qp, struct lpfc_queue *datqp, qp 3732 drivers/scsi/lpfc/lpfc_debugfs.c if (!qp || !datqp) qp 3740 drivers/scsi/lpfc/lpfc_debugfs.c qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2, qp 3741 drivers/scsi/lpfc/lpfc_debugfs.c qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); qp 3745 drivers/scsi/lpfc/lpfc_debugfs.c qp->queue_id, qp->entry_count, qp->entry_size, qp 3746 drivers/scsi/lpfc/lpfc_debugfs.c qp->host_index, qp->hba_index, qp->notify_interval); qp 3760 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_queue *qp; qp 3763 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.hdwq[eqidx].io_cq; qp 3765 drivers/scsi/lpfc/lpfc_debugfs.c *len = __lpfc_idiag_print_cq(qp, "IO", pbuffer, *len); qp 3768 drivers/scsi/lpfc/lpfc_debugfs.c qp->CQ_max_cqe = 0; qp 3774 drivers/scsi/lpfc/lpfc_debugfs.c max_cnt, qp->queue_id); qp 3780 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.nvmet_cqset[eqidx]; qp 3781 drivers/scsi/lpfc/lpfc_debugfs.c *len = __lpfc_idiag_print_cq(qp, "NVMET CQset", pbuffer, *len); qp 3784 drivers/scsi/lpfc/lpfc_debugfs.c qp->CQ_max_cqe = 0; qp 3790 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.nvmet_mrq_hdr[eqidx]; qp 3791 drivers/scsi/lpfc/lpfc_debugfs.c *len = __lpfc_idiag_print_rqpair(qp, qp 3803 drivers/scsi/lpfc/lpfc_debugfs.c __lpfc_idiag_print_eq(struct lpfc_queue *qp, char *eqtype, qp 3806 drivers/scsi/lpfc/lpfc_debugfs.c if (!qp) qp 3812 drivers/scsi/lpfc/lpfc_debugfs.c eqtype, qp->q_cnt_1, qp->q_cnt_2, qp->q_cnt_3, qp 3813 drivers/scsi/lpfc/lpfc_debugfs.c (unsigned long long)qp->q_cnt_4, qp->q_mode); qp 3817 drivers/scsi/lpfc/lpfc_debugfs.c qp->queue_id, qp->entry_count, qp->entry_size, qp 3818 drivers/scsi/lpfc/lpfc_debugfs.c qp->host_index, qp->notify_interval, qp 3819 drivers/scsi/lpfc/lpfc_debugfs.c qp->max_proc_limit, qp->chann); qp 3852 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_queue *qp = NULL; qp 3880 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.hdwq[x].hba_eq; qp 3881 drivers/scsi/lpfc/lpfc_debugfs.c if (!qp) qp 3884 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_eq(qp, "HBA", pbuffer, len); qp 3887 drivers/scsi/lpfc/lpfc_debugfs.c qp->EQ_max_eqe = 0; qp 3894 drivers/scsi/lpfc/lpfc_debugfs.c max_cnt, x, qp->queue_id); qp 3903 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.mbx_cq; qp 3904 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_cq(qp, "MBX", pbuffer, len); qp 3909 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.mbx_wq; qp 3910 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_wq(qp, "MBX", pbuffer, len); qp 3915 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.els_cq; qp 3916 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_cq(qp, "ELS", pbuffer, len); qp 3918 drivers/scsi/lpfc/lpfc_debugfs.c if (qp) qp 3919 drivers/scsi/lpfc/lpfc_debugfs.c qp->CQ_max_cqe = 0; qp 3924 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.els_wq; qp 3925 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_wq(qp, "ELS", pbuffer, len); qp 3929 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.hdr_rq; qp 3930 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_rqpair(qp, phba->sli4_hba.dat_rq, qp 3936 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.nvmels_cq; qp 3937 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_cq(qp, "NVME LS", qp 3940 drivers/scsi/lpfc/lpfc_debugfs.c if (qp) qp 3941 drivers/scsi/lpfc/lpfc_debugfs.c qp->CQ_max_cqe = 0; qp 3946 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.nvmels_wq; qp 3947 drivers/scsi/lpfc/lpfc_debugfs.c len = __lpfc_idiag_print_wq(qp, "NVME LS", qp 4137 drivers/scsi/lpfc/lpfc_debugfs.c struct lpfc_queue *pque, *qp; qp 4174 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.hdwq[qidx].hba_eq; qp 4175 drivers/scsi/lpfc/lpfc_debugfs.c if (qp && qp->queue_id == queid) { qp 4177 drivers/scsi/lpfc/lpfc_debugfs.c rc = lpfc_idiag_que_param_check(qp, qp 4181 drivers/scsi/lpfc/lpfc_debugfs.c idiag.ptr_private = qp; qp 4226 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.hdwq[qidx].io_cq; qp 4227 drivers/scsi/lpfc/lpfc_debugfs.c if (qp && qp->queue_id == queid) { qp 4230 drivers/scsi/lpfc/lpfc_debugfs.c qp, index, count); qp 4233 drivers/scsi/lpfc/lpfc_debugfs.c idiag.ptr_private = qp; qp 4281 drivers/scsi/lpfc/lpfc_debugfs.c qp = phba->sli4_hba.hdwq[qidx].io_wq; qp 4282 drivers/scsi/lpfc/lpfc_debugfs.c if (qp && qp->queue_id == queid) { qp 4285 drivers/scsi/lpfc/lpfc_debugfs.c qp, index, count); qp 4288 drivers/scsi/lpfc/lpfc_debugfs.c idiag.ptr_private = qp; qp 516 drivers/scsi/lpfc/lpfc_debugfs.h struct lpfc_queue *qp; qp 518 drivers/scsi/lpfc/lpfc_debugfs.h qp = phba->sli4_hba.hdwq[qidx].hba_eq; qp 520 drivers/scsi/lpfc/lpfc_debugfs.h pr_err("EQ[Idx:%d|Qid:%d]\n", qidx, qp->queue_id); qp 522 drivers/scsi/lpfc/lpfc_debugfs.h lpfc_debug_dump_q(qp); qp 4779 drivers/scsi/lpfc/lpfc_hbadisc.c struct lpfc_queue *qp = NULL; qp 4782 drivers/scsi/lpfc/lpfc_hbadisc.c list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) { qp 4783 drivers/scsi/lpfc/lpfc_hbadisc.c pring = qp->pring; qp 965 drivers/scsi/lpfc/lpfc_init.c struct lpfc_queue *qp = NULL; qp 990 drivers/scsi/lpfc/lpfc_init.c list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) { qp 991 drivers/scsi/lpfc/lpfc_init.c pring = qp->pring; qp 1044 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 1082 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[idx]; qp 1084 drivers/scsi/lpfc/lpfc_init.c spin_lock(&qp->abts_io_buf_list_lock); qp 1085 drivers/scsi/lpfc/lpfc_init.c list_splice_init(&qp->lpfc_abts_io_buf_list, qp 1093 drivers/scsi/lpfc/lpfc_init.c spin_lock(&qp->io_buf_list_put_lock); qp 1094 drivers/scsi/lpfc/lpfc_init.c list_splice_init(&aborts, &qp->lpfc_io_buf_list_put); qp 1095 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs += qp->abts_scsi_io_bufs; qp 1096 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs += qp->abts_nvme_io_bufs; qp 1097 drivers/scsi/lpfc/lpfc_init.c qp->abts_scsi_io_bufs = 0; qp 1098 drivers/scsi/lpfc/lpfc_init.c qp->abts_nvme_io_bufs = 0; qp 1099 drivers/scsi/lpfc/lpfc_init.c spin_unlock(&qp->io_buf_list_put_lock); qp 1100 drivers/scsi/lpfc/lpfc_init.c spin_unlock(&qp->abts_io_buf_list_lock); qp 3080 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 3087 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[0]; qp 3090 drivers/scsi/lpfc/lpfc_init.c spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); qp 3094 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_put, list) { qp 3097 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs--; qp 3103 drivers/scsi/lpfc/lpfc_init.c spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); qp 3115 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 3122 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[0]; qp 3124 drivers/scsi/lpfc/lpfc_init.c spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); qp 3129 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_put); qp 3131 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs++; qp 3135 drivers/scsi/lpfc/lpfc_init.c spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); qp 3154 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 3183 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[j]; qp 3184 drivers/scsi/lpfc/lpfc_init.c kfree(qp->p_multixri_pool); qp 3191 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[i]; qp 3192 drivers/scsi/lpfc/lpfc_init.c qp->p_multixri_pool = multixri_pool; qp 3200 drivers/scsi/lpfc/lpfc_init.c spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); qp 3204 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_put, list) { qp 3206 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs--; qp 3213 drivers/scsi/lpfc/lpfc_init.c spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); qp 3240 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 3254 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[i]; qp 3255 drivers/scsi/lpfc/lpfc_init.c multixri_pool = qp->p_multixri_pool; qp 3259 drivers/scsi/lpfc/lpfc_init.c qp->p_multixri_pool = NULL; qp 3261 drivers/scsi/lpfc/lpfc_init.c spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); qp 3274 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_put); qp 3275 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs++; qp 3295 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_put); qp 3296 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs++; qp 3304 drivers/scsi/lpfc/lpfc_init.c spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); qp 3617 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 3621 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[idx]; qp 3623 drivers/scsi/lpfc/lpfc_init.c spin_lock(&qp->io_buf_list_put_lock); qp 3625 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_put, qp 3628 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs--; qp 3635 drivers/scsi/lpfc/lpfc_init.c qp->total_io_bufs--; qp 3637 drivers/scsi/lpfc/lpfc_init.c spin_unlock(&qp->io_buf_list_put_lock); qp 3639 drivers/scsi/lpfc/lpfc_init.c spin_lock(&qp->io_buf_list_get_lock); qp 3641 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_get, qp 3644 drivers/scsi/lpfc/lpfc_init.c qp->get_io_bufs--; qp 3651 drivers/scsi/lpfc/lpfc_init.c qp->total_io_bufs--; qp 3653 drivers/scsi/lpfc/lpfc_init.c spin_unlock(&qp->io_buf_list_get_lock); qp 3902 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 3909 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[idx]; qp 3910 drivers/scsi/lpfc/lpfc_init.c spin_lock_irq(&qp->io_buf_list_get_lock); qp 3911 drivers/scsi/lpfc/lpfc_init.c spin_lock(&qp->io_buf_list_put_lock); qp 3914 drivers/scsi/lpfc/lpfc_init.c list_splice_init(&qp->lpfc_io_buf_list_get, &blist); qp 3915 drivers/scsi/lpfc/lpfc_init.c list_splice(&qp->lpfc_io_buf_list_put, &blist); qp 3916 drivers/scsi/lpfc/lpfc_init.c INIT_LIST_HEAD(&qp->lpfc_io_buf_list_get); qp 3917 drivers/scsi/lpfc/lpfc_init.c INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); qp 3918 drivers/scsi/lpfc/lpfc_init.c cnt += qp->get_io_bufs + qp->put_io_bufs; qp 3919 drivers/scsi/lpfc/lpfc_init.c qp->get_io_bufs = 0; qp 3920 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs = 0; qp 3921 drivers/scsi/lpfc/lpfc_init.c qp->total_io_bufs = 0; qp 3922 drivers/scsi/lpfc/lpfc_init.c spin_unlock(&qp->io_buf_list_put_lock); qp 3923 drivers/scsi/lpfc/lpfc_init.c spin_unlock_irq(&qp->io_buf_list_get_lock); qp 3963 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 3967 drivers/scsi/lpfc/lpfc_init.c qp = phba->sli4_hba.hdwq; qp 3976 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[idx]; qp 3978 drivers/scsi/lpfc/lpfc_init.c lpfc_cmd->hdwq = qp; qp 3981 drivers/scsi/lpfc/lpfc_init.c spin_lock(&qp->io_buf_list_put_lock); qp 3983 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_io_buf_list_put); qp 3984 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs++; qp 3985 drivers/scsi/lpfc/lpfc_init.c qp->total_io_bufs++; qp 3986 drivers/scsi/lpfc/lpfc_init.c spin_unlock(&qp->io_buf_list_put_lock); qp 8725 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 8757 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[idx]; qp 8758 drivers/scsi/lpfc/lpfc_init.c spin_lock_init(&qp->io_buf_list_get_lock); qp 8759 drivers/scsi/lpfc/lpfc_init.c spin_lock_init(&qp->io_buf_list_put_lock); qp 8760 drivers/scsi/lpfc/lpfc_init.c INIT_LIST_HEAD(&qp->lpfc_io_buf_list_get); qp 8761 drivers/scsi/lpfc/lpfc_init.c INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); qp 8762 drivers/scsi/lpfc/lpfc_init.c qp->get_io_bufs = 0; qp 8763 drivers/scsi/lpfc/lpfc_init.c qp->put_io_bufs = 0; qp 8764 drivers/scsi/lpfc/lpfc_init.c qp->total_io_bufs = 0; qp 8765 drivers/scsi/lpfc/lpfc_init.c spin_lock_init(&qp->abts_io_buf_list_lock); qp 8766 drivers/scsi/lpfc/lpfc_init.c INIT_LIST_HEAD(&qp->lpfc_abts_io_buf_list); qp 8767 drivers/scsi/lpfc/lpfc_init.c qp->abts_scsi_io_bufs = 0; qp 8768 drivers/scsi/lpfc/lpfc_init.c qp->abts_nvme_io_bufs = 0; qp 8769 drivers/scsi/lpfc/lpfc_init.c INIT_LIST_HEAD(&qp->sgl_list); qp 8770 drivers/scsi/lpfc/lpfc_init.c INIT_LIST_HEAD(&qp->cmd_rsp_buf_list); qp 8771 drivers/scsi/lpfc/lpfc_init.c spin_lock_init(&qp->hdwq_lock); qp 8823 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[cpup->hdwq]; qp 8841 drivers/scsi/lpfc/lpfc_init.c qp->hba_eq = qdesc; qp 8858 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[cpup->hdwq]; qp 8859 drivers/scsi/lpfc/lpfc_init.c if (qp->hba_eq) qp 8865 drivers/scsi/lpfc/lpfc_init.c qp->hba_eq = phba->sli4_hba.hdwq[eqcpup->hdwq].hba_eq; qp 9090 drivers/scsi/lpfc/lpfc_init.c __lpfc_sli4_release_queue(struct lpfc_queue **qp) qp 9092 drivers/scsi/lpfc/lpfc_init.c if (*qp != NULL) { qp 9093 drivers/scsi/lpfc/lpfc_init.c lpfc_sli4_queue_free(*qp); qp 9094 drivers/scsi/lpfc/lpfc_init.c *qp = NULL; qp 9358 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 9412 drivers/scsi/lpfc/lpfc_init.c qp = phba->sli4_hba.hdwq; qp 9415 drivers/scsi/lpfc/lpfc_init.c if (!qp) { qp 9437 drivers/scsi/lpfc/lpfc_init.c rc = lpfc_eq_create(phba, qp[cpup->hdwq].hba_eq, qp 9449 drivers/scsi/lpfc/lpfc_init.c qp[cpup->hdwq].hba_eq; qp 9454 drivers/scsi/lpfc/lpfc_init.c qp[cpup->hdwq].hba_eq->queue_id); qp 9466 drivers/scsi/lpfc/lpfc_init.c qp[qidx].io_cq, qp 9467 drivers/scsi/lpfc/lpfc_init.c qp[qidx].io_wq, qp 9495 drivers/scsi/lpfc/lpfc_init.c rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, qp 9516 drivers/scsi/lpfc/lpfc_init.c qp, qp 9528 drivers/scsi/lpfc/lpfc_init.c qp[0].hba_eq, qp 9542 drivers/scsi/lpfc/lpfc_init.c qp[0].hba_eq->queue_id); qp 9554 drivers/scsi/lpfc/lpfc_init.c rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, qp 9578 drivers/scsi/lpfc/lpfc_init.c rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, qp 9717 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 9773 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[qidx]; qp 9774 drivers/scsi/lpfc/lpfc_init.c lpfc_wq_destroy(phba, qp->io_wq); qp 9775 drivers/scsi/lpfc/lpfc_init.c lpfc_cq_destroy(phba, qp->io_cq); qp 11414 drivers/scsi/lpfc/lpfc_init.c struct lpfc_sli4_hdw_queue *qp; qp 11433 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[idx]; qp 11434 drivers/scsi/lpfc/lpfc_init.c io_xri_cmpl = list_empty(&qp->lpfc_abts_io_buf_list); qp 11472 drivers/scsi/lpfc/lpfc_init.c qp = &phba->sli4_hba.hdwq[idx]; qp 11474 drivers/scsi/lpfc/lpfc_init.c &qp->lpfc_abts_io_buf_list); qp 2021 drivers/scsi/lpfc/lpfc_nvme.c struct lpfc_sli4_hdw_queue *qp; qp 2059 drivers/scsi/lpfc/lpfc_nvme.c qp = &phba->sli4_hba.hdwq[idx]; qp 2060 drivers/scsi/lpfc/lpfc_nvme.c qp->empty_io_bufs++; qp 2079 drivers/scsi/lpfc/lpfc_nvme.c struct lpfc_sli4_hdw_queue *qp; qp 2088 drivers/scsi/lpfc/lpfc_nvme.c qp = lpfc_ncmd->hdwq; qp 2096 drivers/scsi/lpfc/lpfc_nvme.c spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); qp 2098 drivers/scsi/lpfc/lpfc_nvme.c &qp->lpfc_abts_io_buf_list); qp 2099 drivers/scsi/lpfc/lpfc_nvme.c qp->abts_nvme_io_bufs++; qp 2100 drivers/scsi/lpfc/lpfc_nvme.c spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); qp 2102 drivers/scsi/lpfc/lpfc_nvme.c lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); qp 470 drivers/scsi/lpfc/lpfc_scsi.c struct lpfc_sli4_hdw_queue *qp; qp 479 drivers/scsi/lpfc/lpfc_scsi.c qp = &phba->sli4_hba.hdwq[idx]; qp 481 drivers/scsi/lpfc/lpfc_scsi.c spin_lock(&qp->abts_io_buf_list_lock); qp 483 drivers/scsi/lpfc/lpfc_scsi.c &qp->lpfc_abts_io_buf_list, list) { qp 491 drivers/scsi/lpfc/lpfc_scsi.c spin_unlock(&qp->abts_io_buf_list_lock); qp 511 drivers/scsi/lpfc/lpfc_scsi.c struct lpfc_sli4_hdw_queue *qp; qp 522 drivers/scsi/lpfc/lpfc_scsi.c qp = &phba->sli4_hba.hdwq[idx]; qp 524 drivers/scsi/lpfc/lpfc_scsi.c spin_lock(&qp->abts_io_buf_list_lock); qp 526 drivers/scsi/lpfc/lpfc_scsi.c &qp->lpfc_abts_io_buf_list, list) { qp 532 drivers/scsi/lpfc/lpfc_scsi.c qp->abts_nvme_io_bufs--; qp 533 drivers/scsi/lpfc/lpfc_scsi.c spin_unlock(&qp->abts_io_buf_list_lock); qp 538 drivers/scsi/lpfc/lpfc_scsi.c qp->abts_scsi_io_bufs--; qp 539 drivers/scsi/lpfc/lpfc_scsi.c spin_unlock(&qp->abts_io_buf_list_lock); qp 559 drivers/scsi/lpfc/lpfc_scsi.c spin_unlock(&qp->abts_io_buf_list_lock); qp 634 drivers/scsi/lpfc/lpfc_scsi.c struct lpfc_sli4_hdw_queue *qp; qp 654 drivers/scsi/lpfc/lpfc_scsi.c qp = &phba->sli4_hba.hdwq[idx]; qp 655 drivers/scsi/lpfc/lpfc_scsi.c qp->empty_io_bufs++; qp 784 drivers/scsi/lpfc/lpfc_scsi.c struct lpfc_sli4_hdw_queue *qp; qp 790 drivers/scsi/lpfc/lpfc_scsi.c qp = psb->hdwq; qp 792 drivers/scsi/lpfc/lpfc_scsi.c spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); qp 794 drivers/scsi/lpfc/lpfc_scsi.c list_add_tail(&psb->list, &qp->lpfc_abts_io_buf_list); qp 795 drivers/scsi/lpfc/lpfc_scsi.c qp->abts_scsi_io_bufs++; qp 796 drivers/scsi/lpfc/lpfc_scsi.c spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); qp 798 drivers/scsi/lpfc/lpfc_scsi.c lpfc_release_io_buf(phba, (struct lpfc_io_buf *)psb, qp); qp 5553 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 5565 drivers/scsi/lpfc/lpfc_sli.c qp = &sli4_hba->hdwq[qidx]; qp 5567 drivers/scsi/lpfc/lpfc_sli.c sli4_hba->sli4_write_cq_db(phba, qp->io_cq, 0, qp 10645 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_queue *qp = NULL; qp 10686 drivers/scsi/lpfc/lpfc_sli.c list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) { qp 10687 drivers/scsi/lpfc/lpfc_sli.c pring = qp->pring; qp 10741 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_queue *qp = NULL; qp 10775 drivers/scsi/lpfc/lpfc_sli.c list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) { qp 10776 drivers/scsi/lpfc/lpfc_sli.c pring = qp->pring; qp 11292 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_queue *qp = NULL; qp 11302 drivers/scsi/lpfc/lpfc_sli.c list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) { qp 11303 drivers/scsi/lpfc/lpfc_sli.c pring = qp->pring; qp 19819 drivers/scsi/lpfc/lpfc_sli.c lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, qp 19834 drivers/scsi/lpfc/lpfc_sli.c qp, wq_access); qp 19857 drivers/scsi/lpfc/lpfc_sli.c lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH); qp 19864 drivers/scsi/lpfc/lpfc_sli.c wq = qp->io_wq; qp 19867 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); qp 19870 drivers/scsi/lpfc/lpfc_sli.c qp, wq_access); qp 19879 drivers/scsi/lpfc/lpfc_sli.c lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH); qp 19886 drivers/scsi/lpfc/lpfc_sli.c wq = qp->io_wq; qp 19897 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); qp 19900 drivers/scsi/lpfc/lpfc_sli.c qp, wq_access); qp 19909 drivers/scsi/lpfc/lpfc_sli.c lpfc_sli4_poll_eq(qp->hba_eq, LPFC_POLL_FASTPATH); qp 19932 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 19938 drivers/scsi/lpfc/lpfc_sli.c qp = &phba->sli4_hba.hdwq[hwqid]; qp 19939 drivers/scsi/lpfc/lpfc_sli.c multixri_pool = qp->p_multixri_pool; qp 19944 drivers/scsi/lpfc/lpfc_sli.c pvt_pool = &qp->p_multixri_pool->pvt_pool; qp 19945 drivers/scsi/lpfc/lpfc_sli.c pbl_pool = &qp->p_multixri_pool->pbl_pool; qp 19946 drivers/scsi/lpfc/lpfc_sli.c txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; qp 20005 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 20007 drivers/scsi/lpfc/lpfc_sli.c qp = &phba->sli4_hba.hdwq[hwqid]; qp 20008 drivers/scsi/lpfc/lpfc_sli.c multixri_pool = qp->p_multixri_pool; qp 20016 drivers/scsi/lpfc/lpfc_sli.c txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; qp 20017 drivers/scsi/lpfc/lpfc_sli.c abts_io_bufs = qp->abts_scsi_io_bufs; qp 20018 drivers/scsi/lpfc/lpfc_sli.c abts_io_bufs += qp->abts_nvme_io_bufs; qp 20045 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 20052 drivers/scsi/lpfc/lpfc_sli.c qp = &phba->sli4_hba.hdwq[hwqid]; qp 20053 drivers/scsi/lpfc/lpfc_sli.c pbl_pool = &qp->p_multixri_pool->pbl_pool; qp 20054 drivers/scsi/lpfc/lpfc_sli.c pvt_pool = &qp->p_multixri_pool->pvt_pool; qp 20057 drivers/scsi/lpfc/lpfc_sli.c lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, qp, mv_to_pub_pool); qp 20058 drivers/scsi/lpfc/lpfc_sli.c lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_from_pvt_pool); qp 20111 drivers/scsi/lpfc/lpfc_sli.c _lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, qp 20124 drivers/scsi/lpfc/lpfc_sli.c lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_to_pvt_pool); qp 20166 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 20171 drivers/scsi/lpfc/lpfc_sli.c qp = &phba->sli4_hba.hdwq[hwqid]; qp 20172 drivers/scsi/lpfc/lpfc_sli.c multixri_pool = qp->p_multixri_pool; qp 20177 drivers/scsi/lpfc/lpfc_sli.c ret = _lpfc_move_xri_pbl_to_pvt(phba, qp, pbl_pool, pvt_pool, count); qp 20200 drivers/scsi/lpfc/lpfc_sli.c phba, qp, pbl_pool, pvt_pool, count); qp 20256 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp) qp 20294 drivers/scsi/lpfc/lpfc_sli.c if (!qp->p_multixri_pool) qp 20297 drivers/scsi/lpfc/lpfc_sli.c pbl_pool = &qp->p_multixri_pool->pbl_pool; qp 20298 drivers/scsi/lpfc/lpfc_sli.c pvt_pool = &qp->p_multixri_pool->pvt_pool; qp 20300 drivers/scsi/lpfc/lpfc_sli.c txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; qp 20301 drivers/scsi/lpfc/lpfc_sli.c abts_io_bufs = qp->abts_scsi_io_bufs; qp 20302 drivers/scsi/lpfc/lpfc_sli.c abts_io_bufs += qp->abts_nvme_io_bufs; qp 20305 drivers/scsi/lpfc/lpfc_sli.c xri_limit = qp->p_multixri_pool->xri_limit; qp 20309 drivers/scsi/lpfc/lpfc_sli.c qp->p_multixri_pool->below_limit_count++; qp 20311 drivers/scsi/lpfc/lpfc_sli.c qp->p_multixri_pool->above_limit_count++; qp 20321 drivers/scsi/lpfc/lpfc_sli.c qp, free_pvt_pool); qp 20328 drivers/scsi/lpfc/lpfc_sli.c qp, free_pub_pool); qp 20335 drivers/scsi/lpfc/lpfc_sli.c lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag, qp 20336 drivers/scsi/lpfc/lpfc_sli.c qp, free_xri); qp 20338 drivers/scsi/lpfc/lpfc_sli.c &qp->lpfc_io_buf_list_put); qp 20339 drivers/scsi/lpfc/lpfc_sli.c qp->put_io_bufs++; qp 20340 drivers/scsi/lpfc/lpfc_sli.c spin_unlock_irqrestore(&qp->io_buf_list_put_lock, qp 20359 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp, qp 20367 drivers/scsi/lpfc/lpfc_sli.c lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, qp, alloc_pvt_pool); qp 20446 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 20451 drivers/scsi/lpfc/lpfc_sli.c qp = &phba->sli4_hba.hdwq[hwqid]; qp 20453 drivers/scsi/lpfc/lpfc_sli.c multixri_pool = qp->p_multixri_pool; qp 20462 drivers/scsi/lpfc/lpfc_sli.c lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, qp, pvt_pool, ndlp); qp 20465 drivers/scsi/lpfc/lpfc_sli.c lpfc_ncmd->hdwq = qp; qp 20480 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 20483 drivers/scsi/lpfc/lpfc_sli.c qp = &phba->sli4_hba.hdwq[idx]; qp 20485 drivers/scsi/lpfc/lpfc_sli.c &qp->lpfc_io_buf_list_get, list) { qp 20494 drivers/scsi/lpfc/lpfc_sli.c qp->get_io_bufs--; qp 20495 drivers/scsi/lpfc/lpfc_sli.c lpfc_cmd->hdwq = qp; qp 20524 drivers/scsi/lpfc/lpfc_sli.c struct lpfc_sli4_hdw_queue *qp; qp 20528 drivers/scsi/lpfc/lpfc_sli.c qp = &phba->sli4_hba.hdwq[hwqid]; qp 20535 drivers/scsi/lpfc/lpfc_sli.c lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag, qp 20536 drivers/scsi/lpfc/lpfc_sli.c qp, alloc_xri_get); qp 20537 drivers/scsi/lpfc/lpfc_sli.c if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) qp 20540 drivers/scsi/lpfc/lpfc_sli.c lpfc_qp_spin_lock(&qp->io_buf_list_put_lock, qp 20541 drivers/scsi/lpfc/lpfc_sli.c qp, alloc_xri_put); qp 20542 drivers/scsi/lpfc/lpfc_sli.c list_splice(&qp->lpfc_io_buf_list_put, qp 20543 drivers/scsi/lpfc/lpfc_sli.c &qp->lpfc_io_buf_list_get); qp 20544 drivers/scsi/lpfc/lpfc_sli.c qp->get_io_bufs += qp->put_io_bufs; qp 20545 drivers/scsi/lpfc/lpfc_sli.c INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); qp 20546 drivers/scsi/lpfc/lpfc_sli.c qp->put_io_bufs = 0; qp 20547 drivers/scsi/lpfc/lpfc_sli.c spin_unlock(&qp->io_buf_list_put_lock); qp 20548 drivers/scsi/lpfc/lpfc_sli.c if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || qp 20552 drivers/scsi/lpfc/lpfc_sli.c spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); qp 705 drivers/scsi/lpfc/lpfc_sli4.h #define lpfc_qp_spin_lock_irqsave(lock, flag, qp, lstat) \ qp 711 drivers/scsi/lpfc/lpfc_sli4.h qp->lock_conflict.lstat++; \ qp 715 drivers/scsi/lpfc/lpfc_sli4.h #define lpfc_qp_spin_lock(lock, qp, lstat) \ qp 721 drivers/scsi/lpfc/lpfc_sli4.h qp->lock_conflict.lstat++; \ qp 726 drivers/scsi/lpfc/lpfc_sli4.h #define lpfc_qp_spin_lock_irqsave(lock, flag, qp, lstat) \ qp 728 drivers/scsi/lpfc/lpfc_sli4.h #define lpfc_qp_spin_lock(lock, qp, lstat) spin_lock(lock) qp 4421 drivers/scsi/ncr53c8xx.c struct list_head *qp; qp 4428 drivers/scsi/ncr53c8xx.c qp = ncr_list_pop(&lp->wait_ccbq); qp 4429 drivers/scsi/ncr53c8xx.c if (!qp) qp 4432 drivers/scsi/ncr53c8xx.c cp = list_entry(qp, struct ccb, link_ccbq); qp 4433 drivers/scsi/ncr53c8xx.c list_add_tail(qp, &lp->busy_ccbq); qp 6537 drivers/scsi/ncr53c8xx.c struct list_head *qp; qp 6551 drivers/scsi/ncr53c8xx.c qp = lp->busy_ccbq.prev; qp 6552 drivers/scsi/ncr53c8xx.c while (qp != &lp->busy_ccbq) { qp 6553 drivers/scsi/ncr53c8xx.c cp2 = list_entry(qp, struct ccb, link_ccbq); qp 6554 drivers/scsi/ncr53c8xx.c qp = qp->prev; qp 7148 drivers/scsi/ncr53c8xx.c struct list_head *qp; qp 7164 drivers/scsi/ncr53c8xx.c qp = ncr_list_pop(&lp->free_ccbq); qp 7165 drivers/scsi/ncr53c8xx.c if (qp) { qp 7166 drivers/scsi/ncr53c8xx.c cp = list_entry(qp, struct ccb, link_ccbq); qp 7172 drivers/scsi/ncr53c8xx.c list_add_tail(qp, &lp->wait_ccbq); qp 238 drivers/scsi/pm8001/pm8001_sas.c uint32_t *qp = (uint32_t *)(((char *) qp 242 drivers/scsi/pm8001/pm8001_sas.c phy->invalid_dword_count = qp[0]; qp 243 drivers/scsi/pm8001/pm8001_sas.c phy->running_disparity_error_count = qp[1]; qp 244 drivers/scsi/pm8001/pm8001_sas.c phy->loss_of_dword_sync_count = qp[3]; qp 245 drivers/scsi/pm8001/pm8001_sas.c phy->phy_reset_problem_count = qp[4]; qp 3545 drivers/scsi/qla2xxx/qla_iocb.c struct qla_qpair *qp = sp->qpair; qp 3549 drivers/scsi/qla2xxx/qla_iocb.c spin_lock_irqsave(qp->qp_lock_ptr, flags); qp 3631 drivers/scsi/qla2xxx/qla_iocb.c qla2x00_start_iocbs(vha, qp->req); qp 3633 drivers/scsi/qla2xxx/qla_iocb.c spin_unlock_irqrestore(qp->qp_lock_ptr, flags); qp 1691 drivers/scsi/qla2xxx/qla_os.c static void qla2x00_abort_srb(struct qla_qpair *qp, srb_t *sp, const int res, qp 1693 drivers/scsi/qla2xxx/qla_os.c __releases(qp->qp_lock_ptr) qp 1694 drivers/scsi/qla2xxx/qla_os.c __acquires(qp->qp_lock_ptr) qp 1697 drivers/scsi/qla2xxx/qla_os.c scsi_qla_host_t *vha = qp->vha; qp 1719 drivers/scsi/qla2xxx/qla_os.c spin_unlock_irqrestore(qp->qp_lock_ptr, *flags); qp 1741 drivers/scsi/qla2xxx/qla_os.c spin_lock_irqsave(qp->qp_lock_ptr, *flags); qp 1750 drivers/scsi/qla2xxx/qla_os.c __qla2x00_abort_all_cmds(struct qla_qpair *qp, int res) qp 1755 drivers/scsi/qla2xxx/qla_os.c scsi_qla_host_t *vha = qp->vha; qp 1763 drivers/scsi/qla2xxx/qla_os.c spin_lock_irqsave(qp->qp_lock_ptr, flags); qp 1764 drivers/scsi/qla2xxx/qla_os.c req = qp->req; qp 1770 drivers/scsi/qla2xxx/qla_os.c qla2x00_abort_srb(qp, sp, res, &flags); qp 1792 drivers/scsi/qla2xxx/qla_os.c spin_unlock_irqrestore(qp->qp_lock_ptr, flags); qp 4190 drivers/scsi/qla2xxx/qla_target.c struct qla_qpair *qpair, *qp; qp 4221 drivers/scsi/qla2xxx/qla_target.c list_for_each_entry(qp, &base_vha->qp_list, qp 4223 drivers/scsi/qla2xxx/qla_target.c if (qp->lun_cnt == 0) { qp 4224 drivers/scsi/qla2xxx/qla_target.c qp->lun_cnt++; qp 4225 drivers/scsi/qla2xxx/qla_target.c h = qla_qpair_to_hint(tgt, qp); qp 4230 drivers/scsi/qla2xxx/qla_target.c qp->lun_cnt--; qp 4235 drivers/scsi/qla2xxx/qla_target.c qpair = qp; qp 4238 drivers/scsi/qla2xxx/qla_target.c if (qp->lun_cnt < lcnt) { qp 4239 drivers/scsi/qla2xxx/qla_target.c lcnt = qp->lun_cnt; qp 4240 drivers/scsi/qla2xxx/qla_target.c qpair = qp; qp 504 drivers/scsi/qlogicpti.h #define for_each_qlogicpti(qp) \ qp 505 drivers/scsi/qlogicpti.h for((qp) = qptichain; (qp); (qp) = (qp)->next) qp 585 drivers/scsi/sym53c8xx_2/sym_glue.c SYM_QUEHEAD *qp; qp 624 drivers/scsi/sym53c8xx_2/sym_glue.c FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) { qp 625 drivers/scsi/sym53c8xx_2/sym_glue.c struct sym_ccb *cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 1529 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD *qp; qp 1543 drivers/scsi/sym53c8xx_2/sym_hipd.c qp = sym_remque_head(&lp->waiting_ccbq); qp 1544 drivers/scsi/sym53c8xx_2/sym_hipd.c if (!qp) qp 1546 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link2_ccbq); qp 1550 drivers/scsi/sym53c8xx_2/sym_hipd.c sym_insque_head(qp, &lp->waiting_ccbq); qp 1559 drivers/scsi/sym53c8xx_2/sym_hipd.c sym_insque_head(qp, &lp->waiting_ccbq); qp 1568 drivers/scsi/sym53c8xx_2/sym_hipd.c sym_insque_tail(qp, &lp->started_ccbq); qp 1630 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD *qp; qp 1633 drivers/scsi/sym53c8xx_2/sym_hipd.c while ((qp = sym_remque_head(&np->comp_ccbq)) != NULL) { qp 1635 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 1934 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD *qp; qp 2022 drivers/scsi/sym53c8xx_2/sym_hipd.c FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) { qp 2024 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 3188 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD qtmp, *qp; qp 3204 drivers/scsi/sym53c8xx_2/sym_hipd.c while ((qp = sym_remque_head(&qtmp)) != NULL) { qp 3206 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 3271 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD *qp; qp 3312 drivers/scsi/sym53c8xx_2/sym_hipd.c FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) { qp 3313 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp,struct sym_ccb,link_ccbq); qp 3344 drivers/scsi/sym53c8xx_2/sym_hipd.c FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) { qp 3345 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 3460 drivers/scsi/sym53c8xx_2/sym_hipd.c FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) { qp 3461 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 4652 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD *qp; qp 4660 drivers/scsi/sym53c8xx_2/sym_hipd.c qp = sym_remque_head(&np->free_ccbq); qp 4661 drivers/scsi/sym53c8xx_2/sym_hipd.c if (!qp) qp 4663 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 5324 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD *qp; qp 5330 drivers/scsi/sym53c8xx_2/sym_hipd.c FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) { qp 5331 drivers/scsi/sym53c8xx_2/sym_hipd.c struct sym_ccb *cp2 = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 5795 drivers/scsi/sym53c8xx_2/sym_hipd.c SYM_QUEHEAD *qp; qp 5812 drivers/scsi/sym53c8xx_2/sym_hipd.c while ((qp = sym_remque_head(&np->free_ccbq)) != NULL) { qp 5813 drivers/scsi/sym53c8xx_2/sym_hipd.c cp = sym_que_entry(qp, struct sym_ccb, link_ccbq); qp 146 drivers/scsi/sym53c8xx_2/sym_misc.h #define FOR_EACH_QUEUED_ELEMENT(head, qp) \ qp 147 drivers/scsi/sym53c8xx_2/sym_misc.h for (qp = (head)->flink; qp != (head); qp = qp->flink) qp 2886 drivers/soc/fsl/qbman/qman.c static int qpool_cleanup(u32 qp) qp 2915 drivers/soc/fsl/qbman/qman.c if (qm_fqd_get_chan(&fqd) == qp) { qp 2931 drivers/soc/fsl/qbman/qman.c int qman_release_pool(u32 qp) qp 2935 drivers/soc/fsl/qbman/qman.c ret = qpool_cleanup(qp); qp 2937 drivers/soc/fsl/qbman/qman.c pr_debug("CHID %d leaked\n", qp); qp 2941 drivers/soc/fsl/qbman/qman.c gen_pool_free(qm_qpalloc, qp | DPAA_GENALLOC_OFF, 1); qp 502 drivers/staging/media/allegro-dvt/allegro-core.c u16 qp; qp 503 fs/cifs/smbdirect.c wc->qp->device, qp 761 fs/cifs/smbdirect.c rc = ib_post_send(info->id->qp, &send_wr, NULL); qp 962 fs/cifs/smbdirect.c rc = ib_post_send(info->id->qp, &send_wr, NULL); qp 1111 fs/cifs/smbdirect.c rc = ib_post_recv(info->id->qp, &recv_wr, NULL); qp 1412 fs/cifs/smbdirect.c ib_drain_qp(info->id->qp); qp 2527 fs/cifs/smbdirect.c rc = ib_post_send(info->id->qp, ®_wr->wr, NULL); qp 2587 fs/cifs/smbdirect.c rc = ib_post_send(info->id->qp, wr, NULL); qp 766 fs/gfs2/quota.c static int gfs2_write_disk_quota(struct gfs2_inode *ip, struct gfs2_quota *qp, qp 785 fs/gfs2/quota.c ptr = qp; qp 842 include/linux/mlx4/device.h int qp; qp 930 include/linux/mlx4/device.h } __packed qp; qp 1147 include/linux/mlx4/device.h int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp); qp 1148 include/linux/mlx4/device.h void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp); qp 1159 include/linux/mlx4/device.h int mlx4_unicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1161 include/linux/mlx4/device.h int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1163 include/linux/mlx4/device.h int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 1166 include/linux/mlx4/device.h int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], qp 480 include/linux/mlx4/qp.h int sqd_event, struct mlx4_qp *qp); qp 482 include/linux/mlx4/qp.h int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, qp 487 include/linux/mlx4/qp.h struct mlx4_qp *qp, enum mlx4_qp_state *qp_state); qp 494 include/linux/mlx4/qp.h void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp); qp 562 include/linux/mlx5/qp.h struct mlx5_core_dct *qp, qp 566 include/linux/mlx5/qp.h struct mlx5_core_qp *qp, qp 571 include/linux/mlx5/qp.h struct mlx5_core_qp *qp); qp 573 include/linux/mlx5/qp.h struct mlx5_core_qp *qp); qp 576 include/linux/mlx5/qp.h int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp, qp 588 include/linux/mlx5/qp.h int mlx5_debug_qp_add(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp); qp 589 include/linux/mlx5/qp.h void mlx5_debug_qp_remove(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp); qp 65 include/linux/ntb_transport.h void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, qp 67 include/linux/ntb_transport.h void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, qp 72 include/linux/ntb_transport.h unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp); qp 73 include/linux/ntb_transport.h unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); qp 77 include/linux/ntb_transport.h void ntb_transport_free_queue(struct ntb_transport_qp *qp); qp 78 include/linux/ntb_transport.h int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, qp 80 include/linux/ntb_transport.h int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, qp 82 include/linux/ntb_transport.h void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len); qp 83 include/linux/ntb_transport.h void ntb_transport_link_up(struct ntb_transport_qp *qp); qp 84 include/linux/ntb_transport.h void ntb_transport_link_down(struct ntb_transport_qp *qp); qp 85 include/linux/ntb_transport.h bool ntb_transport_link_query(struct ntb_transport_qp *qp); qp 86 include/linux/ntb_transport.h unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp); qp 513 include/linux/qed/qed_rdma_if.h struct qed_rdma_qp *qp; qp 541 include/linux/qed/qed_rdma_if.h struct qed_rdma_qp *qp; qp 625 include/linux/qed/qed_rdma_if.h int (*rdma_modify_qp)(void *roce_cxt, struct qed_rdma_qp *qp, qp 628 include/linux/qed/qed_rdma_if.h int (*rdma_query_qp)(void *rdma_cxt, struct qed_rdma_qp *qp, qp 630 include/linux/qed/qed_rdma_if.h int (*rdma_destroy_qp)(void *rdma_cxt, struct qed_rdma_qp *qp); qp 165 include/net/erspan.h } *qp; qp 180 include/net/erspan.h qp = (struct qtag_prefix *)(skb->data + 2 * ETH_ALEN); qp 181 include/net/erspan.h vlan_tci = ntohs(qp->tci); qp 259 include/net/erspan.h } *qp; qp 274 include/net/erspan.h qp = (struct qtag_prefix *)(skb->data + 2 * ETH_ALEN); qp 275 include/net/erspan.h vlan_tci = ntohs(qp->tci); qp 612 include/rdma/ib_mad.h struct ib_qp *qp; qp 833 include/rdma/ib_mad.h struct ib_mad_agent *ib_redirect_mad_qp(struct ib_qp *qp, qp 722 include/rdma/ib_verbs.h struct ib_qp *qp; qp 977 include/rdma/ib_verbs.h struct ib_qp *qp; qp 1691 include/rdma/ib_verbs.h struct ib_qp *qp; qp 2060 include/rdma/ib_verbs.h struct ib_qp *qp; qp 2264 include/rdma/ib_verbs.h int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr, qp 2266 include/rdma/ib_verbs.h int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, qp 2268 include/rdma/ib_verbs.h void (*drain_rq)(struct ib_qp *qp); qp 2269 include/rdma/ib_verbs.h void (*drain_sq)(struct ib_qp *qp); qp 2384 include/rdma/ib_verbs.h int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 2386 include/rdma/ib_verbs.h int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, qp 2388 include/rdma/ib_verbs.h int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata); qp 2426 include/rdma/ib_verbs.h int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); qp 2427 include/rdma/ib_verbs.h int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); qp 2431 include/rdma/ib_verbs.h struct ib_flow *(*create_flow)(struct ib_qp *qp, qp 2528 include/rdma/ib_verbs.h void (*iw_add_ref)(struct ib_qp *qp); qp 2529 include/rdma/ib_verbs.h void (*iw_rem_ref)(struct ib_qp *qp); qp 2544 include/rdma/ib_verbs.h int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp); qp 2549 include/rdma/ib_verbs.h int (*counter_unbind_qp)(struct ib_qp *qp); qp 3613 include/rdma/ib_verbs.h int ib_modify_qp_with_udata(struct ib_qp *qp, qp 3627 include/rdma/ib_verbs.h int ib_modify_qp(struct ib_qp *qp, qp 3642 include/rdma/ib_verbs.h int ib_query_qp(struct ib_qp *qp, qp 3652 include/rdma/ib_verbs.h int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata); qp 3660 include/rdma/ib_verbs.h static inline int ib_destroy_qp(struct ib_qp *qp) qp 3662 include/rdma/ib_verbs.h return ib_destroy_qp_user(qp, NULL); qp 3682 include/rdma/ib_verbs.h int ib_close_qp(struct ib_qp *qp); qp 3697 include/rdma/ib_verbs.h static inline int ib_post_send(struct ib_qp *qp, qp 3703 include/rdma/ib_verbs.h return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy); qp 3714 include/rdma/ib_verbs.h static inline int ib_post_recv(struct ib_qp *qp, qp 3720 include/rdma/ib_verbs.h return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy); qp 4222 include/rdma/ib_verbs.h int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); qp 4230 include/rdma/ib_verbs.h int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); qp 4355 include/rdma/ib_verbs.h void ib_drain_rq(struct ib_qp *qp); qp 4356 include/rdma/ib_verbs.h void ib_drain_sq(struct ib_qp *qp); qp 4357 include/rdma/ib_verbs.h void ib_drain_qp(struct ib_qp *qp); qp 153 include/rdma/iw_cm.h void iw_cm_unbind_qp(struct iw_cm_id *cm_id, struct ib_qp *qp); qp 10 include/rdma/mr_pool.h struct ib_mr *ib_mr_pool_get(struct ib_qp *qp, struct list_head *list); qp 11 include/rdma/mr_pool.h void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr); qp 13 include/rdma/mr_pool.h int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr, qp 15 include/rdma/mr_pool.h void ib_mr_pool_destroy(struct ib_qp *qp, struct list_head *list); qp 131 include/rdma/rdma_cm.h struct ib_qp *qp; qp 50 include/rdma/rdma_counter.h int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port); qp 51 include/rdma/rdma_counter.h int rdma_counter_unbind_qp(struct ib_qp *qp, bool force); qp 77 include/rdma/rdma_vt.h struct rvt_qp __rcu *qp[2]; qp 255 include/rdma/rdma_vt.h bool (*schedule_send)(struct rvt_qp *qp); qp 256 include/rdma/rdma_vt.h bool (*schedule_send_no_lock)(struct rvt_qp *qp); qp 265 include/rdma/rdma_vt.h int (*setup_wqe)(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 272 include/rdma/rdma_vt.h void (*do_send)(struct rvt_qp *qp); qp 287 include/rdma/rdma_vt.h void * (*qp_priv_alloc)(struct rvt_dev_info *rdi, struct rvt_qp *qp); qp 293 include/rdma/rdma_vt.h int (*qp_priv_init)(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 299 include/rdma/rdma_vt.h void (*qp_priv_free)(struct rvt_dev_info *rdi, struct rvt_qp *qp); qp 305 include/rdma/rdma_vt.h void (*notify_qp_reset)(struct rvt_qp *qp); qp 310 include/rdma/rdma_vt.h int (*get_pmtu_from_attr)(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 317 include/rdma/rdma_vt.h void (*flush_qp_waiters)(struct rvt_qp *qp); qp 323 include/rdma/rdma_vt.h void (*stop_send_queue)(struct rvt_qp *qp); qp 328 include/rdma/rdma_vt.h void (*quiesce_qp)(struct rvt_qp *qp); qp 333 include/rdma/rdma_vt.h void (*notify_error_qp)(struct rvt_qp *qp); qp 338 include/rdma/rdma_vt.h u32 (*mtu_from_qp)(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp 386 include/rdma/rdma_vt.h int (*check_modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr, qp 390 include/rdma/rdma_vt.h void (*modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr, qp 400 include/rdma/rdma_vt.h void (*notify_restart_rc)(struct rvt_qp *qp, u32 psn, int wait); qp 563 include/rdma/rdma_vt.h int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key, qp 565 include/rdma/rdma_vt.h int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey); qp 566 include/rdma/rdma_vt.h int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, qp 506 include/rdma/rdmavt_qp.h struct rvt_qp *qp; qp 527 include/rdma/rdmavt_qp.h static inline struct rvt_swqe *rvt_get_swqe_ptr(struct rvt_qp *qp, qp 530 include/rdma/rdmavt_qp.h return (struct rvt_swqe *)((char *)qp->s_wq + qp 532 include/rdma/rdmavt_qp.h qp->s_max_sge * qp 552 include/rdma/rdmavt_qp.h static inline bool rvt_is_user_qp(struct rvt_qp *qp) qp 554 include/rdma/rdmavt_qp.h return !!qp->pid; qp 561 include/rdma/rdmavt_qp.h static inline void rvt_get_qp(struct rvt_qp *qp) qp 563 include/rdma/rdmavt_qp.h atomic_inc(&qp->refcount); qp 570 include/rdma/rdmavt_qp.h static inline void rvt_put_qp(struct rvt_qp *qp) qp 572 include/rdma/rdmavt_qp.h if (qp && atomic_dec_and_test(&qp->refcount)) qp 573 include/rdma/rdmavt_qp.h wake_up(&qp->wait); qp 602 include/rdma/rdmavt_qp.h struct rvt_qp *qp, qp 605 include/rdma/rdmavt_qp.h atomic_inc(&qp->s_reserved_used); qp 623 include/rdma/rdmavt_qp.h static inline void rvt_qp_wqe_unreserve(struct rvt_qp *qp, int flags) qp 626 include/rdma/rdmavt_qp.h atomic_dec(&qp->s_reserved_used); qp 649 include/rdma/rdmavt_qp.h __be32 rvt_compute_aeth(struct rvt_qp *qp); qp 658 include/rdma/rdmavt_qp.h void rvt_get_credit(struct rvt_qp *qp, u32 aeth); qp 676 include/rdma/rdmavt_qp.h static inline u32 rvt_div_round_up_mtu(struct rvt_qp *qp, u32 len) qp 678 include/rdma/rdmavt_qp.h return (len + qp->pmtu - 1) >> qp->log_pmtu; qp 687 include/rdma/rdmavt_qp.h static inline u32 rvt_div_mtu(struct rvt_qp *qp, u32 len) qp 689 include/rdma/rdmavt_qp.h return len >> qp->log_pmtu; qp 718 include/rdma/rdmavt_qp.h struct rvt_qp *qp = NULL; qp 721 include/rdma/rdmavt_qp.h qp = rcu_dereference(rvp->qp[qpn]); qp 725 include/rdma/rdmavt_qp.h for (qp = rcu_dereference(rdi->qp_dev->qp_table[n]); qp; qp 726 include/rdma/rdmavt_qp.h qp = rcu_dereference(qp->next)) qp 727 include/rdma/rdmavt_qp.h if (qp->ibqp.qp_num == qpn) qp 730 include/rdma/rdmavt_qp.h return qp; qp 739 include/rdma/rdmavt_qp.h static inline void rvt_mod_retry_timer_ext(struct rvt_qp *qp, u8 shift) qp 741 include/rdma/rdmavt_qp.h struct ib_qp *ibqp = &qp->ibqp; qp 744 include/rdma/rdmavt_qp.h lockdep_assert_held(&qp->s_lock); qp 745 include/rdma/rdmavt_qp.h qp->s_flags |= RVT_S_TIMER; qp 747 include/rdma/rdmavt_qp.h mod_timer(&qp->s_timer, jiffies + rdi->busy_jiffies + qp 748 include/rdma/rdmavt_qp.h (qp->timeout_jiffies << shift)); qp 751 include/rdma/rdmavt_qp.h static inline void rvt_mod_retry_timer(struct rvt_qp *qp) qp 753 include/rdma/rdmavt_qp.h return rvt_mod_retry_timer_ext(qp, 0); qp 763 include/rdma/rdmavt_qp.h static inline void rvt_put_qp_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) qp 766 include/rdma/rdmavt_qp.h if (qp->allowed_ops == IB_OPCODE_UD) qp 778 include/rdma/rdmavt_qp.h rvt_qp_swqe_incr(struct rvt_qp *qp, u32 val) qp 780 include/rdma/rdmavt_qp.h if (++val >= qp->s_size) qp 785 include/rdma/rdmavt_qp.h int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err); qp 798 include/rdma/rdmavt_qp.h static inline void rvt_recv_cq(struct rvt_qp *qp, struct ib_wc *wc, qp 801 include/rdma/rdmavt_qp.h struct rvt_cq *cq = ibcq_to_rvtcq(qp->ibqp.recv_cq); qp 804 include/rdma/rdmavt_qp.h rvt_error_qp(qp, IB_WC_LOC_QP_OP_ERR); qp 818 include/rdma/rdmavt_qp.h static inline void rvt_send_cq(struct rvt_qp *qp, struct ib_wc *wc, qp 821 include/rdma/rdmavt_qp.h struct rvt_cq *cq = ibcq_to_rvtcq(qp->ibqp.send_cq); qp 824 include/rdma/rdmavt_qp.h rvt_error_qp(qp, IB_WC_LOC_QP_OP_ERR); qp 844 include/rdma/rdmavt_qp.h rvt_qp_complete_swqe(struct rvt_qp *qp, qp 854 include/rdma/rdmavt_qp.h rvt_qp_wqe_unreserve(qp, flags); qp 855 include/rdma/rdmavt_qp.h rvt_put_qp_swqe(qp, wqe); qp 859 include/rdma/rdmavt_qp.h (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) || qp 867 include/rdma/rdmavt_qp.h last = rvt_qp_swqe_incr(qp, qp->s_last); qp 869 include/rdma/rdmavt_qp.h smp_store_release(&qp->s_last, last); qp 875 include/rdma/rdmavt_qp.h .qp = &qp->ibqp, qp 878 include/rdma/rdmavt_qp.h rvt_send_cq(qp, &w, status != IB_WC_SUCCESS); qp 886 include/rdma/rdmavt_qp.h int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only); qp 887 include/rdma/rdmavt_qp.h void rvt_comm_est(struct rvt_qp *qp); qp 888 include/rdma/rdmavt_qp.h void rvt_rc_error(struct rvt_qp *qp, enum ib_wc_status err); qp 891 include/rdma/rdmavt_qp.h void rvt_add_rnr_timer(struct rvt_qp *qp, u32 aeth); qp 892 include/rdma/rdmavt_qp.h void rvt_del_timers_sync(struct rvt_qp *qp); qp 893 include/rdma/rdmavt_qp.h void rvt_stop_rc_timers(struct rvt_qp *qp); qp 894 include/rdma/rdmavt_qp.h void rvt_add_retry_timer_ext(struct rvt_qp *qp, u8 shift); qp 895 include/rdma/rdmavt_qp.h static inline void rvt_add_retry_timer(struct rvt_qp *qp) qp 897 include/rdma/rdmavt_qp.h rvt_add_retry_timer_ext(qp, 0); qp 900 include/rdma/rdmavt_qp.h void rvt_copy_sge(struct rvt_qp *qp, struct rvt_sge_state *ss, qp 903 include/rdma/rdmavt_qp.h void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, qp 905 include/rdma/rdmavt_qp.h void rvt_ruc_loopback(struct rvt_qp *qp); qp 916 include/rdma/rdmavt_qp.h struct rvt_qp *qp; qp 920 include/rdma/rdmavt_qp.h void (*cb)(struct rvt_qp *qp, u64 v); qp 982 include/rdma/rdmavt_qp.h static inline struct rvt_ibport *rvt_to_iport(struct rvt_qp *qp) qp 984 include/rdma/rdmavt_qp.h struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); qp 986 include/rdma/rdmavt_qp.h return rdi->ports[qp->port_num - 1]; qp 997 include/rdma/rdmavt_qp.h static inline bool rvt_rc_credit_avail(struct rvt_qp *qp, struct rvt_swqe *wqe) qp 999 include/rdma/rdmavt_qp.h lockdep_assert_held(&qp->s_lock); qp 1000 include/rdma/rdmavt_qp.h if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) && qp 1001 include/rdma/rdmavt_qp.h rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) { qp 1002 include/rdma/rdmavt_qp.h struct rvt_ibport *rvp = rvt_to_iport(qp); qp 1004 include/rdma/rdmavt_qp.h qp->s_flags |= RVT_S_WAIT_SSN_CREDIT; qp 1013 include/rdma/rdmavt_qp.h void (*cb)(struct rvt_qp *qp, u64 v)); qp 1017 include/rdma/rdmavt_qp.h void (*cb)(struct rvt_qp *qp, u64 v)); qp 1018 include/rdma/rdmavt_qp.h void rvt_qp_mr_clean(struct rvt_qp *qp, u32 lkey); qp 45 include/rdma/rw.h int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, qp 48 include/rdma/rw.h void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, qp 52 include/rdma/rw.h int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 57 include/rdma/rw.h void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 62 include/rdma/rw.h struct ib_send_wr *rdma_rw_ctx_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp, qp 64 include/rdma/rw.h int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, qp 70 include/rdma/rw.h int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr); qp 71 include/rdma/rw.h void rdma_rw_cleanup_mrs(struct ib_qp *qp); qp 171 include/rdma/uverbs_std_types.h struct ib_qp *qp, struct ib_device *device, qp 179 include/rdma/uverbs_std_types.h if (qp) { qp 180 include/rdma/uverbs_std_types.h atomic_inc(&qp->usecnt); qp 181 include/rdma/uverbs_std_types.h ibflow->qp = qp; qp 58 include/trace/events/ib_mad.h __entry->qp_num = wr->mad_agent_priv->qp_info->qp->qp_num; qp 147 include/trace/events/ib_mad.h __entry->qp_num = wr->mad_agent_priv->qp_info->qp->qp_num; qp 213 include/trace/events/ib_mad.h __entry->qp_num = qp_info->qp->qp_num; qp 280 include/uapi/rdma/vmw_pvrdma-abi.h __aligned_u64 qp; qp 149 lib/mpi/mpi-internal.h mpi_limb_t mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs, qp 45 lib/mpi/mpih-div.c mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs, qp 74 lib/mpi/mpih-div.c qp += qextra_limbs; qp 76 lib/mpi/mpih-div.c udiv_qrnnd(qp[i], n1, n1, np[i], d); qp 77 lib/mpi/mpih-div.c qp -= qextra_limbs; qp 80 lib/mpi/mpih-div.c udiv_qrnnd(qp[i], n1, n1, 0, d); qp 122 lib/mpi/mpih-div.c qp[i] = q; qp 143 lib/mpi/mpih-div.c qp[i] = q; qp 216 lib/mpi/mpih-div.c qp[i] = q; qp 423 mm/mempolicy.c struct queue_pages *qp) qp 426 mm/mempolicy.c unsigned long flags = qp->flags; qp 428 mm/mempolicy.c return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT); qp 446 mm/mempolicy.c struct queue_pages *qp = walk->private; qp 460 mm/mempolicy.c if (!queue_pages_required(page, qp)) qp 463 mm/mempolicy.c flags = qp->flags; qp 467 mm/mempolicy.c migrate_page_add(page, qp->pagelist, flags)) { qp 495 mm/mempolicy.c struct queue_pages *qp = walk->private; qp 496 mm/mempolicy.c unsigned long flags = qp->flags; qp 526 mm/mempolicy.c if (!queue_pages_required(page, qp)) qp 540 mm/mempolicy.c if (migrate_page_add(page, qp->pagelist, flags)) qp 559 mm/mempolicy.c struct queue_pages *qp = walk->private; qp 560 mm/mempolicy.c unsigned long flags = qp->flags; qp 570 mm/mempolicy.c if (!queue_pages_required(page, qp)) qp 575 mm/mempolicy.c isolate_huge_page(page, qp->pagelist); qp 617 mm/mempolicy.c struct queue_pages *qp = walk->private; qp 619 mm/mempolicy.c unsigned long flags = qp->flags; qp 637 mm/mempolicy.c if (qp->prev && qp->prev->vm_end < vma->vm_start) qp 641 mm/mempolicy.c qp->prev = vma; qp 684 mm/mempolicy.c struct queue_pages qp = { qp 691 mm/mempolicy.c return walk_page_range(mm, start, end, &queue_pages_walk_ops, &qp); qp 81 net/9p/trans_rdma.c struct ib_qp *qp; qp 367 net/9p/trans_rdma.c if (rdma->qp && !IS_ERR(rdma->qp)) qp 368 net/9p/trans_rdma.c ib_destroy_qp(rdma->qp); qp 405 net/9p/trans_rdma.c return ib_post_recv(rdma->qp, &wr, NULL); qp 510 net/9p/trans_rdma.c err = ib_post_send(rdma->qp, &wr, NULL); qp 714 net/9p/trans_rdma.c rdma->qp = rdma->cm_id->qp; qp 78 net/ipv4/ip_fragment.c static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, qp 84 net/ipv4/ip_fragment.c struct ipq *qp = container_of(q, struct ipq, q); qp 90 net/ipv4/ip_fragment.c qp->ecn = 0; qp 91 net/ipv4/ip_fragment.c qp->peer = q->fqdir->max_dist ? qp 98 net/ipv4/ip_fragment.c struct ipq *qp; qp 100 net/ipv4/ip_fragment.c qp = container_of(q, struct ipq, q); qp 101 net/ipv4/ip_fragment.c if (qp->peer) qp 102 net/ipv4/ip_fragment.c inet_putpeer(qp->peer); qp 139 net/ipv4/ip_fragment.c struct ipq *qp; qp 142 net/ipv4/ip_fragment.c qp = container_of(frag, struct ipq, q); qp 143 net/ipv4/ip_fragment.c net = qp->q.fqdir->net; qp 147 net/ipv4/ip_fragment.c if (qp->q.fqdir->dead) qp 150 net/ipv4/ip_fragment.c spin_lock(&qp->q.lock); qp 152 net/ipv4/ip_fragment.c if (qp->q.flags & INET_FRAG_COMPLETE) qp 155 net/ipv4/ip_fragment.c ipq_kill(qp); qp 159 net/ipv4/ip_fragment.c if (!(qp->q.flags & INET_FRAG_FIRST_IN)) qp 166 net/ipv4/ip_fragment.c head = inet_frag_pull_head(&qp->q); qp 169 net/ipv4/ip_fragment.c head->dev = dev_get_by_index_rcu(net, qp->iif); qp 184 net/ipv4/ip_fragment.c if (frag_expire_skip_icmp(qp->q.key.v4.user) && qp 188 net/ipv4/ip_fragment.c spin_unlock(&qp->q.lock); qp 193 net/ipv4/ip_fragment.c spin_unlock(&qp->q.lock); qp 197 net/ipv4/ip_fragment.c ipq_put(qp); qp 224 net/ipv4/ip_fragment.c static int ip_frag_too_far(struct ipq *qp) qp 226 net/ipv4/ip_fragment.c struct inet_peer *peer = qp->peer; qp 227 net/ipv4/ip_fragment.c unsigned int max = qp->q.fqdir->max_dist; qp 235 net/ipv4/ip_fragment.c start = qp->rid; qp 237 net/ipv4/ip_fragment.c qp->rid = end; qp 239 net/ipv4/ip_fragment.c rc = qp->q.fragments_tail && (end - start) > max; qp 242 net/ipv4/ip_fragment.c __IP_INC_STATS(qp->q.fqdir->net, IPSTATS_MIB_REASMFAILS); qp 247 net/ipv4/ip_fragment.c static int ip_frag_reinit(struct ipq *qp) qp 251 net/ipv4/ip_fragment.c if (!mod_timer(&qp->q.timer, jiffies + qp->q.fqdir->timeout)) { qp 252 net/ipv4/ip_fragment.c refcount_inc(&qp->q.refcnt); qp 256 net/ipv4/ip_fragment.c sum_truesize = inet_frag_rbtree_purge(&qp->q.rb_fragments); qp 257 net/ipv4/ip_fragment.c sub_frag_mem_limit(qp->q.fqdir, sum_truesize); qp 259 net/ipv4/ip_fragment.c qp->q.flags = 0; qp 260 net/ipv4/ip_fragment.c qp->q.len = 0; qp 261 net/ipv4/ip_fragment.c qp->q.meat = 0; qp 262 net/ipv4/ip_fragment.c qp->q.rb_fragments = RB_ROOT; qp 263 net/ipv4/ip_fragment.c qp->q.fragments_tail = NULL; qp 264 net/ipv4/ip_fragment.c qp->q.last_run_head = NULL; qp 265 net/ipv4/ip_fragment.c qp->iif = 0; qp 266 net/ipv4/ip_fragment.c qp->ecn = 0; qp 272 net/ipv4/ip_fragment.c static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) qp 274 net/ipv4/ip_fragment.c struct net *net = qp->q.fqdir->net; qp 282 net/ipv4/ip_fragment.c if (qp->q.flags & INET_FRAG_COMPLETE) qp 286 net/ipv4/ip_fragment.c unlikely(ip_frag_too_far(qp)) && qp 287 net/ipv4/ip_fragment.c unlikely(err = ip_frag_reinit(qp))) { qp 288 net/ipv4/ip_fragment.c ipq_kill(qp); qp 308 net/ipv4/ip_fragment.c if (end < qp->q.len || qp 309 net/ipv4/ip_fragment.c ((qp->q.flags & INET_FRAG_LAST_IN) && end != qp->q.len)) qp 311 net/ipv4/ip_fragment.c qp->q.flags |= INET_FRAG_LAST_IN; qp 312 net/ipv4/ip_fragment.c qp->q.len = end; qp 319 net/ipv4/ip_fragment.c if (end > qp->q.len) { qp 321 net/ipv4/ip_fragment.c if (qp->q.flags & INET_FRAG_LAST_IN) qp 323 net/ipv4/ip_fragment.c qp->q.len = end; qp 342 net/ipv4/ip_fragment.c prev_tail = qp->q.fragments_tail; qp 343 net/ipv4/ip_fragment.c err = inet_frag_queue_insert(&qp->q, skb, offset, end); qp 348 net/ipv4/ip_fragment.c qp->iif = dev->ifindex; qp 350 net/ipv4/ip_fragment.c qp->q.stamp = skb->tstamp; qp 351 net/ipv4/ip_fragment.c qp->q.meat += skb->len; qp 352 net/ipv4/ip_fragment.c qp->ecn |= ecn; qp 353 net/ipv4/ip_fragment.c add_frag_mem_limit(qp->q.fqdir, skb->truesize); qp 355 net/ipv4/ip_fragment.c qp->q.flags |= INET_FRAG_FIRST_IN; qp 359 net/ipv4/ip_fragment.c if (fragsize > qp->q.max_size) qp 360 net/ipv4/ip_fragment.c qp->q.max_size = fragsize; qp 363 net/ipv4/ip_fragment.c fragsize > qp->max_df_size) qp 364 net/ipv4/ip_fragment.c qp->max_df_size = fragsize; qp 366 net/ipv4/ip_fragment.c if (qp->q.flags == (INET_FRAG_FIRST_IN | INET_FRAG_LAST_IN) && qp 367 net/ipv4/ip_fragment.c qp->q.meat == qp->q.len) { qp 371 net/ipv4/ip_fragment.c err = ip_frag_reasm(qp, skb, prev_tail, dev); qp 374 net/ipv4/ip_fragment.c inet_frag_kill(&qp->q); qp 389 net/ipv4/ip_fragment.c inet_frag_kill(&qp->q); qp 396 net/ipv4/ip_fragment.c static bool ip_frag_coalesce_ok(const struct ipq *qp) qp 398 net/ipv4/ip_fragment.c return qp->q.key.v4.user == IP_DEFRAG_LOCAL_DELIVER; qp 402 net/ipv4/ip_fragment.c static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, qp 405 net/ipv4/ip_fragment.c struct net *net = qp->q.fqdir->net; qp 411 net/ipv4/ip_fragment.c ipq_kill(qp); qp 413 net/ipv4/ip_fragment.c ecn = ip_frag_ecn_table[qp->ecn]; qp 420 net/ipv4/ip_fragment.c reasm_data = inet_frag_reasm_prepare(&qp->q, skb, prev_tail); qp 424 net/ipv4/ip_fragment.c len = ip_hdrlen(skb) + qp->q.len; qp 429 net/ipv4/ip_fragment.c inet_frag_reasm_finish(&qp->q, skb, reasm_data, qp 430 net/ipv4/ip_fragment.c ip_frag_coalesce_ok(qp)); qp 433 net/ipv4/ip_fragment.c IPCB(skb)->frag_max_size = max(qp->max_df_size, qp->q.max_size); qp 447 net/ipv4/ip_fragment.c if (qp->max_df_size == qp->q.max_size) { qp 457 net/ipv4/ip_fragment.c qp->q.rb_fragments = RB_ROOT; qp 458 net/ipv4/ip_fragment.c qp->q.fragments_tail = NULL; qp 459 net/ipv4/ip_fragment.c qp->q.last_run_head = NULL; qp 463 net/ipv4/ip_fragment.c net_dbg_ratelimited("queue_glue: no memory for gluing queue %p\n", qp); qp 467 net/ipv4/ip_fragment.c net_info_ratelimited("Oversized IP packet from %pI4\n", &qp->q.key.v4.saddr); qp 478 net/ipv4/ip_fragment.c struct ipq *qp; qp 484 net/ipv4/ip_fragment.c qp = ip_find(net, ip_hdr(skb), user, vif); qp 485 net/ipv4/ip_fragment.c if (qp) { qp 488 net/ipv4/ip_fragment.c spin_lock(&qp->q.lock); qp 490 net/ipv4/ip_fragment.c ret = ip_frag_queue(qp, skb); qp 492 net/ipv4/ip_fragment.c spin_unlock(&qp->q.lock); qp 493 net/ipv4/ip_fragment.c ipq_put(qp); qp 89 net/rds/ib_cm.c ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER); qp 174 net/rds/ib_cm.c err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE); qp 990 net/rds/ib_cm.c ic->i_cm_id ? ic->i_cm_id->qp : NULL); qp 1030 net/rds/ib_cm.c if (ic->i_cm_id->qp) qp 164 net/rds/ib_frmr.c ret = ib_post_send(ibmr->ic->i_cm_id->qp, ®_wr.wr, NULL); qp 273 net/rds/ib_frmr.c if (!i_cm_id || !i_cm_id->qp || !frmr->mr) qp 293 net/rds/ib_frmr.c ret = ib_post_send(i_cm_id->qp, s_wr, NULL); qp 418 net/rds/ib_recv.c ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, NULL); qp 671 net/rds/ib_recv.c ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL); qp 724 net/rds/ib_send.c ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr); qp 818 net/rds/ib_send.c ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr); qp 959 net/rds/ib_send.c ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr); qp 130 net/sched/sch_api.c struct Qdisc_ops *q, **qp; qp 134 net/sched/sch_api.c for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) qp 160 net/sched/sch_api.c *qp = qops; qp 174 net/sched/sch_api.c struct Qdisc_ops *q, **qp; qp 178 net/sched/sch_api.c for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) qp 182 net/sched/sch_api.c *qp = q->next; qp 353 net/smc/smc_cdc.c struct smc_link *link = (struct smc_link *)wc->qp->qp_context; qp 307 net/smc/smc_ib.c port_idx = ibevent->element.qp->port - 1; qp 558 net/smc/smc_llc.c struct smc_link *link = (struct smc_link *)wc->qp->qp_context; qp 71 net/smc/smc_wr.c link = wc->qp->qp_context; qp 339 net/smc/smc_wr.c struct smc_link *link = (struct smc_link *)wc->qp->qp_context; qp 362 net/smc/smc_wr.c link = wc[i].qp->qp_context; qp 447 net/sunrpc/xprtrdma/frwr_ops.c return ib_post_send(ia->ri_id->qp, post_wr, NULL); qp 573 net/sunrpc/xprtrdma/frwr_ops.c rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr); qp 676 net/sunrpc/xprtrdma/frwr_ops.c rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr); qp 493 net/sunrpc/xprtrdma/svc_rdma_transport.c newxprt->sc_qp = newxprt->sc_cm_id->qp; qp 99 net/sunrpc/xprtrdma/verbs.c ib_drain_rq(ia->ri_id->qp); qp 104 net/sunrpc/xprtrdma/verbs.c ib_drain_sq(ia->ri_id->qp); qp 419 net/sunrpc/xprtrdma/verbs.c if (ia->ri_id->qp) { qp 422 net/sunrpc/xprtrdma/verbs.c ia->ri_id->qp = NULL; qp 457 net/sunrpc/xprtrdma/verbs.c if (ia->ri_id->qp) qp 590 net/sunrpc/xprtrdma/verbs.c if (ia->ri_id && ia->ri_id->qp) { qp 593 net/sunrpc/xprtrdma/verbs.c ia->ri_id->qp = NULL; qp 1544 net/sunrpc/xprtrdma/verbs.c rc = ib_post_recv(r_xprt->rx_ia.ri_id->qp, wr,