wqe 691 block/kyber-iosched.c static int kyber_domain_wake(wait_queue_entry_t *wqe, unsigned mode, int flags, wqe 694 block/kyber-iosched.c struct blk_mq_hw_ctx *hctx = READ_ONCE(wqe->private); wqe 695 block/kyber-iosched.c struct sbq_wait *wait = container_of(wqe, struct sbq_wait, wait); wqe 596 drivers/crypto/cavium/cpt/cptvf_main.c struct cptvf_wqe *wqe; wqe 602 drivers/crypto/cavium/cpt/cptvf_main.c wqe = get_cptvf_vq_wqe(cptvf, 0); wqe 603 drivers/crypto/cavium/cpt/cptvf_main.c if (unlikely(!wqe)) { wqe 608 drivers/crypto/cavium/cpt/cptvf_main.c tasklet_hi_schedule(&wqe->twork); wqe 409 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; wqe 411 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(wqe, 0, sizeof(*wqe)); wqe 412 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; wqe 413 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->wr_id = BNXT_QPLIB_FENCE_WRID; wqe 414 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; wqe 415 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; wqe 416 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->bind.zero_based = false; wqe 417 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->bind.parent_l_key = ib_mr->lkey; wqe 418 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->bind.va = (u64)(unsigned long)fence->va; wqe 419 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->bind.length = fence->size; wqe 420 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); wqe 421 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; wqe 437 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe wqe; wqe 440 drivers/infiniband/hw/bnxt_re/ib_verbs.c memcpy(&wqe, fence_wqe, sizeof(wqe)); wqe 441 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.bind.r_key = fence->bind_rkey; wqe 446 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.bind.r_key, qp->qplib_qp.id, pd); wqe 447 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); wqe 1479 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe wqe; wqe 1486 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.num_sge = wr->num_sge; wqe 1487 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); wqe 1488 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.wr_id = wr->wr_id; wqe 1489 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; wqe 1491 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); wqe 1837 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe, wqe 1964 drivers/infiniband/hw/bnxt_re/ib_verbs.c for (i = wqe->num_sge; i; i--) { wqe 1965 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; wqe 1966 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; wqe 1967 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[i].size = wqe->sg_list[i - 1].size; wqe 1991 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[0].addr = sge.addr; wqe 1992 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[0].lkey = sge.lkey; wqe 1993 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[0].size = sge.size; wqe 1994 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->num_sge++; wqe 2011 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe, wqe 2027 drivers/infiniband/hw/bnxt_re/ib_verbs.c ref.addr = wqe->sg_list[0].addr; wqe 2028 drivers/infiniband/hw/bnxt_re/ib_verbs.c ref.lkey = wqe->sg_list[0].lkey; wqe 2029 drivers/infiniband/hw/bnxt_re/ib_verbs.c ref.size = wqe->sg_list[0].size; wqe 2034 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[0].addr = sge.addr; wqe 2035 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[0].lkey = sge.lkey; wqe 2036 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; wqe 2037 drivers/infiniband/hw/bnxt_re/ib_verbs.c sge.size -= wqe->sg_list[0].size; wqe 2043 drivers/infiniband/hw/bnxt_re/ib_verbs.c sqp_entry->wrid = wqe->wr_id; wqe 2045 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->wr_id = rq_prod_index; wqe 2057 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe) wqe 2063 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->send.q_key = ud_wr(wr)->remote_qkey; wqe 2064 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->send.dst_qp = ud_wr(wr)->remote_qpn; wqe 2065 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->send.avid = ah->qplib_ah.id; wqe 2069 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; wqe 2072 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; wqe 2073 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->send.imm_data = wr->ex.imm_data; wqe 2076 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; wqe 2077 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->send.inv_key = wr->ex.invalidate_rkey; wqe 2083 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; wqe 2085 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; wqe 2087 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; wqe 2089 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; wqe 2095 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe) wqe 2099 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; wqe 2102 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; wqe 2103 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->rdma.imm_data = wr->ex.imm_data; wqe 2106 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; wqe 2107 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->rdma.inv_key = wr->ex.invalidate_rkey; wqe 2112 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; wqe 2113 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->rdma.r_key = rdma_wr(wr)->rkey; wqe 2115 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; wqe 2117 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; wqe 2119 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; wqe 2121 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; wqe 2127 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe) wqe 2131 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; wqe 2132 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; wqe 2133 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->atomic.swap_data = atomic_wr(wr)->swap; wqe 2136 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; wqe 2137 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; wqe 2142 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; wqe 2143 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->atomic.r_key = atomic_wr(wr)->rkey; wqe 2145 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; wqe 2147 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; wqe 2149 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; wqe 2154 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe) wqe 2156 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; wqe 2157 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; wqe 2162 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; wqe 2165 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; wqe 2167 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; wqe 2173 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe) wqe 2179 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; wqe 2180 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; wqe 2181 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.page_list = mr->pages; wqe 2182 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.page_list_len = mr->npages; wqe 2183 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.levels = qplib_frpl->hwq.level + 1; wqe 2184 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; wqe 2190 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; wqe 2193 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; wqe 2196 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; wqe 2198 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; wqe 2200 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; wqe 2202 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; wqe 2204 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; wqe 2206 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.l_key = wr->key; wqe 2207 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.length = wr->mr->length; wqe 2208 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1; wqe 2209 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->frmr.va = wr->mr->iova; wqe 2215 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe) wqe 2222 drivers/infiniband/hw/bnxt_re/ib_verbs.c in_data = wqe->inline_data; wqe 2228 drivers/infiniband/hw/bnxt_re/ib_verbs.c if ((sge_len + wqe->inline_len) > wqe 2238 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->inline_len += wr->sg_list[i].length; wqe 2240 drivers/infiniband/hw/bnxt_re/ib_verbs.c return wqe->inline_len; wqe 2245 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe *wqe) wqe 2250 drivers/infiniband/hw/bnxt_re/ib_verbs.c payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); wqe 2252 drivers/infiniband/hw/bnxt_re/ib_verbs.c payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, wqe 2253 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe->num_sge); wqe 2278 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe wqe; wqe 2283 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(&wqe, 0, sizeof(wqe)); wqe 2286 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(&wqe, 0, sizeof(wqe)); wqe 2289 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.num_sge = wr->num_sge; wqe 2297 drivers/infiniband/hw/bnxt_re/ib_verbs.c payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); wqe 2302 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.wr_id = wr->wr_id; wqe 2304 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; wqe 2306 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_send_wqe(qp, wr, &wqe); wqe 2308 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); wqe 2328 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe wqe; wqe 2335 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(&wqe, 0, sizeof(wqe)); wqe 2338 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.num_sge = wr->num_sge; wqe 2346 drivers/infiniband/hw/bnxt_re/ib_verbs.c payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); wqe 2351 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.wr_id = wr->wr_id; wqe 2357 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, wqe 2361 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.rawqp1.lflags |= wqe 2366 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.rawqp1.lflags |= wqe 2374 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_send_wqe(qp, wr, &wqe); wqe 2379 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_rdma_wqe(wr, &wqe); wqe 2383 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_atomic_wqe(wr, &wqe); wqe 2391 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_inv_wqe(wr, &wqe); wqe 2394 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); wqe 2404 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); wqe 2426 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe wqe; wqe 2429 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(&wqe, 0, sizeof(wqe)); wqe 2432 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(&wqe, 0, sizeof(wqe)); wqe 2435 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.num_sge = wr->num_sge; wqe 2442 drivers/infiniband/hw/bnxt_re/ib_verbs.c bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); wqe 2443 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.wr_id = wr->wr_id; wqe 2444 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; wqe 2446 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); wqe 2461 drivers/infiniband/hw/bnxt_re/ib_verbs.c struct bnxt_qplib_swqe wqe; wqe 2469 drivers/infiniband/hw/bnxt_re/ib_verbs.c memset(&wqe, 0, sizeof(wqe)); wqe 2472 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.num_sge = wr->num_sge; wqe 2481 drivers/infiniband/hw/bnxt_re/ib_verbs.c payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wqe 2483 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.wr_id = wr->wr_id; wqe 2484 drivers/infiniband/hw/bnxt_re/ib_verbs.c wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; wqe 2488 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, wqe 2491 drivers/infiniband/hw/bnxt_re/ib_verbs.c rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); wqe 661 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_swqe *wqe) wqe 687 drivers/infiniband/hw/bnxt_re/qplib_fp.c i < wqe->num_sge; i++, hw_sge++) { wqe 688 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); wqe 689 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); wqe 690 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); wqe 692 drivers/infiniband/hw/bnxt_re/qplib_fp.c srqe->wqe_type = wqe->type; wqe 693 drivers/infiniband/hw/bnxt_re/qplib_fp.c srqe->flags = wqe->flags; wqe 694 drivers/infiniband/hw/bnxt_re/qplib_fp.c srqe->wqe_size = wqe->num_sge + wqe 697 drivers/infiniband/hw/bnxt_re/qplib_fp.c srq->swq[next].wr_id = wqe->wr_id; wqe 1522 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_swqe *wqe) wqe 1555 drivers/infiniband/hw/bnxt_re/qplib_fp.c swq->wr_id = wqe->wr_id; wqe 1556 drivers/infiniband/hw/bnxt_re/qplib_fp.c swq->type = wqe->type; wqe 1557 drivers/infiniband/hw/bnxt_re/qplib_fp.c swq->flags = wqe->flags; wqe 1568 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { wqe 1570 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { wqe 1575 drivers/infiniband/hw/bnxt_re/qplib_fp.c data_len = wqe->inline_len; wqe 1577 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); wqe 1581 drivers/infiniband/hw/bnxt_re/qplib_fp.c i < wqe->num_sge; i++, hw_sge++) { wqe 1582 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); wqe 1583 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); wqe 1584 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); wqe 1585 drivers/infiniband/hw/bnxt_re/qplib_fp.c data_len += wqe->sg_list[i].size; wqe 1588 drivers/infiniband/hw/bnxt_re/qplib_fp.c wqe_size16 = wqe->num_sge; wqe 1592 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!wqe->num_sge) wqe 1597 drivers/infiniband/hw/bnxt_re/qplib_fp.c switch (wqe->type) { wqe 1604 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->wqe_type = wqe->type; wqe 1605 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->flags = wqe->flags; wqe 1608 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); wqe 1609 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); wqe 1611 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & wqe 1623 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->wqe_type = wqe->type; wqe 1624 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->flags = wqe->flags; wqe 1628 drivers/infiniband/hw/bnxt_re/qplib_fp.c wqe->send.inv_key); wqe 1631 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->q_key = cpu_to_le32(wqe->send.q_key); wqe 1633 drivers/infiniband/hw/bnxt_re/qplib_fp.c wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); wqe 1635 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->avid = cpu_to_le32(wqe->send.avid & wqe 1656 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->wqe_type = wqe->type; wqe 1657 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->flags = wqe->flags; wqe 1660 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); wqe 1662 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); wqe 1663 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); wqe 1676 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->wqe_type = wqe->type; wqe 1677 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->flags = wqe->flags; wqe 1678 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); wqe 1679 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); wqe 1680 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); wqe 1681 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); wqe 1694 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->wqe_type = wqe->type; wqe 1695 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->flags = wqe->flags; wqe 1696 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); wqe 1704 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->wqe_type = wqe->type; wqe 1705 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->flags = wqe->flags; wqe 1706 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->access_cntl = wqe->frmr.access_cntl | wqe 1709 drivers/infiniband/hw/bnxt_re/qplib_fp.c (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << wqe 1711 drivers/infiniband/hw/bnxt_re/qplib_fp.c (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); wqe 1712 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->l_key = cpu_to_le32(wqe->frmr.l_key); wqe 1713 drivers/infiniband/hw/bnxt_re/qplib_fp.c temp32 = cpu_to_le32(wqe->frmr.length); wqe 1714 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); wqe 1716 drivers/infiniband/hw/bnxt_re/qplib_fp.c ((wqe->frmr.pbl_pg_sz_log << wqe 1719 drivers/infiniband/hw/bnxt_re/qplib_fp.c ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & wqe 1722 drivers/infiniband/hw/bnxt_re/qplib_fp.c for (i = 0; i < wqe->frmr.page_list_len; i++) wqe 1723 drivers/infiniband/hw/bnxt_re/qplib_fp.c wqe->frmr.pbl_ptr[i] = cpu_to_le64( wqe 1724 drivers/infiniband/hw/bnxt_re/qplib_fp.c wqe->frmr.page_list[i] | wqe 1726 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); wqe 1727 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->va = cpu_to_le64(wqe->frmr.va); wqe 1735 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->wqe_type = wqe->type; wqe 1736 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->flags = wqe->flags; wqe 1737 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->access_cntl = wqe->bind.access_cntl; wqe 1738 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->mw_type_zero_based = wqe->bind.mw_type | wqe 1739 drivers/infiniband/hw/bnxt_re/qplib_fp.c (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); wqe 1740 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); wqe 1741 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->l_key = cpu_to_le32(wqe->bind.r_key); wqe 1742 drivers/infiniband/hw/bnxt_re/qplib_fp.c sqe->va = cpu_to_le64(wqe->bind.va); wqe 1743 drivers/infiniband/hw/bnxt_re/qplib_fp.c temp32 = cpu_to_le32(wqe->bind.length); wqe 1744 drivers/infiniband/hw/bnxt_re/qplib_fp.c memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); wqe 1759 drivers/infiniband/hw/bnxt_re/qplib_fp.c opcd_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & wqe 1780 drivers/infiniband/hw/bnxt_re/qplib_fp.c swq->wr_id = wqe->wr_id; wqe 1781 drivers/infiniband/hw/bnxt_re/qplib_fp.c swq->type = wqe->type; wqe 1782 drivers/infiniband/hw/bnxt_re/qplib_fp.c swq->flags = wqe->flags; wqe 1823 drivers/infiniband/hw/bnxt_re/qplib_fp.c struct bnxt_qplib_swqe *wqe) wqe 1846 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq->swq[sw_prod].wr_id = wqe->wr_id; wqe 1855 drivers/infiniband/hw/bnxt_re/qplib_fp.c i < wqe->num_sge; i++, hw_sge++) { wqe 1856 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); wqe 1857 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); wqe 1858 drivers/infiniband/hw/bnxt_re/qplib_fp.c hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); wqe 1860 drivers/infiniband/hw/bnxt_re/qplib_fp.c rqe->wqe_type = wqe->type; wqe 1861 drivers/infiniband/hw/bnxt_re/qplib_fp.c rqe->flags = wqe->flags; wqe 1862 drivers/infiniband/hw/bnxt_re/qplib_fp.c rqe->wqe_size = wqe->num_sge + wqe 1867 drivers/infiniband/hw/bnxt_re/qplib_fp.c if (!wqe->num_sge) wqe 1877 drivers/infiniband/hw/bnxt_re/qplib_fp.c rq->swq[sw_prod].wr_id = wqe->wr_id; wqe 524 drivers/infiniband/hw/bnxt_re/qplib_fp.h struct bnxt_qplib_swqe *wqe); wqe 542 drivers/infiniband/hw/bnxt_re/qplib_fp.h struct bnxt_qplib_swqe *wqe); wqe 545 drivers/infiniband/hw/bnxt_re/qplib_fp.h struct bnxt_qplib_swqe *wqe); wqe 139 drivers/infiniband/hw/cxgb3/cxio_hal.c struct t3_modify_qp_wr *wqe; wqe 140 drivers/infiniband/hw/cxgb3/cxio_hal.c struct sk_buff *skb = alloc_skb(sizeof(*wqe), GFP_KERNEL); wqe 145 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe = skb_put_zero(skb, sizeof(*wqe)); wqe 146 drivers/infiniband/hw/cxgb3/cxio_hal.c build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, wqe 149 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->flags = cpu_to_be32(MODQP_WRITE_EC); wqe 151 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->sge_cmd = cpu_to_be64(sge_cmd); wqe 497 drivers/infiniband/hw/cxgb3/cxio_hal.c struct t3_modify_qp_wr *wqe; wqe 500 drivers/infiniband/hw/cxgb3/cxio_hal.c skb = alloc_skb(sizeof(*wqe), GFP_KERNEL); wqe 541 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe = skb_put_zero(skb, sizeof(*wqe)); wqe 542 drivers/infiniband/hw/cxgb3/cxio_hal.c build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 0, 0, wqe 544 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->flags = cpu_to_be32(MODQP_WRITE_EC); wqe 546 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->sge_cmd = cpu_to_be64(sge_cmd); wqe 547 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->ctx1 = cpu_to_be64(ctx1); wqe 548 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->ctx0 = cpu_to_be64(ctx0); wqe 579 drivers/infiniband/hw/cxgb3/cxio_hal.c __be64 *wqe; wqe 604 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr % wqe 627 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe += (sizeof(struct t3_bypass_wr) >> 3); wqe 631 drivers/infiniband/hw/cxgb3/cxio_hal.c *wqe = cpu_to_be64(utx_cmd); wqe 632 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe++; wqe 638 drivers/infiniband/hw/cxgb3/cxio_hal.c memcpy(wqe, copy_data, copy_len); wqe 640 drivers/infiniband/hw/cxgb3/cxio_hal.c memset(wqe, 0, copy_len); wqe 642 drivers/infiniband/hw/cxgb3/cxio_hal.c memset(((u8 *) wqe) + copy_len, 0, wqe 646 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr % wqe 650 drivers/infiniband/hw/cxgb3/cxio_hal.c ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr; wqe 656 drivers/infiniband/hw/cxgb3/cxio_hal.c build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag, wqe 811 drivers/infiniband/hw/cxgb3/cxio_hal.c struct t3_rdma_init_wr *wqe; wqe 812 drivers/infiniband/hw/cxgb3/cxio_hal.c struct sk_buff *skb = alloc_skb(sizeof(*wqe), GFP_ATOMIC); wqe 816 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe = __skb_put(skb, sizeof(*wqe)); wqe 817 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_INIT)); wqe 818 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(attr->tid) | wqe 819 drivers/infiniband/hw/cxgb3/cxio_hal.c V_FW_RIWR_LEN(sizeof(*wqe) >> 3)); wqe 820 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->wrid.id1 = 0; wqe 821 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->qpid = cpu_to_be32(attr->qpid); wqe 822 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->pdid = cpu_to_be32(attr->pdid); wqe 823 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->scqid = cpu_to_be32(attr->scqid); wqe 824 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->rcqid = cpu_to_be32(attr->rcqid); wqe 825 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->rq_addr = cpu_to_be32(attr->rq_addr - rdev_p->rnic_info.rqt_base); wqe 826 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->rq_size = cpu_to_be32(attr->rq_size); wqe 827 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->mpaattrs = attr->mpaattrs; wqe 828 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->qpcaps = attr->qpcaps; wqe 829 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->ulpdu_size = cpu_to_be16(attr->tcp_emss); wqe 830 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->rqe_count = cpu_to_be16(attr->rqe_count); wqe 831 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->flags_rtr_type = cpu_to_be16(attr->flags | wqe 834 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->ord = cpu_to_be32(attr->ord); wqe 835 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->ird = cpu_to_be32(attr->ird); wqe 836 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->qp_dma_addr = cpu_to_be64(attr->qp_dma_addr); wqe 837 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->qp_dma_size = cpu_to_be32(attr->qp_dma_size); wqe 838 drivers/infiniband/hw/cxgb3/cxio_hal.c wqe->irs = cpu_to_be32(attr->irs); wqe 416 drivers/infiniband/hw/cxgb3/cxio_wr.h static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe) wqe 418 drivers/infiniband/hw/cxgb3/cxio_wr.h return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags)); wqe 427 drivers/infiniband/hw/cxgb3/cxio_wr.h static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op, wqe 431 drivers/infiniband/hw/cxgb3/cxio_wr.h wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) | wqe 435 drivers/infiniband/hw/cxgb3/cxio_wr.h wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) | wqe 439 drivers/infiniband/hw/cxgb3/cxio_wr.h ((union t3_wr *)wqe)->genbit.genbit = cpu_to_be64(genbit); wqe 42 drivers/infiniband/hw/cxgb3/iwch_qp.c static int build_rdma_send(union t3_wr *wqe, const struct ib_send_wr *wr, wqe 51 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.rdmaop = T3_SEND_WITH_SE; wqe 53 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.rdmaop = T3_SEND; wqe 54 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.rem_stag = 0; wqe 58 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.rdmaop = T3_SEND_WITH_SE_INV; wqe 60 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.rdmaop = T3_SEND_WITH_INV; wqe 61 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.rem_stag = cpu_to_be32(wr->ex.invalidate_rkey); wqe 68 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.reserved[0] = 0; wqe 69 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.reserved[1] = 0; wqe 70 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.reserved[2] = 0; wqe 77 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); wqe 78 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); wqe 79 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); wqe 81 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.num_sgle = cpu_to_be32(wr->num_sge); wqe 83 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.plen = cpu_to_be32(plen); wqe 87 drivers/infiniband/hw/cxgb3/iwch_qp.c static int build_rdma_write(union t3_wr *wqe, const struct ib_send_wr *wr, wqe 94 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.rdmaop = T3_RDMA_WRITE; wqe 95 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.reserved[0] = 0; wqe 96 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.reserved[1] = 0; wqe 97 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.reserved[2] = 0; wqe 98 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey); wqe 99 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr); wqe 103 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.sgl[0].stag = wr->ex.imm_data; wqe 104 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.sgl[0].len = cpu_to_be32(0); wqe 105 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.num_sgle = cpu_to_be32(0); wqe 114 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.sgl[i].stag = wqe 116 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.sgl[i].len = wqe 118 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.sgl[i].to = wqe 121 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.num_sgle = cpu_to_be32(wr->num_sge); wqe 124 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->write.plen = cpu_to_be32(plen); wqe 128 drivers/infiniband/hw/cxgb3/iwch_qp.c static int build_rdma_read(union t3_wr *wqe, const struct ib_send_wr *wr, wqe 133 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.rdmaop = T3_READ_REQ; wqe 135 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_inv = 1; wqe 137 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_inv = 0; wqe 138 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.reserved[0] = 0; wqe 139 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.reserved[1] = 0; wqe 140 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.rem_stag = cpu_to_be32(rdma_wr(wr)->rkey); wqe 141 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.rem_to = cpu_to_be64(rdma_wr(wr)->remote_addr); wqe 142 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); wqe 143 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); wqe 144 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); wqe 149 drivers/infiniband/hw/cxgb3/iwch_qp.c static int build_memreg(union t3_wr *wqe, const struct ib_reg_wr *wr, wqe 159 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->fastreg.stag = cpu_to_be32(wr->key); wqe 160 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->fastreg.len = cpu_to_be32(mhp->ibmr.length); wqe 161 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->fastreg.va_base_hi = cpu_to_be32(mhp->ibmr.iova >> 32); wqe 162 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->fastreg.va_base_lo_fbo = wqe 164 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->fastreg.page_type_perms = cpu_to_be32( wqe 169 drivers/infiniband/hw/cxgb3/iwch_qp.c p = &wqe->fastreg.pbl_addrs[0]; wqe 175 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe = (union t3_wr *)(wq->queue + wqe 177 drivers/infiniband/hw/cxgb3/iwch_qp.c build_fw_riwrh((void *)wqe, T3_WR_FASTREG, 0, wqe 182 drivers/infiniband/hw/cxgb3/iwch_qp.c p = &wqe->pbl_frag.pbl_addrs[0]; wqe 192 drivers/infiniband/hw/cxgb3/iwch_qp.c static int build_inv_stag(union t3_wr *wqe, const struct ib_send_wr *wr, wqe 195 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->local_inv.stag = cpu_to_be32(wr->ex.invalidate_rkey); wqe 196 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->local_inv.reserved = 0; wqe 248 drivers/infiniband/hw/cxgb3/iwch_qp.c static int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe, wqe 259 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pagesz[0] = page_size[0]; wqe 260 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pagesz[1] = page_size[1]; wqe 261 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pagesz[2] = page_size[2]; wqe 262 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pagesz[3] = page_size[3]; wqe 263 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); wqe 265 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); wqe 266 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); wqe 269 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].to = cpu_to_be64(((u32)wr->sg_list[i].addr) & wqe 273 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); wqe 276 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].stag = 0; wqe 277 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].len = 0; wqe 278 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].to = 0; wqe 279 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pbl_addr[i] = 0; wqe 288 drivers/infiniband/hw/cxgb3/iwch_qp.c static int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe, wqe 311 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); wqe 324 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pagesz[i] = T3_STAG0_PAGE_SHIFT; wqe 331 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].stag = 0; wqe 332 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); wqe 333 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); wqe 334 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_offset); wqe 338 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pagesz[i] = 0; wqe 339 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].stag = 0; wqe 340 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].len = 0; wqe 341 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.sgl[i].to = 0; wqe 342 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->recv.pbl_addr[i] = 0; wqe 360 drivers/infiniband/hw/cxgb3/iwch_qp.c union t3_wr *wqe; wqe 386 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe = (union t3_wr *) (qhp->wq.queue + idx); wqe 400 drivers/infiniband/hw/cxgb3/iwch_qp.c err = build_rdma_send(wqe, wr, &t3_wr_flit_cnt); wqe 405 drivers/infiniband/hw/cxgb3/iwch_qp.c err = build_rdma_write(wqe, wr, &t3_wr_flit_cnt); wqe 411 drivers/infiniband/hw/cxgb3/iwch_qp.c err = build_rdma_read(wqe, wr, &t3_wr_flit_cnt); wqe 414 drivers/infiniband/hw/cxgb3/iwch_qp.c sqp->read_len = wqe->read.local_len; wqe 420 drivers/infiniband/hw/cxgb3/iwch_qp.c err = build_memreg(wqe, reg_wr(wr), &t3_wr_flit_cnt, wqe 427 drivers/infiniband/hw/cxgb3/iwch_qp.c err = build_inv_stag(wqe, wr, &t3_wr_flit_cnt); wqe 436 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; wqe 443 drivers/infiniband/hw/cxgb3/iwch_qp.c build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, wqe 472 drivers/infiniband/hw/cxgb3/iwch_qp.c union t3_wr *wqe; wqe 496 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe = (union t3_wr *) (qhp->wq.queue + idx); wqe 499 drivers/infiniband/hw/cxgb3/iwch_qp.c err = build_rdma_recv(qhp, wqe, wr); wqe 501 drivers/infiniband/hw/cxgb3/iwch_qp.c err = build_zero_stag_recv(qhp, wqe, wr); wqe 508 drivers/infiniband/hw/cxgb3/iwch_qp.c build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, wqe 513 drivers/infiniband/hw/cxgb3/iwch_qp.c idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); wqe 663 drivers/infiniband/hw/cxgb3/iwch_qp.c union t3_wr *wqe; wqe 673 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe = skb_put_zero(skb, sizeof(struct t3_rdma_read_wr)); wqe 674 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.rdmaop = T3_READ_REQ; wqe 675 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.reserved[0] = 0; wqe 676 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.reserved[1] = 0; wqe 677 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.rem_stag = cpu_to_be32(1); wqe 678 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.rem_to = cpu_to_be64(1); wqe 679 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_stag = cpu_to_be32(1); wqe 680 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_len = cpu_to_be32(0); wqe 681 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->read.local_to = cpu_to_be64(1); wqe 682 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_READ)); wqe 683 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(ep->hwtid)| wqe 694 drivers/infiniband/hw/cxgb3/iwch_qp.c union t3_wr *wqe; wqe 704 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe = skb_put_zero(skb, 40); wqe 705 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.rdmaop = T3_TERMINATE; wqe 708 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.plen = htonl(4); wqe 711 drivers/infiniband/hw/cxgb3/iwch_qp.c term = (struct terminate_message *)wqe->send.sgl; wqe 713 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_SEND) | wqe 715 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)); wqe 804 drivers/infiniband/hw/cxgb3/iwch_qp.c union t3_wr *wqe = qhp->wq.queue; wqe 807 drivers/infiniband/hw/cxgb3/iwch_qp.c while (count < USHRT_MAX && fw_riwrh_opcode((struct fw_riwrh *)wqe) == T3_WR_RCV) { wqe 809 drivers/infiniband/hw/cxgb3/iwch_qp.c wqe++; wqe 478 drivers/infiniband/hw/cxgb4/cq.c c4iw_copy_wr_to_srq(srq, &pwr->wqe, pwr->len16); wqe 485 drivers/infiniband/hw/cxgb4/cq.c t4_ring_srq_db(srq, idx, pwr->len16, &pwr->wqe); wqe 1050 drivers/infiniband/hw/cxgb4/iw_cxgb4.h void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16); wqe 488 drivers/infiniband/hw/cxgb4/qp.c static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, wqe 500 drivers/infiniband/hw/cxgb4/qp.c wqe->send.sendop_pkd = cpu_to_be32( wqe 503 drivers/infiniband/hw/cxgb4/qp.c wqe->send.sendop_pkd = cpu_to_be32( wqe 505 drivers/infiniband/hw/cxgb4/qp.c wqe->send.stag_inv = 0; wqe 509 drivers/infiniband/hw/cxgb4/qp.c wqe->send.sendop_pkd = cpu_to_be32( wqe 512 drivers/infiniband/hw/cxgb4/qp.c wqe->send.sendop_pkd = cpu_to_be32( wqe 514 drivers/infiniband/hw/cxgb4/qp.c wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); wqe 520 drivers/infiniband/hw/cxgb4/qp.c wqe->send.r3 = 0; wqe 521 drivers/infiniband/hw/cxgb4/qp.c wqe->send.r4 = 0; wqe 526 drivers/infiniband/hw/cxgb4/qp.c ret = build_immd(sq, wqe->send.u.immd_src, wr, wqe 530 drivers/infiniband/hw/cxgb4/qp.c size = sizeof(wqe->send) + sizeof(struct fw_ri_immd) + wqe 535 drivers/infiniband/hw/cxgb4/qp.c wqe->send.u.isgl_src, wqe 539 drivers/infiniband/hw/cxgb4/qp.c size = sizeof(wqe->send) + sizeof(struct fw_ri_isgl) + wqe 543 drivers/infiniband/hw/cxgb4/qp.c wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD; wqe 544 drivers/infiniband/hw/cxgb4/qp.c wqe->send.u.immd_src[0].r1 = 0; wqe 545 drivers/infiniband/hw/cxgb4/qp.c wqe->send.u.immd_src[0].r2 = 0; wqe 546 drivers/infiniband/hw/cxgb4/qp.c wqe->send.u.immd_src[0].immdlen = 0; wqe 547 drivers/infiniband/hw/cxgb4/qp.c size = sizeof(wqe->send) + sizeof(struct fw_ri_immd); wqe 551 drivers/infiniband/hw/cxgb4/qp.c wqe->send.plen = cpu_to_be32(plen); wqe 555 drivers/infiniband/hw/cxgb4/qp.c static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, wqe 570 drivers/infiniband/hw/cxgb4/qp.c wqe->write.iw_imm_data.ib_imm_data.imm_data32 = wr->ex.imm_data; wqe 572 drivers/infiniband/hw/cxgb4/qp.c wqe->write.iw_imm_data.ib_imm_data.imm_data32 = 0; wqe 573 drivers/infiniband/hw/cxgb4/qp.c wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey); wqe 574 drivers/infiniband/hw/cxgb4/qp.c wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr); wqe 577 drivers/infiniband/hw/cxgb4/qp.c ret = build_immd(sq, wqe->write.u.immd_src, wr, wqe 581 drivers/infiniband/hw/cxgb4/qp.c size = sizeof(wqe->write) + sizeof(struct fw_ri_immd) + wqe 586 drivers/infiniband/hw/cxgb4/qp.c wqe->write.u.isgl_src, wqe 590 drivers/infiniband/hw/cxgb4/qp.c size = sizeof(wqe->write) + sizeof(struct fw_ri_isgl) + wqe 594 drivers/infiniband/hw/cxgb4/qp.c wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD; wqe 595 drivers/infiniband/hw/cxgb4/qp.c wqe->write.u.immd_src[0].r1 = 0; wqe 596 drivers/infiniband/hw/cxgb4/qp.c wqe->write.u.immd_src[0].r2 = 0; wqe 597 drivers/infiniband/hw/cxgb4/qp.c wqe->write.u.immd_src[0].immdlen = 0; wqe 598 drivers/infiniband/hw/cxgb4/qp.c size = sizeof(wqe->write) + sizeof(struct fw_ri_immd); wqe 602 drivers/infiniband/hw/cxgb4/qp.c wqe->write.plen = cpu_to_be32(plen); wqe 659 drivers/infiniband/hw/cxgb4/qp.c static int build_rdma_read(union t4_wr *wqe, const struct ib_send_wr *wr, wqe 665 drivers/infiniband/hw/cxgb4/qp.c wqe->read.stag_src = cpu_to_be32(rdma_wr(wr)->rkey); wqe 666 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_src_hi = cpu_to_be32((u32)(rdma_wr(wr)->remote_addr wqe 668 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_src_lo = cpu_to_be32((u32)rdma_wr(wr)->remote_addr); wqe 669 drivers/infiniband/hw/cxgb4/qp.c wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey); wqe 670 drivers/infiniband/hw/cxgb4/qp.c wqe->read.plen = cpu_to_be32(wr->sg_list[0].length); wqe 671 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr wqe 673 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr)); wqe 675 drivers/infiniband/hw/cxgb4/qp.c wqe->read.stag_src = cpu_to_be32(2); wqe 676 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_src_hi = 0; wqe 677 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_src_lo = 0; wqe 678 drivers/infiniband/hw/cxgb4/qp.c wqe->read.stag_sink = cpu_to_be32(2); wqe 679 drivers/infiniband/hw/cxgb4/qp.c wqe->read.plen = 0; wqe 680 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_sink_hi = 0; wqe 681 drivers/infiniband/hw/cxgb4/qp.c wqe->read.to_sink_lo = 0; wqe 683 drivers/infiniband/hw/cxgb4/qp.c wqe->read.r2 = 0; wqe 684 drivers/infiniband/hw/cxgb4/qp.c wqe->read.r5 = 0; wqe 685 drivers/infiniband/hw/cxgb4/qp.c *len16 = DIV_ROUND_UP(sizeof(wqe->read), 16); wqe 696 drivers/infiniband/hw/cxgb4/qp.c union t4_wr *wqe; wqe 705 drivers/infiniband/hw/cxgb4/qp.c wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue + wqe 707 drivers/infiniband/hw/cxgb4/qp.c build_rdma_write_cmpl(&qhp->wq.sq, &wqe->write_cmpl, wr, &len16); wqe 747 drivers/infiniband/hw/cxgb4/qp.c wqe->write_cmpl.flags_send = send_signaled ? FW_RI_COMPLETION_FLAG : 0; wqe 748 drivers/infiniband/hw/cxgb4/qp.c wqe->write_cmpl.wrid_send = qhp->wq.sq.pidx; wqe 750 drivers/infiniband/hw/cxgb4/qp.c init_wr_hdr(wqe, write_wrid, FW_RI_RDMA_WRITE_CMPL_WR, wqe 755 drivers/infiniband/hw/cxgb4/qp.c t4_ring_sq_db(&qhp->wq, idx, wqe); wqe 758 drivers/infiniband/hw/cxgb4/qp.c static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, wqe 765 drivers/infiniband/hw/cxgb4/qp.c &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); wqe 769 drivers/infiniband/hw/cxgb4/qp.c sizeof(wqe->recv) + wr->num_sge * sizeof(struct fw_ri_sge), 16); wqe 773 drivers/infiniband/hw/cxgb4/qp.c static int build_srq_recv(union t4_recv_wr *wqe, const struct ib_recv_wr *wr, wqe 778 drivers/infiniband/hw/cxgb4/qp.c ret = build_isgl((__be64 *)wqe, (__be64 *)(wqe + 1), wqe 779 drivers/infiniband/hw/cxgb4/qp.c &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); wqe 782 drivers/infiniband/hw/cxgb4/qp.c *len16 = DIV_ROUND_UP(sizeof(wqe->recv) + wqe 819 drivers/infiniband/hw/cxgb4/qp.c static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, wqe 832 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.qpbinde_to_dcacpu = 0; wqe 833 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.pgsz_shift = ilog2(wr->mr->page_size) - 12; wqe 834 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.addr_type = FW_RI_VA_BASED_TO; wqe 835 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->access); wqe 836 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.len_hi = 0; wqe 837 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.len_lo = cpu_to_be32(mhp->ibmr.length); wqe 838 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.stag = cpu_to_be32(wr->key); wqe 839 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32); wqe 840 drivers/infiniband/hw/cxgb4/qp.c wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & wqe 849 drivers/infiniband/hw/cxgb4/qp.c sglp = (struct fw_ri_dsgl *)(&wqe->fr + 1); wqe 856 drivers/infiniband/hw/cxgb4/qp.c *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*sglp), 16); wqe 858 drivers/infiniband/hw/cxgb4/qp.c imdp = (struct fw_ri_immd *)(&wqe->fr + 1); wqe 877 drivers/infiniband/hw/cxgb4/qp.c *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*imdp) wqe 883 drivers/infiniband/hw/cxgb4/qp.c static int build_inv_stag(union t4_wr *wqe, const struct ib_send_wr *wr, wqe 886 drivers/infiniband/hw/cxgb4/qp.c wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); wqe 887 drivers/infiniband/hw/cxgb4/qp.c wqe->inv.r2 = 0; wqe 888 drivers/infiniband/hw/cxgb4/qp.c *len16 = DIV_ROUND_UP(sizeof(wqe->inv), 16); wqe 1085 drivers/infiniband/hw/cxgb4/qp.c union t4_wr *wqe = NULL; wqe 1141 drivers/infiniband/hw/cxgb4/qp.c wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue + wqe 1160 drivers/infiniband/hw/cxgb4/qp.c err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); wqe 1172 drivers/infiniband/hw/cxgb4/qp.c err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16); wqe 1184 drivers/infiniband/hw/cxgb4/qp.c err = build_rdma_read(wqe, wr, &len16); wqe 1198 drivers/infiniband/hw/cxgb4/qp.c build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr), wqe 1202 drivers/infiniband/hw/cxgb4/qp.c err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), wqe 1216 drivers/infiniband/hw/cxgb4/qp.c err = build_inv_stag(wqe, wr, &len16); wqe 1240 drivers/infiniband/hw/cxgb4/qp.c init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); wqe 1251 drivers/infiniband/hw/cxgb4/qp.c t4_ring_sq_db(&qhp->wq, idx, wqe); wqe 1265 drivers/infiniband/hw/cxgb4/qp.c union t4_recv_wr *wqe = NULL; wqe 1295 drivers/infiniband/hw/cxgb4/qp.c wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue + wqe 1299 drivers/infiniband/hw/cxgb4/qp.c err = build_rdma_recv(qhp, wqe, wr, &len16); wqe 1316 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.opcode = FW_RI_RECV_WR; wqe 1317 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r1 = 0; wqe 1318 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.wrid = qhp->wq.rq.pidx; wqe 1319 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r2[0] = 0; wqe 1320 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r2[1] = 0; wqe 1321 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r2[2] = 0; wqe 1322 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.len16 = len16; wqe 1331 drivers/infiniband/hw/cxgb4/qp.c t4_ring_rq_db(&qhp->wq, idx, wqe); wqe 1340 drivers/infiniband/hw/cxgb4/qp.c static void defer_srq_wr(struct t4_srq *srq, union t4_recv_wr *wqe, wqe 1352 drivers/infiniband/hw/cxgb4/qp.c memcpy(&pwr->wqe, wqe, len16 * 16); wqe 1359 drivers/infiniband/hw/cxgb4/qp.c union t4_recv_wr *wqe, lwqe; wqe 1380 drivers/infiniband/hw/cxgb4/qp.c wqe = &lwqe; wqe 1382 drivers/infiniband/hw/cxgb4/qp.c err = build_srq_recv(wqe, wr, &len16); wqe 1390 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.opcode = FW_RI_RECV_WR; wqe 1391 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r1 = 0; wqe 1392 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.wrid = srq->wq.pidx; wqe 1393 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r2[0] = 0; wqe 1394 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r2[1] = 0; wqe 1395 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.r2[2] = 0; wqe 1396 drivers/infiniband/hw/cxgb4/qp.c wqe->recv.len16 = len16; wqe 1401 drivers/infiniband/hw/cxgb4/qp.c defer_srq_wr(&srq->wq, wqe, wr->wr_id, len16); wqe 1405 drivers/infiniband/hw/cxgb4/qp.c c4iw_copy_wr_to_srq(&srq->wq, wqe, len16); wqe 1418 drivers/infiniband/hw/cxgb4/qp.c t4_ring_srq_db(&srq->wq, idx, len16, wqe); wqe 1562 drivers/infiniband/hw/cxgb4/qp.c struct fw_ri_wr *wqe; wqe 1575 drivers/infiniband/hw/cxgb4/qp.c wqe = __skb_put_zero(skb, sizeof(*wqe)); wqe 1576 drivers/infiniband/hw/cxgb4/qp.c wqe->op_compl = cpu_to_be32(FW_WR_OP_V(FW_RI_INIT_WR)); wqe 1577 drivers/infiniband/hw/cxgb4/qp.c wqe->flowid_len16 = cpu_to_be32( wqe 1579 drivers/infiniband/hw/cxgb4/qp.c FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16))); wqe 1581 drivers/infiniband/hw/cxgb4/qp.c wqe->u.terminate.type = FW_RI_TYPE_TERMINATE; wqe 1582 drivers/infiniband/hw/cxgb4/qp.c wqe->u.terminate.immdlen = cpu_to_be32(sizeof(*term)); wqe 1583 drivers/infiniband/hw/cxgb4/qp.c term = (struct terminate_message *)wqe->u.terminate.termmsg; wqe 1694 drivers/infiniband/hw/cxgb4/qp.c struct fw_ri_wr *wqe; wqe 1706 drivers/infiniband/hw/cxgb4/qp.c wqe = __skb_put_zero(skb, sizeof(*wqe)); wqe 1707 drivers/infiniband/hw/cxgb4/qp.c wqe->op_compl = cpu_to_be32( wqe 1710 drivers/infiniband/hw/cxgb4/qp.c wqe->flowid_len16 = cpu_to_be32( wqe 1712 drivers/infiniband/hw/cxgb4/qp.c FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16))); wqe 1713 drivers/infiniband/hw/cxgb4/qp.c wqe->cookie = (uintptr_t)ep->com.wr_waitp; wqe 1715 drivers/infiniband/hw/cxgb4/qp.c wqe->u.fini.type = FW_RI_TYPE_FINI; wqe 1750 drivers/infiniband/hw/cxgb4/qp.c struct fw_ri_wr *wqe; wqe 1757 drivers/infiniband/hw/cxgb4/qp.c skb = alloc_skb(sizeof(*wqe), GFP_KERNEL); wqe 1770 drivers/infiniband/hw/cxgb4/qp.c wqe = __skb_put_zero(skb, sizeof(*wqe)); wqe 1771 drivers/infiniband/hw/cxgb4/qp.c wqe->op_compl = cpu_to_be32( wqe 1774 drivers/infiniband/hw/cxgb4/qp.c wqe->flowid_len16 = cpu_to_be32( wqe 1776 drivers/infiniband/hw/cxgb4/qp.c FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16))); wqe 1778 drivers/infiniband/hw/cxgb4/qp.c wqe->cookie = (uintptr_t)qhp->ep->com.wr_waitp; wqe 1780 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.type = FW_RI_TYPE_INIT; wqe 1781 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.mpareqbit_p2ptype = wqe 1784 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE; wqe 1786 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE; wqe 1788 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE; wqe 1790 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE; wqe 1792 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE | wqe 1796 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE | wqe 1798 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq)); wqe 1799 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd); wqe 1800 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid); wqe 1801 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid); wqe 1803 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.rq_eqid = cpu_to_be32(FW_RI_INIT_RQEQID_SRQ | wqe 1806 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid); wqe 1807 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size); wqe 1808 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr - wqe 1811 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq); wqe 1812 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq); wqe 1813 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord); wqe 1814 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird); wqe 1815 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.iss = cpu_to_be32(qhp->ep->snd_seq); wqe 1816 drivers/infiniband/hw/cxgb4/qp.c wqe->u.init.irs = cpu_to_be32(qhp->ep->rcv_seq); wqe 1818 drivers/infiniband/hw/cxgb4/qp.c build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init); wqe 2652 drivers/infiniband/hw/cxgb4/qp.c void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16) wqe 2656 drivers/infiniband/hw/cxgb4/qp.c src = (u64 *)wqe; wqe 119 drivers/infiniband/hw/cxgb4/t4.h static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid, wqe 122 drivers/infiniband/hw/cxgb4/t4.h wqe->send.opcode = (u8)opcode; wqe 123 drivers/infiniband/hw/cxgb4/t4.h wqe->send.flags = flags; wqe 124 drivers/infiniband/hw/cxgb4/t4.h wqe->send.wrid = wrid; wqe 125 drivers/infiniband/hw/cxgb4/t4.h wqe->send.r1[0] = 0; wqe 126 drivers/infiniband/hw/cxgb4/t4.h wqe->send.r1[1] = 0; wqe 127 drivers/infiniband/hw/cxgb4/t4.h wqe->send.r1[2] = 0; wqe 128 drivers/infiniband/hw/cxgb4/t4.h wqe->send.len16 = len16; wqe 394 drivers/infiniband/hw/cxgb4/t4.h union t4_recv_wr wqe; wqe 593 drivers/infiniband/hw/cxgb4/t4.h union t4_recv_wr *wqe) wqe 597 drivers/infiniband/hw/cxgb4/t4.h if (inc == 1 && srq->bar2_qid == 0 && wqe) { wqe 600 drivers/infiniband/hw/cxgb4/t4.h pio_copy(srq->bar2_va + SGE_UDB_WCDOORBELL, (u64 *)wqe); wqe 611 drivers/infiniband/hw/cxgb4/t4.h static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe) wqe 617 drivers/infiniband/hw/cxgb4/t4.h if (inc == 1 && wq->sq.bar2_qid == 0 && wqe) { wqe 621 drivers/infiniband/hw/cxgb4/t4.h (u64 *)wqe); wqe 636 drivers/infiniband/hw/cxgb4/t4.h union t4_recv_wr *wqe) wqe 642 drivers/infiniband/hw/cxgb4/t4.h if (inc == 1 && wq->rq.bar2_qid == 0 && wqe) { wqe 646 drivers/infiniband/hw/cxgb4/t4.h (void *)wqe); wqe 980 drivers/infiniband/hw/hfi1/hfi.h struct rvt_swqe *wqe); wqe 1504 drivers/infiniband/hw/hfi1/hfi.h struct rvt_swqe *wqe); wqe 1508 drivers/infiniband/hw/hfi1/hfi.h struct rvt_swqe *wqe); wqe 319 drivers/infiniband/hw/hfi1/qp.c int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) wqe 328 drivers/infiniband/hw/hfi1/qp.c hfi1_setup_tid_rdma_wqe(qp, wqe); wqe 331 drivers/infiniband/hw/hfi1/qp.c if (wqe->length > 0x80000000U) wqe 333 drivers/infiniband/hw/hfi1/qp.c if (wqe->length > qp->pmtu) wqe 346 drivers/infiniband/hw/hfi1/qp.c if (wqe->length > dd->vld[15].mtu) wqe 351 drivers/infiniband/hw/hfi1/qp.c ah = rvt_get_swqe_ah(wqe); wqe 352 drivers/infiniband/hw/hfi1/qp.c if (wqe->length > (1 << ah->log_pmtu)) wqe 364 drivers/infiniband/hw/hfi1/qp.c if (wqe->length <= piothreshold) wqe 652 drivers/infiniband/hw/hfi1/qp.c struct rvt_swqe *wqe; wqe 662 drivers/infiniband/hw/hfi1/qp.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 674 drivers/infiniband/hw/hfi1/qp.c wqe ? wqe->wr.opcode : 0, wqe 680 drivers/infiniband/hw/hfi1/qp.c wqe ? wqe->ssn : 0, wqe 435 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe; wqe 490 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 491 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ? wqe 510 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 536 drivers/infiniband/hw/hfi1/rc.c if ((wqe->wr.send_flags & IB_SEND_FENCE) && wqe 538 drivers/infiniband/hw/hfi1/rc.c (wqe->wr.opcode != IB_WR_TID_RDMA_READ || wqe 547 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_REG_MR || wqe 548 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_LOCAL_INV) { wqe 558 drivers/infiniband/hw/hfi1/rc.c if (!(wqe->wr.send_flags & wqe 562 drivers/infiniband/hw/hfi1/rc.c wqe->wr.ex.invalidate_rkey); wqe 565 drivers/infiniband/hw/hfi1/rc.c rvt_send_complete(qp, wqe, wqe 574 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->psn; wqe 581 drivers/infiniband/hw/hfi1/rc.c len = wqe->length; wqe 590 drivers/infiniband/hw/hfi1/rc.c hfi1_tid_rdma_wqe_interlock(qp, wqe)) wqe 593 drivers/infiniband/hw/hfi1/rc.c switch (wqe->wr.opcode) { wqe 598 drivers/infiniband/hw/hfi1/rc.c if (!rvt_rc_credit_avail(qp, wqe)) wqe 605 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_SEND) { wqe 607 drivers/infiniband/hw/hfi1/rc.c } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { wqe 610 drivers/infiniband/hw/hfi1/rc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 616 drivers/infiniband/hw/hfi1/rc.c wqe->wr.ex.invalidate_rkey); wqe 619 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 632 drivers/infiniband/hw/hfi1/rc.c if (!rvt_rc_credit_avail(qp, wqe)) wqe 636 drivers/infiniband/hw/hfi1/rc.c wqe->rdma_wr.remote_addr, wqe 639 drivers/infiniband/hw/hfi1/rc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 647 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { wqe 653 drivers/infiniband/hw/hfi1/rc.c ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; wqe 655 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 676 drivers/infiniband/hw/hfi1/rc.c hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, wqe 743 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 751 drivers/infiniband/hw/hfi1/rc.c req->comp_seg = delta_psn(bth2, wqe->psn); wqe 758 drivers/infiniband/hw/hfi1/rc.c delta_psn(wqe->lpsn, bth2) + 1; wqe 763 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode, wqe 764 drivers/infiniband/hw/hfi1/rc.c wqe->psn, wqe->lpsn, wqe 784 drivers/infiniband/hw/hfi1/rc.c wqe->rdma_wr.remote_addr, wqe 787 drivers/infiniband/hw/hfi1/rc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 800 drivers/infiniband/hw/hfi1/rc.c wpriv = wqe->priv; wqe 801 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 803 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode, wqe 804 drivers/infiniband/hw/hfi1/rc.c wqe->psn, wqe->lpsn, wqe 806 drivers/infiniband/hw/hfi1/rc.c delta = cmp_psn(qp->s_psn, wqe->psn); wqe 833 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sge = wqe->sg_list[0]; wqe 834 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sg_list = wqe->sg_list + 1; wqe 835 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.num_sge = wqe->wr.num_sge; wqe 836 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.total_len = wqe->length; wqe 837 drivers/infiniband/hw/hfi1/rc.c qp->s_len = wqe->length; wqe 854 drivers/infiniband/hw/hfi1/rc.c wqe->length - req->seg_len * req->cur_seg); wqe 855 drivers/infiniband/hw/hfi1/rc.c delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, wqe 889 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 890 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_OPFN) { wqe 892 drivers/infiniband/hw/hfi1/rc.c put_ib_ateth_swap(wqe->atomic_wr.swap, wqe 894 drivers/infiniband/hw/hfi1/rc.c put_ib_ateth_compare(wqe->atomic_wr.compare_add, wqe 898 drivers/infiniband/hw/hfi1/rc.c put_ib_ateth_swap(wqe->atomic_wr.compare_add, wqe 902 drivers/infiniband/hw/hfi1/rc.c put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, wqe 905 drivers/infiniband/hw/hfi1/rc.c wqe->atomic_wr.rkey); wqe 917 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) { wqe 918 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sge = wqe->sg_list[0]; wqe 919 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.sg_list = wqe->sg_list + 1; wqe 920 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.num_sge = wqe->wr.num_sge; wqe 921 drivers/infiniband/hw/hfi1/rc.c qp->s_sge.total_len = wqe->length; wqe 922 drivers/infiniband/hw/hfi1/rc.c qp->s_len = wqe->length; wqe 929 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 930 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) wqe 931 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->lpsn + 1; wqe 932 drivers/infiniband/hw/hfi1/rc.c else if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) wqe 948 drivers/infiniband/hw/hfi1/rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); wqe 962 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_SEND) { wqe 964 drivers/infiniband/hw/hfi1/rc.c } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { wqe 967 drivers/infiniband/hw/hfi1/rc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 972 drivers/infiniband/hw/hfi1/rc.c ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey); wqe 975 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 993 drivers/infiniband/hw/hfi1/rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); wqe 1007 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { wqe 1012 drivers/infiniband/hw/hfi1/rc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 1014 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 1033 drivers/infiniband/hw/hfi1/rc.c len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu; wqe 1035 drivers/infiniband/hw/hfi1/rc.c wqe->rdma_wr.remote_addr + len, wqe 1038 drivers/infiniband/hw/hfi1/rc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 1039 drivers/infiniband/hw/hfi1/rc.c ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); wqe 1043 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->lpsn + 1; wqe 1057 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 1061 drivers/infiniband/hw/hfi1/rc.c req->comp_seg = delta_psn(qp->s_psn, wqe->psn); wqe 1062 drivers/infiniband/hw/hfi1/rc.c len = wqe->length - (req->comp_seg * remote->max_len); wqe 1066 drivers/infiniband/hw/hfi1/rc.c hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1, wqe 1068 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->lpsn + 1; wqe 1071 drivers/infiniband/hw/hfi1/rc.c priv->pending_tid_w_resp += delta_psn(wqe->lpsn, bth2) + 1; wqe 1075 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode, wqe 1076 drivers/infiniband/hw/hfi1/rc.c wqe->psn, wqe->lpsn, req); wqe 1080 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) wqe 1083 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 1084 drivers/infiniband/hw/hfi1/rc.c wpriv = wqe->priv; wqe 1090 drivers/infiniband/hw/hfi1/rc.c req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps; wqe 1099 drivers/infiniband/hw/hfi1/rc.c hfi1_tid_rdma_restart_req(qp, wqe, &bth2); wqe 1108 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR); wqe 1113 drivers/infiniband/hw/hfi1/rc.c wqe->length - req->seg_len * req->cur_seg); wqe 1117 drivers/infiniband/hw/hfi1/rc.c delta = hfi1_build_tid_rdma_read_packet(wqe, ohdr, &bth1, wqe 1130 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, wqe 1131 drivers/infiniband/hw/hfi1/rc.c wqe->psn, wqe->lpsn, req); wqe 1134 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 1135 drivers/infiniband/hw/hfi1/rc.c delta = cmp_psn(qp->s_psn, wqe->psn); wqe 1141 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode != IB_WR_TID_RDMA_READ || delta == 0 || wqe 1156 drivers/infiniband/hw/hfi1/rc.c wpriv = wqe->priv; wqe 1159 drivers/infiniband/hw/hfi1/rc.c wqe->length - req->seg_len * req->cur_seg); wqe 1160 drivers/infiniband/hw/hfi1/rc.c delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1, wqe 1173 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, wqe 1174 drivers/infiniband/hw/hfi1/rc.c wqe->psn, wqe->lpsn, req); wqe 1178 drivers/infiniband/hw/hfi1/rc.c delta = delta_psn(bth2, wqe->psn); wqe 1180 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) wqe 1461 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe) wqe 1463 drivers/infiniband/hw/hfi1/rc.c u32 opcode = wqe->wr.opcode; wqe 1470 drivers/infiniband/hw/hfi1/rc.c struct tid_rdma_request *req = wqe_to_tid_req(wqe); wqe 1473 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(psn, wqe->lpsn) <= 0) { wqe 1476 drivers/infiniband/hw/hfi1/rc.c cur_seg = (psn - wqe->psn) / priv->pkts_ps; wqe 1481 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode, wqe 1482 drivers/infiniband/hw/hfi1/rc.c wqe->psn, wqe 1483 drivers/infiniband/hw/hfi1/rc.c wqe->lpsn, wqe 1504 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); wqe 1518 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(psn, wqe->psn) <= 0) { wqe 1522 drivers/infiniband/hw/hfi1/rc.c update_num_rd_atomic(qp, psn, wqe); wqe 1532 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, n); wqe 1533 drivers/infiniband/hw/hfi1/rc.c diff = cmp_psn(psn, wqe->psn); wqe 1536 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 1549 drivers/infiniband/hw/hfi1/rc.c update_num_rd_atomic(qp, psn, wqe); wqe 1551 drivers/infiniband/hw/hfi1/rc.c opcode = wqe->wr.opcode; wqe 1610 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1625 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_OPFN) { wqe 1634 drivers/infiniband/hw/hfi1/rc.c wqe = do_rc_completion(qp, wqe, ibp); wqe 1638 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { wqe 1641 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 1646 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, wqe 1659 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 1660 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_TID_RDMA_READ) wqe 1680 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe; wqe 1686 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, n); wqe 1687 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(psn, wqe->lpsn) <= 0) { wqe 1688 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 1689 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_TID_RDMA_READ || wqe 1690 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) wqe 1691 drivers/infiniband/hw/hfi1/rc.c qp->s_sending_psn = wqe->lpsn + 1; wqe 1740 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe; wqe 1783 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, tail); wqe 1784 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 1824 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1825 drivers/infiniband/hw/hfi1/rc.c req = wqe_to_tid_req(wqe); wqe 1826 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && wqe 1832 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 1833 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && wqe 1836 drivers/infiniband/hw/hfi1/rc.c trdma_clean_swqe(qp, wqe); wqe 1837 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); wqe 1839 drivers/infiniband/hw/hfi1/rc.c wqe, wqe 1840 drivers/infiniband/hw/hfi1/rc.c ib_hfi1_wc_opcode[wqe->wr.opcode], wqe 1868 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe, wqe 1879 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_rc_completion(qp, wqe->lpsn); wqe 1880 drivers/infiniband/hw/hfi1/rc.c if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || wqe 1882 drivers/infiniband/hw/hfi1/rc.c trdma_clean_swqe(qp, wqe); wqe 1883 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); wqe 1885 drivers/infiniband/hw/hfi1/rc.c wqe, wqe 1886 drivers/infiniband/hw/hfi1/rc.c ib_hfi1_wc_opcode[wqe->wr.opcode], wqe 1917 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) wqe 1918 drivers/infiniband/hw/hfi1/rc.c update_last_psn(qp, wqe->lpsn); wqe 1929 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 1932 drivers/infiniband/hw/hfi1/rc.c qp->s_psn = wqe->psn; wqe 1939 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1945 drivers/infiniband/hw/hfi1/rc.c return wqe; wqe 2013 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe; wqe 2029 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 2036 drivers/infiniband/hw/hfi1/rc.c while ((diff = delta_psn(ack_psn, wqe->lpsn)) >= 0) { wqe 2043 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ && wqe 2058 drivers/infiniband/hw/hfi1/rc.c if ((wqe->wr.opcode == IB_WR_RDMA_READ && wqe 2060 drivers/infiniband/hw/hfi1/rc.c (wqe->wr.opcode == IB_WR_TID_RDMA_READ && wqe 2062 drivers/infiniband/hw/hfi1/rc.c ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 2063 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && wqe 2065 drivers/infiniband/hw/hfi1/rc.c (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && wqe 2074 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 2075 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { wqe 2076 drivers/infiniband/hw/hfi1/rc.c u64 *vaddr = wqe->sg_list[0].vaddr; wqe 2079 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_OPFN) wqe 2083 drivers/infiniband/hw/hfi1/rc.c (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 2084 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 2085 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { wqe 2104 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) wqe 2107 drivers/infiniband/hw/hfi1/rc.c wqe = do_rc_completion(qp, wqe, ibp); wqe 2112 drivers/infiniband/hw/hfi1/rc.c trace_hfi1_rc_ack_do(qp, aeth, psn, wqe); wqe 2117 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { wqe 2118 drivers/infiniband/hw/hfi1/rc.c if (wqe_to_tid_req(wqe)->ack_pending) wqe 2198 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && wqe 2200 drivers/infiniband/hw/hfi1/rc.c cmp_psn(psn, wqe->psn) >= 0) wqe 2212 drivers/infiniband/hw/hfi1/rc.c if (!(rdi->post_parms[wqe->wr.opcode].flags & wqe 2228 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { wqe 2275 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) wqe 2278 drivers/infiniband/hw/hfi1/rc.c hfi1_trdma_send_complete(qp, wqe, status); wqe 2309 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe; wqe 2315 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 2317 drivers/infiniband/hw/hfi1/rc.c while (cmp_psn(psn, wqe->lpsn) > 0) { wqe 2318 drivers/infiniband/hw/hfi1/rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 2319 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_TID_RDMA_READ || wqe 2320 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_TID_RDMA_WRITE || wqe 2321 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 2322 drivers/infiniband/hw/hfi1/rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) wqe 2324 drivers/infiniband/hw/hfi1/rc.c wqe = do_rc_completion(qp, wqe, ibp); wqe 2353 drivers/infiniband/hw/hfi1/rc.c struct rvt_swqe *wqe; wqe 2397 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 2412 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 2413 drivers/infiniband/hw/hfi1/rc.c if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) wqe 2421 drivers/infiniband/hw/hfi1/rc.c wqe, psn, pmtu); wqe 2428 drivers/infiniband/hw/hfi1/rc.c if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) wqe 2475 drivers/infiniband/hw/hfi1/rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 2477 drivers/infiniband/hw/hfi1/rc.c wqe, psn, pmtu); wqe 2484 drivers/infiniband/hw/hfi1/rc.c if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) wqe 2518 drivers/infiniband/hw/hfi1/rc.c rvt_send_complete(qp, wqe, status); wqe 35 drivers/infiniband/hw/hfi1/rc.h static inline u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, wqe 40 drivers/infiniband/hw/hfi1/rc.h len = delta_psn(psn, wqe->psn) * pmtu; wqe 41 drivers/infiniband/hw/hfi1/rc.h return rvt_restart_sge(ss, wqe, len); wqe 56 drivers/infiniband/hw/hfi1/rc.h struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 377 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); wqe 385 drivers/infiniband/hw/hfi1/tid_rdma.c priv->tid_req.e.swqe = wqe; wqe 386 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->priv = priv; wqe 415 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 420 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); wqe 421 drivers/infiniband/hw/hfi1/tid_rdma.c kfree(wqe->priv); wqe 422 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->priv = NULL; wqe 1614 drivers/infiniband/hw/hfi1/tid_rdma.c void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) wqe 1616 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_swqe_priv *p = wqe->priv; wqe 1696 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, wqe 1700 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_rdma_request *req = wqe_to_tid_req(wqe); wqe 1704 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_swqe_priv *wpriv = wqe->priv; wqe 1741 drivers/infiniband/hw/hfi1/tid_rdma.c rreq->reth.vaddr = cpu_to_be64(wqe->rdma_wr.remote_addr + wqe 1743 drivers/infiniband/hw/hfi1/tid_rdma.c rreq->reth.rkey = cpu_to_be32(wqe->rdma_wr.rkey); wqe 1780 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 1786 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_rdma_request *req = wqe_to_tid_req(wqe); wqe 1793 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_build_read_req(qp, 0, wqe->wr.opcode, wqe->psn, wqe 1794 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 1822 drivers/infiniband/hw/hfi1/tid_rdma.c restart_sge(&qp->s_sge, wqe, req->s_next_psn, wqe 1872 drivers/infiniband/hw/hfi1/tid_rdma.c hdwords = hfi1_build_tid_rdma_read_packet(wqe, ohdr, bth1, bth2, len); wqe 2411 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 2419 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); wqe 2420 drivers/infiniband/hw/hfi1/tid_rdma.c if (cmp_psn(psn, wqe->psn) >= 0 && wqe 2421 drivers/infiniband/hw/hfi1/tid_rdma.c cmp_psn(psn, wqe->lpsn) <= 0) { wqe 2422 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == opcode) wqe 2423 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 2575 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 2582 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, n); wqe 2583 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { wqe 2584 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 2623 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_qp *qp, struct rvt_swqe *wqe) wqe 2630 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 2656 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 2682 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 2686 drivers/infiniband/hw/hfi1/tid_rdma.c while ((int)delta_psn(ack_psn, wqe->lpsn) >= 0) { wqe 2692 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 2693 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->wr.opcode == IB_WR_TID_RDMA_READ || wqe 2694 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 2695 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { wqe 2699 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { wqe 2701 drivers/infiniband/hw/hfi1/tid_rdma.c wqe); wqe 2721 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = do_rc_completion(qp, wqe, ibp); wqe 2730 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) wqe 2733 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 2734 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe->wr.opcode, wqe->psn, wqe 2735 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 2801 drivers/infiniband/hw/hfi1/tid_rdma.c wqe); wqe 3037 drivers/infiniband/hw/hfi1/tid_rdma.c void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 3040 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_rdma_request *req = wqe_to_tid_req(wqe); wqe 3047 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { wqe 3054 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe 3055 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->psn, wqe->lpsn, wqe 3065 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) wqe 3097 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { wqe 3116 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) wqe 3123 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe->psn, wqe 3124 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 3126 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { wqe 3143 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); wqe 3144 drivers/infiniband/hw/hfi1/tid_rdma.c } while (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE); wqe 3145 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 3172 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); wqe 3177 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) wqe 3180 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_swqe_priv *priv = wqe->priv; wqe 3201 drivers/infiniband/hw/hfi1/tid_rdma.c bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) wqe 3211 drivers/infiniband/hw/hfi1/tid_rdma.c switch (wqe->wr.opcode) { wqe 3269 drivers/infiniband/hw/hfi1/tid_rdma.c void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe) wqe 3272 drivers/infiniband/hw/hfi1/tid_rdma.c struct hfi1_swqe_priv *priv = wqe->priv; wqe 3293 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_RDMA_READ) { wqe 3294 drivers/infiniband/hw/hfi1/tid_rdma.c if (hfi1_check_sge_align(qp, &wqe->sg_list[0], wqe 3295 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->wr.num_sge)) { wqe 3299 drivers/infiniband/hw/hfi1/tid_rdma.c } else if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { wqe 3306 drivers/infiniband/hw/hfi1/tid_rdma.c if (!(wqe->rdma_wr.remote_addr & ~PAGE_MASK) && wqe 3307 drivers/infiniband/hw/hfi1/tid_rdma.c !(wqe->length & ~PAGE_MASK)) { wqe 3316 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->wr.opcode = new_opcode; wqe 3318 drivers/infiniband/hw/hfi1/tid_rdma.c min_t(u32, remote->max_len, wqe->length); wqe 3320 drivers/infiniband/hw/hfi1/tid_rdma.c DIV_ROUND_UP(wqe->length, priv->tid_req.seg_len); wqe 3322 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn = wqe->psn; wqe 3323 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { wqe 3326 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn += rvt_div_round_up_mtu(qp, wqe->length) - 1; wqe 3328 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn += priv->tid_req.total_segs - 1; wqe 3343 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_setup_tid_wqe(qp, 1, wqe->wr.opcode, wqe 3344 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->psn, wqe->lpsn, wqe 3353 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 3358 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_rdma_request *req = wqe_to_tid_req(wqe); wqe 3373 drivers/infiniband/hw/hfi1/tid_rdma.c cpu_to_be64(wqe->rdma_wr.remote_addr + (wqe->length - *len)); wqe 3375 drivers/infiniband/hw/hfi1/tid_rdma.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 4031 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 4068 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); wqe 4069 drivers/infiniband/hw/hfi1/tid_rdma.c if (unlikely(wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)) wqe 4072 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 4107 drivers/infiniband/hw/hfi1/tid_rdma.c (wqe->length - (req->comp_seg * req->seg_len))); wqe 4142 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_write_resp(qp, 0, wqe->wr.opcode, wqe->psn, wqe 4143 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 4148 drivers/infiniband/hw/hfi1/tid_rdma.c if (!cmp_psn(psn, wqe->psn)) { wqe 4149 drivers/infiniband/hw/hfi1/tid_rdma.c req->r_last_acked = mask_psn(wqe->psn - 1); wqe 4170 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); wqe 4173 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) wqe 4192 drivers/infiniband/hw/hfi1/tid_rdma.c bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe, wqe 4196 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_rdma_request *req = wqe_to_tid_req(wqe); wqe 4208 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_trdma_send_complete(qp, wqe, IB_WC_REM_INV_RD_REQ_ERR); wqe 4489 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 4524 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 4526 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) wqe 4529 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 4530 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, wqe 4531 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 4547 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, wqe 4548 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 4551 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = do_rc_completion(qp, wqe, wqe 4558 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) wqe 4560 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 4566 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, wqe 4567 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 4574 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && wqe 4621 drivers/infiniband/hw/hfi1/tid_rdma.c if (delta_psn(ack_psn, wqe->lpsn)) wqe 4622 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 4623 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 4691 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, last_acked); wqe 4692 drivers/infiniband/hw/hfi1/tid_rdma.c rptr = wqe_to_tid_req(wqe); wqe 4789 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 4804 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 4805 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_trdma_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); wqe 4808 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 4809 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 4811 drivers/infiniband/hw/hfi1/tid_rdma.c qp, 0, wqe->wr.opcode, wqe->psn, wqe->lpsn, req); wqe 4830 drivers/infiniband/hw/hfi1/tid_rdma.c u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 4836 drivers/infiniband/hw/hfi1/tid_rdma.c struct tid_rdma_request *req = wqe_to_tid_req(wqe); wqe 4975 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 4987 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, i); wqe 4988 drivers/infiniband/hw/hfi1/tid_rdma.c if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) wqe 4999 drivers/infiniband/hw/hfi1/tid_rdma.c struct rvt_swqe *wqe; wqe 5055 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); wqe 5056 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 5057 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe->psn, wqe 5058 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->lpsn, req); wqe 5062 drivers/infiniband/hw/hfi1/tid_rdma.c priv->tid_ss.sge = wqe->sg_list[0]; wqe 5063 drivers/infiniband/hw/hfi1/tid_rdma.c priv->tid_ss.sg_list = wqe->sg_list + 1; wqe 5064 drivers/infiniband/hw/hfi1/tid_rdma.c priv->tid_ss.num_sge = wqe->wr.num_sge; wqe 5065 drivers/infiniband/hw/hfi1/tid_rdma.c priv->tid_ss.total_len = wqe->length; wqe 5068 drivers/infiniband/hw/hfi1/tid_rdma.c hfi1_tid_rdma_restart_req(qp, wqe, &bth2); wqe 5088 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); wqe 5089 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 5090 drivers/infiniband/hw/hfi1/tid_rdma.c len = wqe->length; wqe 5095 drivers/infiniband/hw/hfi1/tid_rdma.c trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe 5096 drivers/infiniband/hw/hfi1/tid_rdma.c wqe->psn, wqe->lpsn, req); wqe 5097 drivers/infiniband/hw/hfi1/tid_rdma.c last = hfi1_build_tid_rdma_packet(wqe, ohdr, &bth1, &bth2, wqe 5123 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, priv->s_tid_cur); wqe 5124 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 5127 drivers/infiniband/hw/hfi1/tid_rdma.c wqe = rvt_get_swqe_ptr(qp, wqe 5130 drivers/infiniband/hw/hfi1/tid_rdma.c req = wqe_to_tid_req(wqe); wqe 5132 drivers/infiniband/hw/hfi1/tid_rdma.c hwords += hfi1_build_tid_rdma_resync(qp, wqe, ohdr, &bth1, wqe 214 drivers/infiniband/hw/hfi1/tid_rdma.h void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe); wqe 221 drivers/infiniband/hw/hfi1/tid_rdma.h static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) wqe 223 drivers/infiniband/hw/hfi1/tid_rdma.h if (!wqe->priv) wqe 225 drivers/infiniband/hw/hfi1/tid_rdma.h __trdma_clean_swqe(qp, wqe); wqe 244 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, wqe 247 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 258 drivers/infiniband/hw/hfi1/tid_rdma.h void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 261 drivers/infiniband/hw/hfi1/tid_rdma.h bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe); wqe 263 drivers/infiniband/hw/hfi1/tid_rdma.h void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); wqe 265 drivers/infiniband/hw/hfi1/tid_rdma.h struct rvt_swqe *wqe) wqe 267 drivers/infiniband/hw/hfi1/tid_rdma.h if (wqe->priv && wqe 268 drivers/infiniband/hw/hfi1/tid_rdma.h (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 269 drivers/infiniband/hw/hfi1/tid_rdma.h wqe->wr.opcode == IB_WR_RDMA_WRITE) && wqe 270 drivers/infiniband/hw/hfi1/tid_rdma.h wqe->length >= TID_RDMA_MIN_SEGMENT_SIZE) wqe 271 drivers/infiniband/hw/hfi1/tid_rdma.h setup_tid_rdma_wqe(qp, wqe); wqe 274 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 289 drivers/infiniband/hw/hfi1/tid_rdma.h bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe, wqe 304 drivers/infiniband/hw/hfi1/tid_rdma.h u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 121 drivers/infiniband/hw/hfi1/trace_rc.h struct rvt_swqe *wqe), wqe 122 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, aeth, psn, wqe), wqe 137 drivers/infiniband/hw/hfi1/trace_rc.h __entry->opcode = wqe->wr.opcode; wqe 138 drivers/infiniband/hw/hfi1/trace_rc.h __entry->spsn = wqe->psn; wqe 139 drivers/infiniband/hw/hfi1/trace_rc.h __entry->lpsn = wqe->lpsn; wqe 156 drivers/infiniband/hw/hfi1/trace_rc.h struct rvt_swqe *wqe), wqe 157 drivers/infiniband/hw/hfi1/trace_rc.h TP_ARGS(qp, aeth, psn, wqe) wqe 780 drivers/infiniband/hw/hfi1/trace_tx.h TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, u32 idx), wqe 781 drivers/infiniband/hw/hfi1/trace_tx.h TP_ARGS(qp, wqe, idx), wqe 784 drivers/infiniband/hw/hfi1/trace_tx.h __field(struct rvt_swqe *, wqe) wqe 796 drivers/infiniband/hw/hfi1/trace_tx.h __entry->wqe = wqe; wqe 797 drivers/infiniband/hw/hfi1/trace_tx.h __entry->wr_id = wqe->wr.wr_id; wqe 800 drivers/infiniband/hw/hfi1/trace_tx.h __entry->length = wqe->length; wqe 802 drivers/infiniband/hw/hfi1/trace_tx.h __entry->ssn = wqe->ssn; wqe 803 drivers/infiniband/hw/hfi1/trace_tx.h __entry->opcode = wqe->wr.opcode; wqe 804 drivers/infiniband/hw/hfi1/trace_tx.h __entry->send_flags = wqe->wr.send_flags; wqe 811 drivers/infiniband/hw/hfi1/trace_tx.h __entry->wqe, wqe 67 drivers/infiniband/hw/hfi1/uc.c struct rvt_swqe *wqe; wqe 90 drivers/infiniband/hw/hfi1/uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 91 drivers/infiniband/hw/hfi1/uc.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); wqe 113 drivers/infiniband/hw/hfi1/uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 129 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.opcode == IB_WR_REG_MR || wqe 130 drivers/infiniband/hw/hfi1/uc.c wqe->wr.opcode == IB_WR_LOCAL_INV) { wqe 138 drivers/infiniband/hw/hfi1/uc.c if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) { wqe 140 drivers/infiniband/hw/hfi1/uc.c qp, wqe->wr.ex.invalidate_rkey); wqe 143 drivers/infiniband/hw/hfi1/uc.c rvt_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR wqe 152 drivers/infiniband/hw/hfi1/uc.c qp->s_psn = wqe->psn; wqe 153 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.sge = wqe->sg_list[0]; wqe 154 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.sg_list = wqe->sg_list + 1; wqe 155 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.num_sge = wqe->wr.num_sge; wqe 156 drivers/infiniband/hw/hfi1/uc.c qp->s_sge.total_len = wqe->length; wqe 157 drivers/infiniband/hw/hfi1/uc.c len = wqe->length; wqe 159 drivers/infiniband/hw/hfi1/uc.c switch (wqe->wr.opcode) { wqe 167 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.opcode == IB_WR_SEND) { wqe 173 drivers/infiniband/hw/hfi1/uc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 176 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 178 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; wqe 186 drivers/infiniband/hw/hfi1/uc.c cpu_to_be64(wqe->rdma_wr.remote_addr); wqe 188 drivers/infiniband/hw/hfi1/uc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 196 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { wqe 202 drivers/infiniband/hw/hfi1/uc.c ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; wqe 204 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 207 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; wqe 227 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.opcode == IB_WR_SEND) { wqe 232 drivers/infiniband/hw/hfi1/uc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 235 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 237 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; wqe 252 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { wqe 258 drivers/infiniband/hw/hfi1/uc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 260 drivers/infiniband/hw/hfi1/uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 263 drivers/infiniband/hw/hfi1/uc.c qp->s_wqe = wqe; wqe 266 drivers/infiniband/hw/hfi1/ud.c static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 274 drivers/infiniband/hw/hfi1/ud.c if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { wqe 275 drivers/infiniband/hw/hfi1/ud.c ohdr->u.ud.imm_data = wqe->wr.ex.imm_data; wqe 281 drivers/infiniband/hw/hfi1/ud.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 285 drivers/infiniband/hw/hfi1/ud.c *pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); wqe 291 drivers/infiniband/hw/hfi1/ud.c ohdr->bth[1] = cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); wqe 292 drivers/infiniband/hw/hfi1/ud.c ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn)); wqe 298 drivers/infiniband/hw/hfi1/ud.c cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : wqe 299 drivers/infiniband/hw/hfi1/ud.c rvt_get_swqe_remote_qkey(wqe)); wqe 304 drivers/infiniband/hw/hfi1/ud.c struct rvt_swqe *wqe) wqe 319 drivers/infiniband/hw/hfi1/ud.c ah_attr = rvt_get_swqe_ah_attr(wqe); wqe 321 drivers/infiniband/hw/hfi1/ud.c extra_bytes = -wqe->length & 3; wqe 322 drivers/infiniband/hw/hfi1/ud.c nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; wqe 325 drivers/infiniband/hw/hfi1/ud.c if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) wqe 365 drivers/infiniband/hw/hfi1/ud.c hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false); wqe 375 drivers/infiniband/hw/hfi1/ud.c struct rvt_swqe *wqe) wqe 383 drivers/infiniband/hw/hfi1/ud.c u32 dest_qp = rvt_get_swqe_remote_qpn(wqe); wqe 391 drivers/infiniband/hw/hfi1/ud.c ah_attr = rvt_get_swqe_ah_attr(wqe); wqe 404 drivers/infiniband/hw/hfi1/ud.c if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) wqe 410 drivers/infiniband/hw/hfi1/ud.c wqe->length); wqe 411 drivers/infiniband/hw/hfi1/ud.c nwords = ((wqe->length + extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC; wqe 453 drivers/infiniband/hw/hfi1/ud.c pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); wqe 457 drivers/infiniband/hw/hfi1/ud.c hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true); wqe 482 drivers/infiniband/hw/hfi1/ud.c struct rvt_swqe *wqe; wqe 501 drivers/infiniband/hw/hfi1/ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 502 drivers/infiniband/hw/hfi1/ud.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); wqe 510 drivers/infiniband/hw/hfi1/ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 518 drivers/infiniband/hw/hfi1/ud.c ah_attr = rvt_get_swqe_ah_attr(wqe); wqe 541 drivers/infiniband/hw/hfi1/ud.c ud_loopback(qp, wqe); wqe 544 drivers/infiniband/hw/hfi1/ud.c rvt_send_complete(qp, wqe, IB_WC_SUCCESS); wqe 550 drivers/infiniband/hw/hfi1/ud.c ps->s_txreq->s_cur_size = wqe->length; wqe 554 drivers/infiniband/hw/hfi1/ud.c qp->s_wqe = wqe; wqe 555 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.sge = wqe->sg_list[0]; wqe 556 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.sg_list = wqe->sg_list + 1; wqe 557 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.num_sge = wqe->wr.num_sge; wqe 558 drivers/infiniband/hw/hfi1/ud.c qp->s_sge.total_len = wqe->length; wqe 635 drivers/infiniband/hw/hfi1/verbs.c if (tx->wqe) { wqe 636 drivers/infiniband/hw/hfi1/verbs.c rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS); wqe 888 drivers/infiniband/hw/hfi1/verbs.c tx->wqe = qp->s_wqe; wqe 376 drivers/infiniband/hw/hfi1/verbs.h static inline struct tid_rdma_request *wqe_to_tid_req(struct rvt_swqe *wqe) wqe 378 drivers/infiniband/hw/hfi1/verbs.h return &((struct hfi1_swqe_priv *)wqe->priv)->tid_req; wqe 434 drivers/infiniband/hw/hfi1/verbs.h int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 494 drivers/infiniband/hw/hfi1/verbs.h struct rvt_swqe *wqe, wqe 497 drivers/infiniband/hw/hfi1/verbs.h trdma_clean_swqe(qp, wqe); wqe 498 drivers/infiniband/hw/hfi1/verbs.h rvt_send_complete(qp, wqe, status); wqe 62 drivers/infiniband/hw/hfi1/verbs_txreq.h struct rvt_swqe *wqe; wqe 75 drivers/infiniband/hw/hns/hns_roce_hw_v1.c void *wqe = NULL; wqe 109 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wqe = get_send_wqe(qp, wqe_idx); wqe 114 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ud_sq_wqe = wqe; wqe 217 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ctrl = wqe; wqe 254 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wqe += sizeof(struct hns_roce_wqe_ctrl_seg); wqe 259 drivers/infiniband/hw/hns/hns_roce_hw_v1.c set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, wqe 265 drivers/infiniband/hw/hns/hns_roce_hw_v1.c set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, wqe 283 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wqe += sizeof(struct hns_roce_wqe_raddr_seg); wqe 285 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dseg = wqe; wqe 297 drivers/infiniband/hw/hns/hns_roce_hw_v1.c memcpy(wqe, ((void *) (uintptr_t) wqe 300 drivers/infiniband/hw/hns/hns_roce_hw_v1.c wqe += wr->sg_list[i].length; wqe 161 drivers/infiniband/hw/hns/hns_roce_hw_v2.c void *wqe, unsigned int *sge_ind, wqe 166 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_wqe_data_seg *dseg = wqe; wqe 187 drivers/infiniband/hw/hns/hns_roce_hw_v2.c memcpy(wqe, ((void *)wr->sg_list[i].addr), wqe 189 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe += wr->sg_list[i].length; wqe 251 drivers/infiniband/hw/hns/hns_roce_hw_v2.c void *wqe = NULL; wqe 296 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe = get_send_wqe(qp, wqe_idx); wqe 312 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ud_sq_wqe = wqe; wqe 437 drivers/infiniband/hw/hns/hns_roce_hw_v2.c rc_sq_wqe = wqe; wqe 473 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe += sizeof(struct hns_roce_v2_rc_send_wqe); wqe 514 drivers/infiniband/hw/hns/hns_roce_hw_v2.c fseg = wqe; wqe 552 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dseg = wqe; wqe 554 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe += sizeof(struct hns_roce_v2_wqe_data_seg); wqe 555 drivers/infiniband/hw/hns/hns_roce_hw_v2.c set_atomic_seg(wqe, atomic_wr(wr)); wqe 562 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe, &sge_idx, wqe 628 drivers/infiniband/hw/hns/hns_roce_hw_v2.c void *wqe = NULL; wqe 661 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe = get_recv_wqe(hr_qp, wqe_idx); wqe 662 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; wqe 6240 drivers/infiniband/hw/hns/hns_roce_hw_v2.c void *wqe; wqe 6270 drivers/infiniband/hw/hns/hns_roce_hw_v2.c wqe = get_srq_wqe(srq, wqe_idx); wqe 6271 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; wqe 51 drivers/infiniband/hw/i40iw/i40iw_ctrl.c void i40iw_insert_wqe_hdr(u64 *wqe, u64 header) wqe 54 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 24, header); wqe 607 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe = NULL; wqe 627 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = cqp->sq_base[*wqe_idx].elem; wqe 629 drivers/infiniband/hw/i40iw/i40iw_ctrl.c I40IW_CQP_INIT_WQE(wqe); wqe 631 drivers/infiniband/hw/i40iw/i40iw_ctrl.c return wqe; wqe 835 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 841 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 842 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 845 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, info->qs_handle); wqe 852 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 855 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 877 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 882 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 883 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 891 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 893 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 915 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 920 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 921 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 924 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, wqe 931 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 934 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 988 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 993 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 994 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 997 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, hmc_fn_id); wqe 998 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, commit_fpm_mem->pa); wqe 1003 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1006 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1050 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1055 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1056 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1059 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, hmc_fn_id); wqe 1060 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, query_fpm_mem->pa); wqe 1065 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1068 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1100 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1103 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1104 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1106 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, info->reach_max); wqe 1115 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, temp); wqe 1123 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1126 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1146 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1149 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1150 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1156 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1159 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1179 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1182 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1183 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1191 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1194 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1214 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1217 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1218 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1221 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, info->port); wqe 1227 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1230 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1259 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1265 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1266 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1276 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, temp); wqe 1281 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 1285 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 1290 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 1298 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, qw2); wqe 1302 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 1306 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 1311 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 1317 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, qw1); wqe 1325 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, temp); wqe 1328 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1346 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1349 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1350 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1355 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1357 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1376 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1379 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1380 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1389 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, temp); wqe 1395 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1398 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1420 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1423 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1424 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1432 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1435 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1452 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1455 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1456 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1460 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1462 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1523 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1527 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1528 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1530 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, ceq->elem_cnt); wqe 1531 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa)); wqe 1532 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0)); wqe 1533 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL)); wqe 1542 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1545 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1603 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1607 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1608 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1610 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, ceq->elem_cnt); wqe 1611 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx); wqe 1618 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1620 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1705 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1710 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1711 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1713 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, aeq->elem_cnt); wqe 1714 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, wqe 1716 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, wqe 1724 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1726 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 1742 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 1747 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 1748 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 1750 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, aeq->elem_cnt); wqe 1751 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx); wqe 1756 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 1759 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2004 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2010 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2011 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2013 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, ccq->cq_uk.cq_size); wqe 2014 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, RS_64_1(ccq, 1)); wqe 2015 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, wqe 2017 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, (ccq->virtual_map ? 0 : ccq->cq_pa)); wqe 2018 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, ccq->shadow_area_pa); wqe 2019 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, wqe 2021 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 56, wqe 2036 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2039 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2063 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2069 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2070 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2072 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, ccq->cq_uk.cq_size); wqe 2073 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, RS_64_1(ccq, 1)); wqe 2074 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, ccq->shadow_area_pa); wqe 2085 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2088 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2162 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2173 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2174 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2177 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, cq->cq_uk.cq_size); wqe 2178 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, RS_64_1(cq, 1)); wqe 2179 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 2183 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa)); wqe 2185 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, cq->shadow_area_pa); wqe 2186 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); wqe 2187 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL)); wqe 2201 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2204 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2222 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2226 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2227 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2229 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, cq->cq_uk.cq_size); wqe 2230 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, RS_64_1(cq, 1)); wqe 2231 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, cq->shadow_area_pa); wqe 2232 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); wqe 2245 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2248 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2268 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2285 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2286 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2318 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, cq_size); wqe 2319 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, RS_64_1(cq, 1)); wqe 2320 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, wqe 2322 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa)); wqe 2323 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, cq->shadow_area_pa); wqe 2324 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, (cq->virtual_map ? first_pm_pbl_idx : 0)); wqe 2325 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL)); wqe 2340 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2343 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2445 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2453 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2454 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2457 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); wqe 2459 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, qp->shadow_area_pa); wqe 2472 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2474 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2494 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2501 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2502 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2514 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 2518 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); wqe 2519 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, qp->shadow_area_pa); wqe 2537 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2540 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2562 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2568 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2569 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2571 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); wqe 2572 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, qp->shadow_area_pa); wqe 2581 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2583 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2604 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2621 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2622 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2634 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, temp); wqe 2639 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, temp); wqe 2649 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2652 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2673 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2678 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2679 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2685 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, temp); wqe 2692 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2695 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2715 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2720 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2721 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2723 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, info->buf_pa); wqe 2732 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2735 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2943 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 2950 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 2951 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 2953 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 2957 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 2960 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 2974 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 2977 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 2997 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3019 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3020 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3024 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, temp); wqe 3026 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3031 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3036 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, info->reg_addr_pa); wqe 3037 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, 0); wqe 3039 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, 0); wqe 3040 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, info->first_pm_pbl_index); wqe 3042 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, info->hmc_fcn_index); wqe 3043 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 56, 0); wqe 3057 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3060 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3080 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3093 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3094 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3100 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3104 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3110 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, temp); wqe 3120 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3123 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3144 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3148 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3149 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3151 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3154 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3162 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3165 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3185 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3189 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3190 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3192 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3199 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3202 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3226 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3229 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3230 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3232 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID)); wqe 3233 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3240 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3243 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3262 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3267 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, wqe 3269 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3276 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, temp); wqe 3279 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3284 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3301 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3304 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_QP_WQE_MIN_SIZE); wqe 3323 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3328 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = qp_uk->sq_base->elem; wqe 3330 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf); wqe 3332 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG))); wqe 3334 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, 0); wqe 3341 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3344 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_QP_WQE_MIN_SIZE); wqe 3357 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3362 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = qp_uk->sq_base->elem; wqe 3364 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf); wqe 3366 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, size); wqe 3368 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, 0); wqe 3375 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3378 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_QP_WQE_MIN_SIZE); wqe 3388 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3393 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = qp_uk->sq_base->elem; wqe 3395 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, 0); wqe 3396 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, 0); wqe 3397 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, 0); wqe 3402 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, ((u64)0xabcd << 32)); wqe 3408 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3411 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_QP_WQE_MIN_SIZE); wqe 3421 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3426 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = qp_uk->sq_base->elem; wqe 3428 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3432 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, 0); wqe 3433 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, 0); wqe 3434 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, 0); wqe 3438 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3441 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, 0); wqe 3442 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, 0); wqe 3443 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, 0); wqe 3449 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3639 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3645 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); wqe 3646 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3649 drivers/infiniband/hw/i40iw/i40iw_ctrl.c I40IW_CQP_INIT_WQE(wqe); wqe 3667 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 16, data); wqe 3671 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 48, wqe 3675 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 56, info->entry[2].data); wqe 3678 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 32, wqe 3682 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 40, info->entry[1].data); wqe 3685 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 0, wqe 3688 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, 8, info->entry[0].data); wqe 3694 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3697 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3757 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3759 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3760 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3766 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3769 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3786 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3788 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3789 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3791 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3799 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3802 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 3824 drivers/infiniband/hw/i40iw/i40iw_ctrl.c u64 *wqe; wqe 3828 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 3829 drivers/infiniband/hw/i40iw/i40iw_ctrl.c if (!wqe) wqe 3831 drivers/infiniband/hw/i40iw/i40iw_ctrl.c set_64bit_val(wqe, wqe 3838 drivers/infiniband/hw/i40iw/i40iw_ctrl.c i40iw_insert_wqe_hdr(wqe, header); wqe 3841 drivers/infiniband/hw/i40iw/i40iw_ctrl.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 114 drivers/infiniband/hw/i40iw/i40iw_d.h #define I40IW_CQP_INIT_WQE(wqe) memset(wqe, 0, 64) wqe 62 drivers/infiniband/hw/i40iw/i40iw_p.h void i40iw_insert_wqe_hdr(u64 *wqe, u64 header); wqe 112 drivers/infiniband/hw/i40iw/i40iw_puda.c u64 *wqe; wqe 117 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = qp->qp_uk.rq_base[wqe_idx].elem; wqe 120 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe_idx, buf, wqe); wqe 122 drivers/infiniband/hw/i40iw/i40iw_puda.c get_64bit_val(wqe, 24, &offset24); wqe 126 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 0, buf->mem.pa); wqe 127 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 8, wqe 129 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_insert_wqe_hdr(wqe, offset24); wqe 208 drivers/infiniband/hw/i40iw/i40iw_puda.c u64 *wqe = NULL; wqe 216 drivers/infiniband/hw/i40iw/i40iw_puda.c return wqe; wqe 217 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = qp->sq_base[*wqe_idx].elem; wqe 219 drivers/infiniband/hw/i40iw/i40iw_puda.c return wqe; wqe 374 drivers/infiniband/hw/i40iw/i40iw_puda.c u64 *wqe; wqe 390 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = i40iw_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx); wqe 391 drivers/infiniband/hw/i40iw/i40iw_puda.c if (!wqe) wqe 406 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 0, info->paddr); wqe 407 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 8, LS_64(info->len, I40IWQPSQ_FRAG_LEN)); wqe 408 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 16, header[0]); wqe 410 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_insert_wqe_hdr(wqe, header[1]); wqe 412 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_debug_buf(qp->dev, I40IW_DEBUG_PUDA, "PUDA SEND WQE", wqe, 32); wqe 519 drivers/infiniband/hw/i40iw/i40iw_puda.c u64 *wqe; wqe 525 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0); wqe 526 drivers/infiniband/hw/i40iw/i40iw_puda.c if (!wqe) wqe 529 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); wqe 530 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 40, qp->shadow_area_pa); wqe 538 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_insert_wqe_hdr(wqe, header); wqe 540 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_debug_buf(cqp->dev, I40IW_DEBUG_PUDA, "PUDA CQE", wqe, 32); wqe 630 drivers/infiniband/hw/i40iw/i40iw_puda.c u64 *wqe; wqe 637 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0); wqe 638 drivers/infiniband/hw/i40iw/i40iw_puda.c if (!wqe) wqe 641 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 0, cq->cq_uk.cq_size); wqe 642 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 8, RS_64_1(cq, 1)); wqe 643 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 16, wqe 646 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 32, cq->cq_pa); wqe 648 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 40, cq->shadow_area_pa); wqe 656 drivers/infiniband/hw/i40iw/i40iw_puda.c i40iw_insert_wqe_hdr(wqe, header); wqe 659 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe, I40IW_CQP_WQE_SIZE * 8); wqe 991 drivers/infiniband/hw/i40iw/i40iw_puda.c u64 *wqe; wqe 994 drivers/infiniband/hw/i40iw/i40iw_puda.c wqe = qp->qp_uk.rq_base[wqe_idx].elem; wqe 995 drivers/infiniband/hw/i40iw/i40iw_puda.c get_64bit_val(wqe, 24, &offset24); wqe 997 drivers/infiniband/hw/i40iw/i40iw_puda.c set_64bit_val(wqe, 24, offset24); wqe 49 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 header, *wqe; wqe 58 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = qp->sq_base[wqe_idx].elem; wqe 69 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 0, 0); wqe 70 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 8, 0); wqe 71 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, 0); wqe 79 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 141 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe = NULL; wqe 183 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = qp->sq_base[*wqe_idx].elem; wqe 196 drivers/infiniband/hw/i40iw/i40iw_uk.c return wqe; wqe 205 drivers/infiniband/hw/i40iw/i40iw_uk.c static void i40iw_set_fragment(u64 *wqe, u32 offset, struct i40iw_sge *sge) wqe 208 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, offset, LS_64(sge->tag_off, I40IWQPSQ_FRAG_TO)); wqe 209 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, (offset + 8), wqe 222 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe = NULL; wqe 234 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = qp->rq_base[*wqe_idx * (qp->rq_wqe_size_multiplier >> 2)].elem; wqe 236 drivers/infiniband/hw/i40iw/i40iw_uk.c return wqe; wqe 250 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 274 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); wqe 275 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 277 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, wqe 290 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_set_fragment(wqe, 0, op_info->lo_sg_list); wqe 293 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_set_fragment(wqe, byte_off, &op_info->lo_sg_list[i]); wqe 299 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 319 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 331 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->lo_addr.len, info->wr_id); wqe 332 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 336 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO)); wqe 344 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_set_fragment(wqe, 0, &op_info->lo_addr); wqe 348 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 367 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 385 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); wqe 386 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 390 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, 0); wqe 400 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_set_fragment(wqe, 0, op_info->sg_list); wqe 403 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_set_fragment(wqe, byte_off, &op_info->sg_list[i]); wqe 409 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 426 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 444 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); wqe 445 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 449 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, wqe 462 drivers/infiniband/hw/i40iw/i40iw_uk.c dest = (u8 *)wqe; wqe 470 drivers/infiniband/hw/i40iw/i40iw_uk.c dest = (u8 *)wqe + 32; wqe 476 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 480 drivers/infiniband/hw/i40iw/i40iw_uk.c memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32); wqe 502 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 520 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); wqe 521 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 535 drivers/infiniband/hw/i40iw/i40iw_uk.c dest = (u8 *)wqe; wqe 543 drivers/infiniband/hw/i40iw/i40iw_uk.c dest = (u8 *)wqe + 32; wqe 549 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 553 drivers/infiniband/hw/i40iw/i40iw_uk.c memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32); wqe 573 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 582 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); wqe 583 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 585 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 0, 0); wqe 586 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 8, wqe 588 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, 0); wqe 597 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 615 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 624 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); wqe 625 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 627 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 0, (uintptr_t)op_info->va); wqe 628 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 8, wqe 631 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, op_info->bind_length); wqe 645 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 661 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 *wqe; wqe 669 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_recv_wqe(qp, &wqe_idx); wqe 670 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 674 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, 0); wqe 680 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_set_fragment(wqe, 0, info->sg_list); wqe 683 drivers/infiniband/hw/i40iw/i40iw_uk.c i40iw_set_fragment(wqe, byte_off, &info->sg_list[i]); wqe 689 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 1132 drivers/infiniband/hw/i40iw/i40iw_uk.c u64 header, *wqe; wqe 1135 drivers/infiniband/hw/i40iw/i40iw_uk.c wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, wr_id); wqe 1136 drivers/infiniband/hw/i40iw/i40iw_uk.c if (!wqe) wqe 1138 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 0, 0); wqe 1139 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 8, 0); wqe 1140 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 16, 0); wqe 1148 drivers/infiniband/hw/i40iw/i40iw_uk.c set_64bit_val(wqe, 24, header); wqe 56 drivers/infiniband/hw/i40iw/i40iw_vf.c u64 *wqe; wqe 59 drivers/infiniband/hw/i40iw/i40iw_vf.c wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch); wqe 60 drivers/infiniband/hw/i40iw/i40iw_vf.c if (!wqe) wqe 66 drivers/infiniband/hw/i40iw/i40iw_vf.c set_64bit_val(wqe, 16, temp); wqe 71 drivers/infiniband/hw/i40iw/i40iw_vf.c set_64bit_val(wqe, 24, header); wqe 74 drivers/infiniband/hw/i40iw/i40iw_vf.c set_64bit_val(wqe, 32, pd_pl_pba); wqe 76 drivers/infiniband/hw/i40iw/i40iw_vf.c i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE VF_PBLE_BP WQE", wqe, I40IW_CQP_WQE_SIZE * 8); wqe 214 drivers/infiniband/hw/mlx4/qp.c __be32 *wqe; wqe 224 drivers/infiniband/hw/mlx4/qp.c wqe = buf + i; wqe 225 drivers/infiniband/hw/mlx4/qp.c *wqe = cpu_to_be32(0xffffffff); wqe 2882 drivers/infiniband/hw/mlx4/qp.c void *wqe, unsigned *mlx_seg_len) wqe 2886 drivers/infiniband/hw/mlx4/qp.c struct mlx4_wqe_mlx_seg *mlx = wqe; wqe 2887 drivers/infiniband/hw/mlx4/qp.c struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; wqe 3033 drivers/infiniband/hw/mlx4/qp.c void *wqe, unsigned *mlx_seg_len) wqe 3037 drivers/infiniband/hw/mlx4/qp.c struct mlx4_wqe_mlx_seg *mlx = wqe; wqe 3038 drivers/infiniband/hw/mlx4/qp.c struct mlx4_wqe_ctrl_seg *ctrl = wqe; wqe 3039 drivers/infiniband/hw/mlx4/qp.c struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; wqe 3407 drivers/infiniband/hw/mlx4/qp.c static void build_tunnel_header(const struct ib_ud_wr *wr, void *wqe, wqe 3410 drivers/infiniband/hw/mlx4/qp.c struct mlx4_wqe_inline_seg *inl = wqe; wqe 3491 drivers/infiniband/hw/mlx4/qp.c static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, wqe 3495 drivers/infiniband/hw/mlx4/qp.c unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16); wqe 3504 drivers/infiniband/hw/mlx4/qp.c memcpy(wqe->header, wr->header, wr->hlen); wqe 3526 drivers/infiniband/hw/mlx4/qp.c static void add_zero_len_inline(void *wqe) wqe 3528 drivers/infiniband/hw/mlx4/qp.c struct mlx4_wqe_inline_seg *inl = wqe; wqe 3529 drivers/infiniband/hw/mlx4/qp.c memset(wqe, 0, 16); wqe 3537 drivers/infiniband/hw/mlx4/qp.c void *wqe; wqe 3599 drivers/infiniband/hw/mlx4/qp.c ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); wqe 3614 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof *ctrl; wqe 3624 drivers/infiniband/hw/mlx4/qp.c set_raddr_seg(wqe, atomic_wr(wr)->remote_addr, wqe 3626 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_raddr_seg); wqe 3628 drivers/infiniband/hw/mlx4/qp.c set_atomic_seg(wqe, atomic_wr(wr)); wqe 3629 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_atomic_seg); wqe 3637 drivers/infiniband/hw/mlx4/qp.c set_raddr_seg(wqe, atomic_wr(wr)->remote_addr, wqe 3639 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_raddr_seg); wqe 3641 drivers/infiniband/hw/mlx4/qp.c set_masked_atomic_seg(wqe, atomic_wr(wr)); wqe 3642 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_masked_atomic_seg); wqe 3652 drivers/infiniband/hw/mlx4/qp.c set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, wqe 3654 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_raddr_seg); wqe 3661 drivers/infiniband/hw/mlx4/qp.c set_local_inv_seg(wqe, wr->ex.invalidate_rkey); wqe 3662 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_local_inval_seg); wqe 3669 drivers/infiniband/hw/mlx4/qp.c set_reg_seg(wqe, reg_wr(wr)); wqe 3670 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof(struct mlx4_wqe_fmr_seg); wqe 3687 drivers/infiniband/hw/mlx4/qp.c wqe += seglen; wqe 3693 drivers/infiniband/hw/mlx4/qp.c set_datagram_seg(wqe, ud_wr(wr)); wqe 3695 drivers/infiniband/hw/mlx4/qp.c *(__be32 *) wqe |= cpu_to_be32(0x80000000); wqe 3696 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_datagram_seg); wqe 3700 drivers/infiniband/hw/mlx4/qp.c set_datagram_seg(wqe, ud_wr(wr)); wqe 3701 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_datagram_seg); wqe 3705 drivers/infiniband/hw/mlx4/qp.c err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen, wqe 3711 drivers/infiniband/hw/mlx4/qp.c lso_wqe = (__be32 *) wqe; wqe 3712 drivers/infiniband/hw/mlx4/qp.c wqe += seglen; wqe 3724 drivers/infiniband/hw/mlx4/qp.c wqe += seglen; wqe 3727 drivers/infiniband/hw/mlx4/qp.c add_zero_len_inline(wqe); wqe 3728 drivers/infiniband/hw/mlx4/qp.c wqe += 16; wqe 3730 drivers/infiniband/hw/mlx4/qp.c build_tunnel_header(ud_wr(wr), wqe, &seglen); wqe 3731 drivers/infiniband/hw/mlx4/qp.c wqe += seglen; wqe 3740 drivers/infiniband/hw/mlx4/qp.c set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe, wqe 3743 drivers/infiniband/hw/mlx4/qp.c wqe += sizeof (struct mlx4_wqe_datagram_seg); wqe 3745 drivers/infiniband/hw/mlx4/qp.c build_tunnel_header(ud_wr(wr), wqe, &seglen); wqe 3746 drivers/infiniband/hw/mlx4/qp.c wqe += seglen; wqe 3758 drivers/infiniband/hw/mlx4/qp.c wqe += seglen; wqe 3773 drivers/infiniband/hw/mlx4/qp.c dseg = wqe; wqe 62 drivers/infiniband/hw/mlx5/odp.c } wqe; wqe 402 drivers/infiniband/hw/mlx5/odp.c pfault->wqe.wq_num : pfault->token; wqe 955 drivers/infiniband/hw/mlx5/odp.c void *wqe, wqe 971 drivers/infiniband/hw/mlx5/odp.c while (wqe < wqe_end) { wqe 972 drivers/infiniband/hw/mlx5/odp.c struct mlx5_wqe_data_seg *dseg = wqe; wqe 982 drivers/infiniband/hw/mlx5/odp.c wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, wqe 985 drivers/infiniband/hw/mlx5/odp.c wqe += sizeof(*dseg); wqe 1027 drivers/infiniband/hw/mlx5/odp.c struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) wqe 1029 drivers/infiniband/hw/mlx5/odp.c struct mlx5_wqe_ctrl_seg *ctrl = *wqe; wqe 1030 drivers/infiniband/hw/mlx5/odp.c u16 wqe_index = pfault->wqe.wqe_index; wqe 1048 drivers/infiniband/hw/mlx5/odp.c *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; wqe 1049 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(*ctrl); wqe 1055 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(struct mlx5_wqe_xrc_seg); wqe 1059 drivers/infiniband/hw/mlx5/odp.c av = *wqe; wqe 1061 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(struct mlx5_av); wqe 1063 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(struct mlx5_base_av); wqe 1070 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(struct mlx5_wqe_raddr_seg); wqe 1074 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(struct mlx5_wqe_raddr_seg); wqe 1075 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(struct mlx5_wqe_atomic_seg); wqe 1087 drivers/infiniband/hw/mlx5/odp.c void **wqe, void **wqe_end, wqe 1097 drivers/infiniband/hw/mlx5/odp.c *wqe_end = *wqe + wqe_size; wqe 1098 drivers/infiniband/hw/mlx5/odp.c *wqe += sizeof(struct mlx5_wqe_srq_next_seg); wqe 1105 drivers/infiniband/hw/mlx5/odp.c void *wqe, void **wqe_end, wqe 1121 drivers/infiniband/hw/mlx5/odp.c *wqe_end = wqe + wqe_size; wqe 1169 drivers/infiniband/hw/mlx5/odp.c u16 wqe_index = pfault->wqe.wqe_index; wqe 1170 drivers/infiniband/hw/mlx5/odp.c void *wqe, *wqe_start = NULL, *wqe_end = NULL; wqe 1178 drivers/infiniband/hw/mlx5/odp.c res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); wqe 1180 drivers/infiniband/hw/mlx5/odp.c mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); wqe 1197 drivers/infiniband/hw/mlx5/odp.c wqe = wqe_start; wqe 1200 drivers/infiniband/hw/mlx5/odp.c ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, wqe 1205 drivers/infiniband/hw/mlx5/odp.c dev, pfault, qp, &wqe, &wqe_end, bytes_copied); wqe 1207 drivers/infiniband/hw/mlx5/odp.c ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, wqe 1212 drivers/infiniband/hw/mlx5/odp.c dev, qp, wqe, &wqe_end, bytes_copied); wqe 1216 drivers/infiniband/hw/mlx5/odp.c ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE, wqe 1221 drivers/infiniband/hw/mlx5/odp.c dev, srq, &wqe, &wqe_end, bytes_copied); wqe 1224 drivers/infiniband/hw/mlx5/odp.c if (ret < 0 || wqe >= wqe_end) wqe 1227 drivers/infiniband/hw/mlx5/odp.c ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped, wqe 1249 drivers/infiniband/hw/mlx5/odp.c pfault->wqe.wq_num, resume_with_error, wqe 1412 drivers/infiniband/hw/mlx5/odp.c (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; wqe 1414 drivers/infiniband/hw/mlx5/odp.c be32_to_cpu(pf_eqe->wqe.token); wqe 1415 drivers/infiniband/hw/mlx5/odp.c pfault->wqe.wq_num = wqe 1416 drivers/infiniband/hw/mlx5/odp.c be32_to_cpu(pf_eqe->wqe.pftype_wq) & wqe 1418 drivers/infiniband/hw/mlx5/odp.c pfault->wqe.wqe_index = wqe 1419 drivers/infiniband/hw/mlx5/odp.c be16_to_cpu(pf_eqe->wqe.wqe_index); wqe 1420 drivers/infiniband/hw/mlx5/odp.c pfault->wqe.packet_size = wqe 1421 drivers/infiniband/hw/mlx5/odp.c be16_to_cpu(pf_eqe->wqe.packet_length); wqe 1425 drivers/infiniband/hw/mlx5/odp.c pfault->wqe.wq_num, wqe 1426 drivers/infiniband/hw/mlx5/odp.c pfault->wqe.wqe_index); wqe 4418 drivers/infiniband/hw/mlx5/qp.c static u8 calc_sig(void *wqe, int size) wqe 4420 drivers/infiniband/hw/mlx5/qp.c u8 *p = wqe; wqe 4430 drivers/infiniband/hw/mlx5/qp.c static u8 wq_sig(void *wqe) wqe 4432 drivers/infiniband/hw/mlx5/qp.c return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4); wqe 4436 drivers/infiniband/hw/mlx5/qp.c void **wqe, int *wqe_sz, void **cur_edge) wqe 4443 drivers/infiniband/hw/mlx5/qp.c seg = *wqe; wqe 4444 drivers/infiniband/hw/mlx5/qp.c *wqe += sizeof(*seg); wqe 4460 drivers/infiniband/hw/mlx5/qp.c handle_post_send_edge(&qp->sq, wqe, wqe 4464 drivers/infiniband/hw/mlx5/qp.c leftlen = *cur_edge - *wqe; wqe 4467 drivers/infiniband/hw/mlx5/qp.c memcpy(*wqe, addr, copysz); wqe 4470 drivers/infiniband/hw/mlx5/qp.c *wqe += copysz; wqe 126 drivers/infiniband/hw/mthca/mthca_cq.c __be32 wqe; wqe 140 drivers/infiniband/hw/mthca/mthca_cq.c __be32 wqe; wqe 307 drivers/infiniband/hw/mthca/mthca_cq.c mthca_free_srq_wqe(srq, be32_to_cpu(cqe->wqe)); wqe 383 drivers/infiniband/hw/mthca/mthca_cq.c be32_to_cpu(cqe->my_qpn), be32_to_cpu(cqe->wqe), wqe 472 drivers/infiniband/hw/mthca/mthca_cq.c cqe->wqe = new_wqe; wqe 506 drivers/infiniband/hw/mthca/mthca_cq.c be32_to_cpu(cqe->wqe)); wqe 535 drivers/infiniband/hw/mthca/mthca_cq.c wqe_index = ((be32_to_cpu(cqe->wqe) - (*cur_qp)->send_wqe_offset) wqe 541 drivers/infiniband/hw/mthca/mthca_cq.c u32 wqe = be32_to_cpu(cqe->wqe); wqe 543 drivers/infiniband/hw/mthca/mthca_cq.c wqe_index = wqe >> srq->wqe_shift; wqe 545 drivers/infiniband/hw/mthca/mthca_cq.c mthca_free_srq_wqe(srq, wqe); wqe 547 drivers/infiniband/hw/mthca/mthca_cq.c s32 wqe; wqe 549 drivers/infiniband/hw/mthca/mthca_cq.c wqe = be32_to_cpu(cqe->wqe); wqe 550 drivers/infiniband/hw/mthca/mthca_cq.c wqe_index = wqe >> wq->wqe_shift; wqe 1628 drivers/infiniband/hw/mthca/mthca_qp.c void *wqe; wqe 1664 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_send_wqe(qp, ind); wqe 1666 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.last = wqe; wqe 1668 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->nda_op = 0; wqe 1669 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->ee_nds = 0; wqe 1670 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->flags = wqe 1678 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data; wqe 1680 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_next_seg); wqe 1688 drivers/infiniband/hw/mthca/mthca_qp.c set_raddr_seg(wqe, atomic_wr(wr)->remote_addr, wqe 1690 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_raddr_seg); wqe 1692 drivers/infiniband/hw/mthca/mthca_qp.c set_atomic_seg(wqe, atomic_wr(wr)); wqe 1693 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_atomic_seg); wqe 1701 drivers/infiniband/hw/mthca/mthca_qp.c set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, wqe 1703 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_raddr_seg); wqe 1718 drivers/infiniband/hw/mthca/mthca_qp.c set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, wqe 1720 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_raddr_seg); wqe 1732 drivers/infiniband/hw/mthca/mthca_qp.c set_tavor_ud_seg(wqe, ud_wr(wr)); wqe 1733 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_tavor_ud_seg); wqe 1739 drivers/infiniband/hw/mthca/mthca_qp.c wqe - sizeof (struct mthca_next_seg), wqe 1740 drivers/infiniband/hw/mthca/mthca_qp.c wqe); wqe 1745 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 1758 drivers/infiniband/hw/mthca/mthca_qp.c mthca_set_data_seg(wqe, wr->sg_list + i); wqe 1759 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 1765 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_data_seg *) wqe)->byte_count = wqe 1767 drivers/infiniband/hw/mthca/mthca_qp.c ((u32 *) wqe)[1] = 0; wqe 1768 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 1840 drivers/infiniband/hw/mthca/mthca_qp.c void *wqe; wqe 1860 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_recv_wqe(qp, ind); wqe 1862 drivers/infiniband/hw/mthca/mthca_qp.c qp->rq.last = wqe; wqe 1864 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->ee_nds = wqe 1866 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->flags = 0; wqe 1868 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_next_seg); wqe 1878 drivers/infiniband/hw/mthca/mthca_qp.c mthca_set_data_seg(wqe, wr->sg_list + i); wqe 1879 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 1932 drivers/infiniband/hw/mthca/mthca_qp.c void *wqe; wqe 1994 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_send_wqe(qp, ind); wqe 1996 drivers/infiniband/hw/mthca/mthca_qp.c qp->sq.last = wqe; wqe 1998 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->flags = wqe 2008 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data; wqe 2010 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_next_seg); wqe 2018 drivers/infiniband/hw/mthca/mthca_qp.c set_raddr_seg(wqe, atomic_wr(wr)->remote_addr, wqe 2020 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_raddr_seg); wqe 2022 drivers/infiniband/hw/mthca/mthca_qp.c set_atomic_seg(wqe, atomic_wr(wr)); wqe 2023 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_atomic_seg); wqe 2031 drivers/infiniband/hw/mthca/mthca_qp.c set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, wqe 2033 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_raddr_seg); wqe 2048 drivers/infiniband/hw/mthca/mthca_qp.c set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, wqe 2050 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_raddr_seg); wqe 2062 drivers/infiniband/hw/mthca/mthca_qp.c set_arbel_ud_seg(wqe, ud_wr(wr)); wqe 2063 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_arbel_ud_seg); wqe 2069 drivers/infiniband/hw/mthca/mthca_qp.c wqe - sizeof (struct mthca_next_seg), wqe 2070 drivers/infiniband/hw/mthca/mthca_qp.c wqe); wqe 2075 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 2088 drivers/infiniband/hw/mthca/mthca_qp.c mthca_set_data_seg(wqe, wr->sg_list + i); wqe 2089 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 2095 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_data_seg *) wqe)->byte_count = wqe 2097 drivers/infiniband/hw/mthca/mthca_qp.c ((u32 *) wqe)[1] = 0; wqe 2098 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 2170 drivers/infiniband/hw/mthca/mthca_qp.c void *wqe; wqe 2189 drivers/infiniband/hw/mthca/mthca_qp.c wqe = get_recv_wqe(qp, ind); wqe 2191 drivers/infiniband/hw/mthca/mthca_qp.c ((struct mthca_next_seg *) wqe)->flags = 0; wqe 2193 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_next_seg); wqe 2202 drivers/infiniband/hw/mthca/mthca_qp.c mthca_set_data_seg(wqe, wr->sg_list + i); wqe 2203 drivers/infiniband/hw/mthca/mthca_qp.c wqe += sizeof (struct mthca_data_seg); wqe 2207 drivers/infiniband/hw/mthca/mthca_qp.c mthca_set_data_seg_inval(wqe); wqe 92 drivers/infiniband/hw/mthca/mthca_srq.c static inline int *wqe_to_link(void *wqe) wqe 94 drivers/infiniband/hw/mthca/mthca_srq.c return (int *) (wqe + offsetof(struct mthca_next_seg, imm)); wqe 158 drivers/infiniband/hw/mthca/mthca_srq.c void *wqe; wqe 185 drivers/infiniband/hw/mthca/mthca_srq.c next = wqe = get_wqe(srq, i); wqe 188 drivers/infiniband/hw/mthca/mthca_srq.c *wqe_to_link(wqe) = i + 1; wqe 191 drivers/infiniband/hw/mthca/mthca_srq.c *wqe_to_link(wqe) = -1; wqe 195 drivers/infiniband/hw/mthca/mthca_srq.c for (scatter = wqe + sizeof (struct mthca_next_seg); wqe 196 drivers/infiniband/hw/mthca/mthca_srq.c (void *) scatter < wqe + (1 << srq->wqe_shift); wqe 495 drivers/infiniband/hw/mthca/mthca_srq.c void *wqe; wqe 504 drivers/infiniband/hw/mthca/mthca_srq.c wqe = get_wqe(srq, ind); wqe 505 drivers/infiniband/hw/mthca/mthca_srq.c next_ind = *wqe_to_link(wqe); wqe 515 drivers/infiniband/hw/mthca/mthca_srq.c srq->last = wqe; wqe 517 drivers/infiniband/hw/mthca/mthca_srq.c ((struct mthca_next_seg *) wqe)->ee_nds = 0; wqe 520 drivers/infiniband/hw/mthca/mthca_srq.c wqe += sizeof (struct mthca_next_seg); wqe 530 drivers/infiniband/hw/mthca/mthca_srq.c mthca_set_data_seg(wqe, wr->sg_list + i); wqe 531 drivers/infiniband/hw/mthca/mthca_srq.c wqe += sizeof (struct mthca_data_seg); wqe 535 drivers/infiniband/hw/mthca/mthca_srq.c mthca_set_data_seg_inval(wqe); wqe 588 drivers/infiniband/hw/mthca/mthca_srq.c void *wqe; wqe 594 drivers/infiniband/hw/mthca/mthca_srq.c wqe = get_wqe(srq, ind); wqe 595 drivers/infiniband/hw/mthca/mthca_srq.c next_ind = *wqe_to_link(wqe); wqe 604 drivers/infiniband/hw/mthca/mthca_srq.c ((struct mthca_next_seg *) wqe)->ee_nds = 0; wqe 607 drivers/infiniband/hw/mthca/mthca_srq.c wqe += sizeof (struct mthca_next_seg); wqe 616 drivers/infiniband/hw/mthca/mthca_srq.c mthca_set_data_seg(wqe, wr->sg_list + i); wqe 617 drivers/infiniband/hw/mthca/mthca_srq.c wqe += sizeof (struct mthca_data_seg); wqe 621 drivers/infiniband/hw/mthca/mthca_srq.c mthca_set_data_seg_inval(wqe); wqe 2933 drivers/infiniband/hw/qedr/verbs.c char *seg_prt, *wqe; wqe 2948 drivers/infiniband/hw/qedr/verbs.c wqe = NULL; wqe 2961 drivers/infiniband/hw/qedr/verbs.c wqe = (char *)qed_chain_produce(&qp->sq.pbl); wqe 2962 drivers/infiniband/hw/qedr/verbs.c seg_prt = wqe; wqe 2981 drivers/infiniband/hw/qedr/verbs.c swap_wqe_data64((u64 *)wqe); wqe 2987 drivers/infiniband/hw/qedr/verbs.c swap_wqe_data64((u64 *)wqe); wqe 3191 drivers/infiniband/hw/qedr/verbs.c struct rdma_sq_common_wqe *wqe; wqe 3201 drivers/infiniband/hw/qedr/verbs.c wqe = qed_chain_produce(&qp->sq.pbl); wqe 3205 drivers/infiniband/hw/qedr/verbs.c wqe->flags = 0; wqe 3206 drivers/infiniband/hw/qedr/verbs.c SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, wqe 3209 drivers/infiniband/hw/qedr/verbs.c SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); wqe 3210 drivers/infiniband/hw/qedr/verbs.c SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, wqe 3212 drivers/infiniband/hw/qedr/verbs.c wqe->prev_wqe_size = qp->prev_wqe_size; wqe 3223 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; wqe 3224 drivers/infiniband/hw/qedr/verbs.c swqe = (struct rdma_sq_send_wqe_1st *)wqe; wqe 3237 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; wqe 3238 drivers/infiniband/hw/qedr/verbs.c swqe = (struct rdma_sq_send_wqe_1st *)wqe; wqe 3250 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; wqe 3251 drivers/infiniband/hw/qedr/verbs.c swqe = (struct rdma_sq_send_wqe_1st *)wqe; wqe 3269 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; wqe 3270 drivers/infiniband/hw/qedr/verbs.c rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; wqe 3283 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; wqe 3284 drivers/infiniband/hw/qedr/verbs.c rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; wqe 3296 drivers/infiniband/hw/qedr/verbs.c SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); wqe 3300 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; wqe 3301 drivers/infiniband/hw/qedr/verbs.c rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; wqe 3315 drivers/infiniband/hw/qedr/verbs.c awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; wqe 3325 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; wqe 3329 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; wqe 3343 drivers/infiniband/hw/qedr/verbs.c iwqe = (struct rdma_sq_local_inv_wqe *)wqe; wqe 3353 drivers/infiniband/hw/qedr/verbs.c wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; wqe 3354 drivers/infiniband/hw/qedr/verbs.c fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; wqe 3381 drivers/infiniband/hw/qedr/verbs.c qed_chain_set_prod(&qp->sq.pbl, value, wqe); wqe 3384 drivers/infiniband/hw/qedr/verbs.c qp->prev_wqe_size = wqe->prev_wqe_size; wqe 250 drivers/infiniband/hw/qib/qib.h struct rvt_swqe *wqe; wqe 386 drivers/infiniband/hw/qib/qib_qp.c struct rvt_swqe *wqe, bool *call_send) wqe 393 drivers/infiniband/hw/qib/qib_qp.c if (wqe->length > 0x80000000U) wqe 395 drivers/infiniband/hw/qib/qib_qp.c if (wqe->length > qp->pmtu) wqe 401 drivers/infiniband/hw/qib/qib_qp.c ah = rvt_get_swqe_ah(wqe); wqe 402 drivers/infiniband/hw/qib/qib_qp.c if (wqe->length > (1 << ah->log_pmtu)) wqe 426 drivers/infiniband/hw/qib/qib_qp.c struct rvt_swqe *wqe; wqe 430 drivers/infiniband/hw/qib/qib_qp.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 437 drivers/infiniband/hw/qib/qib_qp.c wqe->wr.opcode, wqe 443 drivers/infiniband/hw/qib/qib_qp.c wqe->ssn, wqe 42 drivers/infiniband/hw/qib/qib_rc.c static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, wqe 47 drivers/infiniband/hw/qib/qib_rc.c len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; wqe 48 drivers/infiniband/hw/qib/qib_rc.c return rvt_restart_sge(ss, wqe, len); wqe 221 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe; wqe 251 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 252 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? wqe 275 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 296 drivers/infiniband/hw/qib/qib_rc.c if ((wqe->wr.send_flags & IB_SEND_FENCE) && wqe 302 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->psn; wqe 309 drivers/infiniband/hw/qib/qib_rc.c len = wqe->length; wqe 312 drivers/infiniband/hw/qib/qib_rc.c switch (wqe->wr.opcode) { wqe 316 drivers/infiniband/hw/qib/qib_rc.c if (!rvt_rc_credit_avail(qp, wqe)) wqe 323 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_SEND) wqe 328 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 331 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 344 drivers/infiniband/hw/qib/qib_rc.c if (!rvt_rc_credit_avail(qp, wqe)) wqe 348 drivers/infiniband/hw/qib/qib_rc.c cpu_to_be64(wqe->rdma_wr.remote_addr); wqe 350 drivers/infiniband/hw/qib/qib_rc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 358 drivers/infiniband/hw/qib/qib_rc.c if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE) wqe 364 drivers/infiniband/hw/qib/qib_rc.c wqe->rdma_wr.wr.ex.imm_data; wqe 366 drivers/infiniband/hw/qib/qib_rc.c if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED) wqe 391 drivers/infiniband/hw/qib/qib_rc.c cpu_to_be64(wqe->rdma_wr.remote_addr); wqe 393 drivers/infiniband/hw/qib/qib_rc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 420 drivers/infiniband/hw/qib/qib_rc.c if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { wqe 422 drivers/infiniband/hw/qib/qib_rc.c put_ib_ateth_swap(wqe->atomic_wr.swap, wqe 424 drivers/infiniband/hw/qib/qib_rc.c put_ib_ateth_compare(wqe->atomic_wr.compare_add, wqe 428 drivers/infiniband/hw/qib/qib_rc.c put_ib_ateth_swap(wqe->atomic_wr.compare_add, wqe 432 drivers/infiniband/hw/qib/qib_rc.c put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, wqe 435 drivers/infiniband/hw/qib/qib_rc.c wqe->atomic_wr.rkey); wqe 447 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.sge = wqe->sg_list[0]; wqe 448 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.sg_list = wqe->sg_list + 1; wqe 449 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.num_sge = wqe->wr.num_sge; wqe 450 drivers/infiniband/hw/qib/qib_rc.c qp->s_sge.total_len = wqe->length; wqe 451 drivers/infiniband/hw/qib/qib_rc.c qp->s_len = wqe->length; wqe 457 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ) wqe 458 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->lpsn + 1; wqe 473 drivers/infiniband/hw/qib/qib_rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); wqe 486 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_SEND) wqe 491 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 494 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 512 drivers/infiniband/hw/qib/qib_rc.c qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); wqe 525 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE) wqe 530 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 532 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 551 drivers/infiniband/hw/qib/qib_rc.c len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; wqe 553 drivers/infiniband/hw/qib/qib_rc.c cpu_to_be64(wqe->rdma_wr.remote_addr + len); wqe 555 drivers/infiniband/hw/qib/qib_rc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 556 drivers/infiniband/hw/qib/qib_rc.c ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); wqe 560 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->lpsn + 1; wqe 569 drivers/infiniband/hw/qib/qib_rc.c delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8; wqe 738 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); wqe 747 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(psn, wqe->psn) <= 0) { wqe 753 drivers/infiniband/hw/qib/qib_rc.c opcode = wqe->wr.opcode; wqe 761 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, n); wqe 762 drivers/infiniband/hw/qib/qib_rc.c diff = qib_cmp24(psn, wqe->psn); wqe 774 drivers/infiniband/hw/qib/qib_rc.c opcode = wqe->wr.opcode; wqe 822 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 830 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); wqe 839 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ) wqe 858 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe; wqe 863 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, n); wqe 864 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(psn, wqe->lpsn) <= 0) { wqe 865 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ) wqe 866 drivers/infiniband/hw/qib/qib_rc.c qp->s_sending_psn = wqe->lpsn + 1; wqe 884 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe; wqe 918 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 919 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && wqe 923 drivers/infiniband/hw/qib/qib_rc.c wqe, wqe 924 drivers/infiniband/hw/qib/qib_rc.c ib_qib_wc_opcode[wqe->wr.opcode], wqe 951 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe, wqe 959 drivers/infiniband/hw/qib/qib_rc.c if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || wqe 962 drivers/infiniband/hw/qib/qib_rc.c wqe, wqe 963 drivers/infiniband/hw/qib/qib_rc.c ib_qib_wc_opcode[wqe->wr.opcode], wqe 969 drivers/infiniband/hw/qib/qib_rc.c update_last_psn(qp, wqe->lpsn); wqe 980 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 983 drivers/infiniband/hw/qib/qib_rc.c qp->s_psn = wqe->psn; wqe 990 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 992 drivers/infiniband/hw/qib/qib_rc.c return wqe; wqe 1011 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe; wqe 1025 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1032 drivers/infiniband/hw/qib/qib_rc.c while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) { wqe 1039 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ && wqe 1054 drivers/infiniband/hw/qib/qib_rc.c if ((wqe->wr.opcode == IB_WR_RDMA_READ && wqe 1056 drivers/infiniband/hw/qib/qib_rc.c ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 1057 drivers/infiniband/hw/qib/qib_rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && wqe 1076 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 1077 drivers/infiniband/hw/qib/qib_rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { wqe 1078 drivers/infiniband/hw/qib/qib_rc.c u64 *vaddr = wqe->sg_list[0].vaddr; wqe 1082 drivers/infiniband/hw/qib/qib_rc.c (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 1083 drivers/infiniband/hw/qib/qib_rc.c wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 1084 drivers/infiniband/hw/qib/qib_rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { wqe 1098 drivers/infiniband/hw/qib/qib_rc.c wqe = do_rc_completion(qp, wqe, ibp); wqe 1195 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, status); wqe 1226 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe; wqe 1231 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1233 drivers/infiniband/hw/qib/qib_rc.c while (qib_cmp24(psn, wqe->lpsn) > 0) { wqe 1234 drivers/infiniband/hw/qib/qib_rc.c if (wqe->wr.opcode == IB_WR_RDMA_READ || wqe 1235 drivers/infiniband/hw/qib/qib_rc.c wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || wqe 1236 drivers/infiniband/hw/qib/qib_rc.c wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) wqe 1238 drivers/infiniband/hw/qib/qib_rc.c wqe = do_rc_completion(qp, wqe, ibp); wqe 1275 drivers/infiniband/hw/qib/qib_rc.c struct rvt_swqe *wqe; wqe 1339 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1355 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1356 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) wqe 1364 drivers/infiniband/hw/qib/qib_rc.c wqe, psn, pmtu); wqe 1371 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) wqe 1421 drivers/infiniband/hw/qib/qib_rc.c wqe = rvt_get_swqe_ptr(qp, qp->s_acked); wqe 1423 drivers/infiniband/hw/qib/qib_rc.c wqe, psn, pmtu); wqe 1430 drivers/infiniband/hw/qib/qib_rc.c if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) wqe 1466 drivers/infiniband/hw/qib/qib_rc.c rvt_send_complete(qp, wqe, status); wqe 52 drivers/infiniband/hw/qib/qib_uc.c struct rvt_swqe *wqe; wqe 70 drivers/infiniband/hw/qib/qib_uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 71 drivers/infiniband/hw/qib/qib_uc.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); wqe 84 drivers/infiniband/hw/qib/qib_uc.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 97 drivers/infiniband/hw/qib/qib_uc.c qp->s_psn = wqe->psn; wqe 98 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.sge = wqe->sg_list[0]; wqe 99 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.sg_list = wqe->sg_list + 1; wqe 100 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.num_sge = wqe->wr.num_sge; wqe 101 drivers/infiniband/hw/qib/qib_uc.c qp->s_sge.total_len = wqe->length; wqe 102 drivers/infiniband/hw/qib/qib_uc.c len = wqe->length; wqe 104 drivers/infiniband/hw/qib/qib_uc.c switch (wqe->wr.opcode) { wqe 112 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.opcode == IB_WR_SEND) wqe 118 drivers/infiniband/hw/qib/qib_uc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 121 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 123 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; wqe 131 drivers/infiniband/hw/qib/qib_uc.c cpu_to_be64(wqe->rdma_wr.remote_addr); wqe 133 drivers/infiniband/hw/qib/qib_uc.c cpu_to_be32(wqe->rdma_wr.rkey); wqe 141 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE) wqe 147 drivers/infiniband/hw/qib/qib_uc.c ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; wqe 149 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 152 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; wqe 171 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.opcode == IB_WR_SEND) wqe 176 drivers/infiniband/hw/qib/qib_uc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 179 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 181 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; wqe 195 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE) wqe 201 drivers/infiniband/hw/qib/qib_uc.c ohdr->u.imm_data = wqe->wr.ex.imm_data; wqe 203 drivers/infiniband/hw/qib/qib_uc.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 206 drivers/infiniband/hw/qib/qib_uc.c qp->s_wqe = wqe; wqe 237 drivers/infiniband/hw/qib/qib_ud.c struct rvt_swqe *wqe; wqe 257 drivers/infiniband/hw/qib/qib_ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 258 drivers/infiniband/hw/qib/qib_ud.c rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); wqe 266 drivers/infiniband/hw/qib/qib_ud.c wqe = rvt_get_swqe_ptr(qp, qp->s_cur); wqe 274 drivers/infiniband/hw/qib/qib_ud.c ah_attr = rvt_get_swqe_ah_attr(wqe); wqe 299 drivers/infiniband/hw/qib/qib_ud.c qib_ud_loopback(qp, wqe); wqe 302 drivers/infiniband/hw/qib/qib_ud.c rvt_send_complete(qp, wqe, IB_WC_SUCCESS); wqe 308 drivers/infiniband/hw/qib/qib_ud.c extra_bytes = -wqe->length & 3; wqe 309 drivers/infiniband/hw/qib/qib_ud.c nwords = (wqe->length + extra_bytes) >> 2; wqe 313 drivers/infiniband/hw/qib/qib_ud.c qp->s_cur_size = wqe->length; wqe 316 drivers/infiniband/hw/qib/qib_ud.c qp->s_wqe = wqe; wqe 317 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.sge = wqe->sg_list[0]; wqe 318 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.sg_list = wqe->sg_list + 1; wqe 319 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.num_sge = wqe->wr.num_sge; wqe 320 drivers/infiniband/hw/qib/qib_ud.c qp->s_sge.total_len = wqe->length; wqe 338 drivers/infiniband/hw/qib/qib_ud.c if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { wqe 340 drivers/infiniband/hw/qib/qib_ud.c ohdr->u.ud.imm_data = wqe->wr.ex.imm_data; wqe 361 drivers/infiniband/hw/qib/qib_ud.c if (wqe->wr.send_flags & IB_SEND_SOLICITED) wqe 366 drivers/infiniband/hw/qib/qib_ud.c rvt_get_swqe_pkey_index(wqe) : qp->s_pkey_index); wqe 375 drivers/infiniband/hw/qib/qib_ud.c cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); wqe 376 drivers/infiniband/hw/qib/qib_ud.c ohdr->bth[2] = cpu_to_be32(wqe->psn & QIB_PSN_MASK); wqe 382 drivers/infiniband/hw/qib/qib_ud.c cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : wqe 383 drivers/infiniband/hw/qib/qib_ud.c rvt_get_swqe_remote_qkey(wqe)); wqe 720 drivers/infiniband/hw/qib/qib_verbs.c if (tx->wqe) wqe 721 drivers/infiniband/hw/qib/qib_verbs.c rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS); wqe 801 drivers/infiniband/hw/qib/qib_verbs.c tx->wqe = qp->s_wqe; wqe 303 drivers/infiniband/hw/qib/qib_verbs.h int qib_check_send_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 630 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last); wqe 632 drivers/infiniband/sw/rdmavt/qp.c rvt_put_qp_swqe(qp, wqe); wqe 660 drivers/infiniband/sw/rdmavt/qp.c static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey) wqe 664 drivers/infiniband/sw/rdmavt/qp.c for (i = 0; i < wqe->wr.num_sge; i++) { wqe 665 drivers/infiniband/sw/rdmavt/qp.c struct rvt_sge *sge = &wqe->sg_list[i]; wqe 683 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last); wqe 685 drivers/infiniband/sw/rdmavt/qp.c if (rvt_swqe_has_lkey(wqe, lkey)) wqe 1014 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe; wqe 1018 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_swqe_ptr(qp, i); wqe 1019 drivers/infiniband/sw/rdmavt/qp.c kfree(wqe->ud_wr.attr); wqe 1020 drivers/infiniband/sw/rdmavt/qp.c wqe->ud_wr.attr = NULL; wqe 1034 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe; wqe 1038 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_swqe_ptr(qp, i); wqe 1039 drivers/infiniband/sw/rdmavt/qp.c wqe->ud_wr.attr = kzalloc_node(sizeof(*wqe->ud_wr.attr), wqe 1041 drivers/infiniband/sw/rdmavt/qp.c if (!wqe->ud_wr.attr) { wqe 1837 drivers/infiniband/sw/rdmavt/qp.c struct rvt_rwqe *wqe; wqe 1865 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_rwqe_ptr(&qp->r_rq, wq->head); wqe 1866 drivers/infiniband/sw/rdmavt/qp.c wqe->wr_id = wr->wr_id; wqe 1867 drivers/infiniband/sw/rdmavt/qp.c wqe->num_sge = wr->num_sge; wqe 1869 drivers/infiniband/sw/rdmavt/qp.c wqe->sg_list[i].addr = wr->sg_list[i].addr; wqe 1870 drivers/infiniband/sw/rdmavt/qp.c wqe->sg_list[i].length = wr->sg_list[i].length; wqe 1871 drivers/infiniband/sw/rdmavt/qp.c wqe->sg_list[i].lkey = wr->sg_list[i].lkey; wqe 2003 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe; wqe 2077 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_swqe_ptr(qp, qp->s_head); wqe 2080 drivers/infiniband/sw/rdmavt/qp.c memcpy(&wqe->wr, wr, cplen); wqe 2082 drivers/infiniband/sw/rdmavt/qp.c wqe->length = 0; wqe 2094 drivers/infiniband/sw/rdmavt/qp.c ret = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], last_sge, wqe 2098 drivers/infiniband/sw/rdmavt/qp.c wqe->length += length; wqe 2100 drivers/infiniband/sw/rdmavt/qp.c last_sge = &wqe->sg_list[j]; wqe 2103 drivers/infiniband/sw/rdmavt/qp.c wqe->wr.num_sge = j; wqe 2113 drivers/infiniband/sw/rdmavt/qp.c struct rvt_ah *ah = rvt_get_swqe_ah(wqe); wqe 2116 drivers/infiniband/sw/rdmavt/qp.c rdma_copy_ah_attr(wqe->ud_wr.attr, &ah->attr); wqe 2123 drivers/infiniband/sw/rdmavt/qp.c wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY; wqe 2124 drivers/infiniband/sw/rdmavt/qp.c wqe->ssn = 0; wqe 2125 drivers/infiniband/sw/rdmavt/qp.c wqe->psn = 0; wqe 2126 drivers/infiniband/sw/rdmavt/qp.c wqe->lpsn = 0; wqe 2128 drivers/infiniband/sw/rdmavt/qp.c wqe->ssn = qp->s_ssn++; wqe 2129 drivers/infiniband/sw/rdmavt/qp.c wqe->psn = qp->s_next_psn; wqe 2130 drivers/infiniband/sw/rdmavt/qp.c wqe->lpsn = wqe->psn + wqe 2131 drivers/infiniband/sw/rdmavt/qp.c (wqe->length ? wqe 2132 drivers/infiniband/sw/rdmavt/qp.c ((wqe->length - 1) >> log_pmtu) : wqe 2138 drivers/infiniband/sw/rdmavt/qp.c ret = rdi->driver_f.setup_wqe(qp, wqe, call_send); wqe 2144 drivers/infiniband/sw/rdmavt/qp.c qp->s_next_psn = wqe->lpsn + 1; wqe 2147 drivers/infiniband/sw/rdmavt/qp.c wqe->wr.send_flags |= RVT_SEND_RESERVE_USED; wqe 2148 drivers/infiniband/sw/rdmavt/qp.c rvt_qp_wqe_reserve(qp, wqe); wqe 2150 drivers/infiniband/sw/rdmavt/qp.c wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED; wqe 2153 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_post_one_wr(qp, wqe, wr->num_sge); wqe 2161 drivers/infiniband/sw/rdmavt/qp.c rdma_destroy_ah_attr(wqe->ud_wr.attr); wqe 2165 drivers/infiniband/sw/rdmavt/qp.c struct rvt_sge *sge = &wqe->sg_list[--j]; wqe 2251 drivers/infiniband/sw/rdmavt/qp.c struct rvt_rwqe *wqe; wqe 2271 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_rwqe_ptr(&srq->rq, wq->head); wqe 2272 drivers/infiniband/sw/rdmavt/qp.c wqe->wr_id = wr->wr_id; wqe 2273 drivers/infiniband/sw/rdmavt/qp.c wqe->num_sge = wr->num_sge; wqe 2275 drivers/infiniband/sw/rdmavt/qp.c wqe->sg_list[i].addr = wr->sg_list[i].addr; wqe 2276 drivers/infiniband/sw/rdmavt/qp.c wqe->sg_list[i].length = wr->sg_list[i].length; wqe 2277 drivers/infiniband/sw/rdmavt/qp.c wqe->sg_list[i].lkey = wr->sg_list[i].lkey; wqe 2306 drivers/infiniband/sw/rdmavt/qp.c static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe) wqe 2320 drivers/infiniband/sw/rdmavt/qp.c for (i = j = 0; i < wqe->num_sge; i++) { wqe 2321 drivers/infiniband/sw/rdmavt/qp.c if (wqe->sg_list[i].length == 0) wqe 2325 drivers/infiniband/sw/rdmavt/qp.c NULL, rvt_cast_sge(&wqe->sg_list[i]), wqe 2329 drivers/infiniband/sw/rdmavt/qp.c qp->r_len += wqe->sg_list[i].length; wqe 2344 drivers/infiniband/sw/rdmavt/qp.c wc.wr_id = wqe->wr_id; wqe 2414 drivers/infiniband/sw/rdmavt/qp.c struct rvt_rwqe *wqe; wqe 2460 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_rwqe_ptr(rq, tail); wqe 2472 drivers/infiniband/sw/rdmavt/qp.c if (!wr_id_only && !init_sge(qp, wqe)) { wqe 2476 drivers/infiniband/sw/rdmavt/qp.c qp->r_wr_id = wqe->wr_id; wqe 2840 drivers/infiniband/sw/rdmavt/qp.c void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 2851 drivers/infiniband/sw/rdmavt/qp.c trace_rvt_qp_send_completion(qp, wqe, old_last); wqe 2852 drivers/infiniband/sw/rdmavt/qp.c last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode], wqe 2969 drivers/infiniband/sw/rdmavt/qp.c struct rvt_swqe *wqe; wqe 3004 drivers/infiniband/sw/rdmavt/qp.c wqe = rvt_get_swqe_ptr(sqp, sqp->s_last); wqe 3041 drivers/infiniband/sw/rdmavt/qp.c sqp->s_sge.sge = wqe->sg_list[0]; wqe 3042 drivers/infiniband/sw/rdmavt/qp.c sqp->s_sge.sg_list = wqe->sg_list + 1; wqe 3043 drivers/infiniband/sw/rdmavt/qp.c sqp->s_sge.num_sge = wqe->wr.num_sge; wqe 3044 drivers/infiniband/sw/rdmavt/qp.c sqp->s_len = wqe->length; wqe 3045 drivers/infiniband/sw/rdmavt/qp.c switch (wqe->wr.opcode) { wqe 3050 drivers/infiniband/sw/rdmavt/qp.c if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) { wqe 3052 drivers/infiniband/sw/rdmavt/qp.c wqe->wr.ex.invalidate_rkey)) wqe 3066 drivers/infiniband/sw/rdmavt/qp.c if (wqe->length > qp->r_len) wqe 3068 drivers/infiniband/sw/rdmavt/qp.c switch (wqe->wr.opcode) { wqe 3071 drivers/infiniband/sw/rdmavt/qp.c wqe->wr.ex.invalidate_rkey)) { wqe 3074 drivers/infiniband/sw/rdmavt/qp.c wqe->wr.ex.invalidate_rkey; wqe 3079 drivers/infiniband/sw/rdmavt/qp.c wc.ex.imm_data = wqe->wr.ex.imm_data; wqe 3090 drivers/infiniband/sw/rdmavt/qp.c wc.ex.imm_data = wqe->wr.ex.imm_data; wqe 3103 drivers/infiniband/sw/rdmavt/qp.c if (wqe->length == 0) wqe 3105 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length, wqe 3106 drivers/infiniband/sw/rdmavt/qp.c wqe->rdma_wr.remote_addr, wqe 3107 drivers/infiniband/sw/rdmavt/qp.c wqe->rdma_wr.rkey, wqe 3112 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.total_len = wqe->length; wqe 3118 drivers/infiniband/sw/rdmavt/qp.c if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length, wqe 3119 drivers/infiniband/sw/rdmavt/qp.c wqe->rdma_wr.remote_addr, wqe 3120 drivers/infiniband/sw/rdmavt/qp.c wqe->rdma_wr.rkey, wqe 3126 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.sge = wqe->sg_list[0]; wqe 3127 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.sg_list = wqe->sg_list + 1; wqe 3128 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.num_sge = wqe->wr.num_sge; wqe 3129 drivers/infiniband/sw/rdmavt/qp.c qp->r_sge.total_len = wqe->length; wqe 3137 drivers/infiniband/sw/rdmavt/qp.c wqe->atomic_wr.remote_addr, wqe 3138 drivers/infiniband/sw/rdmavt/qp.c wqe->atomic_wr.rkey, wqe 3143 drivers/infiniband/sw/rdmavt/qp.c sdata = wqe->atomic_wr.compare_add; wqe 3145 drivers/infiniband/sw/rdmavt/qp.c (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ? wqe 3148 drivers/infiniband/sw/rdmavt/qp.c sdata, wqe->atomic_wr.swap); wqe 3174 drivers/infiniband/sw/rdmavt/qp.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) wqe 3180 drivers/infiniband/sw/rdmavt/qp.c wc.byte_len = wqe->length; wqe 3187 drivers/infiniband/sw/rdmavt/qp.c rvt_recv_cq(qp, &wc, wqe->wr.send_flags & IB_SEND_SOLICITED); wqe 3195 drivers/infiniband/sw/rdmavt/qp.c rvt_send_complete(sqp, wqe, send_status); wqe 3249 drivers/infiniband/sw/rdmavt/qp.c rvt_send_complete(sqp, wqe, send_status); wqe 199 drivers/infiniband/sw/rdmavt/rc.c u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len) wqe 201 drivers/infiniband/sw/rdmavt/rc.c ss->sge = wqe->sg_list[0]; wqe 202 drivers/infiniband/sw/rdmavt/rc.c ss->sg_list = wqe->sg_list + 1; wqe 203 drivers/infiniband/sw/rdmavt/rc.c ss->num_sge = wqe->wr.num_sge; wqe 204 drivers/infiniband/sw/rdmavt/rc.c ss->total_len = wqe->length; wqe 206 drivers/infiniband/sw/rdmavt/rc.c return wqe->length - len; wqe 239 drivers/infiniband/sw/rdmavt/srq.c struct rvt_rwqe *wqe; wqe 242 drivers/infiniband/sw/rdmavt/srq.c wqe = rvt_get_rwqe_ptr(&srq->rq, tail); wqe 243 drivers/infiniband/sw/rdmavt/srq.c p->wr_id = wqe->wr_id; wqe 244 drivers/infiniband/sw/rdmavt/srq.c p->num_sge = wqe->num_sge; wqe 245 drivers/infiniband/sw/rdmavt/srq.c for (i = 0; i < wqe->num_sge; i++) wqe 246 drivers/infiniband/sw/rdmavt/srq.c p->sg_list[i] = wqe->sg_list[i]; wqe 91 drivers/infiniband/sw/rdmavt/trace_tx.h TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, int wr_num_sge), wqe 92 drivers/infiniband/sw/rdmavt/trace_tx.h TP_ARGS(qp, wqe, wr_num_sge), wqe 96 drivers/infiniband/sw/rdmavt/trace_tx.h __field(struct rvt_swqe *, wqe) wqe 115 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->wqe = wqe; wqe 116 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->wr_id = wqe->wr.wr_id; wqe 119 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->psn = wqe->psn; wqe 120 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->lpsn = wqe->lpsn; wqe 121 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->length = wqe->length; wqe 122 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->opcode = wqe->wr.opcode; wqe 128 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->ssn = wqe->ssn; wqe 129 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->send_flags = wqe->wr.send_flags; wqe 130 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->num_sge = wqe->wr.num_sge; wqe 136 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->wqe, wqe 158 drivers/infiniband/sw/rdmavt/trace_tx.h TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, u32 idx), wqe 159 drivers/infiniband/sw/rdmavt/trace_tx.h TP_ARGS(qp, wqe, idx), wqe 162 drivers/infiniband/sw/rdmavt/trace_tx.h __field(struct rvt_swqe *, wqe) wqe 174 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->wqe = wqe; wqe 175 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->wr_id = wqe->wr.wr_id; wqe 178 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->length = wqe->length; wqe 180 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->ssn = wqe->ssn; wqe 181 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->opcode = wqe->wr.opcode; wqe 182 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->send_flags = wqe->wr.send_flags; wqe 189 drivers/infiniband/sw/rdmavt/trace_tx.h __entry->wqe, wqe 110 drivers/infiniband/sw/rxe/rxe_av.c return (pkt->wqe) ? &pkt->wqe->av : NULL; wqe 166 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe; wqe 171 drivers/infiniband/sw/rxe/rxe_comp.c wqe = queue_head(qp->sq.queue); wqe 172 drivers/infiniband/sw/rxe/rxe_comp.c *wqe_p = wqe; wqe 175 drivers/infiniband/sw/rxe/rxe_comp.c if (!wqe || wqe->state == wqe_state_posted) wqe 179 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->state == wqe_state_done) wqe 183 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->state == wqe_state_error) wqe 199 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe) wqe 206 drivers/infiniband/sw/rxe/rxe_comp.c diff = psn_compare(pkt->psn, wqe->last_psn); wqe 208 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->state == wqe_state_pending) { wqe 209 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->mask & WR_ATOMIC_OR_READ_MASK) wqe 225 drivers/infiniband/sw/rxe/rxe_comp.c if (pkt->psn == wqe->last_psn) wqe 229 drivers/infiniband/sw/rxe/rxe_comp.c } else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) { wqe 238 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe) wqe 260 drivers/infiniband/sw/rxe/rxe_comp.c if ((pkt->psn == wqe->first_psn && wqe 263 drivers/infiniband/sw/rxe/rxe_comp.c (wqe->first_psn == wqe->last_psn && wqe 289 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->wr.opcode != IB_WR_RDMA_READ && wqe 290 drivers/infiniband/sw/rxe/rxe_comp.c wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) { wqe 291 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_FATAL_ERR; wqe 303 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->wr.opcode != IB_WR_ATOMIC_CMP_AND_SWP && wqe 304 drivers/infiniband/sw/rxe/rxe_comp.c wqe->wr.opcode != IB_WR_ATOMIC_FETCH_AND_ADD) wqe 338 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_REM_INV_REQ_ERR; wqe 342 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_REM_ACCESS_ERR; wqe 346 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_REM_OP_ERR; wqe 351 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_REM_OP_ERR; wqe 369 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe) wqe 374 drivers/infiniband/sw/rxe/rxe_comp.c &wqe->dma, payload_addr(pkt), wqe 379 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->dma.resid == 0 && (pkt->mask & RXE_END_MASK)) wqe 387 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe) wqe 394 drivers/infiniband/sw/rxe/rxe_comp.c &wqe->dma, &atomic_orig, wqe 402 drivers/infiniband/sw/rxe/rxe_comp.c static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe, wqe 410 drivers/infiniband/sw/rxe/rxe_comp.c wc->wr_id = wqe->wr.wr_id; wqe 411 drivers/infiniband/sw/rxe/rxe_comp.c wc->status = wqe->status; wqe 412 drivers/infiniband/sw/rxe/rxe_comp.c wc->opcode = wr_to_wc_opcode(wqe->wr.opcode); wqe 413 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM || wqe 414 drivers/infiniband/sw/rxe/rxe_comp.c wqe->wr.opcode == IB_WR_SEND_WITH_IMM) wqe 416 drivers/infiniband/sw/rxe/rxe_comp.c wc->byte_len = wqe->dma.length; wqe 421 drivers/infiniband/sw/rxe/rxe_comp.c uwc->wr_id = wqe->wr.wr_id; wqe 422 drivers/infiniband/sw/rxe/rxe_comp.c uwc->status = wqe->status; wqe 423 drivers/infiniband/sw/rxe/rxe_comp.c uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode); wqe 424 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM || wqe 425 drivers/infiniband/sw/rxe/rxe_comp.c wqe->wr.opcode == IB_WR_SEND_WITH_IMM) wqe 427 drivers/infiniband/sw/rxe/rxe_comp.c uwc->byte_len = wqe->dma.length; wqe 440 drivers/infiniband/sw/rxe/rxe_comp.c static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) wqe 446 drivers/infiniband/sw/rxe/rxe_comp.c (wqe->wr.send_flags & IB_SEND_SIGNALED) || wqe 447 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status != IB_WC_SUCCESS) { wqe 448 drivers/infiniband/sw/rxe/rxe_comp.c make_send_cqe(qp, wqe, &cqe); wqe 455 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->wr.opcode == IB_WR_SEND || wqe 456 drivers/infiniband/sw/rxe/rxe_comp.c wqe->wr.opcode == IB_WR_SEND_WITH_IMM || wqe 457 drivers/infiniband/sw/rxe/rxe_comp.c wqe->wr.opcode == IB_WR_SEND_WITH_INV) wqe 472 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe) wqe 476 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->has_rd_atomic) { wqe 477 drivers/infiniband/sw/rxe/rxe_comp.c wqe->has_rd_atomic = 0; wqe 508 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); wqe 518 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe) wqe 520 drivers/infiniband/sw/rxe/rxe_comp.c if (pkt && wqe->state == wqe_state_pending) { wqe 521 drivers/infiniband/sw/rxe/rxe_comp.c if (psn_compare(wqe->last_psn, qp->comp.psn) >= 0) { wqe 522 drivers/infiniband/sw/rxe/rxe_comp.c qp->comp.psn = (wqe->last_psn + 1) & BTH_PSN_MASK; wqe 532 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); wqe 540 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe; wqe 547 drivers/infiniband/sw/rxe/rxe_comp.c while ((wqe = queue_head(qp->sq.queue))) { wqe 549 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_WR_FLUSH_ERR; wqe 550 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); wqe 561 drivers/infiniband/sw/rxe/rxe_comp.c struct rxe_send_wqe *wqe = NULL; wqe 601 drivers/infiniband/sw/rxe/rxe_comp.c state = get_wqe(qp, pkt, &wqe); wqe 605 drivers/infiniband/sw/rxe/rxe_comp.c state = check_psn(qp, pkt, wqe); wqe 609 drivers/infiniband/sw/rxe/rxe_comp.c state = check_ack(qp, pkt, wqe); wqe 613 drivers/infiniband/sw/rxe/rxe_comp.c state = do_read(qp, pkt, wqe); wqe 617 drivers/infiniband/sw/rxe/rxe_comp.c state = do_atomic(qp, pkt, wqe); wqe 621 drivers/infiniband/sw/rxe/rxe_comp.c if (wqe->state == wqe_state_pending && wqe 622 drivers/infiniband/sw/rxe/rxe_comp.c wqe->last_psn == pkt->psn) wqe 629 drivers/infiniband/sw/rxe/rxe_comp.c state = complete_ack(qp, pkt, wqe); wqe 633 drivers/infiniband/sw/rxe/rxe_comp.c state = complete_wqe(qp, pkt, wqe); wqe 662 drivers/infiniband/sw/rxe/rxe_comp.c if (qp->comp.timeout_retry && wqe) { wqe 693 drivers/infiniband/sw/rxe/rxe_comp.c if (!wqe || (wqe->state == wqe_state_posted)) { wqe 741 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_RETRY_EXC_ERR; wqe 764 drivers/infiniband/sw/rxe/rxe_comp.c wqe->status = IB_WC_RNR_RETRY_EXC_ERR; wqe 770 drivers/infiniband/sw/rxe/rxe_comp.c WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS); wqe 771 drivers/infiniband/sw/rxe/rxe_comp.c do_complete(qp, wqe); wqe 44 drivers/infiniband/sw/rxe/rxe_hdr.h struct rxe_send_wqe *wqe; /* send wqe */ wqe 279 drivers/infiniband/sw/rxe/rxe_loc.h pkt->wqe->state = wqe_state_done; wqe 41 drivers/infiniband/sw/rxe/rxe_req.c static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe, wqe 45 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe, wqe 51 drivers/infiniband/sw/rxe/rxe_req.c int to_send = (wqe->dma.resid > qp->mtu) ? wqe 52 drivers/infiniband/sw/rxe/rxe_req.c qp->mtu : wqe->dma.resid; wqe 54 drivers/infiniband/sw/rxe/rxe_req.c qp->req.opcode = next_opcode(qp, wqe, wqe 55 drivers/infiniband/sw/rxe/rxe_req.c wqe->wr.opcode); wqe 57 drivers/infiniband/sw/rxe/rxe_req.c if (wqe->wr.send_flags & IB_SEND_INLINE) { wqe 58 drivers/infiniband/sw/rxe/rxe_req.c wqe->dma.resid -= to_send; wqe 59 drivers/infiniband/sw/rxe/rxe_req.c wqe->dma.sge_offset += to_send; wqe 61 drivers/infiniband/sw/rxe/rxe_req.c advance_dma_data(&wqe->dma, to_send); wqe 64 drivers/infiniband/sw/rxe/rxe_req.c wqe->iova += qp->mtu; wqe 70 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe; wqe 83 drivers/infiniband/sw/rxe/rxe_req.c wqe = addr_from_index(qp->sq.queue, wqe_index); wqe 84 drivers/infiniband/sw/rxe/rxe_req.c mask = wr_opcode_mask(wqe->wr.opcode, qp); wqe 86 drivers/infiniband/sw/rxe/rxe_req.c if (wqe->state == wqe_state_posted) wqe 89 drivers/infiniband/sw/rxe/rxe_req.c if (wqe->state == wqe_state_done) wqe 92 drivers/infiniband/sw/rxe/rxe_req.c wqe->iova = (mask & WR_ATOMIC_MASK) ? wqe 93 drivers/infiniband/sw/rxe/rxe_req.c wqe->wr.wr.atomic.remote_addr : wqe 95 drivers/infiniband/sw/rxe/rxe_req.c wqe->wr.wr.rdma.remote_addr : wqe 99 drivers/infiniband/sw/rxe/rxe_req.c wqe->dma.resid = wqe->dma.length; wqe 100 drivers/infiniband/sw/rxe/rxe_req.c wqe->dma.cur_sge = 0; wqe 101 drivers/infiniband/sw/rxe/rxe_req.c wqe->dma.sge_offset = 0; wqe 108 drivers/infiniband/sw/rxe/rxe_req.c npsn = (qp->comp.psn - wqe->first_psn) & wqe 110 drivers/infiniband/sw/rxe/rxe_req.c retry_first_write_send(qp, wqe, mask, npsn); wqe 114 drivers/infiniband/sw/rxe/rxe_req.c npsn = (wqe->dma.length - wqe->dma.resid) / wqe 116 drivers/infiniband/sw/rxe/rxe_req.c wqe->iova += npsn * qp->mtu; wqe 120 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_posted; wqe 134 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe = queue_head(qp->sq.queue); wqe 150 drivers/infiniband/sw/rxe/rxe_req.c if (wqe && ((qp->req.wqe_index != wqe 152 drivers/infiniband/sw/rxe/rxe_req.c (wqe->state != wqe_state_posted))) { wqe 177 drivers/infiniband/sw/rxe/rxe_req.c wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index); wqe 181 drivers/infiniband/sw/rxe/rxe_req.c (wqe->state != wqe_state_processing))) wqe 184 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely((wqe->wr.send_flags & IB_SEND_FENCE) && wqe 190 drivers/infiniband/sw/rxe/rxe_req.c wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp); wqe 191 drivers/infiniband/sw/rxe/rxe_req.c return wqe; wqe 317 drivers/infiniband/sw/rxe/rxe_req.c static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe, wqe 320 drivers/infiniband/sw/rxe/rxe_req.c int fits = (wqe->dma.resid <= qp->mtu); wqe 348 drivers/infiniband/sw/rxe/rxe_req.c static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe) wqe 352 drivers/infiniband/sw/rxe/rxe_req.c if (wqe->has_rd_atomic) wqe 360 drivers/infiniband/sw/rxe/rxe_req.c wqe->has_rd_atomic = 1; wqe 379 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe, wqe 386 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wr *ibwr = &wqe->wr; wqe 407 drivers/infiniband/sw/rxe/rxe_req.c pkt->wqe = wqe; wqe 440 drivers/infiniband/sw/rxe/rxe_req.c reth_set_va(pkt, wqe->iova); wqe 441 drivers/infiniband/sw/rxe/rxe_req.c reth_set_len(pkt, wqe->dma.resid); wqe 451 drivers/infiniband/sw/rxe/rxe_req.c atmeth_set_va(pkt, wqe->iova); wqe 473 drivers/infiniband/sw/rxe/rxe_req.c static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe, wqe 487 drivers/infiniband/sw/rxe/rxe_req.c if (wqe->wr.send_flags & IB_SEND_INLINE) { wqe 488 drivers/infiniband/sw/rxe/rxe_req.c u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset]; wqe 493 drivers/infiniband/sw/rxe/rxe_req.c wqe->dma.resid -= paylen; wqe 494 drivers/infiniband/sw/rxe/rxe_req.c wqe->dma.sge_offset += paylen; wqe 496 drivers/infiniband/sw/rxe/rxe_req.c err = copy_data(qp->pd, 0, &wqe->dma, wqe 518 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe, wqe 523 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_pending; wqe 525 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_processing; wqe 530 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe, wqe 535 drivers/infiniband/sw/rxe/rxe_req.c int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu; wqe 542 drivers/infiniband/sw/rxe/rxe_req.c wqe->first_psn = qp->req.psn; wqe 543 drivers/infiniband/sw/rxe/rxe_req.c wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK; wqe 547 drivers/infiniband/sw/rxe/rxe_req.c qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK; wqe 552 drivers/infiniband/sw/rxe/rxe_req.c static void save_state(struct rxe_send_wqe *wqe, wqe 557 drivers/infiniband/sw/rxe/rxe_req.c rollback_wqe->state = wqe->state; wqe 558 drivers/infiniband/sw/rxe/rxe_req.c rollback_wqe->first_psn = wqe->first_psn; wqe 559 drivers/infiniband/sw/rxe/rxe_req.c rollback_wqe->last_psn = wqe->last_psn; wqe 563 drivers/infiniband/sw/rxe/rxe_req.c static void rollback_state(struct rxe_send_wqe *wqe, wqe 568 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = rollback_wqe->state; wqe 569 drivers/infiniband/sw/rxe/rxe_req.c wqe->first_psn = rollback_wqe->first_psn; wqe 570 drivers/infiniband/sw/rxe/rxe_req.c wqe->last_psn = rollback_wqe->last_psn; wqe 574 drivers/infiniband/sw/rxe/rxe_req.c static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe, wqe 594 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_send_wqe *wqe; wqe 623 drivers/infiniband/sw/rxe/rxe_req.c wqe = req_next_wqe(qp); wqe 624 drivers/infiniband/sw/rxe/rxe_req.c if (unlikely(!wqe)) wqe 627 drivers/infiniband/sw/rxe/rxe_req.c if (wqe->mask & WR_REG_MASK) { wqe 628 drivers/infiniband/sw/rxe/rxe_req.c if (wqe->wr.opcode == IB_WR_LOCAL_INV) { wqe 633 drivers/infiniband/sw/rxe/rxe_req.c wqe->wr.ex.invalidate_rkey >> 8); wqe 636 drivers/infiniband/sw/rxe/rxe_req.c wqe->wr.ex.invalidate_rkey); wqe 637 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_error; wqe 638 drivers/infiniband/sw/rxe/rxe_req.c wqe->status = IB_WC_MW_BIND_ERR; wqe 643 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_done; wqe 644 drivers/infiniband/sw/rxe/rxe_req.c wqe->status = IB_WC_SUCCESS; wqe 645 drivers/infiniband/sw/rxe/rxe_req.c } else if (wqe->wr.opcode == IB_WR_REG_MR) { wqe 646 drivers/infiniband/sw/rxe/rxe_req.c struct rxe_mem *rmr = to_rmr(wqe->wr.wr.reg.mr); wqe 649 drivers/infiniband/sw/rxe/rxe_req.c rmr->access = wqe->wr.wr.reg.access; wqe 650 drivers/infiniband/sw/rxe/rxe_req.c rmr->lkey = wqe->wr.wr.reg.key; wqe 651 drivers/infiniband/sw/rxe/rxe_req.c rmr->rkey = wqe->wr.wr.reg.key; wqe 652 drivers/infiniband/sw/rxe/rxe_req.c rmr->iova = wqe->wr.wr.reg.mr->iova; wqe 653 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_done; wqe 654 drivers/infiniband/sw/rxe/rxe_req.c wqe->status = IB_WC_SUCCESS; wqe 658 drivers/infiniband/sw/rxe/rxe_req.c if ((wqe->wr.send_flags & IB_SEND_SIGNALED) || wqe 679 drivers/infiniband/sw/rxe/rxe_req.c opcode = next_opcode(qp, wqe, wqe->wr.opcode); wqe 681 drivers/infiniband/sw/rxe/rxe_req.c wqe->status = IB_WC_LOC_QP_OP_ERR; wqe 687 drivers/infiniband/sw/rxe/rxe_req.c if (check_init_depth(qp, wqe)) wqe 692 drivers/infiniband/sw/rxe/rxe_req.c payload = (mask & RXE_WRITE_OR_SEND) ? wqe->dma.resid : 0; wqe 702 drivers/infiniband/sw/rxe/rxe_req.c wqe->first_psn = qp->req.psn; wqe 703 drivers/infiniband/sw/rxe/rxe_req.c wqe->last_psn = qp->req.psn; wqe 708 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_done; wqe 709 drivers/infiniband/sw/rxe/rxe_req.c wqe->status = IB_WC_SUCCESS; wqe 717 drivers/infiniband/sw/rxe/rxe_req.c skb = init_req_packet(qp, wqe, opcode, payload, &pkt); wqe 723 drivers/infiniband/sw/rxe/rxe_req.c if (fill_packet(qp, wqe, &pkt, skb, payload)) { wqe 735 drivers/infiniband/sw/rxe/rxe_req.c save_state(wqe, qp, &rollback_wqe, &rollback_psn); wqe 736 drivers/infiniband/sw/rxe/rxe_req.c update_wqe_state(qp, wqe, &pkt); wqe 737 drivers/infiniband/sw/rxe/rxe_req.c update_wqe_psn(qp, wqe, &pkt, payload); wqe 742 drivers/infiniband/sw/rxe/rxe_req.c rollback_state(wqe, qp, &rollback_wqe, rollback_psn); wqe 752 drivers/infiniband/sw/rxe/rxe_req.c update_state(qp, wqe, &pkt, payload); wqe 757 drivers/infiniband/sw/rxe/rxe_req.c wqe->status = IB_WC_LOC_PROT_ERR; wqe 758 drivers/infiniband/sw/rxe/rxe_req.c wqe->state = wqe_state_error; wqe 320 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_recv_wqe *wqe; wqe 328 drivers/infiniband/sw/rxe/rxe_resp.c wqe = queue_head(q); wqe 329 drivers/infiniband/sw/rxe/rxe_resp.c if (!wqe) { wqe 335 drivers/infiniband/sw/rxe/rxe_resp.c memcpy(&qp->resp.srq_wqe, wqe, sizeof(qp->resp.srq_wqe)); wqe 337 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = &qp->resp.srq_wqe.wqe; wqe 364 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { wqe 368 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = queue_head(qp->rq.queue); wqe 369 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { wqe 395 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = queue_head(qp->rq.queue); wqe 396 drivers/infiniband/sw/rxe/rxe_resp.c return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR; wqe 512 drivers/infiniband/sw/rxe/rxe_resp.c err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma, wqe 843 drivers/infiniband/sw/rxe/rxe_resp.c struct rxe_recv_wqe *wqe = qp->resp.wqe; wqe 846 drivers/infiniband/sw/rxe/rxe_resp.c if (unlikely(!wqe)) wqe 854 drivers/infiniband/sw/rxe/rxe_resp.c uwc->wr_id = wqe->wr_id; wqe 858 drivers/infiniband/sw/rxe/rxe_resp.c wc->wr_id = wqe->wr_id; wqe 869 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.length : wqe->dma.length - wqe->dma.resid; wqe 942 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = NULL; wqe 1176 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { wqe 1188 drivers/infiniband/sw/rxe/rxe_resp.c if (qp->resp.wqe) { wqe 1189 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe->dma.resid = qp->resp.wqe->dma.length; wqe 1190 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe->dma.cur_sge = 0; wqe 1191 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe->dma.sge_offset = 0; wqe 1391 drivers/infiniband/sw/rxe/rxe_resp.c qp->resp.wqe = NULL; wqe 588 drivers/infiniband/sw/rxe/rxe_verbs.c struct rxe_send_wqe *wqe) wqe 595 drivers/infiniband/sw/rxe/rxe_verbs.c init_send_wr(qp, &wqe->wr, ibwr); wqe 600 drivers/infiniband/sw/rxe/rxe_verbs.c memcpy(&wqe->av, &to_rah(ud_wr(ibwr)->ah)->av, sizeof(wqe->av)); wqe 603 drivers/infiniband/sw/rxe/rxe_verbs.c p = wqe->dma.inline_data; wqe 613 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->mask = mask; wqe 614 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->state = wqe_state_posted; wqe 617 drivers/infiniband/sw/rxe/rxe_verbs.c memcpy(wqe->dma.sge, ibwr->sg_list, wqe 620 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->iova = mask & WR_ATOMIC_MASK ? atomic_wr(ibwr)->remote_addr : wqe 622 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->mask = mask; wqe 623 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->dma.length = length; wqe 624 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->dma.resid = length; wqe 625 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->dma.num_sge = num_sge; wqe 626 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->dma.cur_sge = 0; wqe 627 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->dma.sge_offset = 0; wqe 628 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->state = wqe_state_posted; wqe 629 drivers/infiniband/sw/rxe/rxe_verbs.c wqe->ssn = atomic_add_return(1, &qp->ssn); wqe 209 drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_recv_wqe *wqe; wqe 221 drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_recv_wqe wqe; wqe 496 drivers/infiniband/sw/siw/siw.h #define tx_type(wqe) ((wqe)->sqe.opcode) wqe 497 drivers/infiniband/sw/siw/siw.h #define rx_type(wqe) ((wqe)->rqe.opcode) wqe 498 drivers/infiniband/sw/siw/siw.h #define tx_flags(wqe) ((wqe)->sqe.flags) wqe 262 drivers/infiniband/sw/siw/siw_mem.c void siw_wqe_put_mem(struct siw_wqe *wqe, enum siw_opcode op) wqe 271 drivers/infiniband/sw/siw/siw_mem.c if (!(wqe->sqe.flags & SIW_WQE_INLINE)) wqe 272 drivers/infiniband/sw/siw/siw_mem.c siw_unref_mem_sgl(wqe->mem, wqe->sqe.num_sge); wqe 276 drivers/infiniband/sw/siw/siw_mem.c siw_unref_mem_sgl(wqe->mem, wqe->rqe.num_sge); wqe 280 drivers/infiniband/sw/siw/siw_mem.c siw_unref_mem_sgl(wqe->mem, 1); wqe 21 drivers/infiniband/sw/siw/siw_mem.h void siw_wqe_put_mem(struct siw_wqe *wqe, enum siw_opcode op); wqe 262 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); wqe 268 drivers/infiniband/sw/siw/siw_qp.c if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { wqe 272 drivers/infiniband/sw/siw/siw_qp.c memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); wqe 274 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_QUEUED; wqe 275 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.flags = 0; wqe 276 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.num_sge = 1; wqe 277 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].length = 0; wqe 278 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].laddr = 0; wqe 279 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].lkey = 0; wqe 284 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.rkey = 1; wqe 285 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.raddr = 0; wqe 286 drivers/infiniband/sw/siw/siw_qp.c wqe->processed = 0; wqe 289 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.opcode = SIW_OP_WRITE; wqe 293 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.opcode = SIW_OP_READ; wqe 299 drivers/infiniband/sw/siw/siw_qp.c siw_read_to_orq(rreq, &wqe->sqe); wqe 309 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_IDLE; wqe 461 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); wqe 479 drivers/infiniband/sw/siw/siw_qp.c rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); wqe 481 drivers/infiniband/sw/siw/siw_qp.c rreq->ddp_mo = htonl(wqe->processed); wqe 482 drivers/infiniband/sw/siw/siw_qp.c rreq->sink_stag = htonl(wqe->sqe.rkey); wqe 483 drivers/infiniband/sw/siw/siw_qp.c rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); wqe 484 drivers/infiniband/sw/siw/siw_qp.c rreq->read_size = htonl(wqe->sqe.sge[0].length); wqe 485 drivers/infiniband/sw/siw/siw_qp.c rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); wqe 487 drivers/infiniband/sw/siw/siw_qp.c cpu_to_be64(wqe->sqe.sge[0].laddr); wqe 889 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); wqe 905 drivers/infiniband/sw/siw/siw_qp.c memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); wqe 906 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_QUEUED; wqe 909 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.opcode = SIW_OP_READ_RESPONSE; wqe 910 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.flags = 0; wqe 912 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.num_sge = 1; wqe 913 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].length = irqe->sge[0].length; wqe 914 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; wqe 915 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; wqe 917 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.num_sge = 0; wqe 923 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[1].length = irqe->sge[1].length; wqe 925 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.rkey = irqe->rkey; wqe 926 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.raddr = irqe->raddr; wqe 928 drivers/infiniband/sw/siw/siw_qp.c wqe->processed = 0; wqe 939 drivers/infiniband/sw/siw/siw_qp.c memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); wqe 940 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_QUEUED; wqe 943 drivers/infiniband/sw/siw/siw_qp.c memcpy(&wqe->sqe, sqe, sizeof(*sqe)); wqe 945 drivers/infiniband/sw/siw/siw_qp.c if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { wqe 949 drivers/infiniband/sw/siw/siw_qp.c if (wqe->sqe.flags & SIW_WQE_INLINE) { wqe 950 drivers/infiniband/sw/siw/siw_qp.c if (wqe->sqe.opcode != SIW_OP_SEND && wqe 951 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.opcode != SIW_OP_WRITE) { wqe 955 drivers/infiniband/sw/siw/siw_qp.c if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { wqe 959 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; wqe 960 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.sge[0].lkey = 0; wqe 961 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.num_sge = 1; wqe 963 drivers/infiniband/sw/siw/siw_qp.c if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { wqe 965 drivers/infiniband/sw/siw/siw_qp.c if (unlikely(wqe->sqe.opcode == SIW_OP_READ || wqe 966 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.opcode == wqe 980 drivers/infiniband/sw/siw/siw_qp.c } else if (wqe->sqe.opcode == SIW_OP_READ || wqe 981 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { wqe 984 drivers/infiniband/sw/siw/siw_qp.c wqe->sqe.num_sge = 1; wqe 994 drivers/infiniband/sw/siw/siw_qp.c siw_read_to_orq(rreq, &wqe->sqe); wqe 1012 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_IDLE; wqe 1178 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = tx_wqe(qp); wqe 1198 drivers/infiniband/sw/siw/siw_qp.c if (wqe->wr_status != SIW_WR_IDLE) { wqe 1200 drivers/infiniband/sw/siw/siw_qp.c tx_type(wqe), wqe->wr_status); wqe 1202 drivers/infiniband/sw/siw/siw_qp.c siw_wqe_put_mem(wqe, tx_type(wqe)); wqe 1204 drivers/infiniband/sw/siw/siw_qp.c if (tx_type(wqe) != SIW_OP_READ_RESPONSE && wqe 1205 drivers/infiniband/sw/siw/siw_qp.c ((tx_type(wqe) != SIW_OP_READ && wqe 1206 drivers/infiniband/sw/siw/siw_qp.c tx_type(wqe) != SIW_OP_READ_LOCAL_INV) || wqe 1207 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status == SIW_WR_QUEUED)) wqe 1212 drivers/infiniband/sw/siw/siw_qp.c siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, wqe 1215 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_IDLE; wqe 1253 drivers/infiniband/sw/siw/siw_qp.c struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; wqe 1258 drivers/infiniband/sw/siw/siw_qp.c if (wqe->wr_status != SIW_WR_IDLE) { wqe 1260 drivers/infiniband/sw/siw/siw_qp.c rx_type(wqe), wqe->wr_status); wqe 1262 drivers/infiniband/sw/siw/siw_qp.c siw_wqe_put_mem(wqe, rx_type(wqe)); wqe 1264 drivers/infiniband/sw/siw/siw_qp.c if (rx_type(wqe) == SIW_OP_RECEIVE) { wqe 1265 drivers/infiniband/sw/siw/siw_qp.c siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, wqe 1267 drivers/infiniband/sw/siw/siw_qp.c } else if (rx_type(wqe) != SIW_OP_READ && wqe 1268 drivers/infiniband/sw/siw/siw_qp.c rx_type(wqe) != SIW_OP_READ_RESPONSE && wqe 1269 drivers/infiniband/sw/siw/siw_qp.c rx_type(wqe) != SIW_OP_WRITE) { wqe 1270 drivers/infiniband/sw/siw/siw_qp.c siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); wqe 1272 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_IDLE; wqe 1274 drivers/infiniband/sw/siw/siw_qp.c wqe = &qp->rx_tagged.wqe_active; wqe 1276 drivers/infiniband/sw/siw/siw_qp.c if (wqe->wr_status != SIW_WR_IDLE) { wqe 1277 drivers/infiniband/sw/siw/siw_qp.c siw_wqe_put_mem(wqe, rx_type(wqe)); wqe 1278 drivers/infiniband/sw/siw/siw_qp.c wqe->wr_status = SIW_WR_IDLE; wqe 168 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = &frx->wqe_active; wqe 175 drivers/infiniband/sw/siw/siw_qp_rx.c srx->ddp_stag = wqe->sqe.sge[0].lkey; wqe 176 drivers/infiniband/sw/siw/siw_qp_rx.c srx->ddp_to = wqe->sqe.sge[0].laddr; wqe 203 drivers/infiniband/sw/siw/siw_qp_rx.c (wqe->processed + srx->fpdu_part_rem != wqe->bytes))) { wqe 206 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->processed + srx->fpdu_part_rem, wqe->bytes); wqe 280 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = &frx->wqe_active; wqe 300 drivers/infiniband/sw/siw/siw_qp_rx.c if (unlikely(ddp_mo != wqe->processed)) { wqe 302 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(rx_qp(srx)), ddp_mo, wqe->processed); wqe 315 drivers/infiniband/sw/siw/siw_qp_rx.c if (unlikely(wqe->bytes < wqe->processed + srx->fpdu_part_rem)) { wqe 317 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->bytes, wqe->processed, srx->fpdu_part_rem); wqe 318 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->wc_status = SIW_WC_LOC_LEN_ERR; wqe 333 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = NULL; wqe 356 drivers/infiniband/sw/siw/siw_qp_rx.c wqe = rx_wqe(&qp->rx_untagged); wqe 357 drivers/infiniband/sw/siw/siw_qp_rx.c rx_type(wqe) = SIW_OP_RECEIVE; wqe 358 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->wr_status = SIW_WR_INPROGRESS; wqe 359 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->bytes = 0; wqe 360 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->processed = 0; wqe 362 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->rqe.id = rqe->id; wqe 363 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->rqe.num_sge = num_sge; wqe 366 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->rqe.sge[i].laddr = rqe->sge[i].laddr; wqe 367 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->rqe.sge[i].lkey = rqe->sge[i].lkey; wqe 368 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->rqe.sge[i].length = rqe->sge[i].length; wqe 369 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->bytes += wqe->rqe.sge[i].length; wqe 370 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->mem[i] = NULL; wqe 404 drivers/infiniband/sw/siw/siw_qp_rx.c return wqe; wqe 424 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe; wqe 430 drivers/infiniband/sw/siw/siw_qp_rx.c wqe = siw_rqe_get(qp); wqe 431 drivers/infiniband/sw/siw/siw_qp_rx.c if (unlikely(!wqe)) { wqe 438 drivers/infiniband/sw/siw/siw_qp_rx.c wqe = rx_wqe(frx); wqe 459 drivers/infiniband/sw/siw/siw_qp_rx.c sge = &wqe->rqe.sge[frx->sge_idx]; wqe 469 drivers/infiniband/sw/siw/siw_qp_rx.c mem = &wqe->mem[frx->sge_idx]; wqe 499 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->processed += rcvd_bytes; wqe 519 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->processed += rcvd_bytes; wqe 559 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = rx_wqe(frx); wqe 572 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->rqe.num_sge = 1; wqe 573 drivers/infiniband/sw/siw/siw_qp_rx.c rx_type(wqe) = SIW_OP_WRITE; wqe 574 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->wr_status = SIW_WR_INPROGRESS; wqe 740 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = NULL; wqe 748 drivers/infiniband/sw/siw/siw_qp_rx.c wqe = rx_wqe(&qp->rx_tagged); wqe 749 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->sqe.id = orqe->id; wqe 750 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->sqe.opcode = orqe->opcode; wqe 751 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->sqe.sge[0].laddr = orqe->sge[0].laddr; wqe 752 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->sqe.sge[0].lkey = orqe->sge[0].lkey; wqe 753 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->sqe.sge[0].length = orqe->sge[0].length; wqe 754 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->sqe.flags = orqe->flags; wqe 755 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->sqe.num_sge = 1; wqe 756 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->bytes = orqe->sge[0].length; wqe 757 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->processed = 0; wqe 758 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->mem[0] = NULL; wqe 761 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->wr_status = SIW_WR_INPROGRESS; wqe 781 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = rx_wqe(frx); wqe 787 drivers/infiniband/sw/siw/siw_qp_rx.c if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { wqe 789 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(qp), wqe->wr_status, wqe->sqe.opcode); wqe 808 drivers/infiniband/sw/siw/siw_qp_rx.c if (unlikely(wqe->wr_status != SIW_WR_INPROGRESS)) { wqe 810 drivers/infiniband/sw/siw/siw_qp_rx.c qp_id(qp), wqe->wr_status); wqe 818 drivers/infiniband/sw/siw/siw_qp_rx.c sge = wqe->sqe.sge; /* there is only one */ wqe 819 drivers/infiniband/sw/siw/siw_qp_rx.c mem = &wqe->mem[0]; wqe 826 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->bytes); wqe 829 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->wc_status = SIW_WC_LOC_PROT_ERR; wqe 846 drivers/infiniband/sw/siw/siw_qp_rx.c (void *)(uintptr_t)(sge->laddr + wqe->processed), wqe 849 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_rx_umem(srx, mem_p->umem, sge->laddr + wqe->processed, wqe 853 drivers/infiniband/sw/siw/siw_qp_rx.c sge->laddr + wqe->processed, bytes); wqe 855 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->wc_status = SIW_WC_GENERAL_ERR; wqe 861 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->processed += rv; wqe 1208 drivers/infiniband/sw/siw/siw_qp_rx.c struct siw_wqe *wqe = rx_wqe(qp->rx_fpdu); wqe 1209 drivers/infiniband/sw/siw/siw_qp_rx.c enum siw_wc_status wc_status = wqe->wc_status; wqe 1216 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->rqe.flags |= SIW_WQE_SOLICITED; wqe 1221 drivers/infiniband/sw/siw/siw_qp_rx.c if (wqe->wr_status == SIW_WR_IDLE) wqe 1245 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_rqe_complete(qp, &wqe->rqe, wqe->processed, wqe 1249 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_rqe_complete(qp, &wqe->rqe, wqe->processed, wqe 1252 drivers/infiniband/sw/siw/siw_qp_rx.c siw_wqe_put_mem(wqe, SIW_OP_RECEIVE); wqe 1256 drivers/infiniband/sw/siw/siw_qp_rx.c if (wqe->wr_status == SIW_WR_IDLE) wqe 1268 drivers/infiniband/sw/siw/siw_qp_rx.c rx_type(wqe) == SIW_OP_READ_LOCAL_INV) { wqe 1272 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_invalidate_stag(qp->pd, wqe->sqe.sge[0].lkey); wqe 1287 drivers/infiniband/sw/siw/siw_qp_rx.c if ((wqe->sqe.flags & SIW_WQE_SIGNALLED) || error != 0) wqe 1288 drivers/infiniband/sw/siw/siw_qp_rx.c rv = siw_sqe_complete(qp, &wqe->sqe, wqe->processed, wqe 1290 drivers/infiniband/sw/siw/siw_qp_rx.c siw_wqe_put_mem(wqe, SIW_OP_READ); wqe 1307 drivers/infiniband/sw/siw/siw_qp_rx.c if (wqe->wr_status == SIW_WR_IDLE) wqe 1325 drivers/infiniband/sw/siw/siw_qp_rx.c wqe->wr_status = SIW_WR_IDLE; wqe 42 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_wqe *wqe = &c_tx->wqe_active; wqe 43 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_sge *sge = &wqe->sqe.sge[0]; wqe 46 drivers/infiniband/sw/siw/siw_qp_tx.c if (bytes > MAX_HDR_INLINE || wqe->sqe.num_sge != 1) wqe 52 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_flags(wqe) & SIW_WQE_INLINE) { wqe 53 drivers/infiniband/sw/siw/siw_qp_tx.c memcpy(paddr, &wqe->sqe.sge[1], bytes); wqe 55 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_mem *mem = wqe->mem[0]; wqe 121 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_wqe *wqe = &c_tx->wqe_active; wqe 125 drivers/infiniband/sw/siw/siw_qp_tx.c switch (tx_type(wqe)) { wqe 137 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey); wqe 139 drivers/infiniband/sw/siw/siw_qp_tx.c cpu_to_be64(wqe->sqe.sge[0].laddr); wqe 140 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey); wqe 141 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr); wqe 142 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length); wqe 149 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_flags(wqe) & SIW_WQE_SOLICITED) wqe 171 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_flags(wqe) & SIW_WQE_SOLICITED) wqe 185 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey); wqe 197 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey); wqe 198 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr); wqe 211 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey); wqe 212 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr); wqe 221 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe)); wqe 231 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed = data; wqe 246 drivers/infiniband/sw/siw/siw_qp_tx.c cpu_to_be64(wqe->sqe.raddr); wqe 279 drivers/infiniband/sw/siw/siw_qp_tx.c if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH && wqe 280 drivers/infiniband/sw/siw/siw_qp_tx.c !(tx_flags(wqe) & SIW_WQE_SIGNALLED)) wqe 426 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_wqe *wqe = &c_tx->wqe_active; wqe 427 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx]; wqe 454 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed += data_len; wqe 461 drivers/infiniband/sw/siw/siw_qp_tx.c if (!(tx_flags(wqe) & SIW_WQE_INLINE)) { wqe 462 drivers/infiniband/sw/siw/siw_qp_tx.c mem = wqe->mem[sge_idx]; wqe 501 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed -= c_tx->bytes_unsent; wqe 544 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed -= c_tx->bytes_unsent; wqe 552 drivers/infiniband/sw/siw/siw_qp_tx.c (data_len != 0 || wqe->processed < wqe->bytes)) { wqe 582 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx], wqe 598 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed -= data_len; wqe 609 drivers/infiniband/sw/siw/siw_qp_tx.c if (data_len > 0 && wqe->processed < wqe->bytes) { wqe 630 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed -= data_len - rv; wqe 640 drivers/infiniband/sw/siw/siw_qp_tx.c sge = &wqe->sqe.sge[c_tx->sge_idx]; wqe 693 drivers/infiniband/sw/siw/siw_qp_tx.c static void siw_prepare_fpdu(struct siw_qp *qp, struct siw_wqe *wqe) wqe 707 drivers/infiniband/sw/siw/siw_qp_tx.c c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed); wqe 710 drivers/infiniband/sw/siw/siw_qp_tx.c cpu_to_be64(wqe->sqe.raddr + wqe->processed); wqe 712 drivers/infiniband/sw/siw/siw_qp_tx.c data_len = wqe->bytes - wqe->processed; wqe 751 drivers/infiniband/sw/siw/siw_qp_tx.c static int siw_check_sgl_tx(struct ib_pd *pd, struct siw_wqe *wqe, wqe 754 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_sge *sge = &wqe->sqe.sge[0]; wqe 755 drivers/infiniband/sw/siw/siw_qp_tx.c int i, len, num_sge = wqe->sqe.num_sge; wqe 765 drivers/infiniband/sw/siw/siw_qp_tx.c int rv = siw_check_sge(pd, sge, &wqe->mem[i], perms, 0, wqe 781 drivers/infiniband/sw/siw/siw_qp_tx.c static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe) wqe 788 drivers/infiniband/sw/siw/siw_qp_tx.c if (unlikely(wqe->wr_status == SIW_WR_IDLE)) wqe 794 drivers/infiniband/sw/siw/siw_qp_tx.c if (wqe->wr_status == SIW_WR_QUEUED) { wqe 795 drivers/infiniband/sw/siw/siw_qp_tx.c if (!(wqe->sqe.flags & SIW_WQE_INLINE)) { wqe 796 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_type(wqe) == SIW_OP_READ_RESPONSE) wqe 797 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->sqe.num_sge = 1; wqe 799 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_type(wqe) != SIW_OP_READ && wqe 800 drivers/infiniband/sw/siw/siw_qp_tx.c tx_type(wqe) != SIW_OP_READ_LOCAL_INV) { wqe 806 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_check_sgl_tx(qp->pd, wqe, 0); wqe 808 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_type(wqe) == wqe 814 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->bytes = rv; wqe 816 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->bytes = 0; wqe 819 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->bytes = wqe->sqe.sge[0].length; wqe 821 drivers/infiniband/sw/siw/siw_qp_tx.c if (wqe->bytes > SIW_MAX_INLINE) { wqe 825 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->sqe.sge[0].laddr = wqe 826 drivers/infiniband/sw/siw/siw_qp_tx.c (u64)(uintptr_t)&wqe->sqe.sge[1]; wqe 829 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->wr_status = SIW_WR_INPROGRESS; wqe 830 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed = 0; wqe 837 drivers/infiniband/sw/siw/siw_qp_tx.c siw_prepare_fpdu(qp, wqe); wqe 847 drivers/infiniband/sw/siw/siw_qp_tx.c tx_type(wqe), wqe->wr_status, wqe->bytes, wqe->processed, wqe 848 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->sqe.id); wqe 855 drivers/infiniband/sw/siw/siw_qp_tx.c enum siw_opcode tx_type = tx_type(wqe); wqe 872 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->processed = wqe->bytes; wqe 901 drivers/infiniband/sw/siw/siw_qp_tx.c siw_prepare_fpdu(qp, wqe); wqe 966 drivers/infiniband/sw/siw/siw_qp_tx.c static int siw_qp_sq_proc_local(struct siw_qp *qp, struct siw_wqe *wqe) wqe 970 drivers/infiniband/sw/siw/siw_qp_tx.c switch (tx_type(wqe)) { wqe 972 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_fastreg_mr(qp->pd, &wqe->sqe); wqe 976 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_invalidate_stag(qp->pd, wqe->sqe.rkey); wqe 1014 drivers/infiniband/sw/siw/siw_qp_tx.c struct siw_wqe *wqe = tx_wqe(qp); wqe 1019 drivers/infiniband/sw/siw/siw_qp_tx.c siw_dbg_qp(qp, "enter for type %d\n", tx_type(wqe)); wqe 1029 drivers/infiniband/sw/siw/siw_qp_tx.c tx_type = tx_type(wqe); wqe 1032 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_qp_sq_proc_tx(qp, wqe); wqe 1034 drivers/infiniband/sw/siw/siw_qp_tx.c rv = siw_qp_sq_proc_local(qp, wqe); wqe 1044 drivers/infiniband/sw/siw/siw_qp_tx.c siw_wqe_put_mem(wqe, tx_type); wqe 1049 drivers/infiniband/sw/siw/siw_qp_tx.c if (tx_flags(wqe) & SIW_WQE_SIGNALLED) wqe 1050 drivers/infiniband/sw/siw/siw_qp_tx.c siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, wqe 1062 drivers/infiniband/sw/siw/siw_qp_tx.c siw_wqe_put_mem(wqe, tx_type); wqe 1072 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->wr_status = SIW_WR_IDLE; wqe 1104 drivers/infiniband/sw/siw/siw_qp_tx.c tx_type(wqe), rv); wqe 1130 drivers/infiniband/sw/siw/siw_qp_tx.c siw_wqe_put_mem(wqe, tx_type); wqe 1135 drivers/infiniband/sw/siw/siw_qp_tx.c siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, wqe 1147 drivers/infiniband/sw/siw/siw_qp_tx.c siw_wqe_put_mem(wqe, SIW_OP_READ_RESPONSE); wqe 1155 drivers/infiniband/sw/siw/siw_qp_tx.c wqe->wr_status = SIW_WR_IDLE; wqe 742 drivers/infiniband/sw/siw/siw_verbs.c struct siw_wqe *wqe = tx_wqe(qp); wqe 934 drivers/infiniband/sw/siw/siw_verbs.c if (wqe->wr_status != SIW_WR_IDLE) { wqe 2942 drivers/net/ethernet/broadcom/cnic_defs.h __le16 wqe; wqe 3085 drivers/net/ethernet/broadcom/cnic_defs.h __le16 wqe; wqe 54 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c #define CMDQ_WQE_HEADER(wqe) ((struct hinic_cmdq_header *)(wqe)) wqe 182 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped, wqe 196 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c wqe_lcmd = &wqe->wqe_lcmd; wqe 202 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c wqe_scmd = &wqe->direct_wqe.wqe_scmd; wqe 214 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c CMDQ_WQE_HEADER(wqe)->header_info = wqe 223 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c saved_data = CMDQ_WQE_HEADER(wqe)->saved_data; wqe 227 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c CMDQ_WQE_HEADER(wqe)->saved_data |= wqe 230 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c CMDQ_WQE_HEADER(wqe)->saved_data = saved_data; wqe 239 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe, wqe 242 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd; wqe 248 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe, wqe 255 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; wqe 269 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, wqe 276 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe, wqe 283 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; wqe 300 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx, wqe 533 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_wqe *wqe) wqe 535 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info); wqe 542 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; wqe 546 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; wqe 567 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_wqe *wqe) wqe 569 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; wqe 582 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c clear_wqe_complete_bit(cmdq, wqe); wqe 170 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h struct hinic_sq_wqe *wqe, struct hinic_sge *sges, wqe 182 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h struct hinic_sq_wqe *wqe, struct sk_buff *skb, wqe 195 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h void hinic_sq_get_sges(struct hinic_sq_wqe *wqe, struct hinic_sge *sges, wqe 202 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h struct hinic_rq_wqe *wqe, struct sk_buff *skb); wqe 216 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *wqe, wqe 220 drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h struct hinic_rq_wqe *wqe, struct hinic_sge *sge); wqe 73 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c #define WQE_IN_RANGE(wqe, start, end) \ wqe 74 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c (((unsigned long)(wqe) >= (unsigned long)(start)) && \ wqe 75 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c ((unsigned long)(wqe) < (unsigned long)(end))) wqe 77 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c #define WQE_SHADOW_PAGE(wq, wqe) \ wqe 78 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \ wqe 870 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe) wqe 874 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c return WQE_IN_RANGE(wqe, wq->shadow_wqe, wqe 884 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, wqe 891 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c if (wqe_shadow(wq, wqe)) { wqe 892 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c curr_pg = WQE_SHADOW_PAGE(wq, wqe); wqe 108 drivers/net/ethernet/huawei/hinic/hinic_hw_wq.h void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, wqe 494 drivers/net/ethernet/mellanox/mlx5/core/en.h struct mlx5e_tx_wqe *wqe; wqe 638 drivers/net/ethernet/mellanox/mlx5/core/en.h } wqe; wqe 911 drivers/net/ethernet/mellanox/mlx5/core/en.h struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more); wqe 927 drivers/net/ethernet/mellanox/mlx5/core/en.h return mlx5_wq_cyc_get_size(&rq->wqe.wq); wqe 937 drivers/net/ethernet/mellanox/mlx5/core/en.h return rq->wqe.wq.cur_sz; wqe 258 drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c rq->mpwqe.wq.head : mlx5_wq_cyc_get_head(&rq->wqe.wq); wqe 40 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h void *wqe; wqe 43 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h wqe = mlx5_wq_cyc_get_wqe(wq, *pi); wqe 44 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h memset(wqe, 0, size); wqe 46 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h return wqe; wqe 53 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); wqe 54 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; wqe 63 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h return wqe; wqe 70 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); wqe 71 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; wqe 81 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h return wqe; wqe 188 drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h mlx5_wq_cyc_reset(&rq->wqe.wq); wqe 194 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c session->wqe = mlx5e_xdpsq_fetch_wqe(sq, &pi); wqe 196 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c prefetchw(session->wqe->data); wqe 209 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c struct mlx5_wqe_ctrl_seg *cseg = &session->wqe->ctrl; wqe 225 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c session->wqe = NULL; /* Close session */ wqe 235 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c if (unlikely(!sq->mpwqe.wqe)) { wqe 306 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); wqe 308 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; wqe 309 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c struct mlx5_wqe_eth_seg *eseg = &wqe->eth; wqe 310 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c struct mlx5_wqe_data_seg *dseg = wqe->data; wqe 317 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c prefetchw(wqe); wqe 523 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c if (sq->mpwqe.wqe) wqe 535 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c if (xdpsq->mpwqe.wqe) wqe 164 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; wqe 193 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h struct mlx5e_tx_wqe *wqe; wqe 196 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h wqe = mlx5_wq_cyc_get_wqe(wq, *pi); wqe 197 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h memset(wqe, 0, sizeof(*wqe)); wqe 199 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h return wqe; wqe 103 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c if (sq->mpwqe.wqe) wqe 113 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c if (sq->mpwqe.wqe) wqe 107 drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h struct mlx5e_tx_wqe **wqe, wqe 112 drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h skb = mlx5e_tls_handle_tx_skb(dev, sq, skb, wqe, pi); wqe 120 drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h skb = mlx5e_ipsec_handle_tx_skb(dev, *wqe, skb); wqe 237 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c struct mlx5e_tx_wqe *wqe, wqe 279 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c mlx5e_ipsec_set_swp(skb, &wqe->eth, x->props.mode, xo); wqe 56 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h struct mlx5e_tx_wqe *wqe, wqe 93 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h struct mlx5e_tx_wqe **wqe, u16 *pi); wqe 52 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn, wqe 56 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; wqe 57 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; wqe 72 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c fill_static_params_ctx(wqe->tls_static_params_ctx, priv_tx); wqe 86 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c build_progress_params(struct mlx5e_tx_wqe *wqe, u16 pc, u32 sqn, wqe 90 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; wqe 102 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c fill_progress_params_ctx(wqe->tls_progress_params_ctx, priv_tx); wqe 151 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c struct mlx5e_tx_wqe *wqe; wqe 154 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c wqe = mlx5e_sq_fetch_wqe(sq, MLX5E_KTLS_PROGRESS_WQE_SZ, &pi); wqe 155 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence); wqe 275 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c struct mlx5e_dump_wqe *wqe; wqe 281 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c wqe = mlx5e_sq_fetch_wqe(sq, sizeof(*wqe), &pi); wqe 283 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; wqe 285 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c cseg = &wqe->ctrl; wqe 286 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c dseg = &wqe->data; wqe 428 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c struct mlx5e_tx_wqe **wqe, u16 *pi) wqe 452 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c *wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi); wqe 463 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c *wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi); wqe 477 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c cseg = &(*wqe)->ctrl; wqe 190 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c struct mlx5e_tx_wqe **wqe, wqe 250 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c mlx5e_sq_xmit(sq, nskb, *wqe, *pi, true); wqe 251 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c *wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi); wqe 262 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c struct mlx5e_tx_wqe **wqe, wqe 273 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c skb = mlx5e_ktls_handle_tx_skb(netdev, sq, skb, wqe, pi); wqe 293 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c skb = mlx5e_tls_handle_ooo(context, sq, skb, wqe, pi, priv->tls); wqe 46 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h struct mlx5e_tx_wqe **wqe, wqe 234 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_umr_wqe *wqe) wqe 236 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; wqe 237 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; wqe 319 drivers/net/ethernet/mellanox/mlx5/core/en_main.c next_frag.di = &rq->wqe.di[0]; wqe 321 drivers/net/ethernet/mellanox/mlx5/core/en_main.c for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { wqe 322 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; wqe 324 drivers/net/ethernet/mellanox/mlx5/core/en_main.c &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; wqe 327 drivers/net/ethernet/mellanox/mlx5/core/en_main.c for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { wqe 349 drivers/net/ethernet/mellanox/mlx5/core/en_main.c int len = wq_sz << rq->wqe.info.log_num_frags; wqe 351 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), wqe 353 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (!rq->wqe.di) wqe 363 drivers/net/ethernet/mellanox/mlx5/core/en_main.c kvfree(rq->wqe.di); wqe 485 drivers/net/ethernet/mellanox/mlx5/core/en_main.c err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, wqe 490 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; wqe 492 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); wqe 495 drivers/net/ethernet/mellanox/mlx5/core/en_main.c num_xsk_frames = wq_sz << rq->wqe.info.log_num_frags; wqe 497 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->wqe.info = rqp->frags_info; wqe 498 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->wqe.frags = wqe 499 drivers/net/ethernet/mellanox/mlx5/core/en_main.c kvzalloc_node(array_size(sizeof(*rq->wqe.frags), wqe 500 drivers/net/ethernet/mellanox/mlx5/core/en_main.c (wq_sz << rq->wqe.info.log_num_frags)), wqe 502 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (!rq->wqe.frags) { wqe 526 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->wqe.skb_from_cqe = xsk ? wqe 574 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_rx_wqe_ll *wqe = wqe 580 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom); wqe 581 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[0].byte_count = cpu_to_be32(byte_count); wqe 582 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[0].lkey = rq->mkey_be; wqe 584 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_rx_wqe_cyc *wqe = wqe 585 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); wqe 588 drivers/net/ethernet/mellanox/mlx5/core/en_main.c for (f = 0; f < rq->wqe.info.num_frags; f++) { wqe 589 drivers/net/ethernet/mellanox/mlx5/core/en_main.c u32 frag_size = rq->wqe.info.arr[f].frag_size | wqe 592 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[f].byte_count = cpu_to_be32(frag_size); wqe 593 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[f].lkey = rq->mkey_be; wqe 596 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { wqe 597 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[f].byte_count = 0; wqe 598 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY); wqe 599 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe->data[f].addr = 0; wqe 627 drivers/net/ethernet/mellanox/mlx5/core/en_main.c kvfree(rq->wqe.frags); wqe 654 drivers/net/ethernet/mellanox/mlx5/core/en_main.c kvfree(rq->wqe.frags); wqe 861 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_rx_wqe_ll *wqe; wqe 865 drivers/net/ethernet/mellanox/mlx5/core/en_main.c wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); wqe 868 drivers/net/ethernet/mellanox/mlx5/core/en_main.c &wqe->next.next_wqe_index); wqe 871 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5_wq_cyc *wq = &rq->wqe.wq; wqe 1496 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i); wqe 1497 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; wqe 1498 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5_wqe_eth_seg *eseg = &wqe->eth; wqe 3127 drivers/net/ethernet/mellanox/mlx5/core/en_main.c err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, wqe 129 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, cqd->wqe_counter + 1); wqe 332 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c return &rq->wqe.frags[ix << rq->wqe.info.log_num_frags]; wqe 335 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, wqe 342 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) { wqe 347 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c wqe->data[i].addr = cpu_to_be64(frag->di->addr + wqe 366 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c for (i = 0; i < rq->wqe.info.num_frags; i++, wi++) wqe 379 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5_wq_cyc *wq = &rq->wqe.wq; wqe 384 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags; wqe 391 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i); wqe 393 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c err = mlx5e_alloc_rx_wqe(rq, wqe, ix + i); wqe 558 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5_wq_cyc *wq = &rq->wqe.wq; wqe 565 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c wqe_bulk = rq->wqe.info.wqe_bulk; wqe 1097 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; wqe 1148 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5_wq_cyc *wq = &rq->wqe.wq; wqe 1164 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe, wqe 1195 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5_wq_cyc *wq = &rq->wqe.wq; wqe 1210 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt); wqe 1337 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5e_rx_wqe_ll *wqe; wqe 1375 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c wqe = mlx5_wq_ll_get_wqe(wq, wqe_id); wqe 1377 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index); wqe 1514 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5_wq_cyc *wq = &rq->wqe.wq; wqe 1529 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe, wqe 1554 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c struct mlx5_wq_cyc *wq = &rq->wqe.wq; wqe 1569 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe, wqe 270 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more) wqe 287 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; wqe 295 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c u8 mode = mlx5e_tx_wqe_inline_mode(sq, &wqe->ctrl, skb); wqe 322 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c struct mlx5_wqe_eth_seg cur_eth = wqe->eth; wqe 325 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c struct mlx5_wqe_ctrl_seg cur_ctrl = wqe->ctrl; wqe 328 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c wqe = mlx5e_sq_fetch_wqe(sq, sizeof(*wqe), &pi); wqe 330 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c wqe->eth = cur_eth; wqe 333 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c wqe->ctrl = cur_ctrl; wqe 339 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c cseg = &wqe->ctrl; wqe 340 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c eseg = &wqe->eth; wqe 341 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c dseg = wqe->data; wqe 388 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c struct mlx5e_tx_wqe *wqe; wqe 393 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c wqe = mlx5e_sq_fetch_wqe(sq, sizeof(*wqe), &pi); wqe 396 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c skb = mlx5e_accel_handle_tx(skb, sq, dev, &wqe, &pi); wqe 400 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c return mlx5e_sq_xmit(sq, skb, wqe, pi, netdev_xmit_more()); wqe 593 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c struct mlx5i_tx_wqe *wqe; wqe 610 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; wqe 647 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c mlx5i_sq_fetch_wqe(sq, &wqe, pi); wqe 651 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c cseg = &wqe->ctrl; wqe 652 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c datagram = &wqe->datagram; wqe 653 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c eseg = &wqe->eth; wqe 654 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c dseg = wqe->data; wqe 131 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe) wqe 138 drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET); wqe 114 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h struct mlx5i_tx_wqe **wqe, wqe 119 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h *wqe = mlx5_wq_cyc_get_wqe(wq, pi); wqe 120 drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h memset(*wqe, 0, sizeof(**wqe)); wqe 251 drivers/net/ethernet/mellanox/mlx5/core/wq.h struct mlx5_wqe_srq_next_seg *wqe = mlx5_wq_ll_get_wqe(wq, ix); wqe 253 drivers/net/ethernet/mellanox/mlx5/core/wq.h return be16_to_cpu(wqe->next_wqe_index); wqe 316 drivers/net/ethernet/mellanox/mlxsw/pci.c static int mlxsw_pci_wqe_frag_map(struct mlxsw_pci *mlxsw_pci, char *wqe, wqe 328 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_address_set(wqe, index, mapaddr); wqe 329 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_byte_count_set(wqe, index, frag_len); wqe 333 drivers/net/ethernet/mellanox/mlxsw/pci.c static void mlxsw_pci_wqe_frag_unmap(struct mlxsw_pci *mlxsw_pci, char *wqe, wqe 337 drivers/net/ethernet/mellanox/mlxsw/pci.c size_t frag_len = mlxsw_pci_wqe_byte_count_get(wqe, index); wqe 338 drivers/net/ethernet/mellanox/mlxsw/pci.c dma_addr_t mapaddr = mlxsw_pci_wqe_address_get(wqe, index); wqe 349 drivers/net/ethernet/mellanox/mlxsw/pci.c char *wqe = elem_info->elem; wqe 360 drivers/net/ethernet/mellanox/mlxsw/pci.c err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data, wqe 377 drivers/net/ethernet/mellanox/mlxsw/pci.c char *wqe; wqe 380 drivers/net/ethernet/mellanox/mlxsw/pci.c wqe = elem_info->elem; wqe 382 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE); wqe 512 drivers/net/ethernet/mellanox/mlxsw/pci.c char *wqe; wqe 520 drivers/net/ethernet/mellanox/mlxsw/pci.c wqe = elem_info->elem; wqe 522 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE); wqe 547 drivers/net/ethernet/mellanox/mlxsw/pci.c char *wqe; wqe 557 drivers/net/ethernet/mellanox/mlxsw/pci.c wqe = elem_info->elem; wqe 558 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE); wqe 581 drivers/net/ethernet/mellanox/mlxsw/pci.c memset(wqe, 0, q->elem_size); wqe 1571 drivers/net/ethernet/mellanox/mlxsw/pci.c char *wqe; wqe 1592 drivers/net/ethernet/mellanox/mlxsw/pci.c wqe = elem_info->elem; wqe 1593 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_c_set(wqe, 1); /* always report completion */ wqe 1594 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_lp_set(wqe, !!tx_info->is_emad); wqe 1595 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_type_set(wqe, MLXSW_PCI_WQE_TYPE_ETHERNET); wqe 1597 drivers/net/ethernet/mellanox/mlxsw/pci.c err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data, wqe 1605 drivers/net/ethernet/mellanox/mlxsw/pci.c err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, i + 1, wqe 1618 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_byte_count_set(wqe, i, 0); wqe 1628 drivers/net/ethernet/mellanox/mlxsw/pci.c mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE); wqe 74 drivers/net/ethernet/mellanox/mlxsw/pci_hw.h MLXSW_ITEM32(pci, wqe, c, 0x00, 31, 1); wqe 86 drivers/net/ethernet/mellanox/mlxsw/pci_hw.h MLXSW_ITEM32(pci, wqe, lp, 0x00, 30, 1); wqe 91 drivers/net/ethernet/mellanox/mlxsw/pci_hw.h MLXSW_ITEM32(pci, wqe, type, 0x00, 23, 4); wqe 96 drivers/net/ethernet/mellanox/mlxsw/pci_hw.h MLXSW_ITEM16_INDEXED(pci, wqe, byte_count, 0x02, 0, 14, 0x02, 0x00, false); wqe 102 drivers/net/ethernet/mellanox/mlxsw/pci_hw.h MLXSW_ITEM64_INDEXED(pci, wqe, address, 0x08, 0, 64, 0x8, 0x0, false); wqe 89 drivers/scsi/bfa/bfa.h bfa_reqq_winit(struct bfa_reqq_wait_s *wqe, void (*qresume) (void *cbarg), wqe 92 drivers/scsi/bfa/bfa.h wqe->qresume = qresume; wqe 93 drivers/scsi/bfa/bfa.h wqe->cbarg = cbarg; wqe 695 drivers/scsi/bfa/bfa_core.c struct bfa_reqq_wait_s *wqe; wqe 706 drivers/scsi/bfa/bfa_core.c wqe = (struct bfa_reqq_wait_s *) qe; wqe 707 drivers/scsi/bfa/bfa_core.c wqe->qresume(wqe->cbarg); wqe 621 drivers/scsi/bfa/bfa_svc.c struct bfa_fcxp_wqe_s *wqe; wqe 624 drivers/scsi/bfa/bfa_svc.c bfa_q_deq(&mod->req_wait_q, &wqe); wqe 626 drivers/scsi/bfa/bfa_svc.c bfa_q_deq(&mod->rsp_wait_q, &wqe); wqe 628 drivers/scsi/bfa/bfa_svc.c if (wqe) { wqe 631 drivers/scsi/bfa/bfa_svc.c bfa_fcxp_init(fcxp, wqe->caller, wqe->bfa, wqe->nreq_sgles, wqe 632 drivers/scsi/bfa/bfa_svc.c wqe->nrsp_sgles, wqe->req_sga_cbfn, wqe 633 drivers/scsi/bfa/bfa_svc.c wqe->req_sglen_cbfn, wqe->rsp_sga_cbfn, wqe 634 drivers/scsi/bfa/bfa_svc.c wqe->rsp_sglen_cbfn); wqe 636 drivers/scsi/bfa/bfa_svc.c wqe->alloc_cbfn(wqe->alloc_cbarg, fcxp); wqe 1062 drivers/scsi/bfa/bfa_svc.c bfa_fcxp_req_rsp_alloc_wait(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe, wqe 1077 drivers/scsi/bfa/bfa_svc.c wqe->alloc_cbfn = alloc_cbfn; wqe 1078 drivers/scsi/bfa/bfa_svc.c wqe->alloc_cbarg = alloc_cbarg; wqe 1079 drivers/scsi/bfa/bfa_svc.c wqe->caller = caller; wqe 1080 drivers/scsi/bfa/bfa_svc.c wqe->bfa = bfa; wqe 1081 drivers/scsi/bfa/bfa_svc.c wqe->nreq_sgles = nreq_sgles; wqe 1082 drivers/scsi/bfa/bfa_svc.c wqe->nrsp_sgles = nrsp_sgles; wqe 1083 drivers/scsi/bfa/bfa_svc.c wqe->req_sga_cbfn = req_sga_cbfn; wqe 1084 drivers/scsi/bfa/bfa_svc.c wqe->req_sglen_cbfn = req_sglen_cbfn; wqe 1085 drivers/scsi/bfa/bfa_svc.c wqe->rsp_sga_cbfn = rsp_sga_cbfn; wqe 1086 drivers/scsi/bfa/bfa_svc.c wqe->rsp_sglen_cbfn = rsp_sglen_cbfn; wqe 1089 drivers/scsi/bfa/bfa_svc.c list_add_tail(&wqe->qe, &mod->req_wait_q); wqe 1091 drivers/scsi/bfa/bfa_svc.c list_add_tail(&wqe->qe, &mod->rsp_wait_q); wqe 1095 drivers/scsi/bfa/bfa_svc.c bfa_fcxp_walloc_cancel(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe) wqe 1099 drivers/scsi/bfa/bfa_svc.c WARN_ON(!bfa_q_is_on_q(&mod->req_wait_q, wqe) || wqe 1100 drivers/scsi/bfa/bfa_svc.c !bfa_q_is_on_q(&mod->rsp_wait_q, wqe)); wqe 1101 drivers/scsi/bfa/bfa_svc.c list_del(&wqe->qe); wqe 1178 drivers/scsi/bfa/bfa_svc.c bfa_reqq_wait(lps->bfa, lps->reqq, &lps->wqe); wqe 1295 drivers/scsi/bfa/bfa_svc.c bfa_reqq_wcancel(&lps->wqe); wqe 1324 drivers/scsi/bfa/bfa_svc.c bfa_reqq_wait(lps->bfa, lps->reqq, &lps->wqe); wqe 1345 drivers/scsi/bfa/bfa_svc.c bfa_reqq_wait(lps->bfa, lps->reqq, &lps->wqe); wqe 1383 drivers/scsi/bfa/bfa_svc.c bfa_reqq_wcancel(&lps->wqe); wqe 1394 drivers/scsi/bfa/bfa_svc.c bfa_reqq_wcancel(&lps->wqe); wqe 1445 drivers/scsi/bfa/bfa_svc.c bfa_reqq_wcancel(&lps->wqe); wqe 1504 drivers/scsi/bfa/bfa_svc.c bfa_reqq_winit(&lps->wqe, bfa_lps_reqq_resume, lps); wqe 5265 drivers/scsi/bfa/bfa_svc.c struct bfa_sgpg_wqe_s *wqe; wqe 5279 drivers/scsi/bfa/bfa_svc.c wqe = bfa_q_first(&mod->sgpg_wait_q); wqe 5280 drivers/scsi/bfa/bfa_svc.c if (mod->free_sgpgs < wqe->nsgpg) wqe 5283 drivers/scsi/bfa/bfa_svc.c nsgpg = wqe->nsgpg; wqe 5284 drivers/scsi/bfa/bfa_svc.c bfa_sgpg_malloc(bfa, &wqe->sgpg_q, nsgpg); wqe 5285 drivers/scsi/bfa/bfa_svc.c wqe->nsgpg -= nsgpg; wqe 5286 drivers/scsi/bfa/bfa_svc.c if (wqe->nsgpg == 0) { wqe 5287 drivers/scsi/bfa/bfa_svc.c list_del(&wqe->qe); wqe 5288 drivers/scsi/bfa/bfa_svc.c wqe->cbfn(wqe->cbarg); wqe 5294 drivers/scsi/bfa/bfa_svc.c bfa_sgpg_wait(struct bfa_s *bfa, struct bfa_sgpg_wqe_s *wqe, int nsgpg) wqe 5301 drivers/scsi/bfa/bfa_svc.c wqe->nsgpg_total = wqe->nsgpg = nsgpg; wqe 5311 drivers/scsi/bfa/bfa_svc.c list_splice_tail_init(&mod->sgpg_q, &wqe->sgpg_q); wqe 5312 drivers/scsi/bfa/bfa_svc.c wqe->nsgpg -= mod->free_sgpgs; wqe 5316 drivers/scsi/bfa/bfa_svc.c list_add_tail(&wqe->qe, &mod->sgpg_wait_q); wqe 5320 drivers/scsi/bfa/bfa_svc.c bfa_sgpg_wcancel(struct bfa_s *bfa, struct bfa_sgpg_wqe_s *wqe) wqe 5324 drivers/scsi/bfa/bfa_svc.c WARN_ON(!bfa_q_is_on_q(&mod->sgpg_wait_q, wqe)); wqe 5325 drivers/scsi/bfa/bfa_svc.c list_del(&wqe->qe); wqe 5327 drivers/scsi/bfa/bfa_svc.c if (wqe->nsgpg_total != wqe->nsgpg) wqe 5328 drivers/scsi/bfa/bfa_svc.c bfa_sgpg_mfree(bfa, &wqe->sgpg_q, wqe 5329 drivers/scsi/bfa/bfa_svc.c wqe->nsgpg_total - wqe->nsgpg); wqe 5333 drivers/scsi/bfa/bfa_svc.c bfa_sgpg_winit(struct bfa_sgpg_wqe_s *wqe, void (*cbfn) (void *cbarg), wqe 5336 drivers/scsi/bfa/bfa_svc.c INIT_LIST_HEAD(&wqe->sgpg_q); wqe 5337 drivers/scsi/bfa/bfa_svc.c wqe->cbfn = cbfn; wqe 5338 drivers/scsi/bfa/bfa_svc.c wqe->cbarg = cbarg; wqe 70 drivers/scsi/bfa/bfa_svc.h void bfa_sgpg_winit(struct bfa_sgpg_wqe_s *wqe, wqe 72 drivers/scsi/bfa/bfa_svc.h void bfa_sgpg_wait(struct bfa_s *bfa, struct bfa_sgpg_wqe_s *wqe, int nsgpgs); wqe 73 drivers/scsi/bfa/bfa_svc.h void bfa_sgpg_wcancel(struct bfa_s *bfa, struct bfa_sgpg_wqe_s *wqe); wqe 410 drivers/scsi/bfa/bfa_svc.h struct bfa_reqq_wait_s wqe; /* request wait queue element */ wqe 609 drivers/scsi/bfa/bfa_svc.h void bfa_fcxp_req_rsp_alloc_wait(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe, wqe 619 drivers/scsi/bfa/bfa_svc.h struct bfa_fcxp_wqe_s *wqe); wqe 640 drivers/scsi/bnx2fc/57xx_hsi_bnx2fc.h __le16 wqe; wqe 783 drivers/scsi/bnx2fc/57xx_hsi_bnx2fc.h __le16 wqe; wqe 835 drivers/scsi/bnx2fc/57xx_hsi_bnx2fc.h __le16 wqe; wqe 899 drivers/scsi/bnx2fc/57xx_hsi_bnx2fc.h __le32 wqe; wqe 914 drivers/scsi/bnx2fc/57xx_hsi_bnx2fc.h __le16 wqe; wqe 993 drivers/scsi/bnx2fc/57xx_hsi_bnx2fc.h __le16 wqe; wqe 485 drivers/scsi/bnx2fc/bnx2fc.h u16 wqe; wqe 580 drivers/scsi/bnx2fc/bnx2fc.h void bnx2fc_process_cq_compl(struct bnx2fc_rport *tgt, u16 wqe); wqe 663 drivers/scsi/bnx2fc/bnx2fc_fcoe.c bnx2fc_process_cq_compl(work->tgt, work->wqe); wqe 2658 drivers/scsi/bnx2fc/bnx2fc_fcoe.c bnx2fc_process_cq_compl(work->tgt, work->wqe); wqe 626 drivers/scsi/bnx2fc/bnx2fc_hwi.c static void bnx2fc_process_unsol_compl(struct bnx2fc_rport *tgt, u16 wqe) wqe 644 drivers/scsi/bnx2fc/bnx2fc_hwi.c BNX2FC_TGT_DBG(tgt, "Entered UNSOL COMPLETION wqe = 0x%x\n", wqe); wqe 645 drivers/scsi/bnx2fc/bnx2fc_hwi.c switch (wqe & FCOE_UNSOLICITED_CQE_SUBTYPE) { wqe 647 drivers/scsi/bnx2fc/bnx2fc_hwi.c frame_len = (wqe & FCOE_UNSOLICITED_CQE_PKT_LEN) >> wqe 866 drivers/scsi/bnx2fc/bnx2fc_hwi.c void bnx2fc_process_cq_compl(struct bnx2fc_rport *tgt, u16 wqe) wqe 881 drivers/scsi/bnx2fc/bnx2fc_hwi.c xid = wqe & FCOE_PEND_WQ_CQE_TASK_ID; wqe 990 drivers/scsi/bnx2fc/bnx2fc_hwi.c static struct bnx2fc_work *bnx2fc_alloc_work(struct bnx2fc_rport *tgt, u16 wqe) wqe 999 drivers/scsi/bnx2fc/bnx2fc_hwi.c work->wqe = wqe; wqe 1004 drivers/scsi/bnx2fc/bnx2fc_hwi.c static void bnx2fc_pending_work(struct bnx2fc_rport *tgt, unsigned int wqe) wqe 1006 drivers/scsi/bnx2fc/bnx2fc_hwi.c unsigned int cpu = wqe % num_possible_cpus(); wqe 1013 drivers/scsi/bnx2fc/bnx2fc_hwi.c work = bnx2fc_alloc_work(tgt, wqe); wqe 1022 drivers/scsi/bnx2fc/bnx2fc_hwi.c bnx2fc_process_cq_compl(tgt, wqe); wqe 1032 drivers/scsi/bnx2fc/bnx2fc_hwi.c u16 wqe; wqe 1050 drivers/scsi/bnx2fc/bnx2fc_hwi.c while (((wqe = cqe->wqe) & FCOE_CQE_TOGGLE_BIT) == wqe 1055 drivers/scsi/bnx2fc/bnx2fc_hwi.c if (wqe & FCOE_CQE_CQE_TYPE) { wqe 1057 drivers/scsi/bnx2fc/bnx2fc_hwi.c bnx2fc_process_unsol_compl(tgt, wqe); wqe 1059 drivers/scsi/bnx2fc/bnx2fc_hwi.c bnx2fc_pending_work(tgt, wqe); wqe 1382 drivers/scsi/bnx2fc/bnx2fc_hwi.c sqe->wqe = xid << FCOE_SQE_TASK_ID_SHIFT; wqe 1383 drivers/scsi/bnx2fc/bnx2fc_hwi.c sqe->wqe |= tgt->sq_curr_toggle_bit << FCOE_SQE_TOGGLE_BIT_SHIFT; wqe 76 drivers/scsi/lpfc/lpfc_nvme.c union lpfc_wqe128 *wqe; wqe 79 drivers/scsi/lpfc/lpfc_nvme.c wqe = &lpfc_iread_cmd_template; wqe 80 drivers/scsi/lpfc/lpfc_nvme.c memset(wqe, 0, sizeof(union lpfc_wqe128)); wqe 93 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); wqe 94 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); wqe 95 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); wqe 96 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); wqe 103 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); wqe 104 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); wqe 105 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); wqe 106 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); wqe 107 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); wqe 108 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); wqe 111 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); wqe 112 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 113 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); wqe 120 drivers/scsi/lpfc/lpfc_nvme.c wqe = &lpfc_iwrite_cmd_template; wqe 121 drivers/scsi/lpfc/lpfc_nvme.c memset(wqe, 0, sizeof(union lpfc_wqe128)); wqe 134 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); wqe 135 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); wqe 136 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); wqe 137 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); wqe 144 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); wqe 145 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); wqe 146 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); wqe 147 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); wqe 148 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); wqe 149 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); wqe 152 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); wqe 153 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 154 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); wqe 161 drivers/scsi/lpfc/lpfc_nvme.c wqe = &lpfc_icmnd_cmd_template; wqe 162 drivers/scsi/lpfc/lpfc_nvme.c memset(wqe, 0, sizeof(union lpfc_wqe128)); wqe 173 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); wqe 174 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); wqe 175 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); wqe 176 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); wqe 183 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); wqe 184 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); wqe 185 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); wqe 186 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); wqe 187 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); wqe 188 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); wqe 191 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); wqe 192 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 193 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); wqe 429 drivers/scsi/lpfc/lpfc_nvme.c union lpfc_wqe128 *wqe; wqe 440 drivers/scsi/lpfc/lpfc_nvme.c wqe = &genwqe->wqe; wqe 442 drivers/scsi/lpfc/lpfc_nvme.c memset(wqe, 0, sizeof(union lpfc_wqe)); wqe 473 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; wqe 474 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.tus.f.bdeSize = first_len; wqe 475 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.addrLow = bpl[0].addrLow; wqe 476 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.addrHigh = bpl[0].addrHigh; wqe 479 drivers/scsi/lpfc/lpfc_nvme.c wqe->gen_req.request_payload_len = first_len; wqe 484 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); wqe 485 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); wqe 486 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); wqe 487 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); wqe 488 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); wqe 491 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, wqe 493 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); wqe 496 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); wqe 497 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); wqe 498 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); wqe 499 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); wqe 502 drivers/scsi/lpfc/lpfc_nvme.c wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; wqe 505 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); wqe 508 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); wqe 509 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); wqe 510 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); wqe 511 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); wqe 512 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); wqe 515 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 516 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); wqe 709 drivers/scsi/lpfc/lpfc_nvme.c struct lpfc_iocbq *wqe, *next_wqe; wqe 745 drivers/scsi/lpfc/lpfc_nvme.c list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { wqe 747 drivers/scsi/lpfc/lpfc_nvme.c if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) { wqe 748 drivers/scsi/lpfc/lpfc_nvme.c wqe->iocb_flag |= LPFC_DRIVER_ABORTED; wqe 749 drivers/scsi/lpfc/lpfc_nvme.c list_add_tail(&wqe->dlist, &abort_list); wqe 756 drivers/scsi/lpfc/lpfc_nvme.c list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) { wqe 759 drivers/scsi/lpfc/lpfc_nvme.c list_del_init(&wqe->dlist); wqe 760 drivers/scsi/lpfc/lpfc_nvme.c lpfc_sli_issue_abort_iotag(phba, pring, wqe); wqe 773 drivers/scsi/lpfc/lpfc_nvme.c union lpfc_wqe128 *wqe; wqe 782 drivers/scsi/lpfc/lpfc_nvme.c wqe = &lpfc_ncmd->cur_iocbq.wqe; wqe 798 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; wqe 799 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.tus.f.bdeSize = 56; wqe 800 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.addrHigh = 0; wqe 801 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.addrLow = 64; /* Word 16 */ wqe 813 drivers/scsi/lpfc/lpfc_nvme.c wptr = &wqe->words[16]; /* WQE ptr */ wqe 838 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; wqe 839 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; wqe 840 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.addrHigh = sgl->addr_hi; wqe 841 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.bde.addrLow = sgl->addr_lo; wqe 844 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); wqe 845 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); wqe 1206 drivers/scsi/lpfc/lpfc_nvme.c union lpfc_wqe128 *wqe = &pwqeq->wqe; wqe 1219 drivers/scsi/lpfc/lpfc_nvme.c memcpy(&wqe->words[7], wqe 1224 drivers/scsi/lpfc/lpfc_nvme.c wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; wqe 1231 drivers/scsi/lpfc/lpfc_nvme.c wqe->fcp_iwrite.initial_xfer_len = wqe 1234 drivers/scsi/lpfc/lpfc_nvme.c wqe->fcp_iwrite.initial_xfer_len = wqe 1237 drivers/scsi/lpfc/lpfc_nvme.c wqe->fcp_iwrite.initial_xfer_len = 0; wqe 1242 drivers/scsi/lpfc/lpfc_nvme.c memcpy(&wqe->words[7], wqe 1247 drivers/scsi/lpfc/lpfc_nvme.c wqe->fcp_iread.total_xfer_len = nCmd->payload_length; wqe 1250 drivers/scsi/lpfc/lpfc_nvme.c wqe->fcp_iread.rsrvd5 = 0; wqe 1256 drivers/scsi/lpfc/lpfc_nvme.c memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], wqe 1262 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_erp, &wqe->generic.wqe_com, 1); wqe 1269 drivers/scsi/lpfc/lpfc_nvme.c bf_set(payload_offset_len, &wqe->fcp_icmd, wqe 1273 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, wqe 1275 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); wqe 1278 drivers/scsi/lpfc/lpfc_nvme.c wqe->generic.wqe_com.abort_tag = pwqeq->iotag; wqe 1281 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); wqe 1312 drivers/scsi/lpfc/lpfc_nvme.c union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; wqe 1446 drivers/scsi/lpfc/lpfc_nvme.c &wqe->words[13]; wqe 1455 drivers/scsi/lpfc/lpfc_nvme.c memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); wqe 1456 drivers/scsi/lpfc/lpfc_nvme.c bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); wqe 1918 drivers/scsi/lpfc/lpfc_nvme.c abts_wqe = &abts_buf->wqe; wqe 2024 drivers/scsi/lpfc/lpfc_nvme.c union lpfc_wqe128 *wqe; wqe 2030 drivers/scsi/lpfc/lpfc_nvme.c wqe = &pwqeq->wqe; wqe 2051 drivers/scsi/lpfc/lpfc_nvme.c memset(wqe, 0, sizeof(union lpfc_wqe)); wqe 2726 drivers/scsi/lpfc/lpfc_nvme.c if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == wqe 88 drivers/scsi/lpfc/lpfc_nvmet.c union lpfc_wqe128 *wqe; wqe 91 drivers/scsi/lpfc/lpfc_nvmet.c wqe = &lpfc_tsend_cmd_template; wqe 92 drivers/scsi/lpfc/lpfc_nvmet.c memset(wqe, 0, sizeof(union lpfc_wqe128)); wqe 105 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE); wqe 106 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF); wqe 107 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3); wqe 108 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI); wqe 109 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1); wqe 116 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1); wqe 117 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1); wqe 118 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0); wqe 119 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1); wqe 120 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE); wqe 121 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12); wqe 124 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND); wqe 125 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 126 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0); wqe 127 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0); wqe 128 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0); wqe 129 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0); wqe 136 drivers/scsi/lpfc/lpfc_nvmet.c wqe = &lpfc_treceive_cmd_template; wqe 137 drivers/scsi/lpfc/lpfc_nvmet.c memset(wqe, 0, sizeof(union lpfc_wqe128)); wqe 142 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN; wqe 151 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE); wqe 152 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF); wqe 153 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3); wqe 154 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI); wqe 155 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0); wqe 162 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1); wqe 163 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0); wqe 164 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1); wqe 165 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ); wqe 166 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12); wqe 167 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1); wqe 170 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE); wqe 171 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 172 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0); wqe 173 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0); wqe 174 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0); wqe 175 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1); wqe 182 drivers/scsi/lpfc/lpfc_nvmet.c wqe = &lpfc_trsp_cmd_template; wqe 183 drivers/scsi/lpfc/lpfc_nvmet.c memset(wqe, 0, sizeof(union lpfc_wqe128)); wqe 194 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE); wqe 195 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED); wqe 196 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3); wqe 197 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI); wqe 198 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */ wqe 205 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1); wqe 206 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1); wqe 207 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0); wqe 208 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0); wqe 209 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE); wqe 210 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3); wqe 213 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP); wqe 214 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 215 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0); wqe 216 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0); wqe 217 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0); wqe 218 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0); wqe 892 drivers/scsi/lpfc/lpfc_nvmet.c bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow; wqe 893 drivers/scsi/lpfc/lpfc_nvmet.c bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh; wqe 1294 drivers/scsi/lpfc/lpfc_nvmet.c union lpfc_wqe128 *wqe; wqe 1392 drivers/scsi/lpfc/lpfc_nvmet.c wqe = &nvmewqe->wqe; wqe 1395 drivers/scsi/lpfc/lpfc_nvmet.c memset(wqe, 0, sizeof(union lpfc_wqe)); wqe 2471 drivers/scsi/lpfc/lpfc_nvmet.c union lpfc_wqe128 *wqe; wqe 2509 drivers/scsi/lpfc/lpfc_nvmet.c wqe = &nvmewqe->wqe; wqe 2510 drivers/scsi/lpfc/lpfc_nvmet.c memset(wqe, 0, sizeof(union lpfc_wqe)); wqe 2513 drivers/scsi/lpfc/lpfc_nvmet.c wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; wqe 2514 drivers/scsi/lpfc/lpfc_nvmet.c wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize; wqe 2515 drivers/scsi/lpfc/lpfc_nvmet.c wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf)); wqe 2516 drivers/scsi/lpfc/lpfc_nvmet.c wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf)); wqe 2523 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0); wqe 2524 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1); wqe 2525 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0); wqe 2526 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP); wqe 2527 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME); wqe 2530 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, wqe 2532 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag); wqe 2535 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com, wqe 2537 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI); wqe 2538 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3); wqe 2539 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0); wqe 2542 drivers/scsi/lpfc/lpfc_nvmet.c wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag; wqe 2545 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag); wqe 2547 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid); wqe 2550 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1); wqe 2551 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); wqe 2552 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com, wqe 2554 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0); wqe 2557 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com, wqe 2559 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com, wqe 2563 drivers/scsi/lpfc/lpfc_nvmet.c wqe->xmit_sequence.xmit_len = rspsize; wqe 2596 drivers/scsi/lpfc/lpfc_nvmet.c union lpfc_wqe128 *wqe; wqe 2654 drivers/scsi/lpfc/lpfc_nvmet.c wqe = &nvmewqe->wqe; wqe 2667 drivers/scsi/lpfc/lpfc_nvmet.c memcpy(&wqe->words[7], wqe 2674 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; wqe 2675 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel); wqe 2676 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr)); wqe 2677 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.bde.addrHigh = wqe 2681 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.payload_offset_len = 0; wqe 2684 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.relative_offset = ctxp->offset; wqe 2687 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.reserved = 0; wqe 2690 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com, wqe 2692 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com, wqe 2698 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag; wqe 2701 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag); wqe 2702 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid); wqe 2706 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0); wqe 2712 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; wqe 2737 drivers/scsi/lpfc/lpfc_nvmet.c &wqe->fcp_tsend.wqe_com, 1); wqe 2739 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1); wqe 2740 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1); wqe 2741 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, wqe 2743 drivers/scsi/lpfc/lpfc_nvmet.c memcpy(&wqe->words[16], rsp->rspaddr, wqe 2750 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0); wqe 2756 drivers/scsi/lpfc/lpfc_nvmet.c memcpy(&wqe->words[3], wqe 2771 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; wqe 2772 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN; wqe 2773 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_treceive.bde.addrLow = wqe 2775 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_treceive.bde.addrHigh = wqe 2779 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_treceive.relative_offset = ctxp->offset; wqe 2782 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com, wqe 2784 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com, wqe 2790 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag; wqe 2793 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag); wqe 2794 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid); wqe 2798 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0); wqe 2804 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0); wqe 2809 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; wqe 2835 drivers/scsi/lpfc/lpfc_nvmet.c memcpy(&wqe->words[4], wqe 2841 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; wqe 2842 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen; wqe 2843 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_trsp.bde.addrLow = wqe 2845 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_trsp.bde.addrHigh = wqe 2849 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_trsp.response_len = rsp->rsplen; wqe 2852 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com, wqe 2854 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com, wqe 2860 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag; wqe 2863 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag); wqe 2864 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid); wqe 2868 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1); wqe 2874 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1); wqe 2875 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1); wqe 2876 drivers/scsi/lpfc/lpfc_nvmet.c bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, wqe 2878 drivers/scsi/lpfc/lpfc_nvmet.c memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen); wqe 2883 drivers/scsi/lpfc/lpfc_nvmet.c wqe->fcp_trsp.rsvd_12_15[0] = 0; wqe 2916 drivers/scsi/lpfc/lpfc_nvmet.c bde = (struct ulp_bde64 *)&wqe->words[13]; wqe 3170 drivers/scsi/lpfc/lpfc_nvmet.c wqe_abts = &abts_wqeq->wqe; wqe 3283 drivers/scsi/lpfc/lpfc_nvmet.c abts_wqe = &abts_wqeq->wqe; wqe 138 drivers/scsi/lpfc/lpfc_sli.c lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe) wqe 162 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_wqec, &wqe->generic.wqe_com, 1); wqe 164 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_wqec, &wqe->generic.wqe_com, 0); wqe 166 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id); wqe 167 drivers/scsi/lpfc/lpfc_sli.c lpfc_sli4_pcimem_bcopy(wqe, temp_wqe, q->entry_size); wqe 9282 drivers/scsi/lpfc/lpfc_sli.c union lpfc_wqe128 *wqe) wqe 9311 drivers/scsi/lpfc/lpfc_sli.c memset(wqe, 0, sizeof(union lpfc_wqe128)); wqe 9313 drivers/scsi/lpfc/lpfc_sli.c memcpy(wqe, &iocbq->iocb, sizeof(union lpfc_wqe)); wqe 9315 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.wqe_com.word7 = 0; wqe 9316 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.wqe_com.word10 = 0; wqe 9330 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrHigh = le32_to_cpu(bpl->addrHigh); wqe 9331 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrLow = le32_to_cpu(bpl->addrLow); wqe 9335 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.tus.w = le32_to_cpu(bpl->tus.w); wqe 9336 drivers/scsi/lpfc/lpfc_sli.c xmit_len = wqe->generic.bde.tus.f.bdeSize; wqe 9362 drivers/scsi/lpfc/lpfc_sli.c wqe->els_req.payload_len = xmit_len; wqe 9364 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_tmo, &wqe->els_req.wqe_com, wqe 9367 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_req64_vf, &wqe->els_req, 0); wqe 9369 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_req64_vfid, &wqe->els_req, 0); wqe 9371 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, wqe 9373 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->els_req.wqe_com, ct); wqe 9374 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pu, &wqe->els_req.wqe_com, 0); wqe 9390 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_req64_sp, &wqe->els_req, 1); wqe 9391 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_req64_sid, &wqe->els_req, wqe 9396 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_req64_sid, &wqe->els_req, 0); wqe 9397 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->els_req.wqe_com, 1); wqe 9398 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, wqe 9401 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->els_req.wqe_com, 0); wqe 9402 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, wqe 9406 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_temp_rpi, &wqe->els_req.wqe_com, wqe 9408 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_els_id, &wqe->els_req.wqe_com, els_id); wqe 9409 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->els_req.wqe_com, 1); wqe 9410 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->els_req.wqe_com, LPFC_WQE_IOD_READ); wqe 9411 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_qosd, &wqe->els_req.wqe_com, 1); wqe 9412 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->els_req.wqe_com, LPFC_WQE_LENLOC_NONE); wqe 9413 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ebde_cnt, &wqe->els_req.wqe_com, 0); wqe 9414 drivers/scsi/lpfc/lpfc_sli.c wqe->els_req.max_response_payload_len = total_len - xmit_len; wqe 9417 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, wqe 9419 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, wqe 9425 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xo, &wqe->xmit_sequence.wge_ctl, 1); wqe 9429 drivers/scsi/lpfc/lpfc_sli.c wqe->xmit_sequence.rsvd3 = 0; wqe 9432 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0); wqe 9433 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1); wqe 9434 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, wqe 9436 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com, wqe 9438 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0); wqe 9439 drivers/scsi/lpfc/lpfc_sli.c wqe->xmit_sequence.xmit_len = xmit_len; wqe 9444 drivers/scsi/lpfc/lpfc_sli.c wqe->xmit_bcast64.seq_payload_len = xmit_len; wqe 9448 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->xmit_bcast64.wqe_com, wqe 9450 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->xmit_bcast64.wqe_com, 1); wqe 9451 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->xmit_bcast64.wqe_com, LPFC_WQE_IOD_WRITE); wqe 9452 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->xmit_bcast64.wqe_com, wqe 9454 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ebde_cnt, &wqe->xmit_bcast64.wqe_com, 0); wqe 9460 drivers/scsi/lpfc/lpfc_sli.c bf_set(payload_offset_len, &wqe->fcp_iwrite, wqe 9462 drivers/scsi/lpfc/lpfc_sli.c bf_set(cmd_buff_len, &wqe->fcp_iwrite, wqe 9466 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_erp, &wqe->fcp_iwrite.wqe_com, wqe 9468 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lnk, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpXS); wqe 9470 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); wqe 9471 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, wqe 9473 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpPU); wqe 9474 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 1); wqe 9476 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1); wqe 9477 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1); wqe 9479 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com, wqe 9482 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com, wqe 9490 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); wqe 9492 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0); wqe 9507 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.tus.f.bdeFlags = wqe 9509 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.tus.f.bdeSize = sgl->sge_len; wqe 9510 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrHigh = 0; wqe 9511 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrLow = 88; /* Word 22 */ wqe 9513 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); wqe 9514 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); wqe 9517 drivers/scsi/lpfc/lpfc_sli.c ptr = &wqe->words[22]; wqe 9524 drivers/scsi/lpfc/lpfc_sli.c bf_set(payload_offset_len, &wqe->fcp_iread, wqe 9526 drivers/scsi/lpfc/lpfc_sli.c bf_set(cmd_buff_len, &wqe->fcp_iread, wqe 9530 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_erp, &wqe->fcp_iread.wqe_com, wqe 9532 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lnk, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpXS); wqe 9534 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); wqe 9535 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, wqe 9537 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpPU); wqe 9538 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 1); wqe 9540 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1); wqe 9541 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccpe, &wqe->fcp_iread.wqe_com, 1); wqe 9543 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com, wqe 9546 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com, wqe 9554 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); wqe 9556 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0); wqe 9571 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.tus.f.bdeFlags = wqe 9573 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.tus.f.bdeSize = sgl->sge_len; wqe 9574 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrHigh = 0; wqe 9575 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrLow = 88; /* Word 22 */ wqe 9577 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); wqe 9578 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); wqe 9581 drivers/scsi/lpfc/lpfc_sli.c ptr = &wqe->words[22]; wqe 9588 drivers/scsi/lpfc/lpfc_sli.c bf_set(payload_offset_len, &wqe->fcp_icmd, wqe 9590 drivers/scsi/lpfc/lpfc_sli.c bf_set(cmd_buff_len, &wqe->fcp_icmd, wqe 9593 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); wqe 9595 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 1); wqe 9596 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE); wqe 9597 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); wqe 9598 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, wqe 9600 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_erp, &wqe->fcp_icmd.wqe_com, wqe 9603 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1); wqe 9604 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccpe, &wqe->fcp_icmd.wqe_com, 1); wqe 9606 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com, wqe 9609 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com, wqe 9628 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.tus.f.bdeFlags = wqe 9630 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.tus.f.bdeSize = sgl->sge_len; wqe 9631 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrHigh = 0; wqe 9632 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.bde.addrLow = 88; /* Word 22 */ wqe 9634 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); wqe 9635 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); wqe 9638 drivers/scsi/lpfc/lpfc_sli.c ptr = &wqe->words[22]; wqe 9656 drivers/scsi/lpfc/lpfc_sli.c wqe->gen_req.request_payload_len = xmit_len; wqe 9667 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->gen_req.wqe_com, 0); wqe 9668 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_tmo, &wqe->gen_req.wqe_com, iocbq->iocb.ulpTimeout); wqe 9669 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pu, &wqe->gen_req.wqe_com, iocbq->iocb.ulpPU); wqe 9670 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); wqe 9671 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); wqe 9672 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); wqe 9673 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); wqe 9674 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); wqe 9675 drivers/scsi/lpfc/lpfc_sli.c wqe->gen_req.max_response_payload_len = total_len - xmit_len; wqe 9682 drivers/scsi/lpfc/lpfc_sli.c wqe->xmit_els_rsp.response_payload_len = xmit_len; wqe 9684 drivers/scsi/lpfc/lpfc_sli.c wqe->xmit_els_rsp.word4 = 0; wqe 9686 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest, wqe 9693 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1); wqe 9694 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, wqe 9698 drivers/scsi/lpfc/lpfc_sli.c &wqe->xmit_els_rsp.wqe_dest, 0); wqe 9702 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, wqe 9704 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_pu, &wqe->xmit_els_rsp.wqe_com, iocbq->iocb.ulpPU); wqe 9705 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_rcvoxid, &wqe->xmit_els_rsp.wqe_com, wqe 9708 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com, wqe 9710 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->xmit_els_rsp.wqe_com, 1); wqe 9711 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_iod, &wqe->xmit_els_rsp.wqe_com, LPFC_WQE_IOD_WRITE); wqe 9712 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_qosd, &wqe->xmit_els_rsp.wqe_com, 1); wqe 9713 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->xmit_els_rsp.wqe_com, wqe 9715 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ebde_cnt, &wqe->xmit_els_rsp.wqe_com, 0); wqe 9716 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_rsp_temp_rpi, &wqe->xmit_els_rsp, wqe 9721 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1); wqe 9722 drivers/scsi/lpfc/lpfc_sli.c bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, wqe 9724 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, 1); wqe 9725 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com, wqe 9748 drivers/scsi/lpfc/lpfc_sli.c bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); wqe 9750 drivers/scsi/lpfc/lpfc_sli.c bf_set(abort_cmd_ia, &wqe->abort_cmd, 0); wqe 9751 drivers/scsi/lpfc/lpfc_sli.c bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); wqe 9753 drivers/scsi/lpfc/lpfc_sli.c wqe->abort_cmd.rsrvd5 = 0; wqe 9754 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->abort_cmd.wqe_com, wqe 9761 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); wqe 9762 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); wqe 9763 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, wqe 9775 drivers/scsi/lpfc/lpfc_sli.c memset(wqe, 0, sizeof(*wqe)); wqe 9777 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_oxid, &wqe->xmit_bls_rsp, wqe 9785 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp, wqe 9792 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp, wqe 9795 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_seqcnthi, &wqe->xmit_bls_rsp, 0xffff); wqe 9796 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xmit_bls_pt, &wqe->xmit_bls_rsp.wqe_dest, 0x1); wqe 9799 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_els_did, &wqe->xmit_bls_rsp.wqe_dest, wqe 9801 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_temprpi, &wqe->xmit_bls_rsp, wqe 9803 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ct, &wqe->xmit_bls_rsp.wqe_com, 1); wqe 9804 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_ctxt_tag, &wqe->xmit_bls_rsp.wqe_com, wqe 9806 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_qosd, &wqe->xmit_bls_rsp.wqe_com, 1); wqe 9807 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->xmit_bls_rsp.wqe_com, wqe 9812 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_rjt_vspec, &wqe->xmit_bls_rsp, wqe 9814 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_rjt_expc, &wqe->xmit_bls_rsp, wqe 9816 drivers/scsi/lpfc/lpfc_sli.c bf_set(xmit_bls_rsp64_rjt_rsnc, &wqe->xmit_bls_rsp, wqe 9822 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_SEND_FRAME); wqe 9823 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_sof, &wqe->generic.wqe_com, 0x2E); /* SOF byte */ wqe 9824 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_eof, &wqe->generic.wqe_com, 0x41); /* EOF byte */ wqe 9825 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_lenloc, &wqe->generic.wqe_com, 1); wqe 9826 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xbl, &wqe->generic.wqe_com, 1); wqe 9827 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); wqe 9828 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xc, &wqe->generic.wqe_com, 1); wqe 9829 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cmd_type, &wqe->generic.wqe_com, 0xA); wqe 9830 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 9831 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag); wqe 9832 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag); wqe 9849 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_PASSTHRU); wqe 9851 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_STRIP); wqe 9853 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_INSERT); wqe 9856 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag); wqe 9857 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag); wqe 9858 drivers/scsi/lpfc/lpfc_sli.c wqe->generic.wqe_com.abort_tag = abort_tag; wqe 9859 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cmd_type, &wqe->generic.wqe_com, command_type); wqe 9860 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cmnd, &wqe->generic.wqe_com, cmnd); wqe 9861 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_class, &wqe->generic.wqe_com, iocbq->iocb.ulpClass); wqe 9862 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); wqe 9885 drivers/scsi/lpfc/lpfc_sli.c union lpfc_wqe128 wqe; wqe 9952 drivers/scsi/lpfc/lpfc_sli.c if (lpfc_sli4_iocb2wqe(phba, piocb, &wqe)) wqe 9955 drivers/scsi/lpfc/lpfc_sli.c if (lpfc_sli4_wq_put(wq, &wqe)) wqe 17929 drivers/scsi/lpfc/lpfc_sli.c union lpfc_wqe *wqe; wqe 17979 drivers/scsi/lpfc/lpfc_sli.c wqe = (union lpfc_wqe *)&iocbq->iocb; wqe 17981 drivers/scsi/lpfc/lpfc_sli.c wqe->send_frame.frame_len = frame_len; wqe 17982 drivers/scsi/lpfc/lpfc_sli.c wqe->send_frame.fc_hdr_wd0 = be32_to_cpu(*((uint32_t *)fc_hdr)); wqe 17983 drivers/scsi/lpfc/lpfc_sli.c wqe->send_frame.fc_hdr_wd1 = be32_to_cpu(*((uint32_t *)fc_hdr + 1)); wqe 17984 drivers/scsi/lpfc/lpfc_sli.c wqe->send_frame.fc_hdr_wd2 = be32_to_cpu(*((uint32_t *)fc_hdr + 2)); wqe 17985 drivers/scsi/lpfc/lpfc_sli.c wqe->send_frame.fc_hdr_wd3 = be32_to_cpu(*((uint32_t *)fc_hdr + 3)); wqe 17986 drivers/scsi/lpfc/lpfc_sli.c wqe->send_frame.fc_hdr_wd4 = be32_to_cpu(*((uint32_t *)fc_hdr + 4)); wqe 17987 drivers/scsi/lpfc/lpfc_sli.c wqe->send_frame.fc_hdr_wd5 = be32_to_cpu(*((uint32_t *)fc_hdr + 5)); wqe 19605 drivers/scsi/lpfc/lpfc_sli.c union lpfc_wqe128 wqe; wqe 19661 drivers/scsi/lpfc/lpfc_sli.c else if (lpfc_sli4_iocb2wqe(phba, piocbq, &wqe)) wqe 19663 drivers/scsi/lpfc/lpfc_sli.c else if (lpfc_sli4_wq_put(wq, &wqe)) wqe 19713 drivers/scsi/lpfc/lpfc_sli.c union lpfc_wqe128 *wqe; wqe 19724 drivers/scsi/lpfc/lpfc_sli.c wqe = &pwqeq->wqe; wqe 19727 drivers/scsi/lpfc/lpfc_sli.c cmd = bf_get(wqe_cmnd, &wqe->generic.wqe_com); wqe 19797 drivers/scsi/lpfc/lpfc_sli.c } else if (wqe->gen_req.bde.tus.f.bdeFlags == BUFF_TYPE_BDE_64) { wqe 19802 drivers/scsi/lpfc/lpfc_sli.c sgl->addr_hi = cpu_to_le32(wqe->gen_req.bde.addrHigh); wqe 19803 drivers/scsi/lpfc/lpfc_sli.c sgl->addr_lo = cpu_to_le32(wqe->gen_req.bde.addrLow); wqe 19807 drivers/scsi/lpfc/lpfc_sli.c sgl->sge_len = cpu_to_le32(wqe->gen_req.bde.tus.f.bdeSize); wqe 19822 drivers/scsi/lpfc/lpfc_sli.c union lpfc_wqe128 *wqe = &pwqe->wqe; wqe 19846 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com, wqe 19848 drivers/scsi/lpfc/lpfc_sli.c ret = lpfc_sli4_wq_put(phba->sli4_hba.nvmels_wq, wqe); wqe 19867 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); wqe 19871 drivers/scsi/lpfc/lpfc_sli.c ret = lpfc_sli4_wq_put(wq, wqe); wqe 19895 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com, wqe 19897 drivers/scsi/lpfc/lpfc_sli.c bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); wqe 19901 drivers/scsi/lpfc/lpfc_sli.c ret = lpfc_sli4_wq_put(wq, wqe); wqe 69 drivers/scsi/lpfc/lpfc_sli.h union lpfc_wqe128 wqe; /* SLI-4 */ wqe 1350 drivers/staging/octeon/octeon-stubs.h cvmx_wqe_t *wqe = (void *)(unsigned long)scr_addr; wqe 1352 drivers/staging/octeon/octeon-stubs.h return wqe; wqe 639 include/linux/mlx5/device.h } __packed wqe; wqe 265 include/rdma/rdma_vt.h int (*setup_wqe)(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 582 include/rdma/rdmavt_qp.h static inline void rvt_put_swqe(struct rvt_swqe *wqe) wqe 586 include/rdma/rdmavt_qp.h for (i = 0; i < wqe->wr.num_sge; i++) { wqe 587 include/rdma/rdmavt_qp.h struct rvt_sge *sge = &wqe->sg_list[i]; wqe 603 include/rdma/rdmavt_qp.h struct rvt_swqe *wqe) wqe 668 include/rdma/rdmavt_qp.h u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len); wqe 763 include/rdma/rdmavt_qp.h static inline void rvt_put_qp_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) wqe 765 include/rdma/rdmavt_qp.h rvt_put_swqe(wqe); wqe 767 include/rdma/rdmavt_qp.h rdma_destroy_ah_attr(wqe->ud_wr.attr); wqe 845 include/rdma/rdmavt_qp.h struct rvt_swqe *wqe, wqe 852 include/rdma/rdmavt_qp.h int flags = wqe->wr.send_flags; wqe 855 include/rdma/rdmavt_qp.h rvt_put_qp_swqe(qp, wqe); wqe 863 include/rdma/rdmavt_qp.h wr_id = wqe->wr.wr_id; wqe 864 include/rdma/rdmavt_qp.h byte_len = wqe->length; wqe 903 include/rdma/rdmavt_qp.h void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, wqe 997 include/rdma/rdmavt_qp.h static inline bool rvt_rc_credit_avail(struct rvt_qp *qp, struct rvt_swqe *wqe) wqe 1001 include/rdma/rdmavt_qp.h rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {