Lines Matching refs:wqe
42 static u32 restart_sge(struct ipath_sge_state *ss, struct ipath_swqe *wqe, in restart_sge() argument
47 len = ((psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; in restart_sge()
48 ss->sge = wqe->sg_list[0]; in restart_sge()
49 ss->sg_list = wqe->sg_list + 1; in restart_sge()
50 ss->num_sge = wqe->wr.num_sge; in restart_sge()
52 return wqe->length - len; in restart_sge()
62 static void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe) in ipath_init_restart() argument
66 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, in ipath_init_restart()
218 struct ipath_swqe *wqe; in ipath_make_rc_req() local
256 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_make_rc_req()
257 ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); in ipath_make_rc_req()
272 wqe = get_swqe_ptr(qp, qp->s_cur); in ipath_make_rc_req()
294 if ((wqe->wr.send_flags & IB_SEND_FENCE) && in ipath_make_rc_req()
299 wqe->psn = qp->s_next_psn; in ipath_make_rc_req()
307 len = wqe->length; in ipath_make_rc_req()
310 switch (wqe->wr.opcode) { in ipath_make_rc_req()
315 ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { in ipath_make_rc_req()
319 wqe->lpsn = wqe->psn; in ipath_make_rc_req()
321 wqe->lpsn += (len - 1) / pmtu; in ipath_make_rc_req()
326 if (wqe->wr.opcode == IB_WR_SEND) in ipath_make_rc_req()
331 ohdr->u.imm_data = wqe->wr.ex.imm_data; in ipath_make_rc_req()
334 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in ipath_make_rc_req()
348 ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { in ipath_make_rc_req()
353 cpu_to_be64(wqe->wr.wr.rdma.remote_addr); in ipath_make_rc_req()
355 cpu_to_be32(wqe->wr.wr.rdma.rkey); in ipath_make_rc_req()
358 wqe->lpsn = wqe->psn; in ipath_make_rc_req()
360 wqe->lpsn += (len - 1) / pmtu; in ipath_make_rc_req()
365 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) in ipath_make_rc_req()
371 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; in ipath_make_rc_req()
373 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in ipath_make_rc_req()
401 wqe->lpsn = qp->s_next_psn++; in ipath_make_rc_req()
404 cpu_to_be64(wqe->wr.wr.rdma.remote_addr); in ipath_make_rc_req()
406 cpu_to_be32(wqe->wr.wr.rdma.rkey); in ipath_make_rc_req()
431 wqe->lpsn = wqe->psn; in ipath_make_rc_req()
433 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { in ipath_make_rc_req()
436 wqe->wr.wr.atomic.swap); in ipath_make_rc_req()
438 wqe->wr.wr.atomic.compare_add); in ipath_make_rc_req()
442 wqe->wr.wr.atomic.compare_add); in ipath_make_rc_req()
446 wqe->wr.wr.atomic.remote_addr >> 32); in ipath_make_rc_req()
448 wqe->wr.wr.atomic.remote_addr); in ipath_make_rc_req()
450 wqe->wr.wr.atomic.rkey); in ipath_make_rc_req()
461 qp->s_sge.sge = wqe->sg_list[0]; in ipath_make_rc_req()
462 qp->s_sge.sg_list = wqe->sg_list + 1; in ipath_make_rc_req()
463 qp->s_sge.num_sge = wqe->wr.num_sge; in ipath_make_rc_req()
464 qp->s_len = wqe->length; in ipath_make_rc_req()
471 if (wqe->wr.opcode == IB_WR_RDMA_READ) in ipath_make_rc_req()
472 qp->s_psn = wqe->lpsn + 1; in ipath_make_rc_req()
495 ipath_init_restart(qp, wqe); in ipath_make_rc_req()
510 if (wqe->wr.opcode == IB_WR_SEND) in ipath_make_rc_req()
515 ohdr->u.imm_data = wqe->wr.ex.imm_data; in ipath_make_rc_req()
518 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in ipath_make_rc_req()
531 ipath_init_restart(qp, wqe); in ipath_make_rc_req()
546 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) in ipath_make_rc_req()
551 ohdr->u.imm_data = wqe->wr.ex.imm_data; in ipath_make_rc_req()
553 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in ipath_make_rc_req()
567 ipath_init_restart(qp, wqe); in ipath_make_rc_req()
568 len = ((qp->s_psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; in ipath_make_rc_req()
570 cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len); in ipath_make_rc_req()
572 cpu_to_be32(wqe->wr.wr.rdma.rkey); in ipath_make_rc_req()
577 qp->s_psn = wqe->lpsn + 1; in ipath_make_rc_req()
727 struct ipath_swqe *wqe = get_swqe_ptr(qp, n); in reset_psn() local
736 if (ipath_cmp24(psn, wqe->psn) <= 0) { in reset_psn()
742 opcode = wqe->wr.opcode; in reset_psn()
750 wqe = get_swqe_ptr(qp, n); in reset_psn()
751 diff = ipath_cmp24(psn, wqe->psn); in reset_psn()
763 opcode = wqe->wr.opcode; in reset_psn()
807 struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last); in ipath_restart_rc() local
811 ipath_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in ipath_restart_rc()
829 if (wqe->wr.opcode == IB_WR_RDMA_READ) in ipath_restart_rc()
863 struct ipath_swqe *wqe; in do_rc_ack() local
888 wqe = get_swqe_ptr(qp, qp->s_last); in do_rc_ack()
894 while ((diff = ipath_cmp24(ack_psn, wqe->lpsn)) >= 0) { in do_rc_ack()
901 if (wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
916 if ((wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
918 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
919 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && in do_rc_ack()
925 update_last_psn(qp, wqe->psn - 1); in do_rc_ack()
927 ipath_restart_rc(qp, wqe->psn); in do_rc_ack()
934 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
935 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) in do_rc_ack()
936 *(u64 *) wqe->sg_list[0].vaddr = val; in do_rc_ack()
938 (wqe->wr.opcode == IB_WR_RDMA_READ || in do_rc_ack()
939 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
940 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { in do_rc_ack()
950 (wqe->wr.send_flags & IB_SEND_SIGNALED)) { in do_rc_ack()
952 wc.wr_id = wqe->wr.wr_id; in do_rc_ack()
954 wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; in do_rc_ack()
955 wc.byte_len = wqe->length; in do_rc_ack()
974 wqe = get_swqe_ptr(qp, qp->s_cur); in do_rc_ack()
976 qp->s_psn = wqe->psn; in do_rc_ack()
984 wqe = get_swqe_ptr(qp, qp->s_last); in do_rc_ack()
1032 if (wqe->wr.opcode == IB_WR_RDMA_READ) in do_rc_ack()
1079 ipath_send_complete(qp, wqe, status); in do_rc_ack()
1125 struct ipath_swqe *wqe; in ipath_rc_rcv_resp() local
1162 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_rc_rcv_resp()
1189 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_rc_rcv_resp()
1190 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in ipath_rc_rcv_resp()
1199 wqe, psn, pmtu); in ipath_rc_rcv_resp()
1212 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in ipath_rc_rcv_resp()
1261 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_rc_rcv_resp()
1263 wqe, psn, pmtu); in ipath_rc_rcv_resp()
1276 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in ipath_rc_rcv_resp()
1310 ipath_send_complete(qp, wqe, status); in ipath_rc_rcv_resp()