82 static int qib_init_sge(struct qib_qp *qp, struct qib_rwqe *wqe)  in qib_init_sge()  argument
90 	rkt = &to_idev(qp->ibqp.device)->lk_table;  in qib_init_sge()
91 	pd = to_ipd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd);  in qib_init_sge()
92 	ss = &qp->r_sge;  in qib_init_sge()
93 	ss->sg_list = qp->r_sg_list;  in qib_init_sge()
94 	qp->r_len = 0;  in qib_init_sge()
102 		qp->r_len += wqe->sg_list[i].length;  in qib_init_sge()
106 	ss->total_len = qp->r_len;  in qib_init_sge()
121 	wc.qp = &qp->ibqp;  in qib_init_sge()
123 	qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);  in qib_init_sge()
139 int qib_get_rwqe(struct qib_qp *qp, int wr_id_only)  in qib_get_rwqe()  argument
150 	if (qp->ibqp.srq) {  in qib_get_rwqe()
151 		srq = to_isrq(qp->ibqp.srq);  in qib_get_rwqe()
157 		rq = &qp->r_rq;  in qib_get_rwqe()
161 	if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) {  in qib_get_rwqe()
186 	if (!wr_id_only && !qib_init_sge(qp, wqe)) {  in qib_get_rwqe()
190 	qp->r_wr_id = wqe->wr_id;  in qib_get_rwqe()
193 	set_bit(QIB_R_WRID_VALID, &qp->r_aflags);  in qib_get_rwqe()
213 			ev.device = qp->ibqp.device;  in qib_get_rwqe()
214 			ev.element.srq = qp->ibqp.srq;  in qib_get_rwqe()
230 void qib_migrate_qp(struct qib_qp *qp)  in qib_migrate_qp()  argument
234 	qp->s_mig_state = IB_MIG_MIGRATED;  in qib_migrate_qp()
235 	qp->remote_ah_attr = qp->alt_ah_attr;  in qib_migrate_qp()
236 	qp->port_num = qp->alt_ah_attr.port_num;  in qib_migrate_qp()
237 	qp->s_pkey_index = qp->s_alt_pkey_index;  in qib_migrate_qp()
239 	ev.device = qp->ibqp.device;  in qib_migrate_qp()
240 	ev.element.qp = &qp->ibqp;  in qib_migrate_qp()
242 	qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);  in qib_migrate_qp()
269 		      int has_grh, struct qib_qp *qp, u32 bth0)  in qib_ruc_check_hdr()  argument
274 	if (qp->s_mig_state == IB_MIG_ARMED && (bth0 & IB_BTH_MIG_REQ)) {  in qib_ruc_check_hdr()
276 			if (qp->alt_ah_attr.ah_flags & IB_AH_GRH)  in qib_ruc_check_hdr()
279 			if (!(qp->alt_ah_attr.ah_flags & IB_AH_GRH))  in qib_ruc_check_hdr()
281 			guid = get_sguid(ibp, qp->alt_ah_attr.grh.sgid_index);  in qib_ruc_check_hdr()
285 			    qp->alt_ah_attr.grh.dgid.global.subnet_prefix,  in qib_ruc_check_hdr()
286 			    qp->alt_ah_attr.grh.dgid.global.interface_id))  in qib_ruc_check_hdr()
290 				 qib_get_pkey(ibp, qp->s_alt_pkey_index))) {  in qib_ruc_check_hdr()
294 				      0, qp->ibqp.qp_num,  in qib_ruc_check_hdr()
299 		if (be16_to_cpu(hdr->lrh[3]) != qp->alt_ah_attr.dlid ||  in qib_ruc_check_hdr()
300 		    ppd_from_ibp(ibp)->port != qp->alt_ah_attr.port_num)  in qib_ruc_check_hdr()
302 		spin_lock_irqsave(&qp->s_lock, flags);  in qib_ruc_check_hdr()
303 		qib_migrate_qp(qp);  in qib_ruc_check_hdr()
304 		spin_unlock_irqrestore(&qp->s_lock, flags);  in qib_ruc_check_hdr()
307 			if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)  in qib_ruc_check_hdr()
310 			if (!(qp->remote_ah_attr.ah_flags & IB_AH_GRH))  in qib_ruc_check_hdr()
313 					 qp->remote_ah_attr.grh.sgid_index);  in qib_ruc_check_hdr()
317 			    qp->remote_ah_attr.grh.dgid.global.subnet_prefix,  in qib_ruc_check_hdr()
318 			    qp->remote_ah_attr.grh.dgid.global.interface_id))  in qib_ruc_check_hdr()
322 				 qib_get_pkey(ibp, qp->s_pkey_index))) {  in qib_ruc_check_hdr()
326 				      0, qp->ibqp.qp_num,  in qib_ruc_check_hdr()
331 		if (be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid ||  in qib_ruc_check_hdr()
332 		    ppd_from_ibp(ibp)->port != qp->port_num)  in qib_ruc_check_hdr()
334 		if (qp->s_mig_state == IB_MIG_REARM &&  in qib_ruc_check_hdr()
336 			qp->s_mig_state = IB_MIG_ARMED;  in qib_ruc_check_hdr()
359 	struct qib_qp *qp;  in qib_ruc_loopback()  local
374 	qp = qib_lookup_qpn(ibp, sqp->remote_qpn);  in qib_ruc_loopback()
410 	if (!qp || !(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) ||  in qib_ruc_loopback()
411 	    qp->ibqp.qp_type != sqp->ibqp.qp_type) {  in qib_ruc_loopback()
438 		ret = qib_get_rwqe(qp, 0);  in qib_ruc_loopback()
446 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))  in qib_ruc_loopback()
450 		ret = qib_get_rwqe(qp, 1);  in qib_ruc_loopback()
457 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))  in qib_ruc_loopback()
461 		if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, wqe->length,  in qib_ruc_loopback()
466 		qp->r_sge.sg_list = NULL;  in qib_ruc_loopback()
467 		qp->r_sge.num_sge = 1;  in qib_ruc_loopback()
468 		qp->r_sge.total_len = wqe->length;  in qib_ruc_loopback()
472 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))  in qib_ruc_loopback()
474 		if (unlikely(!qib_rkey_ok(qp, &sqp->s_sge.sge, wqe->length,  in qib_ruc_loopback()
482 		qp->r_sge.sge = wqe->sg_list[0];  in qib_ruc_loopback()
483 		qp->r_sge.sg_list = wqe->sg_list + 1;  in qib_ruc_loopback()
484 		qp->r_sge.num_sge = wqe->wr.num_sge;  in qib_ruc_loopback()
485 		qp->r_sge.total_len = wqe->length;  in qib_ruc_loopback()
490 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))  in qib_ruc_loopback()
492 		if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),  in qib_ruc_loopback()
498 		maddr = (atomic64_t *) qp->r_sge.sge.vaddr;  in qib_ruc_loopback()
503 			(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,  in qib_ruc_loopback()
505 		qib_put_mr(qp->r_sge.sge.mr);  in qib_ruc_loopback()
506 		qp->r_sge.num_sge = 0;  in qib_ruc_loopback()
523 		qib_copy_sge(&qp->r_sge, sge->vaddr, len, release);  in qib_ruc_loopback()
546 		qib_put_ss(&qp->r_sge);  in qib_ruc_loopback()
548 	if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags))  in qib_ruc_loopback()
555 	wc.wr_id = qp->r_wr_id;  in qib_ruc_loopback()
558 	wc.qp = &qp->ibqp;  in qib_ruc_loopback()
559 	wc.src_qp = qp->remote_qpn;  in qib_ruc_loopback()
560 	wc.slid = qp->remote_ah_attr.dlid;  in qib_ruc_loopback()
561 	wc.sl = qp->remote_ah_attr.sl;  in qib_ruc_loopback()
564 	qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,  in qib_ruc_loopback()
577 	if (qp->ibqp.qp_type == IB_QPT_UC)  in qib_ruc_loopback()
596 		usecs_to_jiffies(ib_qib_rnr_table[qp->r_min_rnr_timer]);  in qib_ruc_loopback()
615 	qib_rc_error(qp, wc.status);  in qib_ruc_loopback()
629 			ev.element.qp = &sqp->ibqp;  in qib_ruc_loopback()
640 	if (qp && atomic_dec_and_test(&qp->refcount))  in qib_ruc_loopback()
641 		wake_up(&qp->wait);  in qib_ruc_loopback()
675 void qib_make_ruc_header(struct qib_qp *qp, struct qib_other_headers *ohdr,  in qib_make_ruc_header()  argument
678 	struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);  in qib_make_ruc_header()
684 	extra_bytes = -qp->s_cur_size & 3;  in qib_make_ruc_header()
685 	nwords = (qp->s_cur_size + extra_bytes) >> 2;  in qib_make_ruc_header()
687 	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {  in qib_make_ruc_header()
688 		qp->s_hdrwords += qib_make_grh(ibp, &qp->s_hdr->u.l.grh,  in qib_make_ruc_header()
689 					       &qp->remote_ah_attr.grh,  in qib_make_ruc_header()
690 					       qp->s_hdrwords, nwords);  in qib_make_ruc_header()
693 	lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 |  in qib_make_ruc_header()
694 		qp->remote_ah_attr.sl << 4;  in qib_make_ruc_header()
695 	qp->s_hdr->lrh[0] = cpu_to_be16(lrh0);  in qib_make_ruc_header()
696 	qp->s_hdr->lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);  in qib_make_ruc_header()
697 	qp->s_hdr->lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);  in qib_make_ruc_header()
698 	qp->s_hdr->lrh[3] = cpu_to_be16(ppd_from_ibp(ibp)->lid |  in qib_make_ruc_header()
699 				       qp->remote_ah_attr.src_path_bits);  in qib_make_ruc_header()
700 	bth0 |= qib_get_pkey(ibp, qp->s_pkey_index);  in qib_make_ruc_header()
702 	if (qp->s_mig_state == IB_MIG_MIGRATED)  in qib_make_ruc_header()
705 	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);  in qib_make_ruc_header()
720 	struct qib_qp *qp = container_of(work, struct qib_qp, s_work);  in qib_do_send()  local
721 	struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);  in qib_do_send()
723 	int (*make_req)(struct qib_qp *qp);  in qib_do_send()
726 	if ((qp->ibqp.qp_type == IB_QPT_RC ||  in qib_do_send()
727 	     qp->ibqp.qp_type == IB_QPT_UC) &&  in qib_do_send()
728 	    (qp->remote_ah_attr.dlid & ~((1 << ppd->lmc) - 1)) == ppd->lid) {  in qib_do_send()
729 		qib_ruc_loopback(qp);  in qib_do_send()
733 	if (qp->ibqp.qp_type == IB_QPT_RC)  in qib_do_send()
735 	else if (qp->ibqp.qp_type == IB_QPT_UC)  in qib_do_send()
740 	spin_lock_irqsave(&qp->s_lock, flags);  in qib_do_send()
743 	if (!qib_send_ok(qp)) {  in qib_do_send()
744 		spin_unlock_irqrestore(&qp->s_lock, flags);  in qib_do_send()
748 	qp->s_flags |= QIB_S_BUSY;  in qib_do_send()
750 	spin_unlock_irqrestore(&qp->s_lock, flags);  in qib_do_send()
754 		if (qp->s_hdrwords != 0) {  in qib_do_send()
759 			if (qib_verbs_send(qp, qp->s_hdr, qp->s_hdrwords,  in qib_do_send()
760 					   qp->s_cur_sge, qp->s_cur_size))  in qib_do_send()
763 			qp->s_hdrwords = 0;  in qib_do_send()
765 	} while (make_req(qp));  in qib_do_send()
771 void qib_send_complete(struct qib_qp *qp, struct qib_swqe *wqe,  in qib_send_complete()  argument
777 	if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_OR_FLUSH_SEND))  in qib_send_complete()
785 	if (qp->ibqp.qp_type == IB_QPT_UD ||  in qib_send_complete()
786 	    qp->ibqp.qp_type == IB_QPT_SMI ||  in qib_send_complete()
787 	    qp->ibqp.qp_type == IB_QPT_GSI)  in qib_send_complete()
791 	if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) ||  in qib_send_complete()
800 		wc.qp = &qp->ibqp;  in qib_send_complete()
803 		qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc,  in qib_send_complete()
807 	last = qp->s_last;  in qib_send_complete()
809 	if (++last >= qp->s_size)  in qib_send_complete()
811 	qp->s_last = last;  in qib_send_complete()
812 	if (qp->s_acked == old_last)  in qib_send_complete()
813 		qp->s_acked = last;  in qib_send_complete()
814 	if (qp->s_cur == old_last)  in qib_send_complete()
815 		qp->s_cur = last;  in qib_send_complete()
816 	if (qp->s_tail == old_last)  in qib_send_complete()
817 		qp->s_tail = last;  in qib_send_complete()
818 	if (qp->state == IB_QPS_SQD && last == qp->s_cur)  in qib_send_complete()
819 		qp->s_draining = 0;  in qib_send_complete()