81 static int qib_init_sge(struct qib_qp *qp, struct qib_rwqe *wqe)  in qib_init_sge()  argument
89 	rkt = &to_idev(qp->ibqp.device)->lk_table;  in qib_init_sge()
90 	pd = to_ipd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd);  in qib_init_sge()
91 	ss = &qp->r_sge;  in qib_init_sge()
92 	ss->sg_list = qp->r_sg_list;  in qib_init_sge()
93 	qp->r_len = 0;  in qib_init_sge()
101 		qp->r_len += wqe->sg_list[i].length;  in qib_init_sge()
105 	ss->total_len = qp->r_len;  in qib_init_sge()
120 	wc.qp = &qp->ibqp;  in qib_init_sge()
122 	qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);  in qib_init_sge()
138 int qib_get_rwqe(struct qib_qp *qp, int wr_id_only)  in qib_get_rwqe()  argument
149 	if (qp->ibqp.srq) {  in qib_get_rwqe()
150 		srq = to_isrq(qp->ibqp.srq);  in qib_get_rwqe()
156 		rq = &qp->r_rq;  in qib_get_rwqe()
160 	if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) {  in qib_get_rwqe()
185 	if (!wr_id_only && !qib_init_sge(qp, wqe)) {  in qib_get_rwqe()
189 	qp->r_wr_id = wqe->wr_id;  in qib_get_rwqe()
192 	set_bit(QIB_R_WRID_VALID, &qp->r_aflags);  in qib_get_rwqe()
212 			ev.device = qp->ibqp.device;  in qib_get_rwqe()
213 			ev.element.srq = qp->ibqp.srq;  in qib_get_rwqe()
229 void qib_migrate_qp(struct qib_qp *qp)  in qib_migrate_qp()  argument
233 	qp->s_mig_state = IB_MIG_MIGRATED;  in qib_migrate_qp()
234 	qp->remote_ah_attr = qp->alt_ah_attr;  in qib_migrate_qp()
235 	qp->port_num = qp->alt_ah_attr.port_num;  in qib_migrate_qp()
236 	qp->s_pkey_index = qp->s_alt_pkey_index;  in qib_migrate_qp()
238 	ev.device = qp->ibqp.device;  in qib_migrate_qp()
239 	ev.element.qp = &qp->ibqp;  in qib_migrate_qp()
241 	qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);  in qib_migrate_qp()
268 		      int has_grh, struct qib_qp *qp, u32 bth0)  in qib_ruc_check_hdr()  argument
273 	if (qp->s_mig_state == IB_MIG_ARMED && (bth0 & IB_BTH_MIG_REQ)) {  in qib_ruc_check_hdr()
275 			if (qp->alt_ah_attr.ah_flags & IB_AH_GRH)  in qib_ruc_check_hdr()
278 			if (!(qp->alt_ah_attr.ah_flags & IB_AH_GRH))  in qib_ruc_check_hdr()
280 			guid = get_sguid(ibp, qp->alt_ah_attr.grh.sgid_index);  in qib_ruc_check_hdr()
284 			    qp->alt_ah_attr.grh.dgid.global.subnet_prefix,  in qib_ruc_check_hdr()
285 			    qp->alt_ah_attr.grh.dgid.global.interface_id))  in qib_ruc_check_hdr()
289 				 qib_get_pkey(ibp, qp->s_alt_pkey_index))) {  in qib_ruc_check_hdr()
293 				      0, qp->ibqp.qp_num,  in qib_ruc_check_hdr()
298 		if (be16_to_cpu(hdr->lrh[3]) != qp->alt_ah_attr.dlid ||  in qib_ruc_check_hdr()
299 		    ppd_from_ibp(ibp)->port != qp->alt_ah_attr.port_num)  in qib_ruc_check_hdr()
301 		spin_lock_irqsave(&qp->s_lock, flags);  in qib_ruc_check_hdr()
302 		qib_migrate_qp(qp);  in qib_ruc_check_hdr()
303 		spin_unlock_irqrestore(&qp->s_lock, flags);  in qib_ruc_check_hdr()
306 			if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)  in qib_ruc_check_hdr()
309 			if (!(qp->remote_ah_attr.ah_flags & IB_AH_GRH))  in qib_ruc_check_hdr()
312 					 qp->remote_ah_attr.grh.sgid_index);  in qib_ruc_check_hdr()
316 			    qp->remote_ah_attr.grh.dgid.global.subnet_prefix,  in qib_ruc_check_hdr()
317 			    qp->remote_ah_attr.grh.dgid.global.interface_id))  in qib_ruc_check_hdr()
321 				 qib_get_pkey(ibp, qp->s_pkey_index))) {  in qib_ruc_check_hdr()
325 				      0, qp->ibqp.qp_num,  in qib_ruc_check_hdr()
330 		if (be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid ||  in qib_ruc_check_hdr()
331 		    ppd_from_ibp(ibp)->port != qp->port_num)  in qib_ruc_check_hdr()
333 		if (qp->s_mig_state == IB_MIG_REARM &&  in qib_ruc_check_hdr()
335 			qp->s_mig_state = IB_MIG_ARMED;  in qib_ruc_check_hdr()
358 	struct qib_qp *qp;  in qib_ruc_loopback()  local
373 	qp = qib_lookup_qpn(ibp, sqp->remote_qpn);  in qib_ruc_loopback()
409 	if (!qp || !(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) ||  in qib_ruc_loopback()
410 	    qp->ibqp.qp_type != sqp->ibqp.qp_type) {  in qib_ruc_loopback()
437 		ret = qib_get_rwqe(qp, 0);  in qib_ruc_loopback()
445 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))  in qib_ruc_loopback()
449 		ret = qib_get_rwqe(qp, 1);  in qib_ruc_loopback()
456 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))  in qib_ruc_loopback()
460 		if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, wqe->length,  in qib_ruc_loopback()
465 		qp->r_sge.sg_list = NULL;  in qib_ruc_loopback()
466 		qp->r_sge.num_sge = 1;  in qib_ruc_loopback()
467 		qp->r_sge.total_len = wqe->length;  in qib_ruc_loopback()
471 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))  in qib_ruc_loopback()
473 		if (unlikely(!qib_rkey_ok(qp, &sqp->s_sge.sge, wqe->length,  in qib_ruc_loopback()
481 		qp->r_sge.sge = wqe->sg_list[0];  in qib_ruc_loopback()
482 		qp->r_sge.sg_list = wqe->sg_list + 1;  in qib_ruc_loopback()
483 		qp->r_sge.num_sge = wqe->wr.num_sge;  in qib_ruc_loopback()
484 		qp->r_sge.total_len = wqe->length;  in qib_ruc_loopback()
489 		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))  in qib_ruc_loopback()
491 		if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),  in qib_ruc_loopback()
497 		maddr = (atomic64_t *) qp->r_sge.sge.vaddr;  in qib_ruc_loopback()
502 			(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,  in qib_ruc_loopback()
504 		qib_put_mr(qp->r_sge.sge.mr);  in qib_ruc_loopback()
505 		qp->r_sge.num_sge = 0;  in qib_ruc_loopback()
522 		qib_copy_sge(&qp->r_sge, sge->vaddr, len, release);  in qib_ruc_loopback()
545 		qib_put_ss(&qp->r_sge);  in qib_ruc_loopback()
547 	if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags))  in qib_ruc_loopback()
554 	wc.wr_id = qp->r_wr_id;  in qib_ruc_loopback()
557 	wc.qp = &qp->ibqp;  in qib_ruc_loopback()
558 	wc.src_qp = qp->remote_qpn;  in qib_ruc_loopback()
559 	wc.slid = qp->remote_ah_attr.dlid;  in qib_ruc_loopback()
560 	wc.sl = qp->remote_ah_attr.sl;  in qib_ruc_loopback()
563 	qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,  in qib_ruc_loopback()
576 	if (qp->ibqp.qp_type == IB_QPT_UC)  in qib_ruc_loopback()
595 		usecs_to_jiffies(ib_qib_rnr_table[qp->r_min_rnr_timer]);  in qib_ruc_loopback()
614 	qib_rc_error(qp, wc.status);  in qib_ruc_loopback()
628 			ev.element.qp = &sqp->ibqp;  in qib_ruc_loopback()
639 	if (qp && atomic_dec_and_test(&qp->refcount))  in qib_ruc_loopback()
640 		wake_up(&qp->wait);  in qib_ruc_loopback()
674 void qib_make_ruc_header(struct qib_qp *qp, struct qib_other_headers *ohdr,  in qib_make_ruc_header()  argument
677 	struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);  in qib_make_ruc_header()
683 	extra_bytes = -qp->s_cur_size & 3;  in qib_make_ruc_header()
684 	nwords = (qp->s_cur_size + extra_bytes) >> 2;  in qib_make_ruc_header()
686 	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {  in qib_make_ruc_header()
687 		qp->s_hdrwords += qib_make_grh(ibp, &qp->s_hdr->u.l.grh,  in qib_make_ruc_header()
688 					       &qp->remote_ah_attr.grh,  in qib_make_ruc_header()
689 					       qp->s_hdrwords, nwords);  in qib_make_ruc_header()
692 	lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 |  in qib_make_ruc_header()
693 		qp->remote_ah_attr.sl << 4;  in qib_make_ruc_header()
694 	qp->s_hdr->lrh[0] = cpu_to_be16(lrh0);  in qib_make_ruc_header()
695 	qp->s_hdr->lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);  in qib_make_ruc_header()
696 	qp->s_hdr->lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);  in qib_make_ruc_header()
697 	qp->s_hdr->lrh[3] = cpu_to_be16(ppd_from_ibp(ibp)->lid |  in qib_make_ruc_header()
698 				       qp->remote_ah_attr.src_path_bits);  in qib_make_ruc_header()
699 	bth0 |= qib_get_pkey(ibp, qp->s_pkey_index);  in qib_make_ruc_header()
701 	if (qp->s_mig_state == IB_MIG_MIGRATED)  in qib_make_ruc_header()
704 	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);  in qib_make_ruc_header()
719 	struct qib_qp *qp = container_of(work, struct qib_qp, s_work);  in qib_do_send()  local
720 	struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);  in qib_do_send()
722 	int (*make_req)(struct qib_qp *qp);  in qib_do_send()
725 	if ((qp->ibqp.qp_type == IB_QPT_RC ||  in qib_do_send()
726 	     qp->ibqp.qp_type == IB_QPT_UC) &&  in qib_do_send()
727 	    (qp->remote_ah_attr.dlid & ~((1 << ppd->lmc) - 1)) == ppd->lid) {  in qib_do_send()
728 		qib_ruc_loopback(qp);  in qib_do_send()
732 	if (qp->ibqp.qp_type == IB_QPT_RC)  in qib_do_send()
734 	else if (qp->ibqp.qp_type == IB_QPT_UC)  in qib_do_send()
739 	spin_lock_irqsave(&qp->s_lock, flags);  in qib_do_send()
742 	if (!qib_send_ok(qp)) {  in qib_do_send()
743 		spin_unlock_irqrestore(&qp->s_lock, flags);  in qib_do_send()
747 	qp->s_flags |= QIB_S_BUSY;  in qib_do_send()
749 	spin_unlock_irqrestore(&qp->s_lock, flags);  in qib_do_send()
753 		if (qp->s_hdrwords != 0) {  in qib_do_send()
758 			if (qib_verbs_send(qp, qp->s_hdr, qp->s_hdrwords,  in qib_do_send()
759 					   qp->s_cur_sge, qp->s_cur_size))  in qib_do_send()
762 			qp->s_hdrwords = 0;  in qib_do_send()
764 	} while (make_req(qp));  in qib_do_send()
770 void qib_send_complete(struct qib_qp *qp, struct qib_swqe *wqe,  in qib_send_complete()  argument
776 	if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_OR_FLUSH_SEND))  in qib_send_complete()
784 	if (qp->ibqp.qp_type == IB_QPT_UD ||  in qib_send_complete()
785 	    qp->ibqp.qp_type == IB_QPT_SMI ||  in qib_send_complete()
786 	    qp->ibqp.qp_type == IB_QPT_GSI)  in qib_send_complete()
790 	if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) ||  in qib_send_complete()
799 		wc.qp = &qp->ibqp;  in qib_send_complete()
802 		qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc,  in qib_send_complete()
806 	last = qp->s_last;  in qib_send_complete()
808 	if (++last >= qp->s_size)  in qib_send_complete()
810 	qp->s_last = last;  in qib_send_complete()
811 	if (qp->s_acked == old_last)  in qib_send_complete()
812 		qp->s_acked = last;  in qib_send_complete()
813 	if (qp->s_cur == old_last)  in qib_send_complete()
814 		qp->s_cur = last;  in qib_send_complete()
815 	if (qp->s_tail == old_last)  in qib_send_complete()
816 		qp->s_tail = last;  in qib_send_complete()
817 	if (qp->state == IB_QPS_SQD && last == qp->s_cur)  in qib_send_complete()
818 		qp->s_draining = 0;  in qib_send_complete()