43 static u32 restart_sge(struct qib_sge_state *ss, struct qib_swqe *wqe,  in restart_sge()  argument
48 	len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu;  in restart_sge()
49 	ss->sge = wqe->sg_list[0];  in restart_sge()
50 	ss->sg_list = wqe->sg_list + 1;  in restart_sge()
51 	ss->num_sge = wqe->wr.num_sge;  in restart_sge()
52 	ss->total_len = wqe->length;  in restart_sge()
54 	return wqe->length - len;  in restart_sge()
236 	struct qib_swqe *wqe;  in qib_make_rc_req()  local
273 		wqe = get_swqe_ptr(qp, qp->s_last);  in qib_make_rc_req()
274 		qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ?  in qib_make_rc_req()
297 	wqe = get_swqe_ptr(qp, qp->s_cur);  in qib_make_rc_req()
318 			if ((wqe->wr.send_flags & IB_SEND_FENCE) &&  in qib_make_rc_req()
323 			wqe->psn = qp->s_next_psn;  in qib_make_rc_req()
331 		len = wqe->length;  in qib_make_rc_req()
334 		switch (wqe->wr.opcode) {  in qib_make_rc_req()
339 			    qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {  in qib_make_rc_req()
343 			wqe->lpsn = wqe->psn;  in qib_make_rc_req()
345 				wqe->lpsn += (len - 1) / pmtu;  in qib_make_rc_req()
350 			if (wqe->wr.opcode == IB_WR_SEND)  in qib_make_rc_req()
355 				ohdr->u.imm_data = wqe->wr.ex.imm_data;  in qib_make_rc_req()
358 			if (wqe->wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
372 			    qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {  in qib_make_rc_req()
378 				cpu_to_be64(wqe->rdma_wr.remote_addr);  in qib_make_rc_req()
380 				cpu_to_be32(wqe->rdma_wr.rkey);  in qib_make_rc_req()
383 			wqe->lpsn = wqe->psn;  in qib_make_rc_req()
385 				wqe->lpsn += (len - 1) / pmtu;  in qib_make_rc_req()
390 			if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE)  in qib_make_rc_req()
396 					wqe->rdma_wr.wr.ex.imm_data;  in qib_make_rc_req()
398 				if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
426 				wqe->lpsn = qp->s_next_psn++;  in qib_make_rc_req()
430 				cpu_to_be64(wqe->rdma_wr.remote_addr);  in qib_make_rc_req()
432 				cpu_to_be32(wqe->rdma_wr.rkey);  in qib_make_rc_req()
458 				wqe->lpsn = wqe->psn;  in qib_make_rc_req()
460 			if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {  in qib_make_rc_req()
463 					wqe->atomic_wr.swap);  in qib_make_rc_req()
465 					wqe->atomic_wr.compare_add);  in qib_make_rc_req()
469 					wqe->atomic_wr.compare_add);  in qib_make_rc_req()
473 				wqe->atomic_wr.remote_addr >> 32);  in qib_make_rc_req()
475 				wqe->atomic_wr.remote_addr);  in qib_make_rc_req()
477 				wqe->atomic_wr.rkey);  in qib_make_rc_req()
489 		qp->s_sge.sge = wqe->sg_list[0];  in qib_make_rc_req()
490 		qp->s_sge.sg_list = wqe->sg_list + 1;  in qib_make_rc_req()
491 		qp->s_sge.num_sge = wqe->wr.num_sge;  in qib_make_rc_req()
492 		qp->s_sge.total_len = wqe->length;  in qib_make_rc_req()
493 		qp->s_len = wqe->length;  in qib_make_rc_req()
499 		if (wqe->wr.opcode == IB_WR_RDMA_READ)  in qib_make_rc_req()
500 			qp->s_psn = wqe->lpsn + 1;  in qib_make_rc_req()
518 		qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);  in qib_make_rc_req()
533 		if (wqe->wr.opcode == IB_WR_SEND)  in qib_make_rc_req()
538 			ohdr->u.imm_data = wqe->wr.ex.imm_data;  in qib_make_rc_req()
541 		if (wqe->wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
559 		qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);  in qib_make_rc_req()
574 		if (wqe->wr.opcode == IB_WR_RDMA_WRITE)  in qib_make_rc_req()
579 			ohdr->u.imm_data = wqe->wr.ex.imm_data;  in qib_make_rc_req()
581 			if (wqe->wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
600 		len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;  in qib_make_rc_req()
602 			cpu_to_be64(wqe->rdma_wr.remote_addr + len);  in qib_make_rc_req()
604 			cpu_to_be32(wqe->rdma_wr.rkey);  in qib_make_rc_req()
605 		ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);  in qib_make_rc_req()
609 		qp->s_psn = wqe->lpsn + 1;  in qib_make_rc_req()
618 	delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8;  in qib_make_rc_req()
788 	struct qib_swqe *wqe = get_swqe_ptr(qp, n);  in reset_psn()  local
797 	if (qib_cmp24(psn, wqe->psn) <= 0) {  in reset_psn()
803 	opcode = wqe->wr.opcode;  in reset_psn()
811 		wqe = get_swqe_ptr(qp, n);  in reset_psn()
812 		diff = qib_cmp24(psn, wqe->psn);  in reset_psn()
824 		opcode = wqe->wr.opcode;  in reset_psn()
872 	struct qib_swqe *wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_restart_rc()  local
880 			qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);  in qib_restart_rc()
889 	if (wqe->wr.opcode == IB_WR_RDMA_READ)  in qib_restart_rc()
948 	struct qib_swqe *wqe;  in reset_sending_psn()  local
953 		wqe = get_swqe_ptr(qp, n);  in reset_sending_psn()
954 		if (qib_cmp24(psn, wqe->lpsn) <= 0) {  in reset_sending_psn()
955 			if (wqe->wr.opcode == IB_WR_RDMA_READ)  in reset_sending_psn()
956 				qp->s_sending_psn = wqe->lpsn + 1;  in reset_sending_psn()
974 	struct qib_swqe *wqe;  in qib_rc_send_complete()  local
1010 		wqe = get_swqe_ptr(qp, qp->s_last);  in qib_rc_send_complete()
1011 		if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&  in qib_rc_send_complete()
1014 		for (i = 0; i < wqe->wr.num_sge; i++) {  in qib_rc_send_complete()
1015 			struct qib_sge *sge = &wqe->sg_list[i];  in qib_rc_send_complete()
1021 		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {  in qib_rc_send_complete()
1023 			wc.wr_id = wqe->wr.wr_id;  in qib_rc_send_complete()
1025 			wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];  in qib_rc_send_complete()
1026 			wc.byte_len = wqe->length;  in qib_rc_send_complete()
1057 					 struct qib_swqe *wqe,  in do_rc_completion()  argument
1068 	if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||  in do_rc_completion()
1070 		for (i = 0; i < wqe->wr.num_sge; i++) {  in do_rc_completion()
1071 			struct qib_sge *sge = &wqe->sg_list[i];  in do_rc_completion()
1077 		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {  in do_rc_completion()
1079 			wc.wr_id = wqe->wr.wr_id;  in do_rc_completion()
1081 			wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];  in do_rc_completion()
1082 			wc.byte_len = wqe->length;  in do_rc_completion()
1092 	update_last_psn(qp, wqe->lpsn);  in do_rc_completion()
1103 		wqe = get_swqe_ptr(qp, qp->s_cur);  in do_rc_completion()
1106 			qp->s_psn = wqe->psn;  in do_rc_completion()
1113 		wqe = get_swqe_ptr(qp, qp->s_acked);  in do_rc_completion()
1115 	return wqe;  in do_rc_completion()
1134 	struct qib_swqe *wqe;  in do_rc_ack()  local
1154 	wqe = get_swqe_ptr(qp, qp->s_acked);  in do_rc_ack()
1161 	while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) {  in do_rc_ack()
1168 		if (wqe->wr.opcode == IB_WR_RDMA_READ &&  in do_rc_ack()
1183 		if ((wqe->wr.opcode == IB_WR_RDMA_READ &&  in do_rc_ack()
1185 		    ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in do_rc_ack()
1186 		      wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&  in do_rc_ack()
1205 		if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in do_rc_ack()
1206 		    wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {  in do_rc_ack()
1207 			u64 *vaddr = wqe->sg_list[0].vaddr;  in do_rc_ack()
1211 		    (wqe->wr.opcode == IB_WR_RDMA_READ ||  in do_rc_ack()
1212 		     wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in do_rc_ack()
1213 		     wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {  in do_rc_ack()
1227 		wqe = do_rc_completion(qp, wqe, ibp);  in do_rc_ack()
1325 				qib_send_complete(qp, wqe, status);  in do_rc_ack()
1355 	struct qib_swqe *wqe;  in rdma_seq_err()  local
1363 	wqe = get_swqe_ptr(qp, qp->s_acked);  in rdma_seq_err()
1365 	while (qib_cmp24(psn, wqe->lpsn) > 0) {  in rdma_seq_err()
1366 		if (wqe->wr.opcode == IB_WR_RDMA_READ ||  in rdma_seq_err()
1367 		    wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in rdma_seq_err()
1368 		    wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)  in rdma_seq_err()
1370 		wqe = do_rc_completion(qp, wqe, ibp);  in rdma_seq_err()
1407 	struct qib_swqe *wqe;  in qib_rc_rcv_resp()  local
1471 	wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_rc_rcv_resp()
1490 		wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_rc_rcv_resp()
1491 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))  in qib_rc_rcv_resp()
1499 						  wqe, psn, pmtu);  in qib_rc_rcv_resp()
1506 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))  in qib_rc_rcv_resp()
1556 		wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_rc_rcv_resp()
1558 						  wqe, psn, pmtu);  in qib_rc_rcv_resp()
1565 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))  in qib_rc_rcv_resp()
1600 		qib_send_complete(qp, wqe, status);  in qib_rc_rcv_resp()