43 static u32 restart_sge(struct qib_sge_state *ss, struct qib_swqe *wqe,  in restart_sge()  argument
48 	len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu;  in restart_sge()
49 	ss->sge = wqe->sg_list[0];  in restart_sge()
50 	ss->sg_list = wqe->sg_list + 1;  in restart_sge()
51 	ss->num_sge = wqe->wr.num_sge;  in restart_sge()
52 	ss->total_len = wqe->length;  in restart_sge()
54 	return wqe->length - len;  in restart_sge()
236 	struct qib_swqe *wqe;  in qib_make_rc_req()  local
273 		wqe = get_swqe_ptr(qp, qp->s_last);  in qib_make_rc_req()
274 		qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ?  in qib_make_rc_req()
297 	wqe = get_swqe_ptr(qp, qp->s_cur);  in qib_make_rc_req()
318 			if ((wqe->wr.send_flags & IB_SEND_FENCE) &&  in qib_make_rc_req()
323 			wqe->psn = qp->s_next_psn;  in qib_make_rc_req()
331 		len = wqe->length;  in qib_make_rc_req()
334 		switch (wqe->wr.opcode) {  in qib_make_rc_req()
339 			    qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {  in qib_make_rc_req()
343 			wqe->lpsn = wqe->psn;  in qib_make_rc_req()
345 				wqe->lpsn += (len - 1) / pmtu;  in qib_make_rc_req()
350 			if (wqe->wr.opcode == IB_WR_SEND)  in qib_make_rc_req()
355 				ohdr->u.imm_data = wqe->wr.ex.imm_data;  in qib_make_rc_req()
358 			if (wqe->wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
372 			    qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {  in qib_make_rc_req()
377 				cpu_to_be64(wqe->wr.wr.rdma.remote_addr);  in qib_make_rc_req()
379 				cpu_to_be32(wqe->wr.wr.rdma.rkey);  in qib_make_rc_req()
382 			wqe->lpsn = wqe->psn;  in qib_make_rc_req()
384 				wqe->lpsn += (len - 1) / pmtu;  in qib_make_rc_req()
389 			if (wqe->wr.opcode == IB_WR_RDMA_WRITE)  in qib_make_rc_req()
395 				ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;  in qib_make_rc_req()
397 				if (wqe->wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
425 				wqe->lpsn = qp->s_next_psn++;  in qib_make_rc_req()
428 				cpu_to_be64(wqe->wr.wr.rdma.remote_addr);  in qib_make_rc_req()
430 				cpu_to_be32(wqe->wr.wr.rdma.rkey);  in qib_make_rc_req()
456 				wqe->lpsn = wqe->psn;  in qib_make_rc_req()
458 			if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {  in qib_make_rc_req()
461 					wqe->wr.wr.atomic.swap);  in qib_make_rc_req()
463 					wqe->wr.wr.atomic.compare_add);  in qib_make_rc_req()
467 					wqe->wr.wr.atomic.compare_add);  in qib_make_rc_req()
471 				wqe->wr.wr.atomic.remote_addr >> 32);  in qib_make_rc_req()
473 				wqe->wr.wr.atomic.remote_addr);  in qib_make_rc_req()
475 				wqe->wr.wr.atomic.rkey);  in qib_make_rc_req()
487 		qp->s_sge.sge = wqe->sg_list[0];  in qib_make_rc_req()
488 		qp->s_sge.sg_list = wqe->sg_list + 1;  in qib_make_rc_req()
489 		qp->s_sge.num_sge = wqe->wr.num_sge;  in qib_make_rc_req()
490 		qp->s_sge.total_len = wqe->length;  in qib_make_rc_req()
491 		qp->s_len = wqe->length;  in qib_make_rc_req()
497 		if (wqe->wr.opcode == IB_WR_RDMA_READ)  in qib_make_rc_req()
498 			qp->s_psn = wqe->lpsn + 1;  in qib_make_rc_req()
516 		qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);  in qib_make_rc_req()
531 		if (wqe->wr.opcode == IB_WR_SEND)  in qib_make_rc_req()
536 			ohdr->u.imm_data = wqe->wr.ex.imm_data;  in qib_make_rc_req()
539 		if (wqe->wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
557 		qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);  in qib_make_rc_req()
572 		if (wqe->wr.opcode == IB_WR_RDMA_WRITE)  in qib_make_rc_req()
577 			ohdr->u.imm_data = wqe->wr.ex.imm_data;  in qib_make_rc_req()
579 			if (wqe->wr.send_flags & IB_SEND_SOLICITED)  in qib_make_rc_req()
598 		len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;  in qib_make_rc_req()
600 			cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len);  in qib_make_rc_req()
602 			cpu_to_be32(wqe->wr.wr.rdma.rkey);  in qib_make_rc_req()
603 		ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);  in qib_make_rc_req()
607 		qp->s_psn = wqe->lpsn + 1;  in qib_make_rc_req()
616 	delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8;  in qib_make_rc_req()
786 	struct qib_swqe *wqe = get_swqe_ptr(qp, n);  in reset_psn()  local
795 	if (qib_cmp24(psn, wqe->psn) <= 0) {  in reset_psn()
801 	opcode = wqe->wr.opcode;  in reset_psn()
809 		wqe = get_swqe_ptr(qp, n);  in reset_psn()
810 		diff = qib_cmp24(psn, wqe->psn);  in reset_psn()
822 		opcode = wqe->wr.opcode;  in reset_psn()
870 	struct qib_swqe *wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_restart_rc()  local
878 			qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);  in qib_restart_rc()
887 	if (wqe->wr.opcode == IB_WR_RDMA_READ)  in qib_restart_rc()
946 	struct qib_swqe *wqe;  in reset_sending_psn()  local
951 		wqe = get_swqe_ptr(qp, n);  in reset_sending_psn()
952 		if (qib_cmp24(psn, wqe->lpsn) <= 0) {  in reset_sending_psn()
953 			if (wqe->wr.opcode == IB_WR_RDMA_READ)  in reset_sending_psn()
954 				qp->s_sending_psn = wqe->lpsn + 1;  in reset_sending_psn()
972 	struct qib_swqe *wqe;  in qib_rc_send_complete()  local
1008 		wqe = get_swqe_ptr(qp, qp->s_last);  in qib_rc_send_complete()
1009 		if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&  in qib_rc_send_complete()
1012 		for (i = 0; i < wqe->wr.num_sge; i++) {  in qib_rc_send_complete()
1013 			struct qib_sge *sge = &wqe->sg_list[i];  in qib_rc_send_complete()
1019 		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {  in qib_rc_send_complete()
1021 			wc.wr_id = wqe->wr.wr_id;  in qib_rc_send_complete()
1023 			wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];  in qib_rc_send_complete()
1024 			wc.byte_len = wqe->length;  in qib_rc_send_complete()
1055 					 struct qib_swqe *wqe,  in do_rc_completion()  argument
1066 	if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||  in do_rc_completion()
1068 		for (i = 0; i < wqe->wr.num_sge; i++) {  in do_rc_completion()
1069 			struct qib_sge *sge = &wqe->sg_list[i];  in do_rc_completion()
1075 		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {  in do_rc_completion()
1077 			wc.wr_id = wqe->wr.wr_id;  in do_rc_completion()
1079 			wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];  in do_rc_completion()
1080 			wc.byte_len = wqe->length;  in do_rc_completion()
1090 	update_last_psn(qp, wqe->lpsn);  in do_rc_completion()
1101 		wqe = get_swqe_ptr(qp, qp->s_cur);  in do_rc_completion()
1104 			qp->s_psn = wqe->psn;  in do_rc_completion()
1111 		wqe = get_swqe_ptr(qp, qp->s_acked);  in do_rc_completion()
1113 	return wqe;  in do_rc_completion()
1132 	struct qib_swqe *wqe;  in do_rc_ack()  local
1152 	wqe = get_swqe_ptr(qp, qp->s_acked);  in do_rc_ack()
1159 	while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) {  in do_rc_ack()
1166 		if (wqe->wr.opcode == IB_WR_RDMA_READ &&  in do_rc_ack()
1181 		if ((wqe->wr.opcode == IB_WR_RDMA_READ &&  in do_rc_ack()
1183 		    ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in do_rc_ack()
1184 		      wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&  in do_rc_ack()
1203 		if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in do_rc_ack()
1204 		    wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {  in do_rc_ack()
1205 			u64 *vaddr = wqe->sg_list[0].vaddr;  in do_rc_ack()
1209 		    (wqe->wr.opcode == IB_WR_RDMA_READ ||  in do_rc_ack()
1210 		     wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in do_rc_ack()
1211 		     wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {  in do_rc_ack()
1225 		wqe = do_rc_completion(qp, wqe, ibp);  in do_rc_ack()
1323 				qib_send_complete(qp, wqe, status);  in do_rc_ack()
1353 	struct qib_swqe *wqe;  in rdma_seq_err()  local
1361 	wqe = get_swqe_ptr(qp, qp->s_acked);  in rdma_seq_err()
1363 	while (qib_cmp24(psn, wqe->lpsn) > 0) {  in rdma_seq_err()
1364 		if (wqe->wr.opcode == IB_WR_RDMA_READ ||  in rdma_seq_err()
1365 		    wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||  in rdma_seq_err()
1366 		    wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)  in rdma_seq_err()
1368 		wqe = do_rc_completion(qp, wqe, ibp);  in rdma_seq_err()
1405 	struct qib_swqe *wqe;  in qib_rc_rcv_resp()  local
1469 	wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_rc_rcv_resp()
1488 		wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_rc_rcv_resp()
1489 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))  in qib_rc_rcv_resp()
1497 						  wqe, psn, pmtu);  in qib_rc_rcv_resp()
1504 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))  in qib_rc_rcv_resp()
1554 		wqe = get_swqe_ptr(qp, qp->s_acked);  in qib_rc_rcv_resp()
1556 						  wqe, psn, pmtu);  in qib_rc_rcv_resp()
1563 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))  in qib_rc_rcv_resp()
1598 		qib_send_complete(qp, wqe, status);  in qib_rc_rcv_resp()