807 				    struct qib_pportdata *ppd,  in qib_user_sdma_queue_pkts()  argument
905 						ppd->ibmaxlen) {  in qib_user_sdma_queue_pkts()
1050 static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd,  in qib_user_sdma_queue_clean()  argument
1053 	struct qib_devdata *dd = ppd->dd;  in qib_user_sdma_queue_clean()
1072 		s64 descd = ppd->sdma_descq_removed - pkt->added;  in qib_user_sdma_queue_clean()
1115 static int qib_user_sdma_hwqueue_clean(struct qib_pportdata *ppd)  in qib_user_sdma_hwqueue_clean()  argument
1120 	spin_lock_irqsave(&ppd->sdma_lock, flags);  in qib_user_sdma_hwqueue_clean()
1121 	ret = qib_sdma_make_progress(ppd);  in qib_user_sdma_hwqueue_clean()
1122 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);  in qib_user_sdma_hwqueue_clean()
1128 void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,  in qib_user_sdma_queue_drain()  argument
1131 	struct qib_devdata *dd = ppd->dd;  in qib_user_sdma_queue_drain()
1144 		qib_user_sdma_hwqueue_clean(ppd);  in qib_user_sdma_queue_drain()
1145 		qib_user_sdma_queue_clean(ppd, pq);  in qib_user_sdma_queue_drain()
1156 		spin_lock_irqsave(&ppd->sdma_lock, flags);  in qib_user_sdma_queue_drain()
1162 					&ppd->sdma_userpending, list) {  in qib_user_sdma_queue_drain()
1170 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);  in qib_user_sdma_queue_drain()
1211 static void qib_user_sdma_send_frag(struct qib_pportdata *ppd,  in qib_user_sdma_send_frag()  argument
1221 	descqp = &ppd->sdma_descq[tail].qw[0];  in qib_user_sdma_send_frag()
1228 		if (ppd->sdma_intrequest) {  in qib_user_sdma_send_frag()
1230 			ppd->sdma_intrequest = 0;  in qib_user_sdma_send_frag()
1238 void qib_user_sdma_send_desc(struct qib_pportdata *ppd,  in qib_user_sdma_send_desc()  argument
1241 	struct qib_devdata *dd = ppd->dd;  in qib_user_sdma_send_desc()
1246 	nfree = qib_sdma_descq_freecnt(ppd);  in qib_user_sdma_send_desc()
1252 	tail_c = tail = ppd->sdma_descq_tail;  in qib_user_sdma_send_desc()
1253 	gen_c = gen = ppd->sdma_generation;  in qib_user_sdma_send_desc()
1263 			qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail, gen);  in qib_user_sdma_send_desc()
1266 			if (++tail == ppd->sdma_descq_cnt) {  in qib_user_sdma_send_desc()
1269 				ppd->sdma_intrequest = 1;  in qib_user_sdma_send_desc()
1270 			} else if (tail == (ppd->sdma_descq_cnt>>1)) {  in qib_user_sdma_send_desc()
1271 				ppd->sdma_intrequest = 1;  in qib_user_sdma_send_desc()
1285 					ppd->sdma_descq[dtail].qw[0] |=  in qib_user_sdma_send_desc()
1287 					if (++dtail == ppd->sdma_descq_cnt)  in qib_user_sdma_send_desc()
1298 		ppd->sdma_descq_added += c;  in qib_user_sdma_send_desc()
1301 			pkt->added = ppd->sdma_descq_added;  in qib_user_sdma_send_desc()
1309 		if (!nfree || (nsent<<2) > ppd->sdma_descq_cnt)  in qib_user_sdma_send_desc()
1314 	if (ppd->sdma_descq_tail != tail_c) {  in qib_user_sdma_send_desc()
1315 		ppd->sdma_generation = gen_c;  in qib_user_sdma_send_desc()
1316 		dd->f_sdma_update_tail(ppd, tail_c);  in qib_user_sdma_send_desc()
1324 static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd,  in qib_user_sdma_push_pkts()  argument
1330 	if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE)))  in qib_user_sdma_push_pkts()
1335 		spin_lock_irqsave(&ppd->sdma_lock, flags);  in qib_user_sdma_push_pkts()
1336 		if (unlikely(!__qib_sdma_running(ppd))) {  in qib_user_sdma_push_pkts()
1337 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);  in qib_user_sdma_push_pkts()
1341 		list_splice_tail_init(pktlist, &ppd->sdma_userpending);  in qib_user_sdma_push_pkts()
1342 		qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);  in qib_user_sdma_push_pkts()
1343 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);  in qib_user_sdma_push_pkts()
1362 		spin_lock_irqsave(&ppd->sdma_lock, flags);  in qib_user_sdma_push_pkts()
1363 		if (unlikely(!__qib_sdma_running(ppd))) {  in qib_user_sdma_push_pkts()
1364 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);  in qib_user_sdma_push_pkts()
1367 		qib_user_sdma_send_desc(ppd, pktlist);  in qib_user_sdma_push_pkts()
1369 			qib_sdma_make_progress(ppd);  in qib_user_sdma_push_pkts()
1370 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);  in qib_user_sdma_push_pkts()
1382 	struct qib_pportdata *ppd = rcd->ppd;  in qib_user_sdma_writev()  local
1392 	if (!qib_sdma_running(ppd))  in qib_user_sdma_writev()
1396 	if (pq->added > ppd->sdma_descq_removed)  in qib_user_sdma_writev()
1397 		qib_user_sdma_hwqueue_clean(ppd);  in qib_user_sdma_writev()
1400 		qib_user_sdma_queue_clean(ppd, pq);  in qib_user_sdma_writev()
1406 		ret = qib_user_sdma_queue_pkts(dd, ppd, pq,  in qib_user_sdma_writev()
1420 			if (qib_sdma_descq_freecnt(ppd) < ndesc) {  in qib_user_sdma_writev()
1421 				qib_user_sdma_hwqueue_clean(ppd);  in qib_user_sdma_writev()
1423 					qib_user_sdma_queue_clean(ppd, pq);  in qib_user_sdma_writev()
1426 			ret = qib_user_sdma_push_pkts(ppd, pq, &list, mxp);  in qib_user_sdma_writev()
1444 int qib_user_sdma_make_progress(struct qib_pportdata *ppd,  in qib_user_sdma_make_progress()  argument
1450 	qib_user_sdma_hwqueue_clean(ppd);  in qib_user_sdma_make_progress()
1451 	ret = qib_user_sdma_queue_clean(ppd, pq);  in qib_user_sdma_make_progress()