147 static void ipath_user_sdma_init_frag(struct ipath_user_sdma_pkt *pkt,  in ipath_user_sdma_init_frag()  argument
153 	pkt->addr[i].offset = offset;  in ipath_user_sdma_init_frag()
154 	pkt->addr[i].length = len;  in ipath_user_sdma_init_frag()
155 	pkt->addr[i].put_page = put_page;  in ipath_user_sdma_init_frag()
156 	pkt->addr[i].dma_mapped = dma_mapped;  in ipath_user_sdma_init_frag()
157 	pkt->addr[i].page = page;  in ipath_user_sdma_init_frag()
158 	pkt->addr[i].kvaddr = kvaddr;  in ipath_user_sdma_init_frag()
159 	pkt->addr[i].addr = dma_addr;  in ipath_user_sdma_init_frag()
162 static void ipath_user_sdma_init_header(struct ipath_user_sdma_pkt *pkt,  in ipath_user_sdma_init_header()  argument
168 	pkt->naddr = 1;  in ipath_user_sdma_init_header()
169 	pkt->counter = counter;  in ipath_user_sdma_init_header()
170 	ipath_user_sdma_init_frag(pkt, 0, offset, len, 0, dma_mapped, page,  in ipath_user_sdma_init_header()
176 				    struct ipath_user_sdma_pkt *pkt,  in ipath_user_sdma_coalesce()  argument
215 	ipath_user_sdma_init_frag(pkt, 1, 0, len, 0, 1, page, mpage_save,  in ipath_user_sdma_coalesce()
217 	pkt->naddr = 2;  in ipath_user_sdma_coalesce()
249 					  struct ipath_user_sdma_pkt *pkt,  in ipath_user_sdma_free_pkt_frag()  argument
254 	if (pkt->addr[i].page) {  in ipath_user_sdma_free_pkt_frag()
255 		if (pkt->addr[i].dma_mapped)  in ipath_user_sdma_free_pkt_frag()
257 				       pkt->addr[i].addr,  in ipath_user_sdma_free_pkt_frag()
258 				       pkt->addr[i].length,  in ipath_user_sdma_free_pkt_frag()
261 		if (pkt->addr[i].kvaddr)  in ipath_user_sdma_free_pkt_frag()
262 			kunmap(pkt->addr[i].page);  in ipath_user_sdma_free_pkt_frag()
264 		if (pkt->addr[i].put_page)  in ipath_user_sdma_free_pkt_frag()
265 			put_page(pkt->addr[i].page);  in ipath_user_sdma_free_pkt_frag()
267 			__free_page(pkt->addr[i].page);  in ipath_user_sdma_free_pkt_frag()
268 	} else if (pkt->addr[i].kvaddr)  in ipath_user_sdma_free_pkt_frag()
271 			      pkt->addr[i].kvaddr, pkt->addr[i].addr);  in ipath_user_sdma_free_pkt_frag()
276 				     struct ipath_user_sdma_pkt *pkt,  in ipath_user_sdma_pin_pages()  argument
308 		ipath_user_sdma_init_frag(pkt, pkt->naddr, fofs, flen, 1, 1,  in ipath_user_sdma_pin_pages()
312 		pkt->naddr++;  in ipath_user_sdma_pin_pages()
323 				   struct ipath_user_sdma_pkt *pkt,  in ipath_user_sdma_pin_pkt()  argument
334 		ret = ipath_user_sdma_pin_pages(dd, pkt,  in ipath_user_sdma_pin_pkt()
344 	for (idx = 0; idx < pkt->naddr; idx++)  in ipath_user_sdma_pin_pkt()
345 		ipath_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx);  in ipath_user_sdma_pin_pkt()
353 					struct ipath_user_sdma_pkt *pkt,  in ipath_user_sdma_init_payload()  argument
359 	if (npages >= ARRAY_SIZE(pkt->addr))  in ipath_user_sdma_init_payload()
360 		ret = ipath_user_sdma_coalesce(dd, pkt, iov, niov);  in ipath_user_sdma_init_payload()
362 		ret = ipath_user_sdma_pin_pkt(dd, pq, pkt, iov, niov);  in ipath_user_sdma_init_payload()
372 	struct ipath_user_sdma_pkt *pkt, *pkt_next;  in ipath_user_sdma_free_pkt_list()  local
374 	list_for_each_entry_safe(pkt, pkt_next, list, list) {  in ipath_user_sdma_free_pkt_list()
377 		for (i = 0; i < pkt->naddr; i++)  in ipath_user_sdma_free_pkt_list()
378 			ipath_user_sdma_free_pkt_frag(dev, pq, pkt, i);  in ipath_user_sdma_free_pkt_list()
380 		kmem_cache_free(pq->pkt_slab, pkt);  in ipath_user_sdma_free_pkt_list()
404 	struct ipath_user_sdma_pkt *pkt = NULL;  in ipath_user_sdma_queue_pkts()  local
424 		pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);  in ipath_user_sdma_queue_pkts()
425 		if (!pkt) {  in ipath_user_sdma_queue_pkts()
518 		ipath_user_sdma_init_header(pkt, counter, 0, len, dma_mapped,  in ipath_user_sdma_queue_pkts()
522 			ret = ipath_user_sdma_init_payload(dd, pq, pkt,  in ipath_user_sdma_queue_pkts()
532 		list_add_tail(&pkt->list, list);  in ipath_user_sdma_queue_pkts()
548 	kmem_cache_free(pq->pkt_slab, pkt);  in ipath_user_sdma_queue_pkts()
566 	struct ipath_user_sdma_pkt *pkt;  in ipath_user_sdma_queue_clean()  local
572 	list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) {  in ipath_user_sdma_queue_clean()
573 		s64 descd = dd->ipath_sdma_descq_removed - pkt->added;  in ipath_user_sdma_queue_clean()
578 		list_move_tail(&pkt->list, &free_list);  in ipath_user_sdma_queue_clean()
587 		pkt = list_entry(free_list.prev,  in ipath_user_sdma_queue_clean()
589 		counter = pkt->counter;  in ipath_user_sdma_queue_clean()
685 				      struct ipath_user_sdma_pkt *pkt, int idx,  in ipath_user_sdma_send_frag()  argument
688 	const u64 addr = (u64) pkt->addr[idx].addr +  in ipath_user_sdma_send_frag()
689 		(u64) pkt->addr[idx].offset;  in ipath_user_sdma_send_frag()
690 	const u64 dwlen = (u64) pkt->addr[idx].length / 4;  in ipath_user_sdma_send_frag()
699 	if (idx == pkt->naddr - 1)  in ipath_user_sdma_send_frag()
730 		struct ipath_user_sdma_pkt *pkt =  in ipath_user_sdma_push_pkts()  local
737 		if (pkt->naddr > ipath_sdma_descq_freecnt(dd))  in ipath_user_sdma_push_pkts()
740 		for (i = 0; i < pkt->naddr; i++) {  in ipath_user_sdma_push_pkts()
741 			ipath_user_sdma_send_frag(dd, pkt, i, ofs, tail);  in ipath_user_sdma_push_pkts()
742 			ofs += pkt->addr[i].length >> 2;  in ipath_user_sdma_push_pkts()
763 			for (i = 0; i < pkt->naddr; i++) {  in ipath_user_sdma_push_pkts()
771 		dd->ipath_sdma_descq_added += pkt->naddr;  in ipath_user_sdma_push_pkts()
772 		pkt->added = dd->ipath_sdma_descq_added;  in ipath_user_sdma_push_pkts()
773 		list_move_tail(&pkt->list, &pq->sent);  in ipath_user_sdma_push_pkts()