165 	struct ioat_ring_ent *desc;  in __ioat_start_null_desc()  local
177 	desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);  in __ioat_start_null_desc()
179 	hw = desc->hw;  in __ioat_start_null_desc()
188 	async_tx_ack(&desc->txd);  in __ioat_start_null_desc()
189 	ioat_set_chainaddr(ioat_chan, desc->txd.phys);  in __ioat_start_null_desc()
190 	dump_desc_dbg(ioat_chan, desc);  in __ioat_start_null_desc()
218 		struct ioat_ring_ent *desc;  in __ioat_restart_chan()  local
220 		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);  in __ioat_restart_chan()
221 		ioat_set_chainaddr(ioat_chan, desc->txd.phys);  in __ioat_restart_chan()
296 	struct ioat_ring_ent *desc;  in ioat_alloc_ring_ent()  local
306 	desc = kmem_cache_zalloc(ioat_cache, flags);  in ioat_alloc_ring_ent()
307 	if (!desc) {  in ioat_alloc_ring_ent()
312 	dma_async_tx_descriptor_init(&desc->txd, chan);  in ioat_alloc_ring_ent()
313 	desc->txd.tx_submit = ioat_tx_submit_unlock;  in ioat_alloc_ring_ent()
314 	desc->hw = hw;  in ioat_alloc_ring_ent()
315 	desc->txd.phys = phys;  in ioat_alloc_ring_ent()
316 	return desc;  in ioat_alloc_ring_ent()
319 void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)  in ioat_free_ring_ent()  argument
324 	pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);  in ioat_free_ring_ent()
325 	kmem_cache_free(ioat_cache, desc);  in ioat_free_ring_ent()
532 static bool desc_has_ext(struct ioat_ring_ent *desc)  in desc_has_ext()  argument
534 	struct ioat_dma_descriptor *hw = desc->hw;  in desc_has_ext()
538 		struct ioat_xor_descriptor *xor = desc->xor;  in desc_has_ext()
544 		struct ioat_pq_descriptor *pq = desc->pq;  in desc_has_ext()
591 desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)  in desc_get_errstat()  argument
593 	struct ioat_dma_descriptor *hw = desc->hw;  in desc_get_errstat()
599 		struct ioat_pq_descriptor *pq = desc->pq;  in desc_get_errstat()
608 			*desc->result |= SUM_CHECK_P_RESULT;  in desc_get_errstat()
611 			*desc->result |= SUM_CHECK_Q_RESULT;  in desc_get_errstat()
627 	struct ioat_ring_ent *desc;  in __cleanup()  local
651 		desc = ioat_get_ring_ent(ioat_chan, idx + i);  in __cleanup()
652 		dump_desc_dbg(ioat_chan, desc);  in __cleanup()
656 			desc_get_errstat(ioat_chan, desc);  in __cleanup()
658 		tx = &desc->txd;  in __cleanup()
672 		if (desc_has_ext(desc)) {  in __cleanup()
678 		if (desc->sed) {  in __cleanup()
679 			ioat_free_sed(ioat_dma, desc->sed);  in __cleanup()
680 			desc->sed = NULL;  in __cleanup()
750 	struct ioat_ring_ent *desc;  in ioat_eh()  local
765 	desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);  in ioat_eh()
766 	hw = desc->hw;  in ioat_eh()
767 	dump_desc_dbg(ioat_chan, desc);  in ioat_eh()
772 			*desc->result |= SUM_CHECK_P_RESULT;  in ioat_eh()
779 			*desc->result |= SUM_CHECK_P_RESULT;  in ioat_eh()
783 			*desc->result |= SUM_CHECK_Q_RESULT;  in ioat_eh()
795 		tx = &desc->txd;  in ioat_eh()
810 	*ioat_chan->completion = desc->txd.phys;  in ioat_eh()