88 static void ioat3_eh(struct ioat2_dma_chan *ioat);
335 desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc)  in desc_get_errstat()  argument
371 static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)  in __cleanup()  argument
373 	struct ioat_chan_common *chan = &ioat->base;  in __cleanup()
377 	int idx = ioat->tail, i;  in __cleanup()
381 		__func__, ioat->head, ioat->tail, ioat->issued);  in __cleanup()
393 	active = ioat2_ring_active(ioat);  in __cleanup()
398 		prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));  in __cleanup()
399 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __cleanup()
400 		dump_desc_dbg(ioat, desc);  in __cleanup()
404 			desc_get_errstat(ioat, desc);  in __cleanup()
432 	ioat->tail = idx + i;  in __cleanup()
447 static void ioat3_cleanup(struct ioat2_dma_chan *ioat)  in ioat3_cleanup()  argument
449 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_cleanup()
455 		__cleanup(ioat, phys_complete);  in ioat3_cleanup()
462 			ioat3_eh(ioat);  in ioat3_cleanup()
471 	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);  in ioat3_cleanup_event()  local
472 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_cleanup_event()
474 	ioat3_cleanup(ioat);  in ioat3_cleanup_event()
477 	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);  in ioat3_cleanup_event()
480 static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)  in ioat3_restart_channel()  argument
482 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_restart_channel()
487 		__cleanup(ioat, phys_complete);  in ioat3_restart_channel()
489 	__ioat2_restart_chan(ioat);  in ioat3_restart_channel()
492 static void ioat3_eh(struct ioat2_dma_chan *ioat)  in ioat3_eh()  argument
494 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_eh()
506 		__cleanup(ioat, phys_complete);  in ioat3_eh()
514 	desc = ioat2_get_ring_ent(ioat, ioat->tail);  in ioat3_eh()
516 	dump_desc_dbg(ioat, desc);  in ioat3_eh()
561 	spin_lock_bh(&ioat->prep_lock);  in ioat3_eh()
562 	ioat3_restart_channel(ioat);  in ioat3_eh()
563 	spin_unlock_bh(&ioat->prep_lock);  in ioat3_eh()
566 static void check_active(struct ioat2_dma_chan *ioat)  in check_active()  argument
568 	struct ioat_chan_common *chan = &ioat->base;  in check_active()
570 	if (ioat2_ring_active(ioat)) {  in check_active()
577 	else if (ioat->alloc_order > ioat_get_alloc_order()) {  in check_active()
581 		reshape_ring(ioat, ioat->alloc_order - 1);  in check_active()
586 		if (ioat->alloc_order > ioat_get_alloc_order())  in check_active()
594 	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);  in ioat3_timer_event()  local
595 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_timer_event()
622 		__cleanup(ioat, phys_complete);  in ioat3_timer_event()
624 		spin_lock_bh(&ioat->prep_lock);  in ioat3_timer_event()
625 		ioat3_restart_channel(ioat);  in ioat3_timer_event()
626 		spin_unlock_bh(&ioat->prep_lock);  in ioat3_timer_event()
635 	if (ioat2_ring_active(ioat))  in ioat3_timer_event()
638 		spin_lock_bh(&ioat->prep_lock);  in ioat3_timer_event()
639 		check_active(ioat);  in ioat3_timer_event()
640 		spin_unlock_bh(&ioat->prep_lock);  in ioat3_timer_event()
649 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in ioat3_tx_status()  local
656 	ioat3_cleanup(ioat);  in ioat3_tx_status()
666 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in __ioat3_prep_xor_lock()  local
680 	num_descs = ioat2_xferlen_to_descs(ioat, len);  in __ioat3_prep_xor_lock()
695 	if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0)  in __ioat3_prep_xor_lock()
696 		idx = ioat->head;  in __ioat3_prep_xor_lock()
702 		size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);  in __ioat3_prep_xor_lock()
705 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_xor_lock()
712 		ext = ioat2_get_ring_ent(ioat, idx + i + 1);  in __ioat3_prep_xor_lock()
727 		dump_desc_dbg(ioat, desc);  in __ioat3_prep_xor_lock()
738 	compl_desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_xor_lock()
746 	dump_desc_dbg(ioat, compl_desc);  in __ioat3_prep_xor_lock()
774 dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)  in dump_pq_desc_dbg()  argument
776 	struct device *dev = to_dev(&ioat->base);  in dump_pq_desc_dbg()
800 static void dump_pq16_desc_dbg(struct ioat2_dma_chan *ioat,  in dump_pq16_desc_dbg()  argument
803 	struct device *dev = to_dev(&ioat->base);  in dump_pq16_desc_dbg()
841 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in __ioat3_prep_pq_lock()  local
842 	struct ioat_chan_common *chan = &ioat->base;  in __ioat3_prep_pq_lock()
862 	num_descs = ioat2_xferlen_to_descs(ioat, len);  in __ioat3_prep_pq_lock()
880 	    ioat2_check_space_lock(ioat, num_descs + cb32) == 0)  in __ioat3_prep_pq_lock()
881 		idx = ioat->head;  in __ioat3_prep_pq_lock()
887 		size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);  in __ioat3_prep_pq_lock()
889 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq_lock()
896 		ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);  in __ioat3_prep_pq_lock()
935 	dump_pq_desc_dbg(ioat, desc, ext);  in __ioat3_prep_pq_lock()
943 		compl_desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq_lock()
951 		dump_desc_dbg(ioat, compl_desc);  in __ioat3_prep_pq_lock()
965 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in __ioat3_prep_pq16_lock()  local
966 	struct ioat_chan_common *chan = &ioat->base;  in __ioat3_prep_pq16_lock()
980 	num_descs = ioat2_xferlen_to_descs(ioat, len);  in __ioat3_prep_pq16_lock()
986 	if (num_descs && ioat2_check_space_lock(ioat, num_descs) == 0)  in __ioat3_prep_pq16_lock()
987 		idx = ioat->head;  in __ioat3_prep_pq16_lock()
995 		size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);  in __ioat3_prep_pq16_lock()
997 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq16_lock()
1054 	dump_pq16_desc_dbg(ioat, desc);  in __ioat3_prep_pq16_lock()
1180 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in ioat3_prep_interrupt_lock()  local
1184 	if (ioat2_check_space_lock(ioat, 1) == 0)  in ioat3_prep_interrupt_lock()
1185 		desc = ioat2_get_ring_ent(ioat, ioat->head);  in ioat3_prep_interrupt_lock()
1202 	dump_desc_dbg(ioat, desc);  in ioat3_prep_interrupt_lock()