141 	struct nbpf_desc *desc;  member
172 	struct nbpf_desc desc[NBPF_DESCS_PER_PAGE];  member
381 static int nbpf_start(struct nbpf_desc *desc)  in nbpf_start()  argument
383 	struct nbpf_channel *chan = desc->chan;  in nbpf_start()
384 	struct nbpf_link_desc *ldesc = list_first_entry(&desc->sg, struct nbpf_link_desc, node);  in nbpf_start()
480 	struct nbpf_desc *desc = ldesc->desc;  in nbpf_prep_one()  local
481 	struct nbpf_channel *chan = desc->chan;  in nbpf_prep_one()
585 		struct nbpf_desc *desc = list_first_entry(&chan->active,  in nbpf_issue_pending()  local
587 		if (!nbpf_start(desc))  in nbpf_issue_pending()
588 			chan->running = desc;  in nbpf_issue_pending()
613 			struct nbpf_desc *desc;  in nbpf_tx_status()  local
616 			list_for_each_entry(desc, &chan->active, node)  in nbpf_tx_status()
617 				if (desc->async_tx.cookie == cookie) {  in nbpf_tx_status()
623 				list_for_each_entry(desc, &chan->queued, node)  in nbpf_tx_status()
624 					if (desc->async_tx.cookie == cookie) {  in nbpf_tx_status()
630 			state->residue = found ? desc->length : 0;  in nbpf_tx_status()
644 	struct nbpf_desc *desc = container_of(tx, struct nbpf_desc, async_tx);  in nbpf_tx_submit()  local
645 	struct nbpf_channel *chan = desc->chan;  in nbpf_tx_submit()
651 	list_add_tail(&desc->node, &chan->queued);  in nbpf_tx_submit()
665 	struct nbpf_desc *desc;  in nbpf_desc_page_alloc()  local
689 	for (i = 0, desc = dpage->desc;  in nbpf_desc_page_alloc()
690 	     i < ARRAY_SIZE(dpage->desc);  in nbpf_desc_page_alloc()
691 	     i++, desc++) {  in nbpf_desc_page_alloc()
692 		dma_async_tx_descriptor_init(&desc->async_tx, dchan);  in nbpf_desc_page_alloc()
693 		desc->async_tx.tx_submit = nbpf_tx_submit;  in nbpf_desc_page_alloc()
694 		desc->chan = chan;  in nbpf_desc_page_alloc()
695 		INIT_LIST_HEAD(&desc->sg);  in nbpf_desc_page_alloc()
696 		list_add_tail(&desc->node, &head);  in nbpf_desc_page_alloc()
709 	return ARRAY_SIZE(dpage->desc);  in nbpf_desc_page_alloc()
712 static void nbpf_desc_put(struct nbpf_desc *desc)  in nbpf_desc_put()  argument
714 	struct nbpf_channel *chan = desc->chan;  in nbpf_desc_put()
719 	list_for_each_entry_safe(ldesc, tmp, &desc->sg, node)  in nbpf_desc_put()
722 	list_add(&desc->node, &chan->free);  in nbpf_desc_put()
728 	struct nbpf_desc *desc, *tmp;  in nbpf_scan_acked()  local
733 	list_for_each_entry_safe(desc, tmp, &chan->done, node)  in nbpf_scan_acked()
734 		if (async_tx_test_ack(&desc->async_tx) && desc->user_wait) {  in nbpf_scan_acked()
735 			list_move(&desc->node, &head);  in nbpf_scan_acked()
736 			desc->user_wait = false;  in nbpf_scan_acked()
740 	list_for_each_entry_safe(desc, tmp, &head, node) {  in nbpf_scan_acked()
741 		list_del(&desc->node);  in nbpf_scan_acked()
742 		nbpf_desc_put(desc);  in nbpf_scan_acked()
754 	struct nbpf_desc *desc = NULL;  in nbpf_desc_get()  local
773 		desc = list_first_entry(&chan->free, struct nbpf_desc, node);  in nbpf_desc_get()
774 		list_del(&desc->node);  in nbpf_desc_get()
782 					nbpf_desc_put(desc);  in nbpf_desc_get()
791 			ldesc->desc = desc;  in nbpf_desc_get()
796 			list_move_tail(&ldesc->node, &desc->sg);  in nbpf_desc_get()
800 	} while (!desc);  in nbpf_desc_get()
806 	return desc;  in nbpf_desc_get()
811 	struct nbpf_desc *desc, *tmp;  in nbpf_chan_idle()  local
825 	list_for_each_entry_safe(desc, tmp, &head, node) {  in nbpf_chan_idle()
827 			__func__, desc, desc->async_tx.cookie);  in nbpf_chan_idle()
828 		list_del(&desc->node);  in nbpf_chan_idle()
829 		nbpf_desc_put(desc);  in nbpf_chan_idle()
896 	struct nbpf_desc *desc;  in nbpf_prep_sg()  local
921 	desc = nbpf_desc_get(chan, len);  in nbpf_prep_sg()
922 	if (!desc)  in nbpf_prep_sg()
925 	desc->async_tx.flags = flags;  in nbpf_prep_sg()
926 	desc->async_tx.cookie = -EBUSY;  in nbpf_prep_sg()
927 	desc->user_wait = false;  in nbpf_prep_sg()
933 	list_for_each_entry(ldesc, &desc->sg, node) {  in nbpf_prep_sg()
940 			nbpf_desc_put(desc);  in nbpf_prep_sg()
952 	desc->length = data_len;  in nbpf_prep_sg()
955 	return &desc->async_tx;  in nbpf_prep_sg()
1103 	struct nbpf_desc *desc, *tmp;  in nbpf_chan_tasklet()  local
1112 		list_for_each_entry_safe(desc, tmp, &chan->done, node) {  in nbpf_chan_tasklet()
1113 			if (!desc->user_wait) {  in nbpf_chan_tasklet()
1117 			} else if (async_tx_test_ack(&desc->async_tx)) {  in nbpf_chan_tasklet()
1122 				list_del(&desc->node);  in nbpf_chan_tasklet()
1124 				nbpf_desc_put(desc);  in nbpf_chan_tasklet()
1139 		dma_cookie_complete(&desc->async_tx);  in nbpf_chan_tasklet()
1145 		if (async_tx_test_ack(&desc->async_tx)) {  in nbpf_chan_tasklet()
1146 			list_del(&desc->node);  in nbpf_chan_tasklet()
1149 			desc->user_wait = true;  in nbpf_chan_tasklet()
1153 		callback = desc->async_tx.callback;  in nbpf_chan_tasklet()
1154 		param = desc->async_tx.callback_param;  in nbpf_chan_tasklet()
1163 			nbpf_desc_put(desc);  in nbpf_chan_tasklet()
1171 	struct nbpf_desc *desc;  in nbpf_chan_irq()  local
1183 	desc = chan->running;  in nbpf_chan_irq()
1184 	if (WARN_ON(!desc)) {  in nbpf_chan_irq()
1192 	list_move_tail(&desc->node, &chan->done);  in nbpf_chan_irq()
1196 		desc = list_first_entry(&chan->active,  in nbpf_chan_irq()
1198 		if (!nbpf_start(desc))  in nbpf_chan_irq()
1199 			chan->running = desc;  in nbpf_chan_irq()