525 static inline bool _queue_empty(struct pl330_thread *thrd)  in _queue_empty()  argument
527 	return thrd->req[0].desc == NULL && thrd->req[1].desc == NULL;  in _queue_empty()
530 static inline bool _queue_full(struct pl330_thread *thrd)  in _queue_full()  argument
532 	return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL;  in _queue_full()
535 static inline bool is_manager(struct pl330_thread *thrd)  in is_manager()  argument
537 	return thrd->dmac->manager == thrd;  in is_manager()
541 static inline bool _manager_ns(struct pl330_thread *thrd)  in _manager_ns()  argument
543 	return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false;  in _manager_ns()
899 static bool _until_dmac_idle(struct pl330_thread *thrd)  in _until_dmac_idle()  argument
901 	void __iomem *regs = thrd->dmac->base;  in _until_dmac_idle()
918 static inline void _execute_DBGINSN(struct pl330_thread *thrd,  in _execute_DBGINSN()  argument
921 	void __iomem *regs = thrd->dmac->base;  in _execute_DBGINSN()
927 		val |= (thrd->id << 8); /* Channel Number */  in _execute_DBGINSN()
935 	if (_until_dmac_idle(thrd)) {  in _execute_DBGINSN()
936 		dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");  in _execute_DBGINSN()
944 static inline u32 _state(struct pl330_thread *thrd)  in _state()  argument
946 	void __iomem *regs = thrd->dmac->base;  in _state()
949 	if (is_manager(thrd))  in _state()
952 		val = readl(regs + CS(thrd->id)) & 0xf;  in _state()
968 		if (is_manager(thrd))  in _state()
973 		if (is_manager(thrd))  in _state()
978 		if (is_manager(thrd))  in _state()
983 		if (is_manager(thrd))  in _state()
988 		if (is_manager(thrd))  in _state()
993 		if (is_manager(thrd))  in _state()
1002 static void _stop(struct pl330_thread *thrd)  in _stop()  argument
1004 	void __iomem *regs = thrd->dmac->base;  in _stop()
1007 	if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)  in _stop()
1008 		UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);  in _stop()
1011 	if (_state(thrd) == PL330_STATE_COMPLETING  in _stop()
1012 		  || _state(thrd) == PL330_STATE_KILLING  in _stop()
1013 		  || _state(thrd) == PL330_STATE_STOPPED)  in _stop()
1019 	writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);  in _stop()
1021 	_execute_DBGINSN(thrd, insn, is_manager(thrd));  in _stop()
1025 static bool _trigger(struct pl330_thread *thrd)  in _trigger()  argument
1027 	void __iomem *regs = thrd->dmac->base;  in _trigger()
1036 	if (_state(thrd) != PL330_STATE_STOPPED)  in _trigger()
1039 	idx = 1 - thrd->lstenq;  in _trigger()
1040 	if (thrd->req[idx].desc != NULL) {  in _trigger()
1041 		req = &thrd->req[idx];  in _trigger()
1043 		idx = thrd->lstenq;  in _trigger()
1044 		if (thrd->req[idx].desc != NULL)  in _trigger()
1045 			req = &thrd->req[idx];  in _trigger()
1055 	if (idx == thrd->req_running)  in _trigger()
1063 	if (_manager_ns(thrd) && !ns)  in _trigger()
1064 		dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n",  in _trigger()
1067 	go.chan = thrd->id;  in _trigger()
1073 	writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);  in _trigger()
1076 	_execute_DBGINSN(thrd, insn, true);  in _trigger()
1078 	thrd->req_running = idx;  in _trigger()
1083 static bool _start(struct pl330_thread *thrd)  in _start()  argument
1085 	switch (_state(thrd)) {  in _start()
1087 		UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);  in _start()
1089 		if (_state(thrd) == PL330_STATE_KILLING)  in _start()
1090 			UNTIL(thrd, PL330_STATE_STOPPED)  in _start()
1093 		_stop(thrd);  in _start()
1097 		UNTIL(thrd, PL330_STATE_STOPPED)  in _start()
1100 		return _trigger(thrd);  in _start()
1313 static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,  in _setup_req()  argument
1316 	struct _pl330_req *req = &thrd->req[index];  in _setup_req()
1334 	off += _emit_SEV(dry_run, &buf[off], thrd->ev);  in _setup_req()
1378 static int pl330_submit_req(struct pl330_thread *thrd,  in pl330_submit_req()  argument
1381 	struct pl330_dmac *pl330 = thrd->dmac;  in pl330_submit_req()
1389 		|| pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {  in pl330_submit_req()
1390 		dev_info(thrd->dmac->ddma.dev, "%s:%d\n",  in pl330_submit_req()
1398 		dev_info(thrd->dmac->ddma.dev,  in pl330_submit_req()
1406 	if (_queue_full(thrd)) {  in pl330_submit_req()
1412 	if (!_manager_ns(thrd))  in pl330_submit_req()
1419 	idx = thrd->req[0].desc == NULL ? 0 : 1;  in pl330_submit_req()
1425 	ret = _setup_req(1, thrd, idx, &xs);  in pl330_submit_req()
1437 	thrd->lstenq = idx;  in pl330_submit_req()
1438 	thrd->req[idx].desc = desc;  in pl330_submit_req()
1439 	_setup_req(0, thrd, idx, &xs);  in pl330_submit_req()
1500 			struct pl330_thread *thrd = &pl330->channels[i];  in pl330_dotask()  local
1504 			_stop(thrd);  in pl330_dotask()
1506 			if (readl(regs + FSC) & (1 << thrd->id))  in pl330_dotask()
1512 			dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err);  in pl330_dotask()
1513 			dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err);  in pl330_dotask()
1516 			thrd->req[0].desc = NULL;  in pl330_dotask()
1517 			thrd->req[1].desc = NULL;  in pl330_dotask()
1518 			thrd->req_running = -1;  in pl330_dotask()
1578 			struct pl330_thread *thrd;  in pl330_update()  local
1590 			thrd = &pl330->channels[id];  in pl330_update()
1592 			active = thrd->req_running;  in pl330_update()
1597 			descdone = thrd->req[active].desc;  in pl330_update()
1598 			thrd->req[active].desc = NULL;  in pl330_update()
1600 			thrd->req_running = -1;  in pl330_update()
1603 			_start(thrd);  in pl330_update()
1632 static inline int _alloc_event(struct pl330_thread *thrd)  in _alloc_event()  argument
1634 	struct pl330_dmac *pl330 = thrd->dmac;  in _alloc_event()
1639 			pl330->events[ev] = thrd->id;  in _alloc_event()
1656 	struct pl330_thread *thrd = NULL;  in pl330_request_channel()  local
1668 		thrd = &pl330->channels[i];  in pl330_request_channel()
1669 		if ((thrd->free) && (!_manager_ns(thrd) ||  in pl330_request_channel()
1671 			thrd->ev = _alloc_event(thrd);  in pl330_request_channel()
1672 			if (thrd->ev >= 0) {  in pl330_request_channel()
1673 				thrd->free = false;  in pl330_request_channel()
1674 				thrd->lstenq = 1;  in pl330_request_channel()
1675 				thrd->req[0].desc = NULL;  in pl330_request_channel()
1676 				thrd->req[1].desc = NULL;  in pl330_request_channel()
1677 				thrd->req_running = -1;  in pl330_request_channel()
1681 		thrd = NULL;  in pl330_request_channel()
1686 	return thrd;  in pl330_request_channel()
1690 static inline void _free_event(struct pl330_thread *thrd, int ev)  in _free_event()  argument
1692 	struct pl330_dmac *pl330 = thrd->dmac;  in _free_event()
1696 			&& pl330->events[ev] == thrd->id)  in _free_event()
1700 static void pl330_release_channel(struct pl330_thread *thrd)  in pl330_release_channel()  argument
1705 	if (!thrd || thrd->free)  in pl330_release_channel()
1708 	_stop(thrd);  in pl330_release_channel()
1710 	dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);  in pl330_release_channel()
1711 	dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);  in pl330_release_channel()
1713 	pl330 = thrd->dmac;  in pl330_release_channel()
1716 	_free_event(thrd, thrd->ev);  in pl330_release_channel()
1717 	thrd->free = true;  in pl330_release_channel()
1766 static inline void _reset_thread(struct pl330_thread *thrd)  in _reset_thread()  argument
1768 	struct pl330_dmac *pl330 = thrd->dmac;  in _reset_thread()
1770 	thrd->req[0].mc_cpu = pl330->mcode_cpu  in _reset_thread()
1771 				+ (thrd->id * pl330->mcbufsz);  in _reset_thread()
1772 	thrd->req[0].mc_bus = pl330->mcode_bus  in _reset_thread()
1773 				+ (thrd->id * pl330->mcbufsz);  in _reset_thread()
1774 	thrd->req[0].desc = NULL;  in _reset_thread()
1776 	thrd->req[1].mc_cpu = thrd->req[0].mc_cpu  in _reset_thread()
1778 	thrd->req[1].mc_bus = thrd->req[0].mc_bus  in _reset_thread()
1780 	thrd->req[1].desc = NULL;  in _reset_thread()
1782 	thrd->req_running = -1;  in _reset_thread()
1788 	struct pl330_thread *thrd;  in dmac_alloc_threads()  local
1792 	pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),  in dmac_alloc_threads()
1799 		thrd = &pl330->channels[i];  in dmac_alloc_threads()
1800 		thrd->id = i;  in dmac_alloc_threads()
1801 		thrd->dmac = pl330;  in dmac_alloc_threads()
1802 		_reset_thread(thrd);  in dmac_alloc_threads()
1803 		thrd->free = true;  in dmac_alloc_threads()
1807 	thrd = &pl330->channels[chans];  in dmac_alloc_threads()
1808 	thrd->id = chans;  in dmac_alloc_threads()
1809 	thrd->dmac = pl330;  in dmac_alloc_threads()
1810 	thrd->free = false;  in dmac_alloc_threads()
1811 	pl330->manager = thrd;  in dmac_alloc_threads()
1898 	struct pl330_thread *thrd;  in dmac_free_threads()  local
1903 		thrd = &pl330->channels[i];  in dmac_free_threads()
1904 		pl330_release_channel(thrd);  in dmac_free_threads()
2215 	struct pl330_thread *thrd = pch->thread;  in pl330_get_current_xferred_count()  local
2217 	void __iomem *regs = thrd->dmac->base;  in pl330_get_current_xferred_count()
2223 		val = readl(regs + SA(thrd->id));  in pl330_get_current_xferred_count()
2226 		val = readl(regs + DA(thrd->id));  in pl330_get_current_xferred_count()