1/*
2 * Tegra host1x Command DMA
3 *
4 * Copyright (c) 2010-2013, NVIDIA Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/slab.h>
20#include <linux/scatterlist.h>
21#include <linux/dma-mapping.h>
22
23#include "../cdma.h"
24#include "../channel.h"
25#include "../dev.h"
26#include "../debug.h"
27
28/*
29 * Put the restart at the end of pushbuffer memory
30 */
31static void push_buffer_init(struct push_buffer *pb)
32{
33	*(u32 *)(pb->mapped + pb->size_bytes) = host1x_opcode_restart(0);
34}
35
36/*
37 * Increment timedout buffer's syncpt via CPU.
38 */
39static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
40				u32 syncpt_incrs, u32 syncval, u32 nr_slots)
41{
42	struct host1x *host1x = cdma_to_host1x(cdma);
43	struct push_buffer *pb = &cdma->push_buffer;
44	u32 i;
45
46	for (i = 0; i < syncpt_incrs; i++)
47		host1x_syncpt_incr(cdma->timeout.syncpt);
48
49	/* after CPU incr, ensure shadow is up to date */
50	host1x_syncpt_load(cdma->timeout.syncpt);
51
52	/* NOP all the PB slots */
53	while (nr_slots--) {
54		u32 *p = (u32 *)(pb->mapped + getptr);
55		*(p++) = HOST1X_OPCODE_NOP;
56		*(p++) = HOST1X_OPCODE_NOP;
57		dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
58			&pb->phys, getptr);
59		getptr = (getptr + 8) & (pb->size_bytes - 1);
60	}
61	wmb();
62}
63
64/*
65 * Start channel DMA
66 */
67static void cdma_start(struct host1x_cdma *cdma)
68{
69	struct host1x_channel *ch = cdma_to_channel(cdma);
70
71	if (cdma->running)
72		return;
73
74	cdma->last_pos = cdma->push_buffer.pos;
75
76	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
77			 HOST1X_CHANNEL_DMACTRL);
78
79	/* set base, put and end pointer */
80	host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
81	host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
82	host1x_ch_writel(ch, cdma->push_buffer.phys +
83			 cdma->push_buffer.size_bytes + 4,
84			 HOST1X_CHANNEL_DMAEND);
85
86	/* reset GET */
87	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
88			 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
89			 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
90			 HOST1X_CHANNEL_DMACTRL);
91
92	/* start the command DMA */
93	host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
94
95	cdma->running = true;
96}
97
98/*
99 * Similar to cdma_start(), but rather than starting from an idle
100 * state (where DMA GET is set to DMA PUT), on a timeout we restore
101 * DMA GET from an explicit value (so DMA may again be pending).
102 */
103static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
104{
105	struct host1x *host1x = cdma_to_host1x(cdma);
106	struct host1x_channel *ch = cdma_to_channel(cdma);
107
108	if (cdma->running)
109		return;
110
111	cdma->last_pos = cdma->push_buffer.pos;
112
113	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
114			 HOST1X_CHANNEL_DMACTRL);
115
116	/* set base, end pointer (all of memory) */
117	host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
118	host1x_ch_writel(ch, cdma->push_buffer.phys +
119			 cdma->push_buffer.size_bytes,
120			 HOST1X_CHANNEL_DMAEND);
121
122	/* set GET, by loading the value in PUT (then reset GET) */
123	host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
124	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
125			 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
126			 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
127			 HOST1X_CHANNEL_DMACTRL);
128
129	dev_dbg(host1x->dev,
130		"%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
131		host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
132		host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
133		cdma->last_pos);
134
135	/* deassert GET reset and set PUT */
136	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
137			 HOST1X_CHANNEL_DMACTRL);
138	host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
139
140	/* start the command DMA */
141	host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
142
143	cdma->running = true;
144}
145
146/*
147 * Kick channel DMA into action by writing its PUT offset (if it has changed)
148 */
149static void cdma_flush(struct host1x_cdma *cdma)
150{
151	struct host1x_channel *ch = cdma_to_channel(cdma);
152
153	if (cdma->push_buffer.pos != cdma->last_pos) {
154		host1x_ch_writel(ch, cdma->push_buffer.pos,
155				 HOST1X_CHANNEL_DMAPUT);
156		cdma->last_pos = cdma->push_buffer.pos;
157	}
158}
159
160static void cdma_stop(struct host1x_cdma *cdma)
161{
162	struct host1x_channel *ch = cdma_to_channel(cdma);
163
164	mutex_lock(&cdma->lock);
165	if (cdma->running) {
166		host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
167		host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
168				 HOST1X_CHANNEL_DMACTRL);
169		cdma->running = false;
170	}
171	mutex_unlock(&cdma->lock);
172}
173
174/*
175 * Stops both channel's command processor and CDMA immediately.
176 * Also, tears down the channel and resets corresponding module.
177 */
178static void cdma_freeze(struct host1x_cdma *cdma)
179{
180	struct host1x *host = cdma_to_host1x(cdma);
181	struct host1x_channel *ch = cdma_to_channel(cdma);
182	u32 cmdproc_stop;
183
184	if (cdma->torndown && !cdma->running) {
185		dev_warn(host->dev, "Already torn down\n");
186		return;
187	}
188
189	dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
190
191	cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
192	cmdproc_stop |= BIT(ch->id);
193	host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
194
195	dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
196		__func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
197		host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
198		cdma->last_pos);
199
200	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
201			 HOST1X_CHANNEL_DMACTRL);
202
203	host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
204
205	cdma->running = false;
206	cdma->torndown = true;
207}
208
209static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
210{
211	struct host1x *host1x = cdma_to_host1x(cdma);
212	struct host1x_channel *ch = cdma_to_channel(cdma);
213	u32 cmdproc_stop;
214
215	dev_dbg(host1x->dev,
216		"resuming channel (id %d, DMAGET restart = 0x%x)\n",
217		ch->id, getptr);
218
219	cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
220	cmdproc_stop &= ~(BIT(ch->id));
221	host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
222
223	cdma->torndown = false;
224	cdma_timeout_restart(cdma, getptr);
225}
226
227/*
228 * If this timeout fires, it indicates the current sync_queue entry has
229 * exceeded its TTL and the userctx should be timed out and remaining
230 * submits already issued cleaned up (future submits return an error).
231 */
232static void cdma_timeout_handler(struct work_struct *work)
233{
234	struct host1x_cdma *cdma;
235	struct host1x *host1x;
236	struct host1x_channel *ch;
237
238	u32 syncpt_val;
239
240	u32 prev_cmdproc, cmdproc_stop;
241
242	cdma = container_of(to_delayed_work(work), struct host1x_cdma,
243			    timeout.wq);
244	host1x = cdma_to_host1x(cdma);
245	ch = cdma_to_channel(cdma);
246
247	host1x_debug_dump(cdma_to_host1x(cdma));
248
249	mutex_lock(&cdma->lock);
250
251	if (!cdma->timeout.client) {
252		dev_dbg(host1x->dev,
253			"cdma_timeout: expired, but has no clientid\n");
254		mutex_unlock(&cdma->lock);
255		return;
256	}
257
258	/* stop processing to get a clean snapshot */
259	prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
260	cmdproc_stop = prev_cmdproc | BIT(ch->id);
261	host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
262
263	dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
264		prev_cmdproc, cmdproc_stop);
265
266	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
267
268	/* has buffer actually completed? */
269	if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
270		dev_dbg(host1x->dev,
271			"cdma_timeout: expired, but buffer had completed\n");
272		/* restore */
273		cmdproc_stop = prev_cmdproc & ~(BIT(ch->id));
274		host1x_sync_writel(host1x, cmdproc_stop,
275				   HOST1X_SYNC_CMDPROC_STOP);
276		mutex_unlock(&cdma->lock);
277		return;
278	}
279
280	dev_warn(host1x->dev, "%s: timeout: %d (%s), HW thresh %d, done %d\n",
281		__func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
282		syncpt_val, cdma->timeout.syncpt_val);
283
284	/* stop HW, resetting channel/module */
285	host1x_hw_cdma_freeze(host1x, cdma);
286
287	host1x_cdma_update_sync_queue(cdma, ch->dev);
288	mutex_unlock(&cdma->lock);
289}
290
291/*
292 * Init timeout resources
293 */
294static int cdma_timeout_init(struct host1x_cdma *cdma, u32 syncpt_id)
295{
296	INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
297	cdma->timeout.initialized = true;
298
299	return 0;
300}
301
302/*
303 * Clean up timeout resources
304 */
305static void cdma_timeout_destroy(struct host1x_cdma *cdma)
306{
307	if (cdma->timeout.initialized)
308		cancel_delayed_work(&cdma->timeout.wq);
309	cdma->timeout.initialized = false;
310}
311
312static const struct host1x_cdma_ops host1x_cdma_ops = {
313	.start = cdma_start,
314	.stop = cdma_stop,
315	.flush = cdma_flush,
316
317	.timeout_init = cdma_timeout_init,
318	.timeout_destroy = cdma_timeout_destroy,
319	.freeze = cdma_freeze,
320	.resume = cdma_resume,
321	.timeout_cpu_incr = cdma_timeout_cpu_incr,
322};
323
324static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
325	.init = push_buffer_init,
326};
327