31 OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords)  in OUT_RINGp()  argument
34 	u32 *mem = ttm_kmap_obj_virtual(&chan->push.buffer->kmap, &is_iomem);  in OUT_RINGp()
35 	mem = &mem[chan->dma.cur];  in OUT_RINGp()
40 	chan->dma.cur += nr_dwords;  in OUT_RINGp()
51 READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout)  in READ_GET()  argument
55 	val = nvif_rd32(&chan->user, chan->user_get);  in READ_GET()
56         if (chan->user_get_hi)  in READ_GET()
57                 val |= (uint64_t)nvif_rd32(&chan->user, chan->user_get_hi) << 32;  in READ_GET()
74 	if (val < chan->push.vma.offset ||  in READ_GET()
75 	    val > chan->push.vma.offset + (chan->dma.max << 2))  in READ_GET()
78 	return (val - chan->push.vma.offset) >> 2;  in READ_GET()
82 nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,  in nv50_dma_push()  argument
85 	struct nouveau_cli *cli = (void *)chan->user.client;  in nv50_dma_push()
86 	struct nouveau_bo *pb = chan->push.buffer;  in nv50_dma_push()
88 	int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;  in nv50_dma_push()
95 	BUG_ON(chan->dma.ib_free < 1);  in nv50_dma_push()
100 	chan->dma.ib_put = (chan->dma.ib_put + 1) & chan->dma.ib_max;  in nv50_dma_push()
106 	nvif_wr32(&chan->user, 0x8c, chan->dma.ib_put);  in nv50_dma_push()
107 	chan->dma.ib_free--;  in nv50_dma_push()
111 nv50_dma_push_wait(struct nouveau_channel *chan, int count)  in nv50_dma_push_wait()  argument
115 	while (chan->dma.ib_free < count) {  in nv50_dma_push_wait()
116 		uint32_t get = nvif_rd32(&chan->user, 0x88);  in nv50_dma_push_wait()
128 		chan->dma.ib_free = get - chan->dma.ib_put;  in nv50_dma_push_wait()
129 		if (chan->dma.ib_free <= 0)  in nv50_dma_push_wait()
130 			chan->dma.ib_free += chan->dma.ib_max;  in nv50_dma_push_wait()
137 nv50_dma_wait(struct nouveau_channel *chan, int slots, int count)  in nv50_dma_wait()  argument
142 	ret = nv50_dma_push_wait(chan, slots + 1);  in nv50_dma_wait()
146 	while (chan->dma.free < count) {  in nv50_dma_wait()
147 		int get = READ_GET(chan, &prev_get, &cnt);  in nv50_dma_wait()
155 		if (get <= chan->dma.cur) {  in nv50_dma_wait()
156 			chan->dma.free = chan->dma.max - chan->dma.cur;  in nv50_dma_wait()
157 			if (chan->dma.free >= count)  in nv50_dma_wait()
160 			FIRE_RING(chan);  in nv50_dma_wait()
162 				get = READ_GET(chan, &prev_get, &cnt);  in nv50_dma_wait()
169 			chan->dma.cur = 0;  in nv50_dma_wait()
170 			chan->dma.put = 0;  in nv50_dma_wait()
173 		chan->dma.free = get - chan->dma.cur - 1;  in nv50_dma_wait()
180 nouveau_dma_wait(struct nouveau_channel *chan, int slots, int size)  in nouveau_dma_wait()  argument
185 	if (chan->dma.ib_max)  in nouveau_dma_wait()
186 		return nv50_dma_wait(chan, slots, size);  in nouveau_dma_wait()
188 	while (chan->dma.free < size) {  in nouveau_dma_wait()
189 		get = READ_GET(chan, &prev_get, &cnt);  in nouveau_dma_wait()
205 		if (get <= chan->dma.cur) {  in nouveau_dma_wait()
219 			chan->dma.free = chan->dma.max - chan->dma.cur;  in nouveau_dma_wait()
220 			if (chan->dma.free >= size)  in nouveau_dma_wait()
227 			OUT_RING(chan, chan->push.vma.offset | 0x20000000);  in nouveau_dma_wait()
235 				get = READ_GET(chan, &prev_get, &cnt);  in nouveau_dma_wait()
246 			chan->dma.cur  =  in nouveau_dma_wait()
247 			chan->dma.put  = NOUVEAU_DMA_SKIPS;  in nouveau_dma_wait()
256 		chan->dma.free = get - chan->dma.cur - 1;  in nouveau_dma_wait()