1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24#include "priv.h"
25
26#include <core/gpuobj.h>
27#include <subdev/fb.h>
28#include <subdev/timer.h>
29#include <engine/gr.h>
30
31static void
32nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2])
33{
34	u64 phys = 0xdeadcafe00000000ULL;
35	u32 coverage = 0;
36
37	if (pgt[0]) {
38		/* present, 4KiB pages */
39		phys = 0x00000003 | nvkm_memory_addr(pgt[0]);
40		coverage = (nvkm_memory_size(pgt[0]) >> 3) << 12;
41	} else
42	if (pgt[1]) {
43		/* present, 64KiB pages  */
44		phys = 0x00000001 | nvkm_memory_addr(pgt[1]);
45		coverage = (nvkm_memory_size(pgt[1]) >> 3) << 16;
46	}
47
48	if (phys & 1) {
49		if (coverage <= 32 * 1024 * 1024)
50			phys |= 0x60;
51		else if (coverage <= 64 * 1024 * 1024)
52			phys |= 0x40;
53		else if (coverage <= 128 * 1024 * 1024)
54			phys |= 0x20;
55	}
56
57	nvkm_kmap(pgd);
58	nvkm_wo32(pgd, (pde * 8) + 0, lower_32_bits(phys));
59	nvkm_wo32(pgd, (pde * 8) + 4, upper_32_bits(phys));
60	nvkm_done(pgd);
61}
62
63static inline u64
64vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target)
65{
66	phys |= 1; /* present */
67	phys |= (u64)memtype << 40;
68	phys |= target << 4;
69	if (vma->access & NV_MEM_ACCESS_SYS)
70		phys |= (1 << 6);
71	if (!(vma->access & NV_MEM_ACCESS_WO))
72		phys |= (1 << 3);
73	return phys;
74}
75
76static void
77nv50_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt,
78	    struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
79{
80	struct nvkm_ram *ram = vma->vm->mmu->subdev.device->fb->ram;
81	u32 comp = (mem->memtype & 0x180) >> 7;
82	u32 block, target;
83	int i;
84
85	/* IGPs don't have real VRAM, re-target to stolen system memory */
86	target = 0;
87	if (ram->stolen) {
88		phys += ram->stolen;
89		target = 3;
90	}
91
92	phys  = vm_addr(vma, phys, mem->memtype, target);
93	pte <<= 3;
94	cnt <<= 3;
95
96	nvkm_kmap(pgt);
97	while (cnt) {
98		u32 offset_h = upper_32_bits(phys);
99		u32 offset_l = lower_32_bits(phys);
100
101		for (i = 7; i >= 0; i--) {
102			block = 1 << (i + 3);
103			if (cnt >= block && !(pte & (block - 1)))
104				break;
105		}
106		offset_l |= (i << 7);
107
108		phys += block << (vma->node->type - 3);
109		cnt  -= block;
110		if (comp) {
111			u32 tag = mem->tag->offset + ((delta >> 16) * comp);
112			offset_h |= (tag << 17);
113			delta    += block << (vma->node->type - 3);
114		}
115
116		while (block) {
117			nvkm_wo32(pgt, pte + 0, offset_l);
118			nvkm_wo32(pgt, pte + 4, offset_h);
119			pte += 8;
120			block -= 8;
121		}
122	}
123	nvkm_done(pgt);
124}
125
126static void
127nv50_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt,
128	       struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
129{
130	u32 target = (vma->access & NV_MEM_ACCESS_NOSNOOP) ? 3 : 2;
131	pte <<= 3;
132	nvkm_kmap(pgt);
133	while (cnt--) {
134		u64 phys = vm_addr(vma, (u64)*list++, mem->memtype, target);
135		nvkm_wo32(pgt, pte + 0, lower_32_bits(phys));
136		nvkm_wo32(pgt, pte + 4, upper_32_bits(phys));
137		pte += 8;
138	}
139	nvkm_done(pgt);
140}
141
142static void
143nv50_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt)
144{
145	pte <<= 3;
146	nvkm_kmap(pgt);
147	while (cnt--) {
148		nvkm_wo32(pgt, pte + 0, 0x00000000);
149		nvkm_wo32(pgt, pte + 4, 0x00000000);
150		pte += 8;
151	}
152	nvkm_done(pgt);
153}
154
155static void
156nv50_vm_flush(struct nvkm_vm *vm)
157{
158	struct nvkm_mmu *mmu = vm->mmu;
159	struct nvkm_subdev *subdev = &mmu->subdev;
160	struct nvkm_device *device = subdev->device;
161	int i, vme;
162
163	mutex_lock(&subdev->mutex);
164	for (i = 0; i < NVKM_SUBDEV_NR; i++) {
165		if (!atomic_read(&vm->engref[i]))
166			continue;
167
168		/* unfortunate hw bug workaround... */
169		if (i == NVKM_ENGINE_GR && device->gr) {
170			int ret = nvkm_gr_tlb_flush(device->gr);
171			if (ret != -ENODEV)
172				continue;
173		}
174
175		switch (i) {
176		case NVKM_ENGINE_GR    : vme = 0x00; break;
177		case NVKM_ENGINE_VP    :
178		case NVKM_ENGINE_MSPDEC: vme = 0x01; break;
179		case NVKM_SUBDEV_BAR   : vme = 0x06; break;
180		case NVKM_ENGINE_MSPPP :
181		case NVKM_ENGINE_MPEG  : vme = 0x08; break;
182		case NVKM_ENGINE_BSP   :
183		case NVKM_ENGINE_MSVLD : vme = 0x09; break;
184		case NVKM_ENGINE_CIPHER:
185		case NVKM_ENGINE_SEC   : vme = 0x0a; break;
186		case NVKM_ENGINE_CE0   : vme = 0x0d; break;
187		default:
188			continue;
189		}
190
191		nvkm_wr32(device, 0x100c80, (vme << 16) | 1);
192		if (nvkm_msec(device, 2000,
193			if (!(nvkm_rd32(device, 0x100c80) & 0x00000001))
194				break;
195		) < 0)
196			nvkm_error(subdev, "vm flush timeout: engine %d\n", vme);
197	}
198	mutex_unlock(&subdev->mutex);
199}
200
201static int
202nv50_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset,
203	       struct lock_class_key *key, struct nvkm_vm **pvm)
204{
205	u32 block = (1 << (mmu->func->pgt_bits + 12));
206	if (block > length)
207		block = length;
208
209	return nvkm_vm_create(mmu, offset, length, mm_offset, block, key, pvm);
210}
211
212static const struct nvkm_mmu_func
213nv50_mmu = {
214	.limit = (1ULL << 40),
215	.dma_bits = 40,
216	.pgt_bits  = 29 - 12,
217	.spg_shift = 12,
218	.lpg_shift = 16,
219	.create = nv50_vm_create,
220	.map_pgt = nv50_vm_map_pgt,
221	.map = nv50_vm_map,
222	.map_sg = nv50_vm_map_sg,
223	.unmap = nv50_vm_unmap,
224	.flush = nv50_vm_flush,
225};
226
227int
228nv50_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
229{
230	return nvkm_mmu_new_(&nv50_mmu, device, index, pmmu);
231}
232