This source file includes following definitions.
- nvkm_mmu_ptp_put
- nvkm_mmu_ptp_get
- nvkm_mmu_ptc_find
- nvkm_mmu_ptc_put
- nvkm_mmu_ptc_get
- nvkm_mmu_ptc_dump
- nvkm_mmu_ptc_fini
- nvkm_mmu_ptc_init
- nvkm_mmu_type
- nvkm_mmu_heap
- nvkm_mmu_host
- nvkm_mmu_vram
- nvkm_mmu_oneinit
- nvkm_mmu_init
- nvkm_mmu_dtor
- nvkm_mmu_ctor
- nvkm_mmu_new_
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #include "ummu.h"
25 #include "vmm.h"
26
27 #include <subdev/bar.h>
28 #include <subdev/fb.h>
29
30 #include <nvif/if500d.h>
31 #include <nvif/if900d.h>
32
33 struct nvkm_mmu_ptp {
34 struct nvkm_mmu_pt *pt;
35 struct list_head head;
36 u8 shift;
37 u16 mask;
38 u16 free;
39 };
40
41 static void
42 nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt)
43 {
44 const int slot = pt->base >> pt->ptp->shift;
45 struct nvkm_mmu_ptp *ptp = pt->ptp;
46
47
48
49
50 if (!ptp->free)
51 list_add(&ptp->head, &mmu->ptp.list);
52 ptp->free |= BIT(slot);
53
54
55 if (ptp->free == ptp->mask) {
56 nvkm_mmu_ptc_put(mmu, force, &ptp->pt);
57 list_del(&ptp->head);
58 kfree(ptp);
59 }
60
61 kfree(pt);
62 }
63
64 struct nvkm_mmu_pt *
65 nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero)
66 {
67 struct nvkm_mmu_pt *pt;
68 struct nvkm_mmu_ptp *ptp;
69 int slot;
70
71 if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL)))
72 return NULL;
73
74 ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head);
75 if (!ptp) {
76
77 if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) {
78 kfree(pt);
79 return NULL;
80 }
81
82 ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false);
83 if (!ptp->pt) {
84 kfree(ptp);
85 kfree(pt);
86 return NULL;
87 }
88
89 ptp->shift = order_base_2(size);
90 slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift;
91 ptp->mask = (1 << slot) - 1;
92 ptp->free = ptp->mask;
93 list_add(&ptp->head, &mmu->ptp.list);
94 }
95 pt->ptp = ptp;
96 pt->sub = true;
97
98
99
100
101 slot = __ffs(ptp->free);
102 ptp->free &= ~BIT(slot);
103 if (!ptp->free)
104 list_del(&ptp->head);
105
106 pt->memory = pt->ptp->pt->memory;
107 pt->base = slot << ptp->shift;
108 pt->addr = pt->ptp->pt->addr + pt->base;
109 return pt;
110 }
111
112 struct nvkm_mmu_ptc {
113 struct list_head head;
114 struct list_head item;
115 u32 size;
116 u32 refs;
117 };
118
119 static inline struct nvkm_mmu_ptc *
120 nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size)
121 {
122 struct nvkm_mmu_ptc *ptc;
123
124 list_for_each_entry(ptc, &mmu->ptc.list, head) {
125 if (ptc->size == size)
126 return ptc;
127 }
128
129 ptc = kmalloc(sizeof(*ptc), GFP_KERNEL);
130 if (ptc) {
131 INIT_LIST_HEAD(&ptc->item);
132 ptc->size = size;
133 ptc->refs = 0;
134 list_add(&ptc->head, &mmu->ptc.list);
135 }
136
137 return ptc;
138 }
139
140 void
141 nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt)
142 {
143 struct nvkm_mmu_pt *pt = *ppt;
144 if (pt) {
145
146 if (pt->sub) {
147 mutex_lock(&mmu->ptp.mutex);
148 nvkm_mmu_ptp_put(mmu, force, pt);
149 mutex_unlock(&mmu->ptp.mutex);
150 return;
151 }
152
153
154 mutex_lock(&mmu->ptc.mutex);
155 if (pt->ptc->refs < 8 && !force) {
156 list_add_tail(&pt->head, &pt->ptc->item);
157 pt->ptc->refs++;
158 } else {
159 nvkm_memory_unref(&pt->memory);
160 kfree(pt);
161 }
162 mutex_unlock(&mmu->ptc.mutex);
163 }
164 }
165
166 struct nvkm_mmu_pt *
167 nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero)
168 {
169 struct nvkm_mmu_ptc *ptc;
170 struct nvkm_mmu_pt *pt;
171 int ret;
172
173
174 if (align < 0x1000) {
175 mutex_lock(&mmu->ptp.mutex);
176 pt = nvkm_mmu_ptp_get(mmu, align, zero);
177 mutex_unlock(&mmu->ptp.mutex);
178 return pt;
179 }
180
181
182 mutex_lock(&mmu->ptc.mutex);
183 ptc = nvkm_mmu_ptc_find(mmu, size);
184 if (!ptc) {
185 mutex_unlock(&mmu->ptc.mutex);
186 return NULL;
187 }
188
189
190 pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head);
191 if (pt) {
192 if (zero)
193 nvkm_fo64(pt->memory, 0, 0, size >> 3);
194 list_del(&pt->head);
195 ptc->refs--;
196 mutex_unlock(&mmu->ptc.mutex);
197 return pt;
198 }
199 mutex_unlock(&mmu->ptc.mutex);
200
201
202 if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL)))
203 return NULL;
204 pt->ptc = ptc;
205 pt->sub = false;
206
207 ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
208 size, align, zero, &pt->memory);
209 if (ret) {
210 kfree(pt);
211 return NULL;
212 }
213
214 pt->base = 0;
215 pt->addr = nvkm_memory_addr(pt->memory);
216 return pt;
217 }
218
219 void
220 nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu)
221 {
222 struct nvkm_mmu_ptc *ptc;
223 list_for_each_entry(ptc, &mmu->ptc.list, head) {
224 struct nvkm_mmu_pt *pt, *tt;
225 list_for_each_entry_safe(pt, tt, &ptc->item, head) {
226 nvkm_memory_unref(&pt->memory);
227 list_del(&pt->head);
228 kfree(pt);
229 }
230 }
231 }
232
233 static void
234 nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu)
235 {
236 struct nvkm_mmu_ptc *ptc, *ptct;
237
238 list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) {
239 WARN_ON(!list_empty(&ptc->item));
240 list_del(&ptc->head);
241 kfree(ptc);
242 }
243 }
244
245 static void
246 nvkm_mmu_ptc_init(struct nvkm_mmu *mmu)
247 {
248 mutex_init(&mmu->ptc.mutex);
249 INIT_LIST_HEAD(&mmu->ptc.list);
250 mutex_init(&mmu->ptp.mutex);
251 INIT_LIST_HEAD(&mmu->ptp.list);
252 }
253
254 static void
255 nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type)
256 {
257 if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) {
258 mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type;
259 mmu->type[mmu->type_nr].heap = heap;
260 mmu->type_nr++;
261 }
262 }
263
264 static int
265 nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size)
266 {
267 if (size) {
268 if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) {
269 mmu->heap[mmu->heap_nr].type = type;
270 mmu->heap[mmu->heap_nr].size = size;
271 return mmu->heap_nr++;
272 }
273 }
274 return -EINVAL;
275 }
276
277 static void
278 nvkm_mmu_host(struct nvkm_mmu *mmu)
279 {
280 struct nvkm_device *device = mmu->subdev.device;
281 u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys;
282 int heap;
283
284
285 heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL);
286 nvkm_mmu_type(mmu, heap, type);
287
288
289
290
291
292
293
294 type |= NVKM_MEM_MAPPABLE;
295 if (!device->bar || device->bar->iomap_uncached)
296 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
297 else
298 nvkm_mmu_type(mmu, heap, type);
299
300
301
302
303
304
305
306 type |= NVKM_MEM_COHERENT;
307 if (device->func->cpu_coherent)
308 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
309
310
311 nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED);
312 }
313
314 static void
315 nvkm_mmu_vram(struct nvkm_mmu *mmu)
316 {
317 struct nvkm_device *device = mmu->subdev.device;
318 struct nvkm_mm *mm = &device->fb->ram->vram;
319 const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
320 const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
321 const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
322 u8 type = NVKM_MEM_KIND * !!mmu->func->kind;
323 u8 heap = NVKM_MEM_VRAM;
324 int heapM, heapN, heapU;
325
326
327 heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT);
328
329 heap |= NVKM_MEM_COMP;
330 heap |= NVKM_MEM_DISP;
331 heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT);
332 heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT);
333
334
335
336
337
338 nvkm_mmu_type(mmu, heapU, type);
339 nvkm_mmu_type(mmu, heapN, type);
340 nvkm_mmu_type(mmu, heapM, type);
341
342
343
344
345
346 nvkm_mmu_host(mmu);
347
348
349
350
351 if (device->bar) {
352
353 type |= NVKM_MEM_MAPPABLE;
354 if (!device->bar->iomap_uncached) {
355 nvkm_mmu_type(mmu, heapN, type);
356 nvkm_mmu_type(mmu, heapM, type);
357 }
358
359
360 type |= NVKM_MEM_COHERENT;
361 type |= NVKM_MEM_UNCACHED;
362 nvkm_mmu_type(mmu, heapN, type);
363 nvkm_mmu_type(mmu, heapM, type);
364 }
365 }
366
367 static int
368 nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
369 {
370 struct nvkm_mmu *mmu = nvkm_mmu(subdev);
371
372
373 if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram)
374 nvkm_mmu_vram(mmu);
375 else
376 nvkm_mmu_host(mmu);
377
378 if (mmu->func->vmm.global) {
379 int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL,
380 "gart", &mmu->vmm);
381 if (ret)
382 return ret;
383 }
384
385 return 0;
386 }
387
388 static int
389 nvkm_mmu_init(struct nvkm_subdev *subdev)
390 {
391 struct nvkm_mmu *mmu = nvkm_mmu(subdev);
392 if (mmu->func->init)
393 mmu->func->init(mmu);
394 return 0;
395 }
396
397 static void *
398 nvkm_mmu_dtor(struct nvkm_subdev *subdev)
399 {
400 struct nvkm_mmu *mmu = nvkm_mmu(subdev);
401
402 nvkm_vmm_unref(&mmu->vmm);
403
404 nvkm_mmu_ptc_fini(mmu);
405 return mmu;
406 }
407
408 static const struct nvkm_subdev_func
409 nvkm_mmu = {
410 .dtor = nvkm_mmu_dtor,
411 .oneinit = nvkm_mmu_oneinit,
412 .init = nvkm_mmu_init,
413 };
414
415 void
416 nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device,
417 int index, struct nvkm_mmu *mmu)
418 {
419 nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev);
420 mmu->func = func;
421 mmu->dma_bits = func->dma_bits;
422 nvkm_mmu_ptc_init(mmu);
423 mmu->user.ctor = nvkm_ummu_new;
424 mmu->user.base = func->mmu.user;
425 }
426
427 int
428 nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device,
429 int index, struct nvkm_mmu **pmmu)
430 {
431 if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL)))
432 return -ENOMEM;
433 nvkm_mmu_ctor(func, device, index, *pmmu);
434 return 0;
435 }