This source file includes following definitions.
- nv50_instobj_wr32_slow
- nv50_instobj_rd32_slow
- nv50_instobj_wr32
- nv50_instobj_rd32
- nv50_instobj_kmap
- nv50_instobj_map
- nv50_instobj_release
- nv50_instobj_acquire
- nv50_instobj_boot
- nv50_instobj_size
- nv50_instobj_addr
- nv50_instobj_bar2
- nv50_instobj_target
- nv50_instobj_dtor
- nv50_instobj_new
- nv50_instmem_fini
- nv50_instmem_new
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #define nv50_instmem(p) container_of((p), struct nv50_instmem, base)
25 #include "priv.h"
26
27 #include <core/memory.h>
28 #include <subdev/bar.h>
29 #include <subdev/fb.h>
30 #include <subdev/mmu.h>
31
32 struct nv50_instmem {
33 struct nvkm_instmem base;
34 u64 addr;
35
36
37 struct list_head lru;
38 };
39
40
41
42
43 #define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
44
45 struct nv50_instobj {
46 struct nvkm_instobj base;
47 struct nv50_instmem *imem;
48 struct nvkm_memory *ram;
49 struct nvkm_vma *bar;
50 refcount_t maps;
51 void *map;
52 struct list_head lru;
53 };
54
55 static void
56 nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
57 {
58 struct nv50_instobj *iobj = nv50_instobj(memory);
59 struct nv50_instmem *imem = iobj->imem;
60 struct nvkm_device *device = imem->base.subdev.device;
61 u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
62 u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
63 unsigned long flags;
64
65 spin_lock_irqsave(&imem->base.lock, flags);
66 if (unlikely(imem->addr != base)) {
67 nvkm_wr32(device, 0x001700, base >> 16);
68 imem->addr = base;
69 }
70 nvkm_wr32(device, 0x700000 + addr, data);
71 spin_unlock_irqrestore(&imem->base.lock, flags);
72 }
73
74 static u32
75 nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
76 {
77 struct nv50_instobj *iobj = nv50_instobj(memory);
78 struct nv50_instmem *imem = iobj->imem;
79 struct nvkm_device *device = imem->base.subdev.device;
80 u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
81 u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
82 u32 data;
83 unsigned long flags;
84
85 spin_lock_irqsave(&imem->base.lock, flags);
86 if (unlikely(imem->addr != base)) {
87 nvkm_wr32(device, 0x001700, base >> 16);
88 imem->addr = base;
89 }
90 data = nvkm_rd32(device, 0x700000 + addr);
91 spin_unlock_irqrestore(&imem->base.lock, flags);
92 return data;
93 }
94
95 static const struct nvkm_memory_ptrs
96 nv50_instobj_slow = {
97 .rd32 = nv50_instobj_rd32_slow,
98 .wr32 = nv50_instobj_wr32_slow,
99 };
100
101 static void
102 nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
103 {
104 iowrite32_native(data, nv50_instobj(memory)->map + offset);
105 }
106
107 static u32
108 nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
109 {
110 return ioread32_native(nv50_instobj(memory)->map + offset);
111 }
112
113 static const struct nvkm_memory_ptrs
114 nv50_instobj_fast = {
115 .rd32 = nv50_instobj_rd32,
116 .wr32 = nv50_instobj_wr32,
117 };
118
119 static void
120 nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
121 {
122 struct nv50_instmem *imem = iobj->imem;
123 struct nv50_instobj *eobj;
124 struct nvkm_memory *memory = &iobj->base.memory;
125 struct nvkm_subdev *subdev = &imem->base.subdev;
126 struct nvkm_device *device = subdev->device;
127 struct nvkm_vma *bar = NULL, *ebar;
128 u64 size = nvkm_memory_size(memory);
129 void *emap;
130 int ret;
131
132
133
134
135
136 mutex_unlock(&subdev->mutex);
137 while ((ret = nvkm_vmm_get(vmm, 12, size, &bar))) {
138
139
140
141 mutex_lock(&subdev->mutex);
142 eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);
143 if (eobj) {
144 nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",
145 nvkm_memory_addr(&eobj->base.memory),
146 nvkm_memory_size(&eobj->base.memory),
147 eobj->bar->addr);
148 list_del_init(&eobj->lru);
149 ebar = eobj->bar;
150 eobj->bar = NULL;
151 emap = eobj->map;
152 eobj->map = NULL;
153 }
154 mutex_unlock(&subdev->mutex);
155 if (!eobj)
156 break;
157 iounmap(emap);
158 nvkm_vmm_put(vmm, &ebar);
159 }
160
161 if (ret == 0)
162 ret = nvkm_memory_map(memory, 0, vmm, bar, NULL, 0);
163 mutex_lock(&subdev->mutex);
164 if (ret || iobj->bar) {
165
166 mutex_unlock(&subdev->mutex);
167 nvkm_vmm_put(vmm, &bar);
168 mutex_lock(&subdev->mutex);
169 return;
170 }
171
172
173 iobj->bar = bar;
174 iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +
175 (u32)iobj->bar->addr, size);
176 if (!iobj->map) {
177 nvkm_warn(subdev, "PRAMIN ioremap failed\n");
178 nvkm_vmm_put(vmm, &iobj->bar);
179 }
180 }
181
182 static int
183 nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
184 struct nvkm_vma *vma, void *argv, u32 argc)
185 {
186 memory = nv50_instobj(memory)->ram;
187 return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
188 }
189
190 static void
191 nv50_instobj_release(struct nvkm_memory *memory)
192 {
193 struct nv50_instobj *iobj = nv50_instobj(memory);
194 struct nv50_instmem *imem = iobj->imem;
195 struct nvkm_subdev *subdev = &imem->base.subdev;
196
197 wmb();
198 nvkm_bar_flush(subdev->device->bar);
199
200 if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) {
201
202
203
204 if (likely(iobj->lru.next) && iobj->map) {
205 BUG_ON(!list_empty(&iobj->lru));
206 list_add_tail(&iobj->lru, &imem->lru);
207 }
208
209
210 iobj->base.memory.ptrs = NULL;
211 mutex_unlock(&subdev->mutex);
212 }
213 }
214
215 static void __iomem *
216 nv50_instobj_acquire(struct nvkm_memory *memory)
217 {
218 struct nv50_instobj *iobj = nv50_instobj(memory);
219 struct nvkm_instmem *imem = &iobj->imem->base;
220 struct nvkm_vmm *vmm;
221 void __iomem *map = NULL;
222
223
224 if (refcount_inc_not_zero(&iobj->maps))
225 return iobj->map;
226
227
228
229
230 mutex_lock(&imem->subdev.mutex);
231 if (refcount_inc_not_zero(&iobj->maps)) {
232 mutex_unlock(&imem->subdev.mutex);
233 return iobj->map;
234 }
235
236
237 if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
238 if (!iobj->map)
239 nv50_instobj_kmap(iobj, vmm);
240 map = iobj->map;
241 }
242
243 if (!refcount_inc_not_zero(&iobj->maps)) {
244
245 if (likely(iobj->lru.next))
246 list_del_init(&iobj->lru);
247
248 if (map)
249 iobj->base.memory.ptrs = &nv50_instobj_fast;
250 else
251 iobj->base.memory.ptrs = &nv50_instobj_slow;
252 refcount_set(&iobj->maps, 1);
253 }
254
255 mutex_unlock(&imem->subdev.mutex);
256 return map;
257 }
258
259 static void
260 nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
261 {
262 struct nv50_instobj *iobj = nv50_instobj(memory);
263 struct nvkm_instmem *imem = &iobj->imem->base;
264
265
266
267
268 mutex_lock(&imem->subdev.mutex);
269 if (likely(iobj->lru.next)) {
270 list_del_init(&iobj->lru);
271 iobj->lru.next = NULL;
272 }
273
274 nv50_instobj_kmap(iobj, vmm);
275 nvkm_instmem_boot(imem);
276 mutex_unlock(&imem->subdev.mutex);
277 }
278
279 static u64
280 nv50_instobj_size(struct nvkm_memory *memory)
281 {
282 return nvkm_memory_size(nv50_instobj(memory)->ram);
283 }
284
285 static u64
286 nv50_instobj_addr(struct nvkm_memory *memory)
287 {
288 return nvkm_memory_addr(nv50_instobj(memory)->ram);
289 }
290
291 static u64
292 nv50_instobj_bar2(struct nvkm_memory *memory)
293 {
294 struct nv50_instobj *iobj = nv50_instobj(memory);
295 u64 addr = ~0ULL;
296 if (nv50_instobj_acquire(&iobj->base.memory)) {
297 iobj->lru.next = NULL;
298 addr = iobj->bar->addr;
299 }
300 nv50_instobj_release(&iobj->base.memory);
301 return addr;
302 }
303
304 static enum nvkm_memory_target
305 nv50_instobj_target(struct nvkm_memory *memory)
306 {
307 return nvkm_memory_target(nv50_instobj(memory)->ram);
308 }
309
310 static void *
311 nv50_instobj_dtor(struct nvkm_memory *memory)
312 {
313 struct nv50_instobj *iobj = nv50_instobj(memory);
314 struct nvkm_instmem *imem = &iobj->imem->base;
315 struct nvkm_vma *bar;
316 void *map = map;
317
318 mutex_lock(&imem->subdev.mutex);
319 if (likely(iobj->lru.next))
320 list_del(&iobj->lru);
321 map = iobj->map;
322 bar = iobj->bar;
323 mutex_unlock(&imem->subdev.mutex);
324
325 if (map) {
326 struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device);
327 iounmap(map);
328 if (likely(vmm))
329 nvkm_vmm_put(vmm, &bar);
330 }
331
332 nvkm_memory_unref(&iobj->ram);
333 nvkm_instobj_dtor(imem, &iobj->base);
334 return iobj;
335 }
336
337 static const struct nvkm_memory_func
338 nv50_instobj_func = {
339 .dtor = nv50_instobj_dtor,
340 .target = nv50_instobj_target,
341 .bar2 = nv50_instobj_bar2,
342 .addr = nv50_instobj_addr,
343 .size = nv50_instobj_size,
344 .boot = nv50_instobj_boot,
345 .acquire = nv50_instobj_acquire,
346 .release = nv50_instobj_release,
347 .map = nv50_instobj_map,
348 };
349
350 static int
351 nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
352 struct nvkm_memory **pmemory)
353 {
354 struct nv50_instmem *imem = nv50_instmem(base);
355 struct nv50_instobj *iobj;
356 struct nvkm_device *device = imem->base.subdev.device;
357 u8 page = max(order_base_2(align), 12);
358
359 if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
360 return -ENOMEM;
361 *pmemory = &iobj->base.memory;
362
363 nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
364 iobj->imem = imem;
365 refcount_set(&iobj->maps, 0);
366 INIT_LIST_HEAD(&iobj->lru);
367
368 return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram);
369 }
370
371
372
373
374
375 static void
376 nv50_instmem_fini(struct nvkm_instmem *base)
377 {
378 nv50_instmem(base)->addr = ~0ULL;
379 }
380
381 static const struct nvkm_instmem_func
382 nv50_instmem = {
383 .fini = nv50_instmem_fini,
384 .memory_new = nv50_instobj_new,
385 .zero = false,
386 };
387
388 int
389 nv50_instmem_new(struct nvkm_device *device, int index,
390 struct nvkm_instmem **pimem)
391 {
392 struct nv50_instmem *imem;
393
394 if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))
395 return -ENOMEM;
396 nvkm_instmem_ctor(&nv50_instmem, device, index, &imem->base);
397 INIT_LIST_HEAD(&imem->lru);
398 *pimem = &imem->base;
399 return 0;
400 }