This source file includes following definitions.
- gmc_v10_0_vm_fault_interrupt_state
- gmc_v10_0_process_interrupt
- gmc_v10_0_set_irq_funcs
- gmc_v10_0_get_invalidate_req
- gmc_v10_0_flush_vm_hub
- gmc_v10_0_flush_gpu_tlb
- gmc_v10_0_emit_flush_gpu_tlb
- gmc_v10_0_emit_pasid_mapping
- gmc_v10_0_get_vm_pte_flags
- gmc_v10_0_get_vm_pde
- gmc_v10_0_set_gmc_funcs
- gmc_v10_0_early_init
- gmc_v10_0_late_init
- gmc_v10_0_vram_gtt_location
- gmc_v10_0_mc_init
- gmc_v10_0_gart_init
- gmc_v10_0_get_vbios_fb_size
- gmc_v10_0_sw_init
- gmc_v10_0_gart_fini
- gmc_v10_0_sw_fini
- gmc_v10_0_init_golden_registers
- gmc_v10_0_gart_enable
- gmc_v10_0_hw_init
- gmc_v10_0_gart_disable
- gmc_v10_0_hw_fini
- gmc_v10_0_suspend
- gmc_v10_0_resume
- gmc_v10_0_is_idle
- gmc_v10_0_wait_for_idle
- gmc_v10_0_soft_reset
- gmc_v10_0_set_clockgating_state
- gmc_v10_0_get_clockgating_state
- gmc_v10_0_set_powergating_state
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 #include <linux/firmware.h>
24 #include <linux/pci.h>
25 #include "amdgpu.h"
26 #include "amdgpu_atomfirmware.h"
27 #include "gmc_v10_0.h"
28
29 #include "hdp/hdp_5_0_0_offset.h"
30 #include "hdp/hdp_5_0_0_sh_mask.h"
31 #include "gc/gc_10_1_0_sh_mask.h"
32 #include "mmhub/mmhub_2_0_0_sh_mask.h"
33 #include "dcn/dcn_2_0_0_offset.h"
34 #include "dcn/dcn_2_0_0_sh_mask.h"
35 #include "oss/osssys_5_0_0_offset.h"
36 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
37 #include "navi10_enum.h"
38
39 #include "soc15.h"
40 #include "soc15_common.h"
41
42 #include "nbio_v2_3.h"
43
44 #include "gfxhub_v2_0.h"
45 #include "mmhub_v2_0.h"
46 #include "athub_v2_0.h"
47
48 #define AMDGPU_NUM_OF_VMIDS 8
49
50 #if 0
51 static const struct soc15_reg_golden golden_settings_navi10_hdp[] =
52 {
53
54 };
55 #endif
56
57 static int
58 gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
59 struct amdgpu_irq_src *src, unsigned type,
60 enum amdgpu_interrupt_state state)
61 {
62 struct amdgpu_vmhub *hub;
63 u32 tmp, reg, bits[AMDGPU_MAX_VMHUBS], i;
64
65 bits[AMDGPU_GFXHUB_0] = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
66 GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
67 GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
68 GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
69 GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
70 GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
71 GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
72
73 bits[AMDGPU_MMHUB_0] = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
74 MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
75 MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
76 MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
77 MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
78 MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
79 MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
80
81 switch (state) {
82 case AMDGPU_IRQ_STATE_DISABLE:
83
84 hub = &adev->vmhub[AMDGPU_MMHUB_0];
85 for (i = 0; i < 16; i++) {
86 reg = hub->vm_context0_cntl + i;
87 tmp = RREG32(reg);
88 tmp &= ~bits[AMDGPU_MMHUB_0];
89 WREG32(reg, tmp);
90 }
91
92
93 hub = &adev->vmhub[AMDGPU_GFXHUB_0];
94 for (i = 0; i < 16; i++) {
95 reg = hub->vm_context0_cntl + i;
96 tmp = RREG32(reg);
97 tmp &= ~bits[AMDGPU_GFXHUB_0];
98 WREG32(reg, tmp);
99 }
100 break;
101 case AMDGPU_IRQ_STATE_ENABLE:
102
103 hub = &adev->vmhub[AMDGPU_MMHUB_0];
104 for (i = 0; i < 16; i++) {
105 reg = hub->vm_context0_cntl + i;
106 tmp = RREG32(reg);
107 tmp |= bits[AMDGPU_MMHUB_0];
108 WREG32(reg, tmp);
109 }
110
111
112 hub = &adev->vmhub[AMDGPU_GFXHUB_0];
113 for (i = 0; i < 16; i++) {
114 reg = hub->vm_context0_cntl + i;
115 tmp = RREG32(reg);
116 tmp |= bits[AMDGPU_GFXHUB_0];
117 WREG32(reg, tmp);
118 }
119 break;
120 default:
121 break;
122 }
123
124 return 0;
125 }
126
127 static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,
128 struct amdgpu_irq_src *source,
129 struct amdgpu_iv_entry *entry)
130 {
131 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
132 uint32_t status = 0;
133 u64 addr;
134
135 addr = (u64)entry->src_data[0] << 12;
136 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
137
138 if (!amdgpu_sriov_vf(adev)) {
139
140
141
142
143
144 if (entry->vmid_src == AMDGPU_GFXHUB_0)
145 RREG32(hub->vm_l2_pro_fault_status);
146
147 status = RREG32(hub->vm_l2_pro_fault_status);
148 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
149 }
150
151 if (printk_ratelimit()) {
152 struct amdgpu_task_info task_info;
153
154 memset(&task_info, 0, sizeof(struct amdgpu_task_info));
155 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
156
157 dev_err(adev->dev,
158 "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
159 "for process %s pid %d thread %s pid %d)\n",
160 entry->vmid_src ? "mmhub" : "gfxhub",
161 entry->src_id, entry->ring_id, entry->vmid,
162 entry->pasid, task_info.process_name, task_info.tgid,
163 task_info.task_name, task_info.pid);
164 dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n",
165 addr, entry->client_id);
166 if (!amdgpu_sriov_vf(adev)) {
167 dev_err(adev->dev,
168 "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
169 status);
170 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
171 REG_GET_FIELD(status,
172 GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
173 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
174 REG_GET_FIELD(status,
175 GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
176 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
177 REG_GET_FIELD(status,
178 GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
179 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
180 REG_GET_FIELD(status,
181 GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
182 dev_err(adev->dev, "\t RW: 0x%lx\n",
183 REG_GET_FIELD(status,
184 GCVM_L2_PROTECTION_FAULT_STATUS, RW));
185 }
186 }
187
188 return 0;
189 }
190
191 static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = {
192 .set = gmc_v10_0_vm_fault_interrupt_state,
193 .process = gmc_v10_0_process_interrupt,
194 };
195
196 static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev)
197 {
198 adev->gmc.vm_fault.num_types = 1;
199 adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs;
200 }
201
202 static uint32_t gmc_v10_0_get_invalidate_req(unsigned int vmid,
203 uint32_t flush_type)
204 {
205 u32 req = 0;
206
207
208 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
209 PER_VMID_INVALIDATE_REQ, 1 << vmid);
210 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
211 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
212 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
213 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
214 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
215 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
216 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
217 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
218
219 return req;
220 }
221
222
223
224
225
226
227
228
229 static void gmc_v10_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
230 unsigned int vmhub, uint32_t flush_type)
231 {
232 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
233 u32 inv_req = gmc_v10_0_get_invalidate_req(vmid, flush_type);
234 u32 tmp;
235
236 const unsigned eng = 17;
237 unsigned int i;
238
239 spin_lock(&adev->gmc.invalidate_lock);
240
241
242
243
244
245
246
247
248 if (vmhub == AMDGPU_MMHUB_0 ||
249 vmhub == AMDGPU_MMHUB_1) {
250 for (i = 0; i < adev->usec_timeout; i++) {
251
252 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + eng);
253 if (tmp & 0x1)
254 break;
255 udelay(1);
256 }
257
258 if (i >= adev->usec_timeout)
259 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
260 }
261
262 WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, inv_req);
263
264
265
266
267
268 if (vmhub == AMDGPU_GFXHUB_0)
269 RREG32_NO_KIQ(hub->vm_inv_eng0_req + eng);
270
271
272 for (i = 0; i < adev->usec_timeout; i++) {
273 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
274 tmp &= 1 << vmid;
275 if (tmp)
276 break;
277
278 udelay(1);
279 }
280
281
282 if (vmhub == AMDGPU_MMHUB_0 ||
283 vmhub == AMDGPU_MMHUB_1)
284
285
286
287
288 WREG32_NO_KIQ(hub->vm_inv_eng0_sem + eng, 0);
289
290 spin_unlock(&adev->gmc.invalidate_lock);
291
292 if (i < adev->usec_timeout)
293 return;
294
295 DRM_ERROR("Timeout waiting for VM flush ACK!\n");
296 }
297
298
299
300
301
302
303
304
305
306 static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
307 uint32_t vmhub, uint32_t flush_type)
308 {
309 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
310 struct dma_fence *fence;
311 struct amdgpu_job *job;
312
313 int r;
314
315
316 adev->nbio_funcs->hdp_flush(adev, NULL);
317
318 mutex_lock(&adev->mman.gtt_window_lock);
319
320 if (vmhub == AMDGPU_MMHUB_0) {
321 gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_MMHUB_0, 0);
322 mutex_unlock(&adev->mman.gtt_window_lock);
323 return;
324 }
325
326 BUG_ON(vmhub != AMDGPU_GFXHUB_0);
327
328 if (!adev->mman.buffer_funcs_enabled ||
329 !adev->ib_pool_ready ||
330 adev->in_gpu_reset) {
331 gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_GFXHUB_0, 0);
332 mutex_unlock(&adev->mman.gtt_window_lock);
333 return;
334 }
335
336
337
338
339
340
341 r = amdgpu_job_alloc_with_ib(adev, 16 * 4, &job);
342 if (r)
343 goto error_alloc;
344
345 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
346 job->vm_needs_flush = true;
347 job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
348 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
349 r = amdgpu_job_submit(job, &adev->mman.entity,
350 AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
351 if (r)
352 goto error_submit;
353
354 mutex_unlock(&adev->mman.gtt_window_lock);
355
356 dma_fence_wait(fence, false);
357 dma_fence_put(fence);
358
359 return;
360
361 error_submit:
362 amdgpu_job_free(job);
363
364 error_alloc:
365 mutex_unlock(&adev->mman.gtt_window_lock);
366 DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
367 }
368
369 static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
370 unsigned vmid, uint64_t pd_addr)
371 {
372 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
373 uint32_t req = gmc_v10_0_get_invalidate_req(vmid, 0);
374 unsigned eng = ring->vm_inv_eng;
375
376
377
378
379
380
381
382
383
384 if (ring->funcs->vmhub == AMDGPU_MMHUB_0 ||
385 ring->funcs->vmhub == AMDGPU_MMHUB_1)
386
387 amdgpu_ring_emit_reg_wait(ring,
388 hub->vm_inv_eng0_sem + eng, 0x1, 0x1);
389
390 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),
391 lower_32_bits(pd_addr));
392
393 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
394 upper_32_bits(pd_addr));
395
396 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
397 hub->vm_inv_eng0_ack + eng,
398 req, 1 << vmid);
399
400
401 if (ring->funcs->vmhub == AMDGPU_MMHUB_0 ||
402 ring->funcs->vmhub == AMDGPU_MMHUB_1)
403
404
405
406
407 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + eng, 0);
408
409 return pd_addr;
410 }
411
412 static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
413 unsigned pasid)
414 {
415 struct amdgpu_device *adev = ring->adev;
416 uint32_t reg;
417
418 if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
419 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
420 else
421 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
422
423 amdgpu_ring_emit_wreg(ring, reg, pasid);
424 }
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457 static uint64_t gmc_v10_0_get_vm_pte_flags(struct amdgpu_device *adev,
458 uint32_t flags)
459 {
460 uint64_t pte_flag = 0;
461
462 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
463 pte_flag |= AMDGPU_PTE_EXECUTABLE;
464 if (flags & AMDGPU_VM_PAGE_READABLE)
465 pte_flag |= AMDGPU_PTE_READABLE;
466 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
467 pte_flag |= AMDGPU_PTE_WRITEABLE;
468
469 switch (flags & AMDGPU_VM_MTYPE_MASK) {
470 case AMDGPU_VM_MTYPE_DEFAULT:
471 pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
472 break;
473 case AMDGPU_VM_MTYPE_NC:
474 pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
475 break;
476 case AMDGPU_VM_MTYPE_WC:
477 pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_WC);
478 break;
479 case AMDGPU_VM_MTYPE_CC:
480 pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_CC);
481 break;
482 case AMDGPU_VM_MTYPE_UC:
483 pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_UC);
484 break;
485 default:
486 pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
487 break;
488 }
489
490 if (flags & AMDGPU_VM_PAGE_PRT)
491 pte_flag |= AMDGPU_PTE_PRT;
492
493 return pte_flag;
494 }
495
496 static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level,
497 uint64_t *addr, uint64_t *flags)
498 {
499 if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
500 *addr = adev->vm_manager.vram_base_offset + *addr -
501 adev->gmc.vram_start;
502 BUG_ON(*addr & 0xFFFF00000000003FULL);
503
504 if (!adev->gmc.translate_further)
505 return;
506
507 if (level == AMDGPU_VM_PDB1) {
508
509 if (!(*flags & AMDGPU_PDE_PTE))
510 *flags |= AMDGPU_PDE_BFS(0x9);
511
512 } else if (level == AMDGPU_VM_PDB0) {
513 if (*flags & AMDGPU_PDE_PTE)
514 *flags &= ~AMDGPU_PDE_PTE;
515 else
516 *flags |= AMDGPU_PTE_TF;
517 }
518 }
519
520 static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = {
521 .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb,
522 .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb,
523 .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping,
524 .get_vm_pte_flags = gmc_v10_0_get_vm_pte_flags,
525 .get_vm_pde = gmc_v10_0_get_vm_pde
526 };
527
528 static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev)
529 {
530 if (adev->gmc.gmc_funcs == NULL)
531 adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs;
532 }
533
534 static int gmc_v10_0_early_init(void *handle)
535 {
536 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
537
538 gmc_v10_0_set_gmc_funcs(adev);
539 gmc_v10_0_set_irq_funcs(adev);
540
541 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
542 adev->gmc.shared_aperture_end =
543 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
544 adev->gmc.private_aperture_start = 0x1000000000000000ULL;
545 adev->gmc.private_aperture_end =
546 adev->gmc.private_aperture_start + (4ULL << 30) - 1;
547
548 return 0;
549 }
550
551 static int gmc_v10_0_late_init(void *handle)
552 {
553 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
554 unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
555 unsigned i;
556
557 for(i = 0; i < adev->num_rings; ++i) {
558 struct amdgpu_ring *ring = adev->rings[i];
559 unsigned vmhub = ring->funcs->vmhub;
560
561 ring->vm_inv_eng = vm_inv_eng[vmhub]++;
562 dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n",
563 ring->idx, ring->name, ring->vm_inv_eng,
564 ring->funcs->vmhub);
565 }
566
567
568 for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
569 BUG_ON(vm_inv_eng[i] > 17);
570
571 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
572 }
573
574 static void gmc_v10_0_vram_gtt_location(struct amdgpu_device *adev,
575 struct amdgpu_gmc *mc)
576 {
577 u64 base = 0;
578
579 if (!amdgpu_sriov_vf(adev))
580 base = gfxhub_v2_0_get_fb_location(adev);
581
582 amdgpu_gmc_vram_location(adev, &adev->gmc, base);
583 amdgpu_gmc_gart_location(adev, mc);
584
585
586 adev->vm_manager.vram_base_offset = gfxhub_v2_0_get_mc_fb_offset(adev);
587 }
588
589
590
591
592
593
594
595
596
597
598 static int gmc_v10_0_mc_init(struct amdgpu_device *adev)
599 {
600 int chansize, numchan;
601
602 if (!amdgpu_emu_mode)
603 adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
604 else {
605
606 chansize = 128;
607 numchan = 1;
608 adev->gmc.vram_width = numchan * chansize;
609 }
610
611
612 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
613 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
614
615
616 adev->gmc.mc_vram_size =
617 adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL;
618 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
619 adev->gmc.visible_vram_size = adev->gmc.aper_size;
620
621
622 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
623 adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
624
625
626 if (amdgpu_gart_size == -1) {
627 switch (adev->asic_type) {
628 case CHIP_NAVI10:
629 case CHIP_NAVI14:
630 case CHIP_NAVI12:
631 default:
632 adev->gmc.gart_size = 512ULL << 20;
633 break;
634 }
635 } else
636 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
637
638 gmc_v10_0_vram_gtt_location(adev, &adev->gmc);
639
640 return 0;
641 }
642
643 static int gmc_v10_0_gart_init(struct amdgpu_device *adev)
644 {
645 int r;
646
647 if (adev->gart.bo) {
648 WARN(1, "NAVI10 PCIE GART already initialized\n");
649 return 0;
650 }
651
652
653 r = amdgpu_gart_init(adev);
654 if (r)
655 return r;
656
657 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
658 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(MTYPE_UC) |
659 AMDGPU_PTE_EXECUTABLE;
660
661 return amdgpu_gart_table_vram_alloc(adev);
662 }
663
664 static unsigned gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev)
665 {
666 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
667 unsigned size;
668
669 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
670 size = 9 * 1024 * 1024;
671 } else {
672 u32 viewport;
673 u32 pitch;
674
675 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
676 pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH);
677 size = (REG_GET_FIELD(viewport,
678 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
679 REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) *
680 4);
681 }
682
683 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) {
684 DRM_ERROR("Warning: pre-OS buffer uses most of vram, \
685 be aware of gart table overwrite\n");
686 return 0;
687 }
688
689 return size;
690 }
691
692
693
694 static int gmc_v10_0_sw_init(void *handle)
695 {
696 int r;
697 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
698
699 gfxhub_v2_0_init(adev);
700 mmhub_v2_0_init(adev);
701
702 spin_lock_init(&adev->gmc.invalidate_lock);
703
704 adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev);
705 switch (adev->asic_type) {
706 case CHIP_NAVI10:
707 case CHIP_NAVI14:
708 case CHIP_NAVI12:
709 adev->num_vmhubs = 2;
710
711
712
713
714
715 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
716 break;
717 default:
718 break;
719 }
720
721
722 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC,
723 VMC_1_0__SRCID__VM_FAULT,
724 &adev->gmc.vm_fault);
725 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2,
726 UTCL2_1_0__SRCID__FAULT,
727 &adev->gmc.vm_fault);
728 if (r)
729 return r;
730
731
732
733
734
735 adev->gmc.mc_mask = 0xffffffffffffULL;
736
737
738
739
740
741 if (amdgpu_emu_mode == 1)
742 adev->gmc.stolen_size = 0;
743 else
744 adev->gmc.stolen_size = 9 * 1024 *1024;
745
746 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
747 if (r) {
748 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
749 return r;
750 }
751
752 r = gmc_v10_0_mc_init(adev);
753 if (r)
754 return r;
755
756 adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
757
758
759 r = amdgpu_bo_init(adev);
760 if (r)
761 return r;
762
763 r = gmc_v10_0_gart_init(adev);
764 if (r)
765 return r;
766
767
768
769
770
771
772
773 adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids = AMDGPU_NUM_OF_VMIDS;
774 adev->vm_manager.id_mgr[AMDGPU_MMHUB_0].num_ids = AMDGPU_NUM_OF_VMIDS;
775
776 amdgpu_vm_manager_init(adev);
777
778 return 0;
779 }
780
781
782
783
784
785
786
787
788 static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)
789 {
790 amdgpu_gart_table_vram_free(adev);
791 amdgpu_gart_fini(adev);
792 }
793
794 static int gmc_v10_0_sw_fini(void *handle)
795 {
796 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
797
798 amdgpu_vm_manager_fini(adev);
799 gmc_v10_0_gart_fini(adev);
800 amdgpu_gem_force_release(adev);
801 amdgpu_bo_fini(adev);
802
803 return 0;
804 }
805
806 static void gmc_v10_0_init_golden_registers(struct amdgpu_device *adev)
807 {
808 switch (adev->asic_type) {
809 case CHIP_NAVI10:
810 case CHIP_NAVI14:
811 case CHIP_NAVI12:
812 break;
813 default:
814 break;
815 }
816 }
817
818
819
820
821
822
823 static int gmc_v10_0_gart_enable(struct amdgpu_device *adev)
824 {
825 int r;
826 bool value;
827 u32 tmp;
828
829 if (adev->gart.bo == NULL) {
830 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
831 return -EINVAL;
832 }
833
834 r = amdgpu_gart_table_vram_pin(adev);
835 if (r)
836 return r;
837
838 r = gfxhub_v2_0_gart_enable(adev);
839 if (r)
840 return r;
841
842 r = mmhub_v2_0_gart_enable(adev);
843 if (r)
844 return r;
845
846 tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL);
847 tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK;
848 WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp);
849
850 tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
851 WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
852
853
854 adev->nbio_funcs->hdp_flush(adev, NULL);
855
856 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
857 false : true;
858
859 gfxhub_v2_0_set_fault_enable_default(adev, value);
860 mmhub_v2_0_set_fault_enable_default(adev, value);
861 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB_0, 0);
862 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB_0, 0);
863
864 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
865 (unsigned)(adev->gmc.gart_size >> 20),
866 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
867
868 adev->gart.ready = true;
869
870 return 0;
871 }
872
873 static int gmc_v10_0_hw_init(void *handle)
874 {
875 int r;
876 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
877
878
879 gmc_v10_0_init_golden_registers(adev);
880
881 r = gmc_v10_0_gart_enable(adev);
882 if (r)
883 return r;
884
885 return 0;
886 }
887
888
889
890
891
892
893
894
895 static void gmc_v10_0_gart_disable(struct amdgpu_device *adev)
896 {
897 gfxhub_v2_0_gart_disable(adev);
898 mmhub_v2_0_gart_disable(adev);
899 amdgpu_gart_table_vram_unpin(adev);
900 }
901
902 static int gmc_v10_0_hw_fini(void *handle)
903 {
904 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
905
906 if (amdgpu_sriov_vf(adev)) {
907
908 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
909 return 0;
910 }
911
912 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
913 gmc_v10_0_gart_disable(adev);
914
915 return 0;
916 }
917
918 static int gmc_v10_0_suspend(void *handle)
919 {
920 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
921
922 gmc_v10_0_hw_fini(adev);
923
924 return 0;
925 }
926
927 static int gmc_v10_0_resume(void *handle)
928 {
929 int r;
930 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
931
932 r = gmc_v10_0_hw_init(adev);
933 if (r)
934 return r;
935
936 amdgpu_vmid_reset_all(adev);
937
938 return 0;
939 }
940
941 static bool gmc_v10_0_is_idle(void *handle)
942 {
943
944 return true;
945 }
946
947 static int gmc_v10_0_wait_for_idle(void *handle)
948 {
949
950 return 0;
951 }
952
953 static int gmc_v10_0_soft_reset(void *handle)
954 {
955 return 0;
956 }
957
958 static int gmc_v10_0_set_clockgating_state(void *handle,
959 enum amd_clockgating_state state)
960 {
961 int r;
962 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
963
964 r = mmhub_v2_0_set_clockgating(adev, state);
965 if (r)
966 return r;
967
968 return athub_v2_0_set_clockgating(adev, state);
969 }
970
971 static void gmc_v10_0_get_clockgating_state(void *handle, u32 *flags)
972 {
973 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
974
975 mmhub_v2_0_get_clockgating(adev, flags);
976
977 athub_v2_0_get_clockgating(adev, flags);
978 }
979
980 static int gmc_v10_0_set_powergating_state(void *handle,
981 enum amd_powergating_state state)
982 {
983 return 0;
984 }
985
986 const struct amd_ip_funcs gmc_v10_0_ip_funcs = {
987 .name = "gmc_v10_0",
988 .early_init = gmc_v10_0_early_init,
989 .late_init = gmc_v10_0_late_init,
990 .sw_init = gmc_v10_0_sw_init,
991 .sw_fini = gmc_v10_0_sw_fini,
992 .hw_init = gmc_v10_0_hw_init,
993 .hw_fini = gmc_v10_0_hw_fini,
994 .suspend = gmc_v10_0_suspend,
995 .resume = gmc_v10_0_resume,
996 .is_idle = gmc_v10_0_is_idle,
997 .wait_for_idle = gmc_v10_0_wait_for_idle,
998 .soft_reset = gmc_v10_0_soft_reset,
999 .set_clockgating_state = gmc_v10_0_set_clockgating_state,
1000 .set_powergating_state = gmc_v10_0_set_powergating_state,
1001 .get_clockgating_state = gmc_v10_0_get_clockgating_state,
1002 };
1003
1004 const struct amdgpu_ip_block_version gmc_v10_0_ip_block =
1005 {
1006 .type = AMD_IP_BLOCK_TYPE_GMC,
1007 .major = 10,
1008 .minor = 0,
1009 .rev = 0,
1010 .funcs = &gmc_v10_0_ip_funcs,
1011 };