This source file includes following definitions.
- wait_ready
- write_cmd
- lock_region
- mmu_hw_do_operation_locked
- mmu_hw_do_operation
- panfrost_mmu_enable
- panfrost_mmu_disable
- panfrost_mmu_as_get
- panfrost_mmu_as_put
- panfrost_mmu_reset
- get_pgsize
- panfrost_mmu_flush_range
- mmu_map_sg
- panfrost_mmu_map
- panfrost_mmu_unmap
- mmu_tlb_inv_context_s1
- mmu_tlb_sync_context
- mmu_tlb_flush_walk
- mmu_tlb_flush_leaf
- panfrost_mmu_pgtable_alloc
- panfrost_mmu_pgtable_free
- addr_to_mapping
- panfrost_mmu_map_fault_addr
- access_type_name
- panfrost_mmu_irq_handler
- panfrost_mmu_irq_handler_thread
- panfrost_mmu_init
- panfrost_mmu_fini
1
2
3 #include <linux/atomic.h>
4 #include <linux/bitfield.h>
5 #include <linux/delay.h>
6 #include <linux/dma-mapping.h>
7 #include <linux/interrupt.h>
8 #include <linux/io.h>
9 #include <linux/iopoll.h>
10 #include <linux/io-pgtable.h>
11 #include <linux/iommu.h>
12 #include <linux/platform_device.h>
13 #include <linux/pm_runtime.h>
14 #include <linux/shmem_fs.h>
15 #include <linux/sizes.h>
16
17 #include "panfrost_device.h"
18 #include "panfrost_mmu.h"
19 #include "panfrost_gem.h"
20 #include "panfrost_features.h"
21 #include "panfrost_regs.h"
22
23 #define mmu_write(dev, reg, data) writel(data, dev->iomem + reg)
24 #define mmu_read(dev, reg) readl(dev->iomem + reg)
25
26 static int wait_ready(struct panfrost_device *pfdev, u32 as_nr)
27 {
28 int ret;
29 u32 val;
30
31
32
33 ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + AS_STATUS(as_nr),
34 val, !(val & AS_STATUS_AS_ACTIVE), 10, 1000);
35
36 if (ret)
37 dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n");
38
39 return ret;
40 }
41
42 static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd)
43 {
44 int status;
45
46
47 status = wait_ready(pfdev, as_nr);
48 if (!status)
49 mmu_write(pfdev, AS_COMMAND(as_nr), cmd);
50
51 return status;
52 }
53
54 static void lock_region(struct panfrost_device *pfdev, u32 as_nr,
55 u64 iova, size_t size)
56 {
57 u8 region_width;
58 u64 region = iova & PAGE_MASK;
59
60
61
62
63
64
65
66
67 size = round_up(size, PAGE_SIZE);
68
69 region_width = 10 + fls(size >> PAGE_SHIFT);
70 if ((size >> PAGE_SHIFT) != (1ul << (region_width - 11))) {
71
72 region_width += 1;
73 }
74 region |= region_width;
75
76
77 mmu_write(pfdev, AS_LOCKADDR_LO(as_nr), region & 0xFFFFFFFFUL);
78 mmu_write(pfdev, AS_LOCKADDR_HI(as_nr), (region >> 32) & 0xFFFFFFFFUL);
79 write_cmd(pfdev, as_nr, AS_COMMAND_LOCK);
80 }
81
82
83 static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr,
84 u64 iova, size_t size, u32 op)
85 {
86 if (as_nr < 0)
87 return 0;
88
89 if (op != AS_COMMAND_UNLOCK)
90 lock_region(pfdev, as_nr, iova, size);
91
92
93 write_cmd(pfdev, as_nr, op);
94
95
96 return wait_ready(pfdev, as_nr);
97 }
98
99 static int mmu_hw_do_operation(struct panfrost_device *pfdev,
100 struct panfrost_mmu *mmu,
101 u64 iova, size_t size, u32 op)
102 {
103 int ret;
104
105 spin_lock(&pfdev->as_lock);
106 ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op);
107 spin_unlock(&pfdev->as_lock);
108 return ret;
109 }
110
111 static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
112 {
113 int as_nr = mmu->as;
114 struct io_pgtable_cfg *cfg = &mmu->pgtbl_cfg;
115 u64 transtab = cfg->arm_mali_lpae_cfg.transtab;
116 u64 memattr = cfg->arm_mali_lpae_cfg.memattr;
117
118 mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
119
120 mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL);
121 mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32);
122
123
124
125
126 mmu_write(pfdev, AS_MEMATTR_LO(as_nr), memattr & 0xffffffffUL);
127 mmu_write(pfdev, AS_MEMATTR_HI(as_nr), memattr >> 32);
128
129 write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
130 }
131
132 static void panfrost_mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
133 {
134 mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
135
136 mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0);
137 mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0);
138
139 mmu_write(pfdev, AS_MEMATTR_LO(as_nr), 0);
140 mmu_write(pfdev, AS_MEMATTR_HI(as_nr), 0);
141
142 write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
143 }
144
145 u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
146 {
147 int as;
148
149 spin_lock(&pfdev->as_lock);
150
151 as = mmu->as;
152 if (as >= 0) {
153 int en = atomic_inc_return(&mmu->as_count);
154
155
156
157
158
159 WARN_ON(en >= (NUM_JOB_SLOTS + 1));
160
161 list_move(&mmu->list, &pfdev->as_lru_list);
162 goto out;
163 }
164
165
166 as = ffz(pfdev->as_alloc_mask);
167 if (!(BIT(as) & pfdev->features.as_present)) {
168 struct panfrost_mmu *lru_mmu;
169
170 list_for_each_entry_reverse(lru_mmu, &pfdev->as_lru_list, list) {
171 if (!atomic_read(&lru_mmu->as_count))
172 break;
173 }
174 WARN_ON(&lru_mmu->list == &pfdev->as_lru_list);
175
176 list_del_init(&lru_mmu->list);
177 as = lru_mmu->as;
178
179 WARN_ON(as < 0);
180 lru_mmu->as = -1;
181 }
182
183
184 mmu->as = as;
185 set_bit(as, &pfdev->as_alloc_mask);
186 atomic_set(&mmu->as_count, 1);
187 list_add(&mmu->list, &pfdev->as_lru_list);
188
189 dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask);
190
191 panfrost_mmu_enable(pfdev, mmu);
192
193 out:
194 spin_unlock(&pfdev->as_lock);
195 return as;
196 }
197
198 void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
199 {
200 atomic_dec(&mmu->as_count);
201 WARN_ON(atomic_read(&mmu->as_count) < 0);
202 }
203
204 void panfrost_mmu_reset(struct panfrost_device *pfdev)
205 {
206 struct panfrost_mmu *mmu, *mmu_tmp;
207
208 spin_lock(&pfdev->as_lock);
209
210 pfdev->as_alloc_mask = 0;
211
212 list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) {
213 mmu->as = -1;
214 atomic_set(&mmu->as_count, 0);
215 list_del_init(&mmu->list);
216 }
217
218 spin_unlock(&pfdev->as_lock);
219
220 mmu_write(pfdev, MMU_INT_CLEAR, ~0);
221 mmu_write(pfdev, MMU_INT_MASK, ~0);
222 }
223
224 static size_t get_pgsize(u64 addr, size_t size)
225 {
226 if (addr & (SZ_2M - 1) || size < SZ_2M)
227 return SZ_4K;
228
229 return SZ_2M;
230 }
231
232 static void panfrost_mmu_flush_range(struct panfrost_device *pfdev,
233 struct panfrost_mmu *mmu,
234 u64 iova, size_t size)
235 {
236 if (mmu->as < 0)
237 return;
238
239 pm_runtime_get_noresume(pfdev->dev);
240
241
242 if (pm_runtime_active(pfdev->dev))
243 mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT);
244
245 pm_runtime_put_sync_autosuspend(pfdev->dev);
246 }
247
248 static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu,
249 u64 iova, int prot, struct sg_table *sgt)
250 {
251 unsigned int count;
252 struct scatterlist *sgl;
253 struct io_pgtable_ops *ops = mmu->pgtbl_ops;
254 u64 start_iova = iova;
255
256 for_each_sg(sgt->sgl, sgl, sgt->nents, count) {
257 unsigned long paddr = sg_dma_address(sgl);
258 size_t len = sg_dma_len(sgl);
259
260 dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len);
261
262 while (len) {
263 size_t pgsize = get_pgsize(iova | paddr, len);
264
265 ops->map(ops, iova, paddr, pgsize, prot);
266 iova += pgsize;
267 paddr += pgsize;
268 len -= pgsize;
269 }
270 }
271
272 panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova);
273
274 return 0;
275 }
276
277 int panfrost_mmu_map(struct panfrost_gem_mapping *mapping)
278 {
279 struct panfrost_gem_object *bo = mapping->obj;
280 struct drm_gem_object *obj = &bo->base.base;
281 struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
282 struct sg_table *sgt;
283 int prot = IOMMU_READ | IOMMU_WRITE;
284
285 if (WARN_ON(mapping->active))
286 return 0;
287
288 if (bo->noexec)
289 prot |= IOMMU_NOEXEC;
290
291 sgt = drm_gem_shmem_get_pages_sgt(obj);
292 if (WARN_ON(IS_ERR(sgt)))
293 return PTR_ERR(sgt);
294
295 mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT,
296 prot, sgt);
297 mapping->active = true;
298
299 return 0;
300 }
301
302 void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping)
303 {
304 struct panfrost_gem_object *bo = mapping->obj;
305 struct drm_gem_object *obj = &bo->base.base;
306 struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
307 struct io_pgtable_ops *ops = mapping->mmu->pgtbl_ops;
308 u64 iova = mapping->mmnode.start << PAGE_SHIFT;
309 size_t len = mapping->mmnode.size << PAGE_SHIFT;
310 size_t unmapped_len = 0;
311
312 if (WARN_ON(!mapping->active))
313 return;
314
315 dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx",
316 mapping->mmu->as, iova, len);
317
318 while (unmapped_len < len) {
319 size_t unmapped_page;
320 size_t pgsize = get_pgsize(iova, len - unmapped_len);
321
322 if (ops->iova_to_phys(ops, iova)) {
323 unmapped_page = ops->unmap(ops, iova, pgsize, NULL);
324 WARN_ON(unmapped_page != pgsize);
325 }
326 iova += pgsize;
327 unmapped_len += pgsize;
328 }
329
330 panfrost_mmu_flush_range(pfdev, mapping->mmu,
331 mapping->mmnode.start << PAGE_SHIFT, len);
332 mapping->active = false;
333 }
334
335 static void mmu_tlb_inv_context_s1(void *cookie)
336 {}
337
338 static void mmu_tlb_sync_context(void *cookie)
339 {
340
341
342 }
343
344 static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule,
345 void *cookie)
346 {
347 mmu_tlb_sync_context(cookie);
348 }
349
350 static void mmu_tlb_flush_leaf(unsigned long iova, size_t size, size_t granule,
351 void *cookie)
352 {
353 mmu_tlb_sync_context(cookie);
354 }
355
356 static const struct iommu_flush_ops mmu_tlb_ops = {
357 .tlb_flush_all = mmu_tlb_inv_context_s1,
358 .tlb_flush_walk = mmu_tlb_flush_walk,
359 .tlb_flush_leaf = mmu_tlb_flush_leaf,
360 };
361
362 int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv)
363 {
364 struct panfrost_mmu *mmu = &priv->mmu;
365 struct panfrost_device *pfdev = priv->pfdev;
366
367 INIT_LIST_HEAD(&mmu->list);
368 mmu->as = -1;
369
370 mmu->pgtbl_cfg = (struct io_pgtable_cfg) {
371 .pgsize_bitmap = SZ_4K | SZ_2M,
372 .ias = FIELD_GET(0xff, pfdev->features.mmu_features),
373 .oas = FIELD_GET(0xff00, pfdev->features.mmu_features),
374 .tlb = &mmu_tlb_ops,
375 .iommu_dev = pfdev->dev,
376 };
377
378 mmu->pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &mmu->pgtbl_cfg,
379 priv);
380 if (!mmu->pgtbl_ops)
381 return -EINVAL;
382
383 return 0;
384 }
385
386 void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv)
387 {
388 struct panfrost_device *pfdev = priv->pfdev;
389 struct panfrost_mmu *mmu = &priv->mmu;
390
391 spin_lock(&pfdev->as_lock);
392 if (mmu->as >= 0) {
393 pm_runtime_get_noresume(pfdev->dev);
394 if (pm_runtime_active(pfdev->dev))
395 panfrost_mmu_disable(pfdev, mmu->as);
396 pm_runtime_put_autosuspend(pfdev->dev);
397
398 clear_bit(mmu->as, &pfdev->as_alloc_mask);
399 clear_bit(mmu->as, &pfdev->as_in_use_mask);
400 list_del(&mmu->list);
401 }
402 spin_unlock(&pfdev->as_lock);
403
404 free_io_pgtable_ops(mmu->pgtbl_ops);
405 }
406
407 static struct panfrost_gem_mapping *
408 addr_to_mapping(struct panfrost_device *pfdev, int as, u64 addr)
409 {
410 struct panfrost_gem_mapping *mapping = NULL;
411 struct panfrost_file_priv *priv;
412 struct drm_mm_node *node;
413 u64 offset = addr >> PAGE_SHIFT;
414 struct panfrost_mmu *mmu;
415
416 spin_lock(&pfdev->as_lock);
417 list_for_each_entry(mmu, &pfdev->as_lru_list, list) {
418 if (as == mmu->as)
419 goto found_mmu;
420 }
421 goto out;
422
423 found_mmu:
424 priv = container_of(mmu, struct panfrost_file_priv, mmu);
425
426 spin_lock(&priv->mm_lock);
427
428 drm_mm_for_each_node(node, &priv->mm) {
429 if (offset >= node->start &&
430 offset < (node->start + node->size)) {
431 mapping = drm_mm_node_to_panfrost_mapping(node);
432
433 kref_get(&mapping->refcount);
434 break;
435 }
436 }
437
438 spin_unlock(&priv->mm_lock);
439 out:
440 spin_unlock(&pfdev->as_lock);
441 return mapping;
442 }
443
444 #define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE)
445
446 static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
447 u64 addr)
448 {
449 int ret, i;
450 struct panfrost_gem_mapping *bomapping;
451 struct panfrost_gem_object *bo;
452 struct address_space *mapping;
453 pgoff_t page_offset;
454 struct sg_table *sgt;
455 struct page **pages;
456
457 bomapping = addr_to_mapping(pfdev, as, addr);
458 if (!bomapping)
459 return -ENOENT;
460
461 bo = bomapping->obj;
462 if (!bo->is_heap) {
463 dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)",
464 bomapping->mmnode.start << PAGE_SHIFT);
465 ret = -EINVAL;
466 goto err_bo;
467 }
468 WARN_ON(bomapping->mmu->as != as);
469
470
471 addr &= ~((u64)SZ_2M - 1);
472 page_offset = addr >> PAGE_SHIFT;
473 page_offset -= bomapping->mmnode.start;
474
475 mutex_lock(&bo->base.pages_lock);
476
477 if (!bo->base.pages) {
478 bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M,
479 sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO);
480 if (!bo->sgts) {
481 mutex_unlock(&bo->base.pages_lock);
482 ret = -ENOMEM;
483 goto err_bo;
484 }
485
486 pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT,
487 sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
488 if (!pages) {
489 kfree(bo->sgts);
490 bo->sgts = NULL;
491 mutex_unlock(&bo->base.pages_lock);
492 ret = -ENOMEM;
493 goto err_bo;
494 }
495 bo->base.pages = pages;
496 bo->base.pages_use_count = 1;
497 } else
498 pages = bo->base.pages;
499
500 mapping = bo->base.base.filp->f_mapping;
501 mapping_set_unevictable(mapping);
502
503 for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) {
504 pages[i] = shmem_read_mapping_page(mapping, i);
505 if (IS_ERR(pages[i])) {
506 mutex_unlock(&bo->base.pages_lock);
507 ret = PTR_ERR(pages[i]);
508 goto err_pages;
509 }
510 }
511
512 mutex_unlock(&bo->base.pages_lock);
513
514 sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)];
515 ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
516 NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
517 if (ret)
518 goto err_pages;
519
520 if (!dma_map_sg(pfdev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL)) {
521 ret = -EINVAL;
522 goto err_map;
523 }
524
525 mmu_map_sg(pfdev, bomapping->mmu, addr,
526 IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt);
527
528 bomapping->active = true;
529
530 dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr);
531
532 panfrost_gem_mapping_put(bomapping);
533
534 return 0;
535
536 err_map:
537 sg_free_table(sgt);
538 err_pages:
539 drm_gem_shmem_put_pages(&bo->base);
540 err_bo:
541 drm_gem_object_put_unlocked(&bo->base.base);
542 return ret;
543 }
544
545 static const char *access_type_name(struct panfrost_device *pfdev,
546 u32 fault_status)
547 {
548 switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
549 case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
550 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU))
551 return "ATOMIC";
552 else
553 return "UNKNOWN";
554 case AS_FAULTSTATUS_ACCESS_TYPE_READ:
555 return "READ";
556 case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
557 return "WRITE";
558 case AS_FAULTSTATUS_ACCESS_TYPE_EX:
559 return "EXECUTE";
560 default:
561 WARN_ON(1);
562 return NULL;
563 }
564 }
565
566 static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data)
567 {
568 struct panfrost_device *pfdev = data;
569
570 if (!mmu_read(pfdev, MMU_INT_STAT))
571 return IRQ_NONE;
572
573 mmu_write(pfdev, MMU_INT_MASK, 0);
574 return IRQ_WAKE_THREAD;
575 }
576
577 static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data)
578 {
579 struct panfrost_device *pfdev = data;
580 u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT);
581 int i, ret;
582
583 for (i = 0; status; i++) {
584 u32 mask = BIT(i) | BIT(i + 16);
585 u64 addr;
586 u32 fault_status;
587 u32 exception_type;
588 u32 access_type;
589 u32 source_id;
590
591 if (!(status & mask))
592 continue;
593
594 fault_status = mmu_read(pfdev, AS_FAULTSTATUS(i));
595 addr = mmu_read(pfdev, AS_FAULTADDRESS_LO(i));
596 addr |= (u64)mmu_read(pfdev, AS_FAULTADDRESS_HI(i)) << 32;
597
598
599 exception_type = fault_status & 0xFF;
600 access_type = (fault_status >> 8) & 0x3;
601 source_id = (fault_status >> 16);
602
603
604 ret = -1;
605 if ((status & mask) == BIT(i) && (exception_type & 0xF8) == 0xC0)
606 ret = panfrost_mmu_map_fault_addr(pfdev, i, addr);
607
608 if (ret)
609
610 dev_err(pfdev->dev,
611 "Unhandled Page fault in AS%d at VA 0x%016llX\n"
612 "Reason: %s\n"
613 "raw fault status: 0x%X\n"
614 "decoded fault status: %s\n"
615 "exception type 0x%X: %s\n"
616 "access type 0x%X: %s\n"
617 "source id 0x%X\n",
618 i, addr,
619 "TODO",
620 fault_status,
621 (fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
622 exception_type, panfrost_exception_name(pfdev, exception_type),
623 access_type, access_type_name(pfdev, fault_status),
624 source_id);
625
626 mmu_write(pfdev, MMU_INT_CLEAR, mask);
627
628 status &= ~mask;
629 }
630
631 mmu_write(pfdev, MMU_INT_MASK, ~0);
632 return IRQ_HANDLED;
633 };
634
635 int panfrost_mmu_init(struct panfrost_device *pfdev)
636 {
637 int err, irq;
638
639 irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");
640 if (irq <= 0)
641 return -ENODEV;
642
643 err = devm_request_threaded_irq(pfdev->dev, irq, panfrost_mmu_irq_handler,
644 panfrost_mmu_irq_handler_thread,
645 IRQF_SHARED, "mmu", pfdev);
646
647 if (err) {
648 dev_err(pfdev->dev, "failed to request mmu irq");
649 return err;
650 }
651
652 return 0;
653 }
654
655 void panfrost_mmu_fini(struct panfrost_device *pfdev)
656 {
657 mmu_write(pfdev, MMU_INT_MASK, 0);
658 }