This source file includes following definitions.
- drm_gem_shmem_create
- drm_gem_shmem_free_object
- drm_gem_shmem_get_pages_locked
- drm_gem_shmem_get_pages
- drm_gem_shmem_put_pages_locked
- drm_gem_shmem_put_pages
- drm_gem_shmem_pin
- drm_gem_shmem_unpin
- drm_gem_shmem_vmap_locked
- drm_gem_shmem_vmap
- drm_gem_shmem_vunmap_locked
- drm_gem_shmem_vunmap
- drm_gem_shmem_create_with_handle
- drm_gem_shmem_madvise
- drm_gem_shmem_purge_locked
- drm_gem_shmem_purge
- drm_gem_shmem_dumb_create
- drm_gem_shmem_fault
- drm_gem_shmem_vm_open
- drm_gem_shmem_vm_close
- drm_gem_shmem_mmap
- drm_gem_shmem_print_info
- drm_gem_shmem_get_sg_table
- drm_gem_shmem_get_pages_sgt
- drm_gem_shmem_prime_import_sg_table
1
2
3
4
5
6 #include <linux/dma-buf.h>
7 #include <linux/export.h>
8 #include <linux/mutex.h>
9 #include <linux/shmem_fs.h>
10 #include <linux/slab.h>
11 #include <linux/vmalloc.h>
12
13 #include <drm/drm.h>
14 #include <drm/drm_device.h>
15 #include <drm/drm_drv.h>
16 #include <drm/drm_gem_shmem_helper.h>
17 #include <drm/drm_prime.h>
18 #include <drm/drm_print.h>
19
20
21
22
23
24
25
26
27 static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
28 .free = drm_gem_shmem_free_object,
29 .print_info = drm_gem_shmem_print_info,
30 .pin = drm_gem_shmem_pin,
31 .unpin = drm_gem_shmem_unpin,
32 .get_sg_table = drm_gem_shmem_get_sg_table,
33 .vmap = drm_gem_shmem_vmap,
34 .vunmap = drm_gem_shmem_vunmap,
35 .vm_ops = &drm_gem_shmem_vm_ops,
36 };
37
38
39
40
41
42
43
44
45
46
47
48
49 struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size)
50 {
51 struct drm_gem_shmem_object *shmem;
52 struct drm_gem_object *obj;
53 int ret;
54
55 size = PAGE_ALIGN(size);
56
57 if (dev->driver->gem_create_object)
58 obj = dev->driver->gem_create_object(dev, size);
59 else
60 obj = kzalloc(sizeof(*shmem), GFP_KERNEL);
61 if (!obj)
62 return ERR_PTR(-ENOMEM);
63
64 if (!obj->funcs)
65 obj->funcs = &drm_gem_shmem_funcs;
66
67 ret = drm_gem_object_init(dev, obj, size);
68 if (ret)
69 goto err_free;
70
71 ret = drm_gem_create_mmap_offset(obj);
72 if (ret)
73 goto err_release;
74
75 shmem = to_drm_gem_shmem_obj(obj);
76 mutex_init(&shmem->pages_lock);
77 mutex_init(&shmem->vmap_lock);
78 INIT_LIST_HEAD(&shmem->madv_list);
79
80
81
82
83
84
85
86
87 mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER |
88 __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
89
90 return shmem;
91
92 err_release:
93 drm_gem_object_release(obj);
94 err_free:
95 kfree(obj);
96
97 return ERR_PTR(ret);
98 }
99 EXPORT_SYMBOL_GPL(drm_gem_shmem_create);
100
101
102
103
104
105
106
107
108 void drm_gem_shmem_free_object(struct drm_gem_object *obj)
109 {
110 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
111
112 WARN_ON(shmem->vmap_use_count);
113
114 if (obj->import_attach) {
115 shmem->pages_use_count--;
116 drm_prime_gem_destroy(obj, shmem->sgt);
117 kvfree(shmem->pages);
118 } else {
119 if (shmem->sgt) {
120 dma_unmap_sg(obj->dev->dev, shmem->sgt->sgl,
121 shmem->sgt->nents, DMA_BIDIRECTIONAL);
122 sg_free_table(shmem->sgt);
123 kfree(shmem->sgt);
124 }
125 if (shmem->pages)
126 drm_gem_shmem_put_pages(shmem);
127 }
128
129 WARN_ON(shmem->pages_use_count);
130
131 drm_gem_object_release(obj);
132 mutex_destroy(&shmem->pages_lock);
133 mutex_destroy(&shmem->vmap_lock);
134 kfree(shmem);
135 }
136 EXPORT_SYMBOL_GPL(drm_gem_shmem_free_object);
137
138 static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem)
139 {
140 struct drm_gem_object *obj = &shmem->base;
141 struct page **pages;
142
143 if (shmem->pages_use_count++ > 0)
144 return 0;
145
146 pages = drm_gem_get_pages(obj);
147 if (IS_ERR(pages)) {
148 DRM_DEBUG_KMS("Failed to get pages (%ld)\n", PTR_ERR(pages));
149 shmem->pages_use_count = 0;
150 return PTR_ERR(pages);
151 }
152
153 shmem->pages = pages;
154
155 return 0;
156 }
157
158
159
160
161
162
163
164
165
166
167
168 int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
169 {
170 int ret;
171
172 ret = mutex_lock_interruptible(&shmem->pages_lock);
173 if (ret)
174 return ret;
175 ret = drm_gem_shmem_get_pages_locked(shmem);
176 mutex_unlock(&shmem->pages_lock);
177
178 return ret;
179 }
180 EXPORT_SYMBOL(drm_gem_shmem_get_pages);
181
182 static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
183 {
184 struct drm_gem_object *obj = &shmem->base;
185
186 if (WARN_ON_ONCE(!shmem->pages_use_count))
187 return;
188
189 if (--shmem->pages_use_count > 0)
190 return;
191
192 drm_gem_put_pages(obj, shmem->pages,
193 shmem->pages_mark_dirty_on_put,
194 shmem->pages_mark_accessed_on_put);
195 shmem->pages = NULL;
196 }
197
198
199
200
201
202
203
204 void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)
205 {
206 mutex_lock(&shmem->pages_lock);
207 drm_gem_shmem_put_pages_locked(shmem);
208 mutex_unlock(&shmem->pages_lock);
209 }
210 EXPORT_SYMBOL(drm_gem_shmem_put_pages);
211
212
213
214
215
216
217
218
219
220
221
222 int drm_gem_shmem_pin(struct drm_gem_object *obj)
223 {
224 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
225
226 return drm_gem_shmem_get_pages(shmem);
227 }
228 EXPORT_SYMBOL(drm_gem_shmem_pin);
229
230
231
232
233
234
235
236
237 void drm_gem_shmem_unpin(struct drm_gem_object *obj)
238 {
239 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
240
241 drm_gem_shmem_put_pages(shmem);
242 }
243 EXPORT_SYMBOL(drm_gem_shmem_unpin);
244
245 static void *drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem)
246 {
247 struct drm_gem_object *obj = &shmem->base;
248 int ret;
249
250 if (shmem->vmap_use_count++ > 0)
251 return shmem->vaddr;
252
253 ret = drm_gem_shmem_get_pages(shmem);
254 if (ret)
255 goto err_zero_use;
256
257 if (obj->import_attach)
258 shmem->vaddr = dma_buf_vmap(obj->import_attach->dmabuf);
259 else
260 shmem->vaddr = vmap(shmem->pages, obj->size >> PAGE_SHIFT,
261 VM_MAP, pgprot_writecombine(PAGE_KERNEL));
262
263 if (!shmem->vaddr) {
264 DRM_DEBUG_KMS("Failed to vmap pages\n");
265 ret = -ENOMEM;
266 goto err_put_pages;
267 }
268
269 return shmem->vaddr;
270
271 err_put_pages:
272 drm_gem_shmem_put_pages(shmem);
273 err_zero_use:
274 shmem->vmap_use_count = 0;
275
276 return ERR_PTR(ret);
277 }
278
279
280
281
282
283
284
285
286
287
288
289 void *drm_gem_shmem_vmap(struct drm_gem_object *obj)
290 {
291 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
292 void *vaddr;
293 int ret;
294
295 ret = mutex_lock_interruptible(&shmem->vmap_lock);
296 if (ret)
297 return ERR_PTR(ret);
298 vaddr = drm_gem_shmem_vmap_locked(shmem);
299 mutex_unlock(&shmem->vmap_lock);
300
301 return vaddr;
302 }
303 EXPORT_SYMBOL(drm_gem_shmem_vmap);
304
305 static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem)
306 {
307 struct drm_gem_object *obj = &shmem->base;
308
309 if (WARN_ON_ONCE(!shmem->vmap_use_count))
310 return;
311
312 if (--shmem->vmap_use_count > 0)
313 return;
314
315 if (obj->import_attach)
316 dma_buf_vunmap(obj->import_attach->dmabuf, shmem->vaddr);
317 else
318 vunmap(shmem->vaddr);
319
320 shmem->vaddr = NULL;
321 drm_gem_shmem_put_pages(shmem);
322 }
323
324
325
326
327
328
329
330 void drm_gem_shmem_vunmap(struct drm_gem_object *obj, void *vaddr)
331 {
332 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
333
334 mutex_lock(&shmem->vmap_lock);
335 drm_gem_shmem_vunmap_locked(shmem);
336 mutex_unlock(&shmem->vmap_lock);
337 }
338 EXPORT_SYMBOL(drm_gem_shmem_vunmap);
339
340 struct drm_gem_shmem_object *
341 drm_gem_shmem_create_with_handle(struct drm_file *file_priv,
342 struct drm_device *dev, size_t size,
343 uint32_t *handle)
344 {
345 struct drm_gem_shmem_object *shmem;
346 int ret;
347
348 shmem = drm_gem_shmem_create(dev, size);
349 if (IS_ERR(shmem))
350 return shmem;
351
352
353
354
355
356 ret = drm_gem_handle_create(file_priv, &shmem->base, handle);
357
358 drm_gem_object_put_unlocked(&shmem->base);
359 if (ret)
360 return ERR_PTR(ret);
361
362 return shmem;
363 }
364 EXPORT_SYMBOL(drm_gem_shmem_create_with_handle);
365
366
367
368
369 int drm_gem_shmem_madvise(struct drm_gem_object *obj, int madv)
370 {
371 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
372
373 mutex_lock(&shmem->pages_lock);
374
375 if (shmem->madv >= 0)
376 shmem->madv = madv;
377
378 madv = shmem->madv;
379
380 mutex_unlock(&shmem->pages_lock);
381
382 return (madv >= 0);
383 }
384 EXPORT_SYMBOL(drm_gem_shmem_madvise);
385
386 void drm_gem_shmem_purge_locked(struct drm_gem_object *obj)
387 {
388 struct drm_device *dev = obj->dev;
389 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
390
391 WARN_ON(!drm_gem_shmem_is_purgeable(shmem));
392
393 dma_unmap_sg(obj->dev->dev, shmem->sgt->sgl,
394 shmem->sgt->nents, DMA_BIDIRECTIONAL);
395 sg_free_table(shmem->sgt);
396 kfree(shmem->sgt);
397 shmem->sgt = NULL;
398
399 drm_gem_shmem_put_pages_locked(shmem);
400
401 shmem->madv = -1;
402
403 drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
404 drm_gem_free_mmap_offset(obj);
405
406
407
408
409
410
411 shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1);
412
413 invalidate_mapping_pages(file_inode(obj->filp)->i_mapping,
414 0, (loff_t)-1);
415 }
416 EXPORT_SYMBOL(drm_gem_shmem_purge_locked);
417
418 bool drm_gem_shmem_purge(struct drm_gem_object *obj)
419 {
420 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
421
422 if (!mutex_trylock(&shmem->pages_lock))
423 return false;
424 drm_gem_shmem_purge_locked(obj);
425 mutex_unlock(&shmem->pages_lock);
426
427 return true;
428 }
429 EXPORT_SYMBOL(drm_gem_shmem_purge);
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448 int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev,
449 struct drm_mode_create_dumb *args)
450 {
451 u32 min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
452 struct drm_gem_shmem_object *shmem;
453
454 if (!args->pitch || !args->size) {
455 args->pitch = min_pitch;
456 args->size = args->pitch * args->height;
457 } else {
458
459 if (args->pitch < min_pitch)
460 args->pitch = min_pitch;
461 if (args->size < args->pitch * args->height)
462 args->size = args->pitch * args->height;
463 }
464
465 shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle);
466
467 return PTR_ERR_OR_ZERO(shmem);
468 }
469 EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create);
470
471 static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
472 {
473 struct vm_area_struct *vma = vmf->vma;
474 struct drm_gem_object *obj = vma->vm_private_data;
475 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
476 loff_t num_pages = obj->size >> PAGE_SHIFT;
477 struct page *page;
478
479 if (vmf->pgoff >= num_pages || WARN_ON_ONCE(!shmem->pages))
480 return VM_FAULT_SIGBUS;
481
482 page = shmem->pages[vmf->pgoff];
483
484 return vmf_insert_page(vma, vmf->address, page);
485 }
486
487 static void drm_gem_shmem_vm_open(struct vm_area_struct *vma)
488 {
489 struct drm_gem_object *obj = vma->vm_private_data;
490 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
491 int ret;
492
493 ret = drm_gem_shmem_get_pages(shmem);
494 WARN_ON_ONCE(ret != 0);
495
496 drm_gem_vm_open(vma);
497 }
498
499 static void drm_gem_shmem_vm_close(struct vm_area_struct *vma)
500 {
501 struct drm_gem_object *obj = vma->vm_private_data;
502 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
503
504 drm_gem_shmem_put_pages(shmem);
505 drm_gem_vm_close(vma);
506 }
507
508 const struct vm_operations_struct drm_gem_shmem_vm_ops = {
509 .fault = drm_gem_shmem_fault,
510 .open = drm_gem_shmem_vm_open,
511 .close = drm_gem_shmem_vm_close,
512 };
513 EXPORT_SYMBOL_GPL(drm_gem_shmem_vm_ops);
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531 int drm_gem_shmem_mmap(struct file *filp, struct vm_area_struct *vma)
532 {
533 struct drm_gem_shmem_object *shmem;
534 int ret;
535
536 ret = drm_gem_mmap(filp, vma);
537 if (ret)
538 return ret;
539
540 shmem = to_drm_gem_shmem_obj(vma->vm_private_data);
541
542 ret = drm_gem_shmem_get_pages(shmem);
543 if (ret) {
544 drm_gem_vm_close(vma);
545 return ret;
546 }
547
548
549 vma->vm_flags &= ~VM_PFNMAP;
550 vma->vm_flags |= VM_MIXEDMAP;
551
552
553 vma->vm_pgoff -= drm_vma_node_start(&shmem->base.vma_node);
554
555 return 0;
556 }
557 EXPORT_SYMBOL_GPL(drm_gem_shmem_mmap);
558
559
560
561
562
563
564
565 void drm_gem_shmem_print_info(struct drm_printer *p, unsigned int indent,
566 const struct drm_gem_object *obj)
567 {
568 const struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
569
570 drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count);
571 drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count);
572 drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr);
573 }
574 EXPORT_SYMBOL(drm_gem_shmem_print_info);
575
576
577
578
579
580
581
582
583
584
585
586
587 struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_object *obj)
588 {
589 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
590
591 return drm_prime_pages_to_sg(shmem->pages, obj->size >> PAGE_SHIFT);
592 }
593 EXPORT_SYMBOL_GPL(drm_gem_shmem_get_sg_table);
594
595
596
597
598
599
600
601
602
603
604
605
606
607 struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_object *obj)
608 {
609 int ret;
610 struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
611 struct sg_table *sgt;
612
613 if (shmem->sgt)
614 return shmem->sgt;
615
616 WARN_ON(obj->import_attach);
617
618 ret = drm_gem_shmem_get_pages(shmem);
619 if (ret)
620 return ERR_PTR(ret);
621
622 sgt = drm_gem_shmem_get_sg_table(&shmem->base);
623 if (IS_ERR(sgt)) {
624 ret = PTR_ERR(sgt);
625 goto err_put_pages;
626 }
627
628 dma_map_sg(obj->dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL);
629
630 shmem->sgt = sgt;
631
632 return sgt;
633
634 err_put_pages:
635 drm_gem_shmem_put_pages(shmem);
636 return ERR_PTR(ret);
637 }
638 EXPORT_SYMBOL_GPL(drm_gem_shmem_get_pages_sgt);
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655 struct drm_gem_object *
656 drm_gem_shmem_prime_import_sg_table(struct drm_device *dev,
657 struct dma_buf_attachment *attach,
658 struct sg_table *sgt)
659 {
660 size_t size = PAGE_ALIGN(attach->dmabuf->size);
661 size_t npages = size >> PAGE_SHIFT;
662 struct drm_gem_shmem_object *shmem;
663 int ret;
664
665 shmem = drm_gem_shmem_create(dev, size);
666 if (IS_ERR(shmem))
667 return ERR_CAST(shmem);
668
669 shmem->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
670 if (!shmem->pages) {
671 ret = -ENOMEM;
672 goto err_free_gem;
673 }
674
675 ret = drm_prime_sg_to_page_addr_arrays(sgt, shmem->pages, NULL, npages);
676 if (ret < 0)
677 goto err_free_array;
678
679 shmem->sgt = sgt;
680 shmem->pages_use_count = 1;
681
682 DRM_DEBUG_PRIME("size = %zu\n", size);
683
684 return &shmem->base;
685
686 err_free_array:
687 kvfree(shmem->pages);
688 err_free_gem:
689 drm_gem_object_put_unlocked(&shmem->base);
690
691 return ERR_PTR(ret);
692 }
693 EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_import_sg_table);