This source file includes following definitions.
- ion_heap_map_kernel
- ion_heap_unmap_kernel
- ion_heap_map_user
- ion_heap_clear_pages
- ion_heap_sglist_zero
- ion_heap_buffer_zero
- ion_heap_pages_zero
- ion_heap_freelist_add
- ion_heap_freelist_size
- _ion_heap_freelist_drain
- ion_heap_freelist_drain
- ion_heap_freelist_shrink
- ion_heap_deferred_free
- ion_heap_init_deferred_free
- ion_heap_shrink_count
- ion_heap_shrink_scan
- ion_heap_init_shrinker
1
2
3
4
5
6
7
8 #include <linux/err.h>
9 #include <linux/freezer.h>
10 #include <linux/kthread.h>
11 #include <linux/mm.h>
12 #include <linux/rtmutex.h>
13 #include <linux/sched.h>
14 #include <uapi/linux/sched/types.h>
15 #include <linux/scatterlist.h>
16 #include <linux/vmalloc.h>
17
18 #include "ion.h"
19
20 void *ion_heap_map_kernel(struct ion_heap *heap,
21 struct ion_buffer *buffer)
22 {
23 struct scatterlist *sg;
24 int i, j;
25 void *vaddr;
26 pgprot_t pgprot;
27 struct sg_table *table = buffer->sg_table;
28 int npages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
29 struct page **pages = vmalloc(array_size(npages,
30 sizeof(struct page *)));
31 struct page **tmp = pages;
32
33 if (!pages)
34 return ERR_PTR(-ENOMEM);
35
36 if (buffer->flags & ION_FLAG_CACHED)
37 pgprot = PAGE_KERNEL;
38 else
39 pgprot = pgprot_writecombine(PAGE_KERNEL);
40
41 for_each_sg(table->sgl, sg, table->nents, i) {
42 int npages_this_entry = PAGE_ALIGN(sg->length) / PAGE_SIZE;
43 struct page *page = sg_page(sg);
44
45 BUG_ON(i >= npages);
46 for (j = 0; j < npages_this_entry; j++)
47 *(tmp++) = page++;
48 }
49 vaddr = vmap(pages, npages, VM_MAP, pgprot);
50 vfree(pages);
51
52 if (!vaddr)
53 return ERR_PTR(-ENOMEM);
54
55 return vaddr;
56 }
57
58 void ion_heap_unmap_kernel(struct ion_heap *heap,
59 struct ion_buffer *buffer)
60 {
61 vunmap(buffer->vaddr);
62 }
63
64 int ion_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer,
65 struct vm_area_struct *vma)
66 {
67 struct sg_table *table = buffer->sg_table;
68 unsigned long addr = vma->vm_start;
69 unsigned long offset = vma->vm_pgoff * PAGE_SIZE;
70 struct scatterlist *sg;
71 int i;
72 int ret;
73
74 for_each_sg(table->sgl, sg, table->nents, i) {
75 struct page *page = sg_page(sg);
76 unsigned long remainder = vma->vm_end - addr;
77 unsigned long len = sg->length;
78
79 if (offset >= sg->length) {
80 offset -= sg->length;
81 continue;
82 } else if (offset) {
83 page += offset / PAGE_SIZE;
84 len = sg->length - offset;
85 offset = 0;
86 }
87 len = min(len, remainder);
88 ret = remap_pfn_range(vma, addr, page_to_pfn(page), len,
89 vma->vm_page_prot);
90 if (ret)
91 return ret;
92 addr += len;
93 if (addr >= vma->vm_end)
94 return 0;
95 }
96
97 return 0;
98 }
99
100 static int ion_heap_clear_pages(struct page **pages, int num, pgprot_t pgprot)
101 {
102 void *addr = vm_map_ram(pages, num, -1, pgprot);
103
104 if (!addr)
105 return -ENOMEM;
106 memset(addr, 0, PAGE_SIZE * num);
107 vm_unmap_ram(addr, num);
108
109 return 0;
110 }
111
112 static int ion_heap_sglist_zero(struct scatterlist *sgl, unsigned int nents,
113 pgprot_t pgprot)
114 {
115 int p = 0;
116 int ret = 0;
117 struct sg_page_iter piter;
118 struct page *pages[32];
119
120 for_each_sg_page(sgl, &piter, nents, 0) {
121 pages[p++] = sg_page_iter_page(&piter);
122 if (p == ARRAY_SIZE(pages)) {
123 ret = ion_heap_clear_pages(pages, p, pgprot);
124 if (ret)
125 return ret;
126 p = 0;
127 }
128 }
129 if (p)
130 ret = ion_heap_clear_pages(pages, p, pgprot);
131
132 return ret;
133 }
134
135 int ion_heap_buffer_zero(struct ion_buffer *buffer)
136 {
137 struct sg_table *table = buffer->sg_table;
138 pgprot_t pgprot;
139
140 if (buffer->flags & ION_FLAG_CACHED)
141 pgprot = PAGE_KERNEL;
142 else
143 pgprot = pgprot_writecombine(PAGE_KERNEL);
144
145 return ion_heap_sglist_zero(table->sgl, table->nents, pgprot);
146 }
147
148 int ion_heap_pages_zero(struct page *page, size_t size, pgprot_t pgprot)
149 {
150 struct scatterlist sg;
151
152 sg_init_table(&sg, 1);
153 sg_set_page(&sg, page, size, 0);
154 return ion_heap_sglist_zero(&sg, 1, pgprot);
155 }
156
157 void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer)
158 {
159 spin_lock(&heap->free_lock);
160 list_add(&buffer->list, &heap->free_list);
161 heap->free_list_size += buffer->size;
162 spin_unlock(&heap->free_lock);
163 wake_up(&heap->waitqueue);
164 }
165
166 size_t ion_heap_freelist_size(struct ion_heap *heap)
167 {
168 size_t size;
169
170 spin_lock(&heap->free_lock);
171 size = heap->free_list_size;
172 spin_unlock(&heap->free_lock);
173
174 return size;
175 }
176
177 static size_t _ion_heap_freelist_drain(struct ion_heap *heap, size_t size,
178 bool skip_pools)
179 {
180 struct ion_buffer *buffer;
181 size_t total_drained = 0;
182
183 if (ion_heap_freelist_size(heap) == 0)
184 return 0;
185
186 spin_lock(&heap->free_lock);
187 if (size == 0)
188 size = heap->free_list_size;
189
190 while (!list_empty(&heap->free_list)) {
191 if (total_drained >= size)
192 break;
193 buffer = list_first_entry(&heap->free_list, struct ion_buffer,
194 list);
195 list_del(&buffer->list);
196 heap->free_list_size -= buffer->size;
197 if (skip_pools)
198 buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE;
199 total_drained += buffer->size;
200 spin_unlock(&heap->free_lock);
201 ion_buffer_destroy(buffer);
202 spin_lock(&heap->free_lock);
203 }
204 spin_unlock(&heap->free_lock);
205
206 return total_drained;
207 }
208
209 size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size)
210 {
211 return _ion_heap_freelist_drain(heap, size, false);
212 }
213
214 size_t ion_heap_freelist_shrink(struct ion_heap *heap, size_t size)
215 {
216 return _ion_heap_freelist_drain(heap, size, true);
217 }
218
219 static int ion_heap_deferred_free(void *data)
220 {
221 struct ion_heap *heap = data;
222
223 while (true) {
224 struct ion_buffer *buffer;
225
226 wait_event_freezable(heap->waitqueue,
227 ion_heap_freelist_size(heap) > 0);
228
229 spin_lock(&heap->free_lock);
230 if (list_empty(&heap->free_list)) {
231 spin_unlock(&heap->free_lock);
232 continue;
233 }
234 buffer = list_first_entry(&heap->free_list, struct ion_buffer,
235 list);
236 list_del(&buffer->list);
237 heap->free_list_size -= buffer->size;
238 spin_unlock(&heap->free_lock);
239 ion_buffer_destroy(buffer);
240 }
241
242 return 0;
243 }
244
245 int ion_heap_init_deferred_free(struct ion_heap *heap)
246 {
247 struct sched_param param = { .sched_priority = 0 };
248
249 INIT_LIST_HEAD(&heap->free_list);
250 init_waitqueue_head(&heap->waitqueue);
251 heap->task = kthread_run(ion_heap_deferred_free, heap,
252 "%s", heap->name);
253 if (IS_ERR(heap->task)) {
254 pr_err("%s: creating thread for deferred free failed\n",
255 __func__);
256 return PTR_ERR_OR_ZERO(heap->task);
257 }
258 sched_setscheduler(heap->task, SCHED_IDLE, ¶m);
259
260 return 0;
261 }
262
263 static unsigned long ion_heap_shrink_count(struct shrinker *shrinker,
264 struct shrink_control *sc)
265 {
266 struct ion_heap *heap = container_of(shrinker, struct ion_heap,
267 shrinker);
268 int total = 0;
269
270 total = ion_heap_freelist_size(heap) / PAGE_SIZE;
271
272 if (heap->ops->shrink)
273 total += heap->ops->shrink(heap, sc->gfp_mask, 0);
274
275 return total;
276 }
277
278 static unsigned long ion_heap_shrink_scan(struct shrinker *shrinker,
279 struct shrink_control *sc)
280 {
281 struct ion_heap *heap = container_of(shrinker, struct ion_heap,
282 shrinker);
283 int freed = 0;
284 int to_scan = sc->nr_to_scan;
285
286 if (to_scan == 0)
287 return 0;
288
289
290
291
292
293 if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
294 freed = ion_heap_freelist_shrink(heap, to_scan * PAGE_SIZE) /
295 PAGE_SIZE;
296
297 to_scan -= freed;
298 if (to_scan <= 0)
299 return freed;
300
301 if (heap->ops->shrink)
302 freed += heap->ops->shrink(heap, sc->gfp_mask, to_scan);
303
304 return freed;
305 }
306
307 int ion_heap_init_shrinker(struct ion_heap *heap)
308 {
309 heap->shrinker.count_objects = ion_heap_shrink_count;
310 heap->shrinker.scan_objects = ion_heap_shrink_scan;
311 heap->shrinker.seeks = DEFAULT_SEEKS;
312 heap->shrinker.batch = 0;
313
314 return register_shrinker(&heap->shrinker);
315 }