This source file includes following definitions.
- check_kcov_mode
- canonicalize_ip
- __sanitizer_cov_trace_pc
- write_comp_data
- __sanitizer_cov_trace_cmp1
- __sanitizer_cov_trace_cmp2
- __sanitizer_cov_trace_cmp4
- __sanitizer_cov_trace_cmp8
- __sanitizer_cov_trace_const_cmp1
- __sanitizer_cov_trace_const_cmp2
- __sanitizer_cov_trace_const_cmp4
- __sanitizer_cov_trace_const_cmp8
- __sanitizer_cov_trace_switch
- kcov_get
- kcov_put
- kcov_task_init
- kcov_task_exit
- kcov_mmap
- kcov_open
- kcov_close
- kcov_fault_in_area
- kcov_ioctl_locked
- kcov_ioctl
- kcov_init
1
2 #define pr_fmt(fmt) "kcov: " fmt
3
4 #define DISABLE_BRANCH_PROFILING
5 #include <linux/atomic.h>
6 #include <linux/compiler.h>
7 #include <linux/errno.h>
8 #include <linux/export.h>
9 #include <linux/types.h>
10 #include <linux/file.h>
11 #include <linux/fs.h>
12 #include <linux/init.h>
13 #include <linux/mm.h>
14 #include <linux/preempt.h>
15 #include <linux/printk.h>
16 #include <linux/sched.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/vmalloc.h>
20 #include <linux/debugfs.h>
21 #include <linux/uaccess.h>
22 #include <linux/kcov.h>
23 #include <linux/refcount.h>
24 #include <asm/setup.h>
25
26
27 #define KCOV_WORDS_PER_CMP 4
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42 struct kcov {
43
44
45
46
47
48 refcount_t refcount;
49
50 spinlock_t lock;
51 enum kcov_mode mode;
52
53 unsigned size;
54
55 void *area;
56
57 struct task_struct *t;
58 };
59
60 static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
61 {
62 unsigned int mode;
63
64
65
66
67
68 if (!in_task())
69 return false;
70 mode = READ_ONCE(t->kcov_mode);
71
72
73
74
75
76
77
78 barrier();
79 return mode == needed_mode;
80 }
81
82 static notrace unsigned long canonicalize_ip(unsigned long ip)
83 {
84 #ifdef CONFIG_RANDOMIZE_BASE
85 ip -= kaslr_offset();
86 #endif
87 return ip;
88 }
89
90
91
92
93
94 void notrace __sanitizer_cov_trace_pc(void)
95 {
96 struct task_struct *t;
97 unsigned long *area;
98 unsigned long ip = canonicalize_ip(_RET_IP_);
99 unsigned long pos;
100
101 t = current;
102 if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t))
103 return;
104
105 area = t->kcov_area;
106
107 pos = READ_ONCE(area[0]) + 1;
108 if (likely(pos < t->kcov_size)) {
109 area[pos] = ip;
110 WRITE_ONCE(area[0], pos);
111 }
112 }
113 EXPORT_SYMBOL(__sanitizer_cov_trace_pc);
114
115 #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
116 static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
117 {
118 struct task_struct *t;
119 u64 *area;
120 u64 count, start_index, end_pos, max_pos;
121
122 t = current;
123 if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t))
124 return;
125
126 ip = canonicalize_ip(ip);
127
128
129
130
131
132 area = (u64 *)t->kcov_area;
133 max_pos = t->kcov_size * sizeof(unsigned long);
134
135 count = READ_ONCE(area[0]);
136
137
138 start_index = 1 + count * KCOV_WORDS_PER_CMP;
139 end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
140 if (likely(end_pos <= max_pos)) {
141 area[start_index] = type;
142 area[start_index + 1] = arg1;
143 area[start_index + 2] = arg2;
144 area[start_index + 3] = ip;
145 WRITE_ONCE(area[0], count + 1);
146 }
147 }
148
149 void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2)
150 {
151 write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_);
152 }
153 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1);
154
155 void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2)
156 {
157 write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_);
158 }
159 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2);
160
161 void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2)
162 {
163 write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_);
164 }
165 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4);
166
167 void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2)
168 {
169 write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_);
170 }
171 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8);
172
173 void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2)
174 {
175 write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
176 _RET_IP_);
177 }
178 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1);
179
180 void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2)
181 {
182 write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
183 _RET_IP_);
184 }
185 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2);
186
187 void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2)
188 {
189 write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
190 _RET_IP_);
191 }
192 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4);
193
194 void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2)
195 {
196 write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
197 _RET_IP_);
198 }
199 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8);
200
201 void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
202 {
203 u64 i;
204 u64 count = cases[0];
205 u64 size = cases[1];
206 u64 type = KCOV_CMP_CONST;
207
208 switch (size) {
209 case 8:
210 type |= KCOV_CMP_SIZE(0);
211 break;
212 case 16:
213 type |= KCOV_CMP_SIZE(1);
214 break;
215 case 32:
216 type |= KCOV_CMP_SIZE(2);
217 break;
218 case 64:
219 type |= KCOV_CMP_SIZE(3);
220 break;
221 default:
222 return;
223 }
224 for (i = 0; i < count; i++)
225 write_comp_data(type, cases[i + 2], val, _RET_IP_);
226 }
227 EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
228 #endif
229
230 static void kcov_get(struct kcov *kcov)
231 {
232 refcount_inc(&kcov->refcount);
233 }
234
235 static void kcov_put(struct kcov *kcov)
236 {
237 if (refcount_dec_and_test(&kcov->refcount)) {
238 vfree(kcov->area);
239 kfree(kcov);
240 }
241 }
242
243 void kcov_task_init(struct task_struct *t)
244 {
245 WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
246 barrier();
247 t->kcov_size = 0;
248 t->kcov_area = NULL;
249 t->kcov = NULL;
250 }
251
252 void kcov_task_exit(struct task_struct *t)
253 {
254 struct kcov *kcov;
255
256 kcov = t->kcov;
257 if (kcov == NULL)
258 return;
259 spin_lock(&kcov->lock);
260 if (WARN_ON(kcov->t != t)) {
261 spin_unlock(&kcov->lock);
262 return;
263 }
264
265 kcov_task_init(t);
266 kcov->t = NULL;
267 kcov->mode = KCOV_MODE_INIT;
268 spin_unlock(&kcov->lock);
269 kcov_put(kcov);
270 }
271
272 static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
273 {
274 int res = 0;
275 void *area;
276 struct kcov *kcov = vma->vm_file->private_data;
277 unsigned long size, off;
278 struct page *page;
279
280 area = vmalloc_user(vma->vm_end - vma->vm_start);
281 if (!area)
282 return -ENOMEM;
283
284 spin_lock(&kcov->lock);
285 size = kcov->size * sizeof(unsigned long);
286 if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 ||
287 vma->vm_end - vma->vm_start != size) {
288 res = -EINVAL;
289 goto exit;
290 }
291 if (!kcov->area) {
292 kcov->area = area;
293 vma->vm_flags |= VM_DONTEXPAND;
294 spin_unlock(&kcov->lock);
295 for (off = 0; off < size; off += PAGE_SIZE) {
296 page = vmalloc_to_page(kcov->area + off);
297 if (vm_insert_page(vma, vma->vm_start + off, page))
298 WARN_ONCE(1, "vm_insert_page() failed");
299 }
300 return 0;
301 }
302 exit:
303 spin_unlock(&kcov->lock);
304 vfree(area);
305 return res;
306 }
307
308 static int kcov_open(struct inode *inode, struct file *filep)
309 {
310 struct kcov *kcov;
311
312 kcov = kzalloc(sizeof(*kcov), GFP_KERNEL);
313 if (!kcov)
314 return -ENOMEM;
315 kcov->mode = KCOV_MODE_DISABLED;
316 refcount_set(&kcov->refcount, 1);
317 spin_lock_init(&kcov->lock);
318 filep->private_data = kcov;
319 return nonseekable_open(inode, filep);
320 }
321
322 static int kcov_close(struct inode *inode, struct file *filep)
323 {
324 kcov_put(filep->private_data);
325 return 0;
326 }
327
328
329
330
331
332
333 static void kcov_fault_in_area(struct kcov *kcov)
334 {
335 unsigned long stride = PAGE_SIZE / sizeof(unsigned long);
336 unsigned long *area = kcov->area;
337 unsigned long offset;
338
339 for (offset = 0; offset < kcov->size; offset += stride)
340 READ_ONCE(area[offset]);
341 }
342
343 static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
344 unsigned long arg)
345 {
346 struct task_struct *t;
347 unsigned long size, unused;
348
349 switch (cmd) {
350 case KCOV_INIT_TRACE:
351
352
353
354
355 if (kcov->mode != KCOV_MODE_DISABLED)
356 return -EBUSY;
357
358
359
360
361
362 size = arg;
363 if (size < 2 || size > INT_MAX / sizeof(unsigned long))
364 return -EINVAL;
365 kcov->size = size;
366 kcov->mode = KCOV_MODE_INIT;
367 return 0;
368 case KCOV_ENABLE:
369
370
371
372
373
374
375
376 if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
377 return -EINVAL;
378 t = current;
379 if (kcov->t != NULL || t->kcov != NULL)
380 return -EBUSY;
381 if (arg == KCOV_TRACE_PC)
382 kcov->mode = KCOV_MODE_TRACE_PC;
383 else if (arg == KCOV_TRACE_CMP)
384 #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
385 kcov->mode = KCOV_MODE_TRACE_CMP;
386 #else
387 return -ENOTSUPP;
388 #endif
389 else
390 return -EINVAL;
391 kcov_fault_in_area(kcov);
392
393 t->kcov_size = kcov->size;
394 t->kcov_area = kcov->area;
395
396 barrier();
397 WRITE_ONCE(t->kcov_mode, kcov->mode);
398 t->kcov = kcov;
399 kcov->t = t;
400
401 kcov_get(kcov);
402 return 0;
403 case KCOV_DISABLE:
404
405 unused = arg;
406 if (unused != 0 || current->kcov != kcov)
407 return -EINVAL;
408 t = current;
409 if (WARN_ON(kcov->t != t))
410 return -EINVAL;
411 kcov_task_init(t);
412 kcov->t = NULL;
413 kcov->mode = KCOV_MODE_INIT;
414 kcov_put(kcov);
415 return 0;
416 default:
417 return -ENOTTY;
418 }
419 }
420
421 static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
422 {
423 struct kcov *kcov;
424 int res;
425
426 kcov = filep->private_data;
427 spin_lock(&kcov->lock);
428 res = kcov_ioctl_locked(kcov, cmd, arg);
429 spin_unlock(&kcov->lock);
430 return res;
431 }
432
433 static const struct file_operations kcov_fops = {
434 .open = kcov_open,
435 .unlocked_ioctl = kcov_ioctl,
436 .compat_ioctl = kcov_ioctl,
437 .mmap = kcov_mmap,
438 .release = kcov_close,
439 };
440
441 static int __init kcov_init(void)
442 {
443
444
445
446
447
448 debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops);
449
450 return 0;
451 }
452
453 device_initcall(kcov_init);