1/* 2 * ring buffer based function tracer 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Based on code from the latency_tracer, that is: 8 * 9 * Copyright (C) 2004-2006 Ingo Molnar 10 * Copyright (C) 2004 Nadia Yvette Chambers 11 */ 12#include <linux/ring_buffer.h> 13#include <linux/debugfs.h> 14#include <linux/uaccess.h> 15#include <linux/ftrace.h> 16#include <linux/slab.h> 17#include <linux/fs.h> 18 19#include "trace.h" 20 21static void tracing_start_function_trace(struct trace_array *tr); 22static void tracing_stop_function_trace(struct trace_array *tr); 23static void 24function_trace_call(unsigned long ip, unsigned long parent_ip, 25 struct ftrace_ops *op, struct pt_regs *pt_regs); 26static void 27function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 28 struct ftrace_ops *op, struct pt_regs *pt_regs); 29static struct tracer_flags func_flags; 30 31/* Our option */ 32enum { 33 TRACE_FUNC_OPT_STACK = 0x1, 34}; 35 36static int allocate_ftrace_ops(struct trace_array *tr) 37{ 38 struct ftrace_ops *ops; 39 40 ops = kzalloc(sizeof(*ops), GFP_KERNEL); 41 if (!ops) 42 return -ENOMEM; 43 44 /* Currently only the non stack verision is supported */ 45 ops->func = function_trace_call; 46 ops->flags = FTRACE_OPS_FL_RECURSION_SAFE; 47 48 tr->ops = ops; 49 ops->private = tr; 50 return 0; 51} 52 53 54int ftrace_create_function_files(struct trace_array *tr, 55 struct dentry *parent) 56{ 57 int ret; 58 59 /* 60 * The top level array uses the "global_ops", and the files are 61 * created on boot up. 62 */ 63 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 64 return 0; 65 66 ret = allocate_ftrace_ops(tr); 67 if (ret) 68 return ret; 69 70 ftrace_create_filter_files(tr->ops, parent); 71 72 return 0; 73} 74 75void ftrace_destroy_function_files(struct trace_array *tr) 76{ 77 ftrace_destroy_filter_files(tr->ops); 78 kfree(tr->ops); 79 tr->ops = NULL; 80} 81 82static int function_trace_init(struct trace_array *tr) 83{ 84 ftrace_func_t func; 85 86 /* 87 * Instance trace_arrays get their ops allocated 88 * at instance creation. Unless it failed 89 * the allocation. 90 */ 91 if (!tr->ops) 92 return -ENOMEM; 93 94 /* Currently only the global instance can do stack tracing */ 95 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && 96 func_flags.val & TRACE_FUNC_OPT_STACK) 97 func = function_stack_trace_call; 98 else 99 func = function_trace_call; 100 101 ftrace_init_array_ops(tr, func); 102 103 tr->trace_buffer.cpu = get_cpu(); 104 put_cpu(); 105 106 tracing_start_cmdline_record(); 107 tracing_start_function_trace(tr); 108 return 0; 109} 110 111static void function_trace_reset(struct trace_array *tr) 112{ 113 tracing_stop_function_trace(tr); 114 tracing_stop_cmdline_record(); 115 ftrace_reset_array_ops(tr); 116} 117 118static void function_trace_start(struct trace_array *tr) 119{ 120 tracing_reset_online_cpus(&tr->trace_buffer); 121} 122 123static void 124function_trace_call(unsigned long ip, unsigned long parent_ip, 125 struct ftrace_ops *op, struct pt_regs *pt_regs) 126{ 127 struct trace_array *tr = op->private; 128 struct trace_array_cpu *data; 129 unsigned long flags; 130 int bit; 131 int cpu; 132 int pc; 133 134 if (unlikely(!tr->function_enabled)) 135 return; 136 137 pc = preempt_count(); 138 preempt_disable_notrace(); 139 140 bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX); 141 if (bit < 0) 142 goto out; 143 144 cpu = smp_processor_id(); 145 data = per_cpu_ptr(tr->trace_buffer.data, cpu); 146 if (!atomic_read(&data->disabled)) { 147 local_save_flags(flags); 148 trace_function(tr, ip, parent_ip, flags, pc); 149 } 150 trace_clear_recursion(bit); 151 152 out: 153 preempt_enable_notrace(); 154} 155 156static void 157function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 158 struct ftrace_ops *op, struct pt_regs *pt_regs) 159{ 160 struct trace_array *tr = op->private; 161 struct trace_array_cpu *data; 162 unsigned long flags; 163 long disabled; 164 int cpu; 165 int pc; 166 167 if (unlikely(!tr->function_enabled)) 168 return; 169 170 /* 171 * Need to use raw, since this must be called before the 172 * recursive protection is performed. 173 */ 174 local_irq_save(flags); 175 cpu = raw_smp_processor_id(); 176 data = per_cpu_ptr(tr->trace_buffer.data, cpu); 177 disabled = atomic_inc_return(&data->disabled); 178 179 if (likely(disabled == 1)) { 180 pc = preempt_count(); 181 trace_function(tr, ip, parent_ip, flags, pc); 182 /* 183 * skip over 5 funcs: 184 * __ftrace_trace_stack, 185 * __trace_stack, 186 * function_stack_trace_call 187 * ftrace_list_func 188 * ftrace_call 189 */ 190 __trace_stack(tr, flags, 5, pc); 191 } 192 193 atomic_dec(&data->disabled); 194 local_irq_restore(flags); 195} 196 197static struct tracer_opt func_opts[] = { 198#ifdef CONFIG_STACKTRACE 199 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) }, 200#endif 201 { } /* Always set a last empty entry */ 202}; 203 204static struct tracer_flags func_flags = { 205 .val = 0, /* By default: all flags disabled */ 206 .opts = func_opts 207}; 208 209static void tracing_start_function_trace(struct trace_array *tr) 210{ 211 tr->function_enabled = 0; 212 register_ftrace_function(tr->ops); 213 tr->function_enabled = 1; 214} 215 216static void tracing_stop_function_trace(struct trace_array *tr) 217{ 218 tr->function_enabled = 0; 219 unregister_ftrace_function(tr->ops); 220} 221 222static int 223func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 224{ 225 switch (bit) { 226 case TRACE_FUNC_OPT_STACK: 227 /* do nothing if already set */ 228 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK)) 229 break; 230 231 unregister_ftrace_function(tr->ops); 232 233 if (set) { 234 tr->ops->func = function_stack_trace_call; 235 register_ftrace_function(tr->ops); 236 } else { 237 tr->ops->func = function_trace_call; 238 register_ftrace_function(tr->ops); 239 } 240 241 break; 242 default: 243 return -EINVAL; 244 } 245 246 return 0; 247} 248 249static struct tracer function_trace __tracer_data = 250{ 251 .name = "function", 252 .init = function_trace_init, 253 .reset = function_trace_reset, 254 .start = function_trace_start, 255 .flags = &func_flags, 256 .set_flag = func_set_flag, 257 .allow_instances = true, 258#ifdef CONFIG_FTRACE_SELFTEST 259 .selftest = trace_selftest_startup_function, 260#endif 261}; 262 263#ifdef CONFIG_DYNAMIC_FTRACE 264static void update_traceon_count(void **data, bool on) 265{ 266 long *count = (long *)data; 267 long old_count = *count; 268 269 /* 270 * Tracing gets disabled (or enabled) once per count. 271 * This function can be called at the same time on multiple CPUs. 272 * It is fine if both disable (or enable) tracing, as disabling 273 * (or enabling) the second time doesn't do anything as the 274 * state of the tracer is already disabled (or enabled). 275 * What needs to be synchronized in this case is that the count 276 * only gets decremented once, even if the tracer is disabled 277 * (or enabled) twice, as the second one is really a nop. 278 * 279 * The memory barriers guarantee that we only decrement the 280 * counter once. First the count is read to a local variable 281 * and a read barrier is used to make sure that it is loaded 282 * before checking if the tracer is in the state we want. 283 * If the tracer is not in the state we want, then the count 284 * is guaranteed to be the old count. 285 * 286 * Next the tracer is set to the state we want (disabled or enabled) 287 * then a write memory barrier is used to make sure that 288 * the new state is visible before changing the counter by 289 * one minus the old counter. This guarantees that another CPU 290 * executing this code will see the new state before seeing 291 * the new counter value, and would not do anything if the new 292 * counter is seen. 293 * 294 * Note, there is no synchronization between this and a user 295 * setting the tracing_on file. But we currently don't care 296 * about that. 297 */ 298 if (!old_count) 299 return; 300 301 /* Make sure we see count before checking tracing state */ 302 smp_rmb(); 303 304 if (on == !!tracing_is_on()) 305 return; 306 307 if (on) 308 tracing_on(); 309 else 310 tracing_off(); 311 312 /* unlimited? */ 313 if (old_count == -1) 314 return; 315 316 /* Make sure tracing state is visible before updating count */ 317 smp_wmb(); 318 319 *count = old_count - 1; 320} 321 322static void 323ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, void **data) 324{ 325 update_traceon_count(data, 1); 326} 327 328static void 329ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, void **data) 330{ 331 update_traceon_count(data, 0); 332} 333 334static void 335ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data) 336{ 337 if (tracing_is_on()) 338 return; 339 340 tracing_on(); 341} 342 343static void 344ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data) 345{ 346 if (!tracing_is_on()) 347 return; 348 349 tracing_off(); 350} 351 352/* 353 * Skip 4: 354 * ftrace_stacktrace() 355 * function_trace_probe_call() 356 * ftrace_ops_list_func() 357 * ftrace_call() 358 */ 359#define STACK_SKIP 4 360 361static void 362ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, void **data) 363{ 364 trace_dump_stack(STACK_SKIP); 365} 366 367static void 368ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, void **data) 369{ 370 long *count = (long *)data; 371 long old_count; 372 long new_count; 373 374 /* 375 * Stack traces should only execute the number of times the 376 * user specified in the counter. 377 */ 378 do { 379 380 if (!tracing_is_on()) 381 return; 382 383 old_count = *count; 384 385 if (!old_count) 386 return; 387 388 /* unlimited? */ 389 if (old_count == -1) { 390 trace_dump_stack(STACK_SKIP); 391 return; 392 } 393 394 new_count = old_count - 1; 395 new_count = cmpxchg(count, old_count, new_count); 396 if (new_count == old_count) 397 trace_dump_stack(STACK_SKIP); 398 399 } while (new_count != old_count); 400} 401 402static int update_count(void **data) 403{ 404 unsigned long *count = (long *)data; 405 406 if (!*count) 407 return 0; 408 409 if (*count != -1) 410 (*count)--; 411 412 return 1; 413} 414 415static void 416ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, void **data) 417{ 418 if (update_count(data)) 419 ftrace_dump(DUMP_ALL); 420} 421 422/* Only dump the current CPU buffer. */ 423static void 424ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, void **data) 425{ 426 if (update_count(data)) 427 ftrace_dump(DUMP_ORIG); 428} 429 430static int 431ftrace_probe_print(const char *name, struct seq_file *m, 432 unsigned long ip, void *data) 433{ 434 long count = (long)data; 435 436 seq_printf(m, "%ps:%s", (void *)ip, name); 437 438 if (count == -1) 439 seq_puts(m, ":unlimited\n"); 440 else 441 seq_printf(m, ":count=%ld\n", count); 442 443 return 0; 444} 445 446static int 447ftrace_traceon_print(struct seq_file *m, unsigned long ip, 448 struct ftrace_probe_ops *ops, void *data) 449{ 450 return ftrace_probe_print("traceon", m, ip, data); 451} 452 453static int 454ftrace_traceoff_print(struct seq_file *m, unsigned long ip, 455 struct ftrace_probe_ops *ops, void *data) 456{ 457 return ftrace_probe_print("traceoff", m, ip, data); 458} 459 460static int 461ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, 462 struct ftrace_probe_ops *ops, void *data) 463{ 464 return ftrace_probe_print("stacktrace", m, ip, data); 465} 466 467static int 468ftrace_dump_print(struct seq_file *m, unsigned long ip, 469 struct ftrace_probe_ops *ops, void *data) 470{ 471 return ftrace_probe_print("dump", m, ip, data); 472} 473 474static int 475ftrace_cpudump_print(struct seq_file *m, unsigned long ip, 476 struct ftrace_probe_ops *ops, void *data) 477{ 478 return ftrace_probe_print("cpudump", m, ip, data); 479} 480 481static struct ftrace_probe_ops traceon_count_probe_ops = { 482 .func = ftrace_traceon_count, 483 .print = ftrace_traceon_print, 484}; 485 486static struct ftrace_probe_ops traceoff_count_probe_ops = { 487 .func = ftrace_traceoff_count, 488 .print = ftrace_traceoff_print, 489}; 490 491static struct ftrace_probe_ops stacktrace_count_probe_ops = { 492 .func = ftrace_stacktrace_count, 493 .print = ftrace_stacktrace_print, 494}; 495 496static struct ftrace_probe_ops dump_probe_ops = { 497 .func = ftrace_dump_probe, 498 .print = ftrace_dump_print, 499}; 500 501static struct ftrace_probe_ops cpudump_probe_ops = { 502 .func = ftrace_cpudump_probe, 503 .print = ftrace_cpudump_print, 504}; 505 506static struct ftrace_probe_ops traceon_probe_ops = { 507 .func = ftrace_traceon, 508 .print = ftrace_traceon_print, 509}; 510 511static struct ftrace_probe_ops traceoff_probe_ops = { 512 .func = ftrace_traceoff, 513 .print = ftrace_traceoff_print, 514}; 515 516static struct ftrace_probe_ops stacktrace_probe_ops = { 517 .func = ftrace_stacktrace, 518 .print = ftrace_stacktrace_print, 519}; 520 521static int 522ftrace_trace_probe_callback(struct ftrace_probe_ops *ops, 523 struct ftrace_hash *hash, char *glob, 524 char *cmd, char *param, int enable) 525{ 526 void *count = (void *)-1; 527 char *number; 528 int ret; 529 530 /* hash funcs only work with set_ftrace_filter */ 531 if (!enable) 532 return -EINVAL; 533 534 if (glob[0] == '!') { 535 unregister_ftrace_function_probe_func(glob+1, ops); 536 return 0; 537 } 538 539 if (!param) 540 goto out_reg; 541 542 number = strsep(¶m, ":"); 543 544 if (!strlen(number)) 545 goto out_reg; 546 547 /* 548 * We use the callback data field (which is a pointer) 549 * as our counter. 550 */ 551 ret = kstrtoul(number, 0, (unsigned long *)&count); 552 if (ret) 553 return ret; 554 555 out_reg: 556 ret = register_ftrace_function_probe(glob, ops, count); 557 558 return ret < 0 ? ret : 0; 559} 560 561static int 562ftrace_trace_onoff_callback(struct ftrace_hash *hash, 563 char *glob, char *cmd, char *param, int enable) 564{ 565 struct ftrace_probe_ops *ops; 566 567 /* we register both traceon and traceoff to this callback */ 568 if (strcmp(cmd, "traceon") == 0) 569 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops; 570 else 571 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; 572 573 return ftrace_trace_probe_callback(ops, hash, glob, cmd, 574 param, enable); 575} 576 577static int 578ftrace_stacktrace_callback(struct ftrace_hash *hash, 579 char *glob, char *cmd, char *param, int enable) 580{ 581 struct ftrace_probe_ops *ops; 582 583 ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; 584 585 return ftrace_trace_probe_callback(ops, hash, glob, cmd, 586 param, enable); 587} 588 589static int 590ftrace_dump_callback(struct ftrace_hash *hash, 591 char *glob, char *cmd, char *param, int enable) 592{ 593 struct ftrace_probe_ops *ops; 594 595 ops = &dump_probe_ops; 596 597 /* Only dump once. */ 598 return ftrace_trace_probe_callback(ops, hash, glob, cmd, 599 "1", enable); 600} 601 602static int 603ftrace_cpudump_callback(struct ftrace_hash *hash, 604 char *glob, char *cmd, char *param, int enable) 605{ 606 struct ftrace_probe_ops *ops; 607 608 ops = &cpudump_probe_ops; 609 610 /* Only dump once. */ 611 return ftrace_trace_probe_callback(ops, hash, glob, cmd, 612 "1", enable); 613} 614 615static struct ftrace_func_command ftrace_traceon_cmd = { 616 .name = "traceon", 617 .func = ftrace_trace_onoff_callback, 618}; 619 620static struct ftrace_func_command ftrace_traceoff_cmd = { 621 .name = "traceoff", 622 .func = ftrace_trace_onoff_callback, 623}; 624 625static struct ftrace_func_command ftrace_stacktrace_cmd = { 626 .name = "stacktrace", 627 .func = ftrace_stacktrace_callback, 628}; 629 630static struct ftrace_func_command ftrace_dump_cmd = { 631 .name = "dump", 632 .func = ftrace_dump_callback, 633}; 634 635static struct ftrace_func_command ftrace_cpudump_cmd = { 636 .name = "cpudump", 637 .func = ftrace_cpudump_callback, 638}; 639 640static int __init init_func_cmd_traceon(void) 641{ 642 int ret; 643 644 ret = register_ftrace_command(&ftrace_traceoff_cmd); 645 if (ret) 646 return ret; 647 648 ret = register_ftrace_command(&ftrace_traceon_cmd); 649 if (ret) 650 goto out_free_traceoff; 651 652 ret = register_ftrace_command(&ftrace_stacktrace_cmd); 653 if (ret) 654 goto out_free_traceon; 655 656 ret = register_ftrace_command(&ftrace_dump_cmd); 657 if (ret) 658 goto out_free_stacktrace; 659 660 ret = register_ftrace_command(&ftrace_cpudump_cmd); 661 if (ret) 662 goto out_free_dump; 663 664 return 0; 665 666 out_free_dump: 667 unregister_ftrace_command(&ftrace_dump_cmd); 668 out_free_stacktrace: 669 unregister_ftrace_command(&ftrace_stacktrace_cmd); 670 out_free_traceon: 671 unregister_ftrace_command(&ftrace_traceon_cmd); 672 out_free_traceoff: 673 unregister_ftrace_command(&ftrace_traceoff_cmd); 674 675 return ret; 676} 677#else 678static inline int init_func_cmd_traceon(void) 679{ 680 return 0; 681} 682#endif /* CONFIG_DYNAMIC_FTRACE */ 683 684static __init int init_function_trace(void) 685{ 686 init_func_cmd_traceon(); 687 return register_tracer(&function_trace); 688} 689core_initcall(init_function_trace); 690