root/kernel/trace/trace.c

DEFINITIONS

1. dummy_set_flag
2. set_cmdline_ftrace
3. set_ftrace_dump_on_oops
4. stop_trace_on_warning
5. boot_alloc_snapshot
6. set_trace_boot_options
7. set_trace_boot_clock
8. set_tracepoint_printk
9. ns2usecs
10. trace_array_get
11. __trace_array_put
12. trace_array_put
13. tracing_check_open_get_tr
14. call_filter_check_discard
15. trace_free_pid_list
16. trace_find_filtered_pid
17. trace_ignore_this_task
18. trace_filter_add_remove_task
19. trace_pid_next
20. trace_pid_start
21. trace_pid_show
22. trace_pid_write
23. buffer_ftrace_now
24. ftrace_now
25. tracing_is_enabled
26. trace_access_lock
27. trace_access_unlock
28. trace_access_lock_init
29. trace_access_lock
30. trace_access_unlock
31. trace_access_lock_init
32. __ftrace_trace_stack
33. ftrace_trace_stack
34. trace_event_setup
35. __trace_buffer_lock_reserve
36. tracer_tracing_on
37. tracing_on
38. __buffer_unlock_commit
39. __trace_puts
40. __trace_bputs
41. tracing_snapshot_instance_cond
42. tracing_snapshot_instance
43. tracing_snapshot
44. tracing_snapshot_cond
45. tracing_cond_snapshot_data
46. tracing_alloc_snapshot_instance
47. free_snapshot
48. tracing_alloc_snapshot
49. tracing_snapshot_alloc
50. tracing_snapshot_cond_enable
51. tracing_snapshot_cond_disable
52. tracing_snapshot
53. tracing_snapshot_cond
54. tracing_alloc_snapshot
55. tracing_snapshot_alloc
56. tracing_cond_snapshot_data
57. tracing_snapshot_cond_enable
58. tracing_snapshot_cond_disable
59. tracer_tracing_off
60. tracing_off
61. disable_trace_on_warning
62. tracer_tracing_is_on
63. tracing_is_on
64. set_buf_size
65. set_tracing_thresh
66. nsecs_to_usecs
67. trace_clock_in_ns
68. trace_parser_get_init
69. trace_parser_put
70. trace_get_user
71. trace_seq_to_buffer
72. __update_max_tr
73. update_max_tr
74. update_max_tr_single
75. wait_on_pipe
76. save_selftest
77. run_tracer_selftest
78. init_trace_selftests
79. run_tracer_selftest
80. register_tracer
81. tracing_reset_cpu
82. tracing_reset_online_cpus
83. tracing_reset_all_online_cpus
84. get_saved_cmdlines
85. set_cmdline
86. allocate_cmdlines_buffer
87. trace_create_savedcmd
88. is_tracing_stopped
89. tracing_start
90. tracing_start_tr
91. tracing_stop
92. tracing_stop_tr
93. trace_save_cmdline
94. __trace_find_cmdline
95. trace_find_cmdline
96. trace_find_tgid
97. trace_save_tgid
98. tracing_record_taskinfo_skip
99. tracing_record_taskinfo
100. tracing_record_taskinfo_sched_switch
101. tracing_record_cmdline
102. tracing_record_tgid
103. trace_handle_return
104. tracing_generic_entry_update
105. trace_buffer_lock_reserve
106. trace_buffered_event_enable
107. enable_trace_buffered_event
108. disable_trace_buffered_event
109. trace_buffered_event_disable
110. trace_event_buffer_lock_reserve
111. output_printk
112. tracepoint_printk_sysctl
113. trace_event_buffer_commit
114. trace_buffer_unlock_commit_regs
115. trace_buffer_unlock_commit_nostack
116. trace_process_export
117. ftrace_exports_enable
118. ftrace_exports_disable
119. ftrace_exports
120. add_trace_export
121. rm_trace_export
122. add_ftrace_export
123. rm_ftrace_export
124. register_ftrace_export
125. unregister_ftrace_export
126. trace_function
127. __ftrace_trace_stack
128. ftrace_trace_stack
129. __trace_stack
130. trace_dump_stack
131. ftrace_trace_userstack
132. ftrace_trace_userstack
133. get_trace_buf
134. put_trace_buf
135. alloc_percpu_trace_buffer
136. trace_printk_init_buffers
137. trace_printk_start_comm
138. trace_printk_start_stop_comm
139. trace_vbprintk
140. __printf
141. __printf
142. __printf
143. __printf
144. __printf
145. trace_iterator_increment
146. peek_next_entry
147. __find_next_entry
148. trace_find_next_entry
149. trace_find_next_entry_inc
150. trace_consume
151. s_next
152. tracing_iter_reset
153. s_start
154. s_stop
155. get_total_entries_cpu
156. get_total_entries
157. trace_total_entries_cpu
158. trace_total_entries
159. print_lat_help_header
160. print_event_info
161. print_func_help_header
162. print_func_help_header_irq
163. print_trace_header
164. test_cpu_buff_start
165. print_trace_fmt
166. print_raw_fmt
167. print_hex_fmt
168. print_bin_fmt
169. trace_empty
170. print_trace_line
171. trace_latency_header
172. trace_default_header
173. test_ftrace_alive
174. show_snapshot_main_help
175. show_snapshot_percpu_help
176. print_snapshot_help
177. print_snapshot_help
178. s_show
179. tracing_get_cpu
180. __tracing_open
181. tracing_open_generic
182. tracing_is_disabled
183. tracing_open_generic_tr
184. tracing_release
185. tracing_release_generic_tr
186. tracing_single_release_tr
187. tracing_open
188. trace_ok_for_array
189. get_tracer_for_array
190. t_next
191. t_start
192. t_stop
193. t_show
194. show_traces_open
195. show_traces_release
196. tracing_write_stub
197. tracing_lseek
198. tracing_cpumask_read
199. tracing_cpumask_write
200. tracing_trace_options_show
201. __set_tracer_option
202. set_tracer_option
203. trace_keep_overwrite
204. set_tracer_flag
205. trace_set_options
206. apply_trace_boot_options
207. tracing_trace_options_write
208. tracing_trace_options_open
209. tracing_readme_read
210. saved_tgids_next
211. saved_tgids_start
212. saved_tgids_stop
213. saved_tgids_show
214. tracing_saved_tgids_open
215. saved_cmdlines_next
216. saved_cmdlines_start
217. saved_cmdlines_stop
218. saved_cmdlines_show
219. tracing_saved_cmdlines_open
220. tracing_saved_cmdlines_size_read
221. free_saved_cmdlines_buffer
222. tracing_resize_saved_cmdlines
223. tracing_saved_cmdlines_size_write
224. update_eval_map
225. eval_map_next
226. eval_map_start
227. eval_map_stop
228. eval_map_show
229. tracing_eval_map_open
230. trace_eval_jmp_to_tail
231. trace_insert_eval_map_file
232. trace_create_eval_file
233. trace_create_eval_file
234. trace_insert_eval_map_file
235. trace_insert_eval_map
236. tracing_set_trace_read
237. tracer_init
238. set_buffer_entries
239. resize_buffer_duplicate_size
240. __tracing_resize_ring_buffer
241. tracing_resize_ring_buffer
242. tracing_update_buffers
243. tracing_set_nop
244. add_tracer_options
245. tracing_set_tracer
246. tracing_set_trace_write
247. tracing_nsecs_read
248. tracing_nsecs_write
249. tracing_thresh_read
250. tracing_thresh_write
251. tracing_max_lat_read
252. tracing_max_lat_write
253. tracing_open_pipe
254. tracing_release_pipe
255. trace_poll
256. tracing_poll_pipe
257. tracing_wait_pipe
258. tracing_read_pipe
259. tracing_spd_release_pipe
260. tracing_fill_pipe_page
261. tracing_splice_read_pipe
262. tracing_entries_read
263. tracing_entries_write
264. tracing_total_entries_read
265. tracing_free_buffer_write
266. tracing_free_buffer_release
267. tracing_mark_write
268. tracing_mark_raw_write
269. tracing_clock_show
270. tracing_set_clock
271. tracing_clock_write
272. tracing_clock_open
273. tracing_time_stamp_mode_show
274. tracing_time_stamp_mode_open
275. tracing_set_time_stamp_abs
276. tracing_snapshot_open
277. tracing_snapshot_write
278. tracing_snapshot_release
279. snapshot_raw_open
280. get_tracing_log_err
281. err_pos
282. tracing_log_err
283. clear_tracing_err_log
284. tracing_err_log_seq_start
285. tracing_err_log_seq_next
286. tracing_err_log_seq_stop
287. tracing_err_log_show_pos
288. tracing_err_log_seq_show
289. tracing_err_log_open
290. tracing_err_log_write
291. tracing_err_log_release
292. tracing_buffers_open
293. tracing_buffers_poll
294. tracing_buffers_read
295. tracing_buffers_release
296. buffer_ref_release
297. buffer_pipe_buf_release
298. buffer_pipe_buf_get
299. buffer_spd_release
300. tracing_buffers_splice_read
301. tracing_stats_read
302. tracing_read_dyn_info
303. ftrace_snapshot
304. ftrace_count_snapshot
305. ftrace_snapshot_print
306. ftrace_snapshot_init
307. ftrace_snapshot_free
308. ftrace_trace_snapshot_callback
309. register_snapshot_cmd
310. register_snapshot_cmd
311. tracing_get_dentry
312. tracing_dentry_percpu
313. trace_create_cpu_file
314. tracing_init_tracefs_percpu
315. trace_options_read
316. trace_options_write
317. get_tr_index
318. trace_options_core_read
319. trace_options_core_write
320. trace_create_file
321. trace_options_init_dentry
322. create_trace_option_file
323. create_trace_option_files
324. create_trace_option_core_file
325. create_trace_options_dir
326. rb_simple_read
327. rb_simple_write
328. buffer_percent_read
329. buffer_percent_write
330. allocate_trace_buffer
331. allocate_trace_buffers
332. free_trace_buffer
333. free_trace_buffers
334. init_trace_flags_index
335. __update_tracer_options
336. update_tracer_options
337. trace_array_create
338. instance_mkdir
339. __remove_instance
340. trace_array_destroy
341. instance_rmdir
342. create_trace_instances
343. init_tracer_tracefs
344. trace_automount
345. tracing_init_dentry
346. trace_eval_init
347. trace_module_add_evals
348. trace_module_remove_evals
349. trace_module_remove_evals
350. trace_module_notify
351. tracer_init_tracefs
352. trace_panic_handler
353. trace_die_handler
354. trace_printk_seq
355. trace_init_global_iter
356. ftrace_dump
357. trace_run_command
358. trace_parse_run_command
359. tracer_alloc_buffers
360. early_trace_init
361. trace_init
362. clear_boot_tracer
363. tracing_set_default_clock

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * ring buffer based function tracer
   4  *
   5  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
   6  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   7  *
   8  * Originally taken from the RT patch by:
   9  *    Arnaldo Carvalho de Melo <acme@redhat.com>
  10  *
  11  * Based on code from the latency_tracer, that is:
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 Nadia Yvette Chambers
  14  */
  15 #include <linux/ring_buffer.h>
  16 #include <generated/utsrelease.h>
  17 #include <linux/stacktrace.h>
  18 #include <linux/writeback.h>
  19 #include <linux/kallsyms.h>
  20 #include <linux/security.h>
  21 #include <linux/seq_file.h>
  22 #include <linux/notifier.h>
  23 #include <linux/irqflags.h>
  24 #include <linux/debugfs.h>
  25 #include <linux/tracefs.h>
  26 #include <linux/pagemap.h>
  27 #include <linux/hardirq.h>
  28 #include <linux/linkage.h>
  29 #include <linux/uaccess.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/ftrace.h>
  32 #include <linux/module.h>
  33 #include <linux/percpu.h>
  34 #include <linux/splice.h>
  35 #include <linux/kdebug.h>
  36 #include <linux/string.h>
  37 #include <linux/mount.h>
  38 #include <linux/rwsem.h>
  39 #include <linux/slab.h>
  40 #include <linux/ctype.h>
  41 #include <linux/init.h>
  42 #include <linux/poll.h>
  43 #include <linux/nmi.h>
  44 #include <linux/fs.h>
  45 #include <linux/trace.h>
  46 #include <linux/sched/clock.h>
  47 #include <linux/sched/rt.h>
  48 
  49 #include "trace.h"
  50 #include "trace_output.h"
  51 
  52 /*
  53  * On boot up, the ring buffer is set to the minimum size, so that
  54  * we do not waste memory on systems that are not using tracing.
  55  */
  56 bool ring_buffer_expanded;
  57 
  58 /*
  59  * We need to change this state when a selftest is running.
  60  * A selftest will lurk into the ring-buffer to count the
  61  * entries inserted during the selftest although some concurrent
  62  * insertions into the ring-buffer such as trace_printk could occurred
  63  * at the same time, giving false positive or negative results.
  64  */
  65 static bool __read_mostly tracing_selftest_running;
  66 
  67 /*
  68  * If a tracer is running, we do not want to run SELFTEST.
  69  */
  70 bool __read_mostly tracing_selftest_disabled;
  71 
  72 /* Pipe tracepoints to printk */
  73 struct trace_iterator *tracepoint_print_iter;
  74 int tracepoint_printk;
  75 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
  76 
  77 /* For tracers that don't implement custom flags */
  78 static struct tracer_opt dummy_tracer_opt[] = {
  79         { }
  80 };
  81 
  82 static int
  83 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
  84 {
  85         return 0;
  86 }
  87 
  88 /*
  89  * To prevent the comm cache from being overwritten when no
  90  * tracing is active, only save the comm when a trace event
  91  * occurred.
  92  */
  93 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
  94 
  95 /*
  96  * Kill all tracing for good (never come back).
  97  * It is initialized to 1 but will turn to zero if the initialization
  98  * of the tracer is successful. But that is the only place that sets
  99  * this back to zero.
 100  */
 101 static int tracing_disabled = 1;
 102 
 103 cpumask_var_t __read_mostly     tracing_buffer_mask;
 104 
 105 /*
 106  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
 107  *
 108  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
 109  * is set, then ftrace_dump is called. This will output the contents
 110  * of the ftrace buffers to the console.  This is very useful for
 111  * capturing traces that lead to crashes and outputing it to a
 112  * serial console.
 113  *
 114  * It is default off, but you can enable it with either specifying
 115  * "ftrace_dump_on_oops" in the kernel command line, or setting
 116  * /proc/sys/kernel/ftrace_dump_on_oops
 117  * Set 1 if you want to dump buffers of all CPUs
 118  * Set 2 if you want to dump the buffer of the CPU that triggered oops
 119  */
 120 
 121 enum ftrace_dump_mode ftrace_dump_on_oops;
 122 
 123 /* When set, tracing will stop when a WARN*() is hit */
 124 int __disable_trace_on_warning;
 125 
 126 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
 127 /* Map of enums to their values, for "eval_map" file */
 128 struct trace_eval_map_head {
 129         struct module                   *mod;
 130         unsigned long                   length;
 131 };
 132 
 133 union trace_eval_map_item;
 134 
 135 struct trace_eval_map_tail {
 136         /*
 137          * "end" is first and points to NULL as it must be different
 138          * than "mod" or "eval_string"
 139          */
 140         union trace_eval_map_item       *next;
 141         const char                      *end;   /* points to NULL */
 142 };
 143 
 144 static DEFINE_MUTEX(trace_eval_mutex);
 145 
 146 /*
 147  * The trace_eval_maps are saved in an array with two extra elements,
 148  * one at the beginning, and one at the end. The beginning item contains
 149  * the count of the saved maps (head.length), and the module they
 150  * belong to if not built in (head.mod). The ending item contains a
 151  * pointer to the next array of saved eval_map items.
 152  */
 153 union trace_eval_map_item {
 154         struct trace_eval_map           map;
 155         struct trace_eval_map_head      head;
 156         struct trace_eval_map_tail      tail;
 157 };
 158 
 159 static union trace_eval_map_item *trace_eval_maps;
 160 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
 161 
 162 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
 163 static void ftrace_trace_userstack(struct ring_buffer *buffer,
 164                                    unsigned long flags, int pc);
 165 
 166 #define MAX_TRACER_SIZE         100
 167 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
 168 static char *default_bootup_tracer;
 169 
 170 static bool allocate_snapshot;
 171 
 172 static int __init set_cmdline_ftrace(char *str)
 173 {
 174         strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
 175         default_bootup_tracer = bootup_tracer_buf;
 176         /* We are using ftrace early, expand it */
 177         ring_buffer_expanded = true;
 178         return 1;
 179 }
 180 __setup("ftrace=", set_cmdline_ftrace);
 181 
 182 static int __init set_ftrace_dump_on_oops(char *str)
 183 {
 184         if (*str++ != '=' || !*str) {
 185                 ftrace_dump_on_oops = DUMP_ALL;
 186                 return 1;
 187         }
 188 
 189         if (!strcmp("orig_cpu", str)) {
 190                 ftrace_dump_on_oops = DUMP_ORIG;
 191                 return 1;
 192         }
 193 
 194         return 0;
 195 }
 196 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
 197 
 198 static int __init stop_trace_on_warning(char *str)
 199 {
 200         if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
 201                 __disable_trace_on_warning = 1;
 202         return 1;
 203 }
 204 __setup("traceoff_on_warning", stop_trace_on_warning);
 205 
 206 static int __init boot_alloc_snapshot(char *str)
 207 {
 208         allocate_snapshot = true;
 209         /* We also need the main ring buffer expanded */
 210         ring_buffer_expanded = true;
 211         return 1;
 212 }
 213 __setup("alloc_snapshot", boot_alloc_snapshot);
 214 
 215 
 216 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
 217 
 218 static int __init set_trace_boot_options(char *str)
 219 {
 220         strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
 221         return 0;
 222 }
 223 __setup("trace_options=", set_trace_boot_options);
 224 
 225 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
 226 static char *trace_boot_clock __initdata;
 227 
 228 static int __init set_trace_boot_clock(char *str)
 229 {
 230         strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
 231         trace_boot_clock = trace_boot_clock_buf;
 232         return 0;
 233 }
 234 __setup("trace_clock=", set_trace_boot_clock);
 235 
 236 static int __init set_tracepoint_printk(char *str)
 237 {
 238         if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
 239                 tracepoint_printk = 1;
 240         return 1;
 241 }
 242 __setup("tp_printk", set_tracepoint_printk);
 243 
 244 unsigned long long ns2usecs(u64 nsec)
 245 {
 246         nsec += 500;
 247         do_div(nsec, 1000);
 248         return nsec;
 249 }
 250 
 251 /* trace_flags holds trace_options default values */
 252 #define TRACE_DEFAULT_FLAGS                                             \
 253         (FUNCTION_DEFAULT_FLAGS |                                       \
 254          TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |                  \
 255          TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO |                \
 256          TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |                 \
 257          TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
 258 
 259 /* trace_options that are only supported by global_trace */
 260 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK |                      \
 261                TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
 262 
 263 /* trace_flags that are default zero for instances */
 264 #define ZEROED_TRACE_FLAGS \
 265         (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
 266 
 267 /*
 268  * The global_trace is the descriptor that holds the top-level tracing
 269  * buffers for the live tracing.
 270  */
 271 static struct trace_array global_trace = {
 272         .trace_flags = TRACE_DEFAULT_FLAGS,
 273 };
 274 
 275 LIST_HEAD(ftrace_trace_arrays);
 276 
 277 int trace_array_get(struct trace_array *this_tr)
 278 {
 279         struct trace_array *tr;
 280         int ret = -ENODEV;
 281 
 282         mutex_lock(&trace_types_lock);
 283         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
 284                 if (tr == this_tr) {
 285                         tr->ref++;
 286                         ret = 0;
 287                         break;
 288                 }
 289         }
 290         mutex_unlock(&trace_types_lock);
 291 
 292         return ret;
 293 }
 294 
 295 static void __trace_array_put(struct trace_array *this_tr)
 296 {
 297         WARN_ON(!this_tr->ref);
 298         this_tr->ref--;
 299 }
 300 
 301 void trace_array_put(struct trace_array *this_tr)
 302 {
 303         mutex_lock(&trace_types_lock);
 304         __trace_array_put(this_tr);
 305         mutex_unlock(&trace_types_lock);
 306 }
 307 
 308 int tracing_check_open_get_tr(struct trace_array *tr)
 309 {
 310         int ret;
 311 
 312         ret = security_locked_down(LOCKDOWN_TRACEFS);
 313         if (ret)
 314                 return ret;
 315 
 316         if (tracing_disabled)
 317                 return -ENODEV;
 318 
 319         if (tr && trace_array_get(tr) < 0)
 320                 return -ENODEV;
 321 
 322         return 0;
 323 }
 324 
 325 int call_filter_check_discard(struct trace_event_call *call, void *rec,
 326                               struct ring_buffer *buffer,
 327                               struct ring_buffer_event *event)
 328 {
 329         if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
 330             !filter_match_preds(call->filter, rec)) {
 331                 __trace_event_discard_commit(buffer, event);
 332                 return 1;
 333         }
 334 
 335         return 0;
 336 }
 337 
 338 void trace_free_pid_list(struct trace_pid_list *pid_list)
 339 {
 340         vfree(pid_list->pids);
 341         kfree(pid_list);
 342 }
 343 
 344 /**
 345  * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
 346  * @filtered_pids: The list of pids to check
 347  * @search_pid: The PID to find in @filtered_pids
 348  *
 349  * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
 350  */
 351 bool
 352 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
 353 {
 354         /*
 355          * If pid_max changed after filtered_pids was created, we
 356          * by default ignore all pids greater than the previous pid_max.
 357          */
 358         if (search_pid >= filtered_pids->pid_max)
 359                 return false;
 360 
 361         return test_bit(search_pid, filtered_pids->pids);
 362 }
 363 
 364 /**
 365  * trace_ignore_this_task - should a task be ignored for tracing
 366  * @filtered_pids: The list of pids to check
 367  * @task: The task that should be ignored if not filtered
 368  *
 369  * Checks if @task should be traced or not from @filtered_pids.
 370  * Returns true if @task should *NOT* be traced.
 371  * Returns false if @task should be traced.
 372  */
 373 bool
 374 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
 375 {
 376         /*
 377          * Return false, because if filtered_pids does not exist,
 378          * all pids are good to trace.
 379          */
 380         if (!filtered_pids)
 381                 return false;
 382 
 383         return !trace_find_filtered_pid(filtered_pids, task->pid);
 384 }
 385 
 386 /**
 387  * trace_filter_add_remove_task - Add or remove a task from a pid_list
 388  * @pid_list: The list to modify
 389  * @self: The current task for fork or NULL for exit
 390  * @task: The task to add or remove
 391  *
 392  * If adding a task, if @self is defined, the task is only added if @self
 393  * is also included in @pid_list. This happens on fork and tasks should
 394  * only be added when the parent is listed. If @self is NULL, then the
 395  * @task pid will be removed from the list, which would happen on exit
 396  * of a task.
 397  */
 398 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
 399                                   struct task_struct *self,
 400                                   struct task_struct *task)
 401 {
 402         if (!pid_list)
 403                 return;
 404 
 405         /* For forks, we only add if the forking task is listed */
 406         if (self) {
 407                 if (!trace_find_filtered_pid(pid_list, self->pid))
 408                         return;
 409         }
 410 
 411         /* Sorry, but we don't support pid_max changing after setting */
 412         if (task->pid >= pid_list->pid_max)
 413                 return;
 414 
 415         /* "self" is set for forks, and NULL for exits */
 416         if (self)
 417                 set_bit(task->pid, pid_list->pids);
 418         else
 419                 clear_bit(task->pid, pid_list->pids);
 420 }
 421 
 422 /**
 423  * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
 424  * @pid_list: The pid list to show
 425  * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
 426  * @pos: The position of the file
 427  *
 428  * This is used by the seq_file "next" operation to iterate the pids
 429  * listed in a trace_pid_list structure.
 430  *
 431  * Returns the pid+1 as we want to display pid of zero, but NULL would
 432  * stop the iteration.
 433  */
 434 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
 435 {
 436         unsigned long pid = (unsigned long)v;
 437 
 438         (*pos)++;
 439 
 440         /* pid already is +1 of the actual prevous bit */
 441         pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
 442 
 443         /* Return pid + 1 to allow zero to be represented */
 444         if (pid < pid_list->pid_max)
 445                 return (void *)(pid + 1);
 446 
 447         return NULL;
 448 }
 449 
 450 /**
 451  * trace_pid_start - Used for seq_file to start reading pid lists
 452  * @pid_list: The pid list to show
 453  * @pos: The position of the file
 454  *
 455  * This is used by seq_file "start" operation to start the iteration
 456  * of listing pids.
 457  *
 458  * Returns the pid+1 as we want to display pid of zero, but NULL would
 459  * stop the iteration.
 460  */
 461 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
 462 {
 463         unsigned long pid;
 464         loff_t l = 0;
 465 
 466         pid = find_first_bit(pid_list->pids, pid_list->pid_max);
 467         if (pid >= pid_list->pid_max)
 468                 return NULL;
 469 
 470         /* Return pid + 1 so that zero can be the exit value */
 471         for (pid++; pid && l < *pos;
 472              pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
 473                 ;
 474         return (void *)pid;
 475 }
 476 
 477 /**
 478  * trace_pid_show - show the current pid in seq_file processing
 479  * @m: The seq_file structure to write into
 480  * @v: A void pointer of the pid (+1) value to display
 481  *
 482  * Can be directly used by seq_file operations to display the current
 483  * pid value.
 484  */
 485 int trace_pid_show(struct seq_file *m, void *v)
 486 {
 487         unsigned long pid = (unsigned long)v - 1;
 488 
 489         seq_printf(m, "%lu\n", pid);
 490         return 0;
 491 }
 492 
 493 /* 128 should be much more than enough */
 494 #define PID_BUF_SIZE            127
 495 
 496 int trace_pid_write(struct trace_pid_list *filtered_pids,
 497                     struct trace_pid_list **new_pid_list,
 498                     const char __user *ubuf, size_t cnt)
 499 {
 500         struct trace_pid_list *pid_list;
 501         struct trace_parser parser;
 502         unsigned long val;
 503         int nr_pids = 0;
 504         ssize_t read = 0;
 505         ssize_t ret = 0;
 506         loff_t pos;
 507         pid_t pid;
 508 
 509         if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
 510                 return -ENOMEM;
 511 
 512         /*
 513          * Always recreate a new array. The write is an all or nothing
 514          * operation. Always create a new array when adding new pids by
 515          * the user. If the operation fails, then the current list is
 516          * not modified.
 517          */
 518         pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
 519         if (!pid_list) {
 520                 trace_parser_put(&parser);
 521                 return -ENOMEM;
 522         }
 523 
 524         pid_list->pid_max = READ_ONCE(pid_max);
 525 
 526         /* Only truncating will shrink pid_max */
 527         if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
 528                 pid_list->pid_max = filtered_pids->pid_max;
 529 
 530         pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
 531         if (!pid_list->pids) {
 532                 trace_parser_put(&parser);
 533                 kfree(pid_list);
 534                 return -ENOMEM;
 535         }
 536 
 537         if (filtered_pids) {
 538                 /* copy the current bits to the new max */
 539                 for_each_set_bit(pid, filtered_pids->pids,
 540                                  filtered_pids->pid_max) {
 541                         set_bit(pid, pid_list->pids);
 542                         nr_pids++;
 543                 }
 544         }
 545 
 546         while (cnt > 0) {
 547 
 548                 pos = 0;
 549 
 550                 ret = trace_get_user(&parser, ubuf, cnt, &pos);
 551                 if (ret < 0 || !trace_parser_loaded(&parser))
 552                         break;
 553 
 554                 read += ret;
 555                 ubuf += ret;
 556                 cnt -= ret;
 557 
 558                 ret = -EINVAL;
 559                 if (kstrtoul(parser.buffer, 0, &val))
 560                         break;
 561                 if (val >= pid_list->pid_max)
 562                         break;
 563 
 564                 pid = (pid_t)val;
 565 
 566                 set_bit(pid, pid_list->pids);
 567                 nr_pids++;
 568 
 569                 trace_parser_clear(&parser);
 570                 ret = 0;
 571         }
 572         trace_parser_put(&parser);
 573 
 574         if (ret < 0) {
 575                 trace_free_pid_list(pid_list);
 576                 return ret;
 577         }
 578 
 579         if (!nr_pids) {
 580                 /* Cleared the list of pids */
 581                 trace_free_pid_list(pid_list);
 582                 read = ret;
 583                 pid_list = NULL;
 584         }
 585 
 586         *new_pid_list = pid_list;
 587 
 588         return read;
 589 }
 590 
 591 static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
 592 {
 593         u64 ts;
 594 
 595         /* Early boot up does not have a buffer yet */
 596         if (!buf->buffer)
 597                 return trace_clock_local();
 598 
 599         ts = ring_buffer_time_stamp(buf->buffer, cpu);
 600         ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
 601 
 602         return ts;
 603 }
 604 
 605 u64 ftrace_now(int cpu)
 606 {
 607         return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
 608 }
 609 
 610 /**
 611  * tracing_is_enabled - Show if global_trace has been disabled
 612  *
 613  * Shows if the global trace has been enabled or not. It uses the
 614  * mirror flag "buffer_disabled" to be used in fast paths such as for
 615  * the irqsoff tracer. But it may be inaccurate due to races. If you
 616  * need to know the accurate state, use tracing_is_on() which is a little
 617  * slower, but accurate.
 618  */
 619 int tracing_is_enabled(void)
 620 {
 621         /*
 622          * For quick access (irqsoff uses this in fast path), just
 623          * return the mirror variable of the state of the ring buffer.
 624          * It's a little racy, but we don't really care.
 625          */
 626         smp_rmb();
 627         return !global_trace.buffer_disabled;
 628 }
 629 
 630 /*
 631  * trace_buf_size is the size in bytes that is allocated
 632  * for a buffer. Note, the number of bytes is always rounded
 633  * to page size.
 634  *
 635  * This number is purposely set to a low number of 16384.
 636  * If the dump on oops happens, it will be much appreciated
 637  * to not have to wait for all that output. Anyway this can be
 638  * boot time and run time configurable.
 639  */
 640 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
 641 
 642 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 643 
 644 /* trace_types holds a link list of available tracers. */
 645 static struct tracer            *trace_types __read_mostly;
 646 
 647 /*
 648  * trace_types_lock is used to protect the trace_types list.
 649  */
 650 DEFINE_MUTEX(trace_types_lock);
 651 
 652 /*
 653  * serialize the access of the ring buffer
 654  *
 655  * ring buffer serializes readers, but it is low level protection.
 656  * The validity of the events (which returns by ring_buffer_peek() ..etc)
 657  * are not protected by ring buffer.
 658  *
 659  * The content of events may become garbage if we allow other process consumes
 660  * these events concurrently:
 661  *   A) the page of the consumed events may become a normal page
 662  *      (not reader page) in ring buffer, and this page will be rewrited
 663  *      by events producer.
 664  *   B) The page of the consumed events may become a page for splice_read,
 665  *      and this page will be returned to system.
 666  *
 667  * These primitives allow multi process access to different cpu ring buffer
 668  * concurrently.
 669  *
 670  * These primitives don't distinguish read-only and read-consume access.
 671  * Multi read-only access are also serialized.
 672  */
 673 
 674 #ifdef CONFIG_SMP
 675 static DECLARE_RWSEM(all_cpu_access_lock);
 676 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
 677 
 678 static inline void trace_access_lock(int cpu)
 679 {
 680         if (cpu == RING_BUFFER_ALL_CPUS) {
 681                 /* gain it for accessing the whole ring buffer. */
 682                 down_write(&all_cpu_access_lock);
 683         } else {
 684                 /* gain it for accessing a cpu ring buffer. */
 685 
 686                 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
 687                 down_read(&all_cpu_access_lock);
 688 
 689                 /* Secondly block other access to this @cpu ring buffer. */
 690                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
 691         }
 692 }
 693 
 694 static inline void trace_access_unlock(int cpu)
 695 {
 696         if (cpu == RING_BUFFER_ALL_CPUS) {
 697                 up_write(&all_cpu_access_lock);
 698         } else {
 699                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
 700                 up_read(&all_cpu_access_lock);
 701         }
 702 }
 703 
 704 static inline void trace_access_lock_init(void)
 705 {
 706         int cpu;
 707 
 708         for_each_possible_cpu(cpu)
 709                 mutex_init(&per_cpu(cpu_access_lock, cpu));
 710 }
 711 
 712 #else
 713 
 714 static DEFINE_MUTEX(access_lock);
 715 
 716 static inline void trace_access_lock(int cpu)
 717 {
 718         (void)cpu;
 719         mutex_lock(&access_lock);
 720 }
 721 
 722 static inline void trace_access_unlock(int cpu)
 723 {
 724         (void)cpu;
 725         mutex_unlock(&access_lock);
 726 }
 727 
 728 static inline void trace_access_lock_init(void)
 729 {
 730 }
 731 
 732 #endif
 733 
 734 #ifdef CONFIG_STACKTRACE
 735 static void __ftrace_trace_stack(struct ring_buffer *buffer,
 736                                  unsigned long flags,
 737                                  int skip, int pc, struct pt_regs *regs);
 738 static inline void ftrace_trace_stack(struct trace_array *tr,
 739                                       struct ring_buffer *buffer,
 740                                       unsigned long flags,
 741                                       int skip, int pc, struct pt_regs *regs);
 742 
 743 #else
 744 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
 745                                         unsigned long flags,
 746                                         int skip, int pc, struct pt_regs *regs)
 747 {
 748 }
 749 static inline void ftrace_trace_stack(struct trace_array *tr,
 750                                       struct ring_buffer *buffer,
 751                                       unsigned long flags,
 752                                       int skip, int pc, struct pt_regs *regs)
 753 {
 754 }
 755 
 756 #endif
 757 
 758 static __always_inline void
 759 trace_event_setup(struct ring_buffer_event *event,
 760                   int type, unsigned long flags, int pc)
 761 {
 762         struct trace_entry *ent = ring_buffer_event_data(event);
 763 
 764         tracing_generic_entry_update(ent, type, flags, pc);
 765 }
 766 
 767 static __always_inline struct ring_buffer_event *
 768 __trace_buffer_lock_reserve(struct ring_buffer *buffer,
 769                           int type,
 770                           unsigned long len,
 771                           unsigned long flags, int pc)
 772 {
 773         struct ring_buffer_event *event;
 774 
 775         event = ring_buffer_lock_reserve(buffer, len);
 776         if (event != NULL)
 777                 trace_event_setup(event, type, flags, pc);
 778 
 779         return event;
 780 }
 781 
 782 void tracer_tracing_on(struct trace_array *tr)
 783 {
 784         if (tr->trace_buffer.buffer)
 785                 ring_buffer_record_on(tr->trace_buffer.buffer);
 786         /*
 787          * This flag is looked at when buffers haven't been allocated
 788          * yet, or by some tracers (like irqsoff), that just want to
 789          * know if the ring buffer has been disabled, but it can handle
 790          * races of where it gets disabled but we still do a record.
 791          * As the check is in the fast path of the tracers, it is more
 792          * important to be fast than accurate.
 793          */
 794         tr->buffer_disabled = 0;
 795         /* Make the flag seen by readers */
 796         smp_wmb();
 797 }
 798 
 799 /**
 800  * tracing_on - enable tracing buffers
 801  *
 802  * This function enables tracing buffers that may have been
 803  * disabled with tracing_off.
 804  */
 805 void tracing_on(void)
 806 {
 807         tracer_tracing_on(&global_trace);
 808 }
 809 EXPORT_SYMBOL_GPL(tracing_on);
 810 
 811 
 812 static __always_inline void
 813 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
 814 {
 815         __this_cpu_write(trace_taskinfo_save, true);
 816 
 817         /* If this is the temp buffer, we need to commit fully */
 818         if (this_cpu_read(trace_buffered_event) == event) {
 819                 /* Length is in event->array[0] */
 820                 ring_buffer_write(buffer, event->array[0], &event->array[1]);
 821                 /* Release the temp buffer */
 822                 this_cpu_dec(trace_buffered_event_cnt);
 823         } else
 824                 ring_buffer_unlock_commit(buffer, event);
 825 }
 826 
 827 /**
 828  * __trace_puts - write a constant string into the trace buffer.
 829  * @ip:    The address of the caller
 830  * @str:   The constant string to write
 831  * @size:  The size of the string.
 832  */
 833 int __trace_puts(unsigned long ip, const char *str, int size)
 834 {
 835         struct ring_buffer_event *event;
 836         struct ring_buffer *buffer;
 837         struct print_entry *entry;
 838         unsigned long irq_flags;
 839         int alloc;
 840         int pc;
 841 
 842         if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
 843                 return 0;
 844 
 845         pc = preempt_count();
 846 
 847         if (unlikely(tracing_selftest_running || tracing_disabled))
 848                 return 0;
 849 
 850         alloc = sizeof(*entry) + size + 2; /* possible \n added */
 851 
 852         local_save_flags(irq_flags);
 853         buffer = global_trace.trace_buffer.buffer;
 854         event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
 855                                             irq_flags, pc);
 856         if (!event)
 857                 return 0;
 858 
 859         entry = ring_buffer_event_data(event);
 860         entry->ip = ip;
 861 
 862         memcpy(&entry->buf, str, size);
 863 
 864         /* Add a newline if necessary */
 865         if (entry->buf[size - 1] != '\n') {
 866                 entry->buf[size] = '\n';
 867                 entry->buf[size + 1] = '\0';
 868         } else
 869                 entry->buf[size] = '\0';
 870 
 871         __buffer_unlock_commit(buffer, event);
 872         ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
 873 
 874         return size;
 875 }
 876 EXPORT_SYMBOL_GPL(__trace_puts);
 877 
 878 /**
 879  * __trace_bputs - write the pointer to a constant string into trace buffer
 880  * @ip:    The address of the caller
 881  * @str:   The constant string to write to the buffer to
 882  */
 883 int __trace_bputs(unsigned long ip, const char *str)
 884 {
 885         struct ring_buffer_event *event;
 886         struct ring_buffer *buffer;
 887         struct bputs_entry *entry;
 888         unsigned long irq_flags;
 889         int size = sizeof(struct bputs_entry);
 890         int pc;
 891 
 892         if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
 893                 return 0;
 894 
 895         pc = preempt_count();
 896 
 897         if (unlikely(tracing_selftest_running || tracing_disabled))
 898                 return 0;
 899 
 900         local_save_flags(irq_flags);
 901         buffer = global_trace.trace_buffer.buffer;
 902         event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
 903                                             irq_flags, pc);
 904         if (!event)
 905                 return 0;
 906 
 907         entry = ring_buffer_event_data(event);
 908         entry->ip                       = ip;
 909         entry->str                      = str;
 910 
 911         __buffer_unlock_commit(buffer, event);
 912         ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
 913 
 914         return 1;
 915 }
 916 EXPORT_SYMBOL_GPL(__trace_bputs);
 917 
 918 #ifdef CONFIG_TRACER_SNAPSHOT
 919 void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data)
 920 {
 921         struct tracer *tracer = tr->current_trace;
 922         unsigned long flags;
 923 
 924         if (in_nmi()) {
 925                 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
 926                 internal_trace_puts("*** snapshot is being ignored        ***\n");
 927                 return;
 928         }
 929 
 930         if (!tr->allocated_snapshot) {
 931                 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
 932                 internal_trace_puts("*** stopping trace here!   ***\n");
 933                 tracing_off();
 934                 return;
 935         }
 936 
 937         /* Note, snapshot can not be used when the tracer uses it */
 938         if (tracer->use_max_tr) {
 939                 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
 940                 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
 941                 return;
 942         }
 943 
 944         local_irq_save(flags);
 945         update_max_tr(tr, current, smp_processor_id(), cond_data);
 946         local_irq_restore(flags);
 947 }
 948 
 949 void tracing_snapshot_instance(struct trace_array *tr)
 950 {
 951         tracing_snapshot_instance_cond(tr, NULL);
 952 }
 953 
 954 /**
 955  * tracing_snapshot - take a snapshot of the current buffer.
 956  *
 957  * This causes a swap between the snapshot buffer and the current live
 958  * tracing buffer. You can use this to take snapshots of the live
 959  * trace when some condition is triggered, but continue to trace.
 960  *
 961  * Note, make sure to allocate the snapshot with either
 962  * a tracing_snapshot_alloc(), or by doing it manually
 963  * with: echo 1 > /sys/kernel/debug/tracing/snapshot
 964  *
 965  * If the snapshot buffer is not allocated, it will stop tracing.
 966  * Basically making a permanent snapshot.
 967  */
 968 void tracing_snapshot(void)
 969 {
 970         struct trace_array *tr = &global_trace;
 971 
 972         tracing_snapshot_instance(tr);
 973 }
 974 EXPORT_SYMBOL_GPL(tracing_snapshot);
 975 
 976 /**
 977  * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
 978  * @tr:         The tracing instance to snapshot
 979  * @cond_data:  The data to be tested conditionally, and possibly saved
 980  *
 981  * This is the same as tracing_snapshot() except that the snapshot is
 982  * conditional - the snapshot will only happen if the
 983  * cond_snapshot.update() implementation receiving the cond_data
 984  * returns true, which means that the trace array's cond_snapshot
 985  * update() operation used the cond_data to determine whether the
 986  * snapshot should be taken, and if it was, presumably saved it along
 987  * with the snapshot.
 988  */
 989 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
 990 {
 991         tracing_snapshot_instance_cond(tr, cond_data);
 992 }
 993 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
 994 
 995 /**
 996  * tracing_snapshot_cond_data - get the user data associated with a snapshot
 997  * @tr:         The tracing instance
 998  *
 999  * When the user enables a conditional snapshot using
1000  * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1001  * with the snapshot.  This accessor is used to retrieve it.
1002  *
1003  * Should not be called from cond_snapshot.update(), since it takes
1004  * the tr->max_lock lock, which the code calling
1005  * cond_snapshot.update() has already done.
1006  *
1007  * Returns the cond_data associated with the trace array's snapshot.
1008  */
1009 void *tracing_cond_snapshot_data(struct trace_array *tr)
1010 {
1011         void *cond_data = NULL;
1012 
1013         arch_spin_lock(&tr->max_lock);
1014 
1015         if (tr->cond_snapshot)
1016                 cond_data = tr->cond_snapshot->cond_data;
1017 
1018         arch_spin_unlock(&tr->max_lock);
1019 
1020         return cond_data;
1021 }
1022 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1023 
1024 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
1025                                         struct trace_buffer *size_buf, int cpu_id);
1026 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
1027 
1028 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1029 {
1030         int ret;
1031 
1032         if (!tr->allocated_snapshot) {
1033 
1034                 /* allocate spare buffer */
1035                 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1036                                    &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
1037                 if (ret < 0)
1038                         return ret;
1039 
1040                 tr->allocated_snapshot = true;
1041         }
1042 
1043         return 0;
1044 }
1045 
1046 static void free_snapshot(struct trace_array *tr)
1047 {
1048         /*
1049          * We don't free the ring buffer. instead, resize it because
1050          * The max_tr ring buffer has some state (e.g. ring->clock) and
1051          * we want preserve it.
1052          */
1053         ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1054         set_buffer_entries(&tr->max_buffer, 1);
1055         tracing_reset_online_cpus(&tr->max_buffer);
1056         tr->allocated_snapshot = false;
1057 }
1058 
1059 /**
1060  * tracing_alloc_snapshot - allocate snapshot buffer.
1061  *
1062  * This only allocates the snapshot buffer if it isn't already
1063  * allocated - it doesn't also take a snapshot.
1064  *
1065  * This is meant to be used in cases where the snapshot buffer needs
1066  * to be set up for events that can't sleep but need to be able to
1067  * trigger a snapshot.
1068  */
1069 int tracing_alloc_snapshot(void)
1070 {
1071         struct trace_array *tr = &global_trace;
1072         int ret;
1073 
1074         ret = tracing_alloc_snapshot_instance(tr);
1075         WARN_ON(ret < 0);
1076 
1077         return ret;
1078 }
1079 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1080 
1081 /**
1082  * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1083  *
1084  * This is similar to tracing_snapshot(), but it will allocate the
1085  * snapshot buffer if it isn't already allocated. Use this only
1086  * where it is safe to sleep, as the allocation may sleep.
1087  *
1088  * This causes a swap between the snapshot buffer and the current live
1089  * tracing buffer. You can use this to take snapshots of the live
1090  * trace when some condition is triggered, but continue to trace.
1091  */
1092 void tracing_snapshot_alloc(void)
1093 {
1094         int ret;
1095 
1096         ret = tracing_alloc_snapshot();
1097         if (ret < 0)
1098                 return;
1099 
1100         tracing_snapshot();
1101 }
1102 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1103 
1104 /**
1105  * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1106  * @tr:         The tracing instance
1107  * @cond_data:  User data to associate with the snapshot
1108  * @update:     Implementation of the cond_snapshot update function
1109  *
1110  * Check whether the conditional snapshot for the given instance has
1111  * already been enabled, or if the current tracer is already using a
1112  * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1113  * save the cond_data and update function inside.
1114  *
1115  * Returns 0 if successful, error otherwise.
1116  */
1117 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1118                                  cond_update_fn_t update)
1119 {
1120         struct cond_snapshot *cond_snapshot;
1121         int ret = 0;
1122 
1123         cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1124         if (!cond_snapshot)
1125                 return -ENOMEM;
1126 
1127         cond_snapshot->cond_data = cond_data;
1128         cond_snapshot->update = update;
1129 
1130         mutex_lock(&trace_types_lock);
1131 
1132         ret = tracing_alloc_snapshot_instance(tr);
1133         if (ret)
1134                 goto fail_unlock;
1135 
1136         if (tr->current_trace->use_max_tr) {
1137                 ret = -EBUSY;
1138                 goto fail_unlock;
1139         }
1140 
1141         /*
1142          * The cond_snapshot can only change to NULL without the
1143          * trace_types_lock. We don't care if we race with it going
1144          * to NULL, but we want to make sure that it's not set to
1145          * something other than NULL when we get here, which we can
1146          * do safely with only holding the trace_types_lock and not
1147          * having to take the max_lock.
1148          */
1149         if (tr->cond_snapshot) {
1150                 ret = -EBUSY;
1151                 goto fail_unlock;
1152         }
1153 
1154         arch_spin_lock(&tr->max_lock);
1155         tr->cond_snapshot = cond_snapshot;
1156         arch_spin_unlock(&tr->max_lock);
1157 
1158         mutex_unlock(&trace_types_lock);
1159 
1160         return ret;
1161 
1162  fail_unlock:
1163         mutex_unlock(&trace_types_lock);
1164         kfree(cond_snapshot);
1165         return ret;
1166 }
1167 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1168 
1169 /**
1170  * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1171  * @tr:         The tracing instance
1172  *
1173  * Check whether the conditional snapshot for the given instance is
1174  * enabled; if so, free the cond_snapshot associated with it,
1175  * otherwise return -EINVAL.
1176  *
1177  * Returns 0 if successful, error otherwise.
1178  */
1179 int tracing_snapshot_cond_disable(struct trace_array *tr)
1180 {
1181         int ret = 0;
1182 
1183         arch_spin_lock(&tr->max_lock);
1184 
1185         if (!tr->cond_snapshot)
1186                 ret = -EINVAL;
1187         else {
1188                 kfree(tr->cond_snapshot);
1189                 tr->cond_snapshot = NULL;
1190         }
1191 
1192         arch_spin_unlock(&tr->max_lock);
1193 
1194         return ret;
1195 }
1196 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1197 #else
1198 void tracing_snapshot(void)
1199 {
1200         WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1201 }
1202 EXPORT_SYMBOL_GPL(tracing_snapshot);
1203 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1204 {
1205         WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1206 }
1207 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1208 int tracing_alloc_snapshot(void)
1209 {
1210         WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1211         return -ENODEV;
1212 }
1213 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1214 void tracing_snapshot_alloc(void)
1215 {
1216         /* Give warning */
1217         tracing_snapshot();
1218 }
1219 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1220 void *tracing_cond_snapshot_data(struct trace_array *tr)
1221 {
1222         return NULL;
1223 }
1224 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1225 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1226 {
1227         return -ENODEV;
1228 }
1229 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1230 int tracing_snapshot_cond_disable(struct trace_array *tr)
1231 {
1232         return false;
1233 }
1234 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1235 #endif /* CONFIG_TRACER_SNAPSHOT */
1236 
1237 void tracer_tracing_off(struct trace_array *tr)
1238 {
1239         if (tr->trace_buffer.buffer)
1240                 ring_buffer_record_off(tr->trace_buffer.buffer);
1241         /*
1242          * This flag is looked at when buffers haven't been allocated
1243          * yet, or by some tracers (like irqsoff), that just want to
1244          * know if the ring buffer has been disabled, but it can handle
1245          * races of where it gets disabled but we still do a record.
1246          * As the check is in the fast path of the tracers, it is more
1247          * important to be fast than accurate.
1248          */
1249         tr->buffer_disabled = 1;
1250         /* Make the flag seen by readers */
1251         smp_wmb();
1252 }
1253 
1254 /**
1255  * tracing_off - turn off tracing buffers
1256  *
1257  * This function stops the tracing buffers from recording data.
1258  * It does not disable any overhead the tracers themselves may
1259  * be causing. This function simply causes all recording to
1260  * the ring buffers to fail.
1261  */
1262 void tracing_off(void)
1263 {
1264         tracer_tracing_off(&global_trace);
1265 }
1266 EXPORT_SYMBOL_GPL(tracing_off);
1267 
1268 void disable_trace_on_warning(void)
1269 {
1270         if (__disable_trace_on_warning)
1271                 tracing_off();
1272 }
1273 
1274 /**
1275  * tracer_tracing_is_on - show real state of ring buffer enabled
1276  * @tr : the trace array to know if ring buffer is enabled
1277  *
1278  * Shows real state of the ring buffer if it is enabled or not.
1279  */
1280 bool tracer_tracing_is_on(struct trace_array *tr)
1281 {
1282         if (tr->trace_buffer.buffer)
1283                 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1284         return !tr->buffer_disabled;
1285 }
1286 
1287 /**
1288  * tracing_is_on - show state of ring buffers enabled
1289  */
1290 int tracing_is_on(void)
1291 {
1292         return tracer_tracing_is_on(&global_trace);
1293 }
1294 EXPORT_SYMBOL_GPL(tracing_is_on);
1295 
1296 static int __init set_buf_size(char *str)
1297 {
1298         unsigned long buf_size;
1299 
1300         if (!str)
1301                 return 0;
1302         buf_size = memparse(str, &str);
1303         /* nr_entries can not be zero */
1304         if (buf_size == 0)
1305                 return 0;
1306         trace_buf_size = buf_size;
1307         return 1;
1308 }
1309 __setup("trace_buf_size=", set_buf_size);
1310 
1311 static int __init set_tracing_thresh(char *str)
1312 {
1313         unsigned long threshold;
1314         int ret;
1315 
1316         if (!str)
1317                 return 0;
1318         ret = kstrtoul(str, 0, &threshold);
1319         if (ret < 0)
1320                 return 0;
1321         tracing_thresh = threshold * 1000;
1322         return 1;
1323 }
1324 __setup("tracing_thresh=", set_tracing_thresh);
1325 
1326 unsigned long nsecs_to_usecs(unsigned long nsecs)
1327 {
1328         return nsecs / 1000;
1329 }
1330 
1331 /*
1332  * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1333  * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1334  * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1335  * of strings in the order that the evals (enum) were defined.
1336  */
1337 #undef C
1338 #define C(a, b) b
1339 
1340 /* These must match the bit postions in trace_iterator_flags */
1341 static const char *trace_options[] = {
1342         TRACE_FLAGS
1343         NULL
1344 };
1345 
1346 static struct {
1347         u64 (*func)(void);
1348         const char *name;
1349         int in_ns;              /* is this clock in nanoseconds? */
1350 } trace_clocks[] = {
1351         { trace_clock_local,            "local",        1 },
1352         { trace_clock_global,           "global",       1 },
1353         { trace_clock_counter,          "counter",      0 },
1354         { trace_clock_jiffies,          "uptime",       0 },
1355         { trace_clock,                  "perf",         1 },
1356         { ktime_get_mono_fast_ns,       "mono",         1 },
1357         { ktime_get_raw_fast_ns,        "mono_raw",     1 },
1358         { ktime_get_boot_fast_ns,       "boot",         1 },
1359         ARCH_TRACE_CLOCKS
1360 };
1361 
1362 bool trace_clock_in_ns(struct trace_array *tr)
1363 {
1364         if (trace_clocks[tr->clock_id].in_ns)
1365                 return true;
1366 
1367         return false;
1368 }
1369 
1370 /*
1371  * trace_parser_get_init - gets the buffer for trace parser
1372  */
1373 int trace_parser_get_init(struct trace_parser *parser, int size)
1374 {
1375         memset(parser, 0, sizeof(*parser));
1376 
1377         parser->buffer = kmalloc(size, GFP_KERNEL);
1378         if (!parser->buffer)
1379                 return 1;
1380 
1381         parser->size = size;
1382         return 0;
1383 }
1384 
1385 /*
1386  * trace_parser_put - frees the buffer for trace parser
1387  */
1388 void trace_parser_put(struct trace_parser *parser)
1389 {
1390         kfree(parser->buffer);
1391         parser->buffer = NULL;
1392 }
1393 
1394 /*
1395  * trace_get_user - reads the user input string separated by  space
1396  * (matched by isspace(ch))
1397  *
1398  * For each string found the 'struct trace_parser' is updated,
1399  * and the function returns.
1400  *
1401  * Returns number of bytes read.
1402  *
1403  * See kernel/trace/trace.h for 'struct trace_parser' details.
1404  */
1405 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1406         size_t cnt, loff_t *ppos)
1407 {
1408         char ch;
1409         size_t read = 0;
1410         ssize_t ret;
1411 
1412         if (!*ppos)
1413                 trace_parser_clear(parser);
1414 
1415         ret = get_user(ch, ubuf++);
1416         if (ret)
1417                 goto out;
1418 
1419         read++;
1420         cnt--;
1421 
1422         /*
1423          * The parser is not finished with the last write,
1424          * continue reading the user input without skipping spaces.
1425          */
1426         if (!parser->cont) {
1427                 /* skip white space */
1428                 while (cnt && isspace(ch)) {
1429                         ret = get_user(ch, ubuf++);
1430                         if (ret)
1431                                 goto out;
1432                         read++;
1433                         cnt--;
1434                 }
1435 
1436                 parser->idx = 0;
1437 
1438                 /* only spaces were written */
1439                 if (isspace(ch) || !ch) {
1440                         *ppos += read;
1441                         ret = read;
1442                         goto out;
1443                 }
1444         }
1445 
1446         /* read the non-space input */
1447         while (cnt && !isspace(ch) && ch) {
1448                 if (parser->idx < parser->size - 1)
1449                         parser->buffer[parser->idx++] = ch;
1450                 else {
1451                         ret = -EINVAL;
1452                         goto out;
1453                 }
1454                 ret = get_user(ch, ubuf++);
1455                 if (ret)
1456                         goto out;
1457                 read++;
1458                 cnt--;
1459         }
1460 
1461         /* We either got finished input or we have to wait for another call. */
1462         if (isspace(ch) || !ch) {
1463                 parser->buffer[parser->idx] = 0;
1464                 parser->cont = false;
1465         } else if (parser->idx < parser->size - 1) {
1466                 parser->cont = true;
1467                 parser->buffer[parser->idx++] = ch;
1468                 /* Make sure the parsed string always terminates with '\0'. */
1469                 parser->buffer[parser->idx] = 0;
1470         } else {
1471                 ret = -EINVAL;
1472                 goto out;
1473         }
1474 
1475         *ppos += read;
1476         ret = read;
1477 
1478 out:
1479         return ret;
1480 }
1481 
1482 /* TODO add a seq_buf_to_buffer() */
1483 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1484 {
1485         int len;
1486 
1487         if (trace_seq_used(s) <= s->seq.readpos)
1488                 return -EBUSY;
1489 
1490         len = trace_seq_used(s) - s->seq.readpos;
1491         if (cnt > len)
1492                 cnt = len;
1493         memcpy(buf, s->buffer + s->seq.readpos, cnt);
1494 
1495         s->seq.readpos += cnt;
1496         return cnt;
1497 }
1498 
1499 unsigned long __read_mostly     tracing_thresh;
1500 
1501 #ifdef CONFIG_TRACER_MAX_TRACE
1502 /*
1503  * Copy the new maximum trace into the separate maximum-trace
1504  * structure. (this way the maximum trace is permanently saved,
1505  * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1506  */
1507 static void
1508 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1509 {
1510         struct trace_buffer *trace_buf = &tr->trace_buffer;
1511         struct trace_buffer *max_buf = &tr->max_buffer;
1512         struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1513         struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1514 
1515         max_buf->cpu = cpu;
1516         max_buf->time_start = data->preempt_timestamp;
1517 
1518         max_data->saved_latency = tr->max_latency;
1519         max_data->critical_start = data->critical_start;
1520         max_data->critical_end = data->critical_end;
1521 
1522         strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1523         max_data->pid = tsk->pid;
1524         /*
1525          * If tsk == current, then use current_uid(), as that does not use
1526          * RCU. The irq tracer can be called out of RCU scope.
1527          */
1528         if (tsk == current)
1529                 max_data->uid = current_uid();
1530         else
1531                 max_data->uid = task_uid(tsk);
1532 
1533         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1534         max_data->policy = tsk->policy;
1535         max_data->rt_priority = tsk->rt_priority;
1536 
1537         /* record this tasks comm */
1538         tracing_record_cmdline(tsk);
1539 }
1540 
1541 /**
1542  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1543  * @tr: tracer
1544  * @tsk: the task with the latency
1545  * @cpu: The cpu that initiated the trace.
1546  * @cond_data: User data associated with a conditional snapshot
1547  *
1548  * Flip the buffers between the @tr and the max_tr and record information
1549  * about which task was the cause of this latency.
1550  */
1551 void
1552 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1553               void *cond_data)
1554 {
1555         if (tr->stop_count)
1556                 return;
1557 
1558         WARN_ON_ONCE(!irqs_disabled());
1559 
1560         if (!tr->allocated_snapshot) {
1561                 /* Only the nop tracer should hit this when disabling */
1562                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1563                 return;
1564         }
1565 
1566         arch_spin_lock(&tr->max_lock);
1567 
1568         /* Inherit the recordable setting from trace_buffer */
1569         if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1570                 ring_buffer_record_on(tr->max_buffer.buffer);
1571         else
1572                 ring_buffer_record_off(tr->max_buffer.buffer);
1573 
1574 #ifdef CONFIG_TRACER_SNAPSHOT
1575         if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1576                 goto out_unlock;
1577 #endif
1578         swap(tr->trace_buffer.buffer, tr->max_buffer.buffer);
1579 
1580         __update_max_tr(tr, tsk, cpu);
1581 
1582  out_unlock:
1583         arch_spin_unlock(&tr->max_lock);
1584 }
1585 
1586 /**
1587  * update_max_tr_single - only copy one trace over, and reset the rest
1588  * @tr: tracer
1589  * @tsk: task with the latency
1590  * @cpu: the cpu of the buffer to copy.
1591  *
1592  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1593  */
1594 void
1595 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1596 {
1597         int ret;
1598 
1599         if (tr->stop_count)
1600                 return;
1601 
1602         WARN_ON_ONCE(!irqs_disabled());
1603         if (!tr->allocated_snapshot) {
1604                 /* Only the nop tracer should hit this when disabling */
1605                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1606                 return;
1607         }
1608 
1609         arch_spin_lock(&tr->max_lock);
1610 
1611         ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1612 
1613         if (ret == -EBUSY) {
1614                 /*
1615                  * We failed to swap the buffer due to a commit taking
1616                  * place on this CPU. We fail to record, but we reset
1617                  * the max trace buffer (no one writes directly to it)
1618                  * and flag that it failed.
1619                  */
1620                 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1621                         "Failed to swap buffers due to commit in progress\n");
1622         }
1623 
1624         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1625 
1626         __update_max_tr(tr, tsk, cpu);
1627         arch_spin_unlock(&tr->max_lock);
1628 }
1629 #endif /* CONFIG_TRACER_MAX_TRACE */
1630 
1631 static int wait_on_pipe(struct trace_iterator *iter, int full)
1632 {
1633         /* Iterators are static, they should be filled or empty */
1634         if (trace_buffer_iter(iter, iter->cpu_file))
1635                 return 0;
1636 
1637         return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1638                                 full);
1639 }
1640 
1641 #ifdef CONFIG_FTRACE_STARTUP_TEST
1642 static bool selftests_can_run;
1643 
1644 struct trace_selftests {
1645         struct list_head                list;
1646         struct tracer                   *type;
1647 };
1648 
1649 static LIST_HEAD(postponed_selftests);
1650 
1651 static int save_selftest(struct tracer *type)
1652 {
1653         struct trace_selftests *selftest;
1654 
1655         selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1656         if (!selftest)
1657                 return -ENOMEM;
1658 
1659         selftest->type = type;
1660         list_add(&selftest->list, &postponed_selftests);
1661         return 0;
1662 }
1663 
1664 static int run_tracer_selftest(struct tracer *type)
1665 {
1666         struct trace_array *tr = &global_trace;
1667         struct tracer *saved_tracer = tr->current_trace;
1668         int ret;
1669 
1670         if (!type->selftest || tracing_selftest_disabled)
1671                 return 0;
1672 
1673         /*
1674          * If a tracer registers early in boot up (before scheduling is
1675          * initialized and such), then do not run its selftests yet.
1676          * Instead, run it a little later in the boot process.
1677          */
1678         if (!selftests_can_run)
1679                 return save_selftest(type);
1680 
1681         /*
1682          * Run a selftest on this tracer.
1683          * Here we reset the trace buffer, and set the current
1684          * tracer to be this tracer. The tracer can then run some
1685          * internal tracing to verify that everything is in order.
1686          * If we fail, we do not register this tracer.
1687          */
1688         tracing_reset_online_cpus(&tr->trace_buffer);
1689 
1690         tr->current_trace = type;
1691 
1692 #ifdef CONFIG_TRACER_MAX_TRACE
1693         if (type->use_max_tr) {
1694                 /* If we expanded the buffers, make sure the max is expanded too */
1695                 if (ring_buffer_expanded)
1696                         ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1697                                            RING_BUFFER_ALL_CPUS);
1698                 tr->allocated_snapshot = true;
1699         }
1700 #endif
1701 
1702         /* the test is responsible for initializing and enabling */
1703         pr_info("Testing tracer %s: ", type->name);
1704         ret = type->selftest(type, tr);
1705         /* the test is responsible for resetting too */
1706         tr->current_trace = saved_tracer;
1707         if (ret) {
1708                 printk(KERN_CONT "FAILED!\n");
1709                 /* Add the warning after printing 'FAILED' */
1710                 WARN_ON(1);
1711                 return -1;
1712         }
1713         /* Only reset on passing, to avoid touching corrupted buffers */
1714         tracing_reset_online_cpus(&tr->trace_buffer);
1715 
1716 #ifdef CONFIG_TRACER_MAX_TRACE
1717         if (type->use_max_tr) {
1718                 tr->allocated_snapshot = false;
1719 
1720                 /* Shrink the max buffer again */
1721                 if (ring_buffer_expanded)
1722                         ring_buffer_resize(tr->max_buffer.buffer, 1,
1723                                            RING_BUFFER_ALL_CPUS);
1724         }
1725 #endif
1726 
1727         printk(KERN_CONT "PASSED\n");
1728         return 0;
1729 }
1730 
1731 static __init int init_trace_selftests(void)
1732 {
1733         struct trace_selftests *p, *n;
1734         struct tracer *t, **last;
1735         int ret;
1736 
1737         selftests_can_run = true;
1738 
1739         mutex_lock(&trace_types_lock);
1740 
1741         if (list_empty(&postponed_selftests))
1742                 goto out;
1743 
1744         pr_info("Running postponed tracer tests:\n");
1745 
1746         tracing_selftest_running = true;
1747         list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1748                 /* This loop can take minutes when sanitizers are enabled, so
1749                  * lets make sure we allow RCU processing.
1750                  */
1751                 cond_resched();
1752                 ret = run_tracer_selftest(p->type);
1753                 /* If the test fails, then warn and remove from available_tracers */
1754                 if (ret < 0) {
1755                         WARN(1, "tracer: %s failed selftest, disabling\n",
1756                              p->type->name);
1757                         last = &trace_types;
1758                         for (t = trace_types; t; t = t->next) {
1759                                 if (t == p->type) {
1760                                         *last = t->next;
1761                                         break;
1762                                 }
1763                                 last = &t->next;
1764                         }
1765                 }
1766                 list_del(&p->list);
1767                 kfree(p);
1768         }
1769         tracing_selftest_running = false;
1770 
1771  out:
1772         mutex_unlock(&trace_types_lock);
1773 
1774         return 0;
1775 }
1776 core_initcall(init_trace_selftests);
1777 #else
1778 static inline int run_tracer_selftest(struct tracer *type)
1779 {
1780         return 0;
1781 }
1782 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1783 
1784 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1785 
1786 static void __init apply_trace_boot_options(void);
1787 
1788 /**
1789  * register_tracer - register a tracer with the ftrace system.
1790  * @type: the plugin for the tracer
1791  *
1792  * Register a new plugin tracer.
1793  */
1794 int __init register_tracer(struct tracer *type)
1795 {
1796         struct tracer *t;
1797         int ret = 0;
1798 
1799         if (!type->name) {
1800                 pr_info("Tracer must have a name\n");
1801                 return -1;
1802         }
1803 
1804         if (strlen(type->name) >= MAX_TRACER_SIZE) {
1805                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1806                 return -1;
1807         }
1808 
1809         if (security_locked_down(LOCKDOWN_TRACEFS)) {
1810                 pr_warning("Can not register tracer %s due to lockdown\n",
1811                            type->name);
1812                 return -EPERM;
1813         }
1814 
1815         mutex_lock(&trace_types_lock);
1816 
1817         tracing_selftest_running = true;
1818 
1819         for (t = trace_types; t; t = t->next) {
1820                 if (strcmp(type->name, t->name) == 0) {
1821                         /* already found */
1822                         pr_info("Tracer %s already registered\n",
1823                                 type->name);
1824                         ret = -1;
1825                         goto out;
1826                 }
1827         }
1828 
1829         if (!type->set_flag)
1830                 type->set_flag = &dummy_set_flag;
1831         if (!type->flags) {
1832                 /*allocate a dummy tracer_flags*/
1833                 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1834                 if (!type->flags) {
1835                         ret = -ENOMEM;
1836                         goto out;
1837                 }
1838                 type->flags->val = 0;
1839                 type->flags->opts = dummy_tracer_opt;
1840         } else
1841                 if (!type->flags->opts)
1842                         type->flags->opts = dummy_tracer_opt;
1843 
1844         /* store the tracer for __set_tracer_option */
1845         type->flags->trace = type;
1846 
1847         ret = run_tracer_selftest(type);
1848         if (ret < 0)
1849                 goto out;
1850 
1851         type->next = trace_types;
1852         trace_types = type;
1853         add_tracer_options(&global_trace, type);
1854 
1855  out:
1856         tracing_selftest_running = false;
1857         mutex_unlock(&trace_types_lock);
1858 
1859         if (ret || !default_bootup_tracer)
1860                 goto out_unlock;
1861 
1862         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1863                 goto out_unlock;
1864 
1865         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1866         /* Do we want this tracer to start on bootup? */
1867         tracing_set_tracer(&global_trace, type->name);
1868         default_bootup_tracer = NULL;
1869 
1870         apply_trace_boot_options();
1871 
1872         /* disable other selftests, since this will break it. */
1873         tracing_selftest_disabled = true;
1874 #ifdef CONFIG_FTRACE_STARTUP_TEST
1875         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1876                type->name);
1877 #endif
1878 
1879  out_unlock:
1880         return ret;
1881 }
1882 
1883 static void tracing_reset_cpu(struct trace_buffer *buf, int cpu)
1884 {
1885         struct ring_buffer *buffer = buf->buffer;
1886 
1887         if (!buffer)
1888                 return;
1889 
1890         ring_buffer_record_disable(buffer);
1891 
1892         /* Make sure all commits have finished */
1893         synchronize_rcu();
1894         ring_buffer_reset_cpu(buffer, cpu);
1895 
1896         ring_buffer_record_enable(buffer);
1897 }
1898 
1899 void tracing_reset_online_cpus(struct trace_buffer *buf)
1900 {
1901         struct ring_buffer *buffer = buf->buffer;
1902         int cpu;
1903 
1904         if (!buffer)
1905                 return;
1906 
1907         ring_buffer_record_disable(buffer);
1908 
1909         /* Make sure all commits have finished */
1910         synchronize_rcu();
1911 
1912         buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1913 
1914         for_each_online_cpu(cpu)
1915                 ring_buffer_reset_cpu(buffer, cpu);
1916 
1917         ring_buffer_record_enable(buffer);
1918 }
1919 
1920 /* Must have trace_types_lock held */
1921 void tracing_reset_all_online_cpus(void)
1922 {
1923         struct trace_array *tr;
1924 
1925         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1926                 if (!tr->clear_trace)
1927                         continue;
1928                 tr->clear_trace = false;
1929                 tracing_reset_online_cpus(&tr->trace_buffer);
1930 #ifdef CONFIG_TRACER_MAX_TRACE
1931                 tracing_reset_online_cpus(&tr->max_buffer);
1932 #endif
1933         }
1934 }
1935 
1936 static int *tgid_map;
1937 
1938 #define SAVED_CMDLINES_DEFAULT 128
1939 #define NO_CMDLINE_MAP UINT_MAX
1940 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1941 struct saved_cmdlines_buffer {
1942         unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1943         unsigned *map_cmdline_to_pid;
1944         unsigned cmdline_num;
1945         int cmdline_idx;
1946         char *saved_cmdlines;
1947 };
1948 static struct saved_cmdlines_buffer *savedcmd;
1949 
1950 /* temporary disable recording */
1951 static atomic_t trace_record_taskinfo_disabled __read_mostly;
1952 
1953 static inline char *get_saved_cmdlines(int idx)
1954 {
1955         return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1956 }
1957 
1958 static inline void set_cmdline(int idx, const char *cmdline)
1959 {
1960         strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1961 }
1962 
1963 static int allocate_cmdlines_buffer(unsigned int val,
1964                                     struct saved_cmdlines_buffer *s)
1965 {
1966         s->map_cmdline_to_pid = kmalloc_array(val,
1967                                               sizeof(*s->map_cmdline_to_pid),
1968                                               GFP_KERNEL);
1969         if (!s->map_cmdline_to_pid)
1970                 return -ENOMEM;
1971 
1972         s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
1973         if (!s->saved_cmdlines) {
1974                 kfree(s->map_cmdline_to_pid);
1975                 return -ENOMEM;
1976         }
1977 
1978         s->cmdline_idx = 0;
1979         s->cmdline_num = val;
1980         memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1981                sizeof(s->map_pid_to_cmdline));
1982         memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1983                val * sizeof(*s->map_cmdline_to_pid));
1984 
1985         return 0;
1986 }
1987 
1988 static int trace_create_savedcmd(void)
1989 {
1990         int ret;
1991 
1992         savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1993         if (!savedcmd)
1994                 return -ENOMEM;
1995 
1996         ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1997         if (ret < 0) {
1998                 kfree(savedcmd);
1999                 savedcmd = NULL;
2000                 return -ENOMEM;
2001         }
2002 
2003         return 0;
2004 }
2005 
2006 int is_tracing_stopped(void)
2007 {
2008         return global_trace.stop_count;
2009 }
2010 
2011 /**
2012  * tracing_start - quick start of the tracer
2013  *
2014  * If tracing is enabled but was stopped by tracing_stop,
2015  * this will start the tracer back up.
2016  */
2017 void tracing_start(void)
2018 {
2019         struct ring_buffer *buffer;
2020         unsigned long flags;
2021 
2022         if (tracing_disabled)
2023                 return;
2024 
2025         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2026         if (--global_trace.stop_count) {
2027                 if (global_trace.stop_count < 0) {
2028                         /* Someone screwed up their debugging */
2029                         WARN_ON_ONCE(1);
2030                         global_trace.stop_count = 0;
2031                 }
2032                 goto out;
2033         }
2034 
2035         /* Prevent the buffers from switching */
2036         arch_spin_lock(&global_trace.max_lock);
2037 
2038         buffer = global_trace.trace_buffer.buffer;
2039         if (buffer)
2040                 ring_buffer_record_enable(buffer);
2041 
2042 #ifdef CONFIG_TRACER_MAX_TRACE
2043         buffer = global_trace.max_buffer.buffer;
2044         if (buffer)
2045                 ring_buffer_record_enable(buffer);
2046 #endif
2047 
2048         arch_spin_unlock(&global_trace.max_lock);
2049 
2050  out:
2051         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2052 }
2053 
2054 static void tracing_start_tr(struct trace_array *tr)
2055 {
2056         struct ring_buffer *buffer;
2057         unsigned long flags;
2058 
2059         if (tracing_disabled)
2060                 return;
2061 
2062         /* If global, we need to also start the max tracer */
2063         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2064                 return tracing_start();
2065 
2066         raw_spin_lock_irqsave(&tr->start_lock, flags);
2067 
2068         if (--tr->stop_count) {
2069                 if (tr->stop_count < 0) {
2070                         /* Someone screwed up their debugging */
2071                         WARN_ON_ONCE(1);
2072                         tr->stop_count = 0;
2073                 }
2074                 goto out;
2075         }
2076 
2077         buffer = tr->trace_buffer.buffer;
2078         if (buffer)
2079                 ring_buffer_record_enable(buffer);
2080 
2081  out:
2082         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2083 }
2084 
2085 /**
2086  * tracing_stop - quick stop of the tracer
2087  *
2088  * Light weight way to stop tracing. Use in conjunction with
2089  * tracing_start.
2090  */
2091 void tracing_stop(void)
2092 {
2093         struct ring_buffer *buffer;
2094         unsigned long flags;
2095 
2096         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2097         if (global_trace.stop_count++)
2098                 goto out;
2099 
2100         /* Prevent the buffers from switching */
2101         arch_spin_lock(&global_trace.max_lock);
2102 
2103         buffer = global_trace.trace_buffer.buffer;
2104         if (buffer)
2105                 ring_buffer_record_disable(buffer);
2106 
2107 #ifdef CONFIG_TRACER_MAX_TRACE
2108         buffer = global_trace.max_buffer.buffer;
2109         if (buffer)
2110                 ring_buffer_record_disable(buffer);
2111 #endif
2112 
2113         arch_spin_unlock(&global_trace.max_lock);
2114 
2115  out:
2116         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2117 }
2118 
2119 static void tracing_stop_tr(struct trace_array *tr)
2120 {
2121         struct ring_buffer *buffer;
2122         unsigned long flags;
2123 
2124         /* If global, we need to also stop the max tracer */
2125         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2126                 return tracing_stop();
2127 
2128         raw_spin_lock_irqsave(&tr->start_lock, flags);
2129         if (tr->stop_count++)
2130                 goto out;
2131 
2132         buffer = tr->trace_buffer.buffer;
2133         if (buffer)
2134                 ring_buffer_record_disable(buffer);
2135 
2136  out:
2137         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2138 }
2139 
2140 static int trace_save_cmdline(struct task_struct *tsk)
2141 {
2142         unsigned pid, idx;
2143 
2144         /* treat recording of idle task as a success */
2145         if (!tsk->pid)
2146                 return 1;
2147 
2148         if (unlikely(tsk->pid > PID_MAX_DEFAULT))
2149                 return 0;
2150 
2151         /*
2152          * It's not the end of the world if we don't get
2153          * the lock, but we also don't want to spin
2154          * nor do we want to disable interrupts,
2155          * so if we miss here, then better luck next time.
2156          */
2157         if (!arch_spin_trylock(&trace_cmdline_lock))
2158                 return 0;
2159 
2160         idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2161         if (idx == NO_CMDLINE_MAP) {
2162                 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2163 
2164                 /*
2165                  * Check whether the cmdline buffer at idx has a pid
2166                  * mapped. We are going to overwrite that entry so we
2167                  * need to clear the map_pid_to_cmdline. Otherwise we
2168                  * would read the new comm for the old pid.
2169                  */
2170                 pid = savedcmd->map_cmdline_to_pid[idx];
2171                 if (pid != NO_CMDLINE_MAP)
2172                         savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
2173 
2174                 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2175                 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
2176 
2177                 savedcmd->cmdline_idx = idx;
2178         }
2179 
2180         set_cmdline(idx, tsk->comm);
2181 
2182         arch_spin_unlock(&trace_cmdline_lock);
2183 
2184         return 1;
2185 }
2186 
2187 static void __trace_find_cmdline(int pid, char comm[])
2188 {
2189         unsigned map;
2190 
2191         if (!pid) {
2192                 strcpy(comm, "<idle>");
2193                 return;
2194         }
2195 
2196         if (WARN_ON_ONCE(pid < 0)) {
2197                 strcpy(comm, "<XXX>");
2198                 return;
2199         }
2200 
2201         if (pid > PID_MAX_DEFAULT) {
2202                 strcpy(comm, "<...>");
2203                 return;
2204         }
2205 
2206         map = savedcmd->map_pid_to_cmdline[pid];
2207         if (map != NO_CMDLINE_MAP)
2208                 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2209         else
2210                 strcpy(comm, "<...>");
2211 }
2212 
2213 void trace_find_cmdline(int pid, char comm[])
2214 {
2215         preempt_disable();
2216         arch_spin_lock(&trace_cmdline_lock);
2217 
2218         __trace_find_cmdline(pid, comm);
2219 
2220         arch_spin_unlock(&trace_cmdline_lock);
2221         preempt_enable();
2222 }
2223 
2224 int trace_find_tgid(int pid)
2225 {
2226         if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2227                 return 0;
2228 
2229         return tgid_map[pid];
2230 }
2231 
2232 static int trace_save_tgid(struct task_struct *tsk)
2233 {
2234         /* treat recording of idle task as a success */
2235         if (!tsk->pid)
2236                 return 1;
2237 
2238         if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2239                 return 0;
2240 
2241         tgid_map[tsk->pid] = tsk->tgid;
2242         return 1;
2243 }
2244 
2245 static bool tracing_record_taskinfo_skip(int flags)
2246 {
2247         if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2248                 return true;
2249         if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2250                 return true;
2251         if (!__this_cpu_read(trace_taskinfo_save))
2252                 return true;
2253         return false;
2254 }
2255 
2256 /**
2257  * tracing_record_taskinfo - record the task info of a task
2258  *
2259  * @task:  task to record
2260  * @flags: TRACE_RECORD_CMDLINE for recording comm
2261  *         TRACE_RECORD_TGID for recording tgid
2262  */
2263 void tracing_record_taskinfo(struct task_struct *task, int flags)
2264 {
2265         bool done;
2266 
2267         if (tracing_record_taskinfo_skip(flags))
2268                 return;
2269 
2270         /*
2271          * Record as much task information as possible. If some fail, continue
2272          * to try to record the others.
2273          */
2274         done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2275         done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2276 
2277         /* If recording any information failed, retry again soon. */
2278         if (!done)
2279                 return;
2280 
2281         __this_cpu_write(trace_taskinfo_save, false);
2282 }
2283 
2284 /**
2285  * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2286  *
2287  * @prev: previous task during sched_switch
2288  * @next: next task during sched_switch
2289  * @flags: TRACE_RECORD_CMDLINE for recording comm
2290  *         TRACE_RECORD_TGID for recording tgid
2291  */
2292 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2293                                           struct task_struct *next, int flags)
2294 {
2295         bool done;
2296 
2297         if (tracing_record_taskinfo_skip(flags))
2298                 return;
2299 
2300         /*
2301          * Record as much task information as possible. If some fail, continue
2302          * to try to record the others.
2303          */
2304         done  = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2305         done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2306         done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2307         done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2308 
2309         /* If recording any information failed, retry again soon. */
2310         if (!done)
2311                 return;
2312 
2313         __this_cpu_write(trace_taskinfo_save, false);
2314 }
2315 
2316 /* Helpers to record a specific task information */
2317 void tracing_record_cmdline(struct task_struct *task)
2318 {
2319         tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2320 }
2321 
2322 void tracing_record_tgid(struct task_struct *task)
2323 {
2324         tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2325 }
2326 
2327 /*
2328  * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2329  * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2330  * simplifies those functions and keeps them in sync.
2331  */
2332 enum print_line_t trace_handle_return(struct trace_seq *s)
2333 {
2334         return trace_seq_has_overflowed(s) ?
2335                 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2336 }
2337 EXPORT_SYMBOL_GPL(trace_handle_return);
2338 
2339 void
2340 tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
2341                              unsigned long flags, int pc)
2342 {
2343         struct task_struct *tsk = current;
2344 
2345         entry->preempt_count            = pc & 0xff;
2346         entry->pid                      = (tsk) ? tsk->pid : 0;
2347         entry->type                     = type;
2348         entry->flags =
2349 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2350                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2351 #else
2352                 TRACE_FLAG_IRQS_NOSUPPORT |
2353 #endif
2354                 ((pc & NMI_MASK    ) ? TRACE_FLAG_NMI     : 0) |
2355                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2356                 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2357                 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2358                 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2359 }
2360 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2361 
2362 struct ring_buffer_event *
2363 trace_buffer_lock_reserve(struct ring_buffer *buffer,
2364                           int type,
2365                           unsigned long len,
2366                           unsigned long flags, int pc)
2367 {
2368         return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2369 }
2370 
2371 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2372 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2373 static int trace_buffered_event_ref;
2374 
2375 /**
2376  * trace_buffered_event_enable - enable buffering events
2377  *
2378  * When events are being filtered, it is quicker to use a temporary
2379  * buffer to write the event data into if there's a likely chance
2380  * that it will not be committed. The discard of the ring buffer
2381  * is not as fast as committing, and is much slower than copying
2382  * a commit.
2383  *
2384  * When an event is to be filtered, allocate per cpu buffers to
2385  * write the event data into, and if the event is filtered and discarded
2386  * it is simply dropped, otherwise, the entire data is to be committed
2387  * in one shot.
2388  */
2389 void trace_buffered_event_enable(void)
2390 {
2391         struct ring_buffer_event *event;
2392         struct page *page;
2393         int cpu;
2394 
2395         WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2396 
2397         if (trace_buffered_event_ref++)
2398                 return;
2399 
2400         for_each_tracing_cpu(cpu) {
2401                 page = alloc_pages_node(cpu_to_node(cpu),
2402                                         GFP_KERNEL | __GFP_NORETRY, 0);
2403                 if (!page)
2404                         goto failed;
2405 
2406                 event = page_address(page);
2407                 memset(event, 0, sizeof(*event));
2408 
2409                 per_cpu(trace_buffered_event, cpu) = event;
2410 
2411                 preempt_disable();
2412                 if (cpu == smp_processor_id() &&
2413                     this_cpu_read(trace_buffered_event) !=
2414                     per_cpu(trace_buffered_event, cpu))
2415                         WARN_ON_ONCE(1);
2416                 preempt_enable();
2417         }
2418 
2419         return;
2420  failed:
2421         trace_buffered_event_disable();
2422 }
2423 
2424 static void enable_trace_buffered_event(void *data)
2425 {
2426         /* Probably not needed, but do it anyway */
2427         smp_rmb();
2428         this_cpu_dec(trace_buffered_event_cnt);
2429 }
2430 
2431 static void disable_trace_buffered_event(void *data)
2432 {
2433         this_cpu_inc(trace_buffered_event_cnt);
2434 }
2435 
2436 /**
2437  * trace_buffered_event_disable - disable buffering events
2438  *
2439  * When a filter is removed, it is faster to not use the buffered
2440  * events, and to commit directly into the ring buffer. Free up
2441  * the temp buffers when there are no more users. This requires
2442  * special synchronization with current events.
2443  */
2444 void trace_buffered_event_disable(void)
2445 {
2446         int cpu;
2447 
2448         WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2449 
2450         if (WARN_ON_ONCE(!trace_buffered_event_ref))
2451                 return;
2452 
2453         if (--trace_buffered_event_ref)
2454                 return;
2455 
2456         preempt_disable();
2457         /* For each CPU, set the buffer as used. */
2458         smp_call_function_many(tracing_buffer_mask,
2459                                disable_trace_buffered_event, NULL, 1);
2460         preempt_enable();
2461 
2462         /* Wait for all current users to finish */
2463         synchronize_rcu();
2464 
2465         for_each_tracing_cpu(cpu) {
2466                 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2467                 per_cpu(trace_buffered_event, cpu) = NULL;
2468         }
2469         /*
2470          * Make sure trace_buffered_event is NULL before clearing
2471          * trace_buffered_event_cnt.
2472          */
2473         smp_wmb();
2474 
2475         preempt_disable();
2476         /* Do the work on each cpu */
2477         smp_call_function_many(tracing_buffer_mask,
2478                                enable_trace_buffered_event, NULL, 1);
2479         preempt_enable();
2480 }
2481 
2482 static struct ring_buffer *temp_buffer;
2483 
2484 struct ring_buffer_event *
2485 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2486                           struct trace_event_file *trace_file,
2487                           int type, unsigned long len,
2488                           unsigned long flags, int pc)
2489 {
2490         struct ring_buffer_event *entry;
2491         int val;
2492 
2493         *current_rb = trace_file->tr->trace_buffer.buffer;
2494 
2495         if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2496              (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2497             (entry = this_cpu_read(trace_buffered_event))) {
2498                 /* Try to use the per cpu buffer first */
2499                 val = this_cpu_inc_return(trace_buffered_event_cnt);
2500                 if (val == 1) {
2501                         trace_event_setup(entry, type, flags, pc);
2502                         entry->array[0] = len;
2503                         return entry;
2504                 }
2505                 this_cpu_dec(trace_buffered_event_cnt);
2506         }
2507 
2508         entry = __trace_buffer_lock_reserve(*current_rb,
2509                                             type, len, flags, pc);
2510         /*
2511          * If tracing is off, but we have triggers enabled
2512          * we still need to look at the event data. Use the temp_buffer
2513          * to store the trace event for the tigger to use. It's recusive
2514          * safe and will not be recorded anywhere.
2515          */
2516         if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2517                 *current_rb = temp_buffer;
2518                 entry = __trace_buffer_lock_reserve(*current_rb,
2519                                                     type, len, flags, pc);
2520         }
2521         return entry;
2522 }
2523 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2524 
2525 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2526 static DEFINE_MUTEX(tracepoint_printk_mutex);
2527 
2528 static void output_printk(struct trace_event_buffer *fbuffer)
2529 {
2530         struct trace_event_call *event_call;
2531         struct trace_event *event;
2532         unsigned long flags;
2533         struct trace_iterator *iter = tracepoint_print_iter;
2534 
2535         /* We should never get here if iter is NULL */
2536         if (WARN_ON_ONCE(!iter))
2537                 return;
2538 
2539         event_call = fbuffer->trace_file->event_call;
2540         if (!event_call || !event_call->event.funcs ||
2541             !event_call->event.funcs->trace)
2542                 return;
2543 
2544         event = &fbuffer->trace_file->event_call->event;
2545 
2546         spin_lock_irqsave(&tracepoint_iter_lock, flags);
2547         trace_seq_init(&iter->seq);
2548         iter->ent = fbuffer->entry;
2549         event_call->event.funcs->trace(iter, 0, event);
2550         trace_seq_putc(&iter->seq, 0);
2551         printk("%s", iter->seq.buffer);
2552 
2553         spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2554 }
2555 
2556 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2557                              void __user *buffer, size_t *lenp,
2558                              loff_t *ppos)
2559 {
2560         int save_tracepoint_printk;
2561         int ret;
2562 
2563         mutex_lock(&tracepoint_printk_mutex);
2564         save_tracepoint_printk = tracepoint_printk;
2565 
2566         ret = proc_dointvec(table, write, buffer, lenp, ppos);
2567 
2568         /*
2569          * This will force exiting early, as tracepoint_printk
2570          * is always zero when tracepoint_printk_iter is not allocated
2571          */
2572         if (!tracepoint_print_iter)
2573                 tracepoint_printk = 0;
2574 
2575         if (save_tracepoint_printk == tracepoint_printk)
2576                 goto out;
2577 
2578         if (tracepoint_printk)
2579                 static_key_enable(&tracepoint_printk_key.key);
2580         else
2581                 static_key_disable(&tracepoint_printk_key.key);
2582 
2583  out:
2584         mutex_unlock(&tracepoint_printk_mutex);
2585 
2586         return ret;
2587 }
2588 
2589 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2590 {
2591         if (static_key_false(&tracepoint_printk_key.key))
2592                 output_printk(fbuffer);
2593 
2594         event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2595                                     fbuffer->event, fbuffer->entry,
2596                                     fbuffer->flags, fbuffer->pc);
2597 }
2598 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2599 
2600 /*
2601  * Skip 3:
2602  *
2603  *   trace_buffer_unlock_commit_regs()
2604  *   trace_event_buffer_commit()
2605  *   trace_event_raw_event_xxx()
2606  */
2607 # define STACK_SKIP 3
2608 
2609 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2610                                      struct ring_buffer *buffer,
2611                                      struct ring_buffer_event *event,
2612                                      unsigned long flags, int pc,
2613                                      struct pt_regs *regs)
2614 {
2615         __buffer_unlock_commit(buffer, event);
2616 
2617         /*
2618          * If regs is not set, then skip the necessary functions.
2619          * Note, we can still get here via blktrace, wakeup tracer
2620          * and mmiotrace, but that's ok if they lose a function or
2621          * two. They are not that meaningful.
2622          */
2623         ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2624         ftrace_trace_userstack(buffer, flags, pc);
2625 }
2626 
2627 /*
2628  * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2629  */
2630 void
2631 trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2632                                    struct ring_buffer_event *event)
2633 {
2634         __buffer_unlock_commit(buffer, event);
2635 }
2636 
2637 static void
2638 trace_process_export(struct trace_export *export,
2639                struct ring_buffer_event *event)
2640 {
2641         struct trace_entry *entry;
2642         unsigned int size = 0;
2643 
2644         entry = ring_buffer_event_data(event);
2645         size = ring_buffer_event_length(event);
2646         export->write(export, entry, size);
2647 }
2648 
2649 static DEFINE_MUTEX(ftrace_export_lock);
2650 
2651 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2652 
2653 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2654 
2655 static inline void ftrace_exports_enable(void)
2656 {
2657         static_branch_enable(&ftrace_exports_enabled);
2658 }
2659 
2660 static inline void ftrace_exports_disable(void)
2661 {
2662         static_branch_disable(&ftrace_exports_enabled);
2663 }
2664 
2665 static void ftrace_exports(struct ring_buffer_event *event)
2666 {
2667         struct trace_export *export;
2668 
2669         preempt_disable_notrace();
2670 
2671         export = rcu_dereference_raw_check(ftrace_exports_list);
2672         while (export) {
2673                 trace_process_export(export, event);
2674                 export = rcu_dereference_raw_check(export->next);
2675         }
2676 
2677         preempt_enable_notrace();
2678 }
2679 
2680 static inline void
2681 add_trace_export(struct trace_export **list, struct trace_export *export)
2682 {
2683         rcu_assign_pointer(export->next, *list);
2684         /*
2685          * We are entering export into the list but another
2686          * CPU might be walking that list. We need to make sure
2687          * the export->next pointer is valid before another CPU sees
2688          * the export pointer included into the list.
2689          */
2690         rcu_assign_pointer(*list, export);
2691 }
2692 
2693 static inline int
2694 rm_trace_export(struct trace_export **list, struct trace_export *export)
2695 {
2696         struct trace_export **p;
2697 
2698         for (p = list; *p != NULL; p = &(*p)->next)
2699                 if (*p == export)
2700                         break;
2701 
2702         if (*p != export)
2703                 return -1;
2704 
2705         rcu_assign_pointer(*p, (*p)->next);
2706 
2707         return 0;
2708 }
2709 
2710 static inline void
2711 add_ftrace_export(struct trace_export **list, struct trace_export *export)
2712 {
2713         if (*list == NULL)
2714                 ftrace_exports_enable();
2715 
2716         add_trace_export(list, export);
2717 }
2718 
2719 static inline int
2720 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2721 {
2722         int ret;
2723 
2724         ret = rm_trace_export(list, export);
2725         if (*list == NULL)
2726                 ftrace_exports_disable();
2727 
2728         return ret;
2729 }
2730 
2731 int register_ftrace_export(struct trace_export *export)
2732 {
2733         if (WARN_ON_ONCE(!export->write))
2734                 return -1;
2735 
2736         mutex_lock(&ftrace_export_lock);
2737 
2738         add_ftrace_export(&ftrace_exports_list, export);
2739 
2740         mutex_unlock(&ftrace_export_lock);
2741 
2742         return 0;
2743 }
2744 EXPORT_SYMBOL_GPL(register_ftrace_export);
2745 
2746 int unregister_ftrace_export(struct trace_export *export)
2747 {
2748         int ret;
2749 
2750         mutex_lock(&ftrace_export_lock);
2751 
2752         ret = rm_ftrace_export(&ftrace_exports_list, export);
2753 
2754         mutex_unlock(&ftrace_export_lock);
2755 
2756         return ret;
2757 }
2758 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2759 
2760 void
2761 trace_function(struct trace_array *tr,
2762                unsigned long ip, unsigned long parent_ip, unsigned long flags,
2763                int pc)
2764 {
2765         struct trace_event_call *call = &event_function;
2766         struct ring_buffer *buffer = tr->trace_buffer.buffer;
2767         struct ring_buffer_event *event;
2768         struct ftrace_entry *entry;
2769 
2770         event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2771                                             flags, pc);
2772         if (!event)
2773                 return;
2774         entry   = ring_buffer_event_data(event);
2775         entry->ip                       = ip;
2776         entry->parent_ip                = parent_ip;
2777 
2778         if (!call_filter_check_discard(call, entry, buffer, event)) {
2779                 if (static_branch_unlikely(&ftrace_exports_enabled))
2780                         ftrace_exports(event);
2781                 __buffer_unlock_commit(buffer, event);
2782         }
2783 }
2784 
2785 #ifdef CONFIG_STACKTRACE
2786 
2787 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2788 #define FTRACE_KSTACK_NESTING   4
2789 
2790 #define FTRACE_KSTACK_ENTRIES   (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2791 
2792 struct ftrace_stack {
2793         unsigned long           calls[FTRACE_KSTACK_ENTRIES];
2794 };
2795 
2796 
2797 struct ftrace_stacks {
2798         struct ftrace_stack     stacks[FTRACE_KSTACK_NESTING];
2799 };
2800 
2801 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2802 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2803 
2804 static void __ftrace_trace_stack(struct ring_buffer *buffer,
2805                                  unsigned long flags,
2806                                  int skip, int pc, struct pt_regs *regs)
2807 {
2808         struct trace_event_call *call = &event_kernel_stack;
2809         struct ring_buffer_event *event;
2810         unsigned int size, nr_entries;
2811         struct ftrace_stack *fstack;
2812         struct stack_entry *entry;
2813         int stackidx;
2814 
2815         /*
2816          * Add one, for this function and the call to save_stack_trace()
2817          * If regs is set, then these functions will not be in the way.
2818          */
2819 #ifndef CONFIG_UNWINDER_ORC
2820         if (!regs)
2821                 skip++;
2822 #endif
2823 
2824         /*
2825          * Since events can happen in NMIs there's no safe way to
2826          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2827          * or NMI comes in, it will just have to use the default
2828          * FTRACE_STACK_SIZE.
2829          */
2830         preempt_disable_notrace();
2831 
2832         stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2833 
2834         /* This should never happen. If it does, yell once and skip */
2835         if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING))
2836                 goto out;
2837 
2838         /*
2839          * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2840          * interrupt will either see the value pre increment or post
2841          * increment. If the interrupt happens pre increment it will have
2842          * restored the counter when it returns.  We just need a barrier to
2843          * keep gcc from moving things around.
2844          */
2845         barrier();
2846 
2847         fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2848         size = ARRAY_SIZE(fstack->calls);
2849 
2850         if (regs) {
2851                 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2852                                                    size, skip);
2853         } else {
2854                 nr_entries = stack_trace_save(fstack->calls, size, skip);
2855         }
2856 
2857         size = nr_entries * sizeof(unsigned long);
2858         event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2859                                             sizeof(*entry) + size, flags, pc);
2860         if (!event)
2861                 goto out;
2862         entry = ring_buffer_event_data(event);
2863 
2864         memcpy(&entry->caller, fstack->calls, size);
2865         entry->size = nr_entries;
2866 
2867         if (!call_filter_check_discard(call, entry, buffer, event))
2868                 __buffer_unlock_commit(buffer, event);
2869 
2870  out:
2871         /* Again, don't let gcc optimize things here */
2872         barrier();
2873         __this_cpu_dec(ftrace_stack_reserve);
2874         preempt_enable_notrace();
2875 
2876 }
2877 
2878 static inline void ftrace_trace_stack(struct trace_array *tr,
2879                                       struct ring_buffer *buffer,
2880                                       unsigned long flags,
2881                                       int skip, int pc, struct pt_regs *regs)
2882 {
2883         if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2884                 return;
2885 
2886         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2887 }
2888 
2889 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2890                    int pc)
2891 {
2892         struct ring_buffer *buffer = tr->trace_buffer.buffer;
2893 
2894         if (rcu_is_watching()) {
2895                 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2896                 return;
2897         }
2898 
2899         /*
2900          * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2901          * but if the above rcu_is_watching() failed, then the NMI
2902          * triggered someplace critical, and rcu_irq_enter() should
2903          * not be called from NMI.
2904          */
2905         if (unlikely(in_nmi()))
2906                 return;
2907 
2908         rcu_irq_enter_irqson();
2909         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2910         rcu_irq_exit_irqson();
2911 }
2912 
2913 /**
2914  * trace_dump_stack - record a stack back trace in the trace buffer
2915  * @skip: Number of functions to skip (helper handlers)
2916  */
2917 void trace_dump_stack(int skip)
2918 {
2919         unsigned long flags;
2920 
2921         if (tracing_disabled || tracing_selftest_running)
2922                 return;
2923 
2924         local_save_flags(flags);
2925 
2926 #ifndef CONFIG_UNWINDER_ORC
2927         /* Skip 1 to skip this function. */
2928         skip++;
2929 #endif
2930         __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2931                              flags, skip, preempt_count(), NULL);
2932 }
2933 EXPORT_SYMBOL_GPL(trace_dump_stack);
2934 
2935 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
2936 static DEFINE_PER_CPU(int, user_stack_count);
2937 
2938 static void
2939 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
2940 {
2941         struct trace_event_call *call = &event_user_stack;
2942         struct ring_buffer_event *event;
2943         struct userstack_entry *entry;
2944 
2945         if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
2946                 return;
2947 
2948         /*
2949          * NMIs can not handle page faults, even with fix ups.
2950          * The save user stack can (and often does) fault.
2951          */
2952         if (unlikely(in_nmi()))
2953                 return;
2954 
2955         /*
2956          * prevent recursion, since the user stack tracing may
2957          * trigger other kernel events.
2958          */
2959         preempt_disable();
2960         if (__this_cpu_read(user_stack_count))
2961                 goto out;
2962 
2963         __this_cpu_inc(user_stack_count);
2964 
2965         event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2966                                             sizeof(*entry), flags, pc);
2967         if (!event)
2968                 goto out_drop_count;
2969         entry   = ring_buffer_event_data(event);
2970 
2971         entry->tgid             = current->tgid;
2972         memset(&entry->caller, 0, sizeof(entry->caller));
2973 
2974         stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
2975         if (!call_filter_check_discard(call, entry, buffer, event))
2976                 __buffer_unlock_commit(buffer, event);
2977 
2978  out_drop_count:
2979         __this_cpu_dec(user_stack_count);
2980  out:
2981         preempt_enable();
2982 }
2983 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
2984 static void ftrace_trace_userstack(struct ring_buffer *buffer,
2985                                    unsigned long flags, int pc)
2986 {
2987 }
2988 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
2989 
2990 #endif /* CONFIG_STACKTRACE */
2991 
2992 /* created for use with alloc_percpu */
2993 struct trace_buffer_struct {
2994         int nesting;
2995         char buffer[4][TRACE_BUF_SIZE];
2996 };
2997 
2998 static struct trace_buffer_struct *trace_percpu_buffer;
2999 
3000 /*
3001  * Thise allows for lockless recording.  If we're nested too deeply, then
3002  * this returns NULL.
3003  */
3004 static char *get_trace_buf(void)
3005 {
3006         struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3007 
3008         if (!buffer || buffer->nesting >= 4)
3009                 return NULL;
3010 
3011         buffer->nesting++;
3012 
3013         /* Interrupts must see nesting incremented before we use the buffer */
3014         barrier();
3015         return &buffer->buffer[buffer->nesting][0];
3016 }
3017 
3018 static void put_trace_buf(void)
3019 {
3020         /* Don't let the decrement of nesting leak before this */
3021         barrier();
3022         this_cpu_dec(trace_percpu_buffer->nesting);
3023 }
3024 
3025 static int alloc_percpu_trace_buffer(void)
3026 {
3027         struct trace_buffer_struct *buffers;
3028 
3029         buffers = alloc_percpu(struct trace_buffer_struct);
3030         if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
3031                 return -ENOMEM;
3032 
3033         trace_percpu_buffer = buffers;
3034         return 0;
3035 }
3036 
3037 static int buffers_allocated;
3038 
3039 void trace_printk_init_buffers(void)
3040 {
3041         if (buffers_allocated)
3042                 return;
3043 
3044         if (alloc_percpu_trace_buffer())
3045                 return;
3046 
3047         /* trace_printk() is for debug use only. Don't use it in production. */
3048 
3049         pr_warn("\n");
3050         pr_warn("**********************************************************\n");
3051         pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
3052         pr_warn("**                                                      **\n");
3053         pr_warn("** trace_printk() being used. Allocating extra memory.  **\n");
3054         pr_warn("**                                                      **\n");
3055         pr_warn("** This means that this is a DEBUG kernel and it is     **\n");
3056         pr_warn("** unsafe for production use.                           **\n");
3057         pr_warn("**                                                      **\n");
3058         pr_warn("** If you see this message and you are not debugging    **\n");
3059         pr_warn("** the kernel, report this immediately to your vendor!  **\n");
3060         pr_warn("**                                                      **\n");
3061         pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
3062         pr_warn("**********************************************************\n");
3063 
3064         /* Expand the buffers to set size */
3065         tracing_update_buffers();
3066 
3067         buffers_allocated = 1;
3068 
3069         /*
3070          * trace_printk_init_buffers() can be called by modules.
3071          * If that happens, then we need to start cmdline recording
3072          * directly here. If the global_trace.buffer is already
3073          * allocated here, then this was called by module code.
3074          */
3075         if (global_trace.trace_buffer.buffer)
3076                 tracing_start_cmdline_record();
3077 }
3078 EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3079 
3080 void trace_printk_start_comm(void)
3081 {
3082         /* Start tracing comms if trace printk is set */
3083         if (!buffers_allocated)
3084                 return;
3085         tracing_start_cmdline_record();
3086 }
3087 
3088 static void trace_printk_start_stop_comm(int enabled)
3089 {
3090         if (!buffers_allocated)
3091                 return;
3092 
3093         if (enabled)
3094                 tracing_start_cmdline_record();
3095         else
3096                 tracing_stop_cmdline_record();
3097 }
3098 
3099 /**
3100  * trace_vbprintk - write binary msg to tracing buffer
3101  * @ip:    The address of the caller
3102  * @fmt:   The string format to write to the buffer
3103  * @args:  Arguments for @fmt
3104  */
3105 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3106 {
3107         struct trace_event_call *call = &event_bprint;
3108         struct ring_buffer_event *event;
3109         struct ring_buffer *buffer;
3110         struct trace_array *tr = &global_trace;
3111         struct bprint_entry *entry;
3112         unsigned long flags;
3113         char *tbuffer;
3114         int len = 0, size, pc;
3115 
3116         if (unlikely(tracing_selftest_running || tracing_disabled))
3117                 return 0;
3118 
3119         /* Don't pollute graph traces with trace_vprintk internals */
3120         pause_graph_tracing();
3121 
3122         pc = preempt_count();
3123         preempt_disable_notrace();
3124 
3125         tbuffer = get_trace_buf();
3126         if (!tbuffer) {
3127                 len = 0;
3128                 goto out_nobuffer;
3129         }
3130 
3131         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3132 
3133         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3134                 goto out;
3135 
3136         local_save_flags(flags);
3137         size = sizeof(*entry) + sizeof(u32) * len;
3138         buffer = tr->trace_buffer.buffer;
3139         event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3140                                             flags, pc);
3141         if (!event)
3142                 goto out;
3143         entry = ring_buffer_event_data(event);
3144         entry->ip                       = ip;
3145         entry->fmt                      = fmt;
3146 
3147         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3148         if (!call_filter_check_discard(call, entry, buffer, event)) {
3149                 __buffer_unlock_commit(buffer, event);
3150                 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
3151         }
3152 
3153 out:
3154         put_trace_buf();
3155 
3156 out_nobuffer:
3157         preempt_enable_notrace();
3158         unpause_graph_tracing();
3159 
3160         return len;
3161 }
3162 EXPORT_SYMBOL_GPL(trace_vbprintk);
3163 
3164 __printf(3, 0)
3165 static int
3166 __trace_array_vprintk(struct ring_buffer *buffer,
3167                       unsigned long ip, const char *fmt, va_list args)
3168 {
3169         struct trace_event_call *call = &event_print;
3170         struct ring_buffer_event *event;
3171         int len = 0, size, pc;
3172         struct print_entry *entry;
3173         unsigned long flags;
3174         char *tbuffer;
3175 
3176         if (tracing_disabled || tracing_selftest_running)
3177                 return 0;
3178 
3179         /* Don't pollute graph traces with trace_vprintk internals */
3180         pause_graph_tracing();
3181 
3182         pc = preempt_count();
3183         preempt_disable_notrace();
3184 
3185 
3186         tbuffer = get_trace_buf();
3187         if (!tbuffer) {
3188                 len = 0;
3189                 goto out_nobuffer;
3190         }
3191 
3192         len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3193 
3194         local_save_flags(flags);
3195         size = sizeof(*entry) + len + 1;
3196         event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3197                                             flags, pc);
3198         if (!event)
3199                 goto out;
3200         entry = ring_buffer_event_data(event);
3201         entry->ip = ip;
3202 
3203         memcpy(&entry->buf, tbuffer, len + 1);
3204         if (!call_filter_check_discard(call, entry, buffer, event)) {
3205                 __buffer_unlock_commit(buffer, event);
3206                 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3207         }
3208 
3209 out:
3210         put_trace_buf();
3211 
3212 out_nobuffer:
3213         preempt_enable_notrace();
3214         unpause_graph_tracing();
3215 
3216         return len;
3217 }
3218 
3219 __printf(3, 0)
3220 int trace_array_vprintk(struct trace_array *tr,
3221                         unsigned long ip, const char *fmt, va_list args)
3222 {
3223         return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
3224 }
3225 
3226 __printf(3, 0)
3227 int trace_array_printk(struct trace_array *tr,
3228                        unsigned long ip, const char *fmt, ...)
3229 {
3230         int ret;
3231         va_list ap;
3232 
3233         if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3234                 return 0;
3235 
3236         va_start(ap, fmt);
3237         ret = trace_array_vprintk(tr, ip, fmt, ap);
3238         va_end(ap);
3239         return ret;
3240 }
3241 EXPORT_SYMBOL_GPL(trace_array_printk);
3242 
3243 __printf(3, 4)
3244 int trace_array_printk_buf(struct ring_buffer *buffer,
3245                            unsigned long ip, const char *fmt, ...)
3246 {
3247         int ret;
3248         va_list ap;
3249 
3250         if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3251                 return 0;
3252 
3253         va_start(ap, fmt);
3254         ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3255         va_end(ap);
3256         return ret;
3257 }
3258 
3259 __printf(2, 0)
3260 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3261 {
3262         return trace_array_vprintk(&global_trace, ip, fmt, args);
3263 }
3264 EXPORT_SYMBOL_GPL(trace_vprintk);
3265 
3266 static void trace_iterator_increment(struct trace_iterator *iter)
3267 {
3268         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3269 
3270         iter->idx++;
3271         if (buf_iter)
3272                 ring_buffer_read(buf_iter, NULL);
3273 }
3274 
3275 static struct trace_entry *
3276 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3277                 unsigned long *lost_events)
3278 {
3279         struct ring_buffer_event *event;
3280         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3281 
3282         if (buf_iter)
3283                 event = ring_buffer_iter_peek(buf_iter, ts);
3284         else
3285                 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
3286                                          lost_events);
3287 
3288         if (event) {
3289                 iter->ent_size = ring_buffer_event_length(event);
3290                 return ring_buffer_event_data(event);
3291         }
3292         iter->ent_size = 0;
3293         return NULL;
3294 }
3295 
3296 static struct trace_entry *
3297 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3298                   unsigned long *missing_events, u64 *ent_ts)
3299 {
3300         struct ring_buffer *buffer = iter->trace_buffer->buffer;
3301         struct trace_entry *ent, *next = NULL;
3302         unsigned long lost_events = 0, next_lost = 0;
3303         int cpu_file = iter->cpu_file;
3304         u64 next_ts = 0, ts;
3305         int next_cpu = -1;
3306         int next_size = 0;
3307         int cpu;
3308 
3309         /*
3310          * If we are in a per_cpu trace file, don't bother by iterating over
3311          * all cpu and peek directly.
3312          */
3313         if (cpu_file > RING_BUFFER_ALL_CPUS) {
3314                 if (ring_buffer_empty_cpu(buffer, cpu_file))
3315                         return NULL;
3316                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3317                 if (ent_cpu)
3318                         *ent_cpu = cpu_file;
3319 
3320                 return ent;
3321         }
3322 
3323         for_each_tracing_cpu(cpu) {
3324 
3325                 if (ring_buffer_empty_cpu(buffer, cpu))
3326                         continue;
3327 
3328                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3329 
3330                 /*
3331                  * Pick the entry with the smallest timestamp:
3332                  */
3333                 if (ent && (!next || ts < next_ts)) {
3334                         next = ent;
3335                         next_cpu = cpu;
3336                         next_ts = ts;
3337                         next_lost = lost_events;
3338                         next_size = iter->ent_size;
3339                 }
3340         }
3341 
3342         iter->ent_size = next_size;
3343 
3344         if (ent_cpu)
3345                 *ent_cpu = next_cpu;
3346 
3347         if (ent_ts)
3348                 *ent_ts = next_ts;
3349 
3350         if (missing_events)
3351                 *missing_events = next_lost;
3352 
3353         return next;
3354 }
3355 
3356 /* Find the next real entry, without updating the iterator itself */
3357 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3358                                           int *ent_cpu, u64 *ent_ts)
3359 {
3360         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3361 }
3362 
3363 /* Find the next real entry, and increment the iterator to the next entry */
3364 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3365 {
3366         iter->ent = __find_next_entry(iter, &iter->cpu,
3367                                       &iter->lost_events, &iter->ts);
3368 
3369         if (iter->ent)
3370                 trace_iterator_increment(iter);
3371 
3372         return iter->ent ? iter : NULL;
3373 }
3374 
3375 static void trace_consume(struct trace_iterator *iter)
3376 {
3377         ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
3378                             &iter->lost_events);
3379 }
3380 
3381 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3382 {
3383         struct trace_iterator *iter = m->private;
3384         int i = (int)*pos;
3385         void *ent;
3386 
3387         WARN_ON_ONCE(iter->leftover);
3388 
3389         (*pos)++;
3390 
3391         /* can't go backwards */
3392         if (iter->idx > i)
3393                 return NULL;
3394 
3395         if (iter->idx < 0)
3396                 ent = trace_find_next_entry_inc(iter);
3397         else
3398                 ent = iter;
3399 
3400         while (ent && iter->idx < i)
3401                 ent = trace_find_next_entry_inc(iter);
3402 
3403         iter->pos = *pos;
3404 
3405         return ent;
3406 }
3407 
3408 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3409 {
3410         struct ring_buffer_event *event;
3411         struct ring_buffer_iter *buf_iter;
3412         unsigned long entries = 0;
3413         u64 ts;
3414 
3415         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
3416 
3417         buf_iter = trace_buffer_iter(iter, cpu);
3418         if (!buf_iter)
3419                 return;
3420 
3421         ring_buffer_iter_reset(buf_iter);
3422 
3423         /*
3424          * We could have the case with the max latency tracers
3425          * that a reset never took place on a cpu. This is evident
3426          * by the timestamp being before the start of the buffer.
3427          */
3428         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3429                 if (ts >= iter->trace_buffer->time_start)
3430                         break;
3431                 entries++;
3432                 ring_buffer_read(buf_iter, NULL);
3433         }
3434 
3435         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
3436 }
3437 
3438 /*
3439  * The current tracer is copied to avoid a global locking
3440  * all around.
3441  */
3442 static void *s_start(struct seq_file *m, loff_t *pos)
3443 {
3444         struct trace_iterator *iter = m->private;
3445         struct trace_array *tr = iter->tr;
3446         int cpu_file = iter->cpu_file;
3447         void *p = NULL;
3448         loff_t l = 0;
3449         int cpu;
3450 
3451         /*
3452          * copy the tracer to avoid using a global lock all around.
3453          * iter->trace is a copy of current_trace, the pointer to the
3454          * name may be used instead of a strcmp(), as iter->trace->name
3455          * will point to the same string as current_trace->name.
3456          */
3457         mutex_lock(&trace_types_lock);
3458         if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3459                 *iter->trace = *tr->current_trace;
3460         mutex_unlock(&trace_types_lock);
3461 
3462 #ifdef CONFIG_TRACER_MAX_TRACE
3463         if (iter->snapshot && iter->trace->use_max_tr)
3464                 return ERR_PTR(-EBUSY);
3465 #endif
3466 
3467         if (!iter->snapshot)
3468                 atomic_inc(&trace_record_taskinfo_disabled);
3469 
3470         if (*pos != iter->pos) {
3471                 iter->ent = NULL;
3472                 iter->cpu = 0;
3473                 iter->idx = -1;
3474 
3475                 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3476                         for_each_tracing_cpu(cpu)
3477                                 tracing_iter_reset(iter, cpu);
3478                 } else
3479                         tracing_iter_reset(iter, cpu_file);
3480 
3481                 iter->leftover = 0;
3482                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3483                         ;
3484 
3485         } else {
3486                 /*
3487                  * If we overflowed the seq_file before, then we want
3488                  * to just reuse the trace_seq buffer again.
3489                  */
3490                 if (iter->leftover)
3491                         p = iter;
3492                 else {
3493                         l = *pos - 1;
3494                         p = s_next(m, p, &l);
3495                 }
3496         }
3497 
3498         trace_event_read_lock();
3499         trace_access_lock(cpu_file);
3500         return p;
3501 }
3502 
3503 static void s_stop(struct seq_file *m, void *p)
3504 {
3505         struct trace_iterator *iter = m->private;
3506 
3507 #ifdef CONFIG_TRACER_MAX_TRACE
3508         if (iter->snapshot && iter->trace->use_max_tr)
3509                 return;
3510 #endif
3511 
3512         if (!iter->snapshot)
3513                 atomic_dec(&trace_record_taskinfo_disabled);
3514 
3515         trace_access_unlock(iter->cpu_file);
3516         trace_event_read_unlock();
3517 }
3518 
3519 static void
3520 get_total_entries_cpu(struct trace_buffer *buf, unsigned long *total,
3521                       unsigned long *entries, int cpu)
3522 {
3523         unsigned long count;
3524 
3525         count = ring_buffer_entries_cpu(buf->buffer, cpu);
3526         /*
3527          * If this buffer has skipped entries, then we hold all
3528          * entries for the trace and we need to ignore the
3529          * ones before the time stamp.
3530          */
3531         if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3532                 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3533                 /* total is the same as the entries */
3534                 *total = count;
3535         } else
3536                 *total = count +
3537                         ring_buffer_overrun_cpu(buf->buffer, cpu);
3538         *entries = count;
3539 }
3540 
3541 static void
3542 get_total_entries(struct trace_buffer *buf,
3543                   unsigned long *total, unsigned long *entries)
3544 {
3545         unsigned long t, e;
3546         int cpu;
3547 
3548         *total = 0;
3549         *entries = 0;
3550 
3551         for_each_tracing_cpu(cpu) {
3552                 get_total_entries_cpu(buf, &t, &e, cpu);
3553                 *total += t;
3554                 *entries += e;
3555         }
3556 }
3557 
3558 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
3559 {
3560         unsigned long total, entries;
3561 
3562         if (!tr)
3563                 tr = &global_trace;
3564 
3565         get_total_entries_cpu(&tr->trace_buffer, &total, &entries, cpu);
3566 
3567         return entries;
3568 }
3569 
3570 unsigned long trace_total_entries(struct trace_array *tr)
3571 {
3572         unsigned long total, entries;
3573 
3574         if (!tr)
3575                 tr = &global_trace;
3576 
3577         get_total_entries(&tr->trace_buffer, &total, &entries);
3578 
3579         return entries;
3580 }
3581 
3582 static void print_lat_help_header(struct seq_file *m)
3583 {
3584         seq_puts(m, "#                  _------=> CPU#            \n"
3585                     "#                 / _-----=> irqs-off        \n"
3586                     "#                | / _----=> need-resched    \n"
3587                     "#                || / _---=> hardirq/softirq \n"
3588                     "#                ||| / _--=> preempt-depth   \n"
3589                     "#                |||| /     delay            \n"
3590                     "#  cmd     pid   ||||| time  |   caller      \n"
3591                     "#     \\   /      |||||  \\    |   /         \n");
3592 }
3593 
3594 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
3595 {
3596         unsigned long total;
3597         unsigned long entries;
3598 
3599         get_total_entries(buf, &total, &entries);
3600         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
3601                    entries, total, num_online_cpus());
3602         seq_puts(m, "#\n");
3603 }
3604 
3605 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m,
3606                                    unsigned int flags)
3607 {
3608         bool tgid = flags & TRACE_ITER_RECORD_TGID;
3609 
3610         print_event_info(buf, m);
3611 
3612         seq_printf(m, "#           TASK-PID   %s  CPU#   TIMESTAMP  FUNCTION\n", tgid ? "TGID     " : "");
3613         seq_printf(m, "#              | |     %s    |       |         |\n",      tgid ? "  |      " : "");
3614 }
3615 
3616 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m,
3617                                        unsigned int flags)
3618 {
3619         bool tgid = flags & TRACE_ITER_RECORD_TGID;
3620         const char *space = "          ";
3621         int prec = tgid ? 10 : 2;
3622 
3623         print_event_info(buf, m);
3624 
3625         seq_printf(m, "#                          %.*s  _-----=> irqs-off\n", prec, space);
3626         seq_printf(m, "#                          %.*s / _----=> need-resched\n", prec, space);
3627         seq_printf(m, "#                          %.*s| / _---=> hardirq/softirq\n", prec, space);
3628         seq_printf(m, "#                          %.*s|| / _--=> preempt-depth\n", prec, space);
3629         seq_printf(m, "#                          %.*s||| /     delay\n", prec, space);
3630         seq_printf(m, "#           TASK-PID %.*sCPU#  ||||    TIMESTAMP  FUNCTION\n", prec, "   TGID   ");
3631         seq_printf(m, "#              | |   %.*s  |   ||||       |         |\n", prec, "     |    ");
3632 }
3633 
3634 void
3635 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3636 {
3637         unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3638         struct trace_buffer *buf = iter->trace_buffer;
3639         struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3640         struct tracer *type = iter->trace;
3641         unsigned long entries;
3642         unsigned long total;
3643         const char *name = "preemption";
3644 
3645         name = type->name;
3646 
3647         get_total_entries(buf, &total, &entries);
3648 
3649         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3650                    name, UTS_RELEASE);
3651         seq_puts(m, "# -----------------------------------"
3652                  "---------------------------------\n");
3653         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3654                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3655                    nsecs_to_usecs(data->saved_latency),
3656                    entries,
3657                    total,
3658                    buf->cpu,
3659 #if defined(CONFIG_PREEMPT_NONE)
3660                    "server",
3661 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3662                    "desktop",
3663 #elif defined(CONFIG_PREEMPT)
3664                    "preempt",
3665 #else
3666                    "unknown",
3667 #endif
3668                    /* These are reserved for later use */
3669                    0, 0, 0, 0);
3670 #ifdef CONFIG_SMP
3671         seq_printf(m, " #P:%d)\n", num_online_cpus());
3672 #else
3673         seq_puts(m, ")\n");
3674 #endif
3675         seq_puts(m, "#    -----------------\n");
3676         seq_printf(m, "#    | task: %.16s-%d "
3677                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3678                    data->comm, data->pid,
3679                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3680                    data->policy, data->rt_priority);
3681         seq_puts(m, "#    -----------------\n");
3682 
3683         if (data->critical_start) {
3684                 seq_puts(m, "#  => started at: ");
3685                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3686                 trace_print_seq(m, &iter->seq);
3687                 seq_puts(m, "\n#  => ended at:   ");
3688                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3689                 trace_print_seq(m, &iter->seq);
3690                 seq_puts(m, "\n#\n");
3691         }
3692 
3693         seq_puts(m, "#\n");
3694 }
3695 
3696 static void test_cpu_buff_start(struct trace_iterator *iter)
3697 {
3698         struct trace_seq *s = &iter->seq;
3699         struct trace_array *tr = iter->tr;
3700 
3701         if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3702                 return;
3703 
3704         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3705                 return;
3706 
3707         if (cpumask_available(iter->started) &&
3708             cpumask_test_cpu(iter->cpu, iter->started))
3709                 return;
3710 
3711         if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3712                 return;
3713 
3714         if (cpumask_available(iter->started))
3715                 cpumask_set_cpu(iter->cpu, iter->started);
3716 
3717         /* Don't print started cpu buffer for the first entry of the trace */
3718         if (iter->idx > 1)
3719                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3720                                 iter->cpu);
3721 }
3722 
3723 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3724 {
3725         struct trace_array *tr = iter->tr;
3726         struct trace_seq *s = &iter->seq;
3727         unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3728         struct trace_entry *entry;
3729         struct trace_event *event;
3730 
3731         entry = iter->ent;
3732 
3733         test_cpu_buff_start(iter);
3734 
3735         event = ftrace_find_event(entry->type);
3736 
3737         if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3738                 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3739                         trace_print_lat_context(iter);
3740                 else
3741                         trace_print_context(iter);
3742         }
3743 
3744         if (trace_seq_has_overflowed(s))
3745                 return TRACE_TYPE_PARTIAL_LINE;
3746 
3747         if (event)
3748                 return event->funcs->trace(iter, sym_flags, event);
3749 
3750         trace_seq_printf(s, "Unknown type %d\n", entry->type);
3751 
3752         return trace_handle_return(s);
3753 }
3754 
3755 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3756 {
3757         struct trace_array *tr = iter->tr;
3758         struct trace_seq *s = &iter->seq;
3759         struct trace_entry *entry;
3760         struct trace_event *event;
3761 
3762         entry = iter->ent;
3763 
3764         if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3765                 trace_seq_printf(s, "%d %d %llu ",
3766                                  entry->pid, iter->cpu, iter->ts);
3767 
3768         if (trace_seq_has_overflowed(s))
3769                 return TRACE_TYPE_PARTIAL_LINE;
3770 
3771         event = ftrace_find_event(entry->type);
3772         if (event)
3773                 return event->funcs->raw(iter, 0, event);
3774 
3775         trace_seq_printf(s, "%d ?\n", entry->type);
3776 
3777         return trace_handle_return(s);
3778 }
3779 
3780 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3781 {
3782         struct trace_array *tr = iter->tr;
3783         struct trace_seq *s = &iter->seq;
3784         unsigned char newline = '\n';
3785         struct trace_entry *entry;
3786         struct trace_event *event;
3787 
3788         entry = iter->ent;
3789 
3790         if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3791                 SEQ_PUT_HEX_FIELD(s, entry->pid);
3792                 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3793                 SEQ_PUT_HEX_FIELD(s, iter->ts);
3794                 if (trace_seq_has_overflowed(s))
3795                         return TRACE_TYPE_PARTIAL_LINE;
3796         }
3797 
3798         event = ftrace_find_event(entry->type);
3799         if (event) {
3800                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3801                 if (ret != TRACE_TYPE_HANDLED)
3802                         return ret;
3803         }
3804 
3805         SEQ_PUT_FIELD(s, newline);
3806 
3807         return trace_handle_return(s);
3808 }
3809 
3810 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3811 {
3812         struct trace_array *tr = iter->tr;
3813         struct trace_seq *s = &iter->seq;
3814         struct trace_entry *entry;
3815         struct trace_event *event;
3816 
3817         entry = iter->ent;
3818 
3819         if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3820                 SEQ_PUT_FIELD(s, entry->pid);
3821                 SEQ_PUT_FIELD(s, iter->cpu);
3822                 SEQ_PUT_FIELD(s, iter->ts);
3823                 if (trace_seq_has_overflowed(s))
3824                         return TRACE_TYPE_PARTIAL_LINE;
3825         }
3826 
3827         event = ftrace_find_event(entry->type);
3828         return event ? event->funcs->binary(iter, 0, event) :
3829                 TRACE_TYPE_HANDLED;
3830 }
3831 
3832 int trace_empty(struct trace_iterator *iter)
3833 {
3834         struct ring_buffer_iter *buf_iter;
3835         int cpu;
3836 
3837         /* If we are looking at one CPU buffer, only check that one */
3838         if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3839                 cpu = iter->cpu_file;
3840                 buf_iter = trace_buffer_iter(iter, cpu);
3841                 if (buf_iter) {
3842                         if (!ring_buffer_iter_empty(buf_iter))
3843                                 return 0;
3844                 } else {
3845                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3846                                 return 0;
3847                 }
3848                 return 1;
3849         }
3850 
3851         for_each_tracing_cpu(cpu) {
3852                 buf_iter = trace_buffer_iter(iter, cpu);
3853                 if (buf_iter) {
3854                         if (!ring_buffer_iter_empty(buf_iter))
3855                                 return 0;
3856                 } else {
3857                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3858                                 return 0;
3859                 }
3860         }
3861 
3862         return 1;
3863 }
3864 
3865 /*  Called with trace_event_read_lock() held. */
3866 enum print_line_t print_trace_line(struct trace_iterator *iter)
3867 {
3868         struct trace_array *tr = iter->tr;
3869         unsigned long trace_flags = tr->trace_flags;
3870         enum print_line_t ret;
3871 
3872         if (iter->lost_events) {
3873                 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3874                                  iter->cpu, iter->lost_events);
3875                 if (trace_seq_has_overflowed(&iter->seq))
3876                         return TRACE_TYPE_PARTIAL_LINE;
3877         }
3878 
3879         if (iter->trace && iter->trace->print_line) {
3880                 ret = iter->trace->print_line(iter);
3881                 if (ret != TRACE_TYPE_UNHANDLED)
3882                         return ret;
3883         }
3884 
3885         if (iter->ent->type == TRACE_BPUTS &&
3886                         trace_flags & TRACE_ITER_PRINTK &&
3887                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3888                 return trace_print_bputs_msg_only(iter);
3889 
3890         if (iter->ent->type == TRACE_BPRINT &&
3891                         trace_flags & TRACE_ITER_PRINTK &&
3892                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3893                 return trace_print_bprintk_msg_only(iter);
3894 
3895         if (iter->ent->type == TRACE_PRINT &&
3896                         trace_flags & TRACE_ITER_PRINTK &&
3897                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3898                 return trace_print_printk_msg_only(iter);
3899 
3900         if (trace_flags & TRACE_ITER_BIN)
3901                 return print_bin_fmt(iter);
3902 
3903         if (trace_flags & TRACE_ITER_HEX)
3904                 return print_hex_fmt(iter);
3905 
3906         if (trace_flags & TRACE_ITER_RAW)
3907                 return print_raw_fmt(iter);
3908 
3909         return print_trace_fmt(iter);
3910 }
3911 
3912 void trace_latency_header(struct seq_file *m)
3913 {
3914         struct trace_iterator *iter = m->private;
3915         struct trace_array *tr = iter->tr;
3916 
3917         /* print nothing if the buffers are empty */
3918         if (trace_empty(iter))
3919                 return;
3920 
3921         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3922                 print_trace_header(m, iter);
3923 
3924         if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
3925                 print_lat_help_header(m);
3926 }
3927 
3928 void trace_default_header(struct seq_file *m)
3929 {
3930         struct trace_iterator *iter = m->private;
3931         struct trace_array *tr = iter->tr;
3932         unsigned long trace_flags = tr->trace_flags;
3933 
3934         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3935                 return;
3936 
3937         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3938                 /* print nothing if the buffers are empty */
3939                 if (trace_empty(iter))
3940                         return;
3941                 print_trace_header(m, iter);
3942                 if (!(trace_flags & TRACE_ITER_VERBOSE))
3943                         print_lat_help_header(m);
3944         } else {
3945                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3946                         if (trace_flags & TRACE_ITER_IRQ_INFO)
3947                                 print_func_help_header_irq(iter->trace_buffer,
3948                                                            m, trace_flags);
3949                         else
3950                                 print_func_help_header(iter->trace_buffer, m,
3951                                                        trace_flags);
3952                 }
3953         }
3954 }
3955 
3956 static void test_ftrace_alive(struct seq_file *m)
3957 {
3958         if (!ftrace_is_dead())
3959                 return;
3960         seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3961                     "#          MAY BE MISSING FUNCTION EVENTS\n");
3962 }
3963 
3964 #ifdef CONFIG_TRACER_MAX_TRACE
3965 static void show_snapshot_main_help(struct seq_file *m)
3966 {
3967         seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3968                     "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3969                     "#                      Takes a snapshot of the main buffer.\n"
3970                     "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3971                     "#                      (Doesn't have to be '2' works with any number that\n"
3972                     "#                       is not a '0' or '1')\n");
3973 }
3974 
3975 static void show_snapshot_percpu_help(struct seq_file *m)
3976 {
3977         seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3978 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3979         seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3980                     "#                      Takes a snapshot of the main buffer for this cpu.\n");
3981 #else
3982         seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3983                     "#                     Must use main snapshot file to allocate.\n");
3984 #endif
3985         seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3986                     "#                      (Doesn't have to be '2' works with any number that\n"
3987                     "#                       is not a '0' or '1')\n");
3988 }
3989 
3990 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3991 {
3992         if (iter->tr->allocated_snapshot)
3993                 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3994         else
3995                 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3996 
3997         seq_puts(m, "# Snapshot commands:\n");
3998         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3999                 show_snapshot_main_help(m);
4000         else
4001                 show_snapshot_percpu_help(m);
4002 }
4003 #else
4004 /* Should never be called */
4005 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4006 #endif
4007 
4008 static int s_show(struct seq_file *m, void *v)
4009 {
4010         struct trace_iterator *iter = v;
4011         int ret;
4012 
4013         if (iter->ent == NULL) {
4014                 if (iter->tr) {
4015                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
4016                         seq_puts(m, "#\n");
4017                         test_ftrace_alive(m);
4018                 }
4019                 if (iter->snapshot && trace_empty(iter))
4020                         print_snapshot_help(m, iter);
4021                 else if (iter->trace && iter->trace->print_header)
4022                         iter->trace->print_header(m);
4023                 else
4024                         trace_default_header(m);
4025 
4026         } else if (iter->leftover) {
4027                 /*
4028                  * If we filled the seq_file buffer earlier, we
4029                  * want to just show it now.
4030                  */
4031                 ret = trace_print_seq(m, &iter->seq);
4032 
4033                 /* ret should this time be zero, but you never know */
4034                 iter->leftover = ret;
4035 
4036         } else {
4037                 print_trace_line(iter);
4038                 ret = trace_print_seq(m, &iter->seq);
4039                 /*
4040                  * If we overflow the seq_file buffer, then it will
4041                  * ask us for this data again at start up.
4042                  * Use that instead.
4043                  *  ret is 0 if seq_file write succeeded.
4044                  *        -1 otherwise.
4045                  */
4046                 iter->leftover = ret;
4047         }
4048 
4049         return 0;
4050 }
4051 
4052 /*
4053  * Should be used after trace_array_get(), trace_types_lock
4054  * ensures that i_cdev was already initialized.
4055  */
4056 static inline int tracing_get_cpu(struct inode *inode)
4057 {
4058         if (inode->i_cdev) /* See trace_create_cpu_file() */
4059                 return (long)inode->i_cdev - 1;
4060         return RING_BUFFER_ALL_CPUS;
4061 }
4062 
4063 static const struct seq_operations tracer_seq_ops = {
4064         .start          = s_start,
4065         .next           = s_next,
4066         .stop           = s_stop,
4067         .show           = s_show,
4068 };
4069 
4070 static struct trace_iterator *
4071 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4072 {
4073         struct trace_array *tr = inode->i_private;
4074         struct trace_iterator *iter;
4075         int cpu;
4076 
4077         if (tracing_disabled)
4078                 return ERR_PTR(-ENODEV);
4079 
4080         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4081         if (!iter)
4082                 return ERR_PTR(-ENOMEM);
4083 
4084         iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4085                                     GFP_KERNEL);
4086         if (!iter->buffer_iter)
4087                 goto release;
4088 
4089         /*
4090          * We make a copy of the current tracer to avoid concurrent
4091          * changes on it while we are reading.
4092          */
4093         mutex_lock(&trace_types_lock);
4094         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4095         if (!iter->trace)
4096                 goto fail;
4097 
4098         *iter->trace = *tr->current_trace;
4099 
4100         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4101                 goto fail;
4102 
4103         iter->tr = tr;
4104 
4105 #ifdef CONFIG_TRACER_MAX_TRACE
4106         /* Currently only the top directory has a snapshot */
4107         if (tr->current_trace->print_max || snapshot)
4108                 iter->trace_buffer = &tr->max_buffer;
4109         else
4110 #endif
4111                 iter->trace_buffer = &tr->trace_buffer;
4112         iter->snapshot = snapshot;
4113         iter->pos = -1;
4114         iter->cpu_file = tracing_get_cpu(inode);
4115         mutex_init(&iter->mutex);
4116 
4117         /* Notify the tracer early; before we stop tracing. */
4118         if (iter->trace && iter->trace->open)
4119                 iter->trace->open(iter);
4120 
4121         /* Annotate start of buffers if we had overruns */
4122         if (ring_buffer_overruns(iter->trace_buffer->buffer))
4123                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4124 
4125         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4126         if (trace_clocks[tr->clock_id].in_ns)
4127                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4128 
4129         /* stop the trace while dumping if we are not opening "snapshot" */
4130         if (!iter->snapshot)
4131                 tracing_stop_tr(tr);
4132 
4133         if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4134                 for_each_tracing_cpu(cpu) {
4135                         iter->buffer_iter[cpu] =
4136                                 ring_buffer_read_prepare(iter->trace_buffer->buffer,
4137                                                          cpu, GFP_KERNEL);
4138                 }
4139                 ring_buffer_read_prepare_sync();
4140                 for_each_tracing_cpu(cpu) {
4141                         ring_buffer_read_start(iter->buffer_iter[cpu]);
4142                         tracing_iter_reset(iter, cpu);
4143                 }
4144         } else {
4145                 cpu = iter->cpu_file;
4146                 iter->buffer_iter[cpu] =
4147                         ring_buffer_read_prepare(iter->trace_buffer->buffer,
4148                                                  cpu, GFP_KERNEL);
4149                 ring_buffer_read_prepare_sync();
4150                 ring_buffer_read_start(iter->buffer_iter[cpu]);
4151                 tracing_iter_reset(iter, cpu);
4152         }
4153 
4154         mutex_unlock(&trace_types_lock);
4155 
4156         return iter;
4157 
4158  fail:
4159         mutex_unlock(&trace_types_lock);
4160         kfree(iter->trace);
4161         kfree(iter->buffer_iter);
4162 release:
4163         seq_release_private(inode, file);
4164         return ERR_PTR(-ENOMEM);
4165 }
4166 
4167 int tracing_open_generic(struct inode *inode, struct file *filp)
4168 {
4169         int ret;
4170 
4171         ret = tracing_check_open_get_tr(NULL);
4172         if (ret)
4173                 return ret;
4174 
4175         filp->private_data = inode->i_private;
4176         return 0;
4177 }
4178 
4179 bool tracing_is_disabled(void)
4180 {
4181         return (tracing_disabled) ? true: false;
4182 }
4183 
4184 /*
4185  * Open and update trace_array ref count.
4186  * Must have the current trace_array passed to it.
4187  */
4188 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4189 {
4190         struct trace_array *tr = inode->i_private;
4191         int ret;
4192 
4193         ret = tracing_check_open_get_tr(tr);
4194         if (ret)
4195                 return ret;
4196 
4197         filp->private_data = inode->i_private;
4198 
4199         return 0;
4200 }
4201 
4202 static int tracing_release(struct inode *inode, struct file *file)
4203 {
4204         struct trace_array *tr = inode->i_private;
4205         struct seq_file *m = file->private_data;
4206         struct trace_iterator *iter;
4207         int cpu;
4208 
4209         if (!(file->f_mode & FMODE_READ)) {
4210                 trace_array_put(tr);
4211                 return 0;
4212         }
4213 
4214         /* Writes do not use seq_file */
4215         iter = m->private;
4216         mutex_lock(&trace_types_lock);
4217 
4218         for_each_tracing_cpu(cpu) {
4219                 if (iter->buffer_iter[cpu])
4220                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
4221         }
4222 
4223         if (iter->trace && iter->trace->close)
4224                 iter->trace->close(iter);
4225 
4226         if (!iter->snapshot)
4227                 /* reenable tracing if it was previously enabled */
4228                 tracing_start_tr(tr);
4229 
4230         __trace_array_put(tr);
4231 
4232         mutex_unlock(&trace_types_lock);
4233 
4234         mutex_destroy(&iter->mutex);
4235         free_cpumask_var(iter->started);
4236         kfree(iter->trace);
4237         kfree(iter->buffer_iter);
4238         seq_release_private(inode, file);
4239 
4240         return 0;
4241 }
4242 
4243 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4244 {
4245         struct trace_array *tr = inode->i_private;
4246 
4247         trace_array_put(tr);
4248         return 0;
4249 }
4250 
4251 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4252 {
4253         struct trace_array *tr = inode->i_private;
4254 
4255         trace_array_put(tr);
4256 
4257         return single_release(inode, file);
4258 }
4259 
4260 static int tracing_open(struct inode *inode, struct file *file)
4261 {
4262         struct trace_array *tr = inode->i_private;
4263         struct trace_iterator *iter;
4264         int ret;
4265 
4266         ret = tracing_check_open_get_tr(tr);
4267         if (ret)
4268                 return ret;
4269 
4270         /* If this file was open for write, then erase contents */
4271         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4272                 int cpu = tracing_get_cpu(inode);
4273                 struct trace_buffer *trace_buf = &tr->trace_buffer;
4274 
4275 #ifdef CONFIG_TRACER_MAX_TRACE
4276                 if (tr->current_trace->print_max)
4277                         trace_buf = &tr->max_buffer;
4278 #endif
4279 
4280                 if (cpu == RING_BUFFER_ALL_CPUS)
4281                         tracing_reset_online_cpus(trace_buf);
4282                 else
4283                         tracing_reset_cpu(trace_buf, cpu);
4284         }
4285 
4286         if (file->f_mode & FMODE_READ) {
4287                 iter = __tracing_open(inode, file, false);
4288                 if (IS_ERR(iter))
4289                         ret = PTR_ERR(iter);
4290                 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4291                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
4292         }
4293 
4294         if (ret < 0)
4295                 trace_array_put(tr);
4296 
4297         return ret;
4298 }
4299 
4300 /*
4301  * Some tracers are not suitable for instance buffers.
4302  * A tracer is always available for the global array (toplevel)
4303  * or if it explicitly states that it is.
4304  */
4305 static bool
4306 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4307 {
4308         return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4309 }
4310 
4311 /* Find the next tracer that this trace array may use */
4312 static struct tracer *
4313 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4314 {
4315         while (t && !trace_ok_for_array(t, tr))
4316                 t = t->next;
4317 
4318         return t;
4319 }
4320 
4321 static void *
4322 t_next(struct seq_file *m, void *v, loff_t *pos)
4323 {
4324         struct trace_array *tr = m->private;
4325         struct tracer *t = v;
4326 
4327         (*pos)++;
4328 
4329         if (t)
4330                 t = get_tracer_for_array(tr, t->next);
4331 
4332         return t;
4333 }
4334 
4335 static void *t_start(struct seq_file *m, loff_t *pos)
4336 {
4337         struct trace_array *tr = m->private;
4338         struct tracer *t;
4339         loff_t l = 0;
4340 
4341         mutex_lock(&trace_types_lock);
4342 
4343         t = get_tracer_for_array(tr, trace_types);
4344         for (; t && l < *pos; t = t_next(m, t, &l))
4345                         ;
4346 
4347         return t;
4348 }
4349 
4350 static void t_stop(struct seq_file *m, void *p)
4351 {
4352         mutex_unlock(&trace_types_lock);
4353 }
4354 
4355 static int t_show(struct seq_file *m, void *v)
4356 {
4357         struct tracer *t = v;
4358 
4359         if (!t)
4360                 return 0;
4361 
4362         seq_puts(m, t->name);
4363         if (t->next)
4364                 seq_putc(m, ' ');
4365         else
4366                 seq_putc(m, '\n');
4367 
4368         return 0;
4369 }
4370 
4371 static const struct seq_operations show_traces_seq_ops = {
4372         .start          = t_start,
4373         .next           = t_next,
4374         .stop           = t_stop,
4375         .show           = t_show,
4376 };
4377 
4378 static int show_traces_open(struct inode *inode, struct file *file)
4379 {
4380         struct trace_array *tr = inode->i_private;
4381         struct seq_file *m;
4382         int ret;
4383 
4384         ret = tracing_check_open_get_tr(tr);
4385         if (ret)
4386                 return ret;
4387 
4388         ret = seq_open(file, &show_traces_seq_ops);
4389         if (ret) {
4390                 trace_array_put(tr);
4391                 return ret;
4392         }
4393 
4394         m = file->private_data;
4395         m->private = tr;
4396 
4397         return 0;
4398 }
4399 
4400 static int show_traces_release(struct inode *inode, struct file *file)
4401 {
4402         struct trace_array *tr = inode->i_private;
4403 
4404         trace_array_put(tr);
4405         return seq_release(inode, file);
4406 }
4407 
4408 static ssize_t
4409 tracing_write_stub(struct file *filp, const char __user *ubuf,
4410                    size_t count, loff_t *ppos)
4411 {
4412         return count;
4413 }
4414 
4415 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4416 {
4417         int ret;
4418 
4419         if (file->f_mode & FMODE_READ)
4420                 ret = seq_lseek(file, offset, whence);
4421         else
4422                 file->f_pos = ret = 0;
4423 
4424         return ret;
4425 }
4426 
4427 static const struct file_operations tracing_fops = {
4428         .open           = tracing_open,
4429         .read           = seq_read,
4430         .write          = tracing_write_stub,
4431         .llseek         = tracing_lseek,
4432         .release        = tracing_release,
4433 };
4434 
4435 static const struct file_operations show_traces_fops = {
4436         .open           = show_traces_open,
4437         .read           = seq_read,
4438         .llseek         = seq_lseek,
4439         .release        = show_traces_release,
4440 };
4441 
4442 static ssize_t
4443 tracing_cpumask_read(struct file *filp, char __user *ubuf,
4444                      size_t count, loff_t *ppos)
4445 {
4446         struct trace_array *tr = file_inode(filp)->i_private;
4447         char *mask_str;
4448         int len;
4449 
4450         len = snprintf(NULL, 0, "%*pb\n",
4451                        cpumask_pr_args(tr->tracing_cpumask)) + 1;
4452         mask_str = kmalloc(len, GFP_KERNEL);
4453         if (!mask_str)
4454                 return -ENOMEM;
4455 
4456         len = snprintf(mask_str, len, "%*pb\n",
4457                        cpumask_pr_args(tr->tracing_cpumask));
4458         if (len >= count) {
4459                 count = -EINVAL;
4460                 goto out_err;
4461         }
4462         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4463 
4464 out_err:
4465         kfree(mask_str);
4466 
4467         return count;
4468 }
4469 
4470 static ssize_t
4471 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4472                       size_t count, loff_t *ppos)
4473 {
4474         struct trace_array *tr = file_inode(filp)->i_private;
4475         cpumask_var_t tracing_cpumask_new;
4476         int err, cpu;
4477 
4478         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4479                 return -ENOMEM;
4480 
4481         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4482         if (err)
4483                 goto err_unlock;
4484 
4485         local_irq_disable();
4486         arch_spin_lock(&tr->max_lock);
4487         for_each_tracing_cpu(cpu) {
4488                 /*
4489                  * Increase/decrease the disabled counter if we are
4490                  * about to flip a bit in the cpumask:
4491                  */
4492                 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4493                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4494                         atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4495                         ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
4496                 }
4497                 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4498                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4499                         atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4500                         ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
4501                 }
4502         }
4503         arch_spin_unlock(&tr->max_lock);
4504         local_irq_enable();
4505 
4506         cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4507         free_cpumask_var(tracing_cpumask_new);
4508 
4509         return count;
4510 
4511 err_unlock:
4512         free_cpumask_var(tracing_cpumask_new);
4513 
4514         return err;
4515 }
4516 
4517 static const struct file_operations tracing_cpumask_fops = {
4518         .open           = tracing_open_generic_tr,
4519         .read           = tracing_cpumask_read,
4520         .write          = tracing_cpumask_write,
4521         .release        = tracing_release_generic_tr,
4522         .llseek         = generic_file_llseek,
4523 };
4524 
4525 static int tracing_trace_options_show(struct seq_file *m, void *v)
4526 {
4527         struct tracer_opt *trace_opts;
4528         struct trace_array *tr = m->private;
4529         u32 tracer_flags;
4530         int i;
4531 
4532         mutex_lock(&trace_types_lock);
4533         tracer_flags = tr->current_trace->flags->val;
4534         trace_opts = tr->current_trace->flags->opts;
4535 
4536         for (i = 0; trace_options[i]; i++) {
4537                 if (tr->trace_flags & (1 << i))
4538                         seq_printf(m, "%s\n", trace_options[i]);
4539                 else
4540                         seq_printf(m, "no%s\n", trace_options[i]);
4541         }
4542 
4543         for (i = 0; trace_opts[i].name; i++) {
4544                 if (tracer_flags & trace_opts[i].bit)
4545                         seq_printf(m, "%s\n", trace_opts[i].name);
4546                 else
4547                         seq_printf(m, "no%s\n", trace_opts[i].name);
4548         }
4549         mutex_unlock(&trace_types_lock);
4550 
4551         return 0;
4552 }
4553 
4554 static int __set_tracer_option(struct trace_array *tr,
4555                                struct tracer_flags *tracer_flags,
4556                                struct tracer_opt *opts, int neg)
4557 {
4558         struct tracer *trace = tracer_flags->trace;
4559         int ret;
4560 
4561         ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4562         if (ret)
4563                 return ret;
4564 
4565         if (neg)
4566                 tracer_flags->val &= ~opts->bit;
4567         else
4568                 tracer_flags->val |= opts->bit;
4569         return 0;
4570 }
4571 
4572 /* Try to assign a tracer specific option */
4573 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4574 {
4575         struct tracer *trace = tr->current_trace;
4576         struct tracer_flags *tracer_flags = trace->flags;
4577         struct tracer_opt *opts = NULL;
4578         int i;
4579 
4580         for (i = 0; tracer_flags->opts[i].name; i++) {
4581                 opts = &tracer_flags->opts[i];
4582 
4583                 if (strcmp(cmp, opts->name) == 0)
4584                         return __set_tracer_option(tr, trace->flags, opts, neg);
4585         }
4586 
4587         return -EINVAL;
4588 }
4589 
4590 /* Some tracers require overwrite to stay enabled */
4591 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4592 {
4593         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4594                 return -1;
4595 
4596         return 0;
4597 }
4598 
4599 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4600 {
4601         if ((mask == TRACE_ITER_RECORD_TGID) ||
4602             (mask == TRACE_ITER_RECORD_CMD))
4603                 lockdep_assert_held(&event_mutex);
4604 
4605         /* do nothing if flag is already set */
4606         if (!!(tr->trace_flags & mask) == !!enabled)
4607                 return 0;
4608 
4609         /* Give the tracer a chance to approve the change */
4610         if (tr->current_trace->flag_changed)
4611                 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4612                         return -EINVAL;
4613 
4614         if (enabled)
4615                 tr->trace_flags |= mask;
4616         else
4617                 tr->trace_flags &= ~mask;
4618 
4619         if (mask == TRACE_ITER_RECORD_CMD)
4620                 trace_event_enable_cmd_record(enabled);
4621 
4622         if (mask == TRACE_ITER_RECORD_TGID) {
4623                 if (!tgid_map)
4624                         tgid_map = kvcalloc(PID_MAX_DEFAULT + 1,
4625                                            sizeof(*tgid_map),
4626                                            GFP_KERNEL);
4627                 if (!tgid_map) {
4628                         tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4629                         return -ENOMEM;
4630                 }
4631 
4632                 trace_event_enable_tgid_record(enabled);
4633         }
4634 
4635         if (mask == TRACE_ITER_EVENT_FORK)
4636                 trace_event_follow_fork(tr, enabled);
4637 
4638         if (mask == TRACE_ITER_FUNC_FORK)
4639                 ftrace_pid_follow_fork(tr, enabled);
4640 
4641         if (mask == TRACE_ITER_OVERWRITE) {
4642                 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
4643 #ifdef CONFIG_TRACER_MAX_TRACE
4644                 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4645 #endif
4646         }
4647 
4648         if (mask == TRACE_ITER_PRINTK) {
4649                 trace_printk_start_stop_comm(enabled);
4650                 trace_printk_control(enabled);
4651         }
4652 
4653         return 0;
4654 }
4655 
4656 static int trace_set_options(struct trace_array *tr, char *option)
4657 {
4658         char *cmp;
4659         int neg = 0;
4660         int ret;
4661         size_t orig_len = strlen(option);
4662         int len;
4663 
4664         cmp = strstrip(option);
4665 
4666         len = str_has_prefix(cmp, "no");
4667         if (len)
4668                 neg = 1;
4669 
4670         cmp += len;
4671 
4672         mutex_lock(&event_mutex);
4673         mutex_lock(&trace_types_lock);
4674 
4675         ret = match_string(trace_options, -1, cmp);
4676         /* If no option could be set, test the specific tracer options */
4677         if (ret < 0)
4678                 ret = set_tracer_option(tr, cmp, neg);
4679         else
4680                 ret = set_tracer_flag(tr, 1 << ret, !neg);
4681 
4682         mutex_unlock(&trace_types_lock);
4683         mutex_unlock(&event_mutex);
4684 
4685         /*
4686          * If the first trailing whitespace is replaced with '\0' by strstrip,
4687          * turn it back into a space.
4688          */
4689         if (orig_len > strlen(option))
4690                 option[strlen(option)] = ' ';
4691 
4692         return ret;
4693 }
4694 
4695 static void __init apply_trace_boot_options(void)
4696 {
4697         char *buf = trace_boot_options_buf;
4698         char *option;
4699 
4700         while (true) {
4701                 option = strsep(&buf, ",");
4702 
4703                 if (!option)
4704                         break;
4705 
4706                 if (*option)
4707                         trace_set_options(&global_trace, option);
4708 
4709                 /* Put back the comma to allow this to be called again */
4710                 if (buf)
4711                         *(buf - 1) = ',';
4712         }
4713 }
4714 
4715 static ssize_t
4716 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4717                         size_t cnt, loff_t *ppos)
4718 {
4719         struct seq_file *m = filp->private_data;
4720         struct trace_array *tr = m->private;
4721         char buf[64];
4722         int ret;
4723 
4724         if (cnt >= sizeof(buf))
4725                 return -EINVAL;
4726 
4727         if (copy_from_user(buf, ubuf, cnt))
4728                 return -EFAULT;
4729 
4730         buf[cnt] = 0;
4731 
4732         ret = trace_set_options(tr, buf);
4733         if (ret < 0)
4734                 return ret;
4735 
4736         *ppos += cnt;
4737 
4738         return cnt;
4739 }
4740 
4741 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4742 {
4743         struct trace_array *tr = inode->i_private;
4744         int ret;
4745 
4746         ret = tracing_check_open_get_tr(tr);
4747         if (ret)
4748                 return ret;
4749 
4750         ret = single_open(file, tracing_trace_options_show, inode->i_private);
4751         if (ret < 0)
4752                 trace_array_put(tr);
4753 
4754         return ret;
4755 }
4756 
4757 static const struct file_operations tracing_iter_fops = {
4758         .open           = tracing_trace_options_open,
4759         .read           = seq_read,
4760         .llseek         = seq_lseek,
4761         .release        = tracing_single_release_tr,
4762         .write          = tracing_trace_options_write,
4763 };
4764 
4765 static const char readme_msg[] =
4766         "tracing mini-HOWTO:\n\n"
4767         "# echo 0 > tracing_on : quick way to disable tracing\n"
4768         "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4769         " Important files:\n"
4770         "  trace\t\t\t- The static contents of the buffer\n"
4771         "\t\t\t  To clear the buffer write into this file: echo > trace\n"
4772         "  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4773         "  current_tracer\t- function and latency tracers\n"
4774         "  available_tracers\t- list of configured tracers for current_tracer\n"
4775         "  error_log\t- error log for failed commands (that support it)\n"
4776         "  buffer_size_kb\t- view and modify size of per cpu buffer\n"
4777         "  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
4778         "  trace_clock\t\t-change the clock used to order events\n"
4779         "       local:   Per cpu clock but may not be synced across CPUs\n"
4780         "      global:   Synced across CPUs but slows tracing down.\n"
4781         "     counter:   Not a clock, but just an increment\n"
4782         "      uptime:   Jiffy counter from time of boot\n"
4783         "        perf:   Same clock that perf events use\n"
4784 #ifdef CONFIG_X86_64
4785         "     x86-tsc:   TSC cycle counter\n"
4786 #endif
4787         "\n  timestamp_mode\t-view the mode used to timestamp events\n"
4788         "       delta:   Delta difference against a buffer-wide timestamp\n"
4789         "    absolute:   Absolute (standalone) timestamp\n"
4790         "\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4791         "\n  trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4792         "  tracing_cpumask\t- Limit which CPUs to trace\n"
4793         "  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4794         "\t\t\t  Remove sub-buffer with rmdir\n"
4795         "  trace_options\t\t- Set format or modify how tracing happens\n"
4796         "\t\t\t  Disable an option by prefixing 'no' to the\n"
4797         "\t\t\t  option name\n"
4798         "  saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4799 #ifdef CONFIG_DYNAMIC_FTRACE
4800         "\n  available_filter_functions - list of functions that can be filtered on\n"
4801         "  set_ftrace_filter\t- echo function name in here to only trace these\n"
4802         "\t\t\t  functions\n"
4803         "\t     accepts: func_full_name or glob-matching-pattern\n"
4804         "\t     modules: Can select a group via module\n"
4805         "\t      Format: :mod:<module-name>\n"
4806         "\t     example: echo :mod:ext3 > set_ftrace_filter\n"
4807         "\t    triggers: a command to perform when function is hit\n"
4808         "\t      Format: <function>:<trigger>[:count]\n"
4809         "\t     trigger: traceon, traceoff\n"
4810         "\t\t      enable_event:<system>:<event>\n"
4811         "\t\t      disable_event:<system>:<event>\n"
4812 #ifdef CONFIG_STACKTRACE
4813         "\t\t      stacktrace\n"
4814 #endif
4815 #ifdef CONFIG_TRACER_SNAPSHOT
4816         "\t\t      snapshot\n"
4817 #endif
4818         "\t\t      dump\n"
4819         "\t\t      cpudump\n"
4820         "\t     example: echo do_fault:traceoff > set_ftrace_filter\n"
4821         "\t              echo do_trap:traceoff:3 > set_ftrace_filter\n"
4822         "\t     The first one will disable tracing every time do_fault is hit\n"
4823         "\t     The second will disable tracing at most 3 times when do_trap is hit\n"
4824         "\t       The first time do trap is hit and it disables tracing, the\n"
4825         "\t       counter will decrement to 2. If tracing is already disabled,\n"
4826         "\t       the counter will not decrement. It only decrements when the\n"
4827         "\t       trigger did work\n"
4828         "\t     To remove trigger without count:\n"
4829         "\t       echo '!<function>:<trigger> > set_ftrace_filter\n"
4830         "\t     To remove trigger with a count:\n"
4831         "\t       echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4832         "  set_ftrace_notrace\t- echo function name in here to never trace.\n"
4833         "\t    accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4834         "\t    modules: Can select a group via module command :mod:\n"
4835         "\t    Does not accept triggers\n"
4836 #endif /* CONFIG_DYNAMIC_FTRACE */
4837 #ifdef CONFIG_FUNCTION_TRACER
4838         "  set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4839         "\t\t    (function)\n"
4840 #endif
4841 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4842         "  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4843         "  set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4844         "  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4845 #endif
4846 #ifdef CONFIG_TRACER_SNAPSHOT
4847         "\n  snapshot\t\t- Like 'trace' but shows the content of the static\n"
4848         "\t\t\t  snapshot buffer. Read the contents for more\n"
4849         "\t\t\t  information\n"
4850 #endif
4851 #ifdef CONFIG_STACK_TRACER
4852         "  stack_trace\t\t- Shows the max stack trace when active\n"
4853         "  stack_max_size\t- Shows current max stack size that was traced\n"
4854         "\t\t\t  Write into this file to reset the max size (trigger a\n"
4855         "\t\t\t  new trace)\n"
4856 #ifdef CONFIG_DYNAMIC_FTRACE
4857         "  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4858         "\t\t\t  traces\n"
4859 #endif
4860 #endif /* CONFIG_STACK_TRACER */
4861 #ifdef CONFIG_DYNAMIC_EVENTS
4862         "  dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
4863         "\t\t\t  Write into this file to define/undefine new trace events.\n"
4864 #endif
4865 #ifdef CONFIG_KPROBE_EVENTS
4866         "  kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
4867         "\t\t\t  Write into this file to define/undefine new trace events.\n"
4868 #endif
4869 #ifdef CONFIG_UPROBE_EVENTS
4870         "  uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
4871         "\t\t\t  Write into this file to define/undefine new trace events.\n"
4872 #endif
4873 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4874         "\t  accepts: event-definitions (one definition per line)\n"
4875         "\t   Format: p[:[<group>/]<event>] <place> [<args>]\n"
4876         "\t           r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
4877 #ifdef CONFIG_HIST_TRIGGERS
4878         "\t           s:[synthetic/]<event> <field> [<field>]\n"
4879 #endif
4880         "\t           -:[<group>/]<event>\n"
4881 #ifdef CONFIG_KPROBE_EVENTS
4882         "\t    place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4883   "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4884 #endif
4885 #ifdef CONFIG_UPROBE_EVENTS
4886   "   place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
4887 #endif
4888         "\t     args: <name>=fetcharg[:type]\n"
4889         "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4890 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4891         "\t           $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
4892 #else
4893         "\t           $stack<index>, $stack, $retval, $comm,\n"
4894 #endif
4895         "\t           +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
4896         "\t     type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
4897         "\t           b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
4898         "\t           <type>\\[<array-size>\\]\n"
4899 #ifdef CONFIG_HIST_TRIGGERS
4900         "\t    field: <stype> <name>;\n"
4901         "\t    stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4902         "\t           [unsigned] char/int/long\n"
4903 #endif
4904 #endif
4905         "  events/\t\t- Directory containing all trace event subsystems:\n"
4906         "      enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4907         "  events/<system>/\t- Directory containing all trace events for <system>:\n"
4908         "      enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4909         "\t\t\t  events\n"
4910         "      filter\t\t- If set, only events passing filter are traced\n"
4911         "  events/<system>/<event>/\t- Directory containing control files for\n"
4912         "\t\t\t  <event>:\n"
4913         "      enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4914         "      filter\t\t- If set, only events passing filter are traced\n"
4915         "      trigger\t\t- If set, a command to perform when event is hit\n"
4916         "\t    Format: <trigger>[:count][if <filter>]\n"
4917         "\t   trigger: traceon, traceoff\n"
4918         "\t            enable_event:<system>:<event>\n"
4919         "\t            disable_event:<system>:<event>\n"
4920 #ifdef CONFIG_HIST_TRIGGERS
4921         "\t            enable_hist:<system>:<event>\n"
4922         "\t            disable_hist:<system>:<event>\n"
4923 #endif
4924 #ifdef CONFIG_STACKTRACE
4925         "\t\t    stacktrace\n"
4926 #endif
4927 #ifdef CONFIG_TRACER_SNAPSHOT
4928         "\t\t    snapshot\n"
4929 #endif
4930 #ifdef CONFIG_HIST_TRIGGERS
4931         "\t\t    hist (see below)\n"
4932 #endif
4933         "\t   example: echo traceoff > events/block/block_unplug/trigger\n"
4934         "\t            echo traceoff:3 > events/block/block_unplug/trigger\n"
4935         "\t            echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4936         "\t                  events/block/block_unplug/trigger\n"
4937         "\t   The first disables tracing every time block_unplug is hit.\n"
4938         "\t   The second disables tracing the first 3 times block_unplug is hit.\n"
4939         "\t   The third enables the kmalloc event the first 3 times block_unplug\n"
4940         "\t     is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4941         "\t   Like function triggers, the counter is only decremented if it\n"
4942         "\t    enabled or disabled tracing.\n"
4943         "\t   To remove a trigger without a count:\n"
4944         "\t     echo '!<trigger> > <system>/<event>/trigger\n"
4945         "\t   To remove a trigger with a count:\n"
4946         "\t     echo '!<trigger>:0 > <system>/<event>/trigger\n"
4947         "\t   Filters can be ignored when removing a trigger.\n"
4948 #ifdef CONFIG_HIST_TRIGGERS
4949         "      hist trigger\t- If set, event hits are aggregated into a hash table\n"
4950         "\t    Format: hist:keys=<field1[,field2,...]>\n"
4951         "\t            [:values=<field1[,field2,...]>]\n"
4952         "\t            [:sort=<field1[,field2,...]>]\n"
4953         "\t            [:size=#entries]\n"
4954         "\t            [:pause][:continue][:clear]\n"
4955         "\t            [:name=histname1]\n"
4956         "\t            [:<handler>.<action>]\n"
4957         "\t            [if <filter>]\n\n"
4958         "\t    When a matching event is hit, an entry is added to a hash\n"
4959         "\t    table using the key(s) and value(s) named, and the value of a\n"
4960         "\t    sum called 'hitcount' is incremented.  Keys and values\n"
4961         "\t    correspond to fields in the event's format description.  Keys\n"
4962         "\t    can be any field, or the special string 'stacktrace'.\n"
4963         "\t    Compound keys consisting of up to two fields can be specified\n"
4964         "\t    by the 'keys' keyword.  Values must correspond to numeric\n"
4965         "\t    fields.  Sort keys consisting of up to two fields can be\n"
4966         "\t    specified using the 'sort' keyword.  The sort direction can\n"
4967         "\t    be modified by appending '.descending' or '.ascending' to a\n"
4968         "\t    sort field.  The 'size' parameter can be used to specify more\n"
4969         "\t    or fewer than the default 2048 entries for the hashtable size.\n"
4970         "\t    If a hist trigger is given a name using the 'name' parameter,\n"
4971         "\t    its histogram data will be shared with other triggers of the\n"
4972         "\t    same name, and trigger hits will update this common data.\n\n"
4973         "\t    Reading the 'hist' file for the event will dump the hash\n"
4974         "\t    table in its entirety to stdout.  If there are multiple hist\n"
4975         "\t    triggers attached to an event, there will be a table for each\n"
4976         "\t    trigger in the output.  The table displayed for a named\n"
4977         "\t    trigger will be the same as any other instance having the\n"
4978         "\t    same name.  The default format used to display a given field\n"
4979         "\t    can be modified by appending any of the following modifiers\n"
4980         "\t    to the field name, as applicable:\n\n"
4981         "\t            .hex        display a number as a hex value\n"
4982         "\t            .sym        display an address as a symbol\n"
4983         "\t            .sym-offset display an address as a symbol and offset\n"
4984         "\t            .execname   display a common_pid as a program name\n"
4985         "\t            .syscall    display a syscall id as a syscall name\n"
4986         "\t            .log2       display log2 value rather than raw number\n"
4987         "\t            .usecs      display a common_timestamp in microseconds\n\n"
4988         "\t    The 'pause' parameter can be used to pause an existing hist\n"
4989         "\t    trigger or to start a hist trigger but not log any events\n"
4990         "\t    until told to do so.  'continue' can be used to start or\n"
4991         "\t    restart a paused hist trigger.\n\n"
4992         "\t    The 'clear' parameter will clear the contents of a running\n"
4993         "\t    hist trigger and leave its current paused/active state\n"
4994         "\t    unchanged.\n\n"
4995         "\t    The enable_hist and disable_hist triggers can be used to\n"
4996         "\t    have one event conditionally start and stop another event's\n"
4997         "\t    already-attached hist trigger.  The syntax is analogous to\n"
4998         "\t    the enable_event and disable_event triggers.\n\n"
4999         "\t    Hist trigger handlers and actions are executed whenever a\n"
5000         "\t    a histogram entry is added or updated.  They take the form:\n\n"
5001         "\t        <handler>.<action>\n\n"
5002         "\t    The available handlers are:\n\n"
5003         "\t        onmatch(matching.event)  - invoke on addition or update\n"
5004         "\t        onmax(var)               - invoke if var exceeds current max\n"
5005         "\t        onchange(var)            - invoke action if var changes\n\n"
5006         "\t    The available actions are:\n\n"
5007         "\t        trace(<synthetic_event>,param list)  - generate synthetic event\n"
5008         "\t        save(field,...)                      - save current event fields\n"
5009 #ifdef CONFIG_TRACER_SNAPSHOT
5010         "\t        snapshot()                           - snapshot the trace buffer\n"
5011 #endif
5012 #endif
5013 ;
5014 
5015 static ssize_t
5016 tracing_readme_read(struct file *filp, char __user *ubuf,
5017                        size_t cnt, loff_t *ppos)
5018 {
5019         return simple_read_from_buffer(ubuf, cnt, ppos,
5020                                         readme_msg, strlen(readme_msg));
5021 }
5022 
5023 static const struct file_operations tracing_readme_fops = {
5024         .open           = tracing_open_generic,
5025         .read           = tracing_readme_read,
5026         .llseek         = generic_file_llseek,
5027 };
5028 
5029 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5030 {
5031         int *ptr = v;
5032 
5033         if (*pos || m->count)
5034                 ptr++;
5035 
5036         (*pos)++;
5037 
5038         for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
5039                 if (trace_find_tgid(*ptr))
5040                         return ptr;
5041         }
5042 
5043         return NULL;
5044 }
5045 
5046 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5047 {
5048         void *v;
5049         loff_t l = 0;
5050 
5051         if (!tgid_map)
5052                 return NULL;
5053 
5054         v = &tgid_map[0];
5055         while (l <= *pos) {
5056                 v = saved_tgids_next(m, v, &l);
5057                 if (!v)
5058                         return NULL;
5059         }
5060 
5061         return v;
5062 }
5063 
5064 static void saved_tgids_stop(struct seq_file *m, void *v)
5065 {
5066 }
5067 
5068 static int saved_tgids_show(struct seq_file *m, void *v)
5069 {
5070         int pid = (int *)v - tgid_map;
5071 
5072         seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5073         return 0;
5074 }
5075 
5076 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5077         .start          = saved_tgids_start,
5078         .stop           = saved_tgids_stop,
5079         .next           = saved_tgids_next,
5080         .show           = saved_tgids_show,
5081 };
5082 
5083 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5084 {
5085         int ret;
5086 
5087         ret = tracing_check_open_get_tr(NULL);
5088         if (ret)
5089                 return ret;
5090 
5091         return seq_open(filp, &tracing_saved_tgids_seq_ops);
5092 }
5093 
5094 
5095 static const struct file_operations tracing_saved_tgids_fops = {
5096         .open           = tracing_saved_tgids_open,
5097         .read           = seq_read,
5098         .llseek         = seq_lseek,
5099         .release        = seq_release,
5100 };
5101 
5102 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5103 {
5104         unsigned int *ptr = v;
5105 
5106         if (*pos || m->count)
5107                 ptr++;
5108 
5109         (*pos)++;
5110 
5111         for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5112              ptr++) {
5113                 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5114                         continue;
5115 
5116                 return ptr;
5117         }
5118 
5119         return NULL;
5120 }
5121 
5122 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5123 {
5124         void *v;
5125         loff_t l = 0;
5126 
5127         preempt_disable();
5128         arch_spin_lock(&trace_cmdline_lock);
5129 
5130         v = &savedcmd->map_cmdline_to_pid[0];
5131         while (l <= *pos) {
5132                 v = saved_cmdlines_next(m, v, &l);
5133                 if (!v)
5134                         return NULL;
5135         }
5136 
5137         return v;
5138 }
5139 
5140 static void saved_cmdlines_stop(struct seq_file *m, void *v)
5141 {
5142         arch_spin_unlock(&trace_cmdline_lock);
5143         preempt_enable();
5144 }
5145 
5146 static int saved_cmdlines_show(struct seq_file *m, void *v)
5147 {
5148         char buf[TASK_COMM_LEN];
5149         unsigned int *pid = v;
5150 
5151         __trace_find_cmdline(*pid, buf);
5152         seq_printf(m, "%d %s\n", *pid, buf);
5153         return 0;
5154 }
5155 
5156 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5157         .start          = saved_cmdlines_start,
5158         .next           = saved_cmdlines_next,
5159         .stop           = saved_cmdlines_stop,
5160         .show           = saved_cmdlines_show,
5161 };
5162 
5163 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5164 {
5165         int ret;
5166 
5167         ret = tracing_check_open_get_tr(NULL);
5168         if (ret)
5169                 return ret;
5170 
5171         return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5172 }
5173 
5174 static const struct file_operations tracing_saved_cmdlines_fops = {
5175         .open           = tracing_saved_cmdlines_open,
5176         .read           = seq_read,
5177         .llseek         = seq_lseek,
5178         .release        = seq_release,
5179 };
5180 
5181 static ssize_t
5182 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5183                                  size_t cnt, loff_t *ppos)
5184 {
5185         char buf[64];
5186         int r;
5187 
5188         arch_spin_lock(&trace_cmdline_lock);
5189         r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5190         arch_spin_unlock(&trace_cmdline_lock);
5191 
5192         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5193 }
5194 
5195 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5196 {
5197         kfree(s->saved_cmdlines);
5198         kfree(s->map_cmdline_to_pid);
5199         kfree(s);
5200 }
5201 
5202 static int tracing_resize_saved_cmdlines(unsigned int val)
5203 {
5204         struct saved_cmdlines_buffer *s, *savedcmd_temp;
5205 
5206         s = kmalloc(sizeof(*s), GFP_KERNEL);
5207         if (!s)
5208                 return -ENOMEM;
5209 
5210         if (allocate_cmdlines_buffer(val, s) < 0) {
5211                 kfree(s);
5212                 return -ENOMEM;
5213         }
5214 
5215         arch_spin_lock(&trace_cmdline_lock);
5216         savedcmd_temp = savedcmd;
5217         savedcmd = s;
5218         arch_spin_unlock(&trace_cmdline_lock);
5219         free_saved_cmdlines_buffer(savedcmd_temp);
5220 
5221         return 0;
5222 }
5223 
5224 static ssize_t
5225 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5226                                   size_t cnt, loff_t *ppos)
5227 {
5228         unsigned long val;
5229         int ret;
5230 
5231         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5232         if (ret)
5233                 return ret;
5234 
5235         /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5236         if (!val || val > PID_MAX_DEFAULT)
5237                 return -EINVAL;
5238 
5239         ret = tracing_resize_saved_cmdlines((unsigned int)val);
5240         if (ret < 0)
5241                 return ret;
5242 
5243         *ppos += cnt;
5244 
5245         return cnt;
5246 }
5247 
5248 static const struct file_operations tracing_saved_cmdlines_size_fops = {
5249         .open           = tracing_open_generic,
5250         .read           = tracing_saved_cmdlines_size_read,
5251         .write          = tracing_saved_cmdlines_size_write,
5252 };
5253 
5254 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5255 static union trace_eval_map_item *
5256 update_eval_map(union trace_eval_map_item *ptr)
5257 {
5258         if (!ptr->map.eval_string) {
5259                 if (ptr->tail.next) {
5260                         ptr = ptr->tail.next;
5261                         /* Set ptr to the next real item (skip head) */
5262                         ptr++;
5263                 } else
5264                         return NULL;
5265         }
5266         return ptr;
5267 }
5268 
5269 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5270 {
5271         union trace_eval_map_item *ptr = v;
5272 
5273         /*
5274          * Paranoid! If ptr points to end, we don't want to increment past it.
5275          * This really should never happen.
5276          */
5277         ptr = update_eval_map(ptr);
5278         if (WARN_ON_ONCE(!ptr))
5279                 return NULL;
5280 
5281         ptr++;
5282 
5283         (*pos)++;
5284 
5285         ptr = update_eval_map(ptr);
5286 
5287         return ptr;
5288 }
5289 
5290 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5291 {
5292         union trace_eval_map_item *v;
5293         loff_t l = 0;
5294 
5295         mutex_lock(&trace_eval_mutex);
5296 
5297         v = trace_eval_maps;
5298         if (v)
5299                 v++;
5300 
5301         while (v && l < *pos) {
5302                 v = eval_map_next(m, v, &l);
5303         }
5304 
5305         return v;
5306 }
5307 
5308 static void eval_map_stop(struct seq_file *m, void *v)
5309 {
5310         mutex_unlock(&trace_eval_mutex);
5311 }
5312 
5313 static int eval_map_show(struct seq_file *m, void *v)
5314 {
5315         union trace_eval_map_item *ptr = v;
5316 
5317         seq_printf(m, "%s %ld (%s)\n",
5318                    ptr->map.eval_string, ptr->map.eval_value,
5319                    ptr->map.system);
5320 
5321         return 0;
5322 }
5323 
5324 static const struct seq_operations tracing_eval_map_seq_ops = {
5325         .start          = eval_map_start,
5326         .next           = eval_map_next,
5327         .stop           = eval_map_stop,
5328         .show           = eval_map_show,
5329 };
5330 
5331 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5332 {
5333         int ret;
5334 
5335         ret = tracing_check_open_get_tr(NULL);
5336         if (ret)
5337                 return ret;
5338 
5339         return seq_open(filp, &tracing_eval_map_seq_ops);
5340 }
5341 
5342 static const struct file_operations tracing_eval_map_fops = {
5343         .open           = tracing_eval_map_open,
5344         .read           = seq_read,
5345         .llseek         = seq_lseek,
5346         .release        = seq_release,
5347 };
5348 
5349 static inline union trace_eval_map_item *
5350 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5351 {
5352         /* Return tail of array given the head */
5353         return ptr + ptr->head.length + 1;
5354 }
5355 
5356 static void
5357 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5358                            int len)
5359 {
5360         struct trace_eval_map **stop;
5361         struct trace_eval_map **map;
5362         union trace_eval_map_item *map_array;
5363         union trace_eval_map_item *ptr;
5364 
5365         stop = start + len;
5366 
5367         /*
5368          * The trace_eval_maps contains the map plus a head and tail item,
5369          * where the head holds the module and length of array, and the
5370          * tail holds a pointer to the next list.
5371          */
5372         map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5373         if (!map_array) {
5374                 pr_warn("Unable to allocate trace eval mapping\n");
5375                 return;
5376         }
5377 
5378         mutex_lock(&trace_eval_mutex);
5379 
5380         if (!trace_eval_maps)
5381                 trace_eval_maps = map_array;
5382         else {
5383                 ptr = trace_eval_maps;
5384                 for (;;) {
5385                         ptr = trace_eval_jmp_to_tail(ptr);
5386                         if (!ptr->tail.next)
5387                                 break;
5388                         ptr = ptr->tail.next;
5389 
5390                 }
5391                 ptr->tail.next = map_array;
5392         }
5393         map_array->head.mod = mod;
5394         map_array->head.length = len;
5395         map_array++;
5396 
5397         for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5398                 map_array->map = **map;
5399                 map_array++;
5400         }
5401         memset(map_array, 0, sizeof(*map_array));
5402 
5403         mutex_unlock(&trace_eval_mutex);
5404 }
5405 
5406 static void trace_create_eval_file(struct dentry *d_tracer)
5407 {
5408         trace_create_file("eval_map", 0444, d_tracer,
5409                           NULL, &tracing_eval_map_fops);
5410 }
5411 
5412 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5413 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5414 static inline void trace_insert_eval_map_file(struct module *mod,
5415                               struct trace_eval_map **start, int len) { }
5416 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5417 
5418 static void trace_insert_eval_map(struct module *mod,
5419                                   struct trace_eval_map **start, int len)
5420 {
5421         struct trace_eval_map **map;
5422 
5423         if (len <= 0)
5424                 return;
5425 
5426         map = start;
5427 
5428         trace_event_eval_update(map, len);
5429 
5430         trace_insert_eval_map_file(mod, start, len);
5431 }
5432 
5433 static ssize_t
5434 tracing_set_trace_read(struct file *filp, char __user *ubuf,
5435                        size_t cnt, loff_t *ppos)
5436 {
5437         struct trace_array *tr = filp->private_data;
5438         char buf[MAX_TRACER_SIZE+2];
5439         int r;
5440 
5441         mutex_lock(&trace_types_lock);
5442         r = sprintf(buf, "%s\n", tr->current_trace->name);
5443         mutex_unlock(&trace_types_lock);
5444 
5445         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5446 }
5447 
5448 int tracer_init(struct tracer *t, struct trace_array *tr)
5449 {
5450         tracing_reset_online_cpus(&tr->trace_buffer);
5451         return t->init(tr);
5452 }
5453 
5454 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
5455 {
5456         int cpu;
5457 
5458         for_each_tracing_cpu(cpu)
5459                 per_cpu_ptr(buf->data, cpu)->entries = val;
5460 }
5461 
5462 #ifdef CONFIG_TRACER_MAX_TRACE
5463 /* resize @tr's buffer to the size of @size_tr's entries */
5464 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5465                                         struct trace_buffer *size_buf, int cpu_id)
5466 {
5467         int cpu, ret = 0;
5468 
5469         if (cpu_id == RING_BUFFER_ALL_CPUS) {
5470                 for_each_tracing_cpu(cpu) {
5471                         ret = ring_buffer_resize(trace_buf->buffer,
5472                                  per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5473                         if (ret < 0)
5474                                 break;
5475                         per_cpu_ptr(trace_buf->data, cpu)->entries =
5476                                 per_cpu_ptr(size_buf->data, cpu)->entries;
5477                 }
5478         } else {
5479                 ret = ring_buffer_resize(trace_buf->buffer,
5480                                  per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5481                 if (ret == 0)
5482                         per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5483                                 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5484         }
5485 
5486         return ret;
5487 }
5488 #endif /* CONFIG_TRACER_MAX_TRACE */
5489 
5490 static int __tracing_resize_ring_buffer(struct trace_array *tr,
5491                                         unsigned long size, int cpu)
5492 {
5493         int ret;
5494 
5495         /*
5496          * If kernel or user changes the size of the ring buffer
5497          * we use the size that was given, and we can forget about
5498          * expanding it later.
5499          */
5500         ring_buffer_expanded = true;
5501 
5502         /* May be called before buffers are initialized */
5503         if (!tr->trace_buffer.buffer)
5504                 return 0;
5505 
5506         ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
5507         if (ret < 0)
5508                 return ret;
5509 
5510 #ifdef CONFIG_TRACER_MAX_TRACE
5511         if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5512             !tr->current_trace->use_max_tr)
5513                 goto out;
5514 
5515         ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5516         if (ret < 0) {
5517                 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5518                                                      &tr->trace_buffer, cpu);
5519                 if (r < 0) {
5520                         /*
5521                          * AARGH! We are left with different
5522                          * size max buffer!!!!
5523                          * The max buffer is our "snapshot" buffer.
5524                          * When a tracer needs a snapshot (one of the
5525                          * latency tracers), it swaps the max buffer
5526                          * with the saved snap shot. We succeeded to
5527                          * update the size of the main buffer, but failed to
5528                          * update the size of the max buffer. But when we tried
5529                          * to reset the main buffer to the original size, we
5530                          * failed there too. This is very unlikely to
5531                          * happen, but if it does, warn and kill all
5532                          * tracing.
5533                          */
5534                         WARN_ON(1);
5535                         tracing_disabled = 1;
5536                 }
5537                 return ret;
5538         }
5539 
5540         if (cpu == RING_BUFFER_ALL_CPUS)
5541                 set_buffer_entries(&tr->max_buffer, size);
5542         else
5543                 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5544 
5545  out:
5546 #endif /* CONFIG_TRACER_MAX_TRACE */
5547 
5548         if (cpu == RING_BUFFER_ALL_CPUS)
5549                 set_buffer_entries(&tr->trace_buffer, size);
5550         else
5551                 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
5552 
5553         return ret;
5554 }
5555 
5556 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5557                                           unsigned long size, int cpu_id)
5558 {
5559         int ret = size;
5560 
5561         mutex_lock(&trace_types_lock);
5562 
5563         if (cpu_id != RING_BUFFER_ALL_CPUS) {
5564                 /* make sure, this cpu is enabled in the mask */
5565                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5566                         ret = -EINVAL;
5567                         goto out;
5568                 }
5569         }
5570 
5571         ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5572         if (ret < 0)
5573                 ret = -ENOMEM;
5574 
5575 out:
5576         mutex_unlock(&trace_types_lock);
5577 
5578         return ret;
5579 }
5580 
5581 
5582 /**
5583  * tracing_update_buffers - used by tracing facility to expand ring buffers
5584  *
5585  * To save on memory when the tracing is never used on a system with it
5586  * configured in. The ring buffers are set to a minimum size. But once
5587  * a user starts to use the tracing facility, then they need to grow
5588  * to their default size.
5589  *
5590  * This function is to be called when a tracer is about to be used.
5591  */
5592 int tracing_update_buffers(void)
5593 {
5594         int ret = 0;
5595 
5596         mutex_lock(&trace_types_lock);
5597         if (!ring_buffer_expanded)
5598                 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5599                                                 RING_BUFFER_ALL_CPUS);
5600         mutex_unlock(&trace_types_lock);
5601 
5602         return ret;
5603 }
5604 
5605 struct trace_option_dentry;
5606 
5607 static void
5608 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5609 
5610 /*
5611  * Used to clear out the tracer before deletion of an instance.
5612  * Must have trace_types_lock held.
5613  */
5614 static void tracing_set_nop(struct trace_array *tr)
5615 {
5616         if (tr->current_trace == &nop_trace)
5617                 return;
5618         
5619         tr->current_trace->enabled--;
5620 
5621         if (tr->current_trace->reset)
5622                 tr->current_trace->reset(tr);
5623 
5624         tr->current_trace = &nop_trace;
5625 }
5626 
5627 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5628 {
5629         /* Only enable if the directory has been created already. */
5630         if (!tr->dir)
5631                 return;
5632 
5633         create_trace_option_files(tr, t);
5634 }
5635 
5636 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5637 {
5638         struct tracer *t;
5639 #ifdef CONFIG_TRACER_MAX_TRACE
5640         bool had_max_tr;
5641 #endif
5642         int ret = 0;
5643 
5644         mutex_lock(&trace_types_lock);
5645 
5646         if (!ring_buffer_expanded) {
5647                 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5648                                                 RING_BUFFER_ALL_CPUS);
5649                 if (ret < 0)
5650                         goto out;
5651                 ret = 0;
5652         }
5653 
5654         for (t = trace_types; t; t = t->next) {
5655                 if (strcmp(t->name, buf) == 0)
5656                         break;
5657         }
5658         if (!t) {
5659                 ret = -EINVAL;
5660                 goto out;
5661         }
5662         if (t == tr->current_trace)
5663                 goto out;
5664 
5665 #ifdef CONFIG_TRACER_SNAPSHOT
5666         if (t->use_max_tr) {
5667                 arch_spin_lock(&tr->max_lock);
5668                 if (tr->cond_snapshot)
5669                         ret = -EBUSY;
5670                 arch_spin_unlock(&tr->max_lock);
5671                 if (ret)
5672                         goto out;
5673         }
5674 #endif
5675         /* Some tracers won't work on kernel command line */
5676         if (system_state < SYSTEM_RUNNING && t->noboot) {
5677                 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5678                         t->name);
5679                 goto out;
5680         }
5681 
5682         /* Some tracers are only allowed for the top level buffer */
5683         if (!trace_ok_for_array(t, tr)) {
5684                 ret = -EINVAL;
5685                 goto out;
5686         }
5687 
5688         /* If trace pipe files are being read, we can't change the tracer */
5689         if (tr->current_trace->ref) {
5690                 ret = -EBUSY;
5691                 goto out;
5692         }
5693 
5694         trace_branch_disable();
5695 
5696         tr->current_trace->enabled--;
5697 
5698         if (tr->current_trace->reset)
5699                 tr->current_trace->reset(tr);
5700 
5701         /* Current trace needs to be nop_trace before synchronize_rcu */
5702         tr->current_trace = &nop_trace;
5703 
5704 #ifdef CONFIG_TRACER_MAX_TRACE
5705         had_max_tr = tr->allocated_snapshot;
5706 
5707         if (had_max_tr && !t->use_max_tr) {
5708                 /*
5709                  * We need to make sure that the update_max_tr sees that
5710                  * current_trace changed to nop_trace to keep it from
5711                  * swapping the buffers after we resize it.
5712                  * The update_max_tr is called from interrupts disabled
5713                  * so a synchronized_sched() is sufficient.
5714                  */
5715                 synchronize_rcu();
5716                 free_snapshot(tr);
5717         }
5718 #endif
5719 
5720 #ifdef CONFIG_TRACER_MAX_TRACE
5721         if (t->use_max_tr && !had_max_tr) {
5722                 ret = tracing_alloc_snapshot_instance(tr);
5723                 if (ret < 0)
5724                         goto out;
5725         }
5726 #endif
5727 
5728         if (t->init) {
5729                 ret = tracer_init(t, tr);
5730                 if (ret)
5731                         goto out;
5732         }
5733 
5734         tr->current_trace = t;
5735         tr->current_trace->enabled++;
5736         trace_branch_enable(tr);
5737  out:
5738         mutex_unlock(&trace_types_lock);
5739 
5740         return ret;
5741 }
5742 
5743 static ssize_t
5744 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5745                         size_t cnt, loff_t *ppos)
5746 {
5747         struct trace_array *tr = filp->private_data;
5748         char buf[MAX_TRACER_SIZE+1];
5749         int i;
5750         size_t ret;
5751         int err;
5752 
5753         ret = cnt;
5754 
5755         if (cnt > MAX_TRACER_SIZE)
5756                 cnt = MAX_TRACER_SIZE;
5757 
5758         if (copy_from_user(buf, ubuf, cnt))
5759                 return -EFAULT;
5760 
5761         buf[cnt] = 0;
5762 
5763         /* strip ending whitespace. */
5764         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5765                 buf[i] = 0;
5766 
5767         err = tracing_set_tracer(tr, buf);
5768         if (err)
5769                 return err;
5770 
5771         *ppos += ret;
5772 
5773         return ret;
5774 }
5775 
5776 static ssize_t
5777 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5778                    size_t cnt, loff_t *ppos)
5779 {
5780         char buf[64];
5781         int r;
5782 
5783         r = snprintf(buf, sizeof(buf), "%ld\n",
5784                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5785         if (r > sizeof(buf))
5786                 r = sizeof(buf);
5787         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5788 }
5789 
5790 static ssize_t
5791 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5792                     size_t cnt, loff_t *ppos)
5793 {
5794         unsigned long val;
5795         int ret;
5796 
5797         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5798         if (ret)
5799                 return ret;
5800 
5801         *ptr = val * 1000;
5802 
5803         return cnt;
5804 }
5805 
5806 static ssize_t
5807 tracing_thresh_read(struct file *filp, char __user *ubuf,
5808                     size_t cnt, loff_t *ppos)
5809 {
5810         return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5811 }
5812 
5813 static ssize_t
5814 tracing_thresh_write(struct file *filp, const char __user *ubuf,
5815                      size_t cnt, loff_t *ppos)
5816 {
5817         struct trace_array *tr = filp->private_data;
5818         int ret;
5819 
5820         mutex_lock(&trace_types_lock);
5821         ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5822         if (ret < 0)
5823                 goto out;
5824 
5825         if (tr->current_trace->update_thresh) {
5826                 ret = tr->current_trace->update_thresh(tr);
5827                 if (ret < 0)
5828                         goto out;
5829         }
5830 
5831         ret = cnt;
5832 out:
5833         mutex_unlock(&trace_types_lock);
5834 
5835         return ret;
5836 }
5837 
5838 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5839 
5840 static ssize_t
5841 tracing_max_lat_read(struct file *filp, char __user *ubuf,
5842                      size_t cnt, loff_t *ppos)
5843 {
5844         return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5845 }
5846 
5847 static ssize_t
5848 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5849                       size_t cnt, loff_t *ppos)
5850 {
5851         return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5852 }
5853 
5854 #endif
5855 
5856 static int tracing_open_pipe(struct inode *inode, struct file *filp)
5857 {
5858         struct trace_array *tr = inode->i_private;
5859         struct trace_iterator *iter;
5860         int ret;
5861 
5862         ret = tracing_check_open_get_tr(tr);
5863         if (ret)
5864                 return ret;
5865 
5866         mutex_lock(&trace_types_lock);
5867 
5868         /* create a buffer to store the information to pass to userspace */
5869         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5870         if (!iter) {
5871                 ret = -ENOMEM;
5872                 __trace_array_put(tr);
5873                 goto out;
5874         }
5875 
5876         trace_seq_init(&iter->seq);
5877         iter->trace = tr->current_trace;
5878 
5879         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5880                 ret = -ENOMEM;
5881                 goto fail;
5882         }
5883 
5884         /* trace pipe does not show start of buffer */
5885         cpumask_setall(iter->started);
5886 
5887         if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5888                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5889 
5890         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5891         if (trace_clocks[tr->clock_id].in_ns)
5892                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5893 
5894         iter->tr = tr;
5895         iter->trace_buffer = &tr->trace_buffer;
5896         iter->cpu_file = tracing_get_cpu(inode);
5897         mutex_init(&iter->mutex);
5898         filp->private_data = iter;
5899 
5900         if (iter->trace->pipe_open)
5901                 iter->trace->pipe_open(iter);
5902 
5903         nonseekable_open(inode, filp);
5904 
5905         tr->current_trace->ref++;
5906 out:
5907         mutex_unlock(&trace_types_lock);
5908         return ret;
5909 
5910 fail:
5911         kfree(iter);
5912         __trace_array_put(tr);
5913         mutex_unlock(&trace_types_lock);
5914         return ret;
5915 }
5916 
5917 static int tracing_release_pipe(struct inode *inode, struct file *file)
5918 {
5919         struct trace_iterator *iter = file->private_data;
5920         struct trace_array *tr = inode->i_private;
5921 
5922         mutex_lock(&trace_types_lock);
5923 
5924         tr->current_trace->ref--;
5925 
5926         if (iter->trace->pipe_close)
5927                 iter->trace->pipe_close(iter);
5928 
5929         mutex_unlock(&trace_types_lock);
5930 
5931         free_cpumask_var(iter->started);
5932         mutex_destroy(&iter->mutex);
5933         kfree(iter);
5934 
5935         trace_array_put(tr);
5936 
5937         return 0;
5938 }
5939 
5940 static __poll_t
5941 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5942 {
5943         struct trace_array *tr = iter->tr;
5944 
5945         /* Iterators are static, they should be filled or empty */
5946         if (trace_buffer_iter(iter, iter->cpu_file))
5947                 return EPOLLIN | EPOLLRDNORM;
5948 
5949         if (tr->trace_flags & TRACE_ITER_BLOCK)
5950                 /*
5951                  * Always select as readable when in blocking mode
5952                  */
5953                 return EPOLLIN | EPOLLRDNORM;
5954         else
5955                 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5956                                              filp, poll_table);
5957 }
5958 
5959 static __poll_t
5960 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5961 {
5962         struct trace_iterator *iter = filp->private_data;
5963 
5964         return trace_poll(iter, filp, poll_table);
5965 }
5966 
5967 /* Must be called with iter->mutex held. */
5968 static int tracing_wait_pipe(struct file *filp)
5969 {
5970         struct trace_iterator *iter = filp->private_data;
5971         int ret;
5972 
5973         while (trace_empty(iter)) {
5974 
5975                 if ((filp->f_flags & O_NONBLOCK)) {
5976                         return -EAGAIN;
5977                 }
5978 
5979                 /*
5980                  * We block until we read something and tracing is disabled.
5981                  * We still block if tracing is disabled, but we have never
5982                  * read anything. This allows a user to cat this file, and
5983                  * then enable tracing. But after we have read something,
5984                  * we give an EOF when tracing is again disabled.
5985                  *
5986                  * iter->pos will be 0 if we haven't read anything.
5987                  */
5988                 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5989                         break;
5990 
5991                 mutex_unlock(&iter->mutex);
5992 
5993                 ret = wait_on_pipe(iter, 0);
5994 
5995                 mutex_lock(&iter->mutex);
5996 
5997                 if (ret)
5998                         return ret;
5999         }
6000 
6001         return 1;
6002 }
6003 
6004 /*
6005  * Consumer reader.
6006  */
6007 static ssize_t
6008 tracing_read_pipe(struct file *filp, char __user *ubuf,
6009                   size_t cnt, loff_t *ppos)
6010 {
6011         struct trace_iterator *iter = filp->private_data;
6012         ssize_t sret;
6013 
6014         /*
6015          * Avoid more than one consumer on a single file descriptor
6016          * This is just a matter of traces coherency, the ring buffer itself
6017          * is protected.
6018          */
6019         mutex_lock(&iter->mutex);
6020 
6021         /* return any leftover data */
6022         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6023         if (sret != -EBUSY)
6024                 goto out;
6025 
6026         trace_seq_init(&iter->seq);
6027 
6028         if (iter->trace->read) {
6029                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6030                 if (sret)
6031                         goto out;
6032         }
6033 
6034 waitagain:
6035         sret = tracing_wait_pipe(filp);
6036         if (sret <= 0)
6037                 goto out;
6038 
6039         /* stop when tracing is finished */
6040         if (trace_empty(iter)) {
6041                 sret = 0;
6042                 goto out;
6043         }
6044 
6045         if (cnt >= PAGE_SIZE)
6046                 cnt = PAGE_SIZE - 1;
6047 
6048         /* reset all but tr, trace, and overruns */
6049         memset(&iter->seq, 0,
6050                sizeof(struct trace_iterator) -
6051                offsetof(struct trace_iterator, seq));
6052         cpumask_clear(iter->started);
6053         trace_seq_init(&iter->seq);
6054         iter->pos = -1;
6055 
6056         trace_event_read_lock();
6057         trace_access_lock(iter->cpu_file);
6058         while (trace_find_next_entry_inc(iter) != NULL) {
6059                 enum print_line_t ret;
6060                 int save_len = iter->seq.seq.len;
6061 
6062                 ret = print_trace_line(iter);
6063                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6064                         /* don't print partial lines */
6065                         iter->seq.seq.len = save_len;
6066                         break;
6067                 }
6068                 if (ret != TRACE_TYPE_NO_CONSUME)
6069                         trace_consume(iter);
6070 
6071                 if (trace_seq_used(&iter->seq) >= cnt)
6072                         break;
6073 
6074                 /*
6075                  * Setting the full flag means we reached the trace_seq buffer
6076                  * size and we should leave by partial output condition above.
6077                  * One of the trace_seq_* functions is not used properly.
6078                  */
6079                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6080                           iter->ent->type);
6081         }
6082         trace_access_unlock(iter->cpu_file);
6083         trace_event_read_unlock();
6084 
6085         /* Now copy what we have to the user */
6086         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6087         if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6088                 trace_seq_init(&iter->seq);
6089 
6090         /*
6091          * If there was nothing to send to user, in spite of consuming trace
6092          * entries, go back to wait for more entries.
6093          */
6094         if (sret == -EBUSY)
6095                 goto waitagain;
6096 
6097 out:
6098         mutex_unlock(&iter->mutex);
6099 
6100         return sret;
6101 }
6102 
6103 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6104                                      unsigned int idx)
6105 {
6106         __free_page(spd->pages[idx]);
6107 }
6108 
6109 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
6110         .confirm                = generic_pipe_buf_confirm,
6111         .release                = generic_pipe_buf_release,
6112         .steal                  = generic_pipe_buf_steal,
6113         .get                    = generic_pipe_buf_get,
6114 };
6115 
6116 static size_t
6117 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6118 {
6119         size_t count;
6120         int save_len;
6121         int ret;
6122 
6123         /* Seq buffer is page-sized, exactly what we need. */
6124         for (;;) {
6125                 save_len = iter->seq.seq.len;
6126                 ret = print_trace_line(iter);
6127 
6128                 if (trace_seq_has_overflowed(&iter->seq)) {
6129                         iter->seq.seq.len = save_len;
6130                         break;
6131                 }
6132 
6133                 /*
6134                  * This should not be hit, because it should only
6135                  * be set if the iter->seq overflowed. But check it
6136                  * anyway to be safe.
6137                  */
6138                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6139                         iter->seq.seq.len = save_len;
6140                         break;
6141                 }
6142 
6143                 count = trace_seq_used(&iter->seq) - save_len;
6144                 if (rem < count) {
6145                         rem = 0;
6146                         iter->seq.seq.len = save_len;
6147                         break;
6148                 }
6149 
6150                 if (ret != TRACE_TYPE_NO_CONSUME)
6151                         trace_consume(iter);
6152                 rem -= count;
6153                 if (!trace_find_next_entry_inc(iter))   {
6154                         rem = 0;
6155                         iter->ent = NULL;
6156                         break;
6157                 }
6158         }
6159 
6160         return rem;
6161 }
6162 
6163 static ssize_t tracing_splice_read_pipe(struct file *filp,
6164                                         loff_t *ppos,
6165                                         struct pipe_inode_info *pipe,
6166                                         size_t len,
6167                                         unsigned int flags)
6168 {
6169         struct page *pages_def[PIPE_DEF_BUFFERS];
6170         struct partial_page partial_def[PIPE_DEF_BUFFERS];
6171         struct trace_iterator *iter = filp->private_data;
6172         struct splice_pipe_desc spd = {
6173                 .pages          = pages_def,
6174                 .partial        = partial_def,
6175                 .nr_pages       = 0, /* This gets updated below. */
6176                 .nr_pages_max   = PIPE_DEF_BUFFERS,
6177                 .ops            = &tracing_pipe_buf_ops,
6178                 .spd_release    = tracing_spd_release_pipe,
6179         };
6180         ssize_t ret;
6181         size_t rem;
6182         unsigned int i;
6183 
6184         if (splice_grow_spd(pipe, &spd))
6185                 return -ENOMEM;
6186 
6187         mutex_lock(&iter->mutex);
6188 
6189         if (iter->trace->splice_read) {
6190                 ret = iter->trace->splice_read(iter, filp,
6191                                                ppos, pipe, len, flags);
6192                 if (ret)
6193                         goto out_err;
6194         }
6195 
6196         ret = tracing_wait_pipe(filp);
6197         if (ret <= 0)
6198                 goto out_err;
6199 
6200         if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6201                 ret = -EFAULT;
6202                 goto out_err;
6203         }
6204 
6205         trace_event_read_lock();
6206         trace_access_lock(iter->cpu_file);
6207 
6208         /* Fill as many pages as possible. */
6209         for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6210                 spd.pages[i] = alloc_page(GFP_KERNEL);
6211                 if (!spd.pages[i])
6212                         break;
6213 
6214                 rem = tracing_fill_pipe_page(rem, iter);
6215 
6216                 /* Copy the data into the page, so we can start over. */
6217                 ret = trace_seq_to_buffer(&iter->seq,
6218                                           page_address(spd.pages[i]),
6219                                           trace_seq_used(&iter->seq));
6220                 if (ret < 0) {
6221                         __free_page(spd.pages[i]);
6222                         break;
6223                 }
6224                 spd.partial[i].offset = 0;
6225                 spd.partial[i].len = trace_seq_used(&iter->seq);
6226 
6227                 trace_seq_init(&iter->seq);
6228         }
6229 
6230         trace_access_unlock(iter->cpu_file);
6231         trace_event_read_unlock();
6232         mutex_unlock(&iter->mutex);
6233 
6234         spd.nr_pages = i;
6235 
6236         if (i)
6237                 ret = splice_to_pipe(pipe, &spd);
6238         else
6239                 ret = 0;
6240 out:
6241         splice_shrink_spd(&spd);
6242         return ret;
6243 
6244 out_err:
6245         mutex_unlock(&iter->mutex);
6246         goto out;
6247 }
6248 
6249 static ssize_t
6250 tracing_entries_read(struct file *filp, char __user *ubuf,
6251                      size_t cnt, loff_t *ppos)
6252 {
6253         struct inode *inode = file_inode(filp);
6254         struct trace_array *tr = inode->i_private;
6255         int cpu = tracing_get_cpu(inode);
6256         char buf[64];
6257         int r = 0;
6258         ssize_t ret;
6259 
6260         mutex_lock(&trace_types_lock);
6261 
6262         if (cpu == RING_BUFFER_ALL_CPUS) {
6263                 int cpu, buf_size_same;
6264                 unsigned long size;
6265 
6266                 size = 0;
6267                 buf_size_same = 1;
6268                 /* check if all cpu sizes are same */
6269                 for_each_tracing_cpu(cpu) {
6270                         /* fill in the size from first enabled cpu */
6271                         if (size == 0)
6272                                 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
6273                         if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
6274                                 buf_size_same = 0;
6275                                 break;
6276                         }
6277                 }
6278 
6279                 if (buf_size_same) {
6280                         if (!ring_buffer_expanded)
6281                                 r = sprintf(buf, "%lu (expanded: %lu)\n",
6282                                             size >> 10,
6283                                             trace_buf_size >> 10);
6284                         else
6285                                 r = sprintf(buf, "%lu\n", size >> 10);
6286                 } else
6287                         r = sprintf(buf, "X\n");
6288         } else
6289                 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
6290 
6291         mutex_unlock(&trace_types_lock);
6292 
6293         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6294         return ret;
6295 }
6296 
6297 static ssize_t
6298 tracing_entries_write(struct file *filp, const char __user *ubuf,
6299                       size_t cnt, loff_t *ppos)
6300 {
6301         struct inode *inode = file_inode(filp);
6302         struct trace_array *tr = inode->i_private;
6303         unsigned long val;
6304         int ret;
6305 
6306         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6307         if (ret)
6308                 return ret;
6309 
6310         /* must have at least 1 entry */
6311         if (!val)
6312                 return -EINVAL;
6313 
6314         /* value is in KB */
6315         val <<= 10;
6316         ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6317         if (ret < 0)
6318                 return ret;
6319 
6320         *ppos += cnt;
6321 
6322         return cnt;
6323 }
6324 
6325 static ssize_t
6326 tracing_total_entries_read(struct file *filp, char __user *ubuf,
6327                                 size_t cnt, loff_t *ppos)
6328 {
6329         struct trace_array *tr = filp->private_data;
6330         char buf[64];
6331         int r, cpu;
6332         unsigned long size = 0, expanded_size = 0;
6333 
6334         mutex_lock(&trace_types_lock);
6335         for_each_tracing_cpu(cpu) {
6336                 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
6337                 if (!ring_buffer_expanded)
6338                         expanded_size += trace_buf_size >> 10;
6339         }
6340         if (ring_buffer_expanded)
6341                 r = sprintf(buf, "%lu\n", size);
6342         else
6343                 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6344         mutex_unlock(&trace_types_lock);
6345 
6346         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6347 }
6348 
6349 static ssize_t
6350 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6351                           size_t cnt, loff_t *ppos)
6352 {
6353         /*
6354          * There is no need to read what the user has written, this function
6355          * is just to make sure that there is no error when "echo" is used
6356          */
6357 
6358         *ppos += cnt;
6359 
6360         return cnt;
6361 }
6362 
6363 static int
6364 tracing_free_buffer_release(struct inode *inode, struct file *filp)
6365 {
6366         struct trace_array *tr = inode->i_private;
6367 
6368         /* disable tracing ? */
6369         if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6370                 tracer_tracing_off(tr);
6371         /* resize the ring buffer to 0 */
6372         tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6373 
6374         trace_array_put(tr);
6375 
6376         return 0;
6377 }
6378 
6379 static ssize_t
6380 tracing_mark_write(struct file *filp, const char __user *ubuf,
6381                                         size_t cnt, loff_t *fpos)
6382 {
6383         struct trace_array *tr = filp->private_data;
6384         struct ring_buffer_event *event;
6385         enum event_trigger_type tt = ETT_NONE;
6386         struct ring_buffer *buffer;
6387         struct print_entry *entry;
6388         unsigned long irq_flags;
6389         ssize_t written;
6390         int size;
6391         int len;
6392 
6393 /* Used in tracing_mark_raw_write() as well */
6394 #define FAULTED_STR "<faulted>"
6395 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6396 
6397         if (tracing_disabled)
6398                 return -EINVAL;
6399 
6400         if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6401                 return -EINVAL;
6402 
6403         if (cnt > TRACE_BUF_SIZE)
6404                 cnt = TRACE_BUF_SIZE;
6405 
6406         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6407 
6408         local_save_flags(irq_flags);
6409         size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6410 
6411         /* If less than "<faulted>", then make sure we can still add that */
6412         if (cnt < FAULTED_SIZE)
6413                 size += FAULTED_SIZE - cnt;
6414 
6415         buffer = tr->trace_buffer.buffer;
6416         event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6417                                             irq_flags, preempt_count());
6418         if (unlikely(!event))
6419                 /* Ring buffer disabled, return as if not open for write */
6420                 return -EBADF;
6421 
6422         entry = ring_buffer_event_data(event);
6423         entry->ip = _THIS_IP_;
6424 
6425         len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6426         if (len) {
6427                 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6428                 cnt = FAULTED_SIZE;
6429                 written = -EFAULT;
6430         } else
6431                 written = cnt;
6432         len = cnt;
6433 
6434         if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6435                 /* do not add \n before testing triggers, but add \0 */
6436                 entry->buf[cnt] = '\0';
6437                 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6438         }
6439 
6440         if (entry->buf[cnt - 1] != '\n') {
6441                 entry->buf[cnt] = '\n';
6442                 entry->buf[cnt + 1] = '\0';
6443         } else
6444                 entry->buf[cnt] = '\0';
6445 
6446         __buffer_unlock_commit(buffer, event);
6447 
6448         if (tt)
6449                 event_triggers_post_call(tr->trace_marker_file, tt);
6450 
6451         if (written > 0)
6452                 *fpos += written;
6453 
6454         return written;
6455 }
6456 
6457 /* Limit it for now to 3K (including tag) */
6458 #define RAW_DATA_MAX_SIZE (1024*3)
6459 
6460 static ssize_t
6461 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6462                                         size_t cnt, loff_t *fpos)
6463 {
6464         struct trace_array *tr = filp->private_data;
6465         struct ring_buffer_event *event;
6466         struct ring_buffer *buffer;
6467         struct raw_data_entry *entry;
6468         unsigned long irq_flags;
6469         ssize_t written;
6470         int size;
6471         int len;
6472 
6473 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6474 
6475         if (tracing_disabled)
6476                 return -EINVAL;
6477 
6478         if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6479                 return -EINVAL;
6480 
6481         /* The marker must at least have a tag id */
6482         if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6483                 return -EINVAL;
6484 
6485         if (cnt > TRACE_BUF_SIZE)
6486                 cnt = TRACE_BUF_SIZE;
6487 
6488         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6489 
6490         local_save_flags(irq_flags);
6491         size = sizeof(*entry) + cnt;
6492         if (cnt < FAULT_SIZE_ID)
6493                 size += FAULT_SIZE_ID - cnt;
6494 
6495         buffer = tr->trace_buffer.buffer;
6496         event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6497                                             irq_flags, preempt_count());
6498         if (!event)
6499                 /* Ring buffer disabled, return as if not open for write */
6500                 return -EBADF;
6501 
6502         entry = ring_buffer_event_data(event);
6503 
6504         len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6505         if (len) {
6506                 entry->id = -1;
6507                 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6508                 written = -EFAULT;
6509         } else
6510                 written = cnt;
6511 
6512         __buffer_unlock_commit(buffer, event);
6513 
6514         if (written > 0)
6515                 *fpos += written;
6516 
6517         return written;
6518 }
6519 
6520 static int tracing_clock_show(struct seq_file *m, void *v)
6521 {
6522         struct trace_array *tr = m->private;
6523         int i;
6524 
6525         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6526                 seq_printf(m,
6527                         "%s%s%s%s", i ? " " : "",
6528                         i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6529                         i == tr->clock_id ? "]" : "");
6530         seq_putc(m, '\n');
6531 
6532         return 0;
6533 }
6534 
6535 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6536 {
6537         int i;
6538 
6539         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6540                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6541                         break;
6542         }
6543         if (i == ARRAY_SIZE(trace_clocks))
6544                 return -EINVAL;
6545 
6546         mutex_lock(&trace_types_lock);
6547 
6548         tr->clock_id = i;
6549 
6550         ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
6551 
6552         /*
6553          * New clock may not be consistent with the previous clock.
6554          * Reset the buffer so that it doesn't have incomparable timestamps.
6555          */
6556         tracing_reset_online_cpus(&tr->trace_buffer);
6557 
6558 #ifdef CONFIG_TRACER_MAX_TRACE
6559         if (tr->max_buffer.buffer)
6560                 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6561         tracing_reset_online_cpus(&tr->max_buffer);
6562 #endif
6563 
6564         mutex_unlock(&trace_types_lock);
6565 
6566         return 0;
6567 }
6568 
6569 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6570                                    size_t cnt, loff_t *fpos)
6571 {
6572         struct seq_file *m = filp->private_data;
6573         struct trace_array *tr = m->private;
6574         char buf[64];
6575         const char *clockstr;
6576         int ret;
6577 
6578         if (cnt >= sizeof(buf))
6579                 return -EINVAL;
6580 
6581         if (copy_from_user(buf, ubuf, cnt))
6582                 return -EFAULT;
6583 
6584         buf[cnt] = 0;
6585 
6586         clockstr = strstrip(buf);
6587 
6588         ret = tracing_set_clock(tr, clockstr);
6589         if (ret)
6590                 return ret;
6591 
6592         *fpos += cnt;
6593 
6594         return cnt;
6595 }
6596 
6597 static int tracing_clock_open(struct inode *inode, struct file *file)
6598 {
6599         struct trace_array *tr = inode->i_private;
6600         int ret;
6601 
6602         ret = tracing_check_open_get_tr(tr);
6603         if (ret)
6604                 return ret;
6605 
6606         ret = single_open(file, tracing_clock_show, inode->i_private);
6607         if (ret < 0)
6608                 trace_array_put(tr);
6609 
6610         return ret;
6611 }
6612 
6613 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6614 {
6615         struct trace_array *tr = m->private;
6616 
6617         mutex_lock(&trace_types_lock);
6618 
6619         if (ring_buffer_time_stamp_abs(tr->trace_buffer.buffer))
6620                 seq_puts(m, "delta [absolute]\n");
6621         else
6622                 seq_puts(m, "[delta] absolute\n");
6623 
6624         mutex_unlock(&trace_types_lock);
6625 
6626         return 0;
6627 }
6628 
6629 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6630 {
6631         struct trace_array *tr = inode->i_private;
6632         int ret;
6633 
6634         ret = tracing_check_open_get_tr(tr);
6635         if (ret)
6636                 return ret;
6637 
6638         ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6639         if (ret < 0)
6640                 trace_array_put(tr);
6641 
6642         return ret;
6643 }
6644 
6645 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6646 {
6647         int ret = 0;
6648 
6649         mutex_lock(&trace_types_lock);
6650 
6651         if (abs && tr->time_stamp_abs_ref++)
6652                 goto out;
6653 
6654         if (!abs) {
6655                 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6656                         ret = -EINVAL;
6657                         goto out;
6658                 }
6659 
6660                 if (--tr->time_stamp_abs_ref)
6661                         goto out;
6662         }
6663 
6664         ring_buffer_set_time_stamp_abs(tr->trace_buffer.buffer, abs);
6665 
6666 #ifdef CONFIG_TRACER_MAX_TRACE
6667         if (tr->max_buffer.buffer)
6668                 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6669 #endif
6670  out:
6671         mutex_unlock(&trace_types_lock);
6672 
6673         return ret;
6674 }
6675 
6676 struct ftrace_buffer_info {
6677         struct trace_iterator   iter;
6678         void                    *spare;
6679         unsigned int            spare_cpu;
6680         unsigned int            read;
6681 };
6682 
6683 #ifdef CONFIG_TRACER_SNAPSHOT
6684 static int tracing_snapshot_open(struct inode *inode, struct file *file)
6685 {
6686         struct trace_array *tr = inode->i_private;
6687         struct trace_iterator *iter;
6688         struct seq_file *m;
6689         int ret;
6690 
6691         ret = tracing_check_open_get_tr(tr);
6692         if (ret)
6693                 return ret;
6694 
6695         if (file->f_mode & FMODE_READ) {
6696                 iter = __tracing_open(inode, file, true);
6697                 if (IS_ERR(iter))
6698                         ret = PTR_ERR(iter);
6699         } else {
6700                 /* Writes still need the seq_file to hold the private data */
6701                 ret = -ENOMEM;
6702                 m = kzalloc(sizeof(*m), GFP_KERNEL);
6703                 if (!m)
6704                         goto out;
6705                 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6706                 if (!iter) {
6707                         kfree(m);
6708                         goto out;
6709                 }
6710                 ret = 0;
6711 
6712                 iter->tr = tr;
6713                 iter->trace_buffer = &tr->max_buffer;
6714                 iter->cpu_file = tracing_get_cpu(inode);
6715                 m->private = iter;
6716                 file->private_data = m;
6717         }
6718 out:
6719         if (ret < 0)
6720                 trace_array_put(tr);
6721 
6722         return ret;
6723 }
6724 
6725 static ssize_t
6726 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6727                        loff_t *ppos)
6728 {
6729         struct seq_file *m = filp->private_data;
6730         struct trace_iterator *iter = m->private;
6731         struct trace_array *tr = iter->tr;
6732         unsigned long val;
6733         int ret;
6734 
6735         ret = tracing_update_buffers();
6736         if (ret < 0)
6737                 return ret;
6738 
6739         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6740         if (ret)
6741                 return ret;
6742 
6743         mutex_lock(&trace_types_lock);
6744 
6745         if (tr->current_trace->use_max_tr) {
6746                 ret = -EBUSY;
6747                 goto out;
6748         }
6749 
6750         arch_spin_lock(&tr->max_lock);
6751         if (tr->cond_snapshot)
6752                 ret = -EBUSY;
6753         arch_spin_unlock(&tr->max_lock);
6754         if (ret)
6755                 goto out;
6756 
6757         switch (val) {
6758         case 0:
6759                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6760                         ret = -EINVAL;
6761                         break;
6762                 }
6763                 if (tr->allocated_snapshot)
6764                         free_snapshot(tr);
6765                 break;
6766         case 1:
6767 /* Only allow per-cpu swap if the ring buffer supports it */
6768 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6769                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6770                         ret = -EINVAL;
6771                         break;
6772                 }
6773 #endif
6774                 if (tr->allocated_snapshot)
6775                         ret = resize_buffer_duplicate_size(&tr->max_buffer,
6776                                         &tr->trace_buffer, iter->cpu_file);
6777                 else
6778                         ret = tracing_alloc_snapshot_instance(tr);
6779                 if (ret < 0)
6780                         break;
6781                 local_irq_disable();
6782                 /* Now, we're going to swap */
6783                 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6784                         update_max_tr(tr, current, smp_processor_id(), NULL);
6785                 else
6786                         update_max_tr_single(tr, current, iter->cpu_file);
6787                 local_irq_enable();
6788                 break;
6789         default:
6790                 if (tr->allocated_snapshot) {
6791                         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6792                                 tracing_reset_online_cpus(&tr->max_buffer);
6793                         else
6794                                 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
6795                 }
6796                 break;
6797         }
6798 
6799         if (ret >= 0) {
6800                 *ppos += cnt;
6801                 ret = cnt;
6802         }
6803 out:
6804         mutex_unlock(&trace_types_lock);
6805         return ret;
6806 }
6807 
6808 static int tracing_snapshot_release(struct inode *inode, struct file *file)
6809 {
6810         struct seq_file *m = file->private_data;
6811         int ret;
6812 
6813         ret = tracing_release(inode, file);
6814 
6815         if (file->f_mode & FMODE_READ)
6816                 return ret;
6817 
6818         /* If write only, the seq_file is just a stub */
6819         if (m)
6820                 kfree(m->private);
6821         kfree(m);
6822 
6823         return 0;
6824 }
6825 
6826 static int tracing_buffers_open(struct inode *inode, struct file *filp);
6827 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6828                                     size_t count, loff_t *ppos);
6829 static int tracing_buffers_release(struct inode *inode, struct file *file);
6830 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6831                    struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6832 
6833 static int snapshot_raw_open(struct inode *inode, struct file *filp)
6834 {
6835         struct ftrace_buffer_info *info;
6836         int ret;
6837 
6838         /* The following checks for tracefs lockdown */
6839         ret = tracing_buffers_open(inode, filp);
6840         if (ret < 0)
6841                 return ret;
6842 
6843         info = filp->private_data;
6844 
6845         if (info->iter.trace->use_max_tr) {
6846                 tracing_buffers_release(inode, filp);
6847                 return -EBUSY;
6848         }
6849 
6850         info->iter.snapshot = true;
6851         info->iter.trace_buffer = &info->iter.tr->max_buffer;
6852 
6853         return ret;
6854 }
6855 
6856 #endif /* CONFIG_TRACER_SNAPSHOT */
6857 
6858 
6859 static const struct file_operations tracing_thresh_fops = {
6860         .open           = tracing_open_generic,
6861         .read           = tracing_thresh_read,
6862         .write          = tracing_thresh_write,
6863         .llseek         = generic_file_llseek,
6864 };
6865 
6866 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6867 static const struct file_operations tracing_max_lat_fops = {
6868         .open           = tracing_open_generic,
6869         .read           = tracing_max_lat_read,
6870         .write          = tracing_max_lat_write,
6871         .llseek         = generic_file_llseek,
6872 };
6873 #endif
6874 
6875 static const struct file_operations set_tracer_fops = {
6876         .open           = tracing_open_generic,
6877         .read           = tracing_set_trace_read,
6878         .write          = tracing_set_trace_write,
6879         .llseek         = generic_file_llseek,
6880 };
6881 
6882 static const struct file_operations tracing_pipe_fops = {
6883         .open           = tracing_open_pipe,
6884         .poll           = tracing_poll_pipe,
6885         .read           = tracing_read_pipe,
6886         .splice_read    = tracing_splice_read_pipe,
6887         .release        = tracing_release_pipe,
6888         .llseek         = no_llseek,
6889 };
6890 
6891 static const struct file_operations tracing_entries_fops = {
6892         .open           = tracing_open_generic_tr,
6893         .read           = tracing_entries_read,
6894         .write          = tracing_entries_write,
6895         .llseek         = generic_file_llseek,
6896         .release        = tracing_release_generic_tr,
6897 };
6898 
6899 static const struct file_operations tracing_total_entries_fops = {
6900         .open           = tracing_open_generic_tr,
6901         .read           = tracing_total_entries_read,
6902         .llseek         = generic_file_llseek,
6903         .release        = tracing_release_generic_tr,
6904 };
6905 
6906 static const struct file_operations tracing_free_buffer_fops = {
6907         .open           = tracing_open_generic_tr,
6908         .write          = tracing_free_buffer_write,
6909         .release        = tracing_free_buffer_release,
6910 };
6911 
6912 static const struct file_operations tracing_mark_fops = {
6913         .open           = tracing_open_generic_tr,
6914         .write          = tracing_mark_write,
6915         .llseek         = generic_file_llseek,
6916         .release        = tracing_release_generic_tr,
6917 };
6918 
6919 static const struct file_operations tracing_mark_raw_fops = {
6920         .open           = tracing_open_generic_tr,
6921         .write          = tracing_mark_raw_write,
6922         .llseek         = generic_file_llseek,
6923         .release        = tracing_release_generic_tr,
6924 };
6925 
6926 static const struct file_operations trace_clock_fops = {
6927         .open           = tracing_clock_open,
6928         .read           = seq_read,
6929         .llseek         = seq_lseek,
6930         .release        = tracing_single_release_tr,
6931         .write          = tracing_clock_write,
6932 };
6933 
6934 static const struct file_operations trace_time_stamp_mode_fops = {
6935         .open           = tracing_time_stamp_mode_open,
6936         .read           = seq_read,
6937         .llseek         = seq_lseek,
6938         .release        = tracing_single_release_tr,
6939 };
6940 
6941 #ifdef CONFIG_TRACER_SNAPSHOT
6942 static const struct file_operations snapshot_fops = {
6943         .open           = tracing_snapshot_open,
6944         .read           = seq_read,
6945         .write          = tracing_snapshot_write,
6946         .llseek         = tracing_lseek,
6947         .release        = tracing_snapshot_release,
6948 };
6949 
6950 static const struct file_operations snapshot_raw_fops = {
6951         .open           = snapshot_raw_open,
6952         .read           = tracing_buffers_read,
6953         .release        = tracing_buffers_release,
6954         .splice_read    = tracing_buffers_splice_read,
6955         .llseek         = no_llseek,
6956 };
6957 
6958 #endif /* CONFIG_TRACER_SNAPSHOT */
6959 
6960 #define TRACING_LOG_ERRS_MAX    8
6961 #define TRACING_LOG_LOC_MAX     128
6962 
6963 #define CMD_PREFIX "  Command: "
6964 
6965 struct err_info {
6966         const char      **errs; /* ptr to loc-specific array of err strings */
6967         u8              type;   /* index into errs -> specific err string */
6968         u8              pos;    /* MAX_FILTER_STR_VAL = 256 */
6969         u64             ts;
6970 };
6971 
6972 struct tracing_log_err {
6973         struct list_head        list;
6974         struct err_info         info;
6975         char                    loc[TRACING_LOG_LOC_MAX]; /* err location */
6976         char                    cmd[MAX_FILTER_STR_VAL]; /* what caused err */
6977 };
6978 
6979 static DEFINE_MUTEX(tracing_err_log_lock);
6980 
6981 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
6982 {
6983         struct tracing_log_err *err;
6984 
6985         if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
6986                 err = kzalloc(sizeof(*err), GFP_KERNEL);
6987                 if (!err)
6988                         err = ERR_PTR(-ENOMEM);
6989                 tr->n_err_log_entries++;
6990 
6991                 return err;
6992         }
6993 
6994         err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
6995         list_del(&err->list);
6996 
6997         return err;
6998 }
6999 
7000 /**
7001  * err_pos - find the position of a string within a command for error careting
7002  * @cmd: The tracing command that caused the error
7003  * @str: The string to position the caret at within @cmd
7004  *
7005  * Finds the position of the first occurence of @str within @cmd.  The
7006  * return value can be passed to tracing_log_err() for caret placement
7007  * within @cmd.
7008  *
7009  * Returns the index within @cmd of the first occurence of @str or 0
7010  * if @str was not found.
7011  */
7012 unsigned int err_pos(char *cmd, const char *str)
7013 {
7014         char *found;
7015 
7016         if (WARN_ON(!strlen(cmd)))
7017                 return 0;
7018 
7019         found = strstr(cmd, str);
7020         if (found)
7021                 return found - cmd;
7022 
7023         return 0;
7024 }
7025 
7026 /**
7027  * tracing_log_err - write an error to the tracing error log
7028  * @tr: The associated trace array for the error (NULL for top level array)
7029  * @loc: A string describing where the error occurred
7030  * @cmd: The tracing command that caused the error
7031  * @errs: The array of loc-specific static error strings
7032  * @type: The index into errs[], which produces the specific static err string
7033  * @pos: The position the caret should be placed in the cmd
7034  *
7035  * Writes an error into tracing/error_log of the form:
7036  *
7037  * <loc>: error: <text>
7038  *   Command: <cmd>
7039  *              ^
7040  *
7041  * tracing/error_log is a small log file containing the last
7042  * TRACING_LOG_ERRS_MAX errors (8).  Memory for errors isn't allocated
7043  * unless there has been a tracing error, and the error log can be
7044  * cleared and have its memory freed by writing the empty string in
7045  * truncation mode to it i.e. echo > tracing/error_log.
7046  *
7047  * NOTE: the @errs array along with the @type param are used to
7048  * produce a static error string - this string is not copied and saved
7049  * when the error is logged - only a pointer to it is saved.  See
7050  * existing callers for examples of how static strings are typically
7051  * defined for use with tracing_log_err().
7052  */
7053 void tracing_log_err(struct trace_array *tr,
7054                      const char *loc, const char *cmd,
7055                      const char **errs, u8 type, u8 pos)
7056 {
7057         struct tracing_log_err *err;
7058 
7059         if (!tr)
7060                 tr = &global_trace;
7061 
7062         mutex_lock(&tracing_err_log_lock);
7063         err = get_tracing_log_err(tr);
7064         if (PTR_ERR(err) == -ENOMEM) {
7065                 mutex_unlock(&tracing_err_log_lock);
7066                 return;
7067         }
7068 
7069         snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7070         snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7071 
7072         err->info.errs = errs;
7073         err->info.type = type;
7074         err->info.pos = pos;
7075         err->info.ts = local_clock();
7076 
7077         list_add_tail(&err->list, &tr->err_log);
7078         mutex_unlock(&tracing_err_log_lock);
7079 }
7080 
7081 static void clear_tracing_err_log(struct trace_array *tr)
7082 {
7083         struct tracing_log_err *err, *next;
7084 
7085         mutex_lock(&tracing_err_log_lock);
7086         list_for_each_entry_safe(err, next, &tr->err_log, list) {
7087                 list_del(&err->list);
7088                 kfree(err);
7089         }
7090 
7091         tr->n_err_log_entries = 0;
7092         mutex_unlock(&tracing_err_log_lock);
7093 }
7094 
7095 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7096 {
7097         struct trace_array *tr = m->private;
7098 
7099         mutex_lock(&tracing_err_log_lock);
7100 
7101         return seq_list_start(&tr->err_log, *pos);
7102 }
7103 
7104 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7105 {
7106         struct trace_array *tr = m->private;
7107 
7108         return seq_list_next(v, &tr->err_log, pos);
7109 }
7110 
7111 static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7112 {
7113         mutex_unlock(&tracing_err_log_lock);
7114 }
7115 
7116 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7117 {
7118         u8 i;
7119 
7120         for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7121                 seq_putc(m, ' ');
7122         for (i = 0; i < pos; i++)
7123                 seq_putc(m, ' ');
7124         seq_puts(m, "^\n");
7125 }
7126 
7127 static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7128 {
7129         struct tracing_log_err *err = v;
7130 
7131         if (err) {
7132                 const char *err_text = err->info.errs[err->info.type];
7133                 u64 sec = err->info.ts;
7134                 u32 nsec;
7135 
7136                 nsec = do_div(sec, NSEC_PER_SEC);
7137                 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7138                            err->loc, err_text);
7139                 seq_printf(m, "%s", err->cmd);
7140                 tracing_err_log_show_pos(m, err->info.pos);
7141         }
7142 
7143         return 0;
7144 }
7145 
7146 static const struct seq_operations tracing_err_log_seq_ops = {
7147         .start  = tracing_err_log_seq_start,
7148         .next   = tracing_err_log_seq_next,
7149         .stop   = tracing_err_log_seq_stop,
7150         .show   = tracing_err_log_seq_show
7151 };
7152 
7153 static int tracing_err_log_open(struct inode *inode, struct file *file)
7154 {
7155         struct trace_array *tr = inode->i_private;
7156         int ret = 0;
7157 
7158         ret = tracing_check_open_get_tr(tr);
7159         if (ret)
7160                 return ret;
7161 
7162         /* If this file was opened for write, then erase contents */
7163         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7164                 clear_tracing_err_log(tr);
7165 
7166         if (file->f_mode & FMODE_READ) {
7167                 ret = seq_open(file, &tracing_err_log_seq_ops);
7168                 if (!ret) {
7169                         struct seq_file *m = file->private_data;
7170                         m->private = tr;
7171                 } else {
7172                         trace_array_put(tr);
7173                 }
7174         }
7175         return ret;
7176 }
7177 
7178 static ssize_t tracing_err_log_write(struct file *file,
7179                                      const char __user *buffer,
7180                                      size_t count, loff_t *ppos)
7181 {
7182         return count;
7183 }
7184 
7185 static int tracing_err_log_release(struct inode *inode, struct file *file)
7186 {
7187         struct trace_array *tr = inode->i_private;
7188 
7189         trace_array_put(tr);
7190 
7191         if (file->f_mode & FMODE_READ)
7192                 seq_release(inode, file);
7193 
7194         return 0;
7195 }
7196 
7197 static const struct file_operations tracing_err_log_fops = {
7198         .open           = tracing_err_log_open,
7199         .write          = tracing_err_log_write,
7200         .read           = seq_read,
7201         .llseek         = seq_lseek,
7202         .release        = tracing_err_log_release,
7203 };
7204 
7205 static int tracing_buffers_open(struct inode *inode, struct file *filp)
7206 {
7207         struct trace_array *tr = inode->i_private;
7208         struct ftrace_buffer_info *info;
7209         int ret;
7210 
7211         ret = tracing_check_open_get_tr(tr);
7212         if (ret)
7213                 return ret;
7214 
7215         info = kzalloc(sizeof(*info), GFP_KERNEL);
7216         if (!info) {
7217                 trace_array_put(tr);
7218                 return -ENOMEM;
7219         }
7220 
7221         mutex_lock(&trace_types_lock);
7222 
7223         info->iter.tr           = tr;
7224         info->iter.cpu_file     = tracing_get_cpu(inode);
7225         info->iter.trace        = tr->current_trace;
7226         info->iter.trace_buffer = &tr->trace_buffer;
7227         info->spare             = NULL;
7228         /* Force reading ring buffer for first read */
7229         info->read              = (unsigned int)-1;
7230 
7231         filp->private_data = info;
7232 
7233         tr->current_trace->ref++;
7234 
7235         mutex_unlock(&trace_types_lock);
7236 
7237         ret = nonseekable_open(inode, filp);
7238         if (ret < 0)
7239                 trace_array_put(tr);
7240 
7241         return ret;
7242 }
7243 
7244 static __poll_t
7245 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7246 {
7247         struct ftrace_buffer_info *info = filp->private_data;
7248         struct trace_iterator *iter = &info->iter;
7249 
7250         return trace_poll(iter, filp, poll_table);
7251 }
7252 
7253 static ssize_t
7254 tracing_buffers_read(struct file *filp, char __user *ubuf,
7255                      size_t count, loff_t *ppos)
7256 {
7257         struct ftrace_buffer_info *info = filp->private_data;
7258         struct trace_iterator *iter = &info->iter;
7259         ssize_t ret = 0;
7260         ssize_t size;
7261 
7262         if (!count)
7263                 return 0;
7264 
7265 #ifdef CONFIG_TRACER_MAX_TRACE
7266         if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7267                 return -EBUSY;
7268 #endif
7269 
7270         if (!info->spare) {
7271                 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
7272                                                           iter->cpu_file);
7273                 if (IS_ERR(info->spare)) {
7274                         ret = PTR_ERR(info->spare);
7275                         info->spare = NULL;
7276                 } else {
7277                         info->spare_cpu = iter->cpu_file;
7278                 }
7279         }
7280         if (!info->spare)
7281                 return ret;
7282 
7283         /* Do we have previous read data to read? */
7284         if (info->read < PAGE_SIZE)
7285                 goto read;
7286 
7287  again:
7288         trace_access_lock(iter->cpu_file);
7289         ret = ring_buffer_read_page(iter->trace_buffer->buffer,
7290                                     &info->spare,
7291                                     count,
7292                                     iter->cpu_file, 0);
7293         trace_access_unlock(iter->cpu_file);
7294 
7295         if (ret < 0) {
7296                 if (trace_empty(iter)) {
7297                         if ((filp->f_flags & O_NONBLOCK))
7298                                 return -EAGAIN;
7299 
7300                         ret = wait_on_pipe(iter, 0);
7301                         if (ret)
7302                                 return ret;
7303 
7304                         goto again;
7305                 }
7306                 return 0;
7307         }
7308 
7309         info->read = 0;
7310  read:
7311         size = PAGE_SIZE - info->read;
7312         if (size > count)
7313                 size = count;
7314 
7315         ret = copy_to_user(ubuf, info->spare + info->read, size);
7316         if (ret == size)
7317                 return -EFAULT;
7318 
7319         size -= ret;
7320 
7321         *ppos += size;
7322         info->read += size;
7323 
7324         return size;
7325 }
7326 
7327 static int tracing_buffers_release(struct inode *inode, struct file *file)
7328 {
7329         struct ftrace_buffer_info *info = file->private_data;
7330         struct trace_iterator *iter = &info->iter;
7331 
7332         mutex_lock(&trace_types_lock);
7333 
7334         iter->tr->current_trace->ref--;
7335 
7336         __trace_array_put(iter->tr);
7337 
7338         if (info->spare)
7339                 ring_buffer_free_read_page(iter->trace_buffer->buffer,
7340                                            info->spare_cpu, info->spare);
7341         kfree(info);
7342 
7343         mutex_unlock(&trace_types_lock);
7344 
7345         return 0;
7346 }
7347 
7348 struct buffer_ref {
7349         struct ring_buffer      *buffer;
7350         void                    *page;
7351         int                     cpu;
7352         refcount_t              refcount;
7353 };
7354 
7355 static void buffer_ref_release(struct buffer_ref *ref)
7356 {
7357         if (!refcount_dec_and_test(&ref->refcount))
7358                 return;
7359         ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7360         kfree(ref);
7361 }
7362 
7363 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7364                                     struct pipe_buffer *buf)
7365 {
7366         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7367 
7368         buffer_ref_release(ref);
7369         buf->private = 0;
7370 }
7371 
7372 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7373                                 struct pipe_buffer *buf)
7374 {
7375         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7376 
7377         if (refcount_read(&ref->refcount) > INT_MAX/2)
7378                 return false;
7379 
7380         refcount_inc(&ref->refcount);
7381         return true;
7382 }
7383 
7384 /* Pipe buffer operations for a buffer. */
7385 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7386         .confirm                = generic_pipe_buf_confirm,
7387         .release                = buffer_pipe_buf_release,
7388         .steal                  = generic_pipe_buf_nosteal,
7389         .get                    = buffer_pipe_buf_get,
7390 };
7391 
7392 /*
7393  * Callback from splice_to_pipe(), if we need to release some pages
7394  * at the end of the spd in case we error'ed out in filling the pipe.
7395  */
7396 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7397 {
7398         struct buffer_ref *ref =
7399                 (struct buffer_ref *)spd->partial[i].private;
7400 
7401         buffer_ref_release(ref);
7402         spd->partial[i].private = 0;
7403 }
7404 
7405 static ssize_t
7406 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7407                             struct pipe_inode_info *pipe, size_t len,
7408                             unsigned int flags)
7409 {
7410         struct ftrace_buffer_info *info = file->private_data;
7411         struct trace_iterator *iter = &info->iter;
7412         struct partial_page partial_def[PIPE_DEF_BUFFERS];
7413         struct page *pages_def[PIPE_DEF_BUFFERS];
7414         struct splice_pipe_desc spd = {
7415                 .pages          = pages_def,
7416                 .partial        = partial_def,
7417                 .nr_pages_max   = PIPE_DEF_BUFFERS,
7418                 .ops            = &buffer_pipe_buf_ops,
7419                 .spd_release    = buffer_spd_release,
7420         };
7421         struct buffer_ref *ref;
7422         int entries, i;
7423         ssize_t ret = 0;
7424 
7425 #ifdef CONFIG_TRACER_MAX_TRACE
7426         if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7427                 return -EBUSY;
7428 #endif
7429 
7430         if (*ppos & (PAGE_SIZE - 1))
7431                 return -EINVAL;
7432 
7433         if (len & (PAGE_SIZE - 1)) {
7434                 if (len < PAGE_SIZE)
7435                         return -EINVAL;
7436                 len &= PAGE_MASK;
7437         }
7438 
7439         if (splice_grow_spd(pipe, &spd))
7440                 return -ENOMEM;
7441 
7442  again:
7443         trace_access_lock(iter->cpu_file);
7444         entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
7445 
7446         for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
7447                 struct page *page;
7448                 int r;
7449 
7450                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
7451                 if (!ref) {
7452                         ret = -ENOMEM;
7453                         break;
7454                 }
7455 
7456                 refcount_set(&ref->refcount, 1);
7457                 ref->buffer = iter->trace_buffer->buffer;
7458                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
7459                 if (IS_ERR(ref->page)) {
7460                         ret = PTR_ERR(ref->page);
7461                         ref->page = NULL;
7462                         kfree(ref);
7463                         break;
7464                 }
7465                 ref->cpu = iter->cpu_file;
7466 
7467                 r = ring_buffer_read_page(ref->buffer, &ref->page,
7468                                           len, iter->cpu_file, 1);
7469                 if (r < 0) {
7470                         ring_buffer_free_read_page(ref->buffer, ref->cpu,
7471                                                    ref->page);
7472                         kfree(ref);
7473                         break;
7474                 }
7475 
7476                 page = virt_to_page(ref->page);
7477 
7478                 spd.pages[i] = page;
7479                 spd.partial[i].len = PAGE_SIZE;
7480                 spd.partial[i].offset = 0;
7481                 spd.partial[i].private = (unsigned long)ref;
7482                 spd.nr_pages++;
7483                 *ppos += PAGE_SIZE;
7484 
7485                 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
7486         }
7487 
7488         trace_access_unlock(iter->cpu_file);
7489         spd.nr_pages = i;
7490 
7491         /* did we read anything? */
7492         if (!spd.nr_pages) {
7493                 if (ret)
7494                         goto out;
7495 
7496                 ret = -EAGAIN;
7497                 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7498                         goto out;
7499 
7500                 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
7501                 if (ret)
7502                         goto out;
7503 
7504                 goto again;
7505         }
7506 
7507         ret = splice_to_pipe(pipe, &spd);
7508 out:
7509         splice_shrink_spd(&spd);
7510 
7511         return ret;
7512 }
7513 
7514 static const struct file_operations tracing_buffers_fops = {
7515         .open           = tracing_buffers_open,
7516         .read           = tracing_buffers_read,
7517         .poll           = tracing_buffers_poll,
7518         .release        = tracing_buffers_release,
7519         .splice_read    = tracing_buffers_splice_read,
7520         .llseek         = no_llseek,
7521 };
7522 
7523 static ssize_t
7524 tracing_stats_read(struct file *filp, char __user *ubuf,
7525                    size_t count, loff_t *ppos)
7526 {
7527         struct inode *inode = file_inode(filp);
7528         struct trace_array *tr = inode->i_private;
7529         struct trace_buffer *trace_buf = &tr->trace_buffer;
7530         int cpu = tracing_get_cpu(inode);
7531         struct trace_seq *s;
7532         unsigned long cnt;
7533         unsigned long long t;
7534         unsigned long usec_rem;
7535 
7536         s = kmalloc(sizeof(*s), GFP_KERNEL);
7537         if (!s)
7538                 return -ENOMEM;
7539 
7540         trace_seq_init(s);
7541 
7542         cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7543         trace_seq_printf(s, "entries: %ld\n", cnt);
7544 
7545         cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7546         trace_seq_printf(s, "overrun: %ld\n", cnt);
7547 
7548         cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7549         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7550 
7551         cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7552         trace_seq_printf(s, "bytes: %ld\n", cnt);
7553 
7554         if (trace_clocks[tr->clock_id].in_ns) {
7555                 /* local or global for trace_clock */
7556                 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7557                 usec_rem = do_div(t, USEC_PER_SEC);
7558                 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7559                                                                 t, usec_rem);
7560 
7561                 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
7562                 usec_rem = do_div(t, USEC_PER_SEC);
7563                 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7564         } else {
7565                 /* counter or tsc mode for trace_clock */
7566                 trace_seq_printf(s, "oldest event ts: %llu\n",
7567                                 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7568 
7569                 trace_seq_printf(s, "now ts: %llu\n",
7570                                 ring_buffer_time_stamp(trace_buf->buffer, cpu));
7571         }
7572 
7573         cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7574         trace_seq_printf(s, "dropped events: %ld\n", cnt);
7575 
7576         cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7577         trace_seq_printf(s, "read events: %ld\n", cnt);
7578 
7579         count = simple_read_from_buffer(ubuf, count, ppos,
7580                                         s->buffer, trace_seq_used(s));
7581 
7582         kfree(s);
7583 
7584         return count;
7585 }
7586 
7587 static const struct file_operations tracing_stats_fops = {
7588         .open           = tracing_open_generic_tr,
7589         .read           = tracing_stats_read,
7590         .llseek         = generic_file_llseek,
7591         .release        = tracing_release_generic_tr,
7592 };
7593 
7594 #ifdef CONFIG_DYNAMIC_FTRACE
7595 
7596 static ssize_t
7597 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7598                   size_t cnt, loff_t *ppos)
7599 {
7600         unsigned long *p = filp->private_data;
7601         char buf[64]; /* Not too big for a shallow stack */
7602         int r;
7603 
7604         r = scnprintf(buf, 63, "%ld", *p);
7605         buf[r++] = '\n';
7606 
7607         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7608 }
7609 
7610 static const struct file_operations tracing_dyn_info_fops = {
7611         .open           = tracing_open_generic,
7612         .read           = tracing_read_dyn_info,
7613         .llseek         = generic_file_llseek,
7614 };
7615 #endif /* CONFIG_DYNAMIC_FTRACE */
7616 
7617 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7618 static void
7619 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7620                 struct trace_array *tr, struct ftrace_probe_ops *ops,
7621                 void *data)
7622 {
7623         tracing_snapshot_instance(tr);
7624 }
7625 
7626 static void
7627 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7628                       struct trace_array *tr, struct ftrace_probe_ops *ops,
7629                       void *data)
7630 {
7631         struct ftrace_func_mapper *mapper = data;
7632         long *count = NULL;
7633 
7634         if (mapper)
7635                 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7636 
7637         if (count) {
7638 
7639                 if (*count <= 0)
7640                         return;
7641 
7642                 (*count)--;
7643         }
7644 
7645         tracing_snapshot_instance(tr);
7646 }
7647 
7648 static int
7649 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7650                       struct ftrace_probe_ops *ops, void *data)
7651 {
7652         struct ftrace_func_mapper *mapper = data;
7653         long *count = NULL;
7654 
7655         seq_printf(m, "%ps:", (void *)ip);
7656 
7657         seq_puts(m, "snapshot");
7658 
7659         if (mapper)
7660                 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7661 
7662         if (count)
7663                 seq_printf(m, ":count=%ld\n", *count);
7664         else
7665                 seq_puts(m, ":unlimited\n");
7666 
7667         return 0;
7668 }
7669 
7670 static int
7671 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7672                      unsigned long ip, void *init_data, void **data)
7673 {
7674         struct ftrace_func_mapper *mapper = *data;
7675 
7676         if (!mapper) {
7677                 mapper = allocate_ftrace_func_mapper();
7678                 if (!mapper)
7679                         return -ENOMEM;
7680                 *data = mapper;
7681         }
7682 
7683         return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7684 }
7685 
7686 static void
7687 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7688                      unsigned long ip, void *data)
7689 {
7690         struct ftrace_func_mapper *mapper = data;
7691 
7692         if (!ip) {
7693                 if (!mapper)
7694                         return;
7695                 free_ftrace_func_mapper(mapper, NULL);
7696                 return;
7697         }
7698 
7699         ftrace_func_mapper_remove_ip(mapper, ip);
7700 }
7701 
7702 static struct ftrace_probe_ops snapshot_probe_ops = {
7703         .func                   = ftrace_snapshot,
7704         .print                  = ftrace_snapshot_print,
7705 };
7706 
7707 static struct ftrace_probe_ops snapshot_count_probe_ops = {
7708         .func                   = ftrace_count_snapshot,
7709         .print                  = ftrace_snapshot_print,
7710         .init                   = ftrace_snapshot_init,
7711         .free                   = ftrace_snapshot_free,
7712 };
7713 
7714 static int
7715 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7716                                char *glob, char *cmd, char *param, int enable)
7717 {
7718         struct ftrace_probe_ops *ops;
7719         void *count = (void *)-1;
7720         char *number;
7721         int ret;
7722 
7723         if (!tr)
7724                 return -ENODEV;
7725 
7726         /* hash funcs only work with set_ftrace_filter */
7727         if (!enable)
7728                 return -EINVAL;
7729 
7730         ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
7731 
7732         if (glob[0] == '!')
7733                 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
7734 
7735         if (!param)
7736                 goto out_reg;
7737 
7738         number = strsep(&param, ":");
7739 
7740         if (!strlen(number))
7741                 goto out_reg;
7742 
7743         /*
7744          * We use the callback data field (which is a pointer)
7745          * as our counter.
7746          */
7747         ret = kstrtoul(number, 0, (unsigned long *)&count);
7748         if (ret)
7749                 return ret;
7750 
7751  out_reg:
7752         ret = tracing_alloc_snapshot_instance(tr);
7753         if (ret < 0)
7754                 goto out;
7755 
7756         ret = register_ftrace_function_probe(glob, tr, ops, count);
7757 
7758  out:
7759         return ret < 0 ? ret : 0;
7760 }
7761 
7762 static struct ftrace_func_command ftrace_snapshot_cmd = {
7763         .name                   = "snapshot",
7764         .func                   = ftrace_trace_snapshot_callback,
7765 };
7766 
7767 static __init int register_snapshot_cmd(void)
7768 {
7769         return register_ftrace_command(&ftrace_snapshot_cmd);
7770 }
7771 #else
7772 static inline __init int register_snapshot_cmd(void) { return 0; }
7773 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7774 
7775 static struct dentry *tracing_get_dentry(struct trace_array *tr)
7776 {
7777         if (WARN_ON(!tr->dir))
7778                 return ERR_PTR(-ENODEV);
7779 
7780         /* Top directory uses NULL as the parent */
7781         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7782                 return NULL;
7783 
7784         /* All sub buffers have a descriptor */
7785         return tr->dir;
7786 }
7787 
7788 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7789 {
7790         struct dentry *d_tracer;
7791 
7792         if (tr->percpu_dir)
7793                 return tr->percpu_dir;
7794 
7795         d_tracer = tracing_get_dentry(tr);
7796         if (IS_ERR(d_tracer))
7797                 return NULL;
7798 
7799         tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7800 
7801         WARN_ONCE(!tr->percpu_dir,
7802                   "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7803 
7804         return tr->percpu_dir;
7805 }
7806 
7807 static struct dentry *
7808 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7809                       void *data, long cpu, const struct file_operations *fops)
7810 {
7811         struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7812 
7813         if (ret) /* See tracing_get_cpu() */
7814                 d_inode(ret)->i_cdev = (void *)(cpu + 1);
7815         return ret;
7816 }
7817 
7818 static void
7819 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
7820 {
7821         struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
7822         struct dentry *d_cpu;
7823         char cpu_dir[30]; /* 30 characters should be more than enough */
7824 
7825         if (!d_percpu)
7826                 return;
7827 
7828         snprintf(cpu_dir, 30, "cpu%ld", cpu);
7829         d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
7830         if (!d_cpu) {
7831                 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
7832                 return;
7833         }
7834 
7835         /* per cpu trace_pipe */
7836         trace_create_cpu_file("trace_pipe", 0444, d_cpu,
7837                                 tr, cpu, &tracing_pipe_fops);
7838 
7839         /* per cpu trace */
7840         trace_create_cpu_file("trace", 0644, d_cpu,
7841                                 tr, cpu, &tracing_fops);
7842 
7843         trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
7844                                 tr, cpu, &tracing_buffers_fops);
7845 
7846         trace_create_cpu_file("stats", 0444, d_cpu,
7847                                 tr, cpu, &tracing_stats_fops);
7848 
7849         trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
7850                                 tr, cpu, &tracing_entries_fops);
7851 
7852 #ifdef CONFIG_TRACER_SNAPSHOT
7853         trace_create_cpu_file("snapshot", 0644, d_cpu,
7854                                 tr, cpu, &snapshot_fops);
7855 
7856         trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
7857                                 tr, cpu, &snapshot_raw_fops);
7858 #endif
7859 }
7860 
7861 #ifdef CONFIG_FTRACE_SELFTEST
7862 /* Let selftest have access to static functions in this file */
7863 #include "trace_selftest.c"
7864 #endif
7865 
7866 static ssize_t
7867 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7868                         loff_t *ppos)
7869 {
7870         struct trace_option_dentry *topt = filp->private_data;
7871         char *buf;
7872 
7873         if (topt->flags->val & topt->opt->bit)
7874                 buf = "1\n";
7875         else
7876                 buf = "0\n";
7877 
7878         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7879 }
7880 
7881 static ssize_t
7882 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7883                          loff_t *ppos)
7884 {
7885         struct trace_option_dentry *topt = filp->private_data;
7886         unsigned long val;
7887         int ret;
7888 
7889         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7890         if (ret)
7891                 return ret;
7892 
7893         if (val != 0 && val != 1)
7894                 return -EINVAL;
7895 
7896         if (!!(topt->flags->val & topt->opt->bit) != val) {
7897                 mutex_lock(&trace_types_lock);
7898                 ret = __set_tracer_option(topt->tr, topt->flags,
7899                                           topt->opt, !val);
7900                 mutex_unlock(&trace_types_lock);
7901                 if (ret)
7902                         return ret;
7903         }
7904 
7905         *ppos += cnt;
7906 
7907         return cnt;
7908 }
7909 
7910 
7911 static const struct file_operations trace_options_fops = {
7912         .open = tracing_open_generic,
7913         .read = trace_options_read,
7914         .write = trace_options_write,
7915         .llseek = generic_file_llseek,
7916 };
7917 
7918 /*
7919  * In order to pass in both the trace_array descriptor as well as the index
7920  * to the flag that the trace option file represents, the trace_array
7921  * has a character array of trace_flags_index[], which holds the index
7922  * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7923  * The address of this character array is passed to the flag option file
7924  * read/write callbacks.
7925  *
7926  * In order to extract both the index and the trace_array descriptor,
7927  * get_tr_index() uses the following algorithm.
7928  *
7929  *   idx = *ptr;
7930  *
7931  * As the pointer itself contains the address of the index (remember
7932  * index[1] == 1).
7933  *
7934  * Then to get the trace_array descriptor, by subtracting that index
7935  * from the ptr, we get to the start of the index itself.
7936  *
7937  *   ptr - idx == &index[0]
7938  *
7939  * Then a simple container_of() from that pointer gets us to the
7940  * trace_array descriptor.
7941  */
7942 static void get_tr_index(void *data, struct trace_array **ptr,
7943                          unsigned int *pindex)
7944 {
7945         *pindex = *(unsigned char *)data;
7946 
7947         *ptr = container_of(data - *pindex, struct trace_array,
7948                             trace_flags_index);
7949 }
7950 
7951 static ssize_t
7952 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7953                         loff_t *ppos)
7954 {
7955         void *tr_index = filp->private_data;
7956         struct trace_array *tr;
7957         unsigned int index;
7958         char *buf;
7959 
7960         get_tr_index(tr_index, &tr, &index);
7961 
7962         if (tr->trace_flags & (1 << index))
7963                 buf = "1\n";
7964         else
7965                 buf = "0\n";
7966 
7967         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7968 }
7969 
7970 static ssize_t
7971 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7972                          loff_t *ppos)
7973 {
7974         void *tr_index = filp->private_data;
7975         struct trace_array *tr;
7976         unsigned int index;
7977         unsigned long val;
7978         int ret;
7979 
7980         get_tr_index(tr_index, &tr, &index);
7981 
7982         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7983         if (ret)
7984                 return ret;
7985 
7986         if (val != 0 && val != 1)
7987                 return -EINVAL;
7988 
7989         mutex_lock(&event_mutex);
7990         mutex_lock(&trace_types_lock);
7991         ret = set_tracer_flag(tr, 1 << index, val);
7992         mutex_unlock(&trace_types_lock);
7993         mutex_unlock(&event_mutex);
7994 
7995         if (ret < 0)
7996                 return ret;
7997 
7998         *ppos += cnt;
7999 
8000         return cnt;
8001 }
8002 
8003 static const struct file_operations trace_options_core_fops = {
8004         .open = tracing_open_generic,
8005         .read = trace_options_core_read,
8006         .write = trace_options_core_write,
8007         .llseek = generic_file_llseek,
8008 };
8009 
8010 struct dentry *trace_create_file(const char *name,
8011                                  umode_t mode,
8012                                  struct dentry *parent,
8013                                  void *data,
8014                                  const struct file_operations *fops)
8015 {
8016         struct dentry *ret;
8017 
8018         ret = tracefs_create_file(name, mode, parent, data, fops);
8019         if (!ret)
8020                 pr_warn("Could not create tracefs '%s' entry\n", name);
8021 
8022         return ret;
8023 }
8024 
8025 
8026 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8027 {
8028         struct dentry *d_tracer;
8029 
8030         if (tr->options)
8031                 return tr->options;
8032 
8033         d_tracer = tracing_get_dentry(tr);
8034         if (IS_ERR(d_tracer))
8035                 return NULL;
8036 
8037         tr->options = tracefs_create_dir("options", d_tracer);
8038         if (!tr->options) {
8039                 pr_warn("Could not create tracefs directory 'options'\n");
8040                 return NULL;
8041         }
8042 
8043         return tr->options;
8044 }
8045 
8046 static void
8047 create_trace_option_file(struct trace_array *tr,
8048                          struct trace_option_dentry *topt,
8049                          struct tracer_flags *flags,
8050                          struct tracer_opt *opt)
8051 {
8052         struct dentry *t_options;
8053 
8054         t_options = trace_options_init_dentry(tr);
8055         if (!t_options)
8056                 return;
8057 
8058         topt->flags = flags;
8059         topt->opt = opt;
8060         topt->tr = tr;
8061 
8062         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
8063                                     &trace_options_fops);
8064 
8065 }
8066 
8067 static void
8068 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8069 {
8070         struct trace_option_dentry *topts;
8071         struct trace_options *tr_topts;
8072         struct tracer_flags *flags;
8073         struct tracer_opt *opts;
8074         int cnt;
8075         int i;
8076 
8077         if (!tracer)
8078                 return;
8079 
8080         flags = tracer->flags;
8081 
8082         if (!flags || !flags->opts)
8083                 return;
8084 
8085         /*
8086          * If this is an instance, only create flags for tracers
8087          * the instance may have.
8088          */
8089         if (!trace_ok_for_array(tracer, tr))
8090                 return;
8091 
8092         for (i = 0; i < tr->nr_topts; i++) {
8093                 /* Make sure there's no duplicate flags. */
8094                 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8095                         return;
8096         }
8097 
8098         opts = flags->opts;
8099 
8100         for (cnt = 0; opts[cnt].name; cnt++)
8101                 ;
8102 
8103         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8104         if (!topts)
8105                 return;
8106 
8107         tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8108                             GFP_KERNEL);
8109         if (!tr_topts) {
8110                 kfree(topts);
8111                 return;
8112         }
8113 
8114         tr->topts = tr_topts;
8115         tr->topts[tr->nr_topts].tracer = tracer;
8116         tr->topts[tr->nr_topts].topts = topts;
8117         tr->nr_topts++;
8118 
8119         for (cnt = 0; opts[cnt].name; cnt++) {
8120                 create_trace_option_file(tr, &topts[cnt], flags,
8121                                          &opts[cnt]);
8122                 WARN_ONCE(topts[cnt].entry == NULL,
8123                           "Failed to create trace option: %s",
8124                           opts[cnt].name);
8125         }
8126 }
8127 
8128 static struct dentry *
8129 create_trace_option_core_file(struct trace_array *tr,
8130                               const char *option, long index)
8131 {
8132         struct dentry *t_options;
8133 
8134         t_options = trace_options_init_dentry(tr);
8135         if (!t_options)
8136                 return NULL;
8137 
8138         return trace_create_file(option, 0644, t_options,
8139                                  (void *)&tr->trace_flags_index[index],
8140                                  &trace_options_core_fops);
8141 }
8142 
8143 static void create_trace_options_dir(struct trace_array *tr)
8144 {
8145         struct dentry *t_options;
8146         bool top_level = tr == &global_trace;
8147         int i;
8148 
8149         t_options = trace_options_init_dentry(tr);
8150         if (!t_options)
8151                 return;
8152 
8153         for (i = 0; trace_options[i]; i++) {
8154                 if (top_level ||
8155                     !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8156                         create_trace_option_core_file(tr, trace_options[i], i);
8157         }
8158 }
8159 
8160 static ssize_t
8161 rb_simple_read(struct file *filp, char __user *ubuf,
8162                size_t cnt, loff_t *ppos)
8163 {
8164         struct trace_array *tr = filp->private_data;
8165         char buf[64];
8166         int r;
8167 
8168         r = tracer_tracing_is_on(tr);
8169         r = sprintf(buf, "%d\n", r);
8170 
8171         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8172 }
8173 
8174 static ssize_t
8175 rb_simple_write(struct file *filp, const char __user *ubuf,
8176                 size_t cnt, loff_t *ppos)
8177 {
8178         struct trace_array *tr = filp->private_data;
8179         struct ring_buffer *buffer = tr->trace_buffer.buffer;
8180         unsigned long val;
8181         int ret;
8182 
8183         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8184         if (ret)
8185                 return ret;
8186 
8187         if (buffer) {
8188                 mutex_lock(&trace_types_lock);
8189                 if (!!val == tracer_tracing_is_on(tr)) {
8190                         val = 0; /* do nothing */
8191                 } else if (val) {
8192                         tracer_tracing_on(tr);
8193                         if (tr->current_trace->start)
8194                                 tr->current_trace->start(tr);
8195                 } else {
8196                         tracer_tracing_off(tr);
8197                         if (tr->current_trace->stop)
8198                                 tr->current_trace->stop(tr);
8199                 }
8200                 mutex_unlock(&trace_types_lock);
8201         }
8202 
8203         (*ppos)++;
8204 
8205         return cnt;
8206 }
8207 
8208 static const struct file_operations rb_simple_fops = {
8209         .open           = tracing_open_generic_tr,
8210         .read           = rb_simple_read,
8211         .write          = rb_simple_write,
8212         .release        = tracing_release_generic_tr,
8213         .llseek         = default_llseek,
8214 };
8215 
8216 static ssize_t
8217 buffer_percent_read(struct file *filp, char __user *ubuf,
8218                     size_t cnt, loff_t *ppos)
8219 {
8220         struct trace_array *tr = filp->private_data;
8221         char buf[64];
8222         int r;
8223 
8224         r = tr->buffer_percent;
8225         r = sprintf(buf, "%d\n", r);
8226 
8227         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8228 }
8229 
8230 static ssize_t
8231 buffer_percent_write(struct file *filp, const char __user *ubuf,
8232                      size_t cnt, loff_t *ppos)
8233 {
8234         struct trace_array *tr = filp->private_data;
8235         unsigned long val;
8236         int ret;
8237 
8238         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8239         if (ret)
8240                 return ret;
8241 
8242         if (val > 100)
8243                 return -EINVAL;
8244 
8245         if (!val)
8246                 val = 1;
8247 
8248         tr->buffer_percent = val;
8249 
8250         (*ppos)++;
8251 
8252         return cnt;
8253 }
8254 
8255 static const struct file_operations buffer_percent_fops = {
8256         .open           = tracing_open_generic_tr,
8257         .read           = buffer_percent_read,
8258         .write          = buffer_percent_write,
8259         .release        = tracing_release_generic_tr,
8260         .llseek         = default_llseek,
8261 };
8262 
8263 static struct dentry *trace_instance_dir;
8264 
8265 static void
8266 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8267 
8268 static int
8269 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
8270 {
8271         enum ring_buffer_flags rb_flags;
8272 
8273         rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
8274 
8275         buf->tr = tr;
8276 
8277         buf->buffer = ring_buffer_alloc(size, rb_flags);
8278         if (!buf->buffer)
8279                 return -ENOMEM;
8280 
8281         buf->data = alloc_percpu(struct trace_array_cpu);
8282         if (!buf->data) {
8283                 ring_buffer_free(buf->buffer);
8284                 buf->buffer = NULL;
8285                 return -ENOMEM;
8286         }
8287 
8288         /* Allocate the first page for all buffers */
8289         set_buffer_entries(&tr->trace_buffer,
8290                            ring_buffer_size(tr->trace_buffer.buffer, 0));
8291 
8292         return 0;
8293 }
8294 
8295 static int allocate_trace_buffers(struct trace_array *tr, int size)
8296 {
8297         int ret;
8298 
8299         ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
8300         if (ret)
8301                 return ret;
8302 
8303 #ifdef CONFIG_TRACER_MAX_TRACE
8304         ret = allocate_trace_buffer(tr, &tr->max_buffer,
8305                                     allocate_snapshot ? size : 1);
8306         if (WARN_ON(ret)) {
8307                 ring_buffer_free(tr->trace_buffer.buffer);
8308                 tr->trace_buffer.buffer = NULL;
8309                 free_percpu(tr->trace_buffer.data);
8310                 tr->trace_buffer.data = NULL;
8311                 return -ENOMEM;
8312         }
8313         tr->allocated_snapshot = allocate_snapshot;
8314 
8315         /*
8316          * Only the top level trace array gets its snapshot allocated
8317          * from the kernel command line.
8318          */
8319         allocate_snapshot = false;
8320 #endif
8321 
8322         /*
8323          * Because of some magic with the way alloc_percpu() works on
8324          * x86_64, we need to synchronize the pgd of all the tables,
8325          * otherwise the trace events that happen in x86_64 page fault
8326          * handlers can't cope with accessing the chance that a
8327          * alloc_percpu()'d memory might be touched in the page fault trace
8328          * event. Oh, and we need to audit all other alloc_percpu() and vmalloc()
8329          * calls in tracing, because something might get triggered within a
8330          * page fault trace event!
8331          */
8332         vmalloc_sync_mappings();
8333 
8334         return 0;
8335 }
8336 
8337 static void free_trace_buffer(struct trace_buffer *buf)
8338 {
8339         if (buf->buffer) {
8340                 ring_buffer_free(buf->buffer);
8341                 buf->buffer = NULL;
8342                 free_percpu(buf->data);
8343                 buf->data = NULL;
8344         }
8345 }
8346 
8347 static void free_trace_buffers(struct trace_array *tr)
8348 {
8349         if (!tr)
8350                 return;
8351 
8352         free_trace_buffer(&tr->trace_buffer);
8353 
8354 #ifdef CONFIG_TRACER_MAX_TRACE
8355         free_trace_buffer(&tr->max_buffer);
8356 #endif
8357 }
8358 
8359 static void init_trace_flags_index(struct trace_array *tr)
8360 {
8361         int i;
8362 
8363         /* Used by the trace options files */
8364         for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8365                 tr->trace_flags_index[i] = i;
8366 }
8367 
8368 static void __update_tracer_options(struct trace_array *tr)
8369 {
8370         struct tracer *t;
8371 
8372         for (t = trace_types; t; t = t->next)
8373                 add_tracer_options(tr, t);
8374 }
8375 
8376 static void update_tracer_options(struct trace_array *tr)
8377 {
8378         mutex_lock(&trace_types_lock);
8379         __update_tracer_options(tr);
8380         mutex_unlock(&trace_types_lock);
8381 }
8382 
8383 struct trace_array *trace_array_create(const char *name)
8384 {
8385         struct trace_array *tr;
8386         int ret;
8387 
8388         mutex_lock(&event_mutex);
8389         mutex_lock(&trace_types_lock);
8390 
8391         ret = -EEXIST;
8392         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8393                 if (tr->name && strcmp(tr->name, name) == 0)
8394                         goto out_unlock;
8395         }
8396 
8397         ret = -ENOMEM;
8398         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8399         if (!tr)
8400                 goto out_unlock;
8401 
8402         tr->name = kstrdup(name, GFP_KERNEL);
8403         if (!tr->name)
8404                 goto out_free_tr;
8405 
8406         if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8407                 goto out_free_tr;
8408 
8409         tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8410 
8411         cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8412 
8413         raw_spin_lock_init(&tr->start_lock);
8414 
8415         tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8416 
8417         tr->current_trace = &nop_trace;
8418 
8419         INIT_LIST_HEAD(&tr->systems);
8420         INIT_LIST_HEAD(&tr->events);
8421         INIT_LIST_HEAD(&tr->hist_vars);
8422         INIT_LIST_HEAD(&tr->err_log);
8423 
8424         if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8425                 goto out_free_tr;
8426 
8427         tr->dir = tracefs_create_dir(name, trace_instance_dir);
8428         if (!tr->dir)
8429                 goto out_free_tr;
8430 
8431         ret = event_trace_add_tracer(tr->dir, tr);
8432         if (ret) {
8433                 tracefs_remove_recursive(tr->dir);
8434                 goto out_free_tr;
8435         }
8436 
8437         ftrace_init_trace_array(tr);
8438 
8439         init_tracer_tracefs(tr, tr->dir);
8440         init_trace_flags_index(tr);
8441         __update_tracer_options(tr);
8442 
8443         list_add(&tr->list, &ftrace_trace_arrays);
8444 
8445         mutex_unlock(&trace_types_lock);
8446         mutex_unlock(&event_mutex);
8447 
8448         return tr;
8449 
8450  out_free_tr:
8451         free_trace_buffers(tr);
8452         free_cpumask_var(tr->tracing_cpumask);
8453         kfree(tr->name);
8454         kfree(tr);
8455 
8456  out_unlock:
8457         mutex_unlock(&trace_types_lock);
8458         mutex_unlock(&event_mutex);
8459 
8460         return ERR_PTR(ret);
8461 }
8462 EXPORT_SYMBOL_GPL(trace_array_create);
8463 
8464 static int instance_mkdir(const char *name)
8465 {
8466         return PTR_ERR_OR_ZERO(trace_array_create(name));
8467 }
8468 
8469 static int __remove_instance(struct trace_array *tr)
8470 {
8471         int i;
8472 
8473         if (tr->ref || (tr->current_trace && tr->current_trace->ref))
8474                 return -EBUSY;
8475 
8476         list_del(&tr->list);
8477 
8478         /* Disable all the flags that were enabled coming in */
8479         for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8480                 if ((1 << i) & ZEROED_TRACE_FLAGS)
8481                         set_tracer_flag(tr, 1 << i, 0);
8482         }
8483 
8484         tracing_set_nop(tr);
8485         clear_ftrace_function_probes(tr);
8486         event_trace_del_tracer(tr);
8487         ftrace_clear_pids(tr);
8488         ftrace_destroy_function_files(tr);
8489         tracefs_remove_recursive(tr->dir);
8490         free_trace_buffers(tr);
8491 
8492         for (i = 0; i < tr->nr_topts; i++) {
8493                 kfree(tr->topts[i].topts);
8494         }
8495         kfree(tr->topts);
8496 
8497         free_cpumask_var(tr->tracing_cpumask);
8498         kfree(tr->name);
8499         kfree(tr);
8500         tr = NULL;
8501 
8502         return 0;
8503 }
8504 
8505 int trace_array_destroy(struct trace_array *tr)
8506 {
8507         int ret;
8508 
8509         if (!tr)
8510                 return -EINVAL;
8511 
8512         mutex_lock(&event_mutex);
8513         mutex_lock(&trace_types_lock);
8514 
8515         ret = __remove_instance(tr);
8516 
8517         mutex_unlock(&trace_types_lock);
8518         mutex_unlock(&event_mutex);
8519 
8520         return ret;
8521 }
8522 EXPORT_SYMBOL_GPL(trace_array_destroy);
8523 
8524 static int instance_rmdir(const char *name)
8525 {
8526         struct trace_array *tr;
8527         int ret;
8528 
8529         mutex_lock(&event_mutex);
8530         mutex_lock(&trace_types_lock);
8531 
8532         ret = -ENODEV;
8533         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8534                 if (tr->name && strcmp(tr->name, name) == 0) {
8535                         ret = __remove_instance(tr);
8536                         break;
8537                 }
8538         }
8539 
8540         mutex_unlock(&trace_types_lock);
8541         mutex_unlock(&event_mutex);
8542 
8543         return ret;
8544 }
8545 
8546 static __init void create_trace_instances(struct dentry *d_tracer)
8547 {
8548         trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8549                                                          instance_mkdir,
8550                                                          instance_rmdir);
8551         if (WARN_ON(!trace_instance_dir))
8552                 return;
8553 }
8554 
8555 static void
8556 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8557 {
8558         struct trace_event_file *file;
8559         int cpu;
8560 
8561         trace_create_file("available_tracers", 0444, d_tracer,
8562                         tr, &show_traces_fops);
8563 
8564         trace_create_file("current_tracer", 0644, d_tracer,
8565                         tr, &set_tracer_fops);
8566 
8567         trace_create_file("tracing_cpumask", 0644, d_tracer,
8568                           tr, &tracing_cpumask_fops);
8569 
8570         trace_create_file("trace_options", 0644, d_tracer,
8571                           tr, &tracing_iter_fops);
8572 
8573         trace_create_file("trace", 0644, d_tracer,
8574                           tr, &tracing_fops);
8575 
8576         trace_create_file("trace_pipe", 0444, d_tracer,
8577                           tr, &tracing_pipe_fops);
8578 
8579         trace_create_file("buffer_size_kb", 0644, d_tracer,
8580                           tr, &tracing_entries_fops);
8581 
8582         trace_create_file("buffer_total_size_kb", 0444, d_tracer,
8583                           tr, &tracing_total_entries_fops);
8584 
8585         trace_create_file("free_buffer", 0200, d_tracer,
8586                           tr, &tracing_free_buffer_fops);
8587 
8588         trace_create_file("trace_marker", 0220, d_tracer,
8589                           tr, &tracing_mark_fops);
8590 
8591         file = __find_event_file(tr, "ftrace", "print");
8592         if (file && file->dir)
8593                 trace_create_file("trigger", 0644, file->dir, file,
8594                                   &event_trigger_fops);
8595         tr->trace_marker_file = file;
8596 
8597         trace_create_file("trace_marker_raw", 0220, d_tracer,
8598                           tr, &tracing_mark_raw_fops);
8599 
8600         trace_create_file("trace_clock", 0644, d_tracer, tr,
8601                           &trace_clock_fops);
8602 
8603         trace_create_file("tracing_on", 0644, d_tracer,
8604                           tr, &rb_simple_fops);
8605 
8606         trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8607                           &trace_time_stamp_mode_fops);
8608 
8609         tr->buffer_percent = 50;
8610 
8611         trace_create_file("buffer_percent", 0444, d_tracer,
8612                         tr, &buffer_percent_fops);
8613 
8614         create_trace_options_dir(tr);
8615 
8616 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
8617         trace_create_file("tracing_max_latency", 0644, d_tracer,
8618                         &tr->max_latency, &tracing_max_lat_fops);
8619 #endif
8620 
8621         if (ftrace_create_function_files(tr, d_tracer))
8622                 WARN(1, "Could not allocate function filter files");
8623 
8624 #ifdef CONFIG_TRACER_SNAPSHOT
8625         trace_create_file("snapshot", 0644, d_tracer,
8626                           tr, &snapshot_fops);
8627 #endif
8628 
8629         trace_create_file("error_log", 0644, d_tracer,
8630                           tr, &tracing_err_log_fops);
8631 
8632         for_each_tracing_cpu(cpu)
8633                 tracing_init_tracefs_percpu(tr, cpu);
8634 
8635         ftrace_init_tracefs(tr, d_tracer);
8636 }
8637 
8638 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
8639 {
8640         struct vfsmount *mnt;
8641         struct file_system_type *type;
8642 
8643         /*
8644          * To maintain backward compatibility for tools that mount
8645          * debugfs to get to the tracing facility, tracefs is automatically
8646          * mounted to the debugfs/tracing directory.
8647          */
8648         type = get_fs_type("tracefs");
8649         if (!type)
8650                 return NULL;
8651         mnt = vfs_submount(mntpt, type, "tracefs", NULL);
8652         put_filesystem(type);
8653         if (IS_ERR(mnt))
8654                 return NULL;
8655         mntget(mnt);
8656 
8657         return mnt;
8658 }
8659 
8660 /**
8661  * tracing_init_dentry - initialize top level trace array
8662  *
8663  * This is called when creating files or directories in the tracing
8664  * directory. It is called via fs_initcall() by any of the boot up code
8665  * and expects to return the dentry of the top level tracing directory.
8666  */
8667 struct dentry *tracing_init_dentry(void)
8668 {
8669         struct trace_array *tr = &global_trace;
8670 
8671         if (security_locked_down(LOCKDOWN_TRACEFS)) {
8672                 pr_warning("Tracing disabled due to lockdown\n");
8673                 return ERR_PTR(-EPERM);
8674         }
8675 
8676         /* The top level trace array uses  NULL as parent */
8677         if (tr->dir)
8678                 return NULL;
8679 
8680         if (WARN_ON(!tracefs_initialized()) ||
8681                 (IS_ENABLED(CONFIG_DEBUG_FS) &&
8682                  WARN_ON(!debugfs_initialized())))
8683                 return ERR_PTR(-ENODEV);
8684 
8685         /*
8686          * As there may still be users that expect the tracing
8687          * files to exist in debugfs/tracing, we must automount
8688          * the tracefs file system there, so older tools still
8689          * work with the newer kerenl.
8690          */
8691         tr->dir = debugfs_create_automount("tracing", NULL,
8692                                            trace_automount, NULL);
8693 
8694         return NULL;
8695 }
8696 
8697 extern struct trace_eval_map *__start_ftrace_eval_maps[];
8698 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8699 
8700 static void __init trace_eval_init(void)
8701 {
8702         int len;
8703 
8704         len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8705         trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8706 }
8707 
8708 #ifdef CONFIG_MODULES
8709 static void trace_module_add_evals(struct module *mod)
8710 {
8711         if (!mod->num_trace_evals)
8712                 return;
8713 
8714         /*
8715          * Modules with bad taint do not have events created, do
8716          * not bother with enums either.
8717          */
8718         if (trace_module_has_bad_taint(mod))
8719                 return;
8720 
8721         trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
8722 }
8723 
8724 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
8725 static void trace_module_remove_evals(struct module *mod)
8726 {
8727         union trace_eval_map_item *map;
8728         union trace_eval_map_item **last = &trace_eval_maps;
8729 
8730         if (!mod->num_trace_evals)
8731                 return;
8732 
8733         mutex_lock(&trace_eval_mutex);
8734 
8735         map = trace_eval_maps;
8736 
8737         while (map) {
8738                 if (map->head.mod == mod)
8739                         break;
8740                 map = trace_eval_jmp_to_tail(map);
8741                 last = &map->tail.next;
8742                 map = map->tail.next;
8743         }
8744         if (!map)
8745                 goto out;
8746 
8747         *last = trace_eval_jmp_to_tail(map)->tail.next;
8748         kfree(map);
8749  out:
8750         mutex_unlock(&trace_eval_mutex);
8751 }
8752 #else
8753 static inline void trace_module_remove_evals(struct module *mod) { }
8754 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
8755 
8756 static int trace_module_notify(struct notifier_block *self,
8757                                unsigned long val, void *data)
8758 {
8759         struct module *mod = data;
8760 
8761         switch (val) {
8762         case MODULE_STATE_COMING:
8763                 trace_module_add_evals(mod);
8764                 break;
8765         case MODULE_STATE_GOING:
8766                 trace_module_remove_evals(mod);
8767                 break;
8768         }
8769 
8770         return 0;
8771 }
8772 
8773 static struct notifier_block trace_module_nb = {
8774         .notifier_call = trace_module_notify,
8775         .priority = 0,
8776 };
8777 #endif /* CONFIG_MODULES */
8778 
8779 static __init int tracer_init_tracefs(void)
8780 {
8781         struct dentry *d_tracer;
8782 
8783         trace_access_lock_init();
8784 
8785         d_tracer = tracing_init_dentry();
8786         if (IS_ERR(d_tracer))
8787                 return 0;
8788 
8789         event_trace_init();
8790 
8791         init_tracer_tracefs(&global_trace, d_tracer);
8792         ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
8793 
8794         trace_create_file("tracing_thresh", 0644, d_tracer,
8795                         &global_trace, &tracing_thresh_fops);
8796 
8797         trace_create_file("README", 0444, d_tracer,
8798                         NULL, &tracing_readme_fops);
8799 
8800         trace_create_file("saved_cmdlines", 0444, d_tracer,
8801                         NULL, &tracing_saved_cmdlines_fops);
8802 
8803         trace_create_file("saved_cmdlines_size", 0644, d_tracer,
8804                           NULL, &tracing_saved_cmdlines_size_fops);
8805 
8806         trace_create_file("saved_tgids", 0444, d_tracer,
8807                         NULL, &tracing_saved_tgids_fops);
8808 
8809         trace_eval_init();
8810 
8811         trace_create_eval_file(d_tracer);
8812 
8813 #ifdef CONFIG_MODULES
8814         register_module_notifier(&trace_module_nb);
8815 #endif
8816 
8817 #ifdef CONFIG_DYNAMIC_FTRACE
8818         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8819                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
8820 #endif
8821 
8822         create_trace_instances(d_tracer);
8823 
8824         update_tracer_options(&global_trace);
8825 
8826         return 0;
8827 }
8828 
8829 static int trace_panic_handler(struct notifier_block *this,
8830                                unsigned long event, void *unused)
8831 {
8832         if (ftrace_dump_on_oops)
8833                 ftrace_dump(ftrace_dump_on_oops);
8834         return NOTIFY_OK;
8835 }
8836 
8837 static struct notifier_block trace_panic_notifier = {
8838         .notifier_call  = trace_panic_handler,
8839         .next           = NULL,
8840         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
8841 };
8842 
8843 static int trace_die_handler(struct notifier_block *self,
8844                              unsigned long val,
8845                              void *data)
8846 {
8847         switch (val) {
8848         case DIE_OOPS:
8849                 if (ftrace_dump_on_oops)
8850                         ftrace_dump(ftrace_dump_on_oops);
8851                 break;
8852         default:
8853                 break;
8854         }
8855         return NOTIFY_OK;
8856 }
8857 
8858 static struct notifier_block trace_die_notifier = {
8859         .notifier_call = trace_die_handler,
8860         .priority = 200
8861 };
8862 
8863 /*
8864  * printk is set to max of 1024, we really don't need it that big.
8865  * Nothing should be printing 1000 characters anyway.
8866  */
8867 #define TRACE_MAX_PRINT         1000
8868 
8869 /*
8870  * Define here KERN_TRACE so that we have one place to modify
8871  * it if we decide to change what log level the ftrace dump
8872  * should be at.
8873  */
8874 #define KERN_TRACE              KERN_EMERG
8875 
8876 void
8877 trace_printk_seq(struct trace_seq *s)
8878 {
8879         /* Probably should print a warning here. */
8880         if (s->seq.len >= TRACE_MAX_PRINT)
8881                 s->seq.len = TRACE_MAX_PRINT;
8882 
8883         /*
8884          * More paranoid code. Although the buffer size is set to
8885          * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
8886          * an extra layer of protection.
8887          */
8888         if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
8889                 s->seq.len = s->seq.size - 1;
8890 
8891         /* should be zero ended, but we are paranoid. */
8892         s->buffer[s->seq.len] = 0;
8893 
8894         printk(KERN_TRACE "%s", s->buffer);
8895 
8896         trace_seq_init(s);
8897 }
8898 
8899 void trace_init_global_iter(struct trace_iterator *iter)
8900 {
8901         iter->tr = &global_trace;
8902         iter->trace = iter->tr->current_trace;
8903         iter->cpu_file = RING_BUFFER_ALL_CPUS;
8904         iter->trace_buffer = &global_trace.trace_buffer;
8905 
8906         if (iter->trace && iter->trace->open)
8907                 iter->trace->open(iter);
8908 
8909         /* Annotate start of buffers if we had overruns */
8910         if (ring_buffer_overruns(iter->trace_buffer->buffer))
8911                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
8912 
8913         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
8914         if (trace_clocks[iter->tr->clock_id].in_ns)
8915                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
8916 }
8917 
8918 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
8919 {
8920         /* use static because iter can be a bit big for the stack */
8921         static struct trace_iterator iter;
8922         static atomic_t dump_running;
8923         struct trace_array *tr = &global_trace;
8924         unsigned int old_userobj;
8925         unsigned long flags;
8926         int cnt = 0, cpu;
8927 
8928         /* Only allow one dump user at a time. */
8929         if (atomic_inc_return(&dump_running) != 1) {
8930                 atomic_dec(&dump_running);
8931                 return;
8932         }
8933 
8934         /*
8935          * Always turn off tracing when we dump.
8936          * We don't need to show trace output of what happens
8937          * between multiple crashes.
8938          *
8939          * If the user does a sysrq-z, then they can re-enable
8940          * tracing with echo 1 > tracing_on.
8941          */
8942         tracing_off();
8943 
8944         local_irq_save(flags);
8945         printk_nmi_direct_enter();
8946 
8947         /* Simulate the iterator */
8948         trace_init_global_iter(&iter);
8949 
8950         for_each_tracing_cpu(cpu) {
8951                 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
8952         }
8953 
8954         old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
8955 
8956         /* don't look at user memory in panic mode */
8957         tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
8958 
8959         switch (oops_dump_mode) {
8960         case DUMP_ALL:
8961                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
8962                 break;
8963         case DUMP_ORIG:
8964                 iter.cpu_file = raw_smp_processor_id();
8965                 break;
8966         case DUMP_NONE:
8967                 goto out_enable;
8968         default:
8969                 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
8970                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
8971         }
8972 
8973         printk(KERN_TRACE "Dumping ftrace buffer:\n");
8974 
8975         /* Did function tracer already get disabled? */
8976         if (ftrace_is_dead()) {
8977                 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
8978                 printk("#          MAY BE MISSING FUNCTION EVENTS\n");
8979         }
8980 
8981         /*
8982          * We need to stop all tracing on all CPUS to read the
8983          * the next buffer. This is a bit expensive, but is
8984          * not done often. We fill all what we can read,
8985          * and then release the locks again.
8986          */
8987 
8988         while (!trace_empty(&iter)) {
8989 
8990                 if (!cnt)
8991                         printk(KERN_TRACE "---------------------------------\n");
8992 
8993                 cnt++;
8994 
8995                 trace_iterator_reset(&iter);
8996                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
8997 
8998                 if (trace_find_next_entry_inc(&iter) != NULL) {
8999                         int ret;
9000 
9001                         ret = print_trace_line(&iter);
9002                         if (ret != TRACE_TYPE_NO_CONSUME)
9003                                 trace_consume(&iter);
9004                 }
9005                 touch_nmi_watchdog();
9006 
9007                 trace_printk_seq(&iter.seq);
9008         }
9009 
9010         if (!cnt)
9011                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
9012         else
9013                 printk(KERN_TRACE "---------------------------------\n");
9014 
9015  out_enable:
9016         tr->trace_flags |= old_userobj;
9017 
9018         for_each_tracing_cpu(cpu) {
9019                 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
9020         }
9021         atomic_dec(&dump_running);
9022         printk_nmi_direct_exit();
9023         local_irq_restore(flags);
9024 }
9025 EXPORT_SYMBOL_GPL(ftrace_dump);
9026 
9027 int trace_run_command(const char *buf, int (*createfn)(int, char **))
9028 {
9029         char **argv;
9030         int argc, ret;
9031 
9032         argc = 0;
9033         ret = 0;
9034         argv = argv_split(GFP_KERNEL, buf, &argc);
9035         if (!argv)
9036                 return -ENOMEM;
9037 
9038         if (argc)
9039                 ret = createfn(argc, argv);
9040 
9041         argv_free(argv);
9042 
9043         return ret;
9044 }
9045 
9046 #define WRITE_BUFSIZE  4096
9047 
9048 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9049                                 size_t count, loff_t *ppos,
9050                                 int (*createfn)(int, char **))
9051 {
9052         char *kbuf, *buf, *tmp;
9053         int ret = 0;
9054         size_t done = 0;
9055         size_t size;
9056 
9057         kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9058         if (!kbuf)
9059                 return -ENOMEM;
9060 
9061         while (done < count) {
9062                 size = count - done;
9063 
9064                 if (size >= WRITE_BUFSIZE)
9065                         size = WRITE_BUFSIZE - 1;
9066 
9067                 if (copy_from_user(kbuf, buffer + done, size)) {
9068                         ret = -EFAULT;
9069                         goto out;
9070                 }
9071                 kbuf[size] = '\0';
9072                 buf = kbuf;
9073                 do {
9074                         tmp = strchr(buf, '\n');
9075                         if (tmp) {
9076                                 *tmp = '\0';
9077                                 size = tmp - buf + 1;
9078                         } else {
9079                                 size = strlen(buf);
9080                                 if (done + size < count) {
9081                                         if (buf != kbuf)
9082                                                 break;
9083                                         /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9084                                         pr_warn("Line length is too long: Should be less than %d\n",
9085                                                 WRITE_BUFSIZE - 2);
9086                                         ret = -EINVAL;
9087                                         goto out;
9088                                 }
9089                         }
9090                         done += size;
9091 
9092                         /* Remove comments */
9093                         tmp = strchr(buf, '#');
9094 
9095                         if (tmp)
9096                                 *tmp = '\0';
9097 
9098                         ret = trace_run_command(buf, createfn);
9099                         if (ret)
9100                                 goto out;
9101                         buf += size;
9102 
9103                 } while (done < count);
9104         }
9105         ret = done;
9106 
9107 out:
9108         kfree(kbuf);
9109 
9110         return ret;
9111 }
9112 
9113 __init static int tracer_alloc_buffers(void)
9114 {
9115         int ring_buf_size;
9116         int ret = -ENOMEM;
9117 
9118 
9119         if (security_locked_down(LOCKDOWN_TRACEFS)) {
9120                 pr_warning("Tracing disabled due to lockdown\n");
9121                 return -EPERM;
9122         }
9123 
9124         /*
9125          * Make sure we don't accidently add more trace options
9126          * than we have bits for.
9127          */
9128         BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9129 
9130         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9131                 goto out;
9132 
9133         if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9134                 goto out_free_buffer_mask;
9135 
9136         /* Only allocate trace_printk buffers if a trace_printk exists */
9137         if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
9138                 /* Must be called before global_trace.buffer is allocated */
9139                 trace_printk_init_buffers();
9140 
9141         /* To save memory, keep the ring buffer size to its minimum */
9142         if (ring_buffer_expanded)
9143                 ring_buf_size = trace_buf_size;
9144         else
9145                 ring_buf_size = 1;
9146 
9147         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9148         cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9149 
9150         raw_spin_lock_init(&global_trace.start_lock);
9151 
9152         /*
9153          * The prepare callbacks allocates some memory for the ring buffer. We
9154          * don't free the buffer if the if the CPU goes down. If we were to free
9155          * the buffer, then the user would lose any trace that was in the
9156          * buffer. The memory will be removed once the "instance" is removed.
9157          */
9158         ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9159                                       "trace/RB:preapre", trace_rb_cpu_prepare,
9160                                       NULL);
9161         if (ret < 0)
9162                 goto out_free_cpumask;
9163         /* Used for event triggers */
9164         ret = -ENOMEM;
9165         temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9166         if (!temp_buffer)
9167                 goto out_rm_hp_state;
9168 
9169         if (trace_create_savedcmd() < 0)
9170                 goto out_free_temp_buffer;
9171 
9172         /* TODO: make the number of buffers hot pluggable with CPUS */
9173         if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9174                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
9175                 WARN_ON(1);
9176                 goto out_free_savedcmd;
9177         }
9178 
9179         if (global_trace.buffer_disabled)
9180                 tracing_off();
9181 
9182         if (trace_boot_clock) {
9183                 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9184                 if (ret < 0)
9185                         pr_warn("Trace clock %s not defined, going back to default\n",
9186                                 trace_boot_clock);
9187         }
9188 
9189         /*
9190          * register_tracer() might reference current_trace, so it
9191          * needs to be set before we register anything. This is
9192          * just a bootstrap of current_trace anyway.
9193          */
9194         global_trace.current_trace = &nop_trace;
9195 
9196         global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9197 
9198         ftrace_init_global_array_ops(&global_trace);
9199 
9200         init_trace_flags_index(&global_trace);
9201 
9202         register_tracer(&nop_trace);
9203 
9204         /* Function tracing may start here (via kernel command line) */
9205         init_function_trace();
9206 
9207         /* All seems OK, enable tracing */
9208         tracing_disabled = 0;
9209 
9210         atomic_notifier_chain_register(&panic_notifier_list,
9211                                        &trace_panic_notifier);
9212 
9213         register_die_notifier(&trace_die_notifier);
9214 
9215         global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9216 
9217         INIT_LIST_HEAD(&global_trace.systems);
9218         INIT_LIST_HEAD(&global_trace.events);
9219         INIT_LIST_HEAD(&global_trace.hist_vars);
9220         INIT_LIST_HEAD(&global_trace.err_log);
9221         list_add(&global_trace.list, &ftrace_trace_arrays);
9222 
9223         apply_trace_boot_options();
9224 
9225         register_snapshot_cmd();
9226 
9227         return 0;
9228 
9229 out_free_savedcmd:
9230         free_saved_cmdlines_buffer(savedcmd);
9231 out_free_temp_buffer:
9232         ring_buffer_free(temp_buffer);
9233 out_rm_hp_state:
9234         cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9235 out_free_cpumask:
9236         free_cpumask_var(global_trace.tracing_cpumask);
9237 out_free_buffer_mask:
9238         free_cpumask_var(tracing_buffer_mask);
9239 out:
9240         return ret;
9241 }
9242 
9243 void __init early_trace_init(void)
9244 {
9245         if (tracepoint_printk) {
9246                 tracepoint_print_iter =
9247                         kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
9248                 if (WARN_ON(!tracepoint_print_iter))
9249                         tracepoint_printk = 0;
9250                 else
9251                         static_key_enable(&tracepoint_printk_key.key);
9252         }
9253         tracer_alloc_buffers();
9254 }
9255 
9256 void __init trace_init(void)
9257 {
9258         trace_event_init();
9259 }
9260 
9261 __init static int clear_boot_tracer(void)
9262 {
9263         /*
9264          * The default tracer at boot buffer is an init section.
9265          * This function is called in lateinit. If we did not
9266          * find the boot tracer, then clear it out, to prevent
9267          * later registration from accessing the buffer that is
9268          * about to be freed.
9269          */
9270         if (!default_bootup_tracer)
9271                 return 0;
9272 
9273         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
9274                default_bootup_tracer);
9275         default_bootup_tracer = NULL;
9276 
9277         return 0;
9278 }
9279 
9280 fs_initcall(tracer_init_tracefs);
9281 late_initcall_sync(clear_boot_tracer);
9282 
9283 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9284 __init static int tracing_set_default_clock(void)
9285 {
9286         /* sched_clock_stable() is determined in late_initcall */
9287         if (!trace_boot_clock && !sched_clock_stable()) {
9288                 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9289                         pr_warn("Can not set tracing clock due to lockdown\n");
9290                         return -EPERM;
9291                 }
9292 
9293                 printk(KERN_WARNING
9294                        "Unstable clock detected, switching default tracing clock to \"global\"\n"
9295                        "If you want to keep using the local clock, then add:\n"
9296                        "  \"trace_clock=local\"\n"
9297                        "on the kernel command line\n");
9298                 tracing_set_clock(&global_trace, "global");
9299         }
9300 
9301         return 0;
9302 }
9303 late_initcall_sync(tracing_set_default_clock);
9304 #endif