43 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)  in trace_kprobe_is_return()  argument
45 	return tk->rp.handler != NULL;  in trace_kprobe_is_return()
48 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)  in trace_kprobe_symbol()  argument
50 	return tk->symbol ? tk->symbol : "unknown";  in trace_kprobe_symbol()
53 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)  in trace_kprobe_offset()  argument
55 	return tk->rp.kp.offset;  in trace_kprobe_offset()
58 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)  in trace_kprobe_has_gone()  argument
60 	return !!(kprobe_gone(&tk->rp.kp));  in trace_kprobe_has_gone()
63 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,  in trace_kprobe_within_module()  argument
67 	const char *name = trace_kprobe_symbol(tk);  in trace_kprobe_within_module()
71 static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)  in trace_kprobe_is_on_module()  argument
73 	return !!strchr(trace_kprobe_symbol(tk), ':');  in trace_kprobe_is_on_module()
76 static int register_kprobe_event(struct trace_kprobe *tk);
77 static int unregister_kprobe_event(struct trace_kprobe *tk);
270 	struct trace_kprobe *tk;  in alloc_trace_kprobe()  local
273 	tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);  in alloc_trace_kprobe()
274 	if (!tk)  in alloc_trace_kprobe()
278 		tk->symbol = kstrdup(symbol, GFP_KERNEL);  in alloc_trace_kprobe()
279 		if (!tk->symbol)  in alloc_trace_kprobe()
281 		tk->rp.kp.symbol_name = tk->symbol;  in alloc_trace_kprobe()
282 		tk->rp.kp.offset = offs;  in alloc_trace_kprobe()
284 		tk->rp.kp.addr = addr;  in alloc_trace_kprobe()
287 		tk->rp.handler = kretprobe_dispatcher;  in alloc_trace_kprobe()
289 		tk->rp.kp.pre_handler = kprobe_dispatcher;  in alloc_trace_kprobe()
296 	tk->tp.call.class = &tk->tp.class;  in alloc_trace_kprobe()
297 	tk->tp.call.name = kstrdup(event, GFP_KERNEL);  in alloc_trace_kprobe()
298 	if (!tk->tp.call.name)  in alloc_trace_kprobe()
306 	tk->tp.class.system = kstrdup(group, GFP_KERNEL);  in alloc_trace_kprobe()
307 	if (!tk->tp.class.system)  in alloc_trace_kprobe()
310 	INIT_LIST_HEAD(&tk->list);  in alloc_trace_kprobe()
311 	INIT_LIST_HEAD(&tk->tp.files);  in alloc_trace_kprobe()
312 	return tk;  in alloc_trace_kprobe()
314 	kfree(tk->tp.call.name);  in alloc_trace_kprobe()
315 	kfree(tk->symbol);  in alloc_trace_kprobe()
316 	kfree(tk);  in alloc_trace_kprobe()
320 static void free_trace_kprobe(struct trace_kprobe *tk)  in free_trace_kprobe()  argument
324 	for (i = 0; i < tk->tp.nr_args; i++)  in free_trace_kprobe()
325 		traceprobe_free_probe_arg(&tk->tp.args[i]);  in free_trace_kprobe()
327 	kfree(tk->tp.call.class->system);  in free_trace_kprobe()
328 	kfree(tk->tp.call.name);  in free_trace_kprobe()
329 	kfree(tk->symbol);  in free_trace_kprobe()
330 	kfree(tk);  in free_trace_kprobe()
336 	struct trace_kprobe *tk;  in find_trace_kprobe()  local
338 	list_for_each_entry(tk, &probe_list, list)  in find_trace_kprobe()
339 		if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&  in find_trace_kprobe()
340 		    strcmp(tk->tp.call.class->system, group) == 0)  in find_trace_kprobe()
341 			return tk;  in find_trace_kprobe()
350 enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)  in enable_trace_kprobe()  argument
364 		list_add_tail_rcu(&link->list, &tk->tp.files);  in enable_trace_kprobe()
366 		tk->tp.flags |= TP_FLAG_TRACE;  in enable_trace_kprobe()
368 		tk->tp.flags |= TP_FLAG_PROFILE;  in enable_trace_kprobe()
370 	if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {  in enable_trace_kprobe()
371 		if (trace_kprobe_is_return(tk))  in enable_trace_kprobe()
372 			ret = enable_kretprobe(&tk->rp);  in enable_trace_kprobe()
374 			ret = enable_kprobe(&tk->rp.kp);  in enable_trace_kprobe()
385 disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)  in disable_trace_kprobe()  argument
392 		link = find_event_file_link(&tk->tp, file);  in disable_trace_kprobe()
400 		if (!list_empty(&tk->tp.files))  in disable_trace_kprobe()
403 		tk->tp.flags &= ~TP_FLAG_TRACE;  in disable_trace_kprobe()
405 		tk->tp.flags &= ~TP_FLAG_PROFILE;  in disable_trace_kprobe()
407 	if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {  in disable_trace_kprobe()
408 		if (trace_kprobe_is_return(tk))  in disable_trace_kprobe()
409 			disable_kretprobe(&tk->rp);  in disable_trace_kprobe()
411 			disable_kprobe(&tk->rp.kp);  in disable_trace_kprobe()
432 static int __register_trace_kprobe(struct trace_kprobe *tk)  in __register_trace_kprobe()  argument
436 	if (trace_probe_is_registered(&tk->tp))  in __register_trace_kprobe()
439 	for (i = 0; i < tk->tp.nr_args; i++)  in __register_trace_kprobe()
440 		traceprobe_update_arg(&tk->tp.args[i]);  in __register_trace_kprobe()
443 	if (trace_probe_is_enabled(&tk->tp))  in __register_trace_kprobe()
444 		tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;  in __register_trace_kprobe()
446 		tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;  in __register_trace_kprobe()
448 	if (trace_kprobe_is_return(tk))  in __register_trace_kprobe()
449 		ret = register_kretprobe(&tk->rp);  in __register_trace_kprobe()
451 		ret = register_kprobe(&tk->rp.kp);  in __register_trace_kprobe()
454 		tk->tp.flags |= TP_FLAG_REGISTERED;  in __register_trace_kprobe()
457 			   trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret);  in __register_trace_kprobe()
458 		if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {  in __register_trace_kprobe()
465 				   tk->rp.kp.addr);  in __register_trace_kprobe()
474 static void __unregister_trace_kprobe(struct trace_kprobe *tk)  in __unregister_trace_kprobe()  argument
476 	if (trace_probe_is_registered(&tk->tp)) {  in __unregister_trace_kprobe()
477 		if (trace_kprobe_is_return(tk))  in __unregister_trace_kprobe()
478 			unregister_kretprobe(&tk->rp);  in __unregister_trace_kprobe()
480 			unregister_kprobe(&tk->rp.kp);  in __unregister_trace_kprobe()
481 		tk->tp.flags &= ~TP_FLAG_REGISTERED;  in __unregister_trace_kprobe()
483 		if (tk->rp.kp.symbol_name)  in __unregister_trace_kprobe()
484 			tk->rp.kp.addr = NULL;  in __unregister_trace_kprobe()
489 static int unregister_trace_kprobe(struct trace_kprobe *tk)  in unregister_trace_kprobe()  argument
492 	if (trace_probe_is_enabled(&tk->tp))  in unregister_trace_kprobe()
496 	if (unregister_kprobe_event(tk))  in unregister_trace_kprobe()
499 	__unregister_trace_kprobe(tk);  in unregister_trace_kprobe()
500 	list_del(&tk->list);  in unregister_trace_kprobe()
506 static int register_trace_kprobe(struct trace_kprobe *tk)  in register_trace_kprobe()  argument
514 	old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),  in register_trace_kprobe()
515 			tk->tp.call.class->system);  in register_trace_kprobe()
524 	ret = register_kprobe_event(tk);  in register_trace_kprobe()
531 	ret = __register_trace_kprobe(tk);  in register_trace_kprobe()
533 		unregister_kprobe_event(tk);  in register_trace_kprobe()
535 		list_add_tail(&tk->list, &probe_list);  in register_trace_kprobe()
547 	struct trace_kprobe *tk;  in trace_kprobe_module_callback()  local
555 	list_for_each_entry(tk, &probe_list, list) {  in trace_kprobe_module_callback()
556 		if (trace_kprobe_within_module(tk, mod)) {  in trace_kprobe_module_callback()
558 			__unregister_trace_kprobe(tk);  in trace_kprobe_module_callback()
559 			ret = __register_trace_kprobe(tk);  in trace_kprobe_module_callback()
563 					   trace_event_name(&tk->tp.call),  in trace_kprobe_module_callback()
597 	struct trace_kprobe *tk;  in create_trace_kprobe()  local
644 		tk = find_trace_kprobe(event, group);  in create_trace_kprobe()
645 		if (!tk) {  in create_trace_kprobe()
651 		ret = unregister_trace_kprobe(tk);  in create_trace_kprobe()
653 			free_trace_kprobe(tk);  in create_trace_kprobe()
700 	tk = alloc_trace_kprobe(group, event, addr, symbol, offset, argc,  in create_trace_kprobe()
702 	if (IS_ERR(tk)) {  in create_trace_kprobe()
704 			(int)PTR_ERR(tk));  in create_trace_kprobe()
705 		return PTR_ERR(tk);  in create_trace_kprobe()
711 		struct probe_arg *parg = &tk->tp.args[i];  in create_trace_kprobe()
714 		tk->tp.nr_args++;  in create_trace_kprobe()
742 							tk->tp.args, i)) {  in create_trace_kprobe()
750 		ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,  in create_trace_kprobe()
759 	ret = register_trace_kprobe(tk);  in create_trace_kprobe()
765 	free_trace_kprobe(tk);  in create_trace_kprobe()
771 	struct trace_kprobe *tk;  in release_all_trace_kprobes()  local
776 	list_for_each_entry(tk, &probe_list, list)  in release_all_trace_kprobes()
777 		if (trace_probe_is_enabled(&tk->tp)) {  in release_all_trace_kprobes()
783 		tk = list_entry(probe_list.next, struct trace_kprobe, list);  in release_all_trace_kprobes()
784 		ret = unregister_trace_kprobe(tk);  in release_all_trace_kprobes()
787 		free_trace_kprobe(tk);  in release_all_trace_kprobes()
815 	struct trace_kprobe *tk = v;  in probes_seq_show()  local
818 	seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');  in probes_seq_show()
819 	seq_printf(m, ":%s/%s", tk->tp.call.class->system,  in probes_seq_show()
820 			trace_event_name(&tk->tp.call));  in probes_seq_show()
822 	if (!tk->symbol)  in probes_seq_show()
823 		seq_printf(m, " 0x%p", tk->rp.kp.addr);  in probes_seq_show()
824 	else if (tk->rp.kp.offset)  in probes_seq_show()
825 		seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),  in probes_seq_show()
826 			   tk->rp.kp.offset);  in probes_seq_show()
828 		seq_printf(m, " %s", trace_kprobe_symbol(tk));  in probes_seq_show()
830 	for (i = 0; i < tk->tp.nr_args; i++)  in probes_seq_show()
831 		seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);  in probes_seq_show()
876 	struct trace_kprobe *tk = v;  in probes_profile_seq_show()  local
879 		   trace_event_name(&tk->tp.call), tk->nhit,  in probes_profile_seq_show()
880 		   tk->rp.kp.nmissed);  in probes_profile_seq_show()
907 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,  in __kprobe_trace_func()  argument
915 	struct trace_event_call *call = &tk->tp.call;  in __kprobe_trace_func()
925 	dsize = __get_data_size(&tk->tp, regs);  in __kprobe_trace_func()
926 	size = sizeof(*entry) + tk->tp.size + dsize;  in __kprobe_trace_func()
935 	entry->ip = (unsigned long)tk->rp.kp.addr;  in __kprobe_trace_func()
936 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in __kprobe_trace_func()
943 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)  in kprobe_trace_func()  argument
947 	list_for_each_entry_rcu(link, &tk->tp.files, list)  in kprobe_trace_func()
948 		__kprobe_trace_func(tk, regs, link->file);  in kprobe_trace_func()
954 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,  in __kretprobe_trace_func()  argument
963 	struct trace_event_call *call = &tk->tp.call;  in __kretprobe_trace_func()
973 	dsize = __get_data_size(&tk->tp, regs);  in __kretprobe_trace_func()
974 	size = sizeof(*entry) + tk->tp.size + dsize;  in __kretprobe_trace_func()
983 	entry->func = (unsigned long)tk->rp.kp.addr;  in __kretprobe_trace_func()
985 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in __kretprobe_trace_func()
992 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,  in kretprobe_trace_func()  argument
997 	list_for_each_entry_rcu(link, &tk->tp.files, list)  in kretprobe_trace_func()
998 		__kretprobe_trace_func(tk, ri, regs, link->file);  in kretprobe_trace_func()
1076 	struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;  in kprobe_event_define_fields()  local
1080 	for (i = 0; i < tk->tp.nr_args; i++) {  in kprobe_event_define_fields()
1081 		struct probe_arg *parg = &tk->tp.args[i];  in kprobe_event_define_fields()
1099 	struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;  in kretprobe_event_define_fields()  local
1104 	for (i = 0; i < tk->tp.nr_args; i++) {  in kretprobe_event_define_fields()
1105 		struct probe_arg *parg = &tk->tp.args[i];  in kretprobe_event_define_fields()
1123 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)  in kprobe_perf_func()  argument
1125 	struct trace_event_call *call = &tk->tp.call;  in kprobe_perf_func()
1139 	dsize = __get_data_size(&tk->tp, regs);  in kprobe_perf_func()
1140 	__size = sizeof(*entry) + tk->tp.size + dsize;  in kprobe_perf_func()
1148 	entry->ip = (unsigned long)tk->rp.kp.addr;  in kprobe_perf_func()
1150 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in kprobe_perf_func()
1157 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,  in kretprobe_perf_func()  argument
1160 	struct trace_event_call *call = &tk->tp.call;  in kretprobe_perf_func()
1174 	dsize = __get_data_size(&tk->tp, regs);  in kretprobe_perf_func()
1175 	__size = sizeof(*entry) + tk->tp.size + dsize;  in kretprobe_perf_func()
1183 	entry->func = (unsigned long)tk->rp.kp.addr;  in kretprobe_perf_func()
1185 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in kretprobe_perf_func()
1200 	struct trace_kprobe *tk = (struct trace_kprobe *)event->data;  in kprobe_register()  local
1205 		return enable_trace_kprobe(tk, file);  in kprobe_register()
1207 		return disable_trace_kprobe(tk, file);  in kprobe_register()
1211 		return enable_trace_kprobe(tk, NULL);  in kprobe_register()
1213 		return disable_trace_kprobe(tk, NULL);  in kprobe_register()
1226 	struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);  in kprobe_dispatcher()  local
1228 	tk->nhit++;  in kprobe_dispatcher()
1230 	if (tk->tp.flags & TP_FLAG_TRACE)  in kprobe_dispatcher()
1231 		kprobe_trace_func(tk, regs);  in kprobe_dispatcher()
1233 	if (tk->tp.flags & TP_FLAG_PROFILE)  in kprobe_dispatcher()
1234 		kprobe_perf_func(tk, regs);  in kprobe_dispatcher()
1243 	struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);  in kretprobe_dispatcher()  local
1245 	tk->nhit++;  in kretprobe_dispatcher()
1247 	if (tk->tp.flags & TP_FLAG_TRACE)  in kretprobe_dispatcher()
1248 		kretprobe_trace_func(tk, ri, regs);  in kretprobe_dispatcher()
1250 	if (tk->tp.flags & TP_FLAG_PROFILE)  in kretprobe_dispatcher()
1251 		kretprobe_perf_func(tk, ri, regs);  in kretprobe_dispatcher()
1265 static int register_kprobe_event(struct trace_kprobe *tk)  in register_kprobe_event()  argument
1267 	struct trace_event_call *call = &tk->tp.call;  in register_kprobe_event()
1272 	if (trace_kprobe_is_return(tk)) {  in register_kprobe_event()
1279 	if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)  in register_kprobe_event()
1288 	call->data = tk;  in register_kprobe_event()
1299 static int unregister_kprobe_event(struct trace_kprobe *tk)  in unregister_kprobe_event()  argument
1304 	ret = trace_remove_event_call(&tk->tp.call);  in unregister_kprobe_event()
1306 		kfree(tk->tp.call.print_fmt);  in unregister_kprobe_event()
1356 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)  in find_trace_probe_file()  argument
1361 		if (file->event_call == &tk->tp.call)  in find_trace_probe_file()
1375 	struct trace_kprobe *tk;  in kprobe_trace_self_tests_init()  local
1393 		tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1394 		if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1398 			file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1403 				enable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()
1414 		tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1415 		if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1419 			file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1424 				enable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()
1434 	tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1435 	if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1439 		file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1444 			disable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()
1447 	tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1448 	if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1452 		file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1457 			disable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()