56 static int ftrace_calc_offset(long ip, long addr)  in ftrace_calc_offset()  argument
58 	return (int)(addr - ip);  in ftrace_calc_offset()
61 static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)  in ftrace_call_replace()  argument
66 	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);  in ftrace_call_replace()
81 static unsigned long text_ip_addr(unsigned long ip)  in text_ip_addr()  argument
91 	if (within(ip, (unsigned long)_text, (unsigned long)_etext))  in text_ip_addr()
92 		ip = (unsigned long)__va(__pa_symbol(ip));  in text_ip_addr()
94 	return ip;  in text_ip_addr()
103 ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,  in ftrace_modify_code_direct()  argument
119 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))  in ftrace_modify_code_direct()
126 	ip = text_ip_addr(ip);  in ftrace_modify_code_direct()
129 	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))  in ftrace_modify_code_direct()
141 	unsigned long ip = rec->ip;  in ftrace_make_nop()  local
143 	old = ftrace_call_replace(ip, addr);  in ftrace_make_nop()
155 		return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_nop()
165 	unsigned long ip = rec->ip;  in ftrace_make_call()  local
168 	new = ftrace_call_replace(ip, addr);  in ftrace_make_call()
171 	return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_call()
208 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
228 static int update_ftrace_func(unsigned long ip, void *new)  in update_ftrace_func()  argument
233 	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);  in update_ftrace_func()
235 	ftrace_update_func = ip;  in update_ftrace_func()
242 	ret = ftrace_modify_code(ip, old, new);  in update_ftrace_func()
251 	unsigned long ip = (unsigned long)(&ftrace_call);  in ftrace_update_ftrace_func()  local
255 	new = ftrace_call_replace(ip, (unsigned long)func);  in ftrace_update_ftrace_func()
256 	ret = update_ftrace_func(ip, new);  in ftrace_update_ftrace_func()
260 		ip = (unsigned long)(&ftrace_regs_call);  in ftrace_update_ftrace_func()
261 		new = ftrace_call_replace(ip, (unsigned long)func);  in ftrace_update_ftrace_func()
262 		ret = update_ftrace_func(ip, new);  in ftrace_update_ftrace_func()
268 static int is_ftrace_caller(unsigned long ip)  in is_ftrace_caller()  argument
270 	if (ip == ftrace_update_func)  in is_ftrace_caller()
285 	unsigned long ip;  in ftrace_int3_handler()  local
290 	ip = regs->ip - 1;  in ftrace_int3_handler()
291 	if (!ftrace_location(ip) && !is_ftrace_caller(ip))  in ftrace_int3_handler()
294 	regs->ip += MCOUNT_INSN_SIZE - 1;  in ftrace_int3_handler()
299 static int ftrace_write(unsigned long ip, const char *val, int size)  in ftrace_write()  argument
301 	ip = text_ip_addr(ip);  in ftrace_write()
303 	if (probe_kernel_write((void *)ip, val, size))  in ftrace_write()
309 static int add_break(unsigned long ip, const char *old)  in add_break()  argument
314 	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))  in add_break()
321 	return ftrace_write(ip, &brk, 1);  in add_break()
327 	unsigned long ip = rec->ip;  in add_brk_on_call()  local
329 	old = ftrace_call_replace(ip, addr);  in add_brk_on_call()
331 	return add_break(rec->ip, old);  in add_brk_on_call()
341 	return add_break(rec->ip, old);  in add_brk_on_nop()
383 	unsigned long ip = rec->ip;  in remove_breakpoint()  local
386 	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))  in remove_breakpoint()
407 		nop = ftrace_call_replace(ip, ftrace_addr);  in remove_breakpoint()
414 		nop = ftrace_call_replace(ip, ftrace_addr);  in remove_breakpoint()
421 	return ftrace_write(ip, nop, 1);  in remove_breakpoint()
424 static int add_update_code(unsigned long ip, unsigned const char *new)  in add_update_code()  argument
427 	ip++;  in add_update_code()
429 	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);  in add_update_code()
434 	unsigned long ip = rec->ip;  in add_update_call()  local
437 	new = ftrace_call_replace(ip, addr);  in add_update_call()
438 	return add_update_code(ip, new);  in add_update_call()
443 	unsigned long ip = rec->ip;  in add_update_nop()  local
447 	return add_update_code(ip, new);  in add_update_nop()
478 	unsigned long ip = rec->ip;  in finish_update_call()  local
481 	new = ftrace_call_replace(ip, addr);  in finish_update_call()
483 	return ftrace_write(ip, new, 1);  in finish_update_call()
488 	unsigned long ip = rec->ip;  in finish_update_nop()  local
493 	return ftrace_write(ip, new, 1);  in finish_update_nop()
604 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,  in ftrace_modify_code()  argument
609 	ret = add_break(ip, old_code);  in ftrace_modify_code()
615 	ret = add_update_code(ip, new_code);  in ftrace_modify_code()
621 	ret = ftrace_write(ip, new_code, 1);  in ftrace_modify_code()
633 	if (ftrace_write(ip, old_code, 1))  in ftrace_modify_code()
654 static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)  in ftrace_jmp_replace()  argument
660 	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);  in ftrace_jmp_replace()
728 	unsigned long ip;  in create_trampoline()  local
766 	ip = (unsigned long)trampoline + size;  in create_trampoline()
769 	jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_return);  in create_trampoline()
828 	unsigned long ip;  in arch_ftrace_update_trampoline()  local
847 	ip = ops->trampoline + offset;  in arch_ftrace_update_trampoline()
852 	new = ftrace_call_replace(ip, (unsigned long)func);  in arch_ftrace_update_trampoline()
853 	ret = update_ftrace_func(ip, new);  in arch_ftrace_update_trampoline()
942 static int ftrace_mod_jmp(unsigned long ip, void *func)  in ftrace_mod_jmp()  argument
946 	new = ftrace_jmp_replace(ip, (unsigned long)func);  in ftrace_mod_jmp()
948 	return update_ftrace_func(ip, new);  in ftrace_mod_jmp()
953 	unsigned long ip = (unsigned long)(&ftrace_graph_call);  in ftrace_enable_ftrace_graph_caller()  local
955 	return ftrace_mod_jmp(ip, &ftrace_graph_caller);  in ftrace_enable_ftrace_graph_caller()
960 	unsigned long ip = (unsigned long)(&ftrace_graph_call);  in ftrace_disable_ftrace_graph_caller()  local
962 	return ftrace_mod_jmp(ip, &ftrace_stub);  in ftrace_disable_ftrace_graph_caller()