This source file includes following definitions.
- __ftrace_modify_code
- arch_ftrace_update_code
- ftrace_nop_replace
- adjust_address
- ftrace_arch_code_modify_prepare
- ftrace_arch_code_modify_post_process
- ftrace_call_replace
- ftrace_modify_code
- ftrace_update_ftrace_func
- ftrace_make_call
- ftrace_modify_call
- ftrace_make_nop
- ftrace_dyn_arch_init
- prepare_ftrace_return
- __ftrace_modify_caller
- ftrace_modify_graph_caller
- ftrace_enable_ftrace_graph_caller
- ftrace_disable_ftrace_graph_caller
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 #include <linux/ftrace.h>
16 #include <linux/uaccess.h>
17 #include <linux/module.h>
18 #include <linux/stop_machine.h>
19
20 #include <asm/cacheflush.h>
21 #include <asm/opcodes.h>
22 #include <asm/ftrace.h>
23 #include <asm/insn.h>
24 #include <asm/set_memory.h>
25
26 #ifdef CONFIG_THUMB2_KERNEL
27 #define NOP 0xf85deb04
28 #else
29 #define NOP 0xe8bd4000
30 #endif
31
32 #ifdef CONFIG_DYNAMIC_FTRACE
33
34 static int __ftrace_modify_code(void *data)
35 {
36 int *command = data;
37
38 set_kernel_text_rw();
39 ftrace_modify_all_code(*command);
40 set_kernel_text_ro();
41
42 return 0;
43 }
44
45 void arch_ftrace_update_code(int command)
46 {
47 stop_machine(__ftrace_modify_code, &command, NULL);
48 }
49
50 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
51 {
52 return NOP;
53 }
54
55 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
56 {
57 return addr;
58 }
59
60 int ftrace_arch_code_modify_prepare(void)
61 {
62 set_all_modules_text_rw();
63 return 0;
64 }
65
66 int ftrace_arch_code_modify_post_process(void)
67 {
68 set_all_modules_text_ro();
69
70 flush_tlb_all();
71 return 0;
72 }
73
74 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr)
75 {
76 return arm_gen_branch_link(pc, addr);
77 }
78
79 static int ftrace_modify_code(unsigned long pc, unsigned long old,
80 unsigned long new, bool validate)
81 {
82 unsigned long replaced;
83
84 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) {
85 old = __opcode_to_mem_thumb32(old);
86 new = __opcode_to_mem_thumb32(new);
87 } else {
88 old = __opcode_to_mem_arm(old);
89 new = __opcode_to_mem_arm(new);
90 }
91
92 if (validate) {
93 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE))
94 return -EFAULT;
95
96 if (replaced != old)
97 return -EINVAL;
98 }
99
100 if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE))
101 return -EPERM;
102
103 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
104
105 return 0;
106 }
107
108 int ftrace_update_ftrace_func(ftrace_func_t func)
109 {
110 unsigned long pc;
111 unsigned long new;
112 int ret;
113
114 pc = (unsigned long)&ftrace_call;
115 new = ftrace_call_replace(pc, (unsigned long)func);
116
117 ret = ftrace_modify_code(pc, 0, new, false);
118
119 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
120 if (!ret) {
121 pc = (unsigned long)&ftrace_regs_call;
122 new = ftrace_call_replace(pc, (unsigned long)func);
123
124 ret = ftrace_modify_code(pc, 0, new, false);
125 }
126 #endif
127
128 return ret;
129 }
130
131 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
132 {
133 unsigned long new, old;
134 unsigned long ip = rec->ip;
135
136 old = ftrace_nop_replace(rec);
137
138 new = ftrace_call_replace(ip, adjust_address(rec, addr));
139
140 return ftrace_modify_code(rec->ip, old, new, true);
141 }
142
143 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
144
145 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
146 unsigned long addr)
147 {
148 unsigned long new, old;
149 unsigned long ip = rec->ip;
150
151 old = ftrace_call_replace(ip, adjust_address(rec, old_addr));
152
153 new = ftrace_call_replace(ip, adjust_address(rec, addr));
154
155 return ftrace_modify_code(rec->ip, old, new, true);
156 }
157
158 #endif
159
160 int ftrace_make_nop(struct module *mod,
161 struct dyn_ftrace *rec, unsigned long addr)
162 {
163 unsigned long ip = rec->ip;
164 unsigned long old;
165 unsigned long new;
166 int ret;
167
168 old = ftrace_call_replace(ip, adjust_address(rec, addr));
169 new = ftrace_nop_replace(rec);
170 ret = ftrace_modify_code(ip, old, new, true);
171
172 return ret;
173 }
174
175 int __init ftrace_dyn_arch_init(void)
176 {
177 return 0;
178 }
179 #endif
180
181 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
182 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
183 unsigned long frame_pointer)
184 {
185 unsigned long return_hooker = (unsigned long) &return_to_handler;
186 unsigned long old;
187
188 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
189 return;
190
191 old = *parent;
192 *parent = return_hooker;
193
194 if (function_graph_enter(old, self_addr, frame_pointer, NULL))
195 *parent = old;
196 }
197
198 #ifdef CONFIG_DYNAMIC_FTRACE
199 extern unsigned long ftrace_graph_call;
200 extern unsigned long ftrace_graph_call_old;
201 extern void ftrace_graph_caller_old(void);
202 extern unsigned long ftrace_graph_regs_call;
203 extern void ftrace_graph_regs_caller(void);
204
205 static int __ftrace_modify_caller(unsigned long *callsite,
206 void (*func) (void), bool enable)
207 {
208 unsigned long caller_fn = (unsigned long) func;
209 unsigned long pc = (unsigned long) callsite;
210 unsigned long branch = arm_gen_branch(pc, caller_fn);
211 unsigned long nop = 0xe1a00000;
212 unsigned long old = enable ? nop : branch;
213 unsigned long new = enable ? branch : nop;
214
215 return ftrace_modify_code(pc, old, new, true);
216 }
217
218 static int ftrace_modify_graph_caller(bool enable)
219 {
220 int ret;
221
222 ret = __ftrace_modify_caller(&ftrace_graph_call,
223 ftrace_graph_caller,
224 enable);
225
226 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
227 if (!ret)
228 ret = __ftrace_modify_caller(&ftrace_graph_regs_call,
229 ftrace_graph_regs_caller,
230 enable);
231 #endif
232
233
234 return ret;
235 }
236
237 int ftrace_enable_ftrace_graph_caller(void)
238 {
239 return ftrace_modify_graph_caller(true);
240 }
241
242 int ftrace_disable_ftrace_graph_caller(void)
243 {
244 return ftrace_modify_graph_caller(false);
245 }
246 #endif
247 #endif