1/*
2 *  linux/arch/x86_64/mcount_64.S
3 *
4 *  Copyright (C) 2014  Steven Rostedt, Red Hat Inc
5 */
6
7#include <linux/linkage.h>
8#include <asm/ptrace.h>
9#include <asm/ftrace.h>
10
11
12	.code64
13	.section .entry.text, "ax"
14
15
16#ifdef CONFIG_FUNCTION_TRACER
17
18#ifdef CC_USING_FENTRY
19# define function_hook	__fentry__
20#else
21# define function_hook	mcount
22#endif
23
24/* All cases save the original rbp (8 bytes) */
25#ifdef CONFIG_FRAME_POINTER
26# ifdef CC_USING_FENTRY
27/* Save parent and function stack frames (rip and rbp) */
28#  define MCOUNT_FRAME_SIZE	(8+16*2)
29# else
30/* Save just function stack frame (rip and rbp) */
31#  define MCOUNT_FRAME_SIZE	(8+16)
32# endif
33#else
34/* No need to save a stack frame */
35# define MCOUNT_FRAME_SIZE	8
36#endif /* CONFIG_FRAME_POINTER */
37
38/* Size of stack used to save mcount regs in save_mcount_regs */
39#define MCOUNT_REG_SIZE		(SS+8 + MCOUNT_FRAME_SIZE)
40
41/*
42 * gcc -pg option adds a call to 'mcount' in most functions.
43 * When -mfentry is used, the call is to 'fentry' and not 'mcount'
44 * and is done before the function's stack frame is set up.
45 * They both require a set of regs to be saved before calling
46 * any C code and restored before returning back to the function.
47 *
48 * On boot up, all these calls are converted into nops. When tracing
49 * is enabled, the call can jump to either ftrace_caller or
50 * ftrace_regs_caller. Callbacks (tracing functions) that require
51 * ftrace_regs_caller (like kprobes) need to have pt_regs passed to
52 * it. For this reason, the size of the pt_regs structure will be
53 * allocated on the stack and the required mcount registers will
54 * be saved in the locations that pt_regs has them in.
55 */
56
57/*
58 * @added: the amount of stack added before calling this
59 *
60 * After this is called, the following registers contain:
61 *
62 *  %rdi - holds the address that called the trampoline
63 *  %rsi - holds the parent function (traced function's return address)
64 *  %rdx - holds the original %rbp
65 */
66.macro save_mcount_regs added=0
67
68	/* Always save the original rbp */
69	pushq %rbp
70
71#ifdef CONFIG_FRAME_POINTER
72	/*
73	 * Stack traces will stop at the ftrace trampoline if the frame pointer
74	 * is not set up properly. If fentry is used, we need to save a frame
75	 * pointer for the parent as well as the function traced, because the
76	 * fentry is called before the stack frame is set up, where as mcount
77	 * is called afterward.
78	 */
79#ifdef CC_USING_FENTRY
80	/* Save the parent pointer (skip orig rbp and our return address) */
81	pushq \added+8*2(%rsp)
82	pushq %rbp
83	movq %rsp, %rbp
84	/* Save the return address (now skip orig rbp, rbp and parent) */
85	pushq \added+8*3(%rsp)
86#else
87	/* Can't assume that rip is before this (unless added was zero) */
88	pushq \added+8(%rsp)
89#endif
90	pushq %rbp
91	movq %rsp, %rbp
92#endif /* CONFIG_FRAME_POINTER */
93
94	/*
95	 * We add enough stack to save all regs.
96	 */
97	subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp
98	movq %rax, RAX(%rsp)
99	movq %rcx, RCX(%rsp)
100	movq %rdx, RDX(%rsp)
101	movq %rsi, RSI(%rsp)
102	movq %rdi, RDI(%rsp)
103	movq %r8, R8(%rsp)
104	movq %r9, R9(%rsp)
105	/*
106	 * Save the original RBP. Even though the mcount ABI does not
107	 * require this, it helps out callers.
108	 */
109	movq MCOUNT_REG_SIZE-8(%rsp), %rdx
110	movq %rdx, RBP(%rsp)
111
112	/* Copy the parent address into %rsi (second parameter) */
113#ifdef CC_USING_FENTRY
114	movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
115#else
116	/* %rdx contains original %rbp */
117	movq 8(%rdx), %rsi
118#endif
119
120	 /* Move RIP to its proper location */
121	movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
122	movq %rdi, RIP(%rsp)
123
124	/*
125	 * Now %rdi (the first parameter) has the return address of
126	 * where ftrace_call returns. But the callbacks expect the
127	 * address of the call itself.
128	 */
129	subq $MCOUNT_INSN_SIZE, %rdi
130	.endm
131
132.macro restore_mcount_regs
133	movq R9(%rsp), %r9
134	movq R8(%rsp), %r8
135	movq RDI(%rsp), %rdi
136	movq RSI(%rsp), %rsi
137	movq RDX(%rsp), %rdx
138	movq RCX(%rsp), %rcx
139	movq RAX(%rsp), %rax
140
141	/* ftrace_regs_caller can modify %rbp */
142	movq RBP(%rsp), %rbp
143
144	addq $MCOUNT_REG_SIZE, %rsp
145
146	.endm
147
148#ifdef CONFIG_DYNAMIC_FTRACE
149
150ENTRY(function_hook)
151	retq
152END(function_hook)
153
154ENTRY(ftrace_caller)
155	/* save_mcount_regs fills in first two parameters */
156	save_mcount_regs
157
158GLOBAL(ftrace_caller_op_ptr)
159	/* Load the ftrace_ops into the 3rd parameter */
160	movq function_trace_op(%rip), %rdx
161
162	/* regs go into 4th parameter (but make it NULL) */
163	movq $0, %rcx
164
165GLOBAL(ftrace_call)
166	call ftrace_stub
167
168	restore_mcount_regs
169
170	/*
171	 * The copied trampoline must call ftrace_return as it
172	 * still may need to call the function graph tracer.
173	 */
174GLOBAL(ftrace_caller_end)
175
176GLOBAL(ftrace_return)
177
178#ifdef CONFIG_FUNCTION_GRAPH_TRACER
179GLOBAL(ftrace_graph_call)
180	jmp ftrace_stub
181#endif
182
183GLOBAL(ftrace_stub)
184	retq
185END(ftrace_caller)
186
187ENTRY(ftrace_regs_caller)
188	/* Save the current flags before any operations that can change them */
189	pushfq
190
191	/* added 8 bytes to save flags */
192	save_mcount_regs 8
193	/* save_mcount_regs fills in first two parameters */
194
195GLOBAL(ftrace_regs_caller_op_ptr)
196	/* Load the ftrace_ops into the 3rd parameter */
197	movq function_trace_op(%rip), %rdx
198
199	/* Save the rest of pt_regs */
200	movq %r15, R15(%rsp)
201	movq %r14, R14(%rsp)
202	movq %r13, R13(%rsp)
203	movq %r12, R12(%rsp)
204	movq %r11, R11(%rsp)
205	movq %r10, R10(%rsp)
206	movq %rbx, RBX(%rsp)
207	/* Copy saved flags */
208	movq MCOUNT_REG_SIZE(%rsp), %rcx
209	movq %rcx, EFLAGS(%rsp)
210	/* Kernel segments */
211	movq $__KERNEL_DS, %rcx
212	movq %rcx, SS(%rsp)
213	movq $__KERNEL_CS, %rcx
214	movq %rcx, CS(%rsp)
215	/* Stack - skipping return address and flags */
216	leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
217	movq %rcx, RSP(%rsp)
218
219	/* regs go into 4th parameter */
220	leaq (%rsp), %rcx
221
222GLOBAL(ftrace_regs_call)
223	call ftrace_stub
224
225	/* Copy flags back to SS, to restore them */
226	movq EFLAGS(%rsp), %rax
227	movq %rax, MCOUNT_REG_SIZE(%rsp)
228
229	/* Handlers can change the RIP */
230	movq RIP(%rsp), %rax
231	movq %rax, MCOUNT_REG_SIZE+8(%rsp)
232
233	/* restore the rest of pt_regs */
234	movq R15(%rsp), %r15
235	movq R14(%rsp), %r14
236	movq R13(%rsp), %r13
237	movq R12(%rsp), %r12
238	movq R10(%rsp), %r10
239	movq RBX(%rsp), %rbx
240
241	restore_mcount_regs
242
243	/* Restore flags */
244	popfq
245
246	/*
247	 * As this jmp to ftrace_return can be a short jump
248	 * it must not be copied into the trampoline.
249	 * The trampoline will add the code to jump
250	 * to the return.
251	 */
252GLOBAL(ftrace_regs_caller_end)
253
254	jmp ftrace_return
255
256END(ftrace_regs_caller)
257
258
259#else /* ! CONFIG_DYNAMIC_FTRACE */
260
261ENTRY(function_hook)
262	cmpq $ftrace_stub, ftrace_trace_function
263	jnz trace
264
265fgraph_trace:
266#ifdef CONFIG_FUNCTION_GRAPH_TRACER
267	cmpq $ftrace_stub, ftrace_graph_return
268	jnz ftrace_graph_caller
269
270	cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
271	jnz ftrace_graph_caller
272#endif
273
274GLOBAL(ftrace_stub)
275	retq
276
277trace:
278	/* save_mcount_regs fills in first two parameters */
279	save_mcount_regs
280
281	call   *ftrace_trace_function
282
283	restore_mcount_regs
284
285	jmp fgraph_trace
286END(function_hook)
287#endif /* CONFIG_DYNAMIC_FTRACE */
288#endif /* CONFIG_FUNCTION_TRACER */
289
290#ifdef CONFIG_FUNCTION_GRAPH_TRACER
291ENTRY(ftrace_graph_caller)
292	/* Saves rbp into %rdx and fills first parameter  */
293	save_mcount_regs
294
295#ifdef CC_USING_FENTRY
296	leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
297	movq $0, %rdx	/* No framepointers needed */
298#else
299	/* Save address of the return address of traced function */
300	leaq 8(%rdx), %rsi
301	/* ftrace does sanity checks against frame pointers */
302	movq (%rdx), %rdx
303#endif
304	call	prepare_ftrace_return
305
306	restore_mcount_regs
307
308	retq
309END(ftrace_graph_caller)
310
311GLOBAL(return_to_handler)
312	subq  $24, %rsp
313
314	/* Save the return values */
315	movq %rax, (%rsp)
316	movq %rdx, 8(%rsp)
317	movq %rbp, %rdi
318
319	call ftrace_return_to_handler
320
321	movq %rax, %rdi
322	movq 8(%rsp), %rdx
323	movq (%rsp), %rax
324	addq $24, %rsp
325	jmp *%rdi
326#endif
327