1/*
2 * Compatibility mode system call entry point for x86-64.
3 *
4 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
5 */
6
7#include <asm/dwarf2.h>
8#include <asm/calling.h>
9#include <asm/asm-offsets.h>
10#include <asm/current.h>
11#include <asm/errno.h>
12#include <asm/ia32_unistd.h>
13#include <asm/thread_info.h>
14#include <asm/segment.h>
15#include <asm/irqflags.h>
16#include <asm/asm.h>
17#include <asm/smap.h>
18#include <linux/linkage.h>
19#include <linux/err.h>
20
21/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
22#include <linux/elf-em.h>
23#define AUDIT_ARCH_I386		(EM_386|__AUDIT_ARCH_LE)
24#define __AUDIT_ARCH_LE	   0x40000000
25
26#ifndef CONFIG_AUDITSYSCALL
27#define sysexit_audit ia32_ret_from_sys_call
28#define sysretl_audit ia32_ret_from_sys_call
29#endif
30
31	.section .entry.text, "ax"
32
33	/* clobbers %rax */
34	.macro  CLEAR_RREGS _r9=rax
35	xorl 	%eax,%eax
36	movq	%rax,R11(%rsp)
37	movq	%rax,R10(%rsp)
38	movq	%\_r9,R9(%rsp)
39	movq	%rax,R8(%rsp)
40	.endm
41
42	/*
43	 * Reload arg registers from stack in case ptrace changed them.
44	 * We don't reload %eax because syscall_trace_enter() returned
45	 * the %rax value we should see.  Instead, we just truncate that
46	 * value to 32 bits again as we did on entry from user mode.
47	 * If it's a new value set by user_regset during entry tracing,
48	 * this matches the normal truncation of the user-mode value.
49	 * If it's -1 to make us punt the syscall, then (u32)-1 is still
50	 * an appropriately invalid value.
51	 */
52	.macro LOAD_ARGS32 _r9=0
53	.if \_r9
54	movl R9(%rsp),%r9d
55	.endif
56	movl RCX(%rsp),%ecx
57	movl RDX(%rsp),%edx
58	movl RSI(%rsp),%esi
59	movl RDI(%rsp),%edi
60	movl %eax,%eax			/* zero extension */
61	.endm
62
63	.macro CFI_STARTPROC32 simple
64	CFI_STARTPROC	\simple
65	CFI_UNDEFINED	r8
66	CFI_UNDEFINED	r9
67	CFI_UNDEFINED	r10
68	CFI_UNDEFINED	r11
69	CFI_UNDEFINED	r12
70	CFI_UNDEFINED	r13
71	CFI_UNDEFINED	r14
72	CFI_UNDEFINED	r15
73	.endm
74
75#ifdef CONFIG_PARAVIRT
76ENTRY(native_usergs_sysret32)
77	swapgs
78	sysretl
79ENDPROC(native_usergs_sysret32)
80
81ENTRY(native_irq_enable_sysexit)
82	swapgs
83	sti
84	sysexit
85ENDPROC(native_irq_enable_sysexit)
86#endif
87
88/*
89 * 32bit SYSENTER instruction entry.
90 *
91 * SYSENTER loads ss, rsp, cs, and rip from previously programmed MSRs.
92 * IF and VM in rflags are cleared (IOW: interrupts are off).
93 * SYSENTER does not save anything on the stack,
94 * and does not save old rip (!!!) and rflags.
95 *
96 * Arguments:
97 * eax  system call number
98 * ebx  arg1
99 * ecx  arg2
100 * edx  arg3
101 * esi  arg4
102 * edi  arg5
103 * ebp  user stack
104 * 0(%ebp) arg6
105 *
106 * This is purely a fast path. For anything complicated we use the int 0x80
107 * path below. We set up a complete hardware stack frame to share code
108 * with the int 0x80 path.
109 */
110ENTRY(ia32_sysenter_target)
111	CFI_STARTPROC32	simple
112	CFI_SIGNAL_FRAME
113	CFI_DEF_CFA	rsp,0
114	CFI_REGISTER	rsp,rbp
115
116	/*
117	 * Interrupts are off on entry.
118	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
119	 * it is too small to ever cause noticeable irq latency.
120	 */
121	SWAPGS_UNSAFE_STACK
122	movq	PER_CPU_VAR(cpu_tss + TSS_sp0), %rsp
123	ENABLE_INTERRUPTS(CLBR_NONE)
124
125	/* Zero-extending 32-bit regs, do not remove */
126	movl	%ebp, %ebp
127	movl	%eax, %eax
128
129	movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
130	CFI_REGISTER rip,r10
131
132	/* Construct struct pt_regs on stack */
133	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
134	pushq_cfi	%rbp			/* pt_regs->sp */
135	CFI_REL_OFFSET	rsp,0
136	pushfq_cfi				/* pt_regs->flags */
137	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
138	pushq_cfi	%r10 /* pt_regs->ip = thread_info->sysenter_return */
139	CFI_REL_OFFSET	rip,0
140	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
141	pushq_cfi_reg	rdi			/* pt_regs->di */
142	pushq_cfi_reg	rsi			/* pt_regs->si */
143	pushq_cfi_reg	rdx			/* pt_regs->dx */
144	pushq_cfi_reg	rcx			/* pt_regs->cx */
145	pushq_cfi_reg	rax			/* pt_regs->ax */
146	cld
147	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
148	CFI_ADJUST_CFA_OFFSET 10*8
149
150	/*
151	 * no need to do an access_ok check here because rbp has been
152	 * 32bit zero extended
153	 */
154	ASM_STAC
1551:	movl	(%rbp),%ebp
156	_ASM_EXTABLE(1b,ia32_badarg)
157	ASM_CLAC
158
159	/*
160	 * Sysenter doesn't filter flags, so we need to clear NT
161	 * ourselves.  To save a few cycles, we can check whether
162	 * NT was set instead of doing an unconditional popfq.
163	 */
164	testl $X86_EFLAGS_NT,EFLAGS(%rsp)
165	jnz sysenter_fix_flags
166sysenter_flags_fixed:
167
168	orl     $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
169	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
170	CFI_REMEMBER_STATE
171	jnz  sysenter_tracesys
172	cmpq	$(IA32_NR_syscalls-1),%rax
173	ja	ia32_badsys
174sysenter_do_call:
175	/* 32bit syscall -> 64bit C ABI argument conversion */
176	movl	%edi,%r8d	/* arg5 */
177	movl	%ebp,%r9d	/* arg6 */
178	xchg	%ecx,%esi	/* rsi:arg2, rcx:arg4 */
179	movl	%ebx,%edi	/* arg1 */
180	movl	%edx,%edx	/* arg3 (zero extension) */
181sysenter_dispatch:
182	call	*ia32_sys_call_table(,%rax,8)
183	movq	%rax,RAX(%rsp)
184	DISABLE_INTERRUPTS(CLBR_NONE)
185	TRACE_IRQS_OFF
186	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
187	jnz	sysexit_audit
188sysexit_from_sys_call:
189	/*
190	 * NB: SYSEXIT is not obviously safe for 64-bit kernels -- an
191	 * NMI between STI and SYSEXIT has poorly specified behavior,
192	 * and and NMI followed by an IRQ with usergs is fatal.  So
193	 * we just pretend we're using SYSEXIT but we really use
194	 * SYSRETL instead.
195	 *
196	 * This code path is still called 'sysexit' because it pairs
197	 * with 'sysenter' and it uses the SYSENTER calling convention.
198	 */
199	andl    $~TS_COMPAT,ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
200	movl	RIP(%rsp),%ecx		/* User %eip */
201	CFI_REGISTER rip,rcx
202	RESTORE_RSI_RDI
203	xorl	%edx,%edx		/* avoid info leaks */
204	xorq	%r8,%r8
205	xorq	%r9,%r9
206	xorq	%r10,%r10
207	movl	EFLAGS(%rsp),%r11d	/* User eflags */
208	/*CFI_RESTORE rflags*/
209	TRACE_IRQS_ON
210
211	/*
212	 * SYSRETL works even on Intel CPUs.  Use it in preference to SYSEXIT,
213	 * since it avoids a dicey window with interrupts enabled.
214	 */
215	movl	RSP(%rsp),%esp
216
217	/*
218	 * USERGS_SYSRET32 does:
219	 *  gsbase = user's gs base
220	 *  eip = ecx
221	 *  rflags = r11
222	 *  cs = __USER32_CS
223	 *  ss = __USER_DS
224	 *
225	 * The prologue set RIP(%rsp) to VDSO32_SYSENTER_RETURN, which does:
226	 *
227	 *  pop %ebp
228	 *  pop %edx
229	 *  pop %ecx
230	 *
231	 * Therefore, we invoke SYSRETL with EDX and R8-R10 zeroed to
232	 * avoid info leaks.  R11 ends up with VDSO32_SYSENTER_RETURN's
233	 * address (already known to user code), and R12-R15 are
234	 * callee-saved and therefore don't contain any interesting
235	 * kernel data.
236	 */
237	USERGS_SYSRET32
238
239	CFI_RESTORE_STATE
240
241#ifdef CONFIG_AUDITSYSCALL
242	.macro auditsys_entry_common
243	movl %esi,%r8d			/* 5th arg: 4th syscall arg */
244	movl %ecx,%r9d			/*swap with edx*/
245	movl %edx,%ecx			/* 4th arg: 3rd syscall arg */
246	movl %r9d,%edx			/* 3rd arg: 2nd syscall arg */
247	movl %ebx,%esi			/* 2nd arg: 1st syscall arg */
248	movl %eax,%edi			/* 1st arg: syscall number */
249	call __audit_syscall_entry
250	movl RAX(%rsp),%eax	/* reload syscall number */
251	cmpq $(IA32_NR_syscalls-1),%rax
252	ja ia32_badsys
253	movl %ebx,%edi			/* reload 1st syscall arg */
254	movl RCX(%rsp),%esi	/* reload 2nd syscall arg */
255	movl RDX(%rsp),%edx	/* reload 3rd syscall arg */
256	movl RSI(%rsp),%ecx	/* reload 4th syscall arg */
257	movl RDI(%rsp),%r8d	/* reload 5th syscall arg */
258	.endm
259
260	.macro auditsys_exit exit
261	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
262	jnz ia32_ret_from_sys_call
263	TRACE_IRQS_ON
264	ENABLE_INTERRUPTS(CLBR_NONE)
265	movl %eax,%esi		/* second arg, syscall return value */
266	cmpl $-MAX_ERRNO,%eax	/* is it an error ? */
267	jbe 1f
268	movslq %eax, %rsi	/* if error sign extend to 64 bits */
2691:	setbe %al		/* 1 if error, 0 if not */
270	movzbl %al,%edi		/* zero-extend that into %edi */
271	call __audit_syscall_exit
272	movq RAX(%rsp),%rax	/* reload syscall return value */
273	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
274	DISABLE_INTERRUPTS(CLBR_NONE)
275	TRACE_IRQS_OFF
276	testl %edi, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
277	jz \exit
278	CLEAR_RREGS
279	jmp int_with_check
280	.endm
281
282sysenter_auditsys:
283	auditsys_entry_common
284	movl %ebp,%r9d			/* reload 6th syscall arg */
285	jmp sysenter_dispatch
286
287sysexit_audit:
288	auditsys_exit sysexit_from_sys_call
289#endif
290
291sysenter_fix_flags:
292	pushq_cfi $(X86_EFLAGS_IF|X86_EFLAGS_FIXED)
293	popfq_cfi
294	jmp sysenter_flags_fixed
295
296sysenter_tracesys:
297#ifdef CONFIG_AUDITSYSCALL
298	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
299	jz	sysenter_auditsys
300#endif
301	SAVE_EXTRA_REGS
302	CLEAR_RREGS
303	movq	$-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
304	movq	%rsp,%rdi        /* &pt_regs -> arg1 */
305	call	syscall_trace_enter
306	LOAD_ARGS32  /* reload args from stack in case ptrace changed it */
307	RESTORE_EXTRA_REGS
308	cmpq	$(IA32_NR_syscalls-1),%rax
309	ja	int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
310	jmp	sysenter_do_call
311	CFI_ENDPROC
312ENDPROC(ia32_sysenter_target)
313
314/*
315 * 32bit SYSCALL instruction entry.
316 *
317 * 32bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
318 * then loads new ss, cs, and rip from previously programmed MSRs.
319 * rflags gets masked by a value from another MSR (so CLD and CLAC
320 * are not needed). SYSCALL does not save anything on the stack
321 * and does not change rsp.
322 *
323 * Note: rflags saving+masking-with-MSR happens only in Long mode
324 * (in legacy 32bit mode, IF, RF and VM bits are cleared and that's it).
325 * Don't get confused: rflags saving+masking depends on Long Mode Active bit
326 * (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
327 * or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
328 *
329 * Arguments:
330 * eax  system call number
331 * ecx  return address
332 * ebx  arg1
333 * ebp  arg2	(note: not saved in the stack frame, should not be touched)
334 * edx  arg3
335 * esi  arg4
336 * edi  arg5
337 * esp  user stack
338 * 0(%esp) arg6
339 *
340 * This is purely a fast path. For anything complicated we use the int 0x80
341 * path below. We set up a complete hardware stack frame to share code
342 * with the int 0x80 path.
343 */
344ENTRY(ia32_cstar_target)
345	CFI_STARTPROC32	simple
346	CFI_SIGNAL_FRAME
347	CFI_DEF_CFA	rsp,0
348	CFI_REGISTER	rip,rcx
349	/*CFI_REGISTER	rflags,r11*/
350
351	/*
352	 * Interrupts are off on entry.
353	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
354	 * it is too small to ever cause noticeable irq latency.
355	 */
356	SWAPGS_UNSAFE_STACK
357	movl	%esp,%r8d
358	CFI_REGISTER	rsp,r8
359	movq	PER_CPU_VAR(kernel_stack),%rsp
360	ENABLE_INTERRUPTS(CLBR_NONE)
361
362	/* Zero-extending 32-bit regs, do not remove */
363	movl	%eax,%eax
364
365	/* Construct struct pt_regs on stack */
366	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
367	pushq_cfi	%r8			/* pt_regs->sp */
368	CFI_REL_OFFSET rsp,0
369	pushq_cfi	%r11			/* pt_regs->flags */
370	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
371	pushq_cfi	%rcx			/* pt_regs->ip */
372	CFI_REL_OFFSET rip,0
373	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
374	pushq_cfi_reg	rdi			/* pt_regs->di */
375	pushq_cfi_reg	rsi			/* pt_regs->si */
376	pushq_cfi_reg	rdx			/* pt_regs->dx */
377	pushq_cfi_reg	rbp			/* pt_regs->cx */
378	movl	%ebp,%ecx
379	pushq_cfi_reg	rax			/* pt_regs->ax */
380	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
381	CFI_ADJUST_CFA_OFFSET 10*8
382
383	/*
384	 * no need to do an access_ok check here because r8 has been
385	 * 32bit zero extended
386	 */
387	ASM_STAC
3881:	movl	(%r8),%r9d
389	_ASM_EXTABLE(1b,ia32_badarg)
390	ASM_CLAC
391	orl     $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
392	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
393	CFI_REMEMBER_STATE
394	jnz   cstar_tracesys
395	cmpq $IA32_NR_syscalls-1,%rax
396	ja  ia32_badsys
397cstar_do_call:
398	/* 32bit syscall -> 64bit C ABI argument conversion */
399	movl	%edi,%r8d	/* arg5 */
400	/* r9 already loaded */	/* arg6 */
401	xchg	%ecx,%esi	/* rsi:arg2, rcx:arg4 */
402	movl	%ebx,%edi	/* arg1 */
403	movl	%edx,%edx	/* arg3 (zero extension) */
404cstar_dispatch:
405	call *ia32_sys_call_table(,%rax,8)
406	movq %rax,RAX(%rsp)
407	DISABLE_INTERRUPTS(CLBR_NONE)
408	TRACE_IRQS_OFF
409	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
410	jnz sysretl_audit
411sysretl_from_sys_call:
412	andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
413	RESTORE_RSI_RDI_RDX
414	movl RIP(%rsp),%ecx
415	CFI_REGISTER rip,rcx
416	movl EFLAGS(%rsp),%r11d
417	/*CFI_REGISTER rflags,r11*/
418	xorq	%r10,%r10
419	xorq	%r9,%r9
420	xorq	%r8,%r8
421	TRACE_IRQS_ON
422	movl RSP(%rsp),%esp
423	CFI_RESTORE rsp
424	/*
425	 * 64bit->32bit SYSRET restores eip from ecx,
426	 * eflags from r11 (but RF and VM bits are forced to 0),
427	 * cs and ss are loaded from MSRs.
428	 * (Note: 32bit->32bit SYSRET is different: since r11
429	 * does not exist, it merely sets eflags.IF=1).
430	 *
431	 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
432	 * descriptor is not reinitialized.  This means that we must
433	 * avoid SYSRET with SS == NULL, which could happen if we schedule,
434	 * exit the kernel, and re-enter using an interrupt vector.  (All
435	 * interrupt entries on x86_64 set SS to NULL.)  We prevent that
436	 * from happening by reloading SS in __switch_to.
437	 */
438	USERGS_SYSRET32
439
440#ifdef CONFIG_AUDITSYSCALL
441cstar_auditsys:
442	CFI_RESTORE_STATE
443	movl %r9d,R9(%rsp)	/* register to be clobbered by call */
444	auditsys_entry_common
445	movl R9(%rsp),%r9d	/* reload 6th syscall arg */
446	jmp cstar_dispatch
447
448sysretl_audit:
449	auditsys_exit sysretl_from_sys_call
450#endif
451
452cstar_tracesys:
453#ifdef CONFIG_AUDITSYSCALL
454	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
455	jz cstar_auditsys
456#endif
457	xchgl %r9d,%ebp
458	SAVE_EXTRA_REGS
459	CLEAR_RREGS r9
460	movq $-ENOSYS,RAX(%rsp)	/* ptrace can change this for a bad syscall */
461	movq %rsp,%rdi        /* &pt_regs -> arg1 */
462	call syscall_trace_enter
463	LOAD_ARGS32 1	/* reload args from stack in case ptrace changed it */
464	RESTORE_EXTRA_REGS
465	xchgl %ebp,%r9d
466	cmpq $(IA32_NR_syscalls-1),%rax
467	ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */
468	jmp cstar_do_call
469END(ia32_cstar_target)
470
471ia32_badarg:
472	ASM_CLAC
473	movq $-EFAULT,%rax
474	jmp ia32_sysret
475	CFI_ENDPROC
476
477/*
478 * Emulated IA32 system calls via int 0x80.
479 *
480 * Arguments:
481 * eax  system call number
482 * ebx  arg1
483 * ecx  arg2
484 * edx  arg3
485 * esi  arg4
486 * edi  arg5
487 * ebp  arg6	(note: not saved in the stack frame, should not be touched)
488 *
489 * Notes:
490 * Uses the same stack frame as the x86-64 version.
491 * All registers except eax must be saved (but ptrace may violate that).
492 * Arguments are zero extended. For system calls that want sign extension and
493 * take long arguments a wrapper is needed. Most calls can just be called
494 * directly.
495 * Assumes it is only called from user space and entered with interrupts off.
496 */
497
498ENTRY(ia32_syscall)
499	CFI_STARTPROC32	simple
500	CFI_SIGNAL_FRAME
501	CFI_DEF_CFA	rsp,5*8
502	/*CFI_REL_OFFSET	ss,4*8 */
503	CFI_REL_OFFSET	rsp,3*8
504	/*CFI_REL_OFFSET	rflags,2*8 */
505	/*CFI_REL_OFFSET	cs,1*8 */
506	CFI_REL_OFFSET	rip,0*8
507
508	/*
509	 * Interrupts are off on entry.
510	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
511	 * it is too small to ever cause noticeable irq latency.
512	 */
513	PARAVIRT_ADJUST_EXCEPTION_FRAME
514	ASM_CLAC			/* Do this early to minimize exposure */
515	SWAPGS
516	ENABLE_INTERRUPTS(CLBR_NONE)
517
518	/* Zero-extending 32-bit regs, do not remove */
519	movl	%eax,%eax
520
521	/* Construct struct pt_regs on stack (iret frame is already on stack) */
522	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
523	pushq_cfi_reg	rdi			/* pt_regs->di */
524	pushq_cfi_reg	rsi			/* pt_regs->si */
525	pushq_cfi_reg	rdx			/* pt_regs->dx */
526	pushq_cfi_reg	rcx			/* pt_regs->cx */
527	pushq_cfi_reg	rax			/* pt_regs->ax */
528	cld
529	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
530	CFI_ADJUST_CFA_OFFSET 10*8
531
532	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
533	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
534	jnz ia32_tracesys
535	cmpq $(IA32_NR_syscalls-1),%rax
536	ja ia32_badsys
537ia32_do_call:
538	/* 32bit syscall -> 64bit C ABI argument conversion */
539	movl %edi,%r8d	/* arg5 */
540	movl %ebp,%r9d	/* arg6 */
541	xchg %ecx,%esi	/* rsi:arg2, rcx:arg4 */
542	movl %ebx,%edi	/* arg1 */
543	movl %edx,%edx	/* arg3 (zero extension) */
544	call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
545ia32_sysret:
546	movq %rax,RAX(%rsp)
547ia32_ret_from_sys_call:
548	CLEAR_RREGS
549	jmp int_ret_from_sys_call
550
551ia32_tracesys:
552	SAVE_EXTRA_REGS
553	CLEAR_RREGS
554	movq $-ENOSYS,RAX(%rsp)	/* ptrace can change this for a bad syscall */
555	movq %rsp,%rdi        /* &pt_regs -> arg1 */
556	call syscall_trace_enter
557	LOAD_ARGS32	/* reload args from stack in case ptrace changed it */
558	RESTORE_EXTRA_REGS
559	cmpq $(IA32_NR_syscalls-1),%rax
560	ja  int_ret_from_sys_call	/* ia32_tracesys has set RAX(%rsp) */
561	jmp ia32_do_call
562END(ia32_syscall)
563
564ia32_badsys:
565	movq $0,ORIG_RAX(%rsp)
566	movq $-ENOSYS,%rax
567	jmp ia32_sysret
568
569	CFI_ENDPROC
570
571	.macro PTREGSCALL label, func
572	ALIGN
573GLOBAL(\label)
574	leaq \func(%rip),%rax
575	jmp  ia32_ptregs_common
576	.endm
577
578	CFI_STARTPROC32
579
580	PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn
581	PTREGSCALL stub32_sigreturn, sys32_sigreturn
582	PTREGSCALL stub32_fork, sys_fork
583	PTREGSCALL stub32_vfork, sys_vfork
584
585	ALIGN
586GLOBAL(stub32_clone)
587	leaq sys_clone(%rip),%rax
588	mov	%r8, %rcx
589	jmp  ia32_ptregs_common
590
591	ALIGN
592ia32_ptregs_common:
593	CFI_ENDPROC
594	CFI_STARTPROC32	simple
595	CFI_SIGNAL_FRAME
596	CFI_DEF_CFA	rsp,SIZEOF_PTREGS
597	CFI_REL_OFFSET	rax,RAX
598	CFI_REL_OFFSET	rcx,RCX
599	CFI_REL_OFFSET	rdx,RDX
600	CFI_REL_OFFSET	rsi,RSI
601	CFI_REL_OFFSET	rdi,RDI
602	CFI_REL_OFFSET	rip,RIP
603/*	CFI_REL_OFFSET	cs,CS*/
604/*	CFI_REL_OFFSET	rflags,EFLAGS*/
605	CFI_REL_OFFSET	rsp,RSP
606/*	CFI_REL_OFFSET	ss,SS*/
607	SAVE_EXTRA_REGS 8
608	call *%rax
609	RESTORE_EXTRA_REGS 8
610	ret
611	CFI_ENDPROC
612END(ia32_ptregs_common)
613