1/*
2 * Asm versions of Xen pv-ops, suitable for either direct use or
3 * inlining.  The inline versions are the same as the direct-use
4 * versions, with the pre- and post-amble chopped off.
5 *
6 * This code is encoded for size rather than absolute efficiency, with
7 * a view to being able to inline as much as possible.
8 *
9 * We only bother with direct forms (ie, vcpu in pda) of the
10 * operations here; the indirect forms are better handled in C, since
11 * they're generally too large to inline anyway.
12 */
13
14#include <asm/errno.h>
15#include <asm/percpu.h>
16#include <asm/processor-flags.h>
17#include <asm/segment.h>
18#include <asm/asm-offsets.h>
19#include <asm/thread_info.h>
20
21#include <xen/interface/xen.h>
22
23#include "xen-asm.h"
24
25ENTRY(xen_adjust_exception_frame)
26	mov 8+0(%rsp), %rcx
27	mov 8+8(%rsp), %r11
28	ret $16
29
30hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
31/*
32 * Xen64 iret frame:
33 *
34 *	ss
35 *	rsp
36 *	rflags
37 *	cs
38 *	rip		<-- standard iret frame
39 *
40 *	flags
41 *
42 *	rcx		}
43 *	r11		}<-- pushed by hypercall page
44 * rsp->rax		}
45 */
46ENTRY(xen_iret)
47	pushq $0
481:	jmp hypercall_iret
49ENDPATCH(xen_iret)
50RELOC(xen_iret, 1b+1)
51
52ENTRY(xen_sysret64)
53	/*
54	 * We're already on the usermode stack at this point, but
55	 * still with the kernel gs, so we can easily switch back
56	 */
57	movq %rsp, PER_CPU_VAR(rsp_scratch)
58	movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
59
60	pushq $__USER_DS
61	pushq PER_CPU_VAR(rsp_scratch)
62	pushq %r11
63	pushq $__USER_CS
64	pushq %rcx
65
66	pushq $VGCF_in_syscall
671:	jmp hypercall_iret
68ENDPATCH(xen_sysret64)
69RELOC(xen_sysret64, 1b+1)
70
71ENTRY(xen_sysret32)
72	/*
73	 * We're already on the usermode stack at this point, but
74	 * still with the kernel gs, so we can easily switch back
75	 */
76	movq %rsp, PER_CPU_VAR(rsp_scratch)
77	movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
78
79	pushq $__USER32_DS
80	pushq PER_CPU_VAR(rsp_scratch)
81	pushq %r11
82	pushq $__USER32_CS
83	pushq %rcx
84
85	pushq $0
861:	jmp hypercall_iret
87ENDPATCH(xen_sysret32)
88RELOC(xen_sysret32, 1b+1)
89
90/*
91 * Xen handles syscall callbacks much like ordinary exceptions, which
92 * means we have:
93 * - kernel gs
94 * - kernel rsp
95 * - an iret-like stack frame on the stack (including rcx and r11):
96 *	ss
97 *	rsp
98 *	rflags
99 *	cs
100 *	rip
101 *	r11
102 * rsp->rcx
103 *
104 * In all the entrypoints, we undo all that to make it look like a
105 * CPU-generated syscall/sysenter and jump to the normal entrypoint.
106 */
107
108.macro undo_xen_syscall
109	mov 0*8(%rsp), %rcx
110	mov 1*8(%rsp), %r11
111	mov 5*8(%rsp), %rsp
112.endm
113
114/* Normal 64-bit system call target */
115ENTRY(xen_syscall_target)
116	undo_xen_syscall
117	jmp entry_SYSCALL_64_after_swapgs
118ENDPROC(xen_syscall_target)
119
120#ifdef CONFIG_IA32_EMULATION
121
122/* 32-bit compat syscall target */
123ENTRY(xen_syscall32_target)
124	undo_xen_syscall
125	jmp entry_SYSCALL_compat
126ENDPROC(xen_syscall32_target)
127
128/* 32-bit compat sysenter target */
129ENTRY(xen_sysenter_target)
130	undo_xen_syscall
131	jmp entry_SYSENTER_compat
132ENDPROC(xen_sysenter_target)
133
134#else /* !CONFIG_IA32_EMULATION */
135
136ENTRY(xen_syscall32_target)
137ENTRY(xen_sysenter_target)
138	lea 16(%rsp), %rsp	/* strip %rcx, %r11 */
139	mov $-ENOSYS, %rax
140	pushq $0
141	jmp hypercall_iret
142ENDPROC(xen_syscall32_target)
143ENDPROC(xen_sysenter_target)
144
145#endif	/* CONFIG_IA32_EMULATION */
146