1
2
3 #include <asm/cache.h>
4
5 #include "entry.h"
6 #include <asm/native/inst.h>
7
8 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
9
10 #define ACCOUNT_GET_STAMP \
11 (pUStk) mov.m r20=ar.itc;
12 #define ACCOUNT_SYS_ENTER \
13 (pUStk) br.call.spnt rp=account_sys_enter \
14 ;;
15 #else
16 #define ACCOUNT_GET_STAMP
17 #define ACCOUNT_SYS_ENTER
18 #endif
19
20 .section ".data..patch.rse", "a"
21 .previous
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48 #define IA64_NATIVE_DO_SAVE_MIN(__COVER,SAVE_IFS,EXTRA,WORKAROUND) \
49 mov r16=IA64_KR(CURRENT); \
50 mov r27=ar.rsc; \
51 mov r20=r1; \
52 mov r25=ar.unat; \
53 MOV_FROM_IPSR(p0,r29); \
54 mov r26=ar.pfs; \
55 MOV_FROM_IIP(r28); \
56 mov r21=ar.fpsr; \
57 __COVER; \
58 ;; \
59 adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16; \
60 ;; \
61 ld1 r17=[r16]; \
62 st1 [r16]=r0; \
63 adds r1=-IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 \
64 \
65 ;; \
66 invala; \
67 SAVE_IFS; \
68 cmp.eq pKStk,pUStk=r0,r17; \
69 ;; \
70 (pUStk) mov ar.rsc=0; \
71 ;; \
72 (pUStk) mov.m r24=ar.rnat; \
73 (pUStk) addl r22=IA64_RBS_OFFSET,r1; \
74 (pKStk) mov r1=sp; \
75 ;; \
76 (pUStk) lfetch.fault.excl.nt1 [r22]; \
77 (pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1; \
78 (pUStk) mov r23=ar.bspstore; \
79 ;; \
80 (pUStk) mov ar.bspstore=r22; \
81 (pKStk) addl r1=-IA64_PT_REGS_SIZE,r1; \
82 ;; \
83 (pUStk) mov r18=ar.bsp; \
84 (pUStk) mov ar.rsc=0x3; \
85 adds r17=2*L1_CACHE_BYTES,r1; \
86 adds r16=PT(CR_IPSR),r1; \
87 ;; \
88 lfetch.fault.excl.nt1 [r17],L1_CACHE_BYTES; \
89 st8 [r16]=r29; \
90 ;; \
91 lfetch.fault.excl.nt1 [r17]; \
92 tbit.nz p15,p0=r29,IA64_PSR_I_BIT; \
93 mov r29=b0 \
94 ;; \
95 WORKAROUND; \
96 adds r16=PT(R8),r1; \
97 adds r17=PT(R9),r1; \
98 (pKStk) mov r18=r0; \
99 ;; \
100 .mem.offset 0,0; st8.spill [r16]=r8,16; \
101 .mem.offset 8,0; st8.spill [r17]=r9,16; \
102 ;; \
103 .mem.offset 0,0; st8.spill [r16]=r10,24; \
104 .mem.offset 8,0; st8.spill [r17]=r11,24; \
105 ;; \
106 st8 [r16]=r28,16; \
107 st8 [r17]=r30,16; \
108 (pUStk) sub r18=r18,r22; \
109 mov r8=ar.ccv; \
110 mov r9=ar.csd; \
111 mov r10=ar.ssd; \
112 movl r11=FPSR_DEFAULT; \
113 ;; \
114 st8 [r16]=r25,16; \
115 st8 [r17]=r26,16; \
116 shl r18=r18,16; \
117 ;; \
118 st8 [r16]=r27,16; \
119 (pUStk) st8 [r17]=r24,16; \
120 (pKStk) adds r17=16,r17; \
121 ;; \
122 (pUStk) st8 [r16]=r23,16; \
123 st8 [r17]=r31,16; \
124 (pKStk) adds r16=16,r16; \
125 ;; \
126 st8 [r16]=r29,16; \
127 st8 [r17]=r18,16; \
128 cmp.eq pNonSys,pSys=r0,r0 \
129 ;; \
130 .mem.offset 0,0; st8.spill [r16]=r20,16; \
131 .mem.offset 8,0; st8.spill [r17]=r12,16; \
132 adds r12=-16,r1; \
133 ;; \
134 .mem.offset 0,0; st8.spill [r16]=r13,16; \
135 .mem.offset 8,0; st8.spill [r17]=r21,16; \
136 mov r13=IA64_KR(CURRENT); \
137 ;; \
138 .mem.offset 0,0; st8.spill [r16]=r15,16; \
139 .mem.offset 8,0; st8.spill [r17]=r14,16; \
140 ;; \
141 .mem.offset 0,0; st8.spill [r16]=r2,16; \
142 .mem.offset 8,0; st8.spill [r17]=r3,16; \
143 ACCOUNT_GET_STAMP \
144 adds r2=IA64_PT_REGS_R16_OFFSET,r1; \
145 ;; \
146 EXTRA; \
147 movl r1=__gp; \
148 ;; \
149 ACCOUNT_SYS_ENTER \
150 bsw.1; \
151 ;;
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167 #define SAVE_REST \
168 .mem.offset 0,0; st8.spill [r2]=r16,16; \
169 .mem.offset 8,0; st8.spill [r3]=r17,16; \
170 ;; \
171 .mem.offset 0,0; st8.spill [r2]=r18,16; \
172 .mem.offset 8,0; st8.spill [r3]=r19,16; \
173 ;; \
174 .mem.offset 0,0; st8.spill [r2]=r20,16; \
175 .mem.offset 8,0; st8.spill [r3]=r21,16; \
176 mov r18=b6; \
177 ;; \
178 .mem.offset 0,0; st8.spill [r2]=r22,16; \
179 .mem.offset 8,0; st8.spill [r3]=r23,16; \
180 mov r19=b7; \
181 ;; \
182 .mem.offset 0,0; st8.spill [r2]=r24,16; \
183 .mem.offset 8,0; st8.spill [r3]=r25,16; \
184 ;; \
185 .mem.offset 0,0; st8.spill [r2]=r26,16; \
186 .mem.offset 8,0; st8.spill [r3]=r27,16; \
187 ;; \
188 .mem.offset 0,0; st8.spill [r2]=r28,16; \
189 .mem.offset 8,0; st8.spill [r3]=r29,16; \
190 ;; \
191 .mem.offset 0,0; st8.spill [r2]=r30,16; \
192 .mem.offset 8,0; st8.spill [r3]=r31,32; \
193 ;; \
194 mov ar.fpsr=r11; \
195 st8 [r2]=r8,8; \
196 adds r24=PT(B6)-PT(F7),r3; \
197 ;; \
198 stf.spill [r2]=f6,32; \
199 stf.spill [r3]=f7,32; \
200 ;; \
201 stf.spill [r2]=f8,32; \
202 stf.spill [r3]=f9,32; \
203 ;; \
204 stf.spill [r2]=f10; \
205 stf.spill [r3]=f11; \
206 adds r25=PT(B7)-PT(F11),r3; \
207 ;; \
208 st8 [r24]=r18,16; \
209 st8 [r25]=r19,16; \
210 ;; \
211 st8 [r24]=r9; \
212 st8 [r25]=r10; \
213 ;;
214
215 #define RSE_WORKAROUND \
216 (pUStk) extr.u r17=r18,3,6; \
217 (pUStk) sub r16=r18,r22; \
218 [1:](pKStk) br.cond.sptk.many 1f; \
219 .xdata4 ".data..patch.rse",1b-. \
220 ;; \
221 cmp.ge p6,p7 = 33,r17; \
222 ;; \
223 (p6) mov r17=0x310; \
224 (p7) mov r17=0x308; \
225 ;; \
226 cmp.leu p1,p0=r16,r17; \
227 (p1) br.cond.sptk.many 1f; \
228 dep.z r17=r26,0,62; \
229 movl r16=2f; \
230 ;; \
231 mov ar.pfs=r17; \
232 dep r27=r0,r27,16,14; \
233 mov b0=r16; \
234 ;; \
235 br.ret.sptk b0; \
236 ;; \
237 2: \
238 mov ar.rsc=r0 \
239 ;; \
240 flushrs; \
241 ;; \
242 mov ar.bspstore=r22 \
243 ;; \
244 mov r18=ar.bsp; \
245 ;; \
246 1: \
247 .pred.rel "mutex", pKStk, pUStk
248
249 #define SAVE_MIN_WITH_COVER DO_SAVE_MIN(COVER, mov r30=cr.ifs, , RSE_WORKAROUND)
250 #define SAVE_MIN_WITH_COVER_R19 DO_SAVE_MIN(COVER, mov r30=cr.ifs, mov r15=r19, RSE_WORKAROUND)
251 #define SAVE_MIN DO_SAVE_MIN( , mov r30=r0, , )