1/*
2 * This file contains the light-weight system call handlers (fsyscall-handlers).
3 *
4 * Copyright (C) 2003 Hewlett-Packard Co
5 * 	David Mosberger-Tang <davidm@hpl.hp.com>
6 *
7 * 25-Sep-03 davidm	Implement fsys_rt_sigprocmask().
8 * 18-Feb-03 louisk	Implement fsys_gettimeofday().
9 * 28-Feb-03 davidm	Fixed several bugs in fsys_gettimeofday().  Tuned it some more,
10 *			probably broke it along the way... ;-)
11 * 13-Jul-04 clameter   Implement fsys_clock_gettime and revise fsys_gettimeofday to make
12 *                      it capable of using memory based clocks without falling back to C code.
13 * 08-Feb-07 Fenghua Yu Implement fsys_getcpu.
14 *
15 */
16
17#include <asm/asmmacro.h>
18#include <asm/errno.h>
19#include <asm/asm-offsets.h>
20#include <asm/percpu.h>
21#include <asm/thread_info.h>
22#include <asm/sal.h>
23#include <asm/signal.h>
24#include <asm/unistd.h>
25
26#include "entry.h"
27#include "paravirt_inst.h"
28
29/*
30 * See Documentation/ia64/fsys.txt for details on fsyscalls.
31 *
32 * On entry to an fsyscall handler:
33 *   r10	= 0 (i.e., defaults to "successful syscall return")
34 *   r11	= saved ar.pfs (a user-level value)
35 *   r15	= system call number
36 *   r16	= "current" task pointer (in normal kernel-mode, this is in r13)
37 *   r32-r39	= system call arguments
38 *   b6		= return address (a user-level value)
39 *   ar.pfs	= previous frame-state (a user-level value)
40 *   PSR.be	= cleared to zero (i.e., little-endian byte order is in effect)
41 *   all other registers may contain values passed in from user-mode
42 *
43 * On return from an fsyscall handler:
44 *   r11	= saved ar.pfs (as passed into the fsyscall handler)
45 *   r15	= system call number (as passed into the fsyscall handler)
46 *   r32-r39	= system call arguments (as passed into the fsyscall handler)
47 *   b6		= return address (as passed into the fsyscall handler)
48 *   ar.pfs	= previous frame-state (as passed into the fsyscall handler)
49 */
50
51ENTRY(fsys_ni_syscall)
52	.prologue
53	.altrp b6
54	.body
55	mov r8=ENOSYS
56	mov r10=-1
57	FSYS_RETURN
58END(fsys_ni_syscall)
59
60ENTRY(fsys_getpid)
61	.prologue
62	.altrp b6
63	.body
64	add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
65	;;
66	ld8 r17=[r17]				// r17 = current->group_leader
67	add r9=TI_FLAGS+IA64_TASK_SIZE,r16
68	;;
69	ld4 r9=[r9]
70	add r17=IA64_TASK_TGIDLINK_OFFSET,r17
71	;;
72	and r9=TIF_ALLWORK_MASK,r9
73	ld8 r17=[r17]				// r17 = current->group_leader->pids[PIDTYPE_PID].pid
74	;;
75	add r8=IA64_PID_LEVEL_OFFSET,r17
76	;;
77	ld4 r8=[r8]				// r8 = pid->level
78	add r17=IA64_PID_UPID_OFFSET,r17	// r17 = &pid->numbers[0]
79	;;
80	shl r8=r8,IA64_UPID_SHIFT
81	;;
82	add r17=r17,r8				// r17 = &pid->numbers[pid->level]
83	;;
84	ld4 r8=[r17]				// r8 = pid->numbers[pid->level].nr
85	;;
86	mov r17=0
87	;;
88	cmp.ne p8,p0=0,r9
89(p8)	br.spnt.many fsys_fallback_syscall
90	FSYS_RETURN
91END(fsys_getpid)
92
93ENTRY(fsys_set_tid_address)
94	.prologue
95	.altrp b6
96	.body
97	add r9=TI_FLAGS+IA64_TASK_SIZE,r16
98	add r17=IA64_TASK_TGIDLINK_OFFSET,r16
99	;;
100	ld4 r9=[r9]
101	tnat.z p6,p7=r32		// check argument register for being NaT
102	ld8 r17=[r17]				// r17 = current->pids[PIDTYPE_PID].pid
103	;;
104	and r9=TIF_ALLWORK_MASK,r9
105	add r8=IA64_PID_LEVEL_OFFSET,r17
106	add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
107	;;
108	ld4 r8=[r8]				// r8 = pid->level
109	add r17=IA64_PID_UPID_OFFSET,r17	// r17 = &pid->numbers[0]
110	;;
111	shl r8=r8,IA64_UPID_SHIFT
112	;;
113	add r17=r17,r8				// r17 = &pid->numbers[pid->level]
114	;;
115	ld4 r8=[r17]				// r8 = pid->numbers[pid->level].nr
116	;;
117	cmp.ne p8,p0=0,r9
118	mov r17=-1
119	;;
120(p6)	st8 [r18]=r32
121(p7)	st8 [r18]=r17
122(p8)	br.spnt.many fsys_fallback_syscall
123	;;
124	mov r17=0			// i must not leak kernel bits...
125	mov r18=0			// i must not leak kernel bits...
126	FSYS_RETURN
127END(fsys_set_tid_address)
128
129#if IA64_GTOD_SEQ_OFFSET !=0
130#error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
131#endif
132#if IA64_ITC_JITTER_OFFSET !=0
133#error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
134#endif
135#define CLOCK_REALTIME 0
136#define CLOCK_MONOTONIC 1
137#define CLOCK_DIVIDE_BY_1000 0x4000
138#define CLOCK_ADD_MONOTONIC 0x8000
139
140ENTRY(fsys_gettimeofday)
141	.prologue
142	.altrp b6
143	.body
144	mov r31 = r32
145	tnat.nz p6,p0 = r33		// guard against NaT argument
146(p6)    br.cond.spnt.few .fail_einval
147	mov r30 = CLOCK_DIVIDE_BY_1000
148	;;
149.gettime:
150	// Register map
151	// Incoming r31 = pointer to address where to place result
152	//          r30 = flags determining how time is processed
153	// r2,r3 = temp r4-r7 preserved
154	// r8 = result nanoseconds
155	// r9 = result seconds
156	// r10 = temporary storage for clock difference
157	// r11 = preserved: saved ar.pfs
158	// r12 = preserved: memory stack
159	// r13 = preserved: thread pointer
160	// r14 = address of mask / mask value
161	// r15 = preserved: system call number
162	// r16 = preserved: current task pointer
163	// r17 = (not used)
164	// r18 = (not used)
165	// r19 = address of itc_lastcycle
166	// r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
167	// r21 = address of mmio_ptr
168	// r22 = address of wall_time or monotonic_time
169	// r23 = address of shift / value
170	// r24 = address mult factor / cycle_last value
171	// r25 = itc_lastcycle value
172	// r26 = address clocksource cycle_last
173	// r27 = (not used)
174	// r28 = sequence number at the beginning of critcal section
175	// r29 = address of itc_jitter
176	// r30 = time processing flags / memory address
177	// r31 = pointer to result
178	// Predicates
179	// p6,p7 short term use
180	// p8 = timesource ar.itc
181	// p9 = timesource mmio64
182	// p10 = timesource mmio32 - not used
183	// p11 = timesource not to be handled by asm code
184	// p12 = memory time source ( = p9 | p10) - not used
185	// p13 = do cmpxchg with itc_lastcycle
186	// p14 = Divide by 1000
187	// p15 = Add monotonic
188	//
189	// Note that instructions are optimized for McKinley. McKinley can
190	// process two bundles simultaneously and therefore we continuously
191	// try to feed the CPU two bundles and then a stop.
192
193	add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
194	tnat.nz p6,p0 = r31		// guard against Nat argument
195(p6)	br.cond.spnt.few .fail_einval
196	movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
197	;;
198	ld4 r2 = [r2]			// process work pending flags
199	movl r29 = itc_jitter_data	// itc_jitter
200	add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20	// wall_time
201	add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
202	mov pr = r30,0xc000	// Set predicates according to function
203	;;
204	and r2 = TIF_ALLWORK_MASK,r2
205	add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
206(p15)	add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20	// monotonic_time
207	;;
208	add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20	// clksrc_cycle_last
209	cmp.ne p6, p0 = 0, r2	// Fallback if work is scheduled
210(p6)	br.cond.spnt.many fsys_fallback_syscall
211	;;
212	// Begin critical section
213.time_redo:
214	ld4.acq r28 = [r20]	// gtod_lock.sequence, Must take first
215	;;
216	and r28 = ~1,r28	// And make sequence even to force retry if odd
217	;;
218	ld8 r30 = [r21]		// clocksource->mmio_ptr
219	add r24 = IA64_CLKSRC_MULT_OFFSET,r20
220	ld4 r2 = [r29]		// itc_jitter value
221	add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
222	add r14 = IA64_CLKSRC_MASK_OFFSET,r20
223	;;
224	ld4 r3 = [r24]		// clocksource mult value
225	ld8 r14 = [r14]         // clocksource mask value
226	cmp.eq p8,p9 = 0,r30	// use cpu timer if no mmio_ptr
227	;;
228	setf.sig f7 = r3	// Setup for mult scaling of counter
229(p8)	cmp.ne p13,p0 = r2,r0	// need itc_jitter compensation, set p13
230	ld4 r23 = [r23]		// clocksource shift value
231	ld8 r24 = [r26]		// get clksrc_cycle_last value
232(p9)	cmp.eq p13,p0 = 0,r30	// if mmio_ptr, clear p13 jitter control
233	;;
234	.pred.rel.mutex p8,p9
235	MOV_FROM_ITC(p8, p6, r2, r10)	// CPU_TIMER. 36 clocks latency!!!
236(p9)	ld8 r2 = [r30]		// MMIO_TIMER. Could also have latency issues..
237(p13)	ld8 r25 = [r19]		// get itc_lastcycle value
238	ld8 r9 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET	// tv_sec
239	;;
240	ld8 r8 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET	// tv_nsec
241(p13)	sub r3 = r25,r2		// Diff needed before comparison (thanks davidm)
242	;;
243(p13)	cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
244	sub r10 = r2,r24	// current_cycle - last_cycle
245	;;
246(p6)	sub r10 = r25,r24	// time we got was less than last_cycle
247(p7)	mov ar.ccv = r25	// more than last_cycle. Prep for cmpxchg
248	;;
249(p7)	cmpxchg8.rel r3 = [r19],r2,ar.ccv
250	;;
251(p7)	cmp.ne p7,p0 = r25,r3	// if cmpxchg not successful
252	;;
253(p7)	sub r10 = r3,r24	// then use new last_cycle instead
254	;;
255	and r10 = r10,r14	// Apply mask
256	;;
257	setf.sig f8 = r10
258	nop.i 123
259	;;
260	// fault check takes 5 cycles and we have spare time
261EX(.fail_efault, probe.w.fault r31, 3)
262	xmpy.l f8 = f8,f7	// nsec_per_cyc*(counter-last_counter)
263	;;
264	getf.sig r2 = f8
265	mf
266	;;
267	ld4 r10 = [r20]		// gtod_lock.sequence
268	shr.u r2 = r2,r23	// shift by factor
269	;;
270	add r8 = r8,r2		// Add xtime.nsecs
271	cmp4.ne p7,p0 = r28,r10
272(p7)	br.cond.dpnt.few .time_redo	// sequence number changed, redo
273	// End critical section.
274	// Now r8=tv->tv_nsec and r9=tv->tv_sec
275	mov r10 = r0
276	movl r2 = 1000000000
277	add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
278(p14)	movl r3 = 2361183241434822607	// Prep for / 1000 hack
279	;;
280.time_normalize:
281	mov r21 = r8
282	cmp.ge p6,p0 = r8,r2
283(p14)	shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
284	;;
285(p14)	setf.sig f8 = r20
286(p6)	sub r8 = r8,r2
287(p6)	add r9 = 1,r9		// two nops before the branch.
288(p14)	setf.sig f7 = r3	// Chances for repeats are 1 in 10000 for gettod
289(p6)	br.cond.dpnt.few .time_normalize
290	;;
291	// Divided by 8 though shift. Now divide by 125
292	// The compiler was able to do that with a multiply
293	// and a shift and we do the same
294EX(.fail_efault, probe.w.fault r23, 3)	// This also costs 5 cycles
295(p14)	xmpy.hu f8 = f8, f7		// xmpy has 5 cycles latency so use it
296	;;
297(p14)	getf.sig r2 = f8
298	;;
299	mov r8 = r0
300(p14)	shr.u r21 = r2, 4
301	;;
302EX(.fail_efault, st8 [r31] = r9)
303EX(.fail_efault, st8 [r23] = r21)
304	FSYS_RETURN
305.fail_einval:
306	mov r8 = EINVAL
307	mov r10 = -1
308	FSYS_RETURN
309.fail_efault:
310	mov r8 = EFAULT
311	mov r10 = -1
312	FSYS_RETURN
313END(fsys_gettimeofday)
314
315ENTRY(fsys_clock_gettime)
316	.prologue
317	.altrp b6
318	.body
319	cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
320	// Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
321(p6)	br.spnt.few fsys_fallback_syscall
322	mov r31 = r33
323	shl r30 = r32,15
324	br.many .gettime
325END(fsys_clock_gettime)
326
327/*
328 * fsys_getcpu doesn't use the third parameter in this implementation. It reads
329 * current_thread_info()->cpu and corresponding node in cpu_to_node_map.
330 */
331ENTRY(fsys_getcpu)
332	.prologue
333	.altrp b6
334	.body
335	;;
336	add r2=TI_FLAGS+IA64_TASK_SIZE,r16
337	tnat.nz p6,p0 = r32			// guard against NaT argument
338	add r3=TI_CPU+IA64_TASK_SIZE,r16
339	;;
340	ld4 r3=[r3]				// M r3 = thread_info->cpu
341	ld4 r2=[r2]				// M r2 = thread_info->flags
342(p6)    br.cond.spnt.few .fail_einval		// B
343	;;
344	tnat.nz p7,p0 = r33			// I guard against NaT argument
345(p7)    br.cond.spnt.few .fail_einval		// B
346	;;
347	cmp.ne p6,p0=r32,r0
348	cmp.ne p7,p0=r33,r0
349	;;
350#ifdef CONFIG_NUMA
351	movl r17=cpu_to_node_map
352	;;
353EX(.fail_efault, (p6) probe.w.fault r32, 3)		// M This takes 5 cycles
354EX(.fail_efault, (p7) probe.w.fault r33, 3)		// M This takes 5 cycles
355	shladd r18=r3,1,r17
356	;;
357	ld2 r20=[r18]				// r20 = cpu_to_node_map[cpu]
358	and r2 = TIF_ALLWORK_MASK,r2
359	;;
360	cmp.ne p8,p0=0,r2
361(p8)	br.spnt.many fsys_fallback_syscall
362	;;
363	;;
364EX(.fail_efault, (p6) st4 [r32] = r3)
365EX(.fail_efault, (p7) st2 [r33] = r20)
366	mov r8=0
367	;;
368#else
369EX(.fail_efault, (p6) probe.w.fault r32, 3)		// M This takes 5 cycles
370EX(.fail_efault, (p7) probe.w.fault r33, 3)		// M This takes 5 cycles
371	and r2 = TIF_ALLWORK_MASK,r2
372	;;
373	cmp.ne p8,p0=0,r2
374(p8)	br.spnt.many fsys_fallback_syscall
375	;;
376EX(.fail_efault, (p6) st4 [r32] = r3)
377EX(.fail_efault, (p7) st2 [r33] = r0)
378	mov r8=0
379	;;
380#endif
381	FSYS_RETURN
382END(fsys_getcpu)
383
384ENTRY(fsys_fallback_syscall)
385	.prologue
386	.altrp b6
387	.body
388	/*
389	 * We only get here from light-weight syscall handlers.  Thus, we already
390	 * know that r15 contains a valid syscall number.  No need to re-check.
391	 */
392	adds r17=-1024,r15
393	movl r14=sys_call_table
394	;;
395	RSM_PSR_I(p0, r26, r27)
396	shladd r18=r17,3,r14
397	;;
398	ld8 r18=[r18]				// load normal (heavy-weight) syscall entry-point
399	MOV_FROM_PSR(p0, r29, r26)		// read psr (12 cyc load latency)
400	mov r27=ar.rsc
401	mov r21=ar.fpsr
402	mov r26=ar.pfs
403END(fsys_fallback_syscall)
404	/* FALL THROUGH */
405GLOBAL_ENTRY(paravirt_fsys_bubble_down)
406	.prologue
407	.altrp b6
408	.body
409	/*
410	 * We get here for syscalls that don't have a lightweight
411	 * handler.  For those, we need to bubble down into the kernel
412	 * and that requires setting up a minimal pt_regs structure,
413	 * and initializing the CPU state more or less as if an
414	 * interruption had occurred.  To make syscall-restarts work,
415	 * we setup pt_regs such that cr_iip points to the second
416	 * instruction in syscall_via_break.  Decrementing the IP
417	 * hence will restart the syscall via break and not
418	 * decrementing IP will return us to the caller, as usual.
419	 * Note that we preserve the value of psr.pp rather than
420	 * initializing it from dcr.pp.  This makes it possible to
421	 * distinguish fsyscall execution from other privileged
422	 * execution.
423	 *
424	 * On entry:
425	 *	- normal fsyscall handler register usage, except
426	 *	  that we also have:
427	 *	- r18: address of syscall entry point
428	 *	- r21: ar.fpsr
429	 *	- r26: ar.pfs
430	 *	- r27: ar.rsc
431	 *	- r29: psr
432	 *
433	 * We used to clear some PSR bits here but that requires slow
434	 * serialization.  Fortuntely, that isn't really necessary.
435	 * The rationale is as follows: we used to clear bits
436	 * ~PSR_PRESERVED_BITS in PSR.L.  Since
437	 * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
438	 * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
439	 * However,
440	 *
441	 * PSR.BE : already is turned off in __kernel_syscall_via_epc()
442	 * PSR.AC : don't care (kernel normally turns PSR.AC on)
443	 * PSR.I  : already turned off by the time paravirt_fsys_bubble_down gets
444	 *	    invoked
445	 * PSR.DFL: always 0 (kernel never turns it on)
446	 * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
447	 *	    initiative
448	 * PSR.DI : always 0 (kernel never turns it on)
449	 * PSR.SI : always 0 (kernel never turns it on)
450	 * PSR.DB : don't care --- kernel never enables kernel-level
451	 *	    breakpoints
452	 * PSR.TB : must be 0 already; if it wasn't zero on entry to
453	 *          __kernel_syscall_via_epc, the branch to paravirt_fsys_bubble_down
454	 *          will trigger a taken branch; the taken-trap-handler then
455	 *          converts the syscall into a break-based system-call.
456	 */
457	/*
458	 * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
459	 * The rest we have to synthesize.
460	 */
461#	define PSR_ONE_BITS		((3 << IA64_PSR_CPL0_BIT)	\
462					 | (0x1 << IA64_PSR_RI_BIT)	\
463					 | IA64_PSR_BN | IA64_PSR_I)
464
465	invala					// M0|1
466	movl r14=ia64_ret_from_syscall		// X
467
468	nop.m 0
469	movl r28=__kernel_syscall_via_break	// X	create cr.iip
470	;;
471
472	mov r2=r16				// A    get task addr to addl-addressable register
473	adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
474	mov r31=pr				// I0   save pr (2 cyc)
475	;;
476	st1 [r16]=r0				// M2|3 clear current->thread.on_ustack flag
477	addl r22=IA64_RBS_OFFSET,r2		// A    compute base of RBS
478	add r3=TI_FLAGS+IA64_TASK_SIZE,r2	// A
479	;;
480	ld4 r3=[r3]				// M0|1 r3 = current_thread_info()->flags
481	lfetch.fault.excl.nt1 [r22]		// M0|1 prefetch register backing-store
482	nop.i 0
483	;;
484	mov ar.rsc=0				// M2   set enforced lazy mode, pl 0, LE, loadrs=0
485#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
486	MOV_FROM_ITC(p0, p6, r30, r23)		// M    get cycle for accounting
487#else
488	nop.m 0
489#endif
490	nop.i 0
491	;;
492	mov r23=ar.bspstore			// M2 (12 cyc) save ar.bspstore
493	mov.m r24=ar.rnat			// M2 (5 cyc) read ar.rnat (dual-issues!)
494	nop.i 0
495	;;
496	mov ar.bspstore=r22			// M2 (6 cyc) switch to kernel RBS
497	movl r8=PSR_ONE_BITS			// X
498	;;
499	mov r25=ar.unat				// M2 (5 cyc) save ar.unat
500	mov r19=b6				// I0   save b6 (2 cyc)
501	mov r20=r1				// A    save caller's gp in r20
502	;;
503	or r29=r8,r29				// A    construct cr.ipsr value to save
504	mov b6=r18				// I0   copy syscall entry-point to b6 (7 cyc)
505	addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
506
507	mov r18=ar.bsp				// M2   save (kernel) ar.bsp (12 cyc)
508	cmp.ne pKStk,pUStk=r0,r0		// A    set pKStk <- 0, pUStk <- 1
509	br.call.sptk.many b7=ia64_syscall_setup	// B
510	;;
511#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
512	// mov.m r30=ar.itc is called in advance
513	add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
514	add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
515	;;
516	ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP	// time at last check in kernel
517	ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE	// time at leave kernel
518	;;
519	ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME	// cumulated stime
520	ld8 r21=[r17]				// cumulated utime
521	sub r22=r19,r18				// stime before leave kernel
522	;;
523	st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP	// update stamp
524	sub r18=r30,r19				// elapsed time in user mode
525	;;
526	add r20=r20,r22				// sum stime
527	add r21=r21,r18				// sum utime
528	;;
529	st8 [r16]=r20				// update stime
530	st8 [r17]=r21				// update utime
531	;;
532#endif
533	mov ar.rsc=0x3				// M2   set eager mode, pl 0, LE, loadrs=0
534	mov rp=r14				// I0   set the real return addr
535	and r3=_TIF_SYSCALL_TRACEAUDIT,r3	// A
536	;;
537	SSM_PSR_I(p0, p6, r22)			// M2   we're on kernel stacks now, reenable irqs
538	cmp.eq p8,p0=r3,r0			// A
539(p10)	br.cond.spnt.many ia64_ret_from_syscall	// B    return if bad call-frame or r15 is a NaT
540
541	nop.m 0
542(p8)	br.call.sptk.many b6=b6			// B    (ignore return address)
543	br.cond.spnt ia64_trace_syscall		// B
544END(paravirt_fsys_bubble_down)
545
546	.rodata
547	.align 8
548	.globl paravirt_fsyscall_table
549
550	data8 paravirt_fsys_bubble_down
551paravirt_fsyscall_table:
552	data8 fsys_ni_syscall
553	data8 0				// exit			// 1025
554	data8 0				// read
555	data8 0				// write
556	data8 0				// open
557	data8 0				// close
558	data8 0				// creat		// 1030
559	data8 0				// link
560	data8 0				// unlink
561	data8 0				// execve
562	data8 0				// chdir
563	data8 0				// fchdir		// 1035
564	data8 0				// utimes
565	data8 0				// mknod
566	data8 0				// chmod
567	data8 0				// chown
568	data8 0				// lseek		// 1040
569	data8 fsys_getpid		// getpid
570	data8 0				// getppid
571	data8 0				// mount
572	data8 0				// umount
573	data8 0				// setuid		// 1045
574	data8 0				// getuid
575	data8 0				// geteuid
576	data8 0				// ptrace
577	data8 0				// access
578	data8 0				// sync			// 1050
579	data8 0				// fsync
580	data8 0				// fdatasync
581	data8 0				// kill
582	data8 0				// rename
583	data8 0				// mkdir		// 1055
584	data8 0				// rmdir
585	data8 0				// dup
586	data8 0				// pipe
587	data8 0				// times
588	data8 0				// brk			// 1060
589	data8 0				// setgid
590	data8 0				// getgid
591	data8 0				// getegid
592	data8 0				// acct
593	data8 0				// ioctl		// 1065
594	data8 0				// fcntl
595	data8 0				// umask
596	data8 0				// chroot
597	data8 0				// ustat
598	data8 0				// dup2			// 1070
599	data8 0				// setreuid
600	data8 0				// setregid
601	data8 0				// getresuid
602	data8 0				// setresuid
603	data8 0				// getresgid		// 1075
604	data8 0				// setresgid
605	data8 0				// getgroups
606	data8 0				// setgroups
607	data8 0				// getpgid
608	data8 0				// setpgid		// 1080
609	data8 0				// setsid
610	data8 0				// getsid
611	data8 0				// sethostname
612	data8 0				// setrlimit
613	data8 0				// getrlimit		// 1085
614	data8 0				// getrusage
615	data8 fsys_gettimeofday		// gettimeofday
616	data8 0				// settimeofday
617	data8 0				// select
618	data8 0				// poll			// 1090
619	data8 0				// symlink
620	data8 0				// readlink
621	data8 0				// uselib
622	data8 0				// swapon
623	data8 0				// swapoff		// 1095
624	data8 0				// reboot
625	data8 0				// truncate
626	data8 0				// ftruncate
627	data8 0				// fchmod
628	data8 0				// fchown		// 1100
629	data8 0				// getpriority
630	data8 0				// setpriority
631	data8 0				// statfs
632	data8 0				// fstatfs
633	data8 0				// gettid		// 1105
634	data8 0				// semget
635	data8 0				// semop
636	data8 0				// semctl
637	data8 0				// msgget
638	data8 0				// msgsnd		// 1110
639	data8 0				// msgrcv
640	data8 0				// msgctl
641	data8 0				// shmget
642	data8 0				// shmat
643	data8 0				// shmdt		// 1115
644	data8 0				// shmctl
645	data8 0				// syslog
646	data8 0				// setitimer
647	data8 0				// getitimer
648	data8 0					 		// 1120
649	data8 0
650	data8 0
651	data8 0				// vhangup
652	data8 0				// lchown
653	data8 0				// remap_file_pages	// 1125
654	data8 0				// wait4
655	data8 0				// sysinfo
656	data8 0				// clone
657	data8 0				// setdomainname
658	data8 0				// newuname		// 1130
659	data8 0				// adjtimex
660	data8 0
661	data8 0				// init_module
662	data8 0				// delete_module
663	data8 0							// 1135
664	data8 0
665	data8 0				// quotactl
666	data8 0				// bdflush
667	data8 0				// sysfs
668	data8 0				// personality		// 1140
669	data8 0				// afs_syscall
670	data8 0				// setfsuid
671	data8 0				// setfsgid
672	data8 0				// getdents
673	data8 0				// flock		// 1145
674	data8 0				// readv
675	data8 0				// writev
676	data8 0				// pread64
677	data8 0				// pwrite64
678	data8 0				// sysctl		// 1150
679	data8 0				// mmap
680	data8 0				// munmap
681	data8 0				// mlock
682	data8 0				// mlockall
683	data8 0				// mprotect		// 1155
684	data8 0				// mremap
685	data8 0				// msync
686	data8 0				// munlock
687	data8 0				// munlockall
688	data8 0				// sched_getparam	// 1160
689	data8 0				// sched_setparam
690	data8 0				// sched_getscheduler
691	data8 0				// sched_setscheduler
692	data8 0				// sched_yield
693	data8 0				// sched_get_priority_max	// 1165
694	data8 0				// sched_get_priority_min
695	data8 0				// sched_rr_get_interval
696	data8 0				// nanosleep
697	data8 0				// nfsservctl
698	data8 0				// prctl		// 1170
699	data8 0				// getpagesize
700	data8 0				// mmap2
701	data8 0				// pciconfig_read
702	data8 0				// pciconfig_write
703	data8 0				// perfmonctl		// 1175
704	data8 0				// sigaltstack
705	data8 0				// rt_sigaction
706	data8 0				// rt_sigpending
707	data8 0				// rt_sigprocmask
708	data8 0				// rt_sigqueueinfo	// 1180
709	data8 0				// rt_sigreturn
710	data8 0				// rt_sigsuspend
711	data8 0				// rt_sigtimedwait
712	data8 0				// getcwd
713	data8 0				// capget		// 1185
714	data8 0				// capset
715	data8 0				// sendfile
716	data8 0
717	data8 0
718	data8 0				// socket		// 1190
719	data8 0				// bind
720	data8 0				// connect
721	data8 0				// listen
722	data8 0				// accept
723	data8 0				// getsockname		// 1195
724	data8 0				// getpeername
725	data8 0				// socketpair
726	data8 0				// send
727	data8 0				// sendto
728	data8 0				// recv			// 1200
729	data8 0				// recvfrom
730	data8 0				// shutdown
731	data8 0				// setsockopt
732	data8 0				// getsockopt
733	data8 0				// sendmsg		// 1205
734	data8 0				// recvmsg
735	data8 0				// pivot_root
736	data8 0				// mincore
737	data8 0				// madvise
738	data8 0				// newstat		// 1210
739	data8 0				// newlstat
740	data8 0				// newfstat
741	data8 0				// clone2
742	data8 0				// getdents64
743	data8 0				// getunwind		// 1215
744	data8 0				// readahead
745	data8 0				// setxattr
746	data8 0				// lsetxattr
747	data8 0				// fsetxattr
748	data8 0				// getxattr		// 1220
749	data8 0				// lgetxattr
750	data8 0				// fgetxattr
751	data8 0				// listxattr
752	data8 0				// llistxattr
753	data8 0				// flistxattr		// 1225
754	data8 0				// removexattr
755	data8 0				// lremovexattr
756	data8 0				// fremovexattr
757	data8 0				// tkill
758	data8 0				// futex		// 1230
759	data8 0				// sched_setaffinity
760	data8 0				// sched_getaffinity
761	data8 fsys_set_tid_address	// set_tid_address
762	data8 0				// fadvise64_64
763	data8 0				// tgkill		// 1235
764	data8 0				// exit_group
765	data8 0				// lookup_dcookie
766	data8 0				// io_setup
767	data8 0				// io_destroy
768	data8 0				// io_getevents		// 1240
769	data8 0				// io_submit
770	data8 0				// io_cancel
771	data8 0				// epoll_create
772	data8 0				// epoll_ctl
773	data8 0				// epoll_wait		// 1245
774	data8 0				// restart_syscall
775	data8 0				// semtimedop
776	data8 0				// timer_create
777	data8 0				// timer_settime
778	data8 0				// timer_gettime 	// 1250
779	data8 0				// timer_getoverrun
780	data8 0				// timer_delete
781	data8 0				// clock_settime
782	data8 fsys_clock_gettime	// clock_gettime
783	data8 0				// clock_getres		// 1255
784	data8 0				// clock_nanosleep
785	data8 0				// fstatfs64
786	data8 0				// statfs64
787	data8 0				// mbind
788	data8 0				// get_mempolicy	// 1260
789	data8 0				// set_mempolicy
790	data8 0				// mq_open
791	data8 0				// mq_unlink
792	data8 0				// mq_timedsend
793	data8 0				// mq_timedreceive	// 1265
794	data8 0				// mq_notify
795	data8 0				// mq_getsetattr
796	data8 0				// kexec_load
797	data8 0				// vserver
798	data8 0				// waitid		// 1270
799	data8 0				// add_key
800	data8 0				// request_key
801	data8 0				// keyctl
802	data8 0				// ioprio_set
803	data8 0				// ioprio_get		// 1275
804	data8 0				// move_pages
805	data8 0				// inotify_init
806	data8 0				// inotify_add_watch
807	data8 0				// inotify_rm_watch
808	data8 0				// migrate_pages	// 1280
809	data8 0				// openat
810	data8 0				// mkdirat
811	data8 0				// mknodat
812	data8 0				// fchownat
813	data8 0				// futimesat		// 1285
814	data8 0				// newfstatat
815	data8 0				// unlinkat
816	data8 0				// renameat
817	data8 0				// linkat
818	data8 0				// symlinkat		// 1290
819	data8 0				// readlinkat
820	data8 0				// fchmodat
821	data8 0				// faccessat
822	data8 0
823	data8 0							// 1295
824	data8 0				// unshare
825	data8 0				// splice
826	data8 0				// set_robust_list
827	data8 0				// get_robust_list
828	data8 0				// sync_file_range	// 1300
829	data8 0				// tee
830	data8 0				// vmsplice
831	data8 0
832	data8 fsys_getcpu		// getcpu		// 1304
833
834	// fill in zeros for the remaining entries
835	.zero:
836	.space paravirt_fsyscall_table + 8*NR_syscalls - .zero, 0
837