1/******************************************************************************
2 * arch/ia64/kernel/paravirt.c
3 *
4 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
5 *                    VA Linux Systems Japan K.K.
6 *     Yaozu (Eddie) Dong <eddie.dong@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
21 *
22 */
23
24#include <linux/init.h>
25
26#include <linux/compiler.h>
27#include <linux/io.h>
28#include <linux/irq.h>
29#include <linux/module.h>
30#include <linux/types.h>
31
32#include <asm/iosapic.h>
33#include <asm/paravirt.h>
34
35/***************************************************************************
36 * general info
37 */
38struct pv_info pv_info = {
39	.kernel_rpl = 0,
40	.paravirt_enabled = 0,
41	.name = "bare hardware"
42};
43
44/***************************************************************************
45 * pv_init_ops
46 * initialization hooks.
47 */
48
49static void __init
50ia64_native_patch_branch(unsigned long tag, unsigned long type);
51
52struct pv_init_ops pv_init_ops =
53{
54#ifdef ASM_SUPPORTED
55	.patch_bundle = ia64_native_patch_bundle,
56#endif
57	.patch_branch = ia64_native_patch_branch,
58};
59
60/***************************************************************************
61 * pv_cpu_ops
62 * intrinsics hooks.
63 */
64
65#ifndef ASM_SUPPORTED
66/* ia64_native_xxx are macros so that we have to make them real functions */
67
68#define DEFINE_VOID_FUNC1(name)					\
69	static void						\
70	ia64_native_ ## name ## _func(unsigned long arg)	\
71	{							\
72		ia64_native_ ## name(arg);			\
73	}
74
75#define DEFINE_VOID_FUNC1_VOID(name)				\
76	static void						\
77	ia64_native_ ## name ## _func(void *arg)		\
78	{							\
79		ia64_native_ ## name(arg);			\
80	}
81
82#define DEFINE_VOID_FUNC2(name)					\
83	static void						\
84	ia64_native_ ## name ## _func(unsigned long arg0,	\
85				      unsigned long arg1)	\
86	{							\
87		ia64_native_ ## name(arg0, arg1);		\
88	}
89
90#define DEFINE_FUNC0(name)			\
91	static unsigned long			\
92	ia64_native_ ## name ## _func(void)	\
93	{					\
94		return ia64_native_ ## name();	\
95	}
96
97#define DEFINE_FUNC1(name, type)			\
98	static unsigned long				\
99	ia64_native_ ## name ## _func(type arg)		\
100	{						\
101		return ia64_native_ ## name(arg);	\
102	}						\
103
104DEFINE_VOID_FUNC1_VOID(fc);
105DEFINE_VOID_FUNC1(intrin_local_irq_restore);
106
107DEFINE_VOID_FUNC2(ptcga);
108DEFINE_VOID_FUNC2(set_rr);
109
110DEFINE_FUNC0(get_psr_i);
111
112DEFINE_FUNC1(thash, unsigned long);
113DEFINE_FUNC1(get_cpuid, int);
114DEFINE_FUNC1(get_pmd, int);
115DEFINE_FUNC1(get_rr, unsigned long);
116
117static void
118ia64_native_ssm_i_func(void)
119{
120	ia64_native_ssm(IA64_PSR_I);
121}
122
123static void
124ia64_native_rsm_i_func(void)
125{
126	ia64_native_rsm(IA64_PSR_I);
127}
128
129static void
130ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
131				unsigned long val2, unsigned long val3,
132				unsigned long val4)
133{
134	ia64_native_set_rr0_to_rr4(val0, val1, val2, val3, val4);
135}
136
137#define CASE_GET_REG(id)				\
138	case _IA64_REG_ ## id:				\
139	res = ia64_native_getreg(_IA64_REG_ ## id);	\
140	break;
141#define CASE_GET_AR(id) CASE_GET_REG(AR_ ## id)
142#define CASE_GET_CR(id) CASE_GET_REG(CR_ ## id)
143
144unsigned long
145ia64_native_getreg_func(int regnum)
146{
147	unsigned long res = -1;
148	switch (regnum) {
149	CASE_GET_REG(GP);
150	/*CASE_GET_REG(IP);*/ /* returned ip value shouldn't be constant */
151	CASE_GET_REG(PSR);
152	CASE_GET_REG(TP);
153	CASE_GET_REG(SP);
154
155	CASE_GET_AR(KR0);
156	CASE_GET_AR(KR1);
157	CASE_GET_AR(KR2);
158	CASE_GET_AR(KR3);
159	CASE_GET_AR(KR4);
160	CASE_GET_AR(KR5);
161	CASE_GET_AR(KR6);
162	CASE_GET_AR(KR7);
163	CASE_GET_AR(RSC);
164	CASE_GET_AR(BSP);
165	CASE_GET_AR(BSPSTORE);
166	CASE_GET_AR(RNAT);
167	CASE_GET_AR(FCR);
168	CASE_GET_AR(EFLAG);
169	CASE_GET_AR(CSD);
170	CASE_GET_AR(SSD);
171	CASE_GET_AR(CFLAG);
172	CASE_GET_AR(FSR);
173	CASE_GET_AR(FIR);
174	CASE_GET_AR(FDR);
175	CASE_GET_AR(CCV);
176	CASE_GET_AR(UNAT);
177	CASE_GET_AR(FPSR);
178	CASE_GET_AR(ITC);
179	CASE_GET_AR(PFS);
180	CASE_GET_AR(LC);
181	CASE_GET_AR(EC);
182
183	CASE_GET_CR(DCR);
184	CASE_GET_CR(ITM);
185	CASE_GET_CR(IVA);
186	CASE_GET_CR(PTA);
187	CASE_GET_CR(IPSR);
188	CASE_GET_CR(ISR);
189	CASE_GET_CR(IIP);
190	CASE_GET_CR(IFA);
191	CASE_GET_CR(ITIR);
192	CASE_GET_CR(IIPA);
193	CASE_GET_CR(IFS);
194	CASE_GET_CR(IIM);
195	CASE_GET_CR(IHA);
196	CASE_GET_CR(LID);
197	CASE_GET_CR(IVR);
198	CASE_GET_CR(TPR);
199	CASE_GET_CR(EOI);
200	CASE_GET_CR(IRR0);
201	CASE_GET_CR(IRR1);
202	CASE_GET_CR(IRR2);
203	CASE_GET_CR(IRR3);
204	CASE_GET_CR(ITV);
205	CASE_GET_CR(PMV);
206	CASE_GET_CR(CMCV);
207	CASE_GET_CR(LRR0);
208	CASE_GET_CR(LRR1);
209
210	default:
211		printk(KERN_CRIT "wrong_getreg %d\n", regnum);
212		break;
213	}
214	return res;
215}
216
217#define CASE_SET_REG(id)				\
218	case _IA64_REG_ ## id:				\
219	ia64_native_setreg(_IA64_REG_ ## id, val);	\
220	break;
221#define CASE_SET_AR(id) CASE_SET_REG(AR_ ## id)
222#define CASE_SET_CR(id) CASE_SET_REG(CR_ ## id)
223
224void
225ia64_native_setreg_func(int regnum, unsigned long val)
226{
227	switch (regnum) {
228	case _IA64_REG_PSR_L:
229		ia64_native_setreg(_IA64_REG_PSR_L, val);
230		ia64_dv_serialize_data();
231		break;
232	CASE_SET_REG(SP);
233	CASE_SET_REG(GP);
234
235	CASE_SET_AR(KR0);
236	CASE_SET_AR(KR1);
237	CASE_SET_AR(KR2);
238	CASE_SET_AR(KR3);
239	CASE_SET_AR(KR4);
240	CASE_SET_AR(KR5);
241	CASE_SET_AR(KR6);
242	CASE_SET_AR(KR7);
243	CASE_SET_AR(RSC);
244	CASE_SET_AR(BSP);
245	CASE_SET_AR(BSPSTORE);
246	CASE_SET_AR(RNAT);
247	CASE_SET_AR(FCR);
248	CASE_SET_AR(EFLAG);
249	CASE_SET_AR(CSD);
250	CASE_SET_AR(SSD);
251	CASE_SET_AR(CFLAG);
252	CASE_SET_AR(FSR);
253	CASE_SET_AR(FIR);
254	CASE_SET_AR(FDR);
255	CASE_SET_AR(CCV);
256	CASE_SET_AR(UNAT);
257	CASE_SET_AR(FPSR);
258	CASE_SET_AR(ITC);
259	CASE_SET_AR(PFS);
260	CASE_SET_AR(LC);
261	CASE_SET_AR(EC);
262
263	CASE_SET_CR(DCR);
264	CASE_SET_CR(ITM);
265	CASE_SET_CR(IVA);
266	CASE_SET_CR(PTA);
267	CASE_SET_CR(IPSR);
268	CASE_SET_CR(ISR);
269	CASE_SET_CR(IIP);
270	CASE_SET_CR(IFA);
271	CASE_SET_CR(ITIR);
272	CASE_SET_CR(IIPA);
273	CASE_SET_CR(IFS);
274	CASE_SET_CR(IIM);
275	CASE_SET_CR(IHA);
276	CASE_SET_CR(LID);
277	CASE_SET_CR(IVR);
278	CASE_SET_CR(TPR);
279	CASE_SET_CR(EOI);
280	CASE_SET_CR(IRR0);
281	CASE_SET_CR(IRR1);
282	CASE_SET_CR(IRR2);
283	CASE_SET_CR(IRR3);
284	CASE_SET_CR(ITV);
285	CASE_SET_CR(PMV);
286	CASE_SET_CR(CMCV);
287	CASE_SET_CR(LRR0);
288	CASE_SET_CR(LRR1);
289	default:
290		printk(KERN_CRIT "wrong setreg %d\n", regnum);
291		break;
292	}
293}
294#else
295
296#define __DEFINE_FUNC(name, code)					\
297	extern const char ia64_native_ ## name ## _direct_start[];	\
298	extern const char ia64_native_ ## name ## _direct_end[];	\
299	asm (".align 32\n"						\
300	     ".proc ia64_native_" #name "_func\n"			\
301	     "ia64_native_" #name "_func:\n"				\
302	     "ia64_native_" #name "_direct_start:\n"			\
303	     code							\
304	     "ia64_native_" #name "_direct_end:\n"			\
305	     "br.cond.sptk.many b6\n"					\
306	     ".endp ia64_native_" #name "_func\n")
307
308#define DEFINE_VOID_FUNC0(name, code)				\
309	extern void						\
310	ia64_native_ ## name ## _func(void);			\
311	__DEFINE_FUNC(name, code)
312
313#define DEFINE_VOID_FUNC1(name, code)				\
314	extern void						\
315	ia64_native_ ## name ## _func(unsigned long arg);	\
316	__DEFINE_FUNC(name, code)
317
318#define DEFINE_VOID_FUNC1_VOID(name, code)			\
319	extern void						\
320	ia64_native_ ## name ## _func(void *arg);		\
321	__DEFINE_FUNC(name, code)
322
323#define DEFINE_VOID_FUNC2(name, code)				\
324	extern void						\
325	ia64_native_ ## name ## _func(unsigned long arg0,	\
326				      unsigned long arg1);	\
327	__DEFINE_FUNC(name, code)
328
329#define DEFINE_FUNC0(name, code)		\
330	extern unsigned long			\
331	ia64_native_ ## name ## _func(void);	\
332	__DEFINE_FUNC(name, code)
333
334#define DEFINE_FUNC1(name, type, code)			\
335	extern unsigned long				\
336	ia64_native_ ## name ## _func(type arg);	\
337	__DEFINE_FUNC(name, code)
338
339DEFINE_VOID_FUNC1_VOID(fc,
340		       "fc r8\n");
341DEFINE_VOID_FUNC1(intrin_local_irq_restore,
342		  ";;\n"
343		  "     cmp.ne p6, p7 = r8, r0\n"
344		  ";;\n"
345		  "(p6) ssm psr.i\n"
346		  "(p7) rsm psr.i\n"
347		  ";;\n"
348		  "(p6) srlz.d\n");
349
350DEFINE_VOID_FUNC2(ptcga,
351		  "ptc.ga r8, r9\n");
352DEFINE_VOID_FUNC2(set_rr,
353		  "mov rr[r8] = r9\n");
354
355/* ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I */
356DEFINE_FUNC0(get_psr_i,
357	     "mov r2 = " __stringify(1 << IA64_PSR_I_BIT) "\n"
358	     "mov r8 = psr\n"
359	     ";;\n"
360	     "and r8 = r2, r8\n");
361
362DEFINE_FUNC1(thash, unsigned long,
363	     "thash r8 = r8\n");
364DEFINE_FUNC1(get_cpuid, int,
365	     "mov r8 = cpuid[r8]\n");
366DEFINE_FUNC1(get_pmd, int,
367	     "mov r8 = pmd[r8]\n");
368DEFINE_FUNC1(get_rr, unsigned long,
369	     "mov r8 = rr[r8]\n");
370
371DEFINE_VOID_FUNC0(ssm_i,
372		  "ssm psr.i\n");
373DEFINE_VOID_FUNC0(rsm_i,
374		  "rsm psr.i\n");
375
376extern void
377ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
378				unsigned long val2, unsigned long val3,
379				unsigned long val4);
380__DEFINE_FUNC(set_rr0_to_rr4,
381	      "mov rr[r0] = r8\n"
382	      "movl r2 = 0x2000000000000000\n"
383	      ";;\n"
384	      "mov rr[r2] = r9\n"
385	      "shl r3 = r2, 1\n"	/* movl r3 = 0x4000000000000000 */
386	      ";;\n"
387	      "add r2 = r2, r3\n"	/* movl r2 = 0x6000000000000000 */
388	      "mov rr[r3] = r10\n"
389	      ";;\n"
390	      "mov rr[r2] = r11\n"
391	      "shl r3 = r3, 1\n"	/* movl r3 = 0x8000000000000000 */
392	      ";;\n"
393	      "mov rr[r3] = r14\n");
394
395extern unsigned long ia64_native_getreg_func(int regnum);
396asm(".global ia64_native_getreg_func\n");
397#define __DEFINE_GET_REG(id, reg)			\
398	"mov r2 = " __stringify(_IA64_REG_ ## id) "\n"	\
399	";;\n"						\
400	"cmp.eq p6, p0 = r2, r8\n"			\
401	";;\n"						\
402	"(p6) mov r8 = " #reg "\n"			\
403	"(p6) br.cond.sptk.many b6\n"			\
404	";;\n"
405#define __DEFINE_GET_AR(id, reg)	__DEFINE_GET_REG(AR_ ## id, ar.reg)
406#define __DEFINE_GET_CR(id, reg)	__DEFINE_GET_REG(CR_ ## id, cr.reg)
407
408__DEFINE_FUNC(getreg,
409	      __DEFINE_GET_REG(GP, gp)
410	      /*__DEFINE_GET_REG(IP, ip)*/ /* returned ip value shouldn't be constant */
411	      __DEFINE_GET_REG(PSR, psr)
412	      __DEFINE_GET_REG(TP, tp)
413	      __DEFINE_GET_REG(SP, sp)
414
415	      __DEFINE_GET_REG(AR_KR0, ar0)
416	      __DEFINE_GET_REG(AR_KR1, ar1)
417	      __DEFINE_GET_REG(AR_KR2, ar2)
418	      __DEFINE_GET_REG(AR_KR3, ar3)
419	      __DEFINE_GET_REG(AR_KR4, ar4)
420	      __DEFINE_GET_REG(AR_KR5, ar5)
421	      __DEFINE_GET_REG(AR_KR6, ar6)
422	      __DEFINE_GET_REG(AR_KR7, ar7)
423	      __DEFINE_GET_AR(RSC, rsc)
424	      __DEFINE_GET_AR(BSP, bsp)
425	      __DEFINE_GET_AR(BSPSTORE, bspstore)
426	      __DEFINE_GET_AR(RNAT, rnat)
427	      __DEFINE_GET_AR(FCR, fcr)
428	      __DEFINE_GET_AR(EFLAG, eflag)
429	      __DEFINE_GET_AR(CSD, csd)
430	      __DEFINE_GET_AR(SSD, ssd)
431	      __DEFINE_GET_REG(AR_CFLAG, ar27)
432	      __DEFINE_GET_AR(FSR, fsr)
433	      __DEFINE_GET_AR(FIR, fir)
434	      __DEFINE_GET_AR(FDR, fdr)
435	      __DEFINE_GET_AR(CCV, ccv)
436	      __DEFINE_GET_AR(UNAT, unat)
437	      __DEFINE_GET_AR(FPSR, fpsr)
438	      __DEFINE_GET_AR(ITC, itc)
439	      __DEFINE_GET_AR(PFS, pfs)
440	      __DEFINE_GET_AR(LC, lc)
441	      __DEFINE_GET_AR(EC, ec)
442
443	      __DEFINE_GET_CR(DCR, dcr)
444	      __DEFINE_GET_CR(ITM, itm)
445	      __DEFINE_GET_CR(IVA, iva)
446	      __DEFINE_GET_CR(PTA, pta)
447	      __DEFINE_GET_CR(IPSR, ipsr)
448	      __DEFINE_GET_CR(ISR, isr)
449	      __DEFINE_GET_CR(IIP, iip)
450	      __DEFINE_GET_CR(IFA, ifa)
451	      __DEFINE_GET_CR(ITIR, itir)
452	      __DEFINE_GET_CR(IIPA, iipa)
453	      __DEFINE_GET_CR(IFS, ifs)
454	      __DEFINE_GET_CR(IIM, iim)
455	      __DEFINE_GET_CR(IHA, iha)
456	      __DEFINE_GET_CR(LID, lid)
457	      __DEFINE_GET_CR(IVR, ivr)
458	      __DEFINE_GET_CR(TPR, tpr)
459	      __DEFINE_GET_CR(EOI, eoi)
460	      __DEFINE_GET_CR(IRR0, irr0)
461	      __DEFINE_GET_CR(IRR1, irr1)
462	      __DEFINE_GET_CR(IRR2, irr2)
463	      __DEFINE_GET_CR(IRR3, irr3)
464	      __DEFINE_GET_CR(ITV, itv)
465	      __DEFINE_GET_CR(PMV, pmv)
466	      __DEFINE_GET_CR(CMCV, cmcv)
467	      __DEFINE_GET_CR(LRR0, lrr0)
468	      __DEFINE_GET_CR(LRR1, lrr1)
469
470	      "mov r8 = -1\n"	/* unsupported case */
471	);
472
473extern void ia64_native_setreg_func(int regnum, unsigned long val);
474asm(".global ia64_native_setreg_func\n");
475#define __DEFINE_SET_REG(id, reg)			\
476	"mov r2 = " __stringify(_IA64_REG_ ## id) "\n"	\
477	";;\n"						\
478	"cmp.eq p6, p0 = r2, r9\n"			\
479	";;\n"						\
480	"(p6) mov " #reg " = r8\n"			\
481	"(p6) br.cond.sptk.many b6\n"			\
482	";;\n"
483#define __DEFINE_SET_AR(id, reg)	__DEFINE_SET_REG(AR_ ## id, ar.reg)
484#define __DEFINE_SET_CR(id, reg)	__DEFINE_SET_REG(CR_ ## id, cr.reg)
485__DEFINE_FUNC(setreg,
486	      "mov r2 = " __stringify(_IA64_REG_PSR_L) "\n"
487	      ";;\n"
488	      "cmp.eq p6, p0 = r2, r9\n"
489	      ";;\n"
490	      "(p6) mov psr.l = r8\n"
491#ifdef HAVE_SERIALIZE_DIRECTIVE
492	      ".serialize.data\n"
493#endif
494	      "(p6) br.cond.sptk.many b6\n"
495	      __DEFINE_SET_REG(GP, gp)
496	      __DEFINE_SET_REG(SP, sp)
497
498	      __DEFINE_SET_REG(AR_KR0, ar0)
499	      __DEFINE_SET_REG(AR_KR1, ar1)
500	      __DEFINE_SET_REG(AR_KR2, ar2)
501	      __DEFINE_SET_REG(AR_KR3, ar3)
502	      __DEFINE_SET_REG(AR_KR4, ar4)
503	      __DEFINE_SET_REG(AR_KR5, ar5)
504	      __DEFINE_SET_REG(AR_KR6, ar6)
505	      __DEFINE_SET_REG(AR_KR7, ar7)
506	      __DEFINE_SET_AR(RSC, rsc)
507	      __DEFINE_SET_AR(BSP, bsp)
508	      __DEFINE_SET_AR(BSPSTORE, bspstore)
509	      __DEFINE_SET_AR(RNAT, rnat)
510	      __DEFINE_SET_AR(FCR, fcr)
511	      __DEFINE_SET_AR(EFLAG, eflag)
512	      __DEFINE_SET_AR(CSD, csd)
513	      __DEFINE_SET_AR(SSD, ssd)
514	      __DEFINE_SET_REG(AR_CFLAG, ar27)
515	      __DEFINE_SET_AR(FSR, fsr)
516	      __DEFINE_SET_AR(FIR, fir)
517	      __DEFINE_SET_AR(FDR, fdr)
518	      __DEFINE_SET_AR(CCV, ccv)
519	      __DEFINE_SET_AR(UNAT, unat)
520	      __DEFINE_SET_AR(FPSR, fpsr)
521	      __DEFINE_SET_AR(ITC, itc)
522	      __DEFINE_SET_AR(PFS, pfs)
523	      __DEFINE_SET_AR(LC, lc)
524	      __DEFINE_SET_AR(EC, ec)
525
526	      __DEFINE_SET_CR(DCR, dcr)
527	      __DEFINE_SET_CR(ITM, itm)
528	      __DEFINE_SET_CR(IVA, iva)
529	      __DEFINE_SET_CR(PTA, pta)
530	      __DEFINE_SET_CR(IPSR, ipsr)
531	      __DEFINE_SET_CR(ISR, isr)
532	      __DEFINE_SET_CR(IIP, iip)
533	      __DEFINE_SET_CR(IFA, ifa)
534	      __DEFINE_SET_CR(ITIR, itir)
535	      __DEFINE_SET_CR(IIPA, iipa)
536	      __DEFINE_SET_CR(IFS, ifs)
537	      __DEFINE_SET_CR(IIM, iim)
538	      __DEFINE_SET_CR(IHA, iha)
539	      __DEFINE_SET_CR(LID, lid)
540	      __DEFINE_SET_CR(IVR, ivr)
541	      __DEFINE_SET_CR(TPR, tpr)
542	      __DEFINE_SET_CR(EOI, eoi)
543	      __DEFINE_SET_CR(IRR0, irr0)
544	      __DEFINE_SET_CR(IRR1, irr1)
545	      __DEFINE_SET_CR(IRR2, irr2)
546	      __DEFINE_SET_CR(IRR3, irr3)
547	      __DEFINE_SET_CR(ITV, itv)
548	      __DEFINE_SET_CR(PMV, pmv)
549	      __DEFINE_SET_CR(CMCV, cmcv)
550	      __DEFINE_SET_CR(LRR0, lrr0)
551	      __DEFINE_SET_CR(LRR1, lrr1)
552	);
553#endif
554
555struct pv_cpu_ops pv_cpu_ops = {
556	.fc		= ia64_native_fc_func,
557	.thash		= ia64_native_thash_func,
558	.get_cpuid	= ia64_native_get_cpuid_func,
559	.get_pmd	= ia64_native_get_pmd_func,
560	.ptcga		= ia64_native_ptcga_func,
561	.get_rr		= ia64_native_get_rr_func,
562	.set_rr		= ia64_native_set_rr_func,
563	.set_rr0_to_rr4	= ia64_native_set_rr0_to_rr4_func,
564	.ssm_i		= ia64_native_ssm_i_func,
565	.getreg		= ia64_native_getreg_func,
566	.setreg		= ia64_native_setreg_func,
567	.rsm_i		= ia64_native_rsm_i_func,
568	.get_psr_i	= ia64_native_get_psr_i_func,
569	.intrin_local_irq_restore
570			= ia64_native_intrin_local_irq_restore_func,
571};
572EXPORT_SYMBOL(pv_cpu_ops);
573
574/******************************************************************************
575 * replacement of hand written assembly codes.
576 */
577
578void
579paravirt_cpu_asm_init(const struct pv_cpu_asm_switch *cpu_asm_switch)
580{
581	extern unsigned long paravirt_switch_to_targ;
582	extern unsigned long paravirt_leave_syscall_targ;
583	extern unsigned long paravirt_work_processed_syscall_targ;
584	extern unsigned long paravirt_leave_kernel_targ;
585
586	paravirt_switch_to_targ = cpu_asm_switch->switch_to;
587	paravirt_leave_syscall_targ = cpu_asm_switch->leave_syscall;
588	paravirt_work_processed_syscall_targ =
589		cpu_asm_switch->work_processed_syscall;
590	paravirt_leave_kernel_targ = cpu_asm_switch->leave_kernel;
591}
592
593/***************************************************************************
594 * pv_iosapic_ops
595 * iosapic read/write hooks.
596 */
597
598static unsigned int
599ia64_native_iosapic_read(char __iomem *iosapic, unsigned int reg)
600{
601	return __ia64_native_iosapic_read(iosapic, reg);
602}
603
604static void
605ia64_native_iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val)
606{
607	__ia64_native_iosapic_write(iosapic, reg, val);
608}
609
610struct pv_iosapic_ops pv_iosapic_ops = {
611	.pcat_compat_init = ia64_native_iosapic_pcat_compat_init,
612	.__get_irq_chip = ia64_native_iosapic_get_irq_chip,
613
614	.__read = ia64_native_iosapic_read,
615	.__write = ia64_native_iosapic_write,
616};
617
618/***************************************************************************
619 * pv_irq_ops
620 * irq operations
621 */
622
623struct pv_irq_ops pv_irq_ops = {
624	.register_ipi = ia64_native_register_ipi,
625
626	.assign_irq_vector = ia64_native_assign_irq_vector,
627	.free_irq_vector = ia64_native_free_irq_vector,
628	.register_percpu_irq = ia64_native_register_percpu_irq,
629
630	.resend_irq = ia64_native_resend_irq,
631};
632
633/***************************************************************************
634 * pv_time_ops
635 * time operations
636 */
637struct static_key paravirt_steal_enabled;
638struct static_key paravirt_steal_rq_enabled;
639
640static int
641ia64_native_do_steal_accounting(unsigned long *new_itm)
642{
643	return 0;
644}
645
646struct pv_time_ops pv_time_ops = {
647	.do_steal_accounting = ia64_native_do_steal_accounting,
648	.sched_clock = ia64_native_sched_clock,
649};
650
651/***************************************************************************
652 * binary pacthing
653 * pv_init_ops.patch_bundle
654 */
655
656#ifdef ASM_SUPPORTED
657#define IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg)	\
658	__DEFINE_FUNC(get_ ## name,			\
659		      ";;\n"				\
660		      "mov r8 = " #reg "\n"		\
661		      ";;\n")
662
663#define IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg)	\
664	__DEFINE_FUNC(set_ ## name,			\
665		      ";;\n"				\
666		      "mov " #reg " = r8\n"		\
667		      ";;\n")
668
669#define IA64_NATIVE_PATCH_DEFINE_REG(name, reg)		\
670	IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg);	\
671	IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg)	\
672
673#define IA64_NATIVE_PATCH_DEFINE_AR(name, reg)			\
674	IA64_NATIVE_PATCH_DEFINE_REG(ar_ ## name, ar.reg)
675
676#define IA64_NATIVE_PATCH_DEFINE_CR(name, reg)			\
677	IA64_NATIVE_PATCH_DEFINE_REG(cr_ ## name, cr.reg)
678
679
680IA64_NATIVE_PATCH_DEFINE_GET_REG(psr, psr);
681IA64_NATIVE_PATCH_DEFINE_GET_REG(tp, tp);
682
683/* IA64_NATIVE_PATCH_DEFINE_SET_REG(psr_l, psr.l); */
684__DEFINE_FUNC(set_psr_l,
685	      ";;\n"
686	      "mov psr.l = r8\n"
687#ifdef HAVE_SERIALIZE_DIRECTIVE
688	      ".serialize.data\n"
689#endif
690	      ";;\n");
691
692IA64_NATIVE_PATCH_DEFINE_REG(gp, gp);
693IA64_NATIVE_PATCH_DEFINE_REG(sp, sp);
694
695IA64_NATIVE_PATCH_DEFINE_REG(kr0, ar0);
696IA64_NATIVE_PATCH_DEFINE_REG(kr1, ar1);
697IA64_NATIVE_PATCH_DEFINE_REG(kr2, ar2);
698IA64_NATIVE_PATCH_DEFINE_REG(kr3, ar3);
699IA64_NATIVE_PATCH_DEFINE_REG(kr4, ar4);
700IA64_NATIVE_PATCH_DEFINE_REG(kr5, ar5);
701IA64_NATIVE_PATCH_DEFINE_REG(kr6, ar6);
702IA64_NATIVE_PATCH_DEFINE_REG(kr7, ar7);
703
704IA64_NATIVE_PATCH_DEFINE_AR(rsc, rsc);
705IA64_NATIVE_PATCH_DEFINE_AR(bsp, bsp);
706IA64_NATIVE_PATCH_DEFINE_AR(bspstore, bspstore);
707IA64_NATIVE_PATCH_DEFINE_AR(rnat, rnat);
708IA64_NATIVE_PATCH_DEFINE_AR(fcr, fcr);
709IA64_NATIVE_PATCH_DEFINE_AR(eflag, eflag);
710IA64_NATIVE_PATCH_DEFINE_AR(csd, csd);
711IA64_NATIVE_PATCH_DEFINE_AR(ssd, ssd);
712IA64_NATIVE_PATCH_DEFINE_REG(ar27, ar27);
713IA64_NATIVE_PATCH_DEFINE_AR(fsr, fsr);
714IA64_NATIVE_PATCH_DEFINE_AR(fir, fir);
715IA64_NATIVE_PATCH_DEFINE_AR(fdr, fdr);
716IA64_NATIVE_PATCH_DEFINE_AR(ccv, ccv);
717IA64_NATIVE_PATCH_DEFINE_AR(unat, unat);
718IA64_NATIVE_PATCH_DEFINE_AR(fpsr, fpsr);
719IA64_NATIVE_PATCH_DEFINE_AR(itc, itc);
720IA64_NATIVE_PATCH_DEFINE_AR(pfs, pfs);
721IA64_NATIVE_PATCH_DEFINE_AR(lc, lc);
722IA64_NATIVE_PATCH_DEFINE_AR(ec, ec);
723
724IA64_NATIVE_PATCH_DEFINE_CR(dcr, dcr);
725IA64_NATIVE_PATCH_DEFINE_CR(itm, itm);
726IA64_NATIVE_PATCH_DEFINE_CR(iva, iva);
727IA64_NATIVE_PATCH_DEFINE_CR(pta, pta);
728IA64_NATIVE_PATCH_DEFINE_CR(ipsr, ipsr);
729IA64_NATIVE_PATCH_DEFINE_CR(isr, isr);
730IA64_NATIVE_PATCH_DEFINE_CR(iip, iip);
731IA64_NATIVE_PATCH_DEFINE_CR(ifa, ifa);
732IA64_NATIVE_PATCH_DEFINE_CR(itir, itir);
733IA64_NATIVE_PATCH_DEFINE_CR(iipa, iipa);
734IA64_NATIVE_PATCH_DEFINE_CR(ifs, ifs);
735IA64_NATIVE_PATCH_DEFINE_CR(iim, iim);
736IA64_NATIVE_PATCH_DEFINE_CR(iha, iha);
737IA64_NATIVE_PATCH_DEFINE_CR(lid, lid);
738IA64_NATIVE_PATCH_DEFINE_CR(ivr, ivr);
739IA64_NATIVE_PATCH_DEFINE_CR(tpr, tpr);
740IA64_NATIVE_PATCH_DEFINE_CR(eoi, eoi);
741IA64_NATIVE_PATCH_DEFINE_CR(irr0, irr0);
742IA64_NATIVE_PATCH_DEFINE_CR(irr1, irr1);
743IA64_NATIVE_PATCH_DEFINE_CR(irr2, irr2);
744IA64_NATIVE_PATCH_DEFINE_CR(irr3, irr3);
745IA64_NATIVE_PATCH_DEFINE_CR(itv, itv);
746IA64_NATIVE_PATCH_DEFINE_CR(pmv, pmv);
747IA64_NATIVE_PATCH_DEFINE_CR(cmcv, cmcv);
748IA64_NATIVE_PATCH_DEFINE_CR(lrr0, lrr0);
749IA64_NATIVE_PATCH_DEFINE_CR(lrr1, lrr1);
750
751static const struct paravirt_patch_bundle_elem ia64_native_patch_bundle_elems[]
752__initdata_or_module =
753{
754#define IA64_NATIVE_PATCH_BUNDLE_ELEM(name, type)		\
755	{							\
756		(void*)ia64_native_ ## name ## _direct_start,	\
757		(void*)ia64_native_ ## name ## _direct_end,	\
758		PARAVIRT_PATCH_TYPE_ ## type,			\
759	}
760
761	IA64_NATIVE_PATCH_BUNDLE_ELEM(fc, FC),
762	IA64_NATIVE_PATCH_BUNDLE_ELEM(thash, THASH),
763	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_cpuid, GET_CPUID),
764	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_pmd, GET_PMD),
765	IA64_NATIVE_PATCH_BUNDLE_ELEM(ptcga, PTCGA),
766	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_rr, GET_RR),
767	IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr, SET_RR),
768	IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr0_to_rr4, SET_RR0_TO_RR4),
769	IA64_NATIVE_PATCH_BUNDLE_ELEM(ssm_i, SSM_I),
770	IA64_NATIVE_PATCH_BUNDLE_ELEM(rsm_i, RSM_I),
771	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_psr_i, GET_PSR_I),
772	IA64_NATIVE_PATCH_BUNDLE_ELEM(intrin_local_irq_restore,
773				      INTRIN_LOCAL_IRQ_RESTORE),
774
775#define IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg)			\
776	{								\
777		(void*)ia64_native_get_ ## name ## _direct_start,	\
778		(void*)ia64_native_get_ ## name ## _direct_end,		\
779		PARAVIRT_PATCH_TYPE_GETREG + _IA64_REG_ ## reg,		\
780	}
781
782#define IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg)			\
783	{								\
784		(void*)ia64_native_set_ ## name ## _direct_start,	\
785		(void*)ia64_native_set_ ## name ## _direct_end,		\
786		PARAVIRT_PATCH_TYPE_SETREG + _IA64_REG_ ## reg,		\
787	}
788
789#define IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(name, reg)		\
790	IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg),	\
791	IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg)		\
792
793#define IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(name, reg)		\
794	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar_ ## name, AR_ ## reg)
795
796#define IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(name, reg)		\
797	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(cr_ ## name, CR_ ## reg)
798
799	IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(psr, PSR),
800	IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(tp, TP),
801
802	IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(psr_l, PSR_L),
803
804	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(gp, GP),
805	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(sp, SP),
806
807	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr0, AR_KR0),
808	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr1, AR_KR1),
809	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr2, AR_KR2),
810	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr3, AR_KR3),
811	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr4, AR_KR4),
812	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr5, AR_KR5),
813	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr6, AR_KR6),
814	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr7, AR_KR7),
815
816	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rsc, RSC),
817	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bsp, BSP),
818	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bspstore, BSPSTORE),
819	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rnat, RNAT),
820	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fcr, FCR),
821	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(eflag, EFLAG),
822	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(csd, CSD),
823	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ssd, SSD),
824	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar27, AR_CFLAG),
825	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fsr, FSR),
826	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fir, FIR),
827	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fdr, FDR),
828	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ccv, CCV),
829	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(unat, UNAT),
830	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fpsr, FPSR),
831	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(itc, ITC),
832	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(pfs, PFS),
833	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(lc, LC),
834	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ec, EC),
835
836	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(dcr, DCR),
837	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itm, ITM),
838	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iva, IVA),
839	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pta, PTA),
840	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ipsr, IPSR),
841	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(isr, ISR),
842	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iip, IIP),
843	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifa, IFA),
844	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itir, ITIR),
845	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iipa, IIPA),
846	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifs, IFS),
847	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iim, IIM),
848	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iha, IHA),
849	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lid, LID),
850	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ivr, IVR),
851	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(tpr, TPR),
852	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(eoi, EOI),
853	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr0, IRR0),
854	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr1, IRR1),
855	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr2, IRR2),
856	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr3, IRR3),
857	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itv, ITV),
858	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pmv, PMV),
859	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(cmcv, CMCV),
860	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr0, LRR0),
861	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr1, LRR1),
862};
863
864unsigned long __init_or_module
865ia64_native_patch_bundle(void *sbundle, void *ebundle, unsigned long type)
866{
867	const unsigned long nelems = sizeof(ia64_native_patch_bundle_elems) /
868		sizeof(ia64_native_patch_bundle_elems[0]);
869
870	return __paravirt_patch_apply_bundle(sbundle, ebundle, type,
871					      ia64_native_patch_bundle_elems,
872					      nelems, NULL);
873}
874#endif /* ASM_SUPPOTED */
875
876extern const char ia64_native_switch_to[];
877extern const char ia64_native_leave_syscall[];
878extern const char ia64_native_work_processed_syscall[];
879extern const char ia64_native_leave_kernel[];
880
881const struct paravirt_patch_branch_target ia64_native_branch_target[]
882__initconst = {
883#define PARAVIRT_BR_TARGET(name, type)			\
884	{						\
885		ia64_native_ ## name,			\
886		PARAVIRT_PATCH_TYPE_BR_ ## type,	\
887	}
888	PARAVIRT_BR_TARGET(switch_to, SWITCH_TO),
889	PARAVIRT_BR_TARGET(leave_syscall, LEAVE_SYSCALL),
890	PARAVIRT_BR_TARGET(work_processed_syscall, WORK_PROCESSED_SYSCALL),
891	PARAVIRT_BR_TARGET(leave_kernel, LEAVE_KERNEL),
892};
893
894static void __init
895ia64_native_patch_branch(unsigned long tag, unsigned long type)
896{
897	const unsigned long nelem =
898		sizeof(ia64_native_branch_target) /
899		sizeof(ia64_native_branch_target[0]);
900	__paravirt_patch_apply_branch(tag, type,
901				      ia64_native_branch_target, nelem);
902}
903