1/******************************************************************************
2 * linux/arch/ia64/xen/paravirt_patch.c
3 *
4 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
5 *                    VA Linux Systems Japan K.K.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20 *
21 */
22
23#include <linux/init.h>
24#include <asm/intrinsics.h>
25#include <asm/kprobes.h>
26#include <asm/paravirt.h>
27#include <asm/paravirt_patch.h>
28
29typedef union ia64_inst {
30        struct {
31		unsigned long long qp : 6;
32		unsigned long long : 31;
33		unsigned long long opcode : 4;
34		unsigned long long reserved : 23;
35        } generic;
36        unsigned long long l;
37} ia64_inst_t;
38
39/*
40 * flush_icache_range() can't be used here.
41 * we are here before cpu_init() which initializes
42 * ia64_i_cache_stride_shift. flush_icache_range() uses it.
43 */
44void __init_or_module
45paravirt_flush_i_cache_range(const void *instr, unsigned long size)
46{
47	extern void paravirt_fc_i(const void *addr);
48	unsigned long i;
49
50	for (i = 0; i < size; i += sizeof(bundle_t))
51		paravirt_fc_i(instr + i);
52}
53
54bundle_t* __init_or_module
55paravirt_get_bundle(unsigned long tag)
56{
57	return (bundle_t *)(tag & ~3UL);
58}
59
60unsigned long __init_or_module
61paravirt_get_slot(unsigned long tag)
62{
63	return tag & 3UL;
64}
65
66unsigned long __init_or_module
67paravirt_get_num_inst(unsigned long stag, unsigned long etag)
68{
69	bundle_t *sbundle = paravirt_get_bundle(stag);
70	unsigned long sslot = paravirt_get_slot(stag);
71	bundle_t *ebundle = paravirt_get_bundle(etag);
72	unsigned long eslot = paravirt_get_slot(etag);
73
74	return (ebundle - sbundle) * 3 + eslot - sslot + 1;
75}
76
77unsigned long __init_or_module
78paravirt_get_next_tag(unsigned long tag)
79{
80	unsigned long slot = paravirt_get_slot(tag);
81
82	switch (slot) {
83	case 0:
84	case 1:
85		return tag + 1;
86	case 2: {
87		bundle_t *bundle = paravirt_get_bundle(tag);
88		return (unsigned long)(bundle + 1);
89	}
90	default:
91		BUG();
92	}
93	/* NOTREACHED */
94}
95
96ia64_inst_t __init_or_module
97paravirt_read_slot0(const bundle_t *bundle)
98{
99	ia64_inst_t inst;
100	inst.l = bundle->quad0.slot0;
101	return inst;
102}
103
104ia64_inst_t __init_or_module
105paravirt_read_slot1(const bundle_t *bundle)
106{
107	ia64_inst_t inst;
108	inst.l = bundle->quad0.slot1_p0 |
109		((unsigned long long)bundle->quad1.slot1_p1 << 18UL);
110	return inst;
111}
112
113ia64_inst_t __init_or_module
114paravirt_read_slot2(const bundle_t *bundle)
115{
116	ia64_inst_t inst;
117	inst.l = bundle->quad1.slot2;
118	return inst;
119}
120
121ia64_inst_t __init_or_module
122paravirt_read_inst(unsigned long tag)
123{
124	bundle_t *bundle = paravirt_get_bundle(tag);
125	unsigned long slot = paravirt_get_slot(tag);
126
127	switch (slot) {
128	case 0:
129		return paravirt_read_slot0(bundle);
130	case 1:
131		return paravirt_read_slot1(bundle);
132	case 2:
133		return paravirt_read_slot2(bundle);
134	default:
135		BUG();
136	}
137	/* NOTREACHED */
138}
139
140void __init_or_module
141paravirt_write_slot0(bundle_t *bundle, ia64_inst_t inst)
142{
143	bundle->quad0.slot0 = inst.l;
144}
145
146void __init_or_module
147paravirt_write_slot1(bundle_t *bundle, ia64_inst_t inst)
148{
149	bundle->quad0.slot1_p0 = inst.l;
150	bundle->quad1.slot1_p1 = inst.l >> 18UL;
151}
152
153void __init_or_module
154paravirt_write_slot2(bundle_t *bundle, ia64_inst_t inst)
155{
156	bundle->quad1.slot2 = inst.l;
157}
158
159void __init_or_module
160paravirt_write_inst(unsigned long tag, ia64_inst_t inst)
161{
162	bundle_t *bundle = paravirt_get_bundle(tag);
163	unsigned long slot = paravirt_get_slot(tag);
164
165	switch (slot) {
166	case 0:
167		paravirt_write_slot0(bundle, inst);
168		break;
169	case 1:
170		paravirt_write_slot1(bundle, inst);
171		break;
172	case 2:
173		paravirt_write_slot2(bundle, inst);
174		break;
175	default:
176		BUG();
177		break;
178	}
179	paravirt_flush_i_cache_range(bundle, sizeof(*bundle));
180}
181
182/* for debug */
183void
184paravirt_print_bundle(const bundle_t *bundle)
185{
186	const unsigned long *quad = (const unsigned long *)bundle;
187	ia64_inst_t slot0 = paravirt_read_slot0(bundle);
188	ia64_inst_t slot1 = paravirt_read_slot1(bundle);
189	ia64_inst_t slot2 = paravirt_read_slot2(bundle);
190
191	printk(KERN_DEBUG
192	       "bundle 0x%p 0x%016lx 0x%016lx\n", bundle, quad[0], quad[1]);
193	printk(KERN_DEBUG
194	       "bundle template 0x%x\n",
195	       bundle->quad0.template);
196	printk(KERN_DEBUG
197	       "slot0 0x%lx slot1_p0 0x%lx slot1_p1 0x%lx slot2 0x%lx\n",
198	       (unsigned long)bundle->quad0.slot0,
199	       (unsigned long)bundle->quad0.slot1_p0,
200	       (unsigned long)bundle->quad1.slot1_p1,
201	       (unsigned long)bundle->quad1.slot2);
202	printk(KERN_DEBUG
203	       "slot0 0x%016llx slot1 0x%016llx slot2 0x%016llx\n",
204	       slot0.l, slot1.l, slot2.l);
205}
206
207static int noreplace_paravirt __init_or_module = 0;
208
209static int __init setup_noreplace_paravirt(char *str)
210{
211	noreplace_paravirt = 1;
212	return 1;
213}
214__setup("noreplace-paravirt", setup_noreplace_paravirt);
215
216#ifdef ASM_SUPPORTED
217static void __init_or_module
218fill_nop_bundle(void *sbundle, void *ebundle)
219{
220	extern const char paravirt_nop_bundle[];
221	extern const unsigned long paravirt_nop_bundle_size;
222
223	void *bundle = sbundle;
224
225	BUG_ON((((unsigned long)sbundle) % sizeof(bundle_t)) != 0);
226	BUG_ON((((unsigned long)ebundle) % sizeof(bundle_t)) != 0);
227
228	while (bundle < ebundle) {
229		memcpy(bundle, paravirt_nop_bundle, paravirt_nop_bundle_size);
230
231		bundle += paravirt_nop_bundle_size;
232	}
233}
234
235/* helper function */
236unsigned long __init_or_module
237__paravirt_patch_apply_bundle(void *sbundle, void *ebundle, unsigned long type,
238			      const struct paravirt_patch_bundle_elem *elems,
239			      unsigned long nelems,
240			      const struct paravirt_patch_bundle_elem **found)
241{
242	unsigned long used = 0;
243	unsigned long i;
244
245	BUG_ON((((unsigned long)sbundle) % sizeof(bundle_t)) != 0);
246	BUG_ON((((unsigned long)ebundle) % sizeof(bundle_t)) != 0);
247
248	found = NULL;
249	for (i = 0; i < nelems; i++) {
250		const struct paravirt_patch_bundle_elem *p = &elems[i];
251		if (p->type == type) {
252			unsigned long need = p->ebundle - p->sbundle;
253			unsigned long room = ebundle - sbundle;
254
255			if (found != NULL)
256				*found = p;
257
258			if (room < need) {
259				/* no room to replace. skip it */
260				printk(KERN_DEBUG
261				       "the space is too small to put "
262				       "bundles. type %ld need %ld room %ld\n",
263				       type, need, room);
264				break;
265			}
266
267			used = need;
268			memcpy(sbundle, p->sbundle, used);
269			break;
270		}
271	}
272
273	return used;
274}
275
276void __init_or_module
277paravirt_patch_apply_bundle(const struct paravirt_patch_site_bundle *start,
278			    const struct paravirt_patch_site_bundle *end)
279{
280	const struct paravirt_patch_site_bundle *p;
281
282	if (noreplace_paravirt)
283		return;
284	if (pv_init_ops.patch_bundle == NULL)
285		return;
286
287	for (p = start; p < end; p++) {
288		unsigned long used;
289
290		used = (*pv_init_ops.patch_bundle)(p->sbundle, p->ebundle,
291						   p->type);
292		if (used == 0)
293			continue;
294
295		fill_nop_bundle(p->sbundle + used, p->ebundle);
296		paravirt_flush_i_cache_range(p->sbundle,
297					     p->ebundle - p->sbundle);
298	}
299	ia64_sync_i();
300	ia64_srlz_i();
301}
302
303/*
304 * nop.i, nop.m, nop.f instruction are same format.
305 * but nop.b has differennt format.
306 * This doesn't support nop.b for now.
307 */
308static void __init_or_module
309fill_nop_inst(unsigned long stag, unsigned long etag)
310{
311	extern const bundle_t paravirt_nop_mfi_inst_bundle[];
312	unsigned long tag;
313	const ia64_inst_t nop_inst =
314		paravirt_read_slot0(paravirt_nop_mfi_inst_bundle);
315
316	for (tag = stag; tag < etag; tag = paravirt_get_next_tag(tag))
317		paravirt_write_inst(tag, nop_inst);
318}
319
320void __init_or_module
321paravirt_patch_apply_inst(const struct paravirt_patch_site_inst *start,
322			  const struct paravirt_patch_site_inst *end)
323{
324	const struct paravirt_patch_site_inst *p;
325
326	if (noreplace_paravirt)
327		return;
328	if (pv_init_ops.patch_inst == NULL)
329		return;
330
331	for (p = start; p < end; p++) {
332		unsigned long tag;
333		bundle_t *sbundle;
334		bundle_t *ebundle;
335
336		tag = (*pv_init_ops.patch_inst)(p->stag, p->etag, p->type);
337		if (tag == p->stag)
338			continue;
339
340		fill_nop_inst(tag, p->etag);
341		sbundle = paravirt_get_bundle(p->stag);
342		ebundle = paravirt_get_bundle(p->etag) + 1;
343		paravirt_flush_i_cache_range(sbundle, (ebundle - sbundle) *
344					     sizeof(bundle_t));
345	}
346	ia64_sync_i();
347	ia64_srlz_i();
348}
349#endif /* ASM_SUPPOTED */
350
351/* brl.cond.sptk.many <target64> X3 */
352typedef union inst_x3_op {
353	ia64_inst_t inst;
354	struct {
355		unsigned long qp: 6;
356		unsigned long btyp: 3;
357		unsigned long unused: 3;
358		unsigned long p: 1;
359		unsigned long imm20b: 20;
360		unsigned long wh: 2;
361		unsigned long d: 1;
362		unsigned long i: 1;
363		unsigned long opcode: 4;
364	};
365	unsigned long l;
366} inst_x3_op_t;
367
368typedef union inst_x3_imm {
369	ia64_inst_t inst;
370	struct {
371		unsigned long unused: 2;
372		unsigned long imm39: 39;
373	};
374	unsigned long l;
375} inst_x3_imm_t;
376
377void __init_or_module
378paravirt_patch_reloc_brl(unsigned long tag, const void *target)
379{
380	unsigned long tag_op = paravirt_get_next_tag(tag);
381	unsigned long tag_imm = tag;
382	bundle_t *bundle = paravirt_get_bundle(tag);
383
384	ia64_inst_t inst_op = paravirt_read_inst(tag_op);
385	ia64_inst_t inst_imm = paravirt_read_inst(tag_imm);
386
387	inst_x3_op_t inst_x3_op = { .l = inst_op.l };
388	inst_x3_imm_t inst_x3_imm = { .l = inst_imm.l };
389
390	unsigned long imm60 =
391		((unsigned long)target - (unsigned long)bundle) >> 4;
392
393	BUG_ON(paravirt_get_slot(tag) != 1); /* MLX */
394	BUG_ON(((unsigned long)target & (sizeof(bundle_t) - 1)) != 0);
395
396	/* imm60[59] 1bit */
397	inst_x3_op.i = (imm60 >> 59) & 1;
398	/* imm60[19:0] 20bit */
399	inst_x3_op.imm20b = imm60 & ((1UL << 20) - 1);
400	/* imm60[58:20] 39bit */
401	inst_x3_imm.imm39 = (imm60 >> 20) & ((1UL << 39) - 1);
402
403	inst_op.l = inst_x3_op.l;
404	inst_imm.l = inst_x3_imm.l;
405
406	paravirt_write_inst(tag_op, inst_op);
407	paravirt_write_inst(tag_imm, inst_imm);
408}
409
410/* br.cond.sptk.many <target25>	B1 */
411typedef union inst_b1 {
412	ia64_inst_t inst;
413	struct {
414		unsigned long qp: 6;
415		unsigned long btype: 3;
416		unsigned long unused: 3;
417		unsigned long p: 1;
418		unsigned long imm20b: 20;
419		unsigned long wh: 2;
420		unsigned long d: 1;
421		unsigned long s: 1;
422		unsigned long opcode: 4;
423	};
424	unsigned long l;
425} inst_b1_t;
426
427void __init
428paravirt_patch_reloc_br(unsigned long tag, const void *target)
429{
430	bundle_t *bundle = paravirt_get_bundle(tag);
431	ia64_inst_t inst = paravirt_read_inst(tag);
432	unsigned long target25 = (unsigned long)target - (unsigned long)bundle;
433	inst_b1_t inst_b1;
434
435	BUG_ON(((unsigned long)target & (sizeof(bundle_t) - 1)) != 0);
436
437	inst_b1.l = inst.l;
438	if (target25 & (1UL << 63))
439		inst_b1.s = 1;
440	else
441		inst_b1.s = 0;
442
443	inst_b1.imm20b = target25 >> 4;
444	inst.l = inst_b1.l;
445
446	paravirt_write_inst(tag, inst);
447}
448
449void __init
450__paravirt_patch_apply_branch(
451	unsigned long tag, unsigned long type,
452	const struct paravirt_patch_branch_target *entries,
453	unsigned int nr_entries)
454{
455	unsigned int i;
456	for (i = 0; i < nr_entries; i++) {
457		if (entries[i].type == type) {
458			paravirt_patch_reloc_br(tag, entries[i].entry);
459			break;
460		}
461	}
462}
463
464static void __init
465paravirt_patch_apply_branch(const struct paravirt_patch_site_branch *start,
466			    const struct paravirt_patch_site_branch *end)
467{
468	const struct paravirt_patch_site_branch *p;
469
470	if (noreplace_paravirt)
471		return;
472	if (pv_init_ops.patch_branch == NULL)
473		return;
474
475	for (p = start; p < end; p++)
476		(*pv_init_ops.patch_branch)(p->tag, p->type);
477
478	ia64_sync_i();
479	ia64_srlz_i();
480}
481
482void __init
483paravirt_patch_apply(void)
484{
485	extern const char __start_paravirt_bundles[];
486	extern const char __stop_paravirt_bundles[];
487	extern const char __start_paravirt_insts[];
488	extern const char __stop_paravirt_insts[];
489	extern const char __start_paravirt_branches[];
490	extern const char __stop_paravirt_branches[];
491
492	paravirt_patch_apply_bundle((const struct paravirt_patch_site_bundle *)
493				    __start_paravirt_bundles,
494				    (const struct paravirt_patch_site_bundle *)
495				    __stop_paravirt_bundles);
496	paravirt_patch_apply_inst((const struct paravirt_patch_site_inst *)
497				  __start_paravirt_insts,
498				  (const struct paravirt_patch_site_inst *)
499				  __stop_paravirt_insts);
500	paravirt_patch_apply_branch((const struct paravirt_patch_site_branch *)
501				    __start_paravirt_branches,
502				    (const struct paravirt_patch_site_branch *)
503				    __stop_paravirt_branches);
504}
505
506/*
507 * Local variables:
508 * mode: C
509 * c-set-style: "linux"
510 * c-basic-offset: 8
511 * tab-width: 8
512 * indent-tabs-mode: t
513 * End:
514 */
515