1/*
2 *  Copyright (C) 2014 ARM Limited
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#include <linux/cpu.h>
10#include <linux/init.h>
11#include <linux/list.h>
12#include <linux/perf_event.h>
13#include <linux/sched.h>
14#include <linux/slab.h>
15#include <linux/sysctl.h>
16
17#include <asm/insn.h>
18#include <asm/opcodes.h>
19#include <asm/system_misc.h>
20#include <asm/traps.h>
21#include <asm/uaccess.h>
22#include <asm/cpufeature.h>
23
24#define CREATE_TRACE_POINTS
25#include "trace-events-emulation.h"
26
27/*
28 * The runtime support for deprecated instruction support can be in one of
29 * following three states -
30 *
31 * 0 = undef
32 * 1 = emulate (software emulation)
33 * 2 = hw (supported in hardware)
34 */
35enum insn_emulation_mode {
36	INSN_UNDEF,
37	INSN_EMULATE,
38	INSN_HW,
39};
40
41enum legacy_insn_status {
42	INSN_DEPRECATED,
43	INSN_OBSOLETE,
44};
45
46struct insn_emulation_ops {
47	const char		*name;
48	enum legacy_insn_status	status;
49	struct undef_hook	*hooks;
50	int			(*set_hw_mode)(bool enable);
51};
52
53struct insn_emulation {
54	struct list_head node;
55	struct insn_emulation_ops *ops;
56	int current_mode;
57	int min;
58	int max;
59};
60
61static LIST_HEAD(insn_emulation);
62static int nr_insn_emulated;
63static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
64
65static void register_emulation_hooks(struct insn_emulation_ops *ops)
66{
67	struct undef_hook *hook;
68
69	BUG_ON(!ops->hooks);
70
71	for (hook = ops->hooks; hook->instr_mask; hook++)
72		register_undef_hook(hook);
73
74	pr_notice("Registered %s emulation handler\n", ops->name);
75}
76
77static void remove_emulation_hooks(struct insn_emulation_ops *ops)
78{
79	struct undef_hook *hook;
80
81	BUG_ON(!ops->hooks);
82
83	for (hook = ops->hooks; hook->instr_mask; hook++)
84		unregister_undef_hook(hook);
85
86	pr_notice("Removed %s emulation handler\n", ops->name);
87}
88
89static void enable_insn_hw_mode(void *data)
90{
91	struct insn_emulation *insn = (struct insn_emulation *)data;
92	if (insn->ops->set_hw_mode)
93		insn->ops->set_hw_mode(true);
94}
95
96static void disable_insn_hw_mode(void *data)
97{
98	struct insn_emulation *insn = (struct insn_emulation *)data;
99	if (insn->ops->set_hw_mode)
100		insn->ops->set_hw_mode(false);
101}
102
103/* Run set_hw_mode(mode) on all active CPUs */
104static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
105{
106	if (!insn->ops->set_hw_mode)
107		return -EINVAL;
108	if (enable)
109		on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
110	else
111		on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
112	return 0;
113}
114
115/*
116 * Run set_hw_mode for all insns on a starting CPU.
117 * Returns:
118 *  0 		- If all the hooks ran successfully.
119 * -EINVAL	- At least one hook is not supported by the CPU.
120 */
121static int run_all_insn_set_hw_mode(unsigned long cpu)
122{
123	int rc = 0;
124	unsigned long flags;
125	struct insn_emulation *insn;
126
127	raw_spin_lock_irqsave(&insn_emulation_lock, flags);
128	list_for_each_entry(insn, &insn_emulation, node) {
129		bool enable = (insn->current_mode == INSN_HW);
130		if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
131			pr_warn("CPU[%ld] cannot support the emulation of %s",
132				cpu, insn->ops->name);
133			rc = -EINVAL;
134		}
135	}
136	raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
137	return rc;
138}
139
140static int update_insn_emulation_mode(struct insn_emulation *insn,
141				       enum insn_emulation_mode prev)
142{
143	int ret = 0;
144
145	switch (prev) {
146	case INSN_UNDEF: /* Nothing to be done */
147		break;
148	case INSN_EMULATE:
149		remove_emulation_hooks(insn->ops);
150		break;
151	case INSN_HW:
152		if (!run_all_cpu_set_hw_mode(insn, false))
153			pr_notice("Disabled %s support\n", insn->ops->name);
154		break;
155	}
156
157	switch (insn->current_mode) {
158	case INSN_UNDEF:
159		break;
160	case INSN_EMULATE:
161		register_emulation_hooks(insn->ops);
162		break;
163	case INSN_HW:
164		ret = run_all_cpu_set_hw_mode(insn, true);
165		if (!ret)
166			pr_notice("Enabled %s support\n", insn->ops->name);
167		break;
168	}
169
170	return ret;
171}
172
173static void register_insn_emulation(struct insn_emulation_ops *ops)
174{
175	unsigned long flags;
176	struct insn_emulation *insn;
177
178	insn = kzalloc(sizeof(*insn), GFP_KERNEL);
179	insn->ops = ops;
180	insn->min = INSN_UNDEF;
181
182	switch (ops->status) {
183	case INSN_DEPRECATED:
184		insn->current_mode = INSN_EMULATE;
185		/* Disable the HW mode if it was turned on at early boot time */
186		run_all_cpu_set_hw_mode(insn, false);
187		insn->max = INSN_HW;
188		break;
189	case INSN_OBSOLETE:
190		insn->current_mode = INSN_UNDEF;
191		insn->max = INSN_EMULATE;
192		break;
193	}
194
195	raw_spin_lock_irqsave(&insn_emulation_lock, flags);
196	list_add(&insn->node, &insn_emulation);
197	nr_insn_emulated++;
198	raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
199
200	/* Register any handlers if required */
201	update_insn_emulation_mode(insn, INSN_UNDEF);
202}
203
204static int emulation_proc_handler(struct ctl_table *table, int write,
205				  void __user *buffer, size_t *lenp,
206				  loff_t *ppos)
207{
208	int ret = 0;
209	struct insn_emulation *insn = (struct insn_emulation *) table->data;
210	enum insn_emulation_mode prev_mode = insn->current_mode;
211
212	table->data = &insn->current_mode;
213	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
214
215	if (ret || !write || prev_mode == insn->current_mode)
216		goto ret;
217
218	ret = update_insn_emulation_mode(insn, prev_mode);
219	if (ret) {
220		/* Mode change failed, revert to previous mode. */
221		insn->current_mode = prev_mode;
222		update_insn_emulation_mode(insn, INSN_UNDEF);
223	}
224ret:
225	table->data = insn;
226	return ret;
227}
228
229static struct ctl_table ctl_abi[] = {
230	{
231		.procname = "abi",
232		.mode = 0555,
233	},
234	{ }
235};
236
237static void register_insn_emulation_sysctl(struct ctl_table *table)
238{
239	unsigned long flags;
240	int i = 0;
241	struct insn_emulation *insn;
242	struct ctl_table *insns_sysctl, *sysctl;
243
244	insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1),
245			      GFP_KERNEL);
246
247	raw_spin_lock_irqsave(&insn_emulation_lock, flags);
248	list_for_each_entry(insn, &insn_emulation, node) {
249		sysctl = &insns_sysctl[i];
250
251		sysctl->mode = 0644;
252		sysctl->maxlen = sizeof(int);
253
254		sysctl->procname = insn->ops->name;
255		sysctl->data = insn;
256		sysctl->extra1 = &insn->min;
257		sysctl->extra2 = &insn->max;
258		sysctl->proc_handler = emulation_proc_handler;
259		i++;
260	}
261	raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
262
263	table->child = insns_sysctl;
264	register_sysctl_table(table);
265}
266
267/*
268 *  Implement emulation of the SWP/SWPB instructions using load-exclusive and
269 *  store-exclusive.
270 *
271 *  Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
272 *  Where: Rt  = destination
273 *	   Rt2 = source
274 *	   Rn  = address
275 */
276
277/*
278 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
279 */
280#define __user_swpX_asm(data, addr, res, temp, B)		\
281	__asm__ __volatile__(					\
282	"0:	ldxr"B"		%w2, [%3]\n"			\
283	"1:	stxr"B"		%w0, %w1, [%3]\n"		\
284	"	cbz		%w0, 2f\n"			\
285	"	mov		%w0, %w4\n"			\
286	"	b		3f\n"				\
287	"2:\n"							\
288	"	mov		%w1, %w2\n"			\
289	"3:\n"							\
290	"	.pushsection	 .fixup,\"ax\"\n"		\
291	"	.align		2\n"				\
292	"4:	mov		%w0, %w5\n"			\
293	"	b		3b\n"				\
294	"	.popsection"					\
295	"	.pushsection	 __ex_table,\"a\"\n"		\
296	"	.align		3\n"				\
297	"	.quad		0b, 4b\n"			\
298	"	.quad		1b, 4b\n"			\
299	"	.popsection\n"					\
300	: "=&r" (res), "+r" (data), "=&r" (temp)		\
301	: "r" (addr), "i" (-EAGAIN), "i" (-EFAULT)		\
302	: "memory")
303
304#define __user_swp_asm(data, addr, res, temp) \
305	__user_swpX_asm(data, addr, res, temp, "")
306#define __user_swpb_asm(data, addr, res, temp) \
307	__user_swpX_asm(data, addr, res, temp, "b")
308
309/*
310 * Bit 22 of the instruction encoding distinguishes between
311 * the SWP and SWPB variants (bit set means SWPB).
312 */
313#define TYPE_SWPB (1 << 22)
314
315/*
316 * Set up process info to signal segmentation fault - called on access error.
317 */
318static void set_segfault(struct pt_regs *regs, unsigned long addr)
319{
320	siginfo_t info;
321
322	down_read(&current->mm->mmap_sem);
323	if (find_vma(current->mm, addr) == NULL)
324		info.si_code = SEGV_MAPERR;
325	else
326		info.si_code = SEGV_ACCERR;
327	up_read(&current->mm->mmap_sem);
328
329	info.si_signo = SIGSEGV;
330	info.si_errno = 0;
331	info.si_addr  = (void *) instruction_pointer(regs);
332
333	pr_debug("SWP{B} emulation: access caused memory abort!\n");
334	arm64_notify_die("Illegal memory access", regs, &info, 0);
335}
336
337static int emulate_swpX(unsigned int address, unsigned int *data,
338			unsigned int type)
339{
340	unsigned int res = 0;
341
342	if ((type != TYPE_SWPB) && (address & 0x3)) {
343		/* SWP to unaligned address not permitted */
344		pr_debug("SWP instruction on unaligned pointer!\n");
345		return -EFAULT;
346	}
347
348	while (1) {
349		unsigned long temp;
350
351		if (type == TYPE_SWPB)
352			__user_swpb_asm(*data, address, res, temp);
353		else
354			__user_swp_asm(*data, address, res, temp);
355
356		if (likely(res != -EAGAIN) || signal_pending(current))
357			break;
358
359		cond_resched();
360	}
361
362	return res;
363}
364
365/*
366 * swp_handler logs the id of calling process, dissects the instruction, sanity
367 * checks the memory location, calls emulate_swpX for the actual operation and
368 * deals with fixup/error handling before returning
369 */
370static int swp_handler(struct pt_regs *regs, u32 instr)
371{
372	u32 destreg, data, type, address = 0;
373	int rn, rt2, res = 0;
374
375	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
376
377	type = instr & TYPE_SWPB;
378
379	switch (arm_check_condition(instr, regs->pstate)) {
380	case ARM_OPCODE_CONDTEST_PASS:
381		break;
382	case ARM_OPCODE_CONDTEST_FAIL:
383		/* Condition failed - return to next instruction */
384		goto ret;
385	case ARM_OPCODE_CONDTEST_UNCOND:
386		/* If unconditional encoding - not a SWP, undef */
387		return -EFAULT;
388	default:
389		return -EINVAL;
390	}
391
392	rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
393	rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
394
395	address = (u32)regs->user_regs.regs[rn];
396	data	= (u32)regs->user_regs.regs[rt2];
397	destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
398
399	pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
400		rn, address, destreg,
401		aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
402
403	/* Check access in reasonable access range for both SWP and SWPB */
404	if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) {
405		pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
406			address);
407		goto fault;
408	}
409
410	res = emulate_swpX(address, &data, type);
411	if (res == -EFAULT)
412		goto fault;
413	else if (res == 0)
414		regs->user_regs.regs[destreg] = data;
415
416ret:
417	if (type == TYPE_SWPB)
418		trace_instruction_emulation("swpb", regs->pc);
419	else
420		trace_instruction_emulation("swp", regs->pc);
421
422	pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
423			current->comm, (unsigned long)current->pid, regs->pc);
424
425	regs->pc += 4;
426	return 0;
427
428fault:
429	set_segfault(regs, address);
430
431	return 0;
432}
433
434/*
435 * Only emulate SWP/SWPB executed in ARM state/User mode.
436 * The kernel must be SWP free and SWP{B} does not exist in Thumb.
437 */
438static struct undef_hook swp_hooks[] = {
439	{
440		.instr_mask	= 0x0fb00ff0,
441		.instr_val	= 0x01000090,
442		.pstate_mask	= COMPAT_PSR_MODE_MASK,
443		.pstate_val	= COMPAT_PSR_MODE_USR,
444		.fn		= swp_handler
445	},
446	{ }
447};
448
449static struct insn_emulation_ops swp_ops = {
450	.name = "swp",
451	.status = INSN_OBSOLETE,
452	.hooks = swp_hooks,
453	.set_hw_mode = NULL,
454};
455
456static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
457{
458	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
459
460	switch (arm_check_condition(instr, regs->pstate)) {
461	case ARM_OPCODE_CONDTEST_PASS:
462		break;
463	case ARM_OPCODE_CONDTEST_FAIL:
464		/* Condition failed - return to next instruction */
465		goto ret;
466	case ARM_OPCODE_CONDTEST_UNCOND:
467		/* If unconditional encoding - not a barrier instruction */
468		return -EFAULT;
469	default:
470		return -EINVAL;
471	}
472
473	switch (aarch32_insn_mcr_extract_crm(instr)) {
474	case 10:
475		/*
476		 * dmb - mcr p15, 0, Rt, c7, c10, 5
477		 * dsb - mcr p15, 0, Rt, c7, c10, 4
478		 */
479		if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
480			dmb(sy);
481			trace_instruction_emulation(
482				"mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
483		} else {
484			dsb(sy);
485			trace_instruction_emulation(
486				"mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
487		}
488		break;
489	case 5:
490		/*
491		 * isb - mcr p15, 0, Rt, c7, c5, 4
492		 *
493		 * Taking an exception or returning from one acts as an
494		 * instruction barrier. So no explicit barrier needed here.
495		 */
496		trace_instruction_emulation(
497			"mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
498		break;
499	}
500
501ret:
502	pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
503			current->comm, (unsigned long)current->pid, regs->pc);
504
505	regs->pc += 4;
506	return 0;
507}
508
509static inline void config_sctlr_el1(u32 clear, u32 set)
510{
511	u32 val;
512
513	asm volatile("mrs %0, sctlr_el1" : "=r" (val));
514	val &= ~clear;
515	val |= set;
516	asm volatile("msr sctlr_el1, %0" : : "r" (val));
517}
518
519static int cp15_barrier_set_hw_mode(bool enable)
520{
521	if (enable)
522		config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
523	else
524		config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
525	return 0;
526}
527
528static struct undef_hook cp15_barrier_hooks[] = {
529	{
530		.instr_mask	= 0x0fff0fdf,
531		.instr_val	= 0x0e070f9a,
532		.pstate_mask	= COMPAT_PSR_MODE_MASK,
533		.pstate_val	= COMPAT_PSR_MODE_USR,
534		.fn		= cp15barrier_handler,
535	},
536	{
537		.instr_mask	= 0x0fff0fff,
538		.instr_val	= 0x0e070f95,
539		.pstate_mask	= COMPAT_PSR_MODE_MASK,
540		.pstate_val	= COMPAT_PSR_MODE_USR,
541		.fn		= cp15barrier_handler,
542	},
543	{ }
544};
545
546static struct insn_emulation_ops cp15_barrier_ops = {
547	.name = "cp15_barrier",
548	.status = INSN_DEPRECATED,
549	.hooks = cp15_barrier_hooks,
550	.set_hw_mode = cp15_barrier_set_hw_mode,
551};
552
553static int setend_set_hw_mode(bool enable)
554{
555	if (!cpu_supports_mixed_endian_el0())
556		return -EINVAL;
557
558	if (enable)
559		config_sctlr_el1(SCTLR_EL1_SED, 0);
560	else
561		config_sctlr_el1(0, SCTLR_EL1_SED);
562	return 0;
563}
564
565static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
566{
567	char *insn;
568
569	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
570
571	if (big_endian) {
572		insn = "setend be";
573		regs->pstate |= COMPAT_PSR_E_BIT;
574	} else {
575		insn = "setend le";
576		regs->pstate &= ~COMPAT_PSR_E_BIT;
577	}
578
579	trace_instruction_emulation(insn, regs->pc);
580	pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
581			current->comm, (unsigned long)current->pid, regs->pc);
582
583	return 0;
584}
585
586static int a32_setend_handler(struct pt_regs *regs, u32 instr)
587{
588	int rc = compat_setend_handler(regs, (instr >> 9) & 1);
589	regs->pc += 4;
590	return rc;
591}
592
593static int t16_setend_handler(struct pt_regs *regs, u32 instr)
594{
595	int rc = compat_setend_handler(regs, (instr >> 3) & 1);
596	regs->pc += 2;
597	return rc;
598}
599
600static struct undef_hook setend_hooks[] = {
601	{
602		.instr_mask	= 0xfffffdff,
603		.instr_val	= 0xf1010000,
604		.pstate_mask	= COMPAT_PSR_MODE_MASK,
605		.pstate_val	= COMPAT_PSR_MODE_USR,
606		.fn		= a32_setend_handler,
607	},
608	{
609		/* Thumb mode */
610		.instr_mask	= 0x0000fff7,
611		.instr_val	= 0x0000b650,
612		.pstate_mask	= (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK),
613		.pstate_val	= (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR),
614		.fn		= t16_setend_handler,
615	},
616	{}
617};
618
619static struct insn_emulation_ops setend_ops = {
620	.name = "setend",
621	.status = INSN_DEPRECATED,
622	.hooks = setend_hooks,
623	.set_hw_mode = setend_set_hw_mode,
624};
625
626static int insn_cpu_hotplug_notify(struct notifier_block *b,
627			      unsigned long action, void *hcpu)
628{
629	int rc = 0;
630	if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING)
631		rc = run_all_insn_set_hw_mode((unsigned long)hcpu);
632
633	return notifier_from_errno(rc);
634}
635
636static struct notifier_block insn_cpu_hotplug_notifier = {
637	.notifier_call = insn_cpu_hotplug_notify,
638};
639
640/*
641 * Invoked as late_initcall, since not needed before init spawned.
642 */
643static int __init armv8_deprecated_init(void)
644{
645	if (IS_ENABLED(CONFIG_SWP_EMULATION))
646		register_insn_emulation(&swp_ops);
647
648	if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
649		register_insn_emulation(&cp15_barrier_ops);
650
651	if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
652		if(system_supports_mixed_endian_el0())
653			register_insn_emulation(&setend_ops);
654		else
655			pr_info("setend instruction emulation is not supported on the system");
656	}
657
658	register_cpu_notifier(&insn_cpu_hotplug_notifier);
659	register_insn_emulation_sysctl(ctl_abi);
660
661	return 0;
662}
663
664late_initcall(armv8_deprecated_init);
665