1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 *   This program is free software; you can redistribute it and/or
5 *   modify it under the terms of the GNU General Public License
6 *   as published by the Free Software Foundation, version 2.
7 *
8 *   This program is distributed in the hope that it will be useful, but
9 *   WITHOUT ANY WARRANTY; without even the implied warranty of
10 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 *   NON INFRINGEMENT.  See the GNU General Public License for
12 *   more details.
13 *
14 * TILE SMP support routines.
15 */
16
17#include <linux/smp.h>
18#include <linux/interrupt.h>
19#include <linux/io.h>
20#include <linux/irq.h>
21#include <linux/irq_work.h>
22#include <linux/module.h>
23#include <asm/cacheflush.h>
24#include <asm/homecache.h>
25
26/*
27 * We write to width and height with a single store in head_NN.S,
28 * so make the variable aligned to "long".
29 */
30HV_Topology smp_topology __write_once __aligned(sizeof(long));
31EXPORT_SYMBOL(smp_topology);
32
33#if CHIP_HAS_IPI()
34static unsigned long __iomem *ipi_mappings[NR_CPUS];
35#endif
36
37/* Does messaging work correctly to the local cpu? */
38bool self_interrupt_ok;
39
40/*
41 * Top-level send_IPI*() functions to send messages to other cpus.
42 */
43
44/* Set by smp_send_stop() to avoid recursive panics. */
45static int stopping_cpus;
46
47static void __send_IPI_many(HV_Recipient *recip, int nrecip, int tag)
48{
49	int sent = 0;
50	while (sent < nrecip) {
51		int rc = hv_send_message(recip, nrecip,
52					 (HV_VirtAddr)&tag, sizeof(tag));
53		if (rc < 0) {
54			if (!stopping_cpus)  /* avoid recursive panic */
55				panic("hv_send_message returned %d", rc);
56			break;
57		}
58		WARN_ONCE(rc == 0, "hv_send_message() returned zero\n");
59		sent += rc;
60	}
61}
62
63void send_IPI_single(int cpu, int tag)
64{
65	HV_Recipient recip = {
66		.y = cpu / smp_width,
67		.x = cpu % smp_width,
68		.state = HV_TO_BE_SENT
69	};
70	__send_IPI_many(&recip, 1, tag);
71}
72
73void send_IPI_many(const struct cpumask *mask, int tag)
74{
75	HV_Recipient recip[NR_CPUS];
76	int cpu;
77	int nrecip = 0;
78	int my_cpu = smp_processor_id();
79	for_each_cpu(cpu, mask) {
80		HV_Recipient *r;
81		BUG_ON(cpu == my_cpu);
82		r = &recip[nrecip++];
83		r->y = cpu / smp_width;
84		r->x = cpu % smp_width;
85		r->state = HV_TO_BE_SENT;
86	}
87	__send_IPI_many(recip, nrecip, tag);
88}
89
90void send_IPI_allbutself(int tag)
91{
92	struct cpumask mask;
93	cpumask_copy(&mask, cpu_online_mask);
94	cpumask_clear_cpu(smp_processor_id(), &mask);
95	send_IPI_many(&mask, tag);
96}
97
98/*
99 * Functions related to starting/stopping cpus.
100 */
101
102/* Handler to start the current cpu. */
103static void smp_start_cpu_interrupt(void)
104{
105	get_irq_regs()->pc = start_cpu_function_addr;
106}
107
108/* Handler to stop the current cpu. */
109static void smp_stop_cpu_interrupt(void)
110{
111	arch_local_irq_disable_all();
112	set_cpu_online(smp_processor_id(), 0);
113	for (;;)
114		asm("nap; nop");
115}
116
117/* This function calls the 'stop' function on all other CPUs in the system. */
118void smp_send_stop(void)
119{
120	stopping_cpus = 1;
121	send_IPI_allbutself(MSG_TAG_STOP_CPU);
122}
123
124/* On panic, just wait; we may get an smp_send_stop() later on. */
125void panic_smp_self_stop(void)
126{
127	while (1)
128		asm("nap; nop");
129}
130
131/*
132 * Dispatch code called from hv_message_intr() for HV_MSG_TILE hv messages.
133 */
134void evaluate_message(int tag)
135{
136	switch (tag) {
137	case MSG_TAG_START_CPU: /* Start up a cpu */
138		smp_start_cpu_interrupt();
139		break;
140
141	case MSG_TAG_STOP_CPU: /* Sent to shut down slave CPU's */
142		smp_stop_cpu_interrupt();
143		break;
144
145	case MSG_TAG_CALL_FUNCTION_MANY: /* Call function on cpumask */
146		generic_smp_call_function_interrupt();
147		break;
148
149	case MSG_TAG_CALL_FUNCTION_SINGLE: /* Call function on one other CPU */
150		generic_smp_call_function_single_interrupt();
151		break;
152
153	case MSG_TAG_IRQ_WORK: /* Invoke IRQ work */
154		irq_work_run();
155		break;
156
157	default:
158		panic("Unknown IPI message tag %d", tag);
159		break;
160	}
161}
162
163
164/*
165 * flush_icache_range() code uses smp_call_function().
166 */
167
168struct ipi_flush {
169	unsigned long start;
170	unsigned long end;
171};
172
173static void ipi_flush_icache_range(void *info)
174{
175	struct ipi_flush *flush = (struct ipi_flush *) info;
176	__flush_icache_range(flush->start, flush->end);
177}
178
179void flush_icache_range(unsigned long start, unsigned long end)
180{
181	struct ipi_flush flush = { start, end };
182
183	/* If invoked with irqs disabled, we can not issue IPIs. */
184	if (irqs_disabled())
185		flush_remote(0, HV_FLUSH_EVICT_L1I, NULL, 0, 0, 0,
186			NULL, NULL, 0);
187	else {
188		preempt_disable();
189		on_each_cpu(ipi_flush_icache_range, &flush, 1);
190		preempt_enable();
191	}
192}
193EXPORT_SYMBOL(flush_icache_range);
194
195
196#ifdef CONFIG_IRQ_WORK
197void arch_irq_work_raise(void)
198{
199	if (arch_irq_work_has_interrupt())
200		send_IPI_single(smp_processor_id(), MSG_TAG_IRQ_WORK);
201}
202#endif
203
204
205/* Called when smp_send_reschedule() triggers IRQ_RESCHEDULE. */
206static irqreturn_t handle_reschedule_ipi(int irq, void *token)
207{
208	__this_cpu_inc(irq_stat.irq_resched_count);
209	scheduler_ipi();
210
211	return IRQ_HANDLED;
212}
213
214static struct irqaction resched_action = {
215	.handler = handle_reschedule_ipi,
216	.name = "resched",
217	.dev_id = handle_reschedule_ipi /* unique token */,
218};
219
220void __init ipi_init(void)
221{
222	int cpu = smp_processor_id();
223	HV_Recipient recip = { .y = cpu_y(cpu), .x = cpu_x(cpu),
224			       .state = HV_TO_BE_SENT };
225	int tag = MSG_TAG_CALL_FUNCTION_SINGLE;
226
227	/*
228	 * Test if we can message ourselves for arch_irq_work_raise.
229	 * This functionality is only available in the Tilera hypervisor
230	 * in versions 4.3.4 and following.
231	 */
232	if (hv_send_message(&recip, 1, (HV_VirtAddr)&tag, sizeof(tag)) == 1)
233		self_interrupt_ok = true;
234	else
235		pr_warn("Older hypervisor: disabling fast irq_work_raise\n");
236
237#if CHIP_HAS_IPI()
238	/* Map IPI trigger MMIO addresses. */
239	for_each_possible_cpu(cpu) {
240		HV_Coord tile;
241		HV_PTE pte;
242		unsigned long offset;
243
244		tile.x = cpu_x(cpu);
245		tile.y = cpu_y(cpu);
246		if (hv_get_ipi_pte(tile, KERNEL_PL, &pte) != 0)
247			panic("Failed to initialize IPI for cpu %d\n", cpu);
248
249		offset = PFN_PHYS(pte_pfn(pte));
250		ipi_mappings[cpu] = ioremap_prot(offset, PAGE_SIZE, pte);
251	}
252#endif
253
254	/* Bind handle_reschedule_ipi() to IRQ_RESCHEDULE. */
255	tile_irq_activate(IRQ_RESCHEDULE, TILE_IRQ_PERCPU);
256	BUG_ON(setup_irq(IRQ_RESCHEDULE, &resched_action));
257}
258
259#if CHIP_HAS_IPI()
260
261void smp_send_reschedule(int cpu)
262{
263	WARN_ON(cpu_is_offline(cpu));
264
265	/*
266	 * We just want to do an MMIO store.  The traditional writeq()
267	 * functions aren't really correct here, since they're always
268	 * directed at the PCI shim.  For now, just do a raw store,
269	 * casting away the __iomem attribute.
270	 */
271	((unsigned long __force *)ipi_mappings[cpu])[IRQ_RESCHEDULE] = 0;
272}
273
274#else
275
276void smp_send_reschedule(int cpu)
277{
278	HV_Coord coord;
279
280	WARN_ON(cpu_is_offline(cpu));
281
282	coord.y = cpu_y(cpu);
283	coord.x = cpu_x(cpu);
284	hv_trigger_ipi(coord, IRQ_RESCHEDULE);
285}
286
287#endif /* CHIP_HAS_IPI() */
288