1 /*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 *
14 * TILE SMP support routines.
15 */
16
17 #include <linux/smp.h>
18 #include <linux/interrupt.h>
19 #include <linux/io.h>
20 #include <linux/irq.h>
21 #include <linux/irq_work.h>
22 #include <linux/module.h>
23 #include <asm/cacheflush.h>
24 #include <asm/homecache.h>
25
26 /*
27 * We write to width and height with a single store in head_NN.S,
28 * so make the variable aligned to "long".
29 */
30 HV_Topology smp_topology __write_once __aligned(sizeof(long));
31 EXPORT_SYMBOL(smp_topology);
32
33 #if CHIP_HAS_IPI()
34 static unsigned long __iomem *ipi_mappings[NR_CPUS];
35 #endif
36
37 /* Does messaging work correctly to the local cpu? */
38 bool self_interrupt_ok;
39
40 /*
41 * Top-level send_IPI*() functions to send messages to other cpus.
42 */
43
44 /* Set by smp_send_stop() to avoid recursive panics. */
45 static int stopping_cpus;
46
__send_IPI_many(HV_Recipient * recip,int nrecip,int tag)47 static void __send_IPI_many(HV_Recipient *recip, int nrecip, int tag)
48 {
49 int sent = 0;
50 while (sent < nrecip) {
51 int rc = hv_send_message(recip, nrecip,
52 (HV_VirtAddr)&tag, sizeof(tag));
53 if (rc < 0) {
54 if (!stopping_cpus) /* avoid recursive panic */
55 panic("hv_send_message returned %d", rc);
56 break;
57 }
58 WARN_ONCE(rc == 0, "hv_send_message() returned zero\n");
59 sent += rc;
60 }
61 }
62
send_IPI_single(int cpu,int tag)63 void send_IPI_single(int cpu, int tag)
64 {
65 HV_Recipient recip = {
66 .y = cpu / smp_width,
67 .x = cpu % smp_width,
68 .state = HV_TO_BE_SENT
69 };
70 __send_IPI_many(&recip, 1, tag);
71 }
72
send_IPI_many(const struct cpumask * mask,int tag)73 void send_IPI_many(const struct cpumask *mask, int tag)
74 {
75 HV_Recipient recip[NR_CPUS];
76 int cpu;
77 int nrecip = 0;
78 int my_cpu = smp_processor_id();
79 for_each_cpu(cpu, mask) {
80 HV_Recipient *r;
81 BUG_ON(cpu == my_cpu);
82 r = &recip[nrecip++];
83 r->y = cpu / smp_width;
84 r->x = cpu % smp_width;
85 r->state = HV_TO_BE_SENT;
86 }
87 __send_IPI_many(recip, nrecip, tag);
88 }
89
send_IPI_allbutself(int tag)90 void send_IPI_allbutself(int tag)
91 {
92 struct cpumask mask;
93 cpumask_copy(&mask, cpu_online_mask);
94 cpumask_clear_cpu(smp_processor_id(), &mask);
95 send_IPI_many(&mask, tag);
96 }
97
98 /*
99 * Functions related to starting/stopping cpus.
100 */
101
102 /* Handler to start the current cpu. */
smp_start_cpu_interrupt(void)103 static void smp_start_cpu_interrupt(void)
104 {
105 get_irq_regs()->pc = start_cpu_function_addr;
106 }
107
108 /* Handler to stop the current cpu. */
smp_stop_cpu_interrupt(void)109 static void smp_stop_cpu_interrupt(void)
110 {
111 arch_local_irq_disable_all();
112 set_cpu_online(smp_processor_id(), 0);
113 for (;;)
114 asm("nap; nop");
115 }
116
117 /* This function calls the 'stop' function on all other CPUs in the system. */
smp_send_stop(void)118 void smp_send_stop(void)
119 {
120 stopping_cpus = 1;
121 send_IPI_allbutself(MSG_TAG_STOP_CPU);
122 }
123
124 /* On panic, just wait; we may get an smp_send_stop() later on. */
panic_smp_self_stop(void)125 void panic_smp_self_stop(void)
126 {
127 while (1)
128 asm("nap; nop");
129 }
130
131 /*
132 * Dispatch code called from hv_message_intr() for HV_MSG_TILE hv messages.
133 */
evaluate_message(int tag)134 void evaluate_message(int tag)
135 {
136 switch (tag) {
137 case MSG_TAG_START_CPU: /* Start up a cpu */
138 smp_start_cpu_interrupt();
139 break;
140
141 case MSG_TAG_STOP_CPU: /* Sent to shut down slave CPU's */
142 smp_stop_cpu_interrupt();
143 break;
144
145 case MSG_TAG_CALL_FUNCTION_MANY: /* Call function on cpumask */
146 generic_smp_call_function_interrupt();
147 break;
148
149 case MSG_TAG_CALL_FUNCTION_SINGLE: /* Call function on one other CPU */
150 generic_smp_call_function_single_interrupt();
151 break;
152
153 case MSG_TAG_IRQ_WORK: /* Invoke IRQ work */
154 irq_work_run();
155 break;
156
157 default:
158 panic("Unknown IPI message tag %d", tag);
159 break;
160 }
161 }
162
163
164 /*
165 * flush_icache_range() code uses smp_call_function().
166 */
167
168 struct ipi_flush {
169 unsigned long start;
170 unsigned long end;
171 };
172
ipi_flush_icache_range(void * info)173 static void ipi_flush_icache_range(void *info)
174 {
175 struct ipi_flush *flush = (struct ipi_flush *) info;
176 __flush_icache_range(flush->start, flush->end);
177 }
178
flush_icache_range(unsigned long start,unsigned long end)179 void flush_icache_range(unsigned long start, unsigned long end)
180 {
181 struct ipi_flush flush = { start, end };
182
183 /* If invoked with irqs disabled, we can not issue IPIs. */
184 if (irqs_disabled())
185 flush_remote(0, HV_FLUSH_EVICT_L1I, NULL, 0, 0, 0,
186 NULL, NULL, 0);
187 else {
188 preempt_disable();
189 on_each_cpu(ipi_flush_icache_range, &flush, 1);
190 preempt_enable();
191 }
192 }
193 EXPORT_SYMBOL(flush_icache_range);
194
195
196 #ifdef CONFIG_IRQ_WORK
arch_irq_work_raise(void)197 void arch_irq_work_raise(void)
198 {
199 if (arch_irq_work_has_interrupt())
200 send_IPI_single(smp_processor_id(), MSG_TAG_IRQ_WORK);
201 }
202 #endif
203
204
205 /* Called when smp_send_reschedule() triggers IRQ_RESCHEDULE. */
handle_reschedule_ipi(int irq,void * token)206 static irqreturn_t handle_reschedule_ipi(int irq, void *token)
207 {
208 __this_cpu_inc(irq_stat.irq_resched_count);
209 scheduler_ipi();
210
211 return IRQ_HANDLED;
212 }
213
214 static struct irqaction resched_action = {
215 .handler = handle_reschedule_ipi,
216 .name = "resched",
217 .dev_id = handle_reschedule_ipi /* unique token */,
218 };
219
ipi_init(void)220 void __init ipi_init(void)
221 {
222 int cpu = smp_processor_id();
223 HV_Recipient recip = { .y = cpu_y(cpu), .x = cpu_x(cpu),
224 .state = HV_TO_BE_SENT };
225 int tag = MSG_TAG_CALL_FUNCTION_SINGLE;
226
227 /*
228 * Test if we can message ourselves for arch_irq_work_raise.
229 * This functionality is only available in the Tilera hypervisor
230 * in versions 4.3.4 and following.
231 */
232 if (hv_send_message(&recip, 1, (HV_VirtAddr)&tag, sizeof(tag)) == 1)
233 self_interrupt_ok = true;
234 else
235 pr_warn("Older hypervisor: disabling fast irq_work_raise\n");
236
237 #if CHIP_HAS_IPI()
238 /* Map IPI trigger MMIO addresses. */
239 for_each_possible_cpu(cpu) {
240 HV_Coord tile;
241 HV_PTE pte;
242 unsigned long offset;
243
244 tile.x = cpu_x(cpu);
245 tile.y = cpu_y(cpu);
246 if (hv_get_ipi_pte(tile, KERNEL_PL, &pte) != 0)
247 panic("Failed to initialize IPI for cpu %d\n", cpu);
248
249 offset = PFN_PHYS(pte_pfn(pte));
250 ipi_mappings[cpu] = ioremap_prot(offset, PAGE_SIZE, pte);
251 }
252 #endif
253
254 /* Bind handle_reschedule_ipi() to IRQ_RESCHEDULE. */
255 tile_irq_activate(IRQ_RESCHEDULE, TILE_IRQ_PERCPU);
256 BUG_ON(setup_irq(IRQ_RESCHEDULE, &resched_action));
257 }
258
259 #if CHIP_HAS_IPI()
260
smp_send_reschedule(int cpu)261 void smp_send_reschedule(int cpu)
262 {
263 WARN_ON(cpu_is_offline(cpu));
264
265 /*
266 * We just want to do an MMIO store. The traditional writeq()
267 * functions aren't really correct here, since they're always
268 * directed at the PCI shim. For now, just do a raw store,
269 * casting away the __iomem attribute.
270 */
271 ((unsigned long __force *)ipi_mappings[cpu])[IRQ_RESCHEDULE] = 0;
272 }
273
274 #else
275
smp_send_reschedule(int cpu)276 void smp_send_reschedule(int cpu)
277 {
278 HV_Coord coord;
279
280 WARN_ON(cpu_is_offline(cpu));
281
282 coord.y = cpu_y(cpu);
283 coord.x = cpu_x(cpu);
284 hv_trigger_ipi(coord, IRQ_RESCHEDULE);
285 }
286
287 #endif /* CHIP_HAS_IPI() */
288