This source file includes following definitions.
- early_nosmt
- early_smt
- __pcpu_sigp_relax
- pcpu_sigp_retry
- pcpu_stopped
- pcpu_running
- pcpu_find_address
- pcpu_ec_call
- pcpu_alloc_lowcore
- pcpu_free_lowcore
- pcpu_prepare_secondary
- pcpu_attach_task
- pcpu_start_fn
- __pcpu_delegate
- pcpu_delegate
- pcpu_set_smt
- smp_call_online_cpu
- smp_call_ipl_cpu
- smp_find_processor_id
- arch_vcpu_is_preempted
- smp_yield_cpu
- smp_emergency_stop
- smp_send_stop
- smp_handle_ext_call
- do_ext_call_interrupt
- arch_send_call_function_ipi_mask
- arch_send_call_function_single_ipi
- smp_send_reschedule
- smp_ctl_bit_callback
- smp_ctl_set_bit
- smp_ctl_clear_bit
- smp_store_status
- smp_save_cpu_vxrs
- smp_save_cpu_regs
- smp_save_dump_cpus
- smp_cpu_set_polarization
- smp_cpu_get_polarization
- smp_get_core_info
- smp_add_core
- __smp_rescan_cpus
- smp_detect_cpus
- smp_init_secondary
- smp_start_secondary
- __cpu_up
- _setup_possible_cpus
- __cpu_disable
- __cpu_die
- cpu_die
- smp_fill_possible_mask
- smp_prepare_cpus
- smp_prepare_boot_cpu
- smp_cpus_done
- smp_setup_processor_id
- setup_profiling_timer
- cpu_configure_show
- cpu_configure_store
- show_cpu_address
- smp_cpu_online
- smp_cpu_pre_down
- smp_add_present_cpu
- smp_rescan_cpus
- rescan_store
- s390_smp_init
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 #define KMSG_COMPONENT "cpu"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21
22 #include <linux/workqueue.h>
23 #include <linux/memblock.h>
24 #include <linux/export.h>
25 #include <linux/init.h>
26 #include <linux/mm.h>
27 #include <linux/err.h>
28 #include <linux/spinlock.h>
29 #include <linux/kernel_stat.h>
30 #include <linux/delay.h>
31 #include <linux/interrupt.h>
32 #include <linux/irqflags.h>
33 #include <linux/cpu.h>
34 #include <linux/slab.h>
35 #include <linux/sched/hotplug.h>
36 #include <linux/sched/task_stack.h>
37 #include <linux/crash_dump.h>
38 #include <linux/kprobes.h>
39 #include <asm/asm-offsets.h>
40 #include <asm/diag.h>
41 #include <asm/switch_to.h>
42 #include <asm/facility.h>
43 #include <asm/ipl.h>
44 #include <asm/setup.h>
45 #include <asm/irq.h>
46 #include <asm/tlbflush.h>
47 #include <asm/vtimer.h>
48 #include <asm/lowcore.h>
49 #include <asm/sclp.h>
50 #include <asm/vdso.h>
51 #include <asm/debug.h>
52 #include <asm/os_info.h>
53 #include <asm/sigp.h>
54 #include <asm/idle.h>
55 #include <asm/nmi.h>
56 #include <asm/stacktrace.h>
57 #include <asm/topology.h>
58 #include "entry.h"
59
60 enum {
61 ec_schedule = 0,
62 ec_call_function_single,
63 ec_stop_cpu,
64 };
65
66 enum {
67 CPU_STATE_STANDBY,
68 CPU_STATE_CONFIGURED,
69 };
70
71 static DEFINE_PER_CPU(struct cpu *, cpu_device);
72
73 struct pcpu {
74 struct lowcore *lowcore;
75 unsigned long ec_mask;
76 unsigned long ec_clk;
77 signed char state;
78 signed char polarization;
79 u16 address;
80 };
81
82 static u8 boot_core_type;
83 static struct pcpu pcpu_devices[NR_CPUS];
84
85 unsigned int smp_cpu_mt_shift;
86 EXPORT_SYMBOL(smp_cpu_mt_shift);
87
88 unsigned int smp_cpu_mtid;
89 EXPORT_SYMBOL(smp_cpu_mtid);
90
91 #ifdef CONFIG_CRASH_DUMP
92 __vector128 __initdata boot_cpu_vector_save_area[__NUM_VXRS];
93 #endif
94
95 static unsigned int smp_max_threads __initdata = -1U;
96
97 static int __init early_nosmt(char *s)
98 {
99 smp_max_threads = 1;
100 return 0;
101 }
102 early_param("nosmt", early_nosmt);
103
104 static int __init early_smt(char *s)
105 {
106 get_option(&s, &smp_max_threads);
107 return 0;
108 }
109 early_param("smt", early_smt);
110
111
112
113
114
115 DEFINE_MUTEX(smp_cpu_state_mutex);
116
117
118
119
120 static inline int __pcpu_sigp_relax(u16 addr, u8 order, unsigned long parm)
121 {
122 int cc;
123
124 while (1) {
125 cc = __pcpu_sigp(addr, order, parm, NULL);
126 if (cc != SIGP_CC_BUSY)
127 return cc;
128 cpu_relax();
129 }
130 }
131
132 static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
133 {
134 int cc, retry;
135
136 for (retry = 0; ; retry++) {
137 cc = __pcpu_sigp(pcpu->address, order, parm, NULL);
138 if (cc != SIGP_CC_BUSY)
139 break;
140 if (retry >= 3)
141 udelay(10);
142 }
143 return cc;
144 }
145
146 static inline int pcpu_stopped(struct pcpu *pcpu)
147 {
148 u32 uninitialized_var(status);
149
150 if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
151 0, &status) != SIGP_CC_STATUS_STORED)
152 return 0;
153 return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
154 }
155
156 static inline int pcpu_running(struct pcpu *pcpu)
157 {
158 if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
159 0, NULL) != SIGP_CC_STATUS_STORED)
160 return 1;
161
162 return 0;
163 }
164
165
166
167
168 static struct pcpu *pcpu_find_address(const struct cpumask *mask, u16 address)
169 {
170 int cpu;
171
172 for_each_cpu(cpu, mask)
173 if (pcpu_devices[cpu].address == address)
174 return pcpu_devices + cpu;
175 return NULL;
176 }
177
178 static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
179 {
180 int order;
181
182 if (test_and_set_bit(ec_bit, &pcpu->ec_mask))
183 return;
184 order = pcpu_running(pcpu) ? SIGP_EXTERNAL_CALL : SIGP_EMERGENCY_SIGNAL;
185 pcpu->ec_clk = get_tod_clock_fast();
186 pcpu_sigp_retry(pcpu, order, 0);
187 }
188
189 static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
190 {
191 unsigned long async_stack, nodat_stack;
192 struct lowcore *lc;
193
194 if (pcpu != &pcpu_devices[0]) {
195 pcpu->lowcore = (struct lowcore *)
196 __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
197 nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
198 if (!pcpu->lowcore || !nodat_stack)
199 goto out;
200 } else {
201 nodat_stack = pcpu->lowcore->nodat_stack - STACK_INIT_OFFSET;
202 }
203 async_stack = stack_alloc();
204 if (!async_stack)
205 goto out;
206 lc = pcpu->lowcore;
207 memcpy(lc, &S390_lowcore, 512);
208 memset((char *) lc + 512, 0, sizeof(*lc) - 512);
209 lc->async_stack = async_stack + STACK_INIT_OFFSET;
210 lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET;
211 lc->cpu_nr = cpu;
212 lc->spinlock_lockval = arch_spin_lockval(cpu);
213 lc->spinlock_index = 0;
214 lc->br_r1_trampoline = 0x07f1;
215 lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW);
216 lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW);
217 if (nmi_alloc_per_cpu(lc))
218 goto out_async;
219 if (vdso_alloc_per_cpu(lc))
220 goto out_mcesa;
221 lowcore_ptr[cpu] = lc;
222 pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc);
223 return 0;
224
225 out_mcesa:
226 nmi_free_per_cpu(lc);
227 out_async:
228 stack_free(async_stack);
229 out:
230 if (pcpu != &pcpu_devices[0]) {
231 free_pages(nodat_stack, THREAD_SIZE_ORDER);
232 free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
233 }
234 return -ENOMEM;
235 }
236
237 static void pcpu_free_lowcore(struct pcpu *pcpu)
238 {
239 unsigned long async_stack, nodat_stack, lowcore;
240
241 nodat_stack = pcpu->lowcore->nodat_stack - STACK_INIT_OFFSET;
242 async_stack = pcpu->lowcore->async_stack - STACK_INIT_OFFSET;
243 lowcore = (unsigned long) pcpu->lowcore;
244
245 pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
246 lowcore_ptr[pcpu - pcpu_devices] = NULL;
247 vdso_free_per_cpu(pcpu->lowcore);
248 nmi_free_per_cpu(pcpu->lowcore);
249 stack_free(async_stack);
250 if (pcpu == &pcpu_devices[0])
251 return;
252 free_pages(nodat_stack, THREAD_SIZE_ORDER);
253 free_pages(lowcore, LC_ORDER);
254 }
255
256 static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
257 {
258 struct lowcore *lc = pcpu->lowcore;
259
260 cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask);
261 cpumask_set_cpu(cpu, mm_cpumask(&init_mm));
262 lc->cpu_nr = cpu;
263 lc->spinlock_lockval = arch_spin_lockval(cpu);
264 lc->spinlock_index = 0;
265 lc->percpu_offset = __per_cpu_offset[cpu];
266 lc->kernel_asce = S390_lowcore.kernel_asce;
267 lc->user_asce = S390_lowcore.kernel_asce;
268 lc->machine_flags = S390_lowcore.machine_flags;
269 lc->user_timer = lc->system_timer =
270 lc->steal_timer = lc->avg_steal_timer = 0;
271 __ctl_store(lc->cregs_save_area, 0, 15);
272 lc->cregs_save_area[1] = lc->kernel_asce;
273 lc->cregs_save_area[7] = lc->vdso_asce;
274 save_access_regs((unsigned int *) lc->access_regs_save_area);
275 memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
276 sizeof(lc->stfle_fac_list));
277 memcpy(lc->alt_stfle_fac_list, S390_lowcore.alt_stfle_fac_list,
278 sizeof(lc->alt_stfle_fac_list));
279 arch_spin_lock_setup(cpu);
280 }
281
282 static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
283 {
284 struct lowcore *lc = pcpu->lowcore;
285
286 lc->kernel_stack = (unsigned long) task_stack_page(tsk)
287 + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
288 lc->current_task = (unsigned long) tsk;
289 lc->lpp = LPP_MAGIC;
290 lc->current_pid = tsk->pid;
291 lc->user_timer = tsk->thread.user_timer;
292 lc->guest_timer = tsk->thread.guest_timer;
293 lc->system_timer = tsk->thread.system_timer;
294 lc->hardirq_timer = tsk->thread.hardirq_timer;
295 lc->softirq_timer = tsk->thread.softirq_timer;
296 lc->steal_timer = 0;
297 }
298
299 static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
300 {
301 struct lowcore *lc = pcpu->lowcore;
302
303 lc->restart_stack = lc->nodat_stack;
304 lc->restart_fn = (unsigned long) func;
305 lc->restart_data = (unsigned long) data;
306 lc->restart_source = -1UL;
307 pcpu_sigp_retry(pcpu, SIGP_RESTART, 0);
308 }
309
310
311
312
313 static void __pcpu_delegate(void (*func)(void*), void *data)
314 {
315 func(data);
316 }
317
318 static void __no_sanitize_address pcpu_delegate(struct pcpu *pcpu,
319 void (*func)(void *),
320 void *data, unsigned long stack)
321 {
322 struct lowcore *lc = lowcore_ptr[pcpu - pcpu_devices];
323 unsigned long source_cpu = stap();
324
325 __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT);
326 if (pcpu->address == source_cpu)
327 CALL_ON_STACK(__pcpu_delegate, stack, 2, func, data);
328
329 pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
330
331 mem_assign_absolute(lc->restart_stack, stack);
332 mem_assign_absolute(lc->restart_fn, (unsigned long) func);
333 mem_assign_absolute(lc->restart_data, (unsigned long) data);
334 mem_assign_absolute(lc->restart_source, source_cpu);
335 __bpon();
336 asm volatile(
337 "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
338 " brc 2,0b # busy, try again\n"
339 "1: sigp 0,%1,%3 # sigp stop to current cpu\n"
340 " brc 2,1b # busy, try again\n"
341 : : "d" (pcpu->address), "d" (source_cpu),
342 "K" (SIGP_RESTART), "K" (SIGP_STOP)
343 : "0", "1", "cc");
344 for (;;) ;
345 }
346
347
348
349
350 static int pcpu_set_smt(unsigned int mtid)
351 {
352 int cc;
353
354 if (smp_cpu_mtid == mtid)
355 return 0;
356 cc = __pcpu_sigp(0, SIGP_SET_MULTI_THREADING, mtid, NULL);
357 if (cc == 0) {
358 smp_cpu_mtid = mtid;
359 smp_cpu_mt_shift = 0;
360 while (smp_cpu_mtid >= (1U << smp_cpu_mt_shift))
361 smp_cpu_mt_shift++;
362 pcpu_devices[0].address = stap();
363 }
364 return cc;
365 }
366
367
368
369
370 void smp_call_online_cpu(void (*func)(void *), void *data)
371 {
372 struct pcpu *pcpu;
373
374
375 pcpu = pcpu_find_address(cpu_online_mask, stap());
376 if (!pcpu)
377
378 pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
379 pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
380 }
381
382
383
384
385 void smp_call_ipl_cpu(void (*func)(void *), void *data)
386 {
387 struct lowcore *lc = pcpu_devices->lowcore;
388
389 if (pcpu_devices[0].address == stap())
390 lc = &S390_lowcore;
391
392 pcpu_delegate(&pcpu_devices[0], func, data,
393 lc->nodat_stack);
394 }
395
396 int smp_find_processor_id(u16 address)
397 {
398 int cpu;
399
400 for_each_present_cpu(cpu)
401 if (pcpu_devices[cpu].address == address)
402 return cpu;
403 return -1;
404 }
405
406 bool notrace arch_vcpu_is_preempted(int cpu)
407 {
408 if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu))
409 return false;
410 if (pcpu_running(pcpu_devices + cpu))
411 return false;
412 return true;
413 }
414 EXPORT_SYMBOL(arch_vcpu_is_preempted);
415
416 void notrace smp_yield_cpu(int cpu)
417 {
418 if (MACHINE_HAS_DIAG9C) {
419 diag_stat_inc_norecursion(DIAG_STAT_X09C);
420 asm volatile("diag %0,0,0x9c"
421 : : "d" (pcpu_devices[cpu].address));
422 } else if (MACHINE_HAS_DIAG44 && !smp_cpu_mtid) {
423 diag_stat_inc_norecursion(DIAG_STAT_X044);
424 asm volatile("diag 0,0,0x44");
425 }
426 }
427
428
429
430
431
432 void notrace smp_emergency_stop(void)
433 {
434 cpumask_t cpumask;
435 u64 end;
436 int cpu;
437
438 cpumask_copy(&cpumask, cpu_online_mask);
439 cpumask_clear_cpu(smp_processor_id(), &cpumask);
440
441 end = get_tod_clock() + (1000000UL << 12);
442 for_each_cpu(cpu, &cpumask) {
443 struct pcpu *pcpu = pcpu_devices + cpu;
444 set_bit(ec_stop_cpu, &pcpu->ec_mask);
445 while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
446 0, NULL) == SIGP_CC_BUSY &&
447 get_tod_clock() < end)
448 cpu_relax();
449 }
450 while (get_tod_clock() < end) {
451 for_each_cpu(cpu, &cpumask)
452 if (pcpu_stopped(pcpu_devices + cpu))
453 cpumask_clear_cpu(cpu, &cpumask);
454 if (cpumask_empty(&cpumask))
455 break;
456 cpu_relax();
457 }
458 }
459 NOKPROBE_SYMBOL(smp_emergency_stop);
460
461
462
463
464 void smp_send_stop(void)
465 {
466 int cpu;
467
468
469 __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT);
470 trace_hardirqs_off();
471
472 debug_set_critical();
473
474 if (oops_in_progress)
475 smp_emergency_stop();
476
477
478 for_each_online_cpu(cpu) {
479 if (cpu == smp_processor_id())
480 continue;
481 pcpu_sigp_retry(pcpu_devices + cpu, SIGP_STOP, 0);
482 while (!pcpu_stopped(pcpu_devices + cpu))
483 cpu_relax();
484 }
485 }
486
487
488
489
490
491 static void smp_handle_ext_call(void)
492 {
493 unsigned long bits;
494
495
496 bits = xchg(&pcpu_devices[smp_processor_id()].ec_mask, 0);
497 if (test_bit(ec_stop_cpu, &bits))
498 smp_stop_cpu();
499 if (test_bit(ec_schedule, &bits))
500 scheduler_ipi();
501 if (test_bit(ec_call_function_single, &bits))
502 generic_smp_call_function_single_interrupt();
503 }
504
505 static void do_ext_call_interrupt(struct ext_code ext_code,
506 unsigned int param32, unsigned long param64)
507 {
508 inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS);
509 smp_handle_ext_call();
510 }
511
512 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
513 {
514 int cpu;
515
516 for_each_cpu(cpu, mask)
517 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
518 }
519
520 void arch_send_call_function_single_ipi(int cpu)
521 {
522 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
523 }
524
525
526
527
528
529
530 void smp_send_reschedule(int cpu)
531 {
532 pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
533 }
534
535
536
537
538 struct ec_creg_mask_parms {
539 unsigned long orval;
540 unsigned long andval;
541 int cr;
542 };
543
544
545
546
547 static void smp_ctl_bit_callback(void *info)
548 {
549 struct ec_creg_mask_parms *pp = info;
550 unsigned long cregs[16];
551
552 __ctl_store(cregs, 0, 15);
553 cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
554 __ctl_load(cregs, 0, 15);
555 }
556
557
558
559
560 void smp_ctl_set_bit(int cr, int bit)
561 {
562 struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr };
563
564 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
565 }
566 EXPORT_SYMBOL(smp_ctl_set_bit);
567
568
569
570
571 void smp_ctl_clear_bit(int cr, int bit)
572 {
573 struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr };
574
575 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
576 }
577 EXPORT_SYMBOL(smp_ctl_clear_bit);
578
579 #ifdef CONFIG_CRASH_DUMP
580
581 int smp_store_status(int cpu)
582 {
583 struct pcpu *pcpu = pcpu_devices + cpu;
584 unsigned long pa;
585
586 pa = __pa(&pcpu->lowcore->floating_pt_save_area);
587 if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_STATUS_AT_ADDRESS,
588 pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
589 return -EIO;
590 if (!MACHINE_HAS_VX && !MACHINE_HAS_GS)
591 return 0;
592 pa = __pa(pcpu->lowcore->mcesad & MCESA_ORIGIN_MASK);
593 if (MACHINE_HAS_GS)
594 pa |= pcpu->lowcore->mcesad & MCESA_LC_MASK;
595 if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS,
596 pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
597 return -EIO;
598 return 0;
599 }
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627 static __init void smp_save_cpu_vxrs(struct save_area *sa, u16 addr,
628 bool is_boot_cpu, unsigned long page)
629 {
630 __vector128 *vxrs = (__vector128 *) page;
631
632 if (is_boot_cpu)
633 vxrs = boot_cpu_vector_save_area;
634 else
635 __pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, page);
636 save_area_add_vxrs(sa, vxrs);
637 }
638
639 static __init void smp_save_cpu_regs(struct save_area *sa, u16 addr,
640 bool is_boot_cpu, unsigned long page)
641 {
642 void *regs = (void *) page;
643
644 if (is_boot_cpu)
645 copy_oldmem_kernel(regs, (void *) __LC_FPREGS_SAVE_AREA, 512);
646 else
647 __pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, page);
648 save_area_add_regs(sa, regs);
649 }
650
651 void __init smp_save_dump_cpus(void)
652 {
653 int addr, boot_cpu_addr, max_cpu_addr;
654 struct save_area *sa;
655 unsigned long page;
656 bool is_boot_cpu;
657
658 if (!(OLDMEM_BASE || ipl_info.type == IPL_TYPE_FCP_DUMP))
659
660 return;
661
662 page = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, 1UL << 31);
663 if (!page)
664 panic("ERROR: Failed to allocate %lx bytes below %lx\n",
665 PAGE_SIZE, 1UL << 31);
666
667
668 pcpu_set_smt(sclp.mtid_prev);
669 boot_cpu_addr = stap();
670 max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev;
671 for (addr = 0; addr <= max_cpu_addr; addr++) {
672 if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0) ==
673 SIGP_CC_NOT_OPERATIONAL)
674 continue;
675 is_boot_cpu = (addr == boot_cpu_addr);
676
677 sa = save_area_alloc(is_boot_cpu);
678 if (!sa)
679 panic("could not allocate memory for save area\n");
680 if (MACHINE_HAS_VX)
681
682 smp_save_cpu_vxrs(sa, addr, is_boot_cpu, page);
683
684
685
686
687
688
689 if (!is_boot_cpu || OLDMEM_BASE)
690
691 smp_save_cpu_regs(sa, addr, is_boot_cpu, page);
692 }
693 memblock_free(page, PAGE_SIZE);
694 diag_dma_ops.diag308_reset();
695 pcpu_set_smt(0);
696 }
697 #endif
698
699 void smp_cpu_set_polarization(int cpu, int val)
700 {
701 pcpu_devices[cpu].polarization = val;
702 }
703
704 int smp_cpu_get_polarization(int cpu)
705 {
706 return pcpu_devices[cpu].polarization;
707 }
708
709 static void __ref smp_get_core_info(struct sclp_core_info *info, int early)
710 {
711 static int use_sigp_detection;
712 int address;
713
714 if (use_sigp_detection || sclp_get_core_info(info, early)) {
715 use_sigp_detection = 1;
716 for (address = 0;
717 address < (SCLP_MAX_CORES << smp_cpu_mt_shift);
718 address += (1U << smp_cpu_mt_shift)) {
719 if (__pcpu_sigp_relax(address, SIGP_SENSE, 0) ==
720 SIGP_CC_NOT_OPERATIONAL)
721 continue;
722 info->core[info->configured].core_id =
723 address >> smp_cpu_mt_shift;
724 info->configured++;
725 }
726 info->combined = info->configured;
727 }
728 }
729
730 static int smp_add_present_cpu(int cpu);
731
732 static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail,
733 bool configured, bool early)
734 {
735 struct pcpu *pcpu;
736 int cpu, nr, i;
737 u16 address;
738
739 nr = 0;
740 if (sclp.has_core_type && core->type != boot_core_type)
741 return nr;
742 cpu = cpumask_first(avail);
743 address = core->core_id << smp_cpu_mt_shift;
744 for (i = 0; (i <= smp_cpu_mtid) && (cpu < nr_cpu_ids); i++) {
745 if (pcpu_find_address(cpu_present_mask, address + i))
746 continue;
747 pcpu = pcpu_devices + cpu;
748 pcpu->address = address + i;
749 if (configured)
750 pcpu->state = CPU_STATE_CONFIGURED;
751 else
752 pcpu->state = CPU_STATE_STANDBY;
753 smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
754 set_cpu_present(cpu, true);
755 if (!early && smp_add_present_cpu(cpu) != 0)
756 set_cpu_present(cpu, false);
757 else
758 nr++;
759 cpumask_clear_cpu(cpu, avail);
760 cpu = cpumask_next(cpu, avail);
761 }
762 return nr;
763 }
764
765 static int __smp_rescan_cpus(struct sclp_core_info *info, bool early)
766 {
767 struct sclp_core_entry *core;
768 cpumask_t avail;
769 bool configured;
770 u16 core_id;
771 int nr, i;
772
773 nr = 0;
774 cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
775
776
777
778
779 if (early) {
780 core_id = pcpu_devices[0].address >> smp_cpu_mt_shift;
781 for (i = 0; i < info->configured; i++) {
782 core = &info->core[i];
783 if (core->core_id == core_id) {
784 nr += smp_add_core(core, &avail, true, early);
785 break;
786 }
787 }
788 }
789 for (i = 0; i < info->combined; i++) {
790 configured = i < info->configured;
791 nr += smp_add_core(&info->core[i], &avail, configured, early);
792 }
793 return nr;
794 }
795
796 void __init smp_detect_cpus(void)
797 {
798 unsigned int cpu, mtid, c_cpus, s_cpus;
799 struct sclp_core_info *info;
800 u16 address;
801
802
803 info = memblock_alloc(sizeof(*info), 8);
804 if (!info)
805 panic("%s: Failed to allocate %zu bytes align=0x%x\n",
806 __func__, sizeof(*info), 8);
807 smp_get_core_info(info, 1);
808
809 if (sclp.has_core_type) {
810 address = stap();
811 for (cpu = 0; cpu < info->combined; cpu++)
812 if (info->core[cpu].core_id == address) {
813
814 boot_core_type = info->core[cpu].type;
815 break;
816 }
817 if (cpu >= info->combined)
818 panic("Could not find boot CPU type");
819 }
820
821
822 mtid = boot_core_type ? sclp.mtid : sclp.mtid_cp;
823 mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1;
824 pcpu_set_smt(mtid);
825
826
827 c_cpus = s_cpus = 0;
828 for (cpu = 0; cpu < info->combined; cpu++) {
829 if (sclp.has_core_type &&
830 info->core[cpu].type != boot_core_type)
831 continue;
832 if (cpu < info->configured)
833 c_cpus += smp_cpu_mtid + 1;
834 else
835 s_cpus += smp_cpu_mtid + 1;
836 }
837 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
838
839
840 get_online_cpus();
841 __smp_rescan_cpus(info, true);
842 put_online_cpus();
843 memblock_free_early((unsigned long)info, sizeof(*info));
844 }
845
846 static void smp_init_secondary(void)
847 {
848 int cpu = smp_processor_id();
849
850 S390_lowcore.last_update_clock = get_tod_clock();
851 restore_access_regs(S390_lowcore.access_regs_save_area);
852 set_cpu_flag(CIF_ASCE_PRIMARY);
853 set_cpu_flag(CIF_ASCE_SECONDARY);
854 cpu_init();
855 preempt_disable();
856 init_cpu_timer();
857 vtime_init();
858 pfault_init();
859 notify_cpu_starting(smp_processor_id());
860 if (topology_cpu_dedicated(cpu))
861 set_cpu_flag(CIF_DEDICATED_CPU);
862 else
863 clear_cpu_flag(CIF_DEDICATED_CPU);
864 set_cpu_online(smp_processor_id(), true);
865 inc_irq_stat(CPU_RST);
866 local_irq_enable();
867 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
868 }
869
870
871
872
873 static void __no_sanitize_address smp_start_secondary(void *cpuvoid)
874 {
875 S390_lowcore.restart_stack = (unsigned long) restart_stack;
876 S390_lowcore.restart_fn = (unsigned long) do_restart;
877 S390_lowcore.restart_data = 0;
878 S390_lowcore.restart_source = -1UL;
879 __ctl_load(S390_lowcore.cregs_save_area, 0, 15);
880 __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT);
881 CALL_ON_STACK(smp_init_secondary, S390_lowcore.kernel_stack, 0);
882 }
883
884
885 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
886 {
887 struct pcpu *pcpu;
888 int base, i, rc;
889
890 pcpu = pcpu_devices + cpu;
891 if (pcpu->state != CPU_STATE_CONFIGURED)
892 return -EIO;
893 base = smp_get_base_cpu(cpu);
894 for (i = 0; i <= smp_cpu_mtid; i++) {
895 if (base + i < nr_cpu_ids)
896 if (cpu_online(base + i))
897 break;
898 }
899
900
901
902
903 if (i > smp_cpu_mtid &&
904 pcpu_sigp_retry(pcpu_devices + base, SIGP_INITIAL_CPU_RESET, 0) !=
905 SIGP_CC_ORDER_CODE_ACCEPTED)
906 return -EIO;
907
908 rc = pcpu_alloc_lowcore(pcpu, cpu);
909 if (rc)
910 return rc;
911 pcpu_prepare_secondary(pcpu, cpu);
912 pcpu_attach_task(pcpu, tidle);
913 pcpu_start_fn(pcpu, smp_start_secondary, NULL);
914
915 while (!cpu_online(cpu))
916 cpu_relax();
917 return 0;
918 }
919
920 static unsigned int setup_possible_cpus __initdata;
921
922 static int __init _setup_possible_cpus(char *s)
923 {
924 get_option(&s, &setup_possible_cpus);
925 return 0;
926 }
927 early_param("possible_cpus", _setup_possible_cpus);
928
929 int __cpu_disable(void)
930 {
931 unsigned long cregs[16];
932
933
934 smp_handle_ext_call();
935 set_cpu_online(smp_processor_id(), false);
936
937 pfault_fini();
938
939 __ctl_store(cregs, 0, 15);
940 cregs[0] &= ~0x0000ee70UL;
941 cregs[6] &= ~0xff000000UL;
942 cregs[14] &= ~0x1f000000UL;
943 __ctl_load(cregs, 0, 15);
944 clear_cpu_flag(CIF_NOHZ_DELAY);
945 return 0;
946 }
947
948 void __cpu_die(unsigned int cpu)
949 {
950 struct pcpu *pcpu;
951
952
953 pcpu = pcpu_devices + cpu;
954 while (!pcpu_stopped(pcpu))
955 cpu_relax();
956 pcpu_free_lowcore(pcpu);
957 cpumask_clear_cpu(cpu, mm_cpumask(&init_mm));
958 cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask);
959 }
960
961 void __noreturn cpu_die(void)
962 {
963 idle_task_exit();
964 __bpon();
965 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
966 for (;;) ;
967 }
968
969 void __init smp_fill_possible_mask(void)
970 {
971 unsigned int possible, sclp_max, cpu;
972
973 sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1;
974 sclp_max = min(smp_max_threads, sclp_max);
975 sclp_max = (sclp.max_cores * sclp_max) ?: nr_cpu_ids;
976 possible = setup_possible_cpus ?: nr_cpu_ids;
977 possible = min(possible, sclp_max);
978 for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++)
979 set_cpu_possible(cpu, true);
980 }
981
982 void __init smp_prepare_cpus(unsigned int max_cpus)
983 {
984
985 if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt))
986 panic("Couldn't request external interrupt 0x1201");
987
988 if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
989 panic("Couldn't request external interrupt 0x1202");
990 }
991
992 void __init smp_prepare_boot_cpu(void)
993 {
994 struct pcpu *pcpu = pcpu_devices;
995
996 WARN_ON(!cpu_present(0) || !cpu_online(0));
997 pcpu->state = CPU_STATE_CONFIGURED;
998 pcpu->lowcore = (struct lowcore *)(unsigned long) store_prefix();
999 S390_lowcore.percpu_offset = __per_cpu_offset[0];
1000 smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
1001 }
1002
1003 void __init smp_cpus_done(unsigned int max_cpus)
1004 {
1005 }
1006
1007 void __init smp_setup_processor_id(void)
1008 {
1009 pcpu_devices[0].address = stap();
1010 S390_lowcore.cpu_nr = 0;
1011 S390_lowcore.spinlock_lockval = arch_spin_lockval(0);
1012 S390_lowcore.spinlock_index = 0;
1013 }
1014
1015
1016
1017
1018
1019
1020
1021 int setup_profiling_timer(unsigned int multiplier)
1022 {
1023 return 0;
1024 }
1025
1026 static ssize_t cpu_configure_show(struct device *dev,
1027 struct device_attribute *attr, char *buf)
1028 {
1029 ssize_t count;
1030
1031 mutex_lock(&smp_cpu_state_mutex);
1032 count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
1033 mutex_unlock(&smp_cpu_state_mutex);
1034 return count;
1035 }
1036
1037 static ssize_t cpu_configure_store(struct device *dev,
1038 struct device_attribute *attr,
1039 const char *buf, size_t count)
1040 {
1041 struct pcpu *pcpu;
1042 int cpu, val, rc, i;
1043 char delim;
1044
1045 if (sscanf(buf, "%d %c", &val, &delim) != 1)
1046 return -EINVAL;
1047 if (val != 0 && val != 1)
1048 return -EINVAL;
1049 get_online_cpus();
1050 mutex_lock(&smp_cpu_state_mutex);
1051 rc = -EBUSY;
1052
1053 cpu = dev->id;
1054 cpu = smp_get_base_cpu(cpu);
1055 if (cpu == 0)
1056 goto out;
1057 for (i = 0; i <= smp_cpu_mtid; i++)
1058 if (cpu_online(cpu + i))
1059 goto out;
1060 pcpu = pcpu_devices + cpu;
1061 rc = 0;
1062 switch (val) {
1063 case 0:
1064 if (pcpu->state != CPU_STATE_CONFIGURED)
1065 break;
1066 rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift);
1067 if (rc)
1068 break;
1069 for (i = 0; i <= smp_cpu_mtid; i++) {
1070 if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1071 continue;
1072 pcpu[i].state = CPU_STATE_STANDBY;
1073 smp_cpu_set_polarization(cpu + i,
1074 POLARIZATION_UNKNOWN);
1075 }
1076 topology_expect_change();
1077 break;
1078 case 1:
1079 if (pcpu->state != CPU_STATE_STANDBY)
1080 break;
1081 rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift);
1082 if (rc)
1083 break;
1084 for (i = 0; i <= smp_cpu_mtid; i++) {
1085 if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1086 continue;
1087 pcpu[i].state = CPU_STATE_CONFIGURED;
1088 smp_cpu_set_polarization(cpu + i,
1089 POLARIZATION_UNKNOWN);
1090 }
1091 topology_expect_change();
1092 break;
1093 default:
1094 break;
1095 }
1096 out:
1097 mutex_unlock(&smp_cpu_state_mutex);
1098 put_online_cpus();
1099 return rc ? rc : count;
1100 }
1101 static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
1102
1103 static ssize_t show_cpu_address(struct device *dev,
1104 struct device_attribute *attr, char *buf)
1105 {
1106 return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
1107 }
1108 static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
1109
1110 static struct attribute *cpu_common_attrs[] = {
1111 &dev_attr_configure.attr,
1112 &dev_attr_address.attr,
1113 NULL,
1114 };
1115
1116 static struct attribute_group cpu_common_attr_group = {
1117 .attrs = cpu_common_attrs,
1118 };
1119
1120 static struct attribute *cpu_online_attrs[] = {
1121 &dev_attr_idle_count.attr,
1122 &dev_attr_idle_time_us.attr,
1123 NULL,
1124 };
1125
1126 static struct attribute_group cpu_online_attr_group = {
1127 .attrs = cpu_online_attrs,
1128 };
1129
1130 static int smp_cpu_online(unsigned int cpu)
1131 {
1132 struct device *s = &per_cpu(cpu_device, cpu)->dev;
1133
1134 return sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1135 }
1136 static int smp_cpu_pre_down(unsigned int cpu)
1137 {
1138 struct device *s = &per_cpu(cpu_device, cpu)->dev;
1139
1140 sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1141 return 0;
1142 }
1143
1144 static int smp_add_present_cpu(int cpu)
1145 {
1146 struct device *s;
1147 struct cpu *c;
1148 int rc;
1149
1150 c = kzalloc(sizeof(*c), GFP_KERNEL);
1151 if (!c)
1152 return -ENOMEM;
1153 per_cpu(cpu_device, cpu) = c;
1154 s = &c->dev;
1155 c->hotpluggable = 1;
1156 rc = register_cpu(c, cpu);
1157 if (rc)
1158 goto out;
1159 rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
1160 if (rc)
1161 goto out_cpu;
1162 rc = topology_cpu_init(c);
1163 if (rc)
1164 goto out_topology;
1165 return 0;
1166
1167 out_topology:
1168 sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
1169 out_cpu:
1170 unregister_cpu(c);
1171 out:
1172 return rc;
1173 }
1174
1175 int __ref smp_rescan_cpus(void)
1176 {
1177 struct sclp_core_info *info;
1178 int nr;
1179
1180 info = kzalloc(sizeof(*info), GFP_KERNEL);
1181 if (!info)
1182 return -ENOMEM;
1183 smp_get_core_info(info, 0);
1184 get_online_cpus();
1185 mutex_lock(&smp_cpu_state_mutex);
1186 nr = __smp_rescan_cpus(info, false);
1187 mutex_unlock(&smp_cpu_state_mutex);
1188 put_online_cpus();
1189 kfree(info);
1190 if (nr)
1191 topology_schedule_update();
1192 return 0;
1193 }
1194
1195 static ssize_t __ref rescan_store(struct device *dev,
1196 struct device_attribute *attr,
1197 const char *buf,
1198 size_t count)
1199 {
1200 int rc;
1201
1202 rc = lock_device_hotplug_sysfs();
1203 if (rc)
1204 return rc;
1205 rc = smp_rescan_cpus();
1206 unlock_device_hotplug();
1207 return rc ? rc : count;
1208 }
1209 static DEVICE_ATTR_WO(rescan);
1210
1211 static int __init s390_smp_init(void)
1212 {
1213 int cpu, rc = 0;
1214
1215 rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan);
1216 if (rc)
1217 return rc;
1218 for_each_present_cpu(cpu) {
1219 rc = smp_add_present_cpu(cpu);
1220 if (rc)
1221 goto out;
1222 }
1223
1224 rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
1225 smp_cpu_online, smp_cpu_pre_down);
1226 rc = rc <= 0 ? rc : 0;
1227 out:
1228 return rc;
1229 }
1230 subsys_initcall(s390_smp_init);