This source file includes following definitions.
- nesting_enabled
- next_runnable_thread
- kvmppc_ipi_thread
- kvmppc_fast_vcpu_kick_hv
- kvmppc_core_start_stolen
- kvmppc_core_end_stolen
- kvmppc_core_vcpu_load_hv
- kvmppc_core_vcpu_put_hv
- kvmppc_set_msr_hv
- kvmppc_set_pvr_hv
- kvmppc_set_arch_compat
- kvmppc_dump_regs
- kvmppc_find_vcpu
- init_vpa
- set_vpa
- vpa_is_registered
- do_h_register_vpa
- kvmppc_update_vpa
- kvmppc_update_vpas
- vcore_stolen_time
- kvmppc_create_dtl_entry
- kvmppc_doorbell_pending
- kvmppc_power8_compatible
- kvmppc_h_set_mode
- kvmppc_copy_guest
- kvmppc_h_page_init
- kvm_arch_vcpu_yield_to
- kvmppc_get_yield_count
- kvmppc_pseries_do_hcall
- kvmppc_nested_cede
- kvmppc_hcall_impl_hv
- kvmppc_emulate_debug_inst
- do_nothing
- kvmppc_read_dpdes
- kvmppc_emulate_doorbell_instr
- kvmppc_handle_exit_hv
- kvmppc_handle_nested_exit
- kvm_arch_vcpu_ioctl_get_sregs_hv
- kvm_arch_vcpu_ioctl_set_sregs_hv
- kvmppc_set_lpcr
- kvmppc_get_one_reg_hv
- kvmppc_set_one_reg_hv
- threads_per_vcore
- kvmppc_vcore_create
- debugfs_timings_open
- debugfs_timings_release
- debugfs_timings_read
- debugfs_timings_write
- debugfs_vcpu_init
- debugfs_vcpu_init
- kvmppc_core_vcpu_create_hv
- kvmhv_set_smt_mode
- unpin_vpa
- kvmppc_core_vcpu_free_hv
- kvmppc_core_check_requests_hv
- kvmppc_set_timer
- kvmppc_end_cede
- kvmppc_remove_runnable
- kvmppc_grab_hwthread
- kvmppc_release_hwthread
- radix_flush_cpu
- kvmppc_prepare_radix_vcpu
- kvmppc_start_thread
- kvmppc_wait_for_nap
- on_primary_thread
- init_vcore_lists
- kvmppc_vcore_preempt
- kvmppc_vcore_end_preempt
- init_core_info
- subcore_config_ok
- init_vcore_to_run
- can_dynamic_split
- can_piggyback
- prepare_threads
- collect_piggybacks
- recheck_signals_and_mmu
- post_guest_process
- kvmppc_clear_host_core
- kvmppc_set_host_core
- set_irq_happened
- kvmppc_run_core
- kvmhv_load_hv_regs_and_go
- kvmhv_p9_guest_entry
- kvmppc_wait_for_exec
- grow_halt_poll_ns
- shrink_halt_poll_ns
- xive_interrupt_pending
- xive_interrupt_pending
- kvmppc_vcpu_woken
- kvmppc_vcore_check_block
- kvmppc_vcore_blocked
- kvmhv_setup_mmu
- kvmppc_run_vcpu
- kvmhv_run_single_vcpu
- kvmppc_vcpu_run_hv
- kvmppc_add_seg_page_size
- kvm_vm_ioctl_get_smmu_info_hv
- kvm_vm_ioctl_get_dirty_log_hv
- kvmppc_core_free_memslot_hv
- kvmppc_core_create_memslot_hv
- kvmppc_core_prepare_memory_region_hv
- kvmppc_core_commit_memory_region_hv
- kvmppc_update_lpcr
- kvmppc_mmu_destroy_hv
- kvmppc_setup_partition_table
- kvmppc_hv_setup_htab_rma
- kvmppc_switch_mmu_to_hpt
- kvmppc_switch_mmu_to_radix
- kvmppc_alloc_host_rm_ops
- kvmppc_free_host_rm_ops
- kvmppc_core_init_vm_hv
- kvmppc_free_vcores
- kvmppc_core_destroy_vm_hv
- kvmppc_core_emulate_op_hv
- kvmppc_core_emulate_mtspr_hv
- kvmppc_core_emulate_mfspr_hv
- kvmppc_core_check_processor_compat_hv
- kvmppc_free_pimap
- kvmppc_alloc_pimap
- kvmppc_set_passthru_irq
- kvmppc_clr_passthru_irq
- kvmppc_irq_bypass_add_producer_hv
- kvmppc_irq_bypass_del_producer_hv
- kvm_arch_vm_ioctl_hv
- init_default_hcalls
- kvmhv_configure_mmu
- kvmhv_enable_nested
- kvmhv_load_from_eaddr
- kvmhv_store_to_eaddr
- kvm_init_subcore_bitmap
- kvmppc_radix_possible
- kvmppc_book3s_init_hv
- kvmppc_book3s_exit_hv
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 #include <linux/kvm_host.h>
19 #include <linux/kernel.h>
20 #include <linux/err.h>
21 #include <linux/slab.h>
22 #include <linux/preempt.h>
23 #include <linux/sched/signal.h>
24 #include <linux/sched/stat.h>
25 #include <linux/delay.h>
26 #include <linux/export.h>
27 #include <linux/fs.h>
28 #include <linux/anon_inodes.h>
29 #include <linux/cpu.h>
30 #include <linux/cpumask.h>
31 #include <linux/spinlock.h>
32 #include <linux/page-flags.h>
33 #include <linux/srcu.h>
34 #include <linux/miscdevice.h>
35 #include <linux/debugfs.h>
36 #include <linux/gfp.h>
37 #include <linux/vmalloc.h>
38 #include <linux/highmem.h>
39 #include <linux/hugetlb.h>
40 #include <linux/kvm_irqfd.h>
41 #include <linux/irqbypass.h>
42 #include <linux/module.h>
43 #include <linux/compiler.h>
44 #include <linux/of.h>
45
46 #include <asm/ftrace.h>
47 #include <asm/reg.h>
48 #include <asm/ppc-opcode.h>
49 #include <asm/asm-prototypes.h>
50 #include <asm/archrandom.h>
51 #include <asm/debug.h>
52 #include <asm/disassemble.h>
53 #include <asm/cputable.h>
54 #include <asm/cacheflush.h>
55 #include <linux/uaccess.h>
56 #include <asm/io.h>
57 #include <asm/kvm_ppc.h>
58 #include <asm/kvm_book3s.h>
59 #include <asm/mmu_context.h>
60 #include <asm/lppaca.h>
61 #include <asm/processor.h>
62 #include <asm/cputhreads.h>
63 #include <asm/page.h>
64 #include <asm/hvcall.h>
65 #include <asm/switch_to.h>
66 #include <asm/smp.h>
67 #include <asm/dbell.h>
68 #include <asm/hmi.h>
69 #include <asm/pnv-pci.h>
70 #include <asm/mmu.h>
71 #include <asm/opal.h>
72 #include <asm/xics.h>
73 #include <asm/xive.h>
74 #include <asm/hw_breakpoint.h>
75
76 #include "book3s.h"
77
78 #define CREATE_TRACE_POINTS
79 #include "trace_hv.h"
80
81
82
83
84
85
86 #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1)
87
88 #define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2)
89
90
91 #define TB_NIL (~(u64)0)
92
93 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
94
95 static int dynamic_mt_modes = 6;
96 module_param(dynamic_mt_modes, int, 0644);
97 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
98 static int target_smt_mode;
99 module_param(target_smt_mode, int, 0644);
100 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
101
102 static bool indep_threads_mode = true;
103 module_param(indep_threads_mode, bool, S_IRUGO | S_IWUSR);
104 MODULE_PARM_DESC(indep_threads_mode, "Independent-threads mode (only on POWER9)");
105
106 static bool one_vm_per_core;
107 module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR);
108 MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires indep_threads_mode=N)");
109
110 #ifdef CONFIG_KVM_XICS
111 static struct kernel_param_ops module_param_ops = {
112 .set = param_set_int,
113 .get = param_get_int,
114 };
115
116 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
117 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
118
119 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
120 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
121 #endif
122
123
124 static bool nested = true;
125 module_param(nested, bool, S_IRUGO | S_IWUSR);
126 MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)");
127
128 static inline bool nesting_enabled(struct kvm *kvm)
129 {
130 return kvm->arch.nested_enable && kvm_is_radix(kvm);
131 }
132
133
134 static bool no_mixing_hpt_and_radix;
135
136 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
137 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
138
139
140
141
142
143
144 #define RWMR_RPA_P8_1THREAD 0x164520C62609AECAUL
145 #define RWMR_RPA_P8_2THREAD 0x7FFF2908450D8DA9UL
146 #define RWMR_RPA_P8_3THREAD 0x164520C62609AECAUL
147 #define RWMR_RPA_P8_4THREAD 0x199A421245058DA9UL
148 #define RWMR_RPA_P8_5THREAD 0x164520C62609AECAUL
149 #define RWMR_RPA_P8_6THREAD 0x164520C62609AECAUL
150 #define RWMR_RPA_P8_7THREAD 0x164520C62609AECAUL
151 #define RWMR_RPA_P8_8THREAD 0x164520C62609AECAUL
152
153 static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
154 RWMR_RPA_P8_1THREAD,
155 RWMR_RPA_P8_1THREAD,
156 RWMR_RPA_P8_2THREAD,
157 RWMR_RPA_P8_3THREAD,
158 RWMR_RPA_P8_4THREAD,
159 RWMR_RPA_P8_5THREAD,
160 RWMR_RPA_P8_6THREAD,
161 RWMR_RPA_P8_7THREAD,
162 RWMR_RPA_P8_8THREAD,
163 };
164
165 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
166 int *ip)
167 {
168 int i = *ip;
169 struct kvm_vcpu *vcpu;
170
171 while (++i < MAX_SMT_THREADS) {
172 vcpu = READ_ONCE(vc->runnable_threads[i]);
173 if (vcpu) {
174 *ip = i;
175 return vcpu;
176 }
177 }
178 return NULL;
179 }
180
181
182 #define for_each_runnable_thread(i, vcpu, vc) \
183 for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
184
185 static bool kvmppc_ipi_thread(int cpu)
186 {
187 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
188
189
190 if (kvmhv_on_pseries())
191 return false;
192
193
194 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
195 msg |= get_hard_smp_processor_id(cpu);
196 smp_mb();
197 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
198 return true;
199 }
200
201
202 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
203 preempt_disable();
204 if (cpu_first_thread_sibling(cpu) ==
205 cpu_first_thread_sibling(smp_processor_id())) {
206 msg |= cpu_thread_in_core(cpu);
207 smp_mb();
208 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
209 preempt_enable();
210 return true;
211 }
212 preempt_enable();
213 }
214
215 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
216 if (cpu >= 0 && cpu < nr_cpu_ids) {
217 if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
218 xics_wake_cpu(cpu);
219 return true;
220 }
221 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
222 return true;
223 }
224 #endif
225
226 return false;
227 }
228
229 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
230 {
231 int cpu;
232 struct swait_queue_head *wqp;
233
234 wqp = kvm_arch_vcpu_wq(vcpu);
235 if (swq_has_sleeper(wqp)) {
236 swake_up_one(wqp);
237 ++vcpu->stat.halt_wakeup;
238 }
239
240 cpu = READ_ONCE(vcpu->arch.thread_cpu);
241 if (cpu >= 0 && kvmppc_ipi_thread(cpu))
242 return;
243
244
245 cpu = vcpu->cpu;
246 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
247 smp_send_reschedule(cpu);
248 }
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
284 {
285 unsigned long flags;
286
287 spin_lock_irqsave(&vc->stoltb_lock, flags);
288 vc->preempt_tb = mftb();
289 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
290 }
291
292 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
293 {
294 unsigned long flags;
295
296 spin_lock_irqsave(&vc->stoltb_lock, flags);
297 if (vc->preempt_tb != TB_NIL) {
298 vc->stolen_tb += mftb() - vc->preempt_tb;
299 vc->preempt_tb = TB_NIL;
300 }
301 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
302 }
303
304 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
305 {
306 struct kvmppc_vcore *vc = vcpu->arch.vcore;
307 unsigned long flags;
308
309
310
311
312
313
314
315 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
316 kvmppc_core_end_stolen(vc);
317
318 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
319 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
320 vcpu->arch.busy_preempt != TB_NIL) {
321 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
322 vcpu->arch.busy_preempt = TB_NIL;
323 }
324 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
325 }
326
327 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
328 {
329 struct kvmppc_vcore *vc = vcpu->arch.vcore;
330 unsigned long flags;
331
332 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
333 kvmppc_core_start_stolen(vc);
334
335 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
336 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
337 vcpu->arch.busy_preempt = mftb();
338 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
339 }
340
341 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
342 {
343
344
345
346
347 if ((msr & MSR_TS_MASK) == MSR_TS_MASK)
348 msr &= ~MSR_TS_MASK;
349 vcpu->arch.shregs.msr = msr;
350 kvmppc_end_cede(vcpu);
351 }
352
353 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
354 {
355 vcpu->arch.pvr = pvr;
356 }
357
358
359 #define PCR_ARCH_300 (PCR_ARCH_207 << 1)
360
361 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
362 {
363 unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
364 struct kvmppc_vcore *vc = vcpu->arch.vcore;
365
366
367 if (cpu_has_feature(CPU_FTR_ARCH_300))
368 host_pcr_bit = PCR_ARCH_300;
369 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
370 host_pcr_bit = PCR_ARCH_207;
371 else if (cpu_has_feature(CPU_FTR_ARCH_206))
372 host_pcr_bit = PCR_ARCH_206;
373 else
374 host_pcr_bit = PCR_ARCH_205;
375
376
377 guest_pcr_bit = host_pcr_bit;
378 if (arch_compat) {
379 switch (arch_compat) {
380 case PVR_ARCH_205:
381 guest_pcr_bit = PCR_ARCH_205;
382 break;
383 case PVR_ARCH_206:
384 case PVR_ARCH_206p:
385 guest_pcr_bit = PCR_ARCH_206;
386 break;
387 case PVR_ARCH_207:
388 guest_pcr_bit = PCR_ARCH_207;
389 break;
390 case PVR_ARCH_300:
391 guest_pcr_bit = PCR_ARCH_300;
392 break;
393 default:
394 return -EINVAL;
395 }
396 }
397
398
399 if (guest_pcr_bit > host_pcr_bit)
400 return -EINVAL;
401
402 spin_lock(&vc->lock);
403 vc->arch_compat = arch_compat;
404
405
406
407
408 vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK;
409 spin_unlock(&vc->lock);
410
411 return 0;
412 }
413
414 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
415 {
416 int r;
417
418 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
419 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
420 vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
421 for (r = 0; r < 16; ++r)
422 pr_err("r%2d = %.16lx r%d = %.16lx\n",
423 r, kvmppc_get_gpr(vcpu, r),
424 r+16, kvmppc_get_gpr(vcpu, r+16));
425 pr_err("ctr = %.16lx lr = %.16lx\n",
426 vcpu->arch.regs.ctr, vcpu->arch.regs.link);
427 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
428 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
429 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
430 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
431 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
432 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
433 pr_err("cr = %.8lx xer = %.16lx dsisr = %.8x\n",
434 vcpu->arch.regs.ccr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
435 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
436 pr_err("fault dar = %.16lx dsisr = %.8x\n",
437 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
438 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
439 for (r = 0; r < vcpu->arch.slb_max; ++r)
440 pr_err(" ESID = %.16llx VSID = %.16llx\n",
441 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
442 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
443 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
444 vcpu->arch.last_inst);
445 }
446
447 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
448 {
449 return kvm_get_vcpu_by_id(kvm, id);
450 }
451
452 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
453 {
454 vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
455 vpa->yield_count = cpu_to_be32(1);
456 }
457
458 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
459 unsigned long addr, unsigned long len)
460 {
461
462 if (addr & (L1_CACHE_BYTES - 1))
463 return -EINVAL;
464 spin_lock(&vcpu->arch.vpa_update_lock);
465 if (v->next_gpa != addr || v->len != len) {
466 v->next_gpa = addr;
467 v->len = addr ? len : 0;
468 v->update_pending = 1;
469 }
470 spin_unlock(&vcpu->arch.vpa_update_lock);
471 return 0;
472 }
473
474
475 struct reg_vpa {
476 u32 dummy;
477 union {
478 __be16 hword;
479 __be32 word;
480 } length;
481 };
482
483 static int vpa_is_registered(struct kvmppc_vpa *vpap)
484 {
485 if (vpap->update_pending)
486 return vpap->next_gpa != 0;
487 return vpap->pinned_addr != NULL;
488 }
489
490 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
491 unsigned long flags,
492 unsigned long vcpuid, unsigned long vpa)
493 {
494 struct kvm *kvm = vcpu->kvm;
495 unsigned long len, nb;
496 void *va;
497 struct kvm_vcpu *tvcpu;
498 int err;
499 int subfunc;
500 struct kvmppc_vpa *vpap;
501
502 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
503 if (!tvcpu)
504 return H_PARAMETER;
505
506 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
507 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
508 subfunc == H_VPA_REG_SLB) {
509
510 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
511 return H_PARAMETER;
512
513
514 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
515 if (va == NULL)
516 return H_PARAMETER;
517 if (subfunc == H_VPA_REG_VPA)
518 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
519 else
520 len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
521 kvmppc_unpin_guest_page(kvm, va, vpa, false);
522
523
524 if (len > nb || len < sizeof(struct reg_vpa))
525 return H_PARAMETER;
526 } else {
527 vpa = 0;
528 len = 0;
529 }
530
531 err = H_PARAMETER;
532 vpap = NULL;
533 spin_lock(&tvcpu->arch.vpa_update_lock);
534
535 switch (subfunc) {
536 case H_VPA_REG_VPA:
537
538
539
540
541
542
543 BUILD_BUG_ON(sizeof(struct lppaca) != 640);
544 if (len < sizeof(struct lppaca))
545 break;
546 vpap = &tvcpu->arch.vpa;
547 err = 0;
548 break;
549
550 case H_VPA_REG_DTL:
551 if (len < sizeof(struct dtl_entry))
552 break;
553 len -= len % sizeof(struct dtl_entry);
554
555
556 err = H_RESOURCE;
557 if (!vpa_is_registered(&tvcpu->arch.vpa))
558 break;
559
560 vpap = &tvcpu->arch.dtl;
561 err = 0;
562 break;
563
564 case H_VPA_REG_SLB:
565
566 err = H_RESOURCE;
567 if (!vpa_is_registered(&tvcpu->arch.vpa))
568 break;
569
570 vpap = &tvcpu->arch.slb_shadow;
571 err = 0;
572 break;
573
574 case H_VPA_DEREG_VPA:
575
576 err = H_RESOURCE;
577 if (vpa_is_registered(&tvcpu->arch.dtl) ||
578 vpa_is_registered(&tvcpu->arch.slb_shadow))
579 break;
580
581 vpap = &tvcpu->arch.vpa;
582 err = 0;
583 break;
584
585 case H_VPA_DEREG_DTL:
586 vpap = &tvcpu->arch.dtl;
587 err = 0;
588 break;
589
590 case H_VPA_DEREG_SLB:
591 vpap = &tvcpu->arch.slb_shadow;
592 err = 0;
593 break;
594 }
595
596 if (vpap) {
597 vpap->next_gpa = vpa;
598 vpap->len = len;
599 vpap->update_pending = 1;
600 }
601
602 spin_unlock(&tvcpu->arch.vpa_update_lock);
603
604 return err;
605 }
606
607 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
608 {
609 struct kvm *kvm = vcpu->kvm;
610 void *va;
611 unsigned long nb;
612 unsigned long gpa;
613
614
615
616
617
618
619
620
621
622 for (;;) {
623 gpa = vpap->next_gpa;
624 spin_unlock(&vcpu->arch.vpa_update_lock);
625 va = NULL;
626 nb = 0;
627 if (gpa)
628 va = kvmppc_pin_guest_page(kvm, gpa, &nb);
629 spin_lock(&vcpu->arch.vpa_update_lock);
630 if (gpa == vpap->next_gpa)
631 break;
632
633 if (va)
634 kvmppc_unpin_guest_page(kvm, va, gpa, false);
635 }
636
637 vpap->update_pending = 0;
638 if (va && nb < vpap->len) {
639
640
641
642
643
644 kvmppc_unpin_guest_page(kvm, va, gpa, false);
645 va = NULL;
646 }
647 if (vpap->pinned_addr)
648 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
649 vpap->dirty);
650 vpap->gpa = gpa;
651 vpap->pinned_addr = va;
652 vpap->dirty = false;
653 if (va)
654 vpap->pinned_end = va + vpap->len;
655 }
656
657 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
658 {
659 if (!(vcpu->arch.vpa.update_pending ||
660 vcpu->arch.slb_shadow.update_pending ||
661 vcpu->arch.dtl.update_pending))
662 return;
663
664 spin_lock(&vcpu->arch.vpa_update_lock);
665 if (vcpu->arch.vpa.update_pending) {
666 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
667 if (vcpu->arch.vpa.pinned_addr)
668 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
669 }
670 if (vcpu->arch.dtl.update_pending) {
671 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
672 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
673 vcpu->arch.dtl_index = 0;
674 }
675 if (vcpu->arch.slb_shadow.update_pending)
676 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
677 spin_unlock(&vcpu->arch.vpa_update_lock);
678 }
679
680
681
682
683
684 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
685 {
686 u64 p;
687 unsigned long flags;
688
689 spin_lock_irqsave(&vc->stoltb_lock, flags);
690 p = vc->stolen_tb;
691 if (vc->vcore_state != VCORE_INACTIVE &&
692 vc->preempt_tb != TB_NIL)
693 p += now - vc->preempt_tb;
694 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
695 return p;
696 }
697
698 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
699 struct kvmppc_vcore *vc)
700 {
701 struct dtl_entry *dt;
702 struct lppaca *vpa;
703 unsigned long stolen;
704 unsigned long core_stolen;
705 u64 now;
706 unsigned long flags;
707
708 dt = vcpu->arch.dtl_ptr;
709 vpa = vcpu->arch.vpa.pinned_addr;
710 now = mftb();
711 core_stolen = vcore_stolen_time(vc, now);
712 stolen = core_stolen - vcpu->arch.stolen_logged;
713 vcpu->arch.stolen_logged = core_stolen;
714 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
715 stolen += vcpu->arch.busy_stolen;
716 vcpu->arch.busy_stolen = 0;
717 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
718 if (!dt || !vpa)
719 return;
720 memset(dt, 0, sizeof(struct dtl_entry));
721 dt->dispatch_reason = 7;
722 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
723 dt->timebase = cpu_to_be64(now + vc->tb_offset);
724 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
725 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
726 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
727 ++dt;
728 if (dt == vcpu->arch.dtl.pinned_end)
729 dt = vcpu->arch.dtl.pinned_addr;
730 vcpu->arch.dtl_ptr = dt;
731
732 smp_wmb();
733 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
734 vcpu->arch.dtl.dirty = true;
735 }
736
737
738 static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
739 {
740 int thr;
741 struct kvmppc_vcore *vc;
742
743 if (vcpu->arch.doorbell_request)
744 return true;
745
746
747
748
749
750 smp_rmb();
751 vc = vcpu->arch.vcore;
752 thr = vcpu->vcpu_id - vc->first_vcpuid;
753 return !!(vc->dpdes & (1 << thr));
754 }
755
756 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
757 {
758 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
759 return true;
760 if ((!vcpu->arch.vcore->arch_compat) &&
761 cpu_has_feature(CPU_FTR_ARCH_207S))
762 return true;
763 return false;
764 }
765
766 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
767 unsigned long resource, unsigned long value1,
768 unsigned long value2)
769 {
770 switch (resource) {
771 case H_SET_MODE_RESOURCE_SET_CIABR:
772 if (!kvmppc_power8_compatible(vcpu))
773 return H_P2;
774 if (value2)
775 return H_P4;
776 if (mflags)
777 return H_UNSUPPORTED_FLAG_START;
778
779 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
780 return H_P3;
781 vcpu->arch.ciabr = value1;
782 return H_SUCCESS;
783 case H_SET_MODE_RESOURCE_SET_DAWR:
784 if (!kvmppc_power8_compatible(vcpu))
785 return H_P2;
786 if (!ppc_breakpoint_available())
787 return H_P2;
788 if (mflags)
789 return H_UNSUPPORTED_FLAG_START;
790 if (value2 & DABRX_HYP)
791 return H_P4;
792 vcpu->arch.dawr = value1;
793 vcpu->arch.dawrx = value2;
794 return H_SUCCESS;
795 default:
796 return H_TOO_HARD;
797 }
798 }
799
800
801 static int kvmppc_copy_guest(struct kvm *kvm, gpa_t to, gpa_t from,
802 unsigned long len)
803 {
804 struct kvm_memory_slot *to_memslot = NULL;
805 struct kvm_memory_slot *from_memslot = NULL;
806 unsigned long to_addr, from_addr;
807 int r;
808
809
810 from_memslot = gfn_to_memslot(kvm, from >> PAGE_SHIFT);
811 if (!from_memslot)
812 return -EFAULT;
813 if ((from + len) >= ((from_memslot->base_gfn + from_memslot->npages)
814 << PAGE_SHIFT))
815 return -EINVAL;
816 from_addr = gfn_to_hva_memslot(from_memslot, from >> PAGE_SHIFT);
817 if (kvm_is_error_hva(from_addr))
818 return -EFAULT;
819 from_addr |= (from & (PAGE_SIZE - 1));
820
821
822 to_memslot = gfn_to_memslot(kvm, to >> PAGE_SHIFT);
823 if (!to_memslot)
824 return -EFAULT;
825 if ((to + len) >= ((to_memslot->base_gfn + to_memslot->npages)
826 << PAGE_SHIFT))
827 return -EINVAL;
828 to_addr = gfn_to_hva_memslot(to_memslot, to >> PAGE_SHIFT);
829 if (kvm_is_error_hva(to_addr))
830 return -EFAULT;
831 to_addr |= (to & (PAGE_SIZE - 1));
832
833
834 r = raw_copy_in_user((void __user *)to_addr, (void __user *)from_addr,
835 len);
836 if (r)
837 return -EFAULT;
838 mark_page_dirty(kvm, to >> PAGE_SHIFT);
839 return 0;
840 }
841
842 static long kvmppc_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags,
843 unsigned long dest, unsigned long src)
844 {
845 u64 pg_sz = SZ_4K;
846 u64 pg_mask = SZ_4K - 1;
847 int ret;
848
849
850 if (flags & ~(H_ICACHE_INVALIDATE | H_ICACHE_SYNCHRONIZE |
851 H_ZERO_PAGE | H_COPY_PAGE | H_PAGE_SET_LOANED))
852 return H_PARAMETER;
853
854
855 if ((dest & pg_mask) || ((flags & H_COPY_PAGE) && (src & pg_mask)))
856 return H_PARAMETER;
857
858
859 if (flags & H_COPY_PAGE) {
860 ret = kvmppc_copy_guest(vcpu->kvm, dest, src, pg_sz);
861 if (ret < 0)
862 return H_PARAMETER;
863 } else if (flags & H_ZERO_PAGE) {
864 ret = kvm_clear_guest(vcpu->kvm, dest, pg_sz);
865 if (ret < 0)
866 return H_PARAMETER;
867 }
868
869
870
871 return H_SUCCESS;
872 }
873
874 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
875 {
876 struct kvmppc_vcore *vcore = target->arch.vcore;
877
878
879
880
881
882
883
884
885
886 spin_lock(&vcore->lock);
887 if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
888 vcore->vcore_state != VCORE_INACTIVE &&
889 vcore->runner)
890 target = vcore->runner;
891 spin_unlock(&vcore->lock);
892
893 return kvm_vcpu_yield_to(target);
894 }
895
896 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
897 {
898 int yield_count = 0;
899 struct lppaca *lppaca;
900
901 spin_lock(&vcpu->arch.vpa_update_lock);
902 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
903 if (lppaca)
904 yield_count = be32_to_cpu(lppaca->yield_count);
905 spin_unlock(&vcpu->arch.vpa_update_lock);
906 return yield_count;
907 }
908
909 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
910 {
911 unsigned long req = kvmppc_get_gpr(vcpu, 3);
912 unsigned long target, ret = H_SUCCESS;
913 int yield_count;
914 struct kvm_vcpu *tvcpu;
915 int idx, rc;
916
917 if (req <= MAX_HCALL_OPCODE &&
918 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
919 return RESUME_HOST;
920
921 switch (req) {
922 case H_CEDE:
923 break;
924 case H_PROD:
925 target = kvmppc_get_gpr(vcpu, 4);
926 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
927 if (!tvcpu) {
928 ret = H_PARAMETER;
929 break;
930 }
931 tvcpu->arch.prodded = 1;
932 smp_mb();
933 if (tvcpu->arch.ceded)
934 kvmppc_fast_vcpu_kick_hv(tvcpu);
935 break;
936 case H_CONFER:
937 target = kvmppc_get_gpr(vcpu, 4);
938 if (target == -1)
939 break;
940 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
941 if (!tvcpu) {
942 ret = H_PARAMETER;
943 break;
944 }
945 yield_count = kvmppc_get_gpr(vcpu, 5);
946 if (kvmppc_get_yield_count(tvcpu) != yield_count)
947 break;
948 kvm_arch_vcpu_yield_to(tvcpu);
949 break;
950 case H_REGISTER_VPA:
951 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
952 kvmppc_get_gpr(vcpu, 5),
953 kvmppc_get_gpr(vcpu, 6));
954 break;
955 case H_RTAS:
956 if (list_empty(&vcpu->kvm->arch.rtas_tokens))
957 return RESUME_HOST;
958
959 idx = srcu_read_lock(&vcpu->kvm->srcu);
960 rc = kvmppc_rtas_hcall(vcpu);
961 srcu_read_unlock(&vcpu->kvm->srcu, idx);
962
963 if (rc == -ENOENT)
964 return RESUME_HOST;
965 else if (rc == 0)
966 break;
967
968
969 return rc;
970 case H_LOGICAL_CI_LOAD:
971 ret = kvmppc_h_logical_ci_load(vcpu);
972 if (ret == H_TOO_HARD)
973 return RESUME_HOST;
974 break;
975 case H_LOGICAL_CI_STORE:
976 ret = kvmppc_h_logical_ci_store(vcpu);
977 if (ret == H_TOO_HARD)
978 return RESUME_HOST;
979 break;
980 case H_SET_MODE:
981 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
982 kvmppc_get_gpr(vcpu, 5),
983 kvmppc_get_gpr(vcpu, 6),
984 kvmppc_get_gpr(vcpu, 7));
985 if (ret == H_TOO_HARD)
986 return RESUME_HOST;
987 break;
988 case H_XIRR:
989 case H_CPPR:
990 case H_EOI:
991 case H_IPI:
992 case H_IPOLL:
993 case H_XIRR_X:
994 if (kvmppc_xics_enabled(vcpu)) {
995 if (xics_on_xive()) {
996 ret = H_NOT_AVAILABLE;
997 return RESUME_GUEST;
998 }
999 ret = kvmppc_xics_hcall(vcpu, req);
1000 break;
1001 }
1002 return RESUME_HOST;
1003 case H_SET_DABR:
1004 ret = kvmppc_h_set_dabr(vcpu, kvmppc_get_gpr(vcpu, 4));
1005 break;
1006 case H_SET_XDABR:
1007 ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4),
1008 kvmppc_get_gpr(vcpu, 5));
1009 break;
1010 #ifdef CONFIG_SPAPR_TCE_IOMMU
1011 case H_GET_TCE:
1012 ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1013 kvmppc_get_gpr(vcpu, 5));
1014 if (ret == H_TOO_HARD)
1015 return RESUME_HOST;
1016 break;
1017 case H_PUT_TCE:
1018 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1019 kvmppc_get_gpr(vcpu, 5),
1020 kvmppc_get_gpr(vcpu, 6));
1021 if (ret == H_TOO_HARD)
1022 return RESUME_HOST;
1023 break;
1024 case H_PUT_TCE_INDIRECT:
1025 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
1026 kvmppc_get_gpr(vcpu, 5),
1027 kvmppc_get_gpr(vcpu, 6),
1028 kvmppc_get_gpr(vcpu, 7));
1029 if (ret == H_TOO_HARD)
1030 return RESUME_HOST;
1031 break;
1032 case H_STUFF_TCE:
1033 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1034 kvmppc_get_gpr(vcpu, 5),
1035 kvmppc_get_gpr(vcpu, 6),
1036 kvmppc_get_gpr(vcpu, 7));
1037 if (ret == H_TOO_HARD)
1038 return RESUME_HOST;
1039 break;
1040 #endif
1041 case H_RANDOM:
1042 if (!powernv_get_random_long(&vcpu->arch.regs.gpr[4]))
1043 ret = H_HARDWARE;
1044 break;
1045
1046 case H_SET_PARTITION_TABLE:
1047 ret = H_FUNCTION;
1048 if (nesting_enabled(vcpu->kvm))
1049 ret = kvmhv_set_partition_table(vcpu);
1050 break;
1051 case H_ENTER_NESTED:
1052 ret = H_FUNCTION;
1053 if (!nesting_enabled(vcpu->kvm))
1054 break;
1055 ret = kvmhv_enter_nested_guest(vcpu);
1056 if (ret == H_INTERRUPT) {
1057 kvmppc_set_gpr(vcpu, 3, 0);
1058 vcpu->arch.hcall_needed = 0;
1059 return -EINTR;
1060 } else if (ret == H_TOO_HARD) {
1061 kvmppc_set_gpr(vcpu, 3, 0);
1062 vcpu->arch.hcall_needed = 0;
1063 return RESUME_HOST;
1064 }
1065 break;
1066 case H_TLB_INVALIDATE:
1067 ret = H_FUNCTION;
1068 if (nesting_enabled(vcpu->kvm))
1069 ret = kvmhv_do_nested_tlbie(vcpu);
1070 break;
1071 case H_COPY_TOFROM_GUEST:
1072 ret = H_FUNCTION;
1073 if (nesting_enabled(vcpu->kvm))
1074 ret = kvmhv_copy_tofrom_guest_nested(vcpu);
1075 break;
1076 case H_PAGE_INIT:
1077 ret = kvmppc_h_page_init(vcpu, kvmppc_get_gpr(vcpu, 4),
1078 kvmppc_get_gpr(vcpu, 5),
1079 kvmppc_get_gpr(vcpu, 6));
1080 break;
1081 default:
1082 return RESUME_HOST;
1083 }
1084 kvmppc_set_gpr(vcpu, 3, ret);
1085 vcpu->arch.hcall_needed = 0;
1086 return RESUME_GUEST;
1087 }
1088
1089
1090
1091
1092
1093
1094
1095 static void kvmppc_nested_cede(struct kvm_vcpu *vcpu)
1096 {
1097 vcpu->arch.shregs.msr |= MSR_EE;
1098 vcpu->arch.ceded = 1;
1099 smp_mb();
1100 if (vcpu->arch.prodded) {
1101 vcpu->arch.prodded = 0;
1102 smp_mb();
1103 vcpu->arch.ceded = 0;
1104 }
1105 }
1106
1107 static int kvmppc_hcall_impl_hv(unsigned long cmd)
1108 {
1109 switch (cmd) {
1110 case H_CEDE:
1111 case H_PROD:
1112 case H_CONFER:
1113 case H_REGISTER_VPA:
1114 case H_SET_MODE:
1115 case H_LOGICAL_CI_LOAD:
1116 case H_LOGICAL_CI_STORE:
1117 #ifdef CONFIG_KVM_XICS
1118 case H_XIRR:
1119 case H_CPPR:
1120 case H_EOI:
1121 case H_IPI:
1122 case H_IPOLL:
1123 case H_XIRR_X:
1124 #endif
1125 case H_PAGE_INIT:
1126 return 1;
1127 }
1128
1129
1130 return kvmppc_hcall_impl_hv_realmode(cmd);
1131 }
1132
1133 static int kvmppc_emulate_debug_inst(struct kvm_run *run,
1134 struct kvm_vcpu *vcpu)
1135 {
1136 u32 last_inst;
1137
1138 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
1139 EMULATE_DONE) {
1140
1141
1142
1143
1144 return RESUME_GUEST;
1145 }
1146
1147 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
1148 run->exit_reason = KVM_EXIT_DEBUG;
1149 run->debug.arch.address = kvmppc_get_pc(vcpu);
1150 return RESUME_HOST;
1151 } else {
1152 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1153 return RESUME_GUEST;
1154 }
1155 }
1156
1157 static void do_nothing(void *x)
1158 {
1159 }
1160
1161 static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
1162 {
1163 int thr, cpu, pcpu, nthreads;
1164 struct kvm_vcpu *v;
1165 unsigned long dpdes;
1166
1167 nthreads = vcpu->kvm->arch.emul_smt_mode;
1168 dpdes = 0;
1169 cpu = vcpu->vcpu_id & ~(nthreads - 1);
1170 for (thr = 0; thr < nthreads; ++thr, ++cpu) {
1171 v = kvmppc_find_vcpu(vcpu->kvm, cpu);
1172 if (!v)
1173 continue;
1174
1175
1176
1177
1178
1179 pcpu = READ_ONCE(v->cpu);
1180 if (pcpu >= 0)
1181 smp_call_function_single(pcpu, do_nothing, NULL, 1);
1182 if (kvmppc_doorbell_pending(v))
1183 dpdes |= 1 << thr;
1184 }
1185 return dpdes;
1186 }
1187
1188
1189
1190
1191
1192
1193
1194 static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
1195 {
1196 u32 inst, rb, thr;
1197 unsigned long arg;
1198 struct kvm *kvm = vcpu->kvm;
1199 struct kvm_vcpu *tvcpu;
1200
1201 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1202 return RESUME_GUEST;
1203 if (get_op(inst) != 31)
1204 return EMULATE_FAIL;
1205 rb = get_rb(inst);
1206 thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1207 switch (get_xop(inst)) {
1208 case OP_31_XOP_MSGSNDP:
1209 arg = kvmppc_get_gpr(vcpu, rb);
1210 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1211 break;
1212 arg &= 0x3f;
1213 if (arg >= kvm->arch.emul_smt_mode)
1214 break;
1215 tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1216 if (!tvcpu)
1217 break;
1218 if (!tvcpu->arch.doorbell_request) {
1219 tvcpu->arch.doorbell_request = 1;
1220 kvmppc_fast_vcpu_kick_hv(tvcpu);
1221 }
1222 break;
1223 case OP_31_XOP_MSGCLRP:
1224 arg = kvmppc_get_gpr(vcpu, rb);
1225 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1226 break;
1227 vcpu->arch.vcore->dpdes = 0;
1228 vcpu->arch.doorbell_request = 0;
1229 break;
1230 case OP_31_XOP_MFSPR:
1231 switch (get_sprn(inst)) {
1232 case SPRN_TIR:
1233 arg = thr;
1234 break;
1235 case SPRN_DPDES:
1236 arg = kvmppc_read_dpdes(vcpu);
1237 break;
1238 default:
1239 return EMULATE_FAIL;
1240 }
1241 kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1242 break;
1243 default:
1244 return EMULATE_FAIL;
1245 }
1246 kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1247 return RESUME_GUEST;
1248 }
1249
1250 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
1251 struct task_struct *tsk)
1252 {
1253 int r = RESUME_HOST;
1254
1255 vcpu->stat.sum_exits++;
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265 if (vcpu->arch.shregs.msr & MSR_HV) {
1266 printk(KERN_EMERG "KVM trap in HV mode!\n");
1267 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1268 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1269 vcpu->arch.shregs.msr);
1270 kvmppc_dump_regs(vcpu);
1271 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1272 run->hw.hardware_exit_reason = vcpu->arch.trap;
1273 return RESUME_HOST;
1274 }
1275 run->exit_reason = KVM_EXIT_UNKNOWN;
1276 run->ready_for_interrupt_injection = 1;
1277 switch (vcpu->arch.trap) {
1278
1279 case BOOK3S_INTERRUPT_HV_DECREMENTER:
1280 vcpu->stat.dec_exits++;
1281 r = RESUME_GUEST;
1282 break;
1283 case BOOK3S_INTERRUPT_EXTERNAL:
1284 case BOOK3S_INTERRUPT_H_DOORBELL:
1285 case BOOK3S_INTERRUPT_H_VIRT:
1286 vcpu->stat.ext_intr_exits++;
1287 r = RESUME_GUEST;
1288 break;
1289
1290 case BOOK3S_INTERRUPT_HMI:
1291 case BOOK3S_INTERRUPT_PERFMON:
1292 case BOOK3S_INTERRUPT_SYSTEM_RESET:
1293 r = RESUME_GUEST;
1294 break;
1295 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1296
1297 machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
1298
1299
1300
1301
1302
1303
1304
1305 if (!vcpu->kvm->arch.fwnmi_enabled) {
1306 ulong flags = vcpu->arch.shregs.msr & 0x083c0000;
1307 kvmppc_core_queue_machine_check(vcpu, flags);
1308 r = RESUME_GUEST;
1309 break;
1310 }
1311
1312
1313 run->exit_reason = KVM_EXIT_NMI;
1314 run->hw.hardware_exit_reason = vcpu->arch.trap;
1315
1316 run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1317
1318 if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1319 run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1320 else
1321 run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1322
1323 r = RESUME_HOST;
1324 break;
1325 case BOOK3S_INTERRUPT_PROGRAM:
1326 {
1327 ulong flags;
1328
1329
1330
1331
1332
1333
1334 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1335 kvmppc_core_queue_program(vcpu, flags);
1336 r = RESUME_GUEST;
1337 break;
1338 }
1339 case BOOK3S_INTERRUPT_SYSCALL:
1340 {
1341
1342 int i;
1343
1344
1345
1346
1347
1348 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1349 for (i = 0; i < 9; ++i)
1350 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1351 run->exit_reason = KVM_EXIT_PAPR_HCALL;
1352 vcpu->arch.hcall_needed = 1;
1353 r = RESUME_HOST;
1354 break;
1355 }
1356
1357
1358
1359
1360
1361
1362
1363 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1364 r = RESUME_PAGE_FAULT;
1365 break;
1366 case BOOK3S_INTERRUPT_H_INST_STORAGE:
1367 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1368 vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr &
1369 DSISR_SRR1_MATCH_64S;
1370 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1371 vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1372 r = RESUME_PAGE_FAULT;
1373 break;
1374
1375
1376
1377
1378
1379
1380
1381 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1382 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1383 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1384 swab32(vcpu->arch.emul_inst) :
1385 vcpu->arch.emul_inst;
1386 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1387 r = kvmppc_emulate_debug_inst(run, vcpu);
1388 } else {
1389 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1390 r = RESUME_GUEST;
1391 }
1392 break;
1393
1394
1395
1396
1397
1398
1399
1400 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
1401 r = EMULATE_FAIL;
1402 if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
1403 cpu_has_feature(CPU_FTR_ARCH_300))
1404 r = kvmppc_emulate_doorbell_instr(vcpu);
1405 if (r == EMULATE_FAIL) {
1406 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1407 r = RESUME_GUEST;
1408 }
1409 break;
1410
1411 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1412 case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1413
1414
1415
1416
1417
1418
1419 r = kvmhv_p9_tm_emulation(vcpu);
1420 break;
1421 #endif
1422
1423 case BOOK3S_INTERRUPT_HV_RM_HARD:
1424 r = RESUME_PASSTHROUGH;
1425 break;
1426 default:
1427 kvmppc_dump_regs(vcpu);
1428 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1429 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1430 vcpu->arch.shregs.msr);
1431 run->hw.hardware_exit_reason = vcpu->arch.trap;
1432 r = RESUME_HOST;
1433 break;
1434 }
1435
1436 return r;
1437 }
1438
1439 static int kvmppc_handle_nested_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
1440 {
1441 int r;
1442 int srcu_idx;
1443
1444 vcpu->stat.sum_exits++;
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454 if (vcpu->arch.shregs.msr & MSR_HV) {
1455 pr_emerg("KVM trap in HV mode while nested!\n");
1456 pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1457 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1458 vcpu->arch.shregs.msr);
1459 kvmppc_dump_regs(vcpu);
1460 return RESUME_HOST;
1461 }
1462 switch (vcpu->arch.trap) {
1463
1464 case BOOK3S_INTERRUPT_HV_DECREMENTER:
1465 vcpu->stat.dec_exits++;
1466 r = RESUME_GUEST;
1467 break;
1468 case BOOK3S_INTERRUPT_EXTERNAL:
1469 vcpu->stat.ext_intr_exits++;
1470 r = RESUME_HOST;
1471 break;
1472 case BOOK3S_INTERRUPT_H_DOORBELL:
1473 case BOOK3S_INTERRUPT_H_VIRT:
1474 vcpu->stat.ext_intr_exits++;
1475 r = RESUME_GUEST;
1476 break;
1477
1478 case BOOK3S_INTERRUPT_HMI:
1479 case BOOK3S_INTERRUPT_PERFMON:
1480 case BOOK3S_INTERRUPT_SYSTEM_RESET:
1481 r = RESUME_GUEST;
1482 break;
1483 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1484
1485 r = RESUME_HOST;
1486
1487 machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
1488 break;
1489
1490
1491
1492
1493
1494
1495 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1496 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1497 r = kvmhv_nested_page_fault(run, vcpu);
1498 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1499 break;
1500 case BOOK3S_INTERRUPT_H_INST_STORAGE:
1501 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1502 vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) &
1503 DSISR_SRR1_MATCH_64S;
1504 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1505 vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1506 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1507 r = kvmhv_nested_page_fault(run, vcpu);
1508 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1509 break;
1510
1511 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1512 case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1513
1514
1515
1516
1517
1518
1519 r = kvmhv_p9_tm_emulation(vcpu);
1520 break;
1521 #endif
1522
1523 case BOOK3S_INTERRUPT_HV_RM_HARD:
1524 vcpu->arch.trap = 0;
1525 r = RESUME_GUEST;
1526 if (!xics_on_xive())
1527 kvmppc_xics_rm_complete(vcpu, 0);
1528 break;
1529 default:
1530 r = RESUME_HOST;
1531 break;
1532 }
1533
1534 return r;
1535 }
1536
1537 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
1538 struct kvm_sregs *sregs)
1539 {
1540 int i;
1541
1542 memset(sregs, 0, sizeof(struct kvm_sregs));
1543 sregs->pvr = vcpu->arch.pvr;
1544 for (i = 0; i < vcpu->arch.slb_max; i++) {
1545 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
1546 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1547 }
1548
1549 return 0;
1550 }
1551
1552 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
1553 struct kvm_sregs *sregs)
1554 {
1555 int i, j;
1556
1557
1558 if (sregs->pvr != vcpu->arch.pvr)
1559 return -EINVAL;
1560
1561 j = 0;
1562 for (i = 0; i < vcpu->arch.slb_nr; i++) {
1563 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
1564 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
1565 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
1566 ++j;
1567 }
1568 }
1569 vcpu->arch.slb_max = j;
1570
1571 return 0;
1572 }
1573
1574 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1575 bool preserve_top32)
1576 {
1577 struct kvm *kvm = vcpu->kvm;
1578 struct kvmppc_vcore *vc = vcpu->arch.vcore;
1579 u64 mask;
1580
1581 spin_lock(&vc->lock);
1582
1583
1584
1585
1586 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
1587 struct kvm_vcpu *vcpu;
1588 int i;
1589
1590 kvm_for_each_vcpu(i, vcpu, kvm) {
1591 if (vcpu->arch.vcore != vc)
1592 continue;
1593 if (new_lpcr & LPCR_ILE)
1594 vcpu->arch.intr_msr |= MSR_LE;
1595 else
1596 vcpu->arch.intr_msr &= ~MSR_LE;
1597 }
1598 }
1599
1600
1601
1602
1603
1604
1605 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
1606 if (cpu_has_feature(CPU_FTR_ARCH_207S))
1607 mask |= LPCR_AIL;
1608
1609
1610
1611
1612 if (cpu_has_feature(CPU_FTR_ARCH_300))
1613 mask |= LPCR_LD;
1614
1615
1616 if (preserve_top32)
1617 mask &= 0xFFFFFFFF;
1618 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
1619 spin_unlock(&vc->lock);
1620 }
1621
1622 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1623 union kvmppc_one_reg *val)
1624 {
1625 int r = 0;
1626 long int i;
1627
1628 switch (id) {
1629 case KVM_REG_PPC_DEBUG_INST:
1630 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
1631 break;
1632 case KVM_REG_PPC_HIOR:
1633 *val = get_reg_val(id, 0);
1634 break;
1635 case KVM_REG_PPC_DABR:
1636 *val = get_reg_val(id, vcpu->arch.dabr);
1637 break;
1638 case KVM_REG_PPC_DABRX:
1639 *val = get_reg_val(id, vcpu->arch.dabrx);
1640 break;
1641 case KVM_REG_PPC_DSCR:
1642 *val = get_reg_val(id, vcpu->arch.dscr);
1643 break;
1644 case KVM_REG_PPC_PURR:
1645 *val = get_reg_val(id, vcpu->arch.purr);
1646 break;
1647 case KVM_REG_PPC_SPURR:
1648 *val = get_reg_val(id, vcpu->arch.spurr);
1649 break;
1650 case KVM_REG_PPC_AMR:
1651 *val = get_reg_val(id, vcpu->arch.amr);
1652 break;
1653 case KVM_REG_PPC_UAMOR:
1654 *val = get_reg_val(id, vcpu->arch.uamor);
1655 break;
1656 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1657 i = id - KVM_REG_PPC_MMCR0;
1658 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
1659 break;
1660 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1661 i = id - KVM_REG_PPC_PMC1;
1662 *val = get_reg_val(id, vcpu->arch.pmc[i]);
1663 break;
1664 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1665 i = id - KVM_REG_PPC_SPMC1;
1666 *val = get_reg_val(id, vcpu->arch.spmc[i]);
1667 break;
1668 case KVM_REG_PPC_SIAR:
1669 *val = get_reg_val(id, vcpu->arch.siar);
1670 break;
1671 case KVM_REG_PPC_SDAR:
1672 *val = get_reg_val(id, vcpu->arch.sdar);
1673 break;
1674 case KVM_REG_PPC_SIER:
1675 *val = get_reg_val(id, vcpu->arch.sier);
1676 break;
1677 case KVM_REG_PPC_IAMR:
1678 *val = get_reg_val(id, vcpu->arch.iamr);
1679 break;
1680 case KVM_REG_PPC_PSPB:
1681 *val = get_reg_val(id, vcpu->arch.pspb);
1682 break;
1683 case KVM_REG_PPC_DPDES:
1684
1685
1686
1687
1688
1689
1690 *val = get_reg_val(id, vcpu->arch.vcore->dpdes |
1691 vcpu->arch.doorbell_request);
1692 break;
1693 case KVM_REG_PPC_VTB:
1694 *val = get_reg_val(id, vcpu->arch.vcore->vtb);
1695 break;
1696 case KVM_REG_PPC_DAWR:
1697 *val = get_reg_val(id, vcpu->arch.dawr);
1698 break;
1699 case KVM_REG_PPC_DAWRX:
1700 *val = get_reg_val(id, vcpu->arch.dawrx);
1701 break;
1702 case KVM_REG_PPC_CIABR:
1703 *val = get_reg_val(id, vcpu->arch.ciabr);
1704 break;
1705 case KVM_REG_PPC_CSIGR:
1706 *val = get_reg_val(id, vcpu->arch.csigr);
1707 break;
1708 case KVM_REG_PPC_TACR:
1709 *val = get_reg_val(id, vcpu->arch.tacr);
1710 break;
1711 case KVM_REG_PPC_TCSCR:
1712 *val = get_reg_val(id, vcpu->arch.tcscr);
1713 break;
1714 case KVM_REG_PPC_PID:
1715 *val = get_reg_val(id, vcpu->arch.pid);
1716 break;
1717 case KVM_REG_PPC_ACOP:
1718 *val = get_reg_val(id, vcpu->arch.acop);
1719 break;
1720 case KVM_REG_PPC_WORT:
1721 *val = get_reg_val(id, vcpu->arch.wort);
1722 break;
1723 case KVM_REG_PPC_TIDR:
1724 *val = get_reg_val(id, vcpu->arch.tid);
1725 break;
1726 case KVM_REG_PPC_PSSCR:
1727 *val = get_reg_val(id, vcpu->arch.psscr);
1728 break;
1729 case KVM_REG_PPC_VPA_ADDR:
1730 spin_lock(&vcpu->arch.vpa_update_lock);
1731 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1732 spin_unlock(&vcpu->arch.vpa_update_lock);
1733 break;
1734 case KVM_REG_PPC_VPA_SLB:
1735 spin_lock(&vcpu->arch.vpa_update_lock);
1736 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1737 val->vpaval.length = vcpu->arch.slb_shadow.len;
1738 spin_unlock(&vcpu->arch.vpa_update_lock);
1739 break;
1740 case KVM_REG_PPC_VPA_DTL:
1741 spin_lock(&vcpu->arch.vpa_update_lock);
1742 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1743 val->vpaval.length = vcpu->arch.dtl.len;
1744 spin_unlock(&vcpu->arch.vpa_update_lock);
1745 break;
1746 case KVM_REG_PPC_TB_OFFSET:
1747 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1748 break;
1749 case KVM_REG_PPC_LPCR:
1750 case KVM_REG_PPC_LPCR_64:
1751 *val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1752 break;
1753 case KVM_REG_PPC_PPR:
1754 *val = get_reg_val(id, vcpu->arch.ppr);
1755 break;
1756 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1757 case KVM_REG_PPC_TFHAR:
1758 *val = get_reg_val(id, vcpu->arch.tfhar);
1759 break;
1760 case KVM_REG_PPC_TFIAR:
1761 *val = get_reg_val(id, vcpu->arch.tfiar);
1762 break;
1763 case KVM_REG_PPC_TEXASR:
1764 *val = get_reg_val(id, vcpu->arch.texasr);
1765 break;
1766 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1767 i = id - KVM_REG_PPC_TM_GPR0;
1768 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1769 break;
1770 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1771 {
1772 int j;
1773 i = id - KVM_REG_PPC_TM_VSR0;
1774 if (i < 32)
1775 for (j = 0; j < TS_FPRWIDTH; j++)
1776 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1777 else {
1778 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1779 val->vval = vcpu->arch.vr_tm.vr[i-32];
1780 else
1781 r = -ENXIO;
1782 }
1783 break;
1784 }
1785 case KVM_REG_PPC_TM_CR:
1786 *val = get_reg_val(id, vcpu->arch.cr_tm);
1787 break;
1788 case KVM_REG_PPC_TM_XER:
1789 *val = get_reg_val(id, vcpu->arch.xer_tm);
1790 break;
1791 case KVM_REG_PPC_TM_LR:
1792 *val = get_reg_val(id, vcpu->arch.lr_tm);
1793 break;
1794 case KVM_REG_PPC_TM_CTR:
1795 *val = get_reg_val(id, vcpu->arch.ctr_tm);
1796 break;
1797 case KVM_REG_PPC_TM_FPSCR:
1798 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1799 break;
1800 case KVM_REG_PPC_TM_AMR:
1801 *val = get_reg_val(id, vcpu->arch.amr_tm);
1802 break;
1803 case KVM_REG_PPC_TM_PPR:
1804 *val = get_reg_val(id, vcpu->arch.ppr_tm);
1805 break;
1806 case KVM_REG_PPC_TM_VRSAVE:
1807 *val = get_reg_val(id, vcpu->arch.vrsave_tm);
1808 break;
1809 case KVM_REG_PPC_TM_VSCR:
1810 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1811 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1812 else
1813 r = -ENXIO;
1814 break;
1815 case KVM_REG_PPC_TM_DSCR:
1816 *val = get_reg_val(id, vcpu->arch.dscr_tm);
1817 break;
1818 case KVM_REG_PPC_TM_TAR:
1819 *val = get_reg_val(id, vcpu->arch.tar_tm);
1820 break;
1821 #endif
1822 case KVM_REG_PPC_ARCH_COMPAT:
1823 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1824 break;
1825 case KVM_REG_PPC_DEC_EXPIRY:
1826 *val = get_reg_val(id, vcpu->arch.dec_expires +
1827 vcpu->arch.vcore->tb_offset);
1828 break;
1829 case KVM_REG_PPC_ONLINE:
1830 *val = get_reg_val(id, vcpu->arch.online);
1831 break;
1832 case KVM_REG_PPC_PTCR:
1833 *val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
1834 break;
1835 default:
1836 r = -EINVAL;
1837 break;
1838 }
1839
1840 return r;
1841 }
1842
1843 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1844 union kvmppc_one_reg *val)
1845 {
1846 int r = 0;
1847 long int i;
1848 unsigned long addr, len;
1849
1850 switch (id) {
1851 case KVM_REG_PPC_HIOR:
1852
1853 if (set_reg_val(id, *val))
1854 r = -EINVAL;
1855 break;
1856 case KVM_REG_PPC_DABR:
1857 vcpu->arch.dabr = set_reg_val(id, *val);
1858 break;
1859 case KVM_REG_PPC_DABRX:
1860 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1861 break;
1862 case KVM_REG_PPC_DSCR:
1863 vcpu->arch.dscr = set_reg_val(id, *val);
1864 break;
1865 case KVM_REG_PPC_PURR:
1866 vcpu->arch.purr = set_reg_val(id, *val);
1867 break;
1868 case KVM_REG_PPC_SPURR:
1869 vcpu->arch.spurr = set_reg_val(id, *val);
1870 break;
1871 case KVM_REG_PPC_AMR:
1872 vcpu->arch.amr = set_reg_val(id, *val);
1873 break;
1874 case KVM_REG_PPC_UAMOR:
1875 vcpu->arch.uamor = set_reg_val(id, *val);
1876 break;
1877 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1878 i = id - KVM_REG_PPC_MMCR0;
1879 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
1880 break;
1881 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1882 i = id - KVM_REG_PPC_PMC1;
1883 vcpu->arch.pmc[i] = set_reg_val(id, *val);
1884 break;
1885 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1886 i = id - KVM_REG_PPC_SPMC1;
1887 vcpu->arch.spmc[i] = set_reg_val(id, *val);
1888 break;
1889 case KVM_REG_PPC_SIAR:
1890 vcpu->arch.siar = set_reg_val(id, *val);
1891 break;
1892 case KVM_REG_PPC_SDAR:
1893 vcpu->arch.sdar = set_reg_val(id, *val);
1894 break;
1895 case KVM_REG_PPC_SIER:
1896 vcpu->arch.sier = set_reg_val(id, *val);
1897 break;
1898 case KVM_REG_PPC_IAMR:
1899 vcpu->arch.iamr = set_reg_val(id, *val);
1900 break;
1901 case KVM_REG_PPC_PSPB:
1902 vcpu->arch.pspb = set_reg_val(id, *val);
1903 break;
1904 case KVM_REG_PPC_DPDES:
1905 vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
1906 break;
1907 case KVM_REG_PPC_VTB:
1908 vcpu->arch.vcore->vtb = set_reg_val(id, *val);
1909 break;
1910 case KVM_REG_PPC_DAWR:
1911 vcpu->arch.dawr = set_reg_val(id, *val);
1912 break;
1913 case KVM_REG_PPC_DAWRX:
1914 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
1915 break;
1916 case KVM_REG_PPC_CIABR:
1917 vcpu->arch.ciabr = set_reg_val(id, *val);
1918
1919 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
1920 vcpu->arch.ciabr &= ~CIABR_PRIV;
1921 break;
1922 case KVM_REG_PPC_CSIGR:
1923 vcpu->arch.csigr = set_reg_val(id, *val);
1924 break;
1925 case KVM_REG_PPC_TACR:
1926 vcpu->arch.tacr = set_reg_val(id, *val);
1927 break;
1928 case KVM_REG_PPC_TCSCR:
1929 vcpu->arch.tcscr = set_reg_val(id, *val);
1930 break;
1931 case KVM_REG_PPC_PID:
1932 vcpu->arch.pid = set_reg_val(id, *val);
1933 break;
1934 case KVM_REG_PPC_ACOP:
1935 vcpu->arch.acop = set_reg_val(id, *val);
1936 break;
1937 case KVM_REG_PPC_WORT:
1938 vcpu->arch.wort = set_reg_val(id, *val);
1939 break;
1940 case KVM_REG_PPC_TIDR:
1941 vcpu->arch.tid = set_reg_val(id, *val);
1942 break;
1943 case KVM_REG_PPC_PSSCR:
1944 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
1945 break;
1946 case KVM_REG_PPC_VPA_ADDR:
1947 addr = set_reg_val(id, *val);
1948 r = -EINVAL;
1949 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
1950 vcpu->arch.dtl.next_gpa))
1951 break;
1952 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
1953 break;
1954 case KVM_REG_PPC_VPA_SLB:
1955 addr = val->vpaval.addr;
1956 len = val->vpaval.length;
1957 r = -EINVAL;
1958 if (addr && !vcpu->arch.vpa.next_gpa)
1959 break;
1960 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
1961 break;
1962 case KVM_REG_PPC_VPA_DTL:
1963 addr = val->vpaval.addr;
1964 len = val->vpaval.length;
1965 r = -EINVAL;
1966 if (addr && (len < sizeof(struct dtl_entry) ||
1967 !vcpu->arch.vpa.next_gpa))
1968 break;
1969 len -= len % sizeof(struct dtl_entry);
1970 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1971 break;
1972 case KVM_REG_PPC_TB_OFFSET:
1973
1974 vcpu->arch.vcore->tb_offset =
1975 ALIGN(set_reg_val(id, *val), 1UL << 24);
1976 break;
1977 case KVM_REG_PPC_LPCR:
1978 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
1979 break;
1980 case KVM_REG_PPC_LPCR_64:
1981 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1982 break;
1983 case KVM_REG_PPC_PPR:
1984 vcpu->arch.ppr = set_reg_val(id, *val);
1985 break;
1986 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1987 case KVM_REG_PPC_TFHAR:
1988 vcpu->arch.tfhar = set_reg_val(id, *val);
1989 break;
1990 case KVM_REG_PPC_TFIAR:
1991 vcpu->arch.tfiar = set_reg_val(id, *val);
1992 break;
1993 case KVM_REG_PPC_TEXASR:
1994 vcpu->arch.texasr = set_reg_val(id, *val);
1995 break;
1996 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1997 i = id - KVM_REG_PPC_TM_GPR0;
1998 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
1999 break;
2000 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
2001 {
2002 int j;
2003 i = id - KVM_REG_PPC_TM_VSR0;
2004 if (i < 32)
2005 for (j = 0; j < TS_FPRWIDTH; j++)
2006 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
2007 else
2008 if (cpu_has_feature(CPU_FTR_ALTIVEC))
2009 vcpu->arch.vr_tm.vr[i-32] = val->vval;
2010 else
2011 r = -ENXIO;
2012 break;
2013 }
2014 case KVM_REG_PPC_TM_CR:
2015 vcpu->arch.cr_tm = set_reg_val(id, *val);
2016 break;
2017 case KVM_REG_PPC_TM_XER:
2018 vcpu->arch.xer_tm = set_reg_val(id, *val);
2019 break;
2020 case KVM_REG_PPC_TM_LR:
2021 vcpu->arch.lr_tm = set_reg_val(id, *val);
2022 break;
2023 case KVM_REG_PPC_TM_CTR:
2024 vcpu->arch.ctr_tm = set_reg_val(id, *val);
2025 break;
2026 case KVM_REG_PPC_TM_FPSCR:
2027 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
2028 break;
2029 case KVM_REG_PPC_TM_AMR:
2030 vcpu->arch.amr_tm = set_reg_val(id, *val);
2031 break;
2032 case KVM_REG_PPC_TM_PPR:
2033 vcpu->arch.ppr_tm = set_reg_val(id, *val);
2034 break;
2035 case KVM_REG_PPC_TM_VRSAVE:
2036 vcpu->arch.vrsave_tm = set_reg_val(id, *val);
2037 break;
2038 case KVM_REG_PPC_TM_VSCR:
2039 if (cpu_has_feature(CPU_FTR_ALTIVEC))
2040 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
2041 else
2042 r = - ENXIO;
2043 break;
2044 case KVM_REG_PPC_TM_DSCR:
2045 vcpu->arch.dscr_tm = set_reg_val(id, *val);
2046 break;
2047 case KVM_REG_PPC_TM_TAR:
2048 vcpu->arch.tar_tm = set_reg_val(id, *val);
2049 break;
2050 #endif
2051 case KVM_REG_PPC_ARCH_COMPAT:
2052 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
2053 break;
2054 case KVM_REG_PPC_DEC_EXPIRY:
2055 vcpu->arch.dec_expires = set_reg_val(id, *val) -
2056 vcpu->arch.vcore->tb_offset;
2057 break;
2058 case KVM_REG_PPC_ONLINE:
2059 i = set_reg_val(id, *val);
2060 if (i && !vcpu->arch.online)
2061 atomic_inc(&vcpu->arch.vcore->online_count);
2062 else if (!i && vcpu->arch.online)
2063 atomic_dec(&vcpu->arch.vcore->online_count);
2064 vcpu->arch.online = i;
2065 break;
2066 case KVM_REG_PPC_PTCR:
2067 vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
2068 break;
2069 default:
2070 r = -EINVAL;
2071 break;
2072 }
2073
2074 return r;
2075 }
2076
2077
2078
2079
2080
2081
2082
2083
2084 static int threads_per_vcore(struct kvm *kvm)
2085 {
2086 if (kvm->arch.threads_indep)
2087 return 1;
2088 return threads_per_subcore;
2089 }
2090
2091 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
2092 {
2093 struct kvmppc_vcore *vcore;
2094
2095 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
2096
2097 if (vcore == NULL)
2098 return NULL;
2099
2100 spin_lock_init(&vcore->lock);
2101 spin_lock_init(&vcore->stoltb_lock);
2102 init_swait_queue_head(&vcore->wq);
2103 vcore->preempt_tb = TB_NIL;
2104 vcore->lpcr = kvm->arch.lpcr;
2105 vcore->first_vcpuid = id;
2106 vcore->kvm = kvm;
2107 INIT_LIST_HEAD(&vcore->preempt_list);
2108
2109 return vcore;
2110 }
2111
2112 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2113 static struct debugfs_timings_element {
2114 const char *name;
2115 size_t offset;
2116 } timings[] = {
2117 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)},
2118 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)},
2119 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)},
2120 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)},
2121 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)},
2122 };
2123
2124 #define N_TIMINGS (ARRAY_SIZE(timings))
2125
2126 struct debugfs_timings_state {
2127 struct kvm_vcpu *vcpu;
2128 unsigned int buflen;
2129 char buf[N_TIMINGS * 100];
2130 };
2131
2132 static int debugfs_timings_open(struct inode *inode, struct file *file)
2133 {
2134 struct kvm_vcpu *vcpu = inode->i_private;
2135 struct debugfs_timings_state *p;
2136
2137 p = kzalloc(sizeof(*p), GFP_KERNEL);
2138 if (!p)
2139 return -ENOMEM;
2140
2141 kvm_get_kvm(vcpu->kvm);
2142 p->vcpu = vcpu;
2143 file->private_data = p;
2144
2145 return nonseekable_open(inode, file);
2146 }
2147
2148 static int debugfs_timings_release(struct inode *inode, struct file *file)
2149 {
2150 struct debugfs_timings_state *p = file->private_data;
2151
2152 kvm_put_kvm(p->vcpu->kvm);
2153 kfree(p);
2154 return 0;
2155 }
2156
2157 static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
2158 size_t len, loff_t *ppos)
2159 {
2160 struct debugfs_timings_state *p = file->private_data;
2161 struct kvm_vcpu *vcpu = p->vcpu;
2162 char *s, *buf_end;
2163 struct kvmhv_tb_accumulator tb;
2164 u64 count;
2165 loff_t pos;
2166 ssize_t n;
2167 int i, loops;
2168 bool ok;
2169
2170 if (!p->buflen) {
2171 s = p->buf;
2172 buf_end = s + sizeof(p->buf);
2173 for (i = 0; i < N_TIMINGS; ++i) {
2174 struct kvmhv_tb_accumulator *acc;
2175
2176 acc = (struct kvmhv_tb_accumulator *)
2177 ((unsigned long)vcpu + timings[i].offset);
2178 ok = false;
2179 for (loops = 0; loops < 1000; ++loops) {
2180 count = acc->seqcount;
2181 if (!(count & 1)) {
2182 smp_rmb();
2183 tb = *acc;
2184 smp_rmb();
2185 if (count == acc->seqcount) {
2186 ok = true;
2187 break;
2188 }
2189 }
2190 udelay(1);
2191 }
2192 if (!ok)
2193 snprintf(s, buf_end - s, "%s: stuck\n",
2194 timings[i].name);
2195 else
2196 snprintf(s, buf_end - s,
2197 "%s: %llu %llu %llu %llu\n",
2198 timings[i].name, count / 2,
2199 tb_to_ns(tb.tb_total),
2200 tb_to_ns(tb.tb_min),
2201 tb_to_ns(tb.tb_max));
2202 s += strlen(s);
2203 }
2204 p->buflen = s - p->buf;
2205 }
2206
2207 pos = *ppos;
2208 if (pos >= p->buflen)
2209 return 0;
2210 if (len > p->buflen - pos)
2211 len = p->buflen - pos;
2212 n = copy_to_user(buf, p->buf + pos, len);
2213 if (n) {
2214 if (n == len)
2215 return -EFAULT;
2216 len -= n;
2217 }
2218 *ppos = pos + len;
2219 return len;
2220 }
2221
2222 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
2223 size_t len, loff_t *ppos)
2224 {
2225 return -EACCES;
2226 }
2227
2228 static const struct file_operations debugfs_timings_ops = {
2229 .owner = THIS_MODULE,
2230 .open = debugfs_timings_open,
2231 .release = debugfs_timings_release,
2232 .read = debugfs_timings_read,
2233 .write = debugfs_timings_write,
2234 .llseek = generic_file_llseek,
2235 };
2236
2237
2238 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2239 {
2240 char buf[16];
2241 struct kvm *kvm = vcpu->kvm;
2242
2243 snprintf(buf, sizeof(buf), "vcpu%u", id);
2244 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
2245 return;
2246 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
2247 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir))
2248 return;
2249 vcpu->arch.debugfs_timings =
2250 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir,
2251 vcpu, &debugfs_timings_ops);
2252 }
2253
2254 #else
2255 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2256 {
2257 }
2258 #endif
2259
2260 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
2261 unsigned int id)
2262 {
2263 struct kvm_vcpu *vcpu;
2264 int err;
2265 int core;
2266 struct kvmppc_vcore *vcore;
2267
2268 err = -ENOMEM;
2269 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2270 if (!vcpu)
2271 goto out;
2272
2273 err = kvm_vcpu_init(vcpu, kvm, id);
2274 if (err)
2275 goto free_vcpu;
2276
2277 vcpu->arch.shared = &vcpu->arch.shregs;
2278 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2279
2280
2281
2282
2283 #ifdef __BIG_ENDIAN__
2284 vcpu->arch.shared_big_endian = true;
2285 #else
2286 vcpu->arch.shared_big_endian = false;
2287 #endif
2288 #endif
2289 vcpu->arch.mmcr[0] = MMCR0_FC;
2290 vcpu->arch.ctrl = CTRL_RUNLATCH;
2291
2292 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
2293 spin_lock_init(&vcpu->arch.vpa_update_lock);
2294 spin_lock_init(&vcpu->arch.tbacct_lock);
2295 vcpu->arch.busy_preempt = TB_NIL;
2296 vcpu->arch.intr_msr = MSR_SF | MSR_ME;
2297
2298
2299
2300
2301
2302
2303
2304
2305 vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB |
2306 HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP;
2307 if (cpu_has_feature(CPU_FTR_HVMODE)) {
2308 vcpu->arch.hfscr &= mfspr(SPRN_HFSCR);
2309 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
2310 vcpu->arch.hfscr |= HFSCR_TM;
2311 }
2312 if (cpu_has_feature(CPU_FTR_TM_COMP))
2313 vcpu->arch.hfscr |= HFSCR_TM;
2314
2315 kvmppc_mmu_book3s_hv_init(vcpu);
2316
2317 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2318
2319 init_waitqueue_head(&vcpu->arch.cpu_run);
2320
2321 mutex_lock(&kvm->lock);
2322 vcore = NULL;
2323 err = -EINVAL;
2324 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2325 if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
2326 pr_devel("KVM: VCPU ID too high\n");
2327 core = KVM_MAX_VCORES;
2328 } else {
2329 BUG_ON(kvm->arch.smt_mode != 1);
2330 core = kvmppc_pack_vcpu_id(kvm, id);
2331 }
2332 } else {
2333 core = id / kvm->arch.smt_mode;
2334 }
2335 if (core < KVM_MAX_VCORES) {
2336 vcore = kvm->arch.vcores[core];
2337 if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
2338 pr_devel("KVM: collision on id %u", id);
2339 vcore = NULL;
2340 } else if (!vcore) {
2341
2342
2343
2344
2345 err = -ENOMEM;
2346 vcore = kvmppc_vcore_create(kvm,
2347 id & ~(kvm->arch.smt_mode - 1));
2348 mutex_lock(&kvm->arch.mmu_setup_lock);
2349 kvm->arch.vcores[core] = vcore;
2350 kvm->arch.online_vcores++;
2351 mutex_unlock(&kvm->arch.mmu_setup_lock);
2352 }
2353 }
2354 mutex_unlock(&kvm->lock);
2355
2356 if (!vcore)
2357 goto uninit_vcpu;
2358
2359 spin_lock(&vcore->lock);
2360 ++vcore->num_threads;
2361 spin_unlock(&vcore->lock);
2362 vcpu->arch.vcore = vcore;
2363 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
2364 vcpu->arch.thread_cpu = -1;
2365 vcpu->arch.prev_cpu = -1;
2366
2367 vcpu->arch.cpu_type = KVM_CPU_3S_64;
2368 kvmppc_sanity_check(vcpu);
2369
2370 debugfs_vcpu_init(vcpu, id);
2371
2372 return vcpu;
2373
2374 uninit_vcpu:
2375 kvm_vcpu_uninit(vcpu);
2376 free_vcpu:
2377 kmem_cache_free(kvm_vcpu_cache, vcpu);
2378 out:
2379 return ERR_PTR(err);
2380 }
2381
2382 static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
2383 unsigned long flags)
2384 {
2385 int err;
2386 int esmt = 0;
2387
2388 if (flags)
2389 return -EINVAL;
2390 if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
2391 return -EINVAL;
2392 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
2393
2394
2395
2396
2397 if (smt_mode > threads_per_subcore)
2398 return -EINVAL;
2399 } else {
2400
2401
2402
2403
2404 esmt = smt_mode;
2405 smt_mode = 1;
2406 }
2407 mutex_lock(&kvm->lock);
2408 err = -EBUSY;
2409 if (!kvm->arch.online_vcores) {
2410 kvm->arch.smt_mode = smt_mode;
2411 kvm->arch.emul_smt_mode = esmt;
2412 err = 0;
2413 }
2414 mutex_unlock(&kvm->lock);
2415
2416 return err;
2417 }
2418
2419 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
2420 {
2421 if (vpa->pinned_addr)
2422 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
2423 vpa->dirty);
2424 }
2425
2426 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
2427 {
2428 spin_lock(&vcpu->arch.vpa_update_lock);
2429 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
2430 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
2431 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
2432 spin_unlock(&vcpu->arch.vpa_update_lock);
2433 kvm_vcpu_uninit(vcpu);
2434 kmem_cache_free(kvm_vcpu_cache, vcpu);
2435 }
2436
2437 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
2438 {
2439
2440 return 1;
2441 }
2442
2443 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
2444 {
2445 unsigned long dec_nsec, now;
2446
2447 now = get_tb();
2448 if (now > vcpu->arch.dec_expires) {
2449
2450 kvmppc_core_queue_dec(vcpu);
2451 kvmppc_core_prepare_to_enter(vcpu);
2452 return;
2453 }
2454 dec_nsec = tb_to_ns(vcpu->arch.dec_expires - now);
2455 hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
2456 vcpu->arch.timer_running = 1;
2457 }
2458
2459 static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
2460 {
2461 vcpu->arch.ceded = 0;
2462 if (vcpu->arch.timer_running) {
2463 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2464 vcpu->arch.timer_running = 0;
2465 }
2466 }
2467
2468 extern int __kvmppc_vcore_entry(void);
2469
2470 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
2471 struct kvm_vcpu *vcpu)
2472 {
2473 u64 now;
2474
2475 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
2476 return;
2477 spin_lock_irq(&vcpu->arch.tbacct_lock);
2478 now = mftb();
2479 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
2480 vcpu->arch.stolen_logged;
2481 vcpu->arch.busy_preempt = now;
2482 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2483 spin_unlock_irq(&vcpu->arch.tbacct_lock);
2484 --vc->n_runnable;
2485 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
2486 }
2487
2488 static int kvmppc_grab_hwthread(int cpu)
2489 {
2490 struct paca_struct *tpaca;
2491 long timeout = 10000;
2492
2493 tpaca = paca_ptrs[cpu];
2494
2495
2496 tpaca->kvm_hstate.kvm_vcpu = NULL;
2497 tpaca->kvm_hstate.kvm_vcore = NULL;
2498 tpaca->kvm_hstate.napping = 0;
2499 smp_wmb();
2500 tpaca->kvm_hstate.hwthread_req = 1;
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511 smp_mb();
2512 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
2513 if (--timeout <= 0) {
2514 pr_err("KVM: couldn't grab cpu %d\n", cpu);
2515 return -EBUSY;
2516 }
2517 udelay(1);
2518 }
2519 return 0;
2520 }
2521
2522 static void kvmppc_release_hwthread(int cpu)
2523 {
2524 struct paca_struct *tpaca;
2525
2526 tpaca = paca_ptrs[cpu];
2527 tpaca->kvm_hstate.hwthread_req = 0;
2528 tpaca->kvm_hstate.kvm_vcpu = NULL;
2529 tpaca->kvm_hstate.kvm_vcore = NULL;
2530 tpaca->kvm_hstate.kvm_split_mode = NULL;
2531 }
2532
2533 static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
2534 {
2535 struct kvm_nested_guest *nested = vcpu->arch.nested;
2536 cpumask_t *cpu_in_guest;
2537 int i;
2538
2539 cpu = cpu_first_thread_sibling(cpu);
2540 if (nested) {
2541 cpumask_set_cpu(cpu, &nested->need_tlb_flush);
2542 cpu_in_guest = &nested->cpu_in_guest;
2543 } else {
2544 cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
2545 cpu_in_guest = &kvm->arch.cpu_in_guest;
2546 }
2547
2548
2549
2550
2551
2552 smp_mb();
2553 for (i = 0; i < threads_per_core; ++i)
2554 if (cpumask_test_cpu(cpu + i, cpu_in_guest))
2555 smp_call_function_single(cpu + i, do_nothing, NULL, 1);
2556 }
2557
2558 static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2559 {
2560 struct kvm_nested_guest *nested = vcpu->arch.nested;
2561 struct kvm *kvm = vcpu->kvm;
2562 int prev_cpu;
2563
2564 if (!cpu_has_feature(CPU_FTR_HVMODE))
2565 return;
2566
2567 if (nested)
2568 prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
2569 else
2570 prev_cpu = vcpu->arch.prev_cpu;
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584 if (prev_cpu != pcpu) {
2585 if (prev_cpu >= 0 &&
2586 cpu_first_thread_sibling(prev_cpu) !=
2587 cpu_first_thread_sibling(pcpu))
2588 radix_flush_cpu(kvm, prev_cpu, vcpu);
2589 if (nested)
2590 nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
2591 else
2592 vcpu->arch.prev_cpu = pcpu;
2593 }
2594 }
2595
2596 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
2597 {
2598 int cpu;
2599 struct paca_struct *tpaca;
2600 struct kvm *kvm = vc->kvm;
2601
2602 cpu = vc->pcpu;
2603 if (vcpu) {
2604 if (vcpu->arch.timer_running) {
2605 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2606 vcpu->arch.timer_running = 0;
2607 }
2608 cpu += vcpu->arch.ptid;
2609 vcpu->cpu = vc->pcpu;
2610 vcpu->arch.thread_cpu = cpu;
2611 cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
2612 }
2613 tpaca = paca_ptrs[cpu];
2614 tpaca->kvm_hstate.kvm_vcpu = vcpu;
2615 tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
2616 tpaca->kvm_hstate.fake_suspend = 0;
2617
2618 smp_wmb();
2619 tpaca->kvm_hstate.kvm_vcore = vc;
2620 if (cpu != smp_processor_id())
2621 kvmppc_ipi_thread(cpu);
2622 }
2623
2624 static void kvmppc_wait_for_nap(int n_threads)
2625 {
2626 int cpu = smp_processor_id();
2627 int i, loops;
2628
2629 if (n_threads <= 1)
2630 return;
2631 for (loops = 0; loops < 1000000; ++loops) {
2632
2633
2634
2635
2636
2637
2638 for (i = 1; i < n_threads; ++i)
2639 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2640 break;
2641 if (i == n_threads) {
2642 HMT_medium();
2643 return;
2644 }
2645 HMT_low();
2646 }
2647 HMT_medium();
2648 for (i = 1; i < n_threads; ++i)
2649 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2650 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
2651 }
2652
2653
2654
2655
2656
2657
2658 static int on_primary_thread(void)
2659 {
2660 int cpu = smp_processor_id();
2661 int thr;
2662
2663
2664 if (cpu_thread_in_subcore(cpu))
2665 return 0;
2666
2667 thr = 0;
2668 while (++thr < threads_per_subcore)
2669 if (cpu_online(cpu + thr))
2670 return 0;
2671
2672
2673 for (thr = 1; thr < threads_per_subcore; ++thr) {
2674 if (kvmppc_grab_hwthread(cpu + thr)) {
2675
2676 do {
2677 kvmppc_release_hwthread(cpu + thr);
2678 } while (--thr > 0);
2679 return 0;
2680 }
2681 }
2682 return 1;
2683 }
2684
2685
2686
2687
2688
2689
2690 struct preempted_vcore_list {
2691 struct list_head list;
2692 spinlock_t lock;
2693 };
2694
2695 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
2696
2697 static void init_vcore_lists(void)
2698 {
2699 int cpu;
2700
2701 for_each_possible_cpu(cpu) {
2702 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
2703 spin_lock_init(&lp->lock);
2704 INIT_LIST_HEAD(&lp->list);
2705 }
2706 }
2707
2708 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
2709 {
2710 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2711
2712 vc->vcore_state = VCORE_PREEMPT;
2713 vc->pcpu = smp_processor_id();
2714 if (vc->num_threads < threads_per_vcore(vc->kvm)) {
2715 spin_lock(&lp->lock);
2716 list_add_tail(&vc->preempt_list, &lp->list);
2717 spin_unlock(&lp->lock);
2718 }
2719
2720
2721 kvmppc_core_start_stolen(vc);
2722 }
2723
2724 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
2725 {
2726 struct preempted_vcore_list *lp;
2727
2728 kvmppc_core_end_stolen(vc);
2729 if (!list_empty(&vc->preempt_list)) {
2730 lp = &per_cpu(preempted_vcores, vc->pcpu);
2731 spin_lock(&lp->lock);
2732 list_del_init(&vc->preempt_list);
2733 spin_unlock(&lp->lock);
2734 }
2735 vc->vcore_state = VCORE_INACTIVE;
2736 }
2737
2738
2739
2740
2741
2742 struct core_info {
2743 int n_subcores;
2744 int max_subcore_threads;
2745 int total_threads;
2746 int subcore_threads[MAX_SUBCORES];
2747 struct kvmppc_vcore *vc[MAX_SUBCORES];
2748 };
2749
2750
2751
2752
2753
2754 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
2755
2756 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
2757 {
2758 memset(cip, 0, sizeof(*cip));
2759 cip->n_subcores = 1;
2760 cip->max_subcore_threads = vc->num_threads;
2761 cip->total_threads = vc->num_threads;
2762 cip->subcore_threads[0] = vc->num_threads;
2763 cip->vc[0] = vc;
2764 }
2765
2766 static bool subcore_config_ok(int n_subcores, int n_threads)
2767 {
2768
2769
2770
2771
2772 if (cpu_has_feature(CPU_FTR_ARCH_300))
2773 return n_subcores <= 4 && n_threads == 1;
2774
2775
2776 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
2777 return false;
2778 if (n_subcores > MAX_SUBCORES)
2779 return false;
2780 if (n_subcores > 1) {
2781 if (!(dynamic_mt_modes & 2))
2782 n_subcores = 4;
2783 if (n_subcores > 2 && !(dynamic_mt_modes & 4))
2784 return false;
2785 }
2786
2787 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
2788 }
2789
2790 static void init_vcore_to_run(struct kvmppc_vcore *vc)
2791 {
2792 vc->entry_exit_map = 0;
2793 vc->in_guest = 0;
2794 vc->napping_threads = 0;
2795 vc->conferring_threads = 0;
2796 vc->tb_offset_applied = 0;
2797 }
2798
2799 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
2800 {
2801 int n_threads = vc->num_threads;
2802 int sub;
2803
2804 if (!cpu_has_feature(CPU_FTR_ARCH_207S))
2805 return false;
2806
2807
2808 if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
2809 return false;
2810
2811
2812 if (no_mixing_hpt_and_radix &&
2813 kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
2814 return false;
2815
2816 if (n_threads < cip->max_subcore_threads)
2817 n_threads = cip->max_subcore_threads;
2818 if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
2819 return false;
2820 cip->max_subcore_threads = n_threads;
2821
2822 sub = cip->n_subcores;
2823 ++cip->n_subcores;
2824 cip->total_threads += vc->num_threads;
2825 cip->subcore_threads[sub] = vc->num_threads;
2826 cip->vc[sub] = vc;
2827 init_vcore_to_run(vc);
2828 list_del_init(&vc->preempt_list);
2829
2830 return true;
2831 }
2832
2833
2834
2835
2836
2837 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
2838 int target_threads)
2839 {
2840 if (cip->total_threads + pvc->num_threads > target_threads)
2841 return false;
2842
2843 return can_dynamic_split(pvc, cip);
2844 }
2845
2846 static void prepare_threads(struct kvmppc_vcore *vc)
2847 {
2848 int i;
2849 struct kvm_vcpu *vcpu;
2850
2851 for_each_runnable_thread(i, vcpu, vc) {
2852 if (signal_pending(vcpu->arch.run_task))
2853 vcpu->arch.ret = -EINTR;
2854 else if (vcpu->arch.vpa.update_pending ||
2855 vcpu->arch.slb_shadow.update_pending ||
2856 vcpu->arch.dtl.update_pending)
2857 vcpu->arch.ret = RESUME_GUEST;
2858 else
2859 continue;
2860 kvmppc_remove_runnable(vc, vcpu);
2861 wake_up(&vcpu->arch.cpu_run);
2862 }
2863 }
2864
2865 static void collect_piggybacks(struct core_info *cip, int target_threads)
2866 {
2867 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2868 struct kvmppc_vcore *pvc, *vcnext;
2869
2870 spin_lock(&lp->lock);
2871 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
2872 if (!spin_trylock(&pvc->lock))
2873 continue;
2874 prepare_threads(pvc);
2875 if (!pvc->n_runnable || !pvc->kvm->arch.mmu_ready) {
2876 list_del_init(&pvc->preempt_list);
2877 if (pvc->runner == NULL) {
2878 pvc->vcore_state = VCORE_INACTIVE;
2879 kvmppc_core_end_stolen(pvc);
2880 }
2881 spin_unlock(&pvc->lock);
2882 continue;
2883 }
2884 if (!can_piggyback(pvc, cip, target_threads)) {
2885 spin_unlock(&pvc->lock);
2886 continue;
2887 }
2888 kvmppc_core_end_stolen(pvc);
2889 pvc->vcore_state = VCORE_PIGGYBACK;
2890 if (cip->total_threads >= target_threads)
2891 break;
2892 }
2893 spin_unlock(&lp->lock);
2894 }
2895
2896 static bool recheck_signals_and_mmu(struct core_info *cip)
2897 {
2898 int sub, i;
2899 struct kvm_vcpu *vcpu;
2900 struct kvmppc_vcore *vc;
2901
2902 for (sub = 0; sub < cip->n_subcores; ++sub) {
2903 vc = cip->vc[sub];
2904 if (!vc->kvm->arch.mmu_ready)
2905 return true;
2906 for_each_runnable_thread(i, vcpu, vc)
2907 if (signal_pending(vcpu->arch.run_task))
2908 return true;
2909 }
2910 return false;
2911 }
2912
2913 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
2914 {
2915 int still_running = 0, i;
2916 u64 now;
2917 long ret;
2918 struct kvm_vcpu *vcpu;
2919
2920 spin_lock(&vc->lock);
2921 now = get_tb();
2922 for_each_runnable_thread(i, vcpu, vc) {
2923
2924
2925
2926
2927
2928
2929
2930 spin_unlock(&vc->lock);
2931
2932 if (now < vcpu->arch.dec_expires &&
2933 kvmppc_core_pending_dec(vcpu))
2934 kvmppc_core_dequeue_dec(vcpu);
2935
2936 trace_kvm_guest_exit(vcpu);
2937
2938 ret = RESUME_GUEST;
2939 if (vcpu->arch.trap)
2940 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
2941 vcpu->arch.run_task);
2942
2943 vcpu->arch.ret = ret;
2944 vcpu->arch.trap = 0;
2945
2946 spin_lock(&vc->lock);
2947 if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
2948 if (vcpu->arch.pending_exceptions)
2949 kvmppc_core_prepare_to_enter(vcpu);
2950 if (vcpu->arch.ceded)
2951 kvmppc_set_timer(vcpu);
2952 else
2953 ++still_running;
2954 } else {
2955 kvmppc_remove_runnable(vc, vcpu);
2956 wake_up(&vcpu->arch.cpu_run);
2957 }
2958 }
2959 if (!is_master) {
2960 if (still_running > 0) {
2961 kvmppc_vcore_preempt(vc);
2962 } else if (vc->runner) {
2963 vc->vcore_state = VCORE_PREEMPT;
2964 kvmppc_core_start_stolen(vc);
2965 } else {
2966 vc->vcore_state = VCORE_INACTIVE;
2967 }
2968 if (vc->n_runnable > 0 && vc->runner == NULL) {
2969
2970 i = -1;
2971 vcpu = next_runnable_thread(vc, &i);
2972 wake_up(&vcpu->arch.cpu_run);
2973 }
2974 }
2975 spin_unlock(&vc->lock);
2976 }
2977
2978
2979
2980
2981
2982
2983 static inline int kvmppc_clear_host_core(unsigned int cpu)
2984 {
2985 int core;
2986
2987 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2988 return 0;
2989
2990
2991
2992
2993
2994 core = cpu >> threads_shift;
2995 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
2996 return 0;
2997 }
2998
2999
3000
3001
3002
3003
3004 static inline int kvmppc_set_host_core(unsigned int cpu)
3005 {
3006 int core;
3007
3008 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
3009 return 0;
3010
3011
3012
3013
3014
3015 core = cpu >> threads_shift;
3016 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
3017 return 0;
3018 }
3019
3020 static void set_irq_happened(int trap)
3021 {
3022 switch (trap) {
3023 case BOOK3S_INTERRUPT_EXTERNAL:
3024 local_paca->irq_happened |= PACA_IRQ_EE;
3025 break;
3026 case BOOK3S_INTERRUPT_H_DOORBELL:
3027 local_paca->irq_happened |= PACA_IRQ_DBELL;
3028 break;
3029 case BOOK3S_INTERRUPT_HMI:
3030 local_paca->irq_happened |= PACA_IRQ_HMI;
3031 break;
3032 case BOOK3S_INTERRUPT_SYSTEM_RESET:
3033 replay_system_reset();
3034 break;
3035 }
3036 }
3037
3038
3039
3040
3041
3042 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
3043 {
3044 struct kvm_vcpu *vcpu;
3045 int i;
3046 int srcu_idx;
3047 struct core_info core_info;
3048 struct kvmppc_vcore *pvc;
3049 struct kvm_split_mode split_info, *sip;
3050 int split, subcore_size, active;
3051 int sub;
3052 bool thr0_done;
3053 unsigned long cmd_bit, stat_bit;
3054 int pcpu, thr;
3055 int target_threads;
3056 int controlled_threads;
3057 int trap;
3058 bool is_power8;
3059 bool hpt_on_radix;
3060
3061
3062
3063
3064
3065 prepare_threads(vc);
3066
3067
3068 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
3069 return;
3070
3071
3072
3073
3074 init_vcore_to_run(vc);
3075 vc->preempt_tb = TB_NIL;
3076
3077
3078
3079
3080
3081
3082 controlled_threads = threads_per_vcore(vc->kvm);
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092 hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
3093 !kvm_is_radix(vc->kvm);
3094 if (((controlled_threads > 1) &&
3095 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) ||
3096 (hpt_on_radix && vc->kvm->arch.threads_indep)) {
3097 for_each_runnable_thread(i, vcpu, vc) {
3098 vcpu->arch.ret = -EBUSY;
3099 kvmppc_remove_runnable(vc, vcpu);
3100 wake_up(&vcpu->arch.cpu_run);
3101 }
3102 goto out;
3103 }
3104
3105
3106
3107
3108
3109 init_core_info(&core_info, vc);
3110 pcpu = smp_processor_id();
3111 target_threads = controlled_threads;
3112 if (target_smt_mode && target_smt_mode < target_threads)
3113 target_threads = target_smt_mode;
3114 if (vc->num_threads < target_threads)
3115 collect_piggybacks(&core_info, target_threads);
3116
3117
3118
3119
3120
3121
3122 pcpu = smp_processor_id();
3123 if (kvm_is_radix(vc->kvm)) {
3124 for (sub = 0; sub < core_info.n_subcores; ++sub)
3125 for_each_runnable_thread(i, vcpu, core_info.vc[sub])
3126 kvmppc_prepare_radix_vcpu(vcpu, pcpu);
3127 }
3128
3129
3130
3131
3132
3133
3134
3135
3136 local_irq_disable();
3137 hard_irq_disable();
3138 if (lazy_irq_pending() || need_resched() ||
3139 recheck_signals_and_mmu(&core_info)) {
3140 local_irq_enable();
3141 vc->vcore_state = VCORE_INACTIVE;
3142
3143 for (sub = 1; sub < core_info.n_subcores; ++sub) {
3144 pvc = core_info.vc[sub];
3145
3146 kvmppc_vcore_preempt(pvc);
3147 spin_unlock(&pvc->lock);
3148 }
3149 for (i = 0; i < controlled_threads; ++i)
3150 kvmppc_release_hwthread(pcpu + i);
3151 return;
3152 }
3153
3154 kvmppc_clear_host_core(pcpu);
3155
3156
3157 subcore_size = threads_per_subcore;
3158 cmd_bit = stat_bit = 0;
3159 split = core_info.n_subcores;
3160 sip = NULL;
3161 is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
3162 && !cpu_has_feature(CPU_FTR_ARCH_300);
3163
3164 if (split > 1 || hpt_on_radix) {
3165 sip = &split_info;
3166 memset(&split_info, 0, sizeof(split_info));
3167 for (sub = 0; sub < core_info.n_subcores; ++sub)
3168 split_info.vc[sub] = core_info.vc[sub];
3169
3170 if (is_power8) {
3171 if (split == 2 && (dynamic_mt_modes & 2)) {
3172 cmd_bit = HID0_POWER8_1TO2LPAR;
3173 stat_bit = HID0_POWER8_2LPARMODE;
3174 } else {
3175 split = 4;
3176 cmd_bit = HID0_POWER8_1TO4LPAR;
3177 stat_bit = HID0_POWER8_4LPARMODE;
3178 }
3179 subcore_size = MAX_SMT_THREADS / split;
3180 split_info.rpr = mfspr(SPRN_RPR);
3181 split_info.pmmar = mfspr(SPRN_PMMAR);
3182 split_info.ldbar = mfspr(SPRN_LDBAR);
3183 split_info.subcore_size = subcore_size;
3184 } else {
3185 split_info.subcore_size = 1;
3186 if (hpt_on_radix) {
3187
3188 split_info.lpcr_req = vc->lpcr;
3189 split_info.lpidr_req = vc->kvm->arch.lpid;
3190 split_info.host_lpcr = vc->kvm->arch.host_lpcr;
3191 split_info.do_set = 1;
3192 }
3193 }
3194
3195
3196 smp_wmb();
3197 }
3198
3199 for (thr = 0; thr < controlled_threads; ++thr) {
3200 struct paca_struct *paca = paca_ptrs[pcpu + thr];
3201
3202 paca->kvm_hstate.tid = thr;
3203 paca->kvm_hstate.napping = 0;
3204 paca->kvm_hstate.kvm_split_mode = sip;
3205 }
3206
3207
3208 if (cmd_bit) {
3209 unsigned long hid0 = mfspr(SPRN_HID0);
3210
3211 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
3212 mb();
3213 mtspr(SPRN_HID0, hid0);
3214 isync();
3215 for (;;) {
3216 hid0 = mfspr(SPRN_HID0);
3217 if (hid0 & stat_bit)
3218 break;
3219 cpu_relax();
3220 }
3221 }
3222
3223
3224
3225
3226
3227
3228 if (is_power8) {
3229 unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
3230 int n_online = atomic_read(&vc->online_count);
3231
3232
3233
3234
3235
3236 if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
3237 n_online >= 1 && n_online <= MAX_SMT_THREADS)
3238 rwmr_val = p8_rwmr_values[n_online];
3239 mtspr(SPRN_RWMR, rwmr_val);
3240 }
3241
3242
3243 active = 0;
3244 for (sub = 0; sub < core_info.n_subcores; ++sub) {
3245 thr = is_power8 ? subcore_thread_map[sub] : sub;
3246 thr0_done = false;
3247 active |= 1 << thr;
3248 pvc = core_info.vc[sub];
3249 pvc->pcpu = pcpu + thr;
3250 for_each_runnable_thread(i, vcpu, pvc) {
3251 kvmppc_start_thread(vcpu, pvc);
3252 kvmppc_create_dtl_entry(vcpu, pvc);
3253 trace_kvm_guest_enter(vcpu);
3254 if (!vcpu->arch.ptid)
3255 thr0_done = true;
3256 active |= 1 << (thr + vcpu->arch.ptid);
3257 }
3258
3259
3260
3261
3262 if (!thr0_done)
3263 kvmppc_start_thread(NULL, pvc);
3264 }
3265
3266
3267
3268
3269
3270 smp_mb();
3271
3272
3273
3274
3275
3276
3277
3278
3279 if (cmd_bit || hpt_on_radix) {
3280 split_info.do_nap = 1;
3281 for (thr = 1; thr < threads_per_subcore; ++thr)
3282 if (!(active & (1 << thr)))
3283 kvmppc_ipi_thread(pcpu + thr);
3284 }
3285
3286 vc->vcore_state = VCORE_RUNNING;
3287 preempt_disable();
3288
3289 trace_kvmppc_run_core(vc, 0);
3290
3291 for (sub = 0; sub < core_info.n_subcores; ++sub)
3292 spin_unlock(&core_info.vc[sub]->lock);
3293
3294 guest_enter_irqoff();
3295
3296 srcu_idx = srcu_read_lock(&vc->kvm->srcu);
3297
3298 this_cpu_disable_ftrace();
3299
3300
3301
3302
3303
3304 trace_hardirqs_on();
3305
3306 trap = __kvmppc_vcore_entry();
3307
3308 trace_hardirqs_off();
3309
3310 this_cpu_enable_ftrace();
3311
3312 srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
3313
3314 set_irq_happened(trap);
3315
3316 spin_lock(&vc->lock);
3317
3318 vc->vcore_state = VCORE_EXITING;
3319
3320
3321 kvmppc_wait_for_nap(controlled_threads);
3322
3323
3324 if (cmd_bit) {
3325 unsigned long hid0 = mfspr(SPRN_HID0);
3326 unsigned long loops = 0;
3327
3328 hid0 &= ~HID0_POWER8_DYNLPARDIS;
3329 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
3330 mb();
3331 mtspr(SPRN_HID0, hid0);
3332 isync();
3333 for (;;) {
3334 hid0 = mfspr(SPRN_HID0);
3335 if (!(hid0 & stat_bit))
3336 break;
3337 cpu_relax();
3338 ++loops;
3339 }
3340 } else if (hpt_on_radix) {
3341
3342 for (thr = 1; thr < controlled_threads; ++thr) {
3343 struct paca_struct *paca = paca_ptrs[pcpu + thr];
3344
3345 while (paca->kvm_hstate.kvm_split_mode) {
3346 HMT_low();
3347 barrier();
3348 }
3349 HMT_medium();
3350 }
3351 }
3352 split_info.do_nap = 0;
3353
3354 kvmppc_set_host_core(pcpu);
3355
3356 local_irq_enable();
3357 guest_exit();
3358
3359
3360 for (i = 0; i < controlled_threads; ++i) {
3361 kvmppc_release_hwthread(pcpu + i);
3362 if (sip && sip->napped[i])
3363 kvmppc_ipi_thread(pcpu + i);
3364 cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
3365 }
3366
3367 spin_unlock(&vc->lock);
3368
3369
3370 smp_mb();
3371
3372 preempt_enable();
3373
3374 for (sub = 0; sub < core_info.n_subcores; ++sub) {
3375 pvc = core_info.vc[sub];
3376 post_guest_process(pvc, pvc == vc);
3377 }
3378
3379 spin_lock(&vc->lock);
3380
3381 out:
3382 vc->vcore_state = VCORE_INACTIVE;
3383 trace_kvmppc_run_core(vc, 1);
3384 }
3385
3386
3387
3388
3389 static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit,
3390 unsigned long lpcr)
3391 {
3392 struct kvmppc_vcore *vc = vcpu->arch.vcore;
3393 s64 hdec;
3394 u64 tb, purr, spurr;
3395 int trap;
3396 unsigned long host_hfscr = mfspr(SPRN_HFSCR);
3397 unsigned long host_ciabr = mfspr(SPRN_CIABR);
3398 unsigned long host_dawr = mfspr(SPRN_DAWR);
3399 unsigned long host_dawrx = mfspr(SPRN_DAWRX);
3400 unsigned long host_psscr = mfspr(SPRN_PSSCR);
3401 unsigned long host_pidr = mfspr(SPRN_PID);
3402
3403 hdec = time_limit - mftb();
3404 if (hdec < 0)
3405 return BOOK3S_INTERRUPT_HV_DECREMENTER;
3406 mtspr(SPRN_HDEC, hdec);
3407
3408 if (vc->tb_offset) {
3409 u64 new_tb = mftb() + vc->tb_offset;
3410 mtspr(SPRN_TBU40, new_tb);
3411 tb = mftb();
3412 if ((tb & 0xffffff) < (new_tb & 0xffffff))
3413 mtspr(SPRN_TBU40, new_tb + 0x1000000);
3414 vc->tb_offset_applied = vc->tb_offset;
3415 }
3416
3417 if (vc->pcr)
3418 mtspr(SPRN_PCR, vc->pcr | PCR_MASK);
3419 mtspr(SPRN_DPDES, vc->dpdes);
3420 mtspr(SPRN_VTB, vc->vtb);
3421
3422 local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR);
3423 local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR);
3424 mtspr(SPRN_PURR, vcpu->arch.purr);
3425 mtspr(SPRN_SPURR, vcpu->arch.spurr);
3426
3427 if (dawr_enabled()) {
3428 mtspr(SPRN_DAWR, vcpu->arch.dawr);
3429 mtspr(SPRN_DAWRX, vcpu->arch.dawrx);
3430 }
3431 mtspr(SPRN_CIABR, vcpu->arch.ciabr);
3432 mtspr(SPRN_IC, vcpu->arch.ic);
3433 mtspr(SPRN_PID, vcpu->arch.pid);
3434
3435 mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC |
3436 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
3437
3438 mtspr(SPRN_HFSCR, vcpu->arch.hfscr);
3439
3440 mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0);
3441 mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1);
3442 mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2);
3443 mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3);
3444
3445 mtspr(SPRN_AMOR, ~0UL);
3446
3447 mtspr(SPRN_LPCR, lpcr);
3448 isync();
3449
3450 kvmppc_xive_push_vcpu(vcpu);
3451
3452 mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0);
3453 mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1);
3454
3455 trap = __kvmhv_vcpu_entry_p9(vcpu);
3456
3457
3458 purr = mfspr(SPRN_PURR);
3459 spurr = mfspr(SPRN_SPURR);
3460 mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr +
3461 purr - vcpu->arch.purr);
3462 mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr +
3463 spurr - vcpu->arch.spurr);
3464 vcpu->arch.purr = purr;
3465 vcpu->arch.spurr = spurr;
3466
3467 vcpu->arch.ic = mfspr(SPRN_IC);
3468 vcpu->arch.pid = mfspr(SPRN_PID);
3469 vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS;
3470
3471 vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0);
3472 vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1);
3473 vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2);
3474 vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3);
3475
3476
3477 mtspr(SPRN_PSSCR, host_psscr |
3478 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
3479 mtspr(SPRN_HFSCR, host_hfscr);
3480 mtspr(SPRN_CIABR, host_ciabr);
3481 mtspr(SPRN_DAWR, host_dawr);
3482 mtspr(SPRN_DAWRX, host_dawrx);
3483 mtspr(SPRN_PID, host_pidr);
3484
3485
3486
3487
3488
3489 asm volatile("eieio; tlbsync; ptesync");
3490
3491 mtspr(SPRN_LPID, vcpu->kvm->arch.host_lpid);
3492 isync();
3493
3494 vc->dpdes = mfspr(SPRN_DPDES);
3495 vc->vtb = mfspr(SPRN_VTB);
3496 mtspr(SPRN_DPDES, 0);
3497 if (vc->pcr)
3498 mtspr(SPRN_PCR, PCR_MASK);
3499
3500 if (vc->tb_offset_applied) {
3501 u64 new_tb = mftb() - vc->tb_offset_applied;
3502 mtspr(SPRN_TBU40, new_tb);
3503 tb = mftb();
3504 if ((tb & 0xffffff) < (new_tb & 0xffffff))
3505 mtspr(SPRN_TBU40, new_tb + 0x1000000);
3506 vc->tb_offset_applied = 0;
3507 }
3508
3509 mtspr(SPRN_HDEC, 0x7fffffff);
3510 mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr);
3511
3512 return trap;
3513 }
3514
3515
3516
3517
3518
3519 int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
3520 unsigned long lpcr)
3521 {
3522 struct kvmppc_vcore *vc = vcpu->arch.vcore;
3523 unsigned long host_dscr = mfspr(SPRN_DSCR);
3524 unsigned long host_tidr = mfspr(SPRN_TIDR);
3525 unsigned long host_iamr = mfspr(SPRN_IAMR);
3526 unsigned long host_amr = mfspr(SPRN_AMR);
3527 s64 dec;
3528 u64 tb;
3529 int trap, save_pmu;
3530
3531 dec = mfspr(SPRN_DEC);
3532 tb = mftb();
3533 if (dec < 512)
3534 return BOOK3S_INTERRUPT_HV_DECREMENTER;
3535 local_paca->kvm_hstate.dec_expires = dec + tb;
3536 if (local_paca->kvm_hstate.dec_expires < time_limit)
3537 time_limit = local_paca->kvm_hstate.dec_expires;
3538
3539 vcpu->arch.ceded = 0;
3540
3541 kvmhv_save_host_pmu();
3542
3543 kvmppc_subcore_enter_guest();
3544
3545 vc->entry_exit_map = 1;
3546 vc->in_guest = 1;
3547
3548 if (vcpu->arch.vpa.pinned_addr) {
3549 struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3550 u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
3551 lp->yield_count = cpu_to_be32(yield_count);
3552 vcpu->arch.vpa.dirty = 1;
3553 }
3554
3555 if (cpu_has_feature(CPU_FTR_TM) ||
3556 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
3557 kvmppc_restore_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
3558
3559 kvmhv_load_guest_pmu(vcpu);
3560
3561 msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
3562 load_fp_state(&vcpu->arch.fp);
3563 #ifdef CONFIG_ALTIVEC
3564 load_vr_state(&vcpu->arch.vr);
3565 #endif
3566 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
3567
3568 mtspr(SPRN_DSCR, vcpu->arch.dscr);
3569 mtspr(SPRN_IAMR, vcpu->arch.iamr);
3570 mtspr(SPRN_PSPB, vcpu->arch.pspb);
3571 mtspr(SPRN_FSCR, vcpu->arch.fscr);
3572 mtspr(SPRN_TAR, vcpu->arch.tar);
3573 mtspr(SPRN_EBBHR, vcpu->arch.ebbhr);
3574 mtspr(SPRN_EBBRR, vcpu->arch.ebbrr);
3575 mtspr(SPRN_BESCR, vcpu->arch.bescr);
3576 mtspr(SPRN_WORT, vcpu->arch.wort);
3577 mtspr(SPRN_TIDR, vcpu->arch.tid);
3578 mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
3579 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
3580 mtspr(SPRN_AMR, vcpu->arch.amr);
3581 mtspr(SPRN_UAMOR, vcpu->arch.uamor);
3582
3583 if (!(vcpu->arch.ctrl & 1))
3584 mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1);
3585
3586 mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb());
3587
3588 if (kvmhv_on_pseries()) {
3589
3590
3591
3592
3593
3594
3595 unsigned long host_psscr;
3596
3597 struct hv_guest_state hvregs;
3598
3599 host_psscr = mfspr(SPRN_PSSCR_PR);
3600 mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
3601 kvmhv_save_hv_regs(vcpu, &hvregs);
3602 hvregs.lpcr = lpcr;
3603 vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
3604 hvregs.version = HV_GUEST_STATE_VERSION;
3605 if (vcpu->arch.nested) {
3606 hvregs.lpid = vcpu->arch.nested->shadow_lpid;
3607 hvregs.vcpu_token = vcpu->arch.nested_vcpu_id;
3608 } else {
3609 hvregs.lpid = vcpu->kvm->arch.lpid;
3610 hvregs.vcpu_token = vcpu->vcpu_id;
3611 }
3612 hvregs.hdec_expiry = time_limit;
3613 trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs),
3614 __pa(&vcpu->arch.regs));
3615 kvmhv_restore_hv_return_state(vcpu, &hvregs);
3616 vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
3617 vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
3618 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
3619 vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
3620 mtspr(SPRN_PSSCR_PR, host_psscr);
3621
3622
3623 if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested &&
3624 kvmppc_get_gpr(vcpu, 3) == H_CEDE) {
3625 kvmppc_nested_cede(vcpu);
3626 kvmppc_set_gpr(vcpu, 3, 0);
3627 trap = 0;
3628 }
3629 } else {
3630 trap = kvmhv_load_hv_regs_and_go(vcpu, time_limit, lpcr);
3631 }
3632
3633 vcpu->arch.slb_max = 0;
3634 dec = mfspr(SPRN_DEC);
3635 if (!(lpcr & LPCR_LD))
3636 dec = (s32) dec;
3637 tb = mftb();
3638 vcpu->arch.dec_expires = dec + tb;
3639 vcpu->cpu = -1;
3640 vcpu->arch.thread_cpu = -1;
3641 vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
3642
3643 vcpu->arch.iamr = mfspr(SPRN_IAMR);
3644 vcpu->arch.pspb = mfspr(SPRN_PSPB);
3645 vcpu->arch.fscr = mfspr(SPRN_FSCR);
3646 vcpu->arch.tar = mfspr(SPRN_TAR);
3647 vcpu->arch.ebbhr = mfspr(SPRN_EBBHR);
3648 vcpu->arch.ebbrr = mfspr(SPRN_EBBRR);
3649 vcpu->arch.bescr = mfspr(SPRN_BESCR);
3650 vcpu->arch.wort = mfspr(SPRN_WORT);
3651 vcpu->arch.tid = mfspr(SPRN_TIDR);
3652 vcpu->arch.amr = mfspr(SPRN_AMR);
3653 vcpu->arch.uamor = mfspr(SPRN_UAMOR);
3654 vcpu->arch.dscr = mfspr(SPRN_DSCR);
3655
3656 mtspr(SPRN_PSPB, 0);
3657 mtspr(SPRN_WORT, 0);
3658 mtspr(SPRN_UAMOR, 0);
3659 mtspr(SPRN_DSCR, host_dscr);
3660 mtspr(SPRN_TIDR, host_tidr);
3661 mtspr(SPRN_IAMR, host_iamr);
3662 mtspr(SPRN_PSPB, 0);
3663
3664 if (host_amr != vcpu->arch.amr)
3665 mtspr(SPRN_AMR, host_amr);
3666
3667 msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
3668 store_fp_state(&vcpu->arch.fp);
3669 #ifdef CONFIG_ALTIVEC
3670 store_vr_state(&vcpu->arch.vr);
3671 #endif
3672 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
3673
3674 if (cpu_has_feature(CPU_FTR_TM) ||
3675 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
3676 kvmppc_save_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
3677
3678 save_pmu = 1;
3679 if (vcpu->arch.vpa.pinned_addr) {
3680 struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3681 u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
3682 lp->yield_count = cpu_to_be32(yield_count);
3683 vcpu->arch.vpa.dirty = 1;
3684 save_pmu = lp->pmcregs_in_use;
3685 }
3686
3687 save_pmu |= nesting_enabled(vcpu->kvm);
3688
3689 kvmhv_save_guest_pmu(vcpu, save_pmu);
3690
3691 vc->entry_exit_map = 0x101;
3692 vc->in_guest = 0;
3693
3694 mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb());
3695 mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
3696
3697 kvmhv_load_host_pmu();
3698
3699 kvmppc_subcore_exit_guest();
3700
3701 return trap;
3702 }
3703
3704
3705
3706
3707
3708 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
3709 struct kvm_vcpu *vcpu, int wait_state)
3710 {
3711 DEFINE_WAIT(wait);
3712
3713 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
3714 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
3715 spin_unlock(&vc->lock);
3716 schedule();
3717 spin_lock(&vc->lock);
3718 }
3719 finish_wait(&vcpu->arch.cpu_run, &wait);
3720 }
3721
3722 static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
3723 {
3724 if (!halt_poll_ns_grow)
3725 return;
3726
3727 vc->halt_poll_ns *= halt_poll_ns_grow;
3728 if (vc->halt_poll_ns < halt_poll_ns_grow_start)
3729 vc->halt_poll_ns = halt_poll_ns_grow_start;
3730 }
3731
3732 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
3733 {
3734 if (halt_poll_ns_shrink == 0)
3735 vc->halt_poll_ns = 0;
3736 else
3737 vc->halt_poll_ns /= halt_poll_ns_shrink;
3738 }
3739
3740 #ifdef CONFIG_KVM_XICS
3741 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3742 {
3743 if (!xics_on_xive())
3744 return false;
3745 return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
3746 vcpu->arch.xive_saved_state.cppr;
3747 }
3748 #else
3749 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3750 {
3751 return false;
3752 }
3753 #endif
3754
3755 static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
3756 {
3757 if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
3758 kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
3759 return true;
3760
3761 return false;
3762 }
3763
3764
3765
3766
3767
3768 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
3769 {
3770 struct kvm_vcpu *vcpu;
3771 int i;
3772
3773 for_each_runnable_thread(i, vcpu, vc) {
3774 if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
3775 return 1;
3776 }
3777
3778 return 0;
3779 }
3780
3781
3782
3783
3784
3785 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
3786 {
3787 ktime_t cur, start_poll, start_wait;
3788 int do_sleep = 1;
3789 u64 block_ns;
3790 DECLARE_SWAITQUEUE(wait);
3791
3792
3793 cur = start_poll = ktime_get();
3794 if (vc->halt_poll_ns) {
3795 ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
3796 ++vc->runner->stat.halt_attempted_poll;
3797
3798 vc->vcore_state = VCORE_POLLING;
3799 spin_unlock(&vc->lock);
3800
3801 do {
3802 if (kvmppc_vcore_check_block(vc)) {
3803 do_sleep = 0;
3804 break;
3805 }
3806 cur = ktime_get();
3807 } while (single_task_running() && ktime_before(cur, stop));
3808
3809 spin_lock(&vc->lock);
3810 vc->vcore_state = VCORE_INACTIVE;
3811
3812 if (!do_sleep) {
3813 ++vc->runner->stat.halt_successful_poll;
3814 goto out;
3815 }
3816 }
3817
3818 prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE);
3819
3820 if (kvmppc_vcore_check_block(vc)) {
3821 finish_swait(&vc->wq, &wait);
3822 do_sleep = 0;
3823
3824 if (vc->halt_poll_ns)
3825 ++vc->runner->stat.halt_successful_poll;
3826 goto out;
3827 }
3828
3829 start_wait = ktime_get();
3830
3831 vc->vcore_state = VCORE_SLEEPING;
3832 trace_kvmppc_vcore_blocked(vc, 0);
3833 spin_unlock(&vc->lock);
3834 schedule();
3835 finish_swait(&vc->wq, &wait);
3836 spin_lock(&vc->lock);
3837 vc->vcore_state = VCORE_INACTIVE;
3838 trace_kvmppc_vcore_blocked(vc, 1);
3839 ++vc->runner->stat.halt_successful_wait;
3840
3841 cur = ktime_get();
3842
3843 out:
3844 block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
3845
3846
3847 if (do_sleep) {
3848 vc->runner->stat.halt_wait_ns +=
3849 ktime_to_ns(cur) - ktime_to_ns(start_wait);
3850
3851 if (vc->halt_poll_ns)
3852 vc->runner->stat.halt_poll_fail_ns +=
3853 ktime_to_ns(start_wait) -
3854 ktime_to_ns(start_poll);
3855 } else {
3856
3857 if (vc->halt_poll_ns)
3858 vc->runner->stat.halt_poll_success_ns +=
3859 ktime_to_ns(cur) -
3860 ktime_to_ns(start_poll);
3861 }
3862
3863
3864 if (halt_poll_ns) {
3865 if (block_ns <= vc->halt_poll_ns)
3866 ;
3867
3868 else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
3869 shrink_halt_poll_ns(vc);
3870
3871 else if (vc->halt_poll_ns < halt_poll_ns &&
3872 block_ns < halt_poll_ns)
3873 grow_halt_poll_ns(vc);
3874 if (vc->halt_poll_ns > halt_poll_ns)
3875 vc->halt_poll_ns = halt_poll_ns;
3876 } else
3877 vc->halt_poll_ns = 0;
3878
3879 trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
3880 }
3881
3882
3883
3884
3885
3886
3887 static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu)
3888 {
3889 int r = 0;
3890 struct kvm *kvm = vcpu->kvm;
3891
3892 mutex_lock(&kvm->arch.mmu_setup_lock);
3893 if (!kvm->arch.mmu_ready) {
3894 if (!kvm_is_radix(kvm))
3895 r = kvmppc_hv_setup_htab_rma(vcpu);
3896 if (!r) {
3897 if (cpu_has_feature(CPU_FTR_ARCH_300))
3898 kvmppc_setup_partition_table(kvm);
3899 kvm->arch.mmu_ready = 1;
3900 }
3901 }
3902 mutex_unlock(&kvm->arch.mmu_setup_lock);
3903 return r;
3904 }
3905
3906 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
3907 {
3908 int n_ceded, i, r;
3909 struct kvmppc_vcore *vc;
3910 struct kvm_vcpu *v;
3911
3912 trace_kvmppc_run_vcpu_enter(vcpu);
3913
3914 kvm_run->exit_reason = 0;
3915 vcpu->arch.ret = RESUME_GUEST;
3916 vcpu->arch.trap = 0;
3917 kvmppc_update_vpas(vcpu);
3918
3919
3920
3921
3922 vc = vcpu->arch.vcore;
3923 spin_lock(&vc->lock);
3924 vcpu->arch.ceded = 0;
3925 vcpu->arch.run_task = current;
3926 vcpu->arch.kvm_run = kvm_run;
3927 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
3928 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
3929 vcpu->arch.busy_preempt = TB_NIL;
3930 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
3931 ++vc->n_runnable;
3932
3933
3934
3935
3936
3937
3938 if (!signal_pending(current)) {
3939 if ((vc->vcore_state == VCORE_PIGGYBACK ||
3940 vc->vcore_state == VCORE_RUNNING) &&
3941 !VCORE_IS_EXITING(vc)) {
3942 kvmppc_create_dtl_entry(vcpu, vc);
3943 kvmppc_start_thread(vcpu, vc);
3944 trace_kvm_guest_enter(vcpu);
3945 } else if (vc->vcore_state == VCORE_SLEEPING) {
3946 swake_up_one(&vc->wq);
3947 }
3948
3949 }
3950
3951 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
3952 !signal_pending(current)) {
3953
3954 if (!vcpu->kvm->arch.mmu_ready) {
3955 spin_unlock(&vc->lock);
3956 r = kvmhv_setup_mmu(vcpu);
3957 spin_lock(&vc->lock);
3958 if (r) {
3959 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3960 kvm_run->fail_entry.
3961 hardware_entry_failure_reason = 0;
3962 vcpu->arch.ret = r;
3963 break;
3964 }
3965 }
3966
3967 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
3968 kvmppc_vcore_end_preempt(vc);
3969
3970 if (vc->vcore_state != VCORE_INACTIVE) {
3971 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
3972 continue;
3973 }
3974 for_each_runnable_thread(i, v, vc) {
3975 kvmppc_core_prepare_to_enter(v);
3976 if (signal_pending(v->arch.run_task)) {
3977 kvmppc_remove_runnable(vc, v);
3978 v->stat.signal_exits++;
3979 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
3980 v->arch.ret = -EINTR;
3981 wake_up(&v->arch.cpu_run);
3982 }
3983 }
3984 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
3985 break;
3986 n_ceded = 0;
3987 for_each_runnable_thread(i, v, vc) {
3988 if (!kvmppc_vcpu_woken(v))
3989 n_ceded += v->arch.ceded;
3990 else
3991 v->arch.ceded = 0;
3992 }
3993 vc->runner = vcpu;
3994 if (n_ceded == vc->n_runnable) {
3995 kvmppc_vcore_blocked(vc);
3996 } else if (need_resched()) {
3997 kvmppc_vcore_preempt(vc);
3998
3999 cond_resched_lock(&vc->lock);
4000 if (vc->vcore_state == VCORE_PREEMPT)
4001 kvmppc_vcore_end_preempt(vc);
4002 } else {
4003 kvmppc_run_core(vc);
4004 }
4005 vc->runner = NULL;
4006 }
4007
4008 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
4009 (vc->vcore_state == VCORE_RUNNING ||
4010 vc->vcore_state == VCORE_EXITING ||
4011 vc->vcore_state == VCORE_PIGGYBACK))
4012 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
4013
4014 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
4015 kvmppc_vcore_end_preempt(vc);
4016
4017 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
4018 kvmppc_remove_runnable(vc, vcpu);
4019 vcpu->stat.signal_exits++;
4020 kvm_run->exit_reason = KVM_EXIT_INTR;
4021 vcpu->arch.ret = -EINTR;
4022 }
4023
4024 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
4025
4026 i = -1;
4027 v = next_runnable_thread(vc, &i);
4028 wake_up(&v->arch.cpu_run);
4029 }
4030
4031 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run);
4032 spin_unlock(&vc->lock);
4033 return vcpu->arch.ret;
4034 }
4035
4036 int kvmhv_run_single_vcpu(struct kvm_run *kvm_run,
4037 struct kvm_vcpu *vcpu, u64 time_limit,
4038 unsigned long lpcr)
4039 {
4040 int trap, r, pcpu;
4041 int srcu_idx, lpid;
4042 struct kvmppc_vcore *vc;
4043 struct kvm *kvm = vcpu->kvm;
4044 struct kvm_nested_guest *nested = vcpu->arch.nested;
4045
4046 trace_kvmppc_run_vcpu_enter(vcpu);
4047
4048 kvm_run->exit_reason = 0;
4049 vcpu->arch.ret = RESUME_GUEST;
4050 vcpu->arch.trap = 0;
4051
4052 vc = vcpu->arch.vcore;
4053 vcpu->arch.ceded = 0;
4054 vcpu->arch.run_task = current;
4055 vcpu->arch.kvm_run = kvm_run;
4056 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
4057 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4058 vcpu->arch.busy_preempt = TB_NIL;
4059 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
4060 vc->runnable_threads[0] = vcpu;
4061 vc->n_runnable = 1;
4062 vc->runner = vcpu;
4063
4064
4065 if (!kvm->arch.mmu_ready)
4066 kvmhv_setup_mmu(vcpu);
4067
4068 if (need_resched())
4069 cond_resched();
4070
4071 kvmppc_update_vpas(vcpu);
4072
4073 init_vcore_to_run(vc);
4074 vc->preempt_tb = TB_NIL;
4075
4076 preempt_disable();
4077 pcpu = smp_processor_id();
4078 vc->pcpu = pcpu;
4079 kvmppc_prepare_radix_vcpu(vcpu, pcpu);
4080
4081 local_irq_disable();
4082 hard_irq_disable();
4083 if (signal_pending(current))
4084 goto sigpend;
4085 if (lazy_irq_pending() || need_resched() || !kvm->arch.mmu_ready)
4086 goto out;
4087
4088 if (!nested) {
4089 kvmppc_core_prepare_to_enter(vcpu);
4090 if (vcpu->arch.doorbell_request) {
4091 vc->dpdes = 1;
4092 smp_wmb();
4093 vcpu->arch.doorbell_request = 0;
4094 }
4095 if (test_bit(BOOK3S_IRQPRIO_EXTERNAL,
4096 &vcpu->arch.pending_exceptions))
4097 lpcr |= LPCR_MER;
4098 } else if (vcpu->arch.pending_exceptions ||
4099 vcpu->arch.doorbell_request ||
4100 xive_interrupt_pending(vcpu)) {
4101 vcpu->arch.ret = RESUME_HOST;
4102 goto out;
4103 }
4104
4105 kvmppc_clear_host_core(pcpu);
4106
4107 local_paca->kvm_hstate.tid = 0;
4108 local_paca->kvm_hstate.napping = 0;
4109 local_paca->kvm_hstate.kvm_split_mode = NULL;
4110 kvmppc_start_thread(vcpu, vc);
4111 kvmppc_create_dtl_entry(vcpu, vc);
4112 trace_kvm_guest_enter(vcpu);
4113
4114 vc->vcore_state = VCORE_RUNNING;
4115 trace_kvmppc_run_core(vc, 0);
4116
4117 if (cpu_has_feature(CPU_FTR_HVMODE)) {
4118 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
4119 mtspr(SPRN_LPID, lpid);
4120 isync();
4121 kvmppc_check_need_tlb_flush(kvm, pcpu, nested);
4122 }
4123
4124 guest_enter_irqoff();
4125
4126 srcu_idx = srcu_read_lock(&kvm->srcu);
4127
4128 this_cpu_disable_ftrace();
4129
4130
4131 trace_hardirqs_on();
4132
4133 trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr);
4134 vcpu->arch.trap = trap;
4135
4136 trace_hardirqs_off();
4137
4138 this_cpu_enable_ftrace();
4139
4140 srcu_read_unlock(&kvm->srcu, srcu_idx);
4141
4142 if (cpu_has_feature(CPU_FTR_HVMODE)) {
4143 mtspr(SPRN_LPID, kvm->arch.host_lpid);
4144 isync();
4145 }
4146
4147 set_irq_happened(trap);
4148
4149 kvmppc_set_host_core(pcpu);
4150
4151 local_irq_enable();
4152 guest_exit();
4153
4154 cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest);
4155
4156 preempt_enable();
4157
4158
4159
4160
4161
4162
4163 if (kvmppc_core_pending_dec(vcpu) &&
4164 ((get_tb() < vcpu->arch.dec_expires) ||
4165 (trap == BOOK3S_INTERRUPT_SYSCALL &&
4166 kvmppc_get_gpr(vcpu, 3) == H_ENTER_NESTED)))
4167 kvmppc_core_dequeue_dec(vcpu);
4168
4169 trace_kvm_guest_exit(vcpu);
4170 r = RESUME_GUEST;
4171 if (trap) {
4172 if (!nested)
4173 r = kvmppc_handle_exit_hv(kvm_run, vcpu, current);
4174 else
4175 r = kvmppc_handle_nested_exit(kvm_run, vcpu);
4176 }
4177 vcpu->arch.ret = r;
4178
4179 if (is_kvmppc_resume_guest(r) && vcpu->arch.ceded &&
4180 !kvmppc_vcpu_woken(vcpu)) {
4181 kvmppc_set_timer(vcpu);
4182 while (vcpu->arch.ceded && !kvmppc_vcpu_woken(vcpu)) {
4183 if (signal_pending(current)) {
4184 vcpu->stat.signal_exits++;
4185 kvm_run->exit_reason = KVM_EXIT_INTR;
4186 vcpu->arch.ret = -EINTR;
4187 break;
4188 }
4189 spin_lock(&vc->lock);
4190 kvmppc_vcore_blocked(vc);
4191 spin_unlock(&vc->lock);
4192 }
4193 }
4194 vcpu->arch.ceded = 0;
4195
4196 vc->vcore_state = VCORE_INACTIVE;
4197 trace_kvmppc_run_core(vc, 1);
4198
4199 done:
4200 kvmppc_remove_runnable(vc, vcpu);
4201 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run);
4202
4203 return vcpu->arch.ret;
4204
4205 sigpend:
4206 vcpu->stat.signal_exits++;
4207 kvm_run->exit_reason = KVM_EXIT_INTR;
4208 vcpu->arch.ret = -EINTR;
4209 out:
4210 local_irq_enable();
4211 preempt_enable();
4212 goto done;
4213 }
4214
4215 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
4216 {
4217 int r;
4218 int srcu_idx;
4219 unsigned long ebb_regs[3] = {};
4220 unsigned long user_tar = 0;
4221 unsigned int user_vrsave;
4222 struct kvm *kvm;
4223
4224 if (!vcpu->arch.sane) {
4225 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4226 return -EINVAL;
4227 }
4228
4229
4230
4231
4232
4233
4234
4235 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
4236 if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
4237 (current->thread.regs->msr & MSR_TM)) {
4238 if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
4239 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4240 run->fail_entry.hardware_entry_failure_reason = 0;
4241 return -EINVAL;
4242 }
4243
4244 mtmsr(mfmsr() | MSR_TM);
4245 current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
4246 current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
4247 current->thread.tm_texasr = mfspr(SPRN_TEXASR);
4248 current->thread.regs->msr &= ~MSR_TM;
4249 }
4250 #endif
4251
4252
4253
4254
4255
4256 if (!vcpu->arch.online) {
4257 atomic_inc(&vcpu->arch.vcore->online_count);
4258 vcpu->arch.online = 1;
4259 }
4260
4261 kvmppc_core_prepare_to_enter(vcpu);
4262
4263
4264 if (signal_pending(current)) {
4265 run->exit_reason = KVM_EXIT_INTR;
4266 return -EINTR;
4267 }
4268
4269 kvm = vcpu->kvm;
4270 atomic_inc(&kvm->arch.vcpus_running);
4271
4272 smp_mb();
4273
4274 flush_all_to_thread(current);
4275
4276
4277 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4278 ebb_regs[0] = mfspr(SPRN_EBBHR);
4279 ebb_regs[1] = mfspr(SPRN_EBBRR);
4280 ebb_regs[2] = mfspr(SPRN_BESCR);
4281 user_tar = mfspr(SPRN_TAR);
4282 }
4283 user_vrsave = mfspr(SPRN_VRSAVE);
4284
4285 vcpu->arch.wqp = &vcpu->arch.vcore->wq;
4286 vcpu->arch.pgdir = current->mm->pgd;
4287 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4288
4289 do {
4290
4291
4292
4293
4294
4295
4296
4297
4298 if (kvm->arch.threads_indep && kvm_is_radix(kvm) &&
4299 !no_mixing_hpt_and_radix)
4300 r = kvmhv_run_single_vcpu(run, vcpu, ~(u64)0,
4301 vcpu->arch.vcore->lpcr);
4302 else
4303 r = kvmppc_run_vcpu(run, vcpu);
4304
4305 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
4306 !(vcpu->arch.shregs.msr & MSR_PR)) {
4307 trace_kvm_hcall_enter(vcpu);
4308 r = kvmppc_pseries_do_hcall(vcpu);
4309 trace_kvm_hcall_exit(vcpu, r);
4310 kvmppc_core_prepare_to_enter(vcpu);
4311 } else if (r == RESUME_PAGE_FAULT) {
4312 srcu_idx = srcu_read_lock(&kvm->srcu);
4313 r = kvmppc_book3s_hv_page_fault(run, vcpu,
4314 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
4315 srcu_read_unlock(&kvm->srcu, srcu_idx);
4316 } else if (r == RESUME_PASSTHROUGH) {
4317 if (WARN_ON(xics_on_xive()))
4318 r = H_SUCCESS;
4319 else
4320 r = kvmppc_xics_rm_complete(vcpu, 0);
4321 }
4322 } while (is_kvmppc_resume_guest(r));
4323
4324
4325 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4326 mtspr(SPRN_EBBHR, ebb_regs[0]);
4327 mtspr(SPRN_EBBRR, ebb_regs[1]);
4328 mtspr(SPRN_BESCR, ebb_regs[2]);
4329 mtspr(SPRN_TAR, user_tar);
4330 mtspr(SPRN_FSCR, current->thread.fscr);
4331 }
4332 mtspr(SPRN_VRSAVE, user_vrsave);
4333
4334 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
4335 atomic_dec(&kvm->arch.vcpus_running);
4336 return r;
4337 }
4338
4339 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
4340 int shift, int sllp)
4341 {
4342 (*sps)->page_shift = shift;
4343 (*sps)->slb_enc = sllp;
4344 (*sps)->enc[0].page_shift = shift;
4345 (*sps)->enc[0].pte_enc = kvmppc_pgsize_lp_encoding(shift, shift);
4346
4347
4348
4349 if (shift != 24) {
4350 int penc = kvmppc_pgsize_lp_encoding(shift, 24);
4351 if (penc != -1) {
4352 (*sps)->enc[1].page_shift = 24;
4353 (*sps)->enc[1].pte_enc = penc;
4354 }
4355 }
4356 (*sps)++;
4357 }
4358
4359 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
4360 struct kvm_ppc_smmu_info *info)
4361 {
4362 struct kvm_ppc_one_seg_page_size *sps;
4363
4364
4365
4366
4367
4368
4369 info->data_keys = 32;
4370 info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
4371
4372
4373 info->flags = KVM_PPC_PAGE_SIZES_REAL | KVM_PPC_1T_SEGMENTS;
4374 info->slb_size = 32;
4375
4376
4377 sps = &info->sps[0];
4378 kvmppc_add_seg_page_size(&sps, 12, 0);
4379 kvmppc_add_seg_page_size(&sps, 16, SLB_VSID_L | SLB_VSID_LP_01);
4380 kvmppc_add_seg_page_size(&sps, 24, SLB_VSID_L);
4381
4382
4383 if (kvmhv_on_pseries())
4384 info->flags |= KVM_PPC_NO_HASH;
4385
4386 return 0;
4387 }
4388
4389
4390
4391
4392 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
4393 struct kvm_dirty_log *log)
4394 {
4395 struct kvm_memslots *slots;
4396 struct kvm_memory_slot *memslot;
4397 int i, r;
4398 unsigned long n;
4399 unsigned long *buf, *p;
4400 struct kvm_vcpu *vcpu;
4401
4402 mutex_lock(&kvm->slots_lock);
4403
4404 r = -EINVAL;
4405 if (log->slot >= KVM_USER_MEM_SLOTS)
4406 goto out;
4407
4408 slots = kvm_memslots(kvm);
4409 memslot = id_to_memslot(slots, log->slot);
4410 r = -ENOENT;
4411 if (!memslot->dirty_bitmap)
4412 goto out;
4413
4414
4415
4416
4417
4418 n = kvm_dirty_bitmap_bytes(memslot);
4419 buf = memslot->dirty_bitmap + n / sizeof(long);
4420 memset(buf, 0, n);
4421
4422 if (kvm_is_radix(kvm))
4423 r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
4424 else
4425 r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
4426 if (r)
4427 goto out;
4428
4429
4430
4431
4432
4433
4434
4435 p = memslot->dirty_bitmap;
4436 for (i = 0; i < n / sizeof(long); ++i)
4437 buf[i] |= xchg(&p[i], 0);
4438
4439
4440
4441 kvm_for_each_vcpu(i, vcpu, kvm) {
4442 spin_lock(&vcpu->arch.vpa_update_lock);
4443 kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
4444 kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
4445 spin_unlock(&vcpu->arch.vpa_update_lock);
4446 }
4447
4448 r = -EFAULT;
4449 if (copy_to_user(log->dirty_bitmap, buf, n))
4450 goto out;
4451
4452 r = 0;
4453 out:
4454 mutex_unlock(&kvm->slots_lock);
4455 return r;
4456 }
4457
4458 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free,
4459 struct kvm_memory_slot *dont)
4460 {
4461 if (!dont || free->arch.rmap != dont->arch.rmap) {
4462 vfree(free->arch.rmap);
4463 free->arch.rmap = NULL;
4464 }
4465 }
4466
4467 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot,
4468 unsigned long npages)
4469 {
4470 slot->arch.rmap = vzalloc(array_size(npages, sizeof(*slot->arch.rmap)));
4471 if (!slot->arch.rmap)
4472 return -ENOMEM;
4473
4474 return 0;
4475 }
4476
4477 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
4478 struct kvm_memory_slot *memslot,
4479 const struct kvm_userspace_memory_region *mem)
4480 {
4481 return 0;
4482 }
4483
4484 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
4485 const struct kvm_userspace_memory_region *mem,
4486 const struct kvm_memory_slot *old,
4487 const struct kvm_memory_slot *new,
4488 enum kvm_mr_change change)
4489 {
4490 unsigned long npages = mem->memory_size >> PAGE_SHIFT;
4491
4492
4493
4494
4495
4496
4497
4498 if (npages)
4499 atomic64_inc(&kvm->arch.mmio_update);
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514 if (change == KVM_MR_FLAGS_ONLY && kvm_is_radix(kvm) &&
4515 ((new->flags ^ old->flags) & KVM_MEM_LOG_DIRTY_PAGES))
4516 kvmppc_radix_flush_memslot(kvm, old);
4517 }
4518
4519
4520
4521
4522
4523
4524 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
4525 {
4526 long int i;
4527 u32 cores_done = 0;
4528
4529 if ((kvm->arch.lpcr & mask) == lpcr)
4530 return;
4531
4532 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
4533
4534 for (i = 0; i < KVM_MAX_VCORES; ++i) {
4535 struct kvmppc_vcore *vc = kvm->arch.vcores[i];
4536 if (!vc)
4537 continue;
4538 spin_lock(&vc->lock);
4539 vc->lpcr = (vc->lpcr & ~mask) | lpcr;
4540 spin_unlock(&vc->lock);
4541 if (++cores_done >= kvm->arch.online_vcores)
4542 break;
4543 }
4544 }
4545
4546 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
4547 {
4548 return;
4549 }
4550
4551 void kvmppc_setup_partition_table(struct kvm *kvm)
4552 {
4553 unsigned long dw0, dw1;
4554
4555 if (!kvm_is_radix(kvm)) {
4556
4557 dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
4558 ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
4559
4560 dw0 |= kvm->arch.sdr1;
4561
4562
4563 dw1 = kvm->arch.process_table;
4564 } else {
4565 dw0 = PATB_HR | radix__get_tree_size() |
4566 __pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
4567 dw1 = PATB_GR | kvm->arch.process_table;
4568 }
4569 kvmhv_set_ptbl_entry(kvm->arch.lpid, dw0, dw1);
4570 }
4571
4572
4573
4574
4575
4576 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
4577 {
4578 int err = 0;
4579 struct kvm *kvm = vcpu->kvm;
4580 unsigned long hva;
4581 struct kvm_memory_slot *memslot;
4582 struct vm_area_struct *vma;
4583 unsigned long lpcr = 0, senc;
4584 unsigned long psize, porder;
4585 int srcu_idx;
4586
4587
4588 if (!kvm->arch.hpt.virt) {
4589 int order = KVM_DEFAULT_HPT_ORDER;
4590 struct kvm_hpt_info info;
4591
4592 err = kvmppc_allocate_hpt(&info, order);
4593
4594
4595
4596 while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
4597 err = kvmppc_allocate_hpt(&info, order);
4598
4599 if (err < 0) {
4600 pr_err("KVM: Couldn't alloc HPT\n");
4601 goto out;
4602 }
4603
4604 kvmppc_set_hpt(kvm, &info);
4605 }
4606
4607
4608 srcu_idx = srcu_read_lock(&kvm->srcu);
4609 memslot = gfn_to_memslot(kvm, 0);
4610
4611
4612 err = -EINVAL;
4613 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
4614 goto out_srcu;
4615
4616
4617 hva = memslot->userspace_addr;
4618 down_read(¤t->mm->mmap_sem);
4619 vma = find_vma(current->mm, hva);
4620 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
4621 goto up_out;
4622
4623 psize = vma_kernel_pagesize(vma);
4624
4625 up_read(¤t->mm->mmap_sem);
4626
4627
4628 if (psize >= 0x1000000)
4629 psize = 0x1000000;
4630 else if (psize >= 0x10000)
4631 psize = 0x10000;
4632 else
4633 psize = 0x1000;
4634 porder = __ilog2(psize);
4635
4636 senc = slb_pgsize_encoding(psize);
4637 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
4638 (VRMA_VSID << SLB_VSID_SHIFT_1T);
4639
4640 kvmppc_map_vrma(vcpu, memslot, porder);
4641
4642
4643 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
4644
4645 lpcr = senc << (LPCR_VRMASD_SH - 4);
4646 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
4647 }
4648
4649
4650 smp_wmb();
4651 err = 0;
4652 out_srcu:
4653 srcu_read_unlock(&kvm->srcu, srcu_idx);
4654 out:
4655 return err;
4656
4657 up_out:
4658 up_read(¤t->mm->mmap_sem);
4659 goto out_srcu;
4660 }
4661
4662
4663
4664
4665
4666 int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
4667 {
4668 if (nesting_enabled(kvm))
4669 kvmhv_release_all_nested(kvm);
4670 kvmppc_rmap_reset(kvm);
4671 kvm->arch.process_table = 0;
4672
4673 spin_lock(&kvm->mmu_lock);
4674 kvm->arch.radix = 0;
4675 spin_unlock(&kvm->mmu_lock);
4676 kvmppc_free_radix(kvm);
4677 kvmppc_update_lpcr(kvm, LPCR_VPM1,
4678 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
4679 return 0;
4680 }
4681
4682
4683
4684
4685
4686 int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
4687 {
4688 int err;
4689
4690 err = kvmppc_init_vm_radix(kvm);
4691 if (err)
4692 return err;
4693 kvmppc_rmap_reset(kvm);
4694
4695 spin_lock(&kvm->mmu_lock);
4696 kvm->arch.radix = 1;
4697 spin_unlock(&kvm->mmu_lock);
4698 kvmppc_free_hpt(&kvm->arch.hpt);
4699 kvmppc_update_lpcr(kvm, LPCR_UPRT | LPCR_GTSE | LPCR_HR,
4700 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
4701 return 0;
4702 }
4703
4704 #ifdef CONFIG_KVM_XICS
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715 void kvmppc_alloc_host_rm_ops(void)
4716 {
4717 struct kvmppc_host_rm_ops *ops;
4718 unsigned long l_ops;
4719 int cpu, core;
4720 int size;
4721
4722
4723 if (kvmppc_host_rm_ops_hv != NULL)
4724 return;
4725
4726 ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
4727 if (!ops)
4728 return;
4729
4730 size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
4731 ops->rm_core = kzalloc(size, GFP_KERNEL);
4732
4733 if (!ops->rm_core) {
4734 kfree(ops);
4735 return;
4736 }
4737
4738 cpus_read_lock();
4739
4740 for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
4741 if (!cpu_online(cpu))
4742 continue;
4743
4744 core = cpu >> threads_shift;
4745 ops->rm_core[core].rm_state.in_host = 1;
4746 }
4747
4748 ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
4749
4750
4751
4752
4753
4754
4755
4756 smp_wmb();
4757 l_ops = (unsigned long) ops;
4758
4759 if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
4760 cpus_read_unlock();
4761 kfree(ops->rm_core);
4762 kfree(ops);
4763 return;
4764 }
4765
4766 cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
4767 "ppc/kvm_book3s:prepare",
4768 kvmppc_set_host_core,
4769 kvmppc_clear_host_core);
4770 cpus_read_unlock();
4771 }
4772
4773 void kvmppc_free_host_rm_ops(void)
4774 {
4775 if (kvmppc_host_rm_ops_hv) {
4776 cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
4777 kfree(kvmppc_host_rm_ops_hv->rm_core);
4778 kfree(kvmppc_host_rm_ops_hv);
4779 kvmppc_host_rm_ops_hv = NULL;
4780 }
4781 }
4782 #endif
4783
4784 static int kvmppc_core_init_vm_hv(struct kvm *kvm)
4785 {
4786 unsigned long lpcr, lpid;
4787 char buf[32];
4788 int ret;
4789
4790 mutex_init(&kvm->arch.mmu_setup_lock);
4791
4792
4793
4794 lpid = kvmppc_alloc_lpid();
4795 if ((long)lpid < 0)
4796 return -ENOMEM;
4797 kvm->arch.lpid = lpid;
4798
4799 kvmppc_alloc_host_rm_ops();
4800
4801 kvmhv_vm_nested_init(kvm);
4802
4803
4804
4805
4806
4807
4808
4809
4810 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4811 cpumask_setall(&kvm->arch.need_tlb_flush);
4812
4813
4814 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
4815 sizeof(kvm->arch.enabled_hcalls));
4816
4817 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4818 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
4819
4820
4821 if (cpu_has_feature(CPU_FTR_HVMODE)) {
4822 kvm->arch.host_lpid = mfspr(SPRN_LPID);
4823 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
4824 lpcr &= LPCR_PECE | LPCR_LPES;
4825 } else {
4826 lpcr = 0;
4827 }
4828 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
4829 LPCR_VPM0 | LPCR_VPM1;
4830 kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
4831 (VRMA_VSID << SLB_VSID_SHIFT_1T);
4832
4833 if (cpu_has_feature(CPU_FTR_ARCH_207S))
4834 lpcr |= LPCR_ONL;
4835
4836
4837
4838
4839
4840
4841
4842 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4843 lpcr &= ~LPCR_VPM0;
4844 lpcr |= LPCR_HVICE | LPCR_HEIC;
4845
4846
4847
4848
4849
4850 if (xics_on_xive())
4851 lpcr |= LPCR_LPES;
4852 }
4853
4854
4855
4856
4857 if (radix_enabled()) {
4858 kvm->arch.radix = 1;
4859 kvm->arch.mmu_ready = 1;
4860 lpcr &= ~LPCR_VPM1;
4861 lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
4862 ret = kvmppc_init_vm_radix(kvm);
4863 if (ret) {
4864 kvmppc_free_lpid(kvm->arch.lpid);
4865 return ret;
4866 }
4867 kvmppc_setup_partition_table(kvm);
4868 }
4869
4870 kvm->arch.lpcr = lpcr;
4871
4872
4873 kvm->arch.resize_hpt = NULL;
4874
4875
4876
4877
4878
4879 if (radix_enabled())
4880 kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX;
4881 else if (cpu_has_feature(CPU_FTR_ARCH_300))
4882 kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;
4883 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
4884 kvm->arch.tlb_sets = POWER8_TLB_SETS;
4885 else
4886 kvm->arch.tlb_sets = POWER7_TLB_SETS;
4887
4888
4889
4890
4891
4892
4893
4894 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4895 if (!indep_threads_mode && !cpu_has_feature(CPU_FTR_HVMODE)) {
4896 pr_warn("KVM: Ignoring indep_threads_mode=N in nested hypervisor\n");
4897 kvm->arch.threads_indep = true;
4898 } else {
4899 kvm->arch.threads_indep = indep_threads_mode;
4900 }
4901 }
4902 if (!kvm->arch.threads_indep)
4903 kvm_hv_vm_activated();
4904
4905
4906
4907
4908
4909
4910
4911
4912 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4913 kvm->arch.smt_mode = threads_per_subcore;
4914 else
4915 kvm->arch.smt_mode = 1;
4916 kvm->arch.emul_smt_mode = 1;
4917
4918
4919
4920
4921 snprintf(buf, sizeof(buf), "vm%d", current->pid);
4922 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
4923 kvmppc_mmu_debugfs_init(kvm);
4924 if (radix_enabled())
4925 kvmhv_radix_debugfs_init(kvm);
4926
4927 return 0;
4928 }
4929
4930 static void kvmppc_free_vcores(struct kvm *kvm)
4931 {
4932 long int i;
4933
4934 for (i = 0; i < KVM_MAX_VCORES; ++i)
4935 kfree(kvm->arch.vcores[i]);
4936 kvm->arch.online_vcores = 0;
4937 }
4938
4939 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
4940 {
4941 debugfs_remove_recursive(kvm->arch.debugfs_dir);
4942
4943 if (!kvm->arch.threads_indep)
4944 kvm_hv_vm_deactivated();
4945
4946 kvmppc_free_vcores(kvm);
4947
4948
4949 if (kvm_is_radix(kvm))
4950 kvmppc_free_radix(kvm);
4951 else
4952 kvmppc_free_hpt(&kvm->arch.hpt);
4953
4954
4955 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4956 if (nesting_enabled(kvm))
4957 kvmhv_release_all_nested(kvm);
4958 kvm->arch.process_table = 0;
4959 kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0);
4960 }
4961 kvmppc_free_lpid(kvm->arch.lpid);
4962
4963 kvmppc_free_pimap(kvm);
4964 }
4965
4966
4967 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
4968 unsigned int inst, int *advance)
4969 {
4970 return EMULATE_FAIL;
4971 }
4972
4973 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
4974 ulong spr_val)
4975 {
4976 return EMULATE_FAIL;
4977 }
4978
4979 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
4980 ulong *spr_val)
4981 {
4982 return EMULATE_FAIL;
4983 }
4984
4985 static int kvmppc_core_check_processor_compat_hv(void)
4986 {
4987 if (cpu_has_feature(CPU_FTR_HVMODE) &&
4988 cpu_has_feature(CPU_FTR_ARCH_206))
4989 return 0;
4990
4991
4992 if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled())
4993 return 0;
4994
4995 return -EIO;
4996 }
4997
4998 #ifdef CONFIG_KVM_XICS
4999
5000 void kvmppc_free_pimap(struct kvm *kvm)
5001 {
5002 kfree(kvm->arch.pimap);
5003 }
5004
5005 static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
5006 {
5007 return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
5008 }
5009
5010 static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5011 {
5012 struct irq_desc *desc;
5013 struct kvmppc_irq_map *irq_map;
5014 struct kvmppc_passthru_irqmap *pimap;
5015 struct irq_chip *chip;
5016 int i, rc = 0;
5017
5018 if (!kvm_irq_bypass)
5019 return 1;
5020
5021 desc = irq_to_desc(host_irq);
5022 if (!desc)
5023 return -EIO;
5024
5025 mutex_lock(&kvm->lock);
5026
5027 pimap = kvm->arch.pimap;
5028 if (pimap == NULL) {
5029
5030 pimap = kvmppc_alloc_pimap();
5031 if (pimap == NULL) {
5032 mutex_unlock(&kvm->lock);
5033 return -ENOMEM;
5034 }
5035 kvm->arch.pimap = pimap;
5036 }
5037
5038
5039
5040
5041
5042
5043 chip = irq_data_get_irq_chip(&desc->irq_data);
5044 if (!chip || !(is_pnv_opal_msi(chip) || is_xive_irq(chip))) {
5045 pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
5046 host_irq, guest_gsi);
5047 mutex_unlock(&kvm->lock);
5048 return -ENOENT;
5049 }
5050
5051
5052
5053
5054
5055
5056 for (i = 0; i < pimap->n_mapped; i++) {
5057 if (guest_gsi == pimap->mapped[i].v_hwirq) {
5058 if (pimap->mapped[i].r_hwirq) {
5059 mutex_unlock(&kvm->lock);
5060 return -EINVAL;
5061 }
5062 break;
5063 }
5064 }
5065
5066 if (i == KVMPPC_PIRQ_MAPPED) {
5067 mutex_unlock(&kvm->lock);
5068 return -EAGAIN;
5069 }
5070
5071 irq_map = &pimap->mapped[i];
5072
5073 irq_map->v_hwirq = guest_gsi;
5074 irq_map->desc = desc;
5075
5076
5077
5078
5079
5080 smp_wmb();
5081 irq_map->r_hwirq = desc->irq_data.hwirq;
5082
5083 if (i == pimap->n_mapped)
5084 pimap->n_mapped++;
5085
5086 if (xics_on_xive())
5087 rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc);
5088 else
5089 kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq);
5090 if (rc)
5091 irq_map->r_hwirq = 0;
5092
5093 mutex_unlock(&kvm->lock);
5094
5095 return 0;
5096 }
5097
5098 static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5099 {
5100 struct irq_desc *desc;
5101 struct kvmppc_passthru_irqmap *pimap;
5102 int i, rc = 0;
5103
5104 if (!kvm_irq_bypass)
5105 return 0;
5106
5107 desc = irq_to_desc(host_irq);
5108 if (!desc)
5109 return -EIO;
5110
5111 mutex_lock(&kvm->lock);
5112 if (!kvm->arch.pimap)
5113 goto unlock;
5114
5115 pimap = kvm->arch.pimap;
5116
5117 for (i = 0; i < pimap->n_mapped; i++) {
5118 if (guest_gsi == pimap->mapped[i].v_hwirq)
5119 break;
5120 }
5121
5122 if (i == pimap->n_mapped) {
5123 mutex_unlock(&kvm->lock);
5124 return -ENODEV;
5125 }
5126
5127 if (xics_on_xive())
5128 rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc);
5129 else
5130 kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
5131
5132
5133 pimap->mapped[i].r_hwirq = 0;
5134
5135
5136
5137
5138
5139 unlock:
5140 mutex_unlock(&kvm->lock);
5141 return rc;
5142 }
5143
5144 static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
5145 struct irq_bypass_producer *prod)
5146 {
5147 int ret = 0;
5148 struct kvm_kernel_irqfd *irqfd =
5149 container_of(cons, struct kvm_kernel_irqfd, consumer);
5150
5151 irqfd->producer = prod;
5152
5153 ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5154 if (ret)
5155 pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
5156 prod->irq, irqfd->gsi, ret);
5157
5158 return ret;
5159 }
5160
5161 static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
5162 struct irq_bypass_producer *prod)
5163 {
5164 int ret;
5165 struct kvm_kernel_irqfd *irqfd =
5166 container_of(cons, struct kvm_kernel_irqfd, consumer);
5167
5168 irqfd->producer = NULL;
5169
5170
5171
5172
5173
5174
5175 ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5176 if (ret)
5177 pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
5178 prod->irq, irqfd->gsi, ret);
5179 }
5180 #endif
5181
5182 static long kvm_arch_vm_ioctl_hv(struct file *filp,
5183 unsigned int ioctl, unsigned long arg)
5184 {
5185 struct kvm *kvm __maybe_unused = filp->private_data;
5186 void __user *argp = (void __user *)arg;
5187 long r;
5188
5189 switch (ioctl) {
5190
5191 case KVM_PPC_ALLOCATE_HTAB: {
5192 u32 htab_order;
5193
5194 r = -EFAULT;
5195 if (get_user(htab_order, (u32 __user *)argp))
5196 break;
5197 r = kvmppc_alloc_reset_hpt(kvm, htab_order);
5198 if (r)
5199 break;
5200 r = 0;
5201 break;
5202 }
5203
5204 case KVM_PPC_GET_HTAB_FD: {
5205 struct kvm_get_htab_fd ghf;
5206
5207 r = -EFAULT;
5208 if (copy_from_user(&ghf, argp, sizeof(ghf)))
5209 break;
5210 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
5211 break;
5212 }
5213
5214 case KVM_PPC_RESIZE_HPT_PREPARE: {
5215 struct kvm_ppc_resize_hpt rhpt;
5216
5217 r = -EFAULT;
5218 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5219 break;
5220
5221 r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
5222 break;
5223 }
5224
5225 case KVM_PPC_RESIZE_HPT_COMMIT: {
5226 struct kvm_ppc_resize_hpt rhpt;
5227
5228 r = -EFAULT;
5229 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5230 break;
5231
5232 r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
5233 break;
5234 }
5235
5236 default:
5237 r = -ENOTTY;
5238 }
5239
5240 return r;
5241 }
5242
5243
5244
5245
5246
5247
5248
5249 static unsigned int default_hcall_list[] = {
5250 H_REMOVE,
5251 H_ENTER,
5252 H_READ,
5253 H_PROTECT,
5254 H_BULK_REMOVE,
5255 H_GET_TCE,
5256 H_PUT_TCE,
5257 H_SET_DABR,
5258 H_SET_XDABR,
5259 H_CEDE,
5260 H_PROD,
5261 H_CONFER,
5262 H_REGISTER_VPA,
5263 #ifdef CONFIG_KVM_XICS
5264 H_EOI,
5265 H_CPPR,
5266 H_IPI,
5267 H_IPOLL,
5268 H_XIRR,
5269 H_XIRR_X,
5270 #endif
5271 0
5272 };
5273
5274 static void init_default_hcalls(void)
5275 {
5276 int i;
5277 unsigned int hcall;
5278
5279 for (i = 0; default_hcall_list[i]; ++i) {
5280 hcall = default_hcall_list[i];
5281 WARN_ON(!kvmppc_hcall_impl_hv(hcall));
5282 __set_bit(hcall / 4, default_enabled_hcalls);
5283 }
5284 }
5285
5286 static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
5287 {
5288 unsigned long lpcr;
5289 int radix;
5290 int err;
5291
5292
5293 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5294 return -ENODEV;
5295
5296
5297 if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
5298 return -EINVAL;
5299
5300
5301 radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
5302 if (!!(cfg->process_table & PATB_GR) != radix)
5303 return -EINVAL;
5304
5305
5306 if ((cfg->process_table & PRTS_MASK) > 24)
5307 return -EINVAL;
5308
5309
5310 if (radix && !radix_enabled())
5311 return -EINVAL;
5312
5313
5314 if (kvmhv_on_pseries() && !radix)
5315 return -EINVAL;
5316
5317 mutex_lock(&kvm->arch.mmu_setup_lock);
5318 if (radix != kvm_is_radix(kvm)) {
5319 if (kvm->arch.mmu_ready) {
5320 kvm->arch.mmu_ready = 0;
5321
5322 smp_mb();
5323 if (atomic_read(&kvm->arch.vcpus_running)) {
5324 kvm->arch.mmu_ready = 1;
5325 err = -EBUSY;
5326 goto out_unlock;
5327 }
5328 }
5329 if (radix)
5330 err = kvmppc_switch_mmu_to_radix(kvm);
5331 else
5332 err = kvmppc_switch_mmu_to_hpt(kvm);
5333 if (err)
5334 goto out_unlock;
5335 }
5336
5337 kvm->arch.process_table = cfg->process_table;
5338 kvmppc_setup_partition_table(kvm);
5339
5340 lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
5341 kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
5342 err = 0;
5343
5344 out_unlock:
5345 mutex_unlock(&kvm->arch.mmu_setup_lock);
5346 return err;
5347 }
5348
5349 static int kvmhv_enable_nested(struct kvm *kvm)
5350 {
5351 if (!nested)
5352 return -EPERM;
5353 if (!cpu_has_feature(CPU_FTR_ARCH_300) || no_mixing_hpt_and_radix)
5354 return -ENODEV;
5355
5356
5357 if (kvm)
5358 kvm->arch.nested_enable = true;
5359 return 0;
5360 }
5361
5362 static int kvmhv_load_from_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5363 int size)
5364 {
5365 int rc = -EINVAL;
5366
5367 if (kvmhv_vcpu_is_radix(vcpu)) {
5368 rc = kvmhv_copy_from_guest_radix(vcpu, *eaddr, ptr, size);
5369
5370 if (rc > 0)
5371 rc = -EINVAL;
5372 }
5373
5374
5375 if (rc && vcpu->arch.nested)
5376 rc = -EAGAIN;
5377
5378 return rc;
5379 }
5380
5381 static int kvmhv_store_to_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5382 int size)
5383 {
5384 int rc = -EINVAL;
5385
5386 if (kvmhv_vcpu_is_radix(vcpu)) {
5387 rc = kvmhv_copy_to_guest_radix(vcpu, *eaddr, ptr, size);
5388
5389 if (rc > 0)
5390 rc = -EINVAL;
5391 }
5392
5393
5394 if (rc && vcpu->arch.nested)
5395 rc = -EAGAIN;
5396
5397 return rc;
5398 }
5399
5400 static struct kvmppc_ops kvm_ops_hv = {
5401 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
5402 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
5403 .get_one_reg = kvmppc_get_one_reg_hv,
5404 .set_one_reg = kvmppc_set_one_reg_hv,
5405 .vcpu_load = kvmppc_core_vcpu_load_hv,
5406 .vcpu_put = kvmppc_core_vcpu_put_hv,
5407 .set_msr = kvmppc_set_msr_hv,
5408 .vcpu_run = kvmppc_vcpu_run_hv,
5409 .vcpu_create = kvmppc_core_vcpu_create_hv,
5410 .vcpu_free = kvmppc_core_vcpu_free_hv,
5411 .check_requests = kvmppc_core_check_requests_hv,
5412 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv,
5413 .flush_memslot = kvmppc_core_flush_memslot_hv,
5414 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
5415 .commit_memory_region = kvmppc_core_commit_memory_region_hv,
5416 .unmap_hva_range = kvm_unmap_hva_range_hv,
5417 .age_hva = kvm_age_hva_hv,
5418 .test_age_hva = kvm_test_age_hva_hv,
5419 .set_spte_hva = kvm_set_spte_hva_hv,
5420 .mmu_destroy = kvmppc_mmu_destroy_hv,
5421 .free_memslot = kvmppc_core_free_memslot_hv,
5422 .create_memslot = kvmppc_core_create_memslot_hv,
5423 .init_vm = kvmppc_core_init_vm_hv,
5424 .destroy_vm = kvmppc_core_destroy_vm_hv,
5425 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
5426 .emulate_op = kvmppc_core_emulate_op_hv,
5427 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
5428 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
5429 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
5430 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv,
5431 .hcall_implemented = kvmppc_hcall_impl_hv,
5432 #ifdef CONFIG_KVM_XICS
5433 .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
5434 .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
5435 #endif
5436 .configure_mmu = kvmhv_configure_mmu,
5437 .get_rmmu_info = kvmhv_get_rmmu_info,
5438 .set_smt_mode = kvmhv_set_smt_mode,
5439 .enable_nested = kvmhv_enable_nested,
5440 .load_from_eaddr = kvmhv_load_from_eaddr,
5441 .store_to_eaddr = kvmhv_store_to_eaddr,
5442 };
5443
5444 static int kvm_init_subcore_bitmap(void)
5445 {
5446 int i, j;
5447 int nr_cores = cpu_nr_cores();
5448 struct sibling_subcore_state *sibling_subcore_state;
5449
5450 for (i = 0; i < nr_cores; i++) {
5451 int first_cpu = i * threads_per_core;
5452 int node = cpu_to_node(first_cpu);
5453
5454
5455 if (paca_ptrs[first_cpu]->sibling_subcore_state)
5456 continue;
5457
5458 sibling_subcore_state =
5459 kzalloc_node(sizeof(struct sibling_subcore_state),
5460 GFP_KERNEL, node);
5461 if (!sibling_subcore_state)
5462 return -ENOMEM;
5463
5464
5465 for (j = 0; j < threads_per_core; j++) {
5466 int cpu = first_cpu + j;
5467
5468 paca_ptrs[cpu]->sibling_subcore_state =
5469 sibling_subcore_state;
5470 }
5471 }
5472 return 0;
5473 }
5474
5475 static int kvmppc_radix_possible(void)
5476 {
5477 return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
5478 }
5479
5480 static int kvmppc_book3s_init_hv(void)
5481 {
5482 int r;
5483
5484 if (!tlbie_capable) {
5485 pr_err("KVM-HV: Host does not support TLBIE\n");
5486 return -ENODEV;
5487 }
5488
5489
5490
5491
5492 r = kvmppc_core_check_processor_compat_hv();
5493 if (r < 0)
5494 return -ENODEV;
5495
5496 r = kvmhv_nested_init();
5497 if (r)
5498 return r;
5499
5500 r = kvm_init_subcore_bitmap();
5501 if (r)
5502 return r;
5503
5504
5505
5506
5507
5508
5509 #ifdef CONFIG_SMP
5510 if (!xics_on_xive() && !kvmhv_on_pseries() &&
5511 !local_paca->kvm_hstate.xics_phys) {
5512 struct device_node *np;
5513
5514 np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
5515 if (!np) {
5516 pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
5517 return -ENODEV;
5518 }
5519
5520 of_node_put(np);
5521 }
5522 #endif
5523
5524 kvm_ops_hv.owner = THIS_MODULE;
5525 kvmppc_hv_ops = &kvm_ops_hv;
5526
5527 init_default_hcalls();
5528
5529 init_vcore_lists();
5530
5531 r = kvmppc_mmu_hv_init();
5532 if (r)
5533 return r;
5534
5535 if (kvmppc_radix_possible())
5536 r = kvmppc_radix_init();
5537
5538
5539
5540
5541
5542 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5543 unsigned int pvr = mfspr(SPRN_PVR);
5544 if ((pvr >> 16) == PVR_POWER9 &&
5545 (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
5546 ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
5547 no_mixing_hpt_and_radix = true;
5548 }
5549
5550 return r;
5551 }
5552
5553 static void kvmppc_book3s_exit_hv(void)
5554 {
5555 kvmppc_free_host_rm_ops();
5556 if (kvmppc_radix_possible())
5557 kvmppc_radix_exit();
5558 kvmppc_hv_ops = NULL;
5559 kvmhv_nested_exit();
5560 }
5561
5562 module_init(kvmppc_book3s_init_hv);
5563 module_exit(kvmppc_book3s_exit_hv);
5564 MODULE_LICENSE("GPL");
5565 MODULE_ALIAS_MISCDEV(KVM_MINOR);
5566 MODULE_ALIAS("devname:kvm");