1 /*
2  * hosting zSeries kernel virtual machines
3  *
4  * Copyright IBM Corp. 2008, 2009
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License (version 2 only)
8  * as published by the Free Software Foundation.
9  *
10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
11  *               Christian Borntraeger <borntraeger@de.ibm.com>
12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
14  *               Jason J. Herne <jjherne@us.ibm.com>
15  */
16 
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
31 #include <asm/etr.h>
32 #include <asm/pgtable.h>
33 #include <asm/nmi.h>
34 #include <asm/switch_to.h>
35 #include <asm/isc.h>
36 #include <asm/sclp.h>
37 #include "kvm-s390.h"
38 #include "gaccess.h"
39 
40 #define KMSG_COMPONENT "kvm-s390"
41 #undef pr_fmt
42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
43 
44 #define CREATE_TRACE_POINTS
45 #include "trace.h"
46 #include "trace-s390.h"
47 
48 #define MEM_OP_MAX_SIZE 65536	/* Maximum transfer size for KVM_S390_MEM_OP */
49 #define LOCAL_IRQS 32
50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 			   (KVM_MAX_VCPUS + LOCAL_IRQS))
52 
53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
54 
55 struct kvm_stats_debugfs_item debugfs_entries[] = {
56 	{ "userspace_handled", VCPU_STAT(exit_userspace) },
57 	{ "exit_null", VCPU_STAT(exit_null) },
58 	{ "exit_validity", VCPU_STAT(exit_validity) },
59 	{ "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 	{ "exit_external_request", VCPU_STAT(exit_external_request) },
61 	{ "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
62 	{ "exit_instruction", VCPU_STAT(exit_instruction) },
63 	{ "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 	{ "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
65 	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
66 	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
67 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
68 	{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
69 	{ "instruction_lctl", VCPU_STAT(instruction_lctl) },
70 	{ "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 	{ "instruction_stctg", VCPU_STAT(instruction_stctg) },
72 	{ "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
73 	{ "deliver_external_call", VCPU_STAT(deliver_external_call) },
74 	{ "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 	{ "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 	{ "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 	{ "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 	{ "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 	{ "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 	{ "exit_wait_state", VCPU_STAT(exit_wait_state) },
81 	{ "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
82 	{ "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 	{ "instruction_spx", VCPU_STAT(instruction_spx) },
84 	{ "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 	{ "instruction_stap", VCPU_STAT(instruction_stap) },
86 	{ "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
87 	{ "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
88 	{ "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 	{ "instruction_chsc", VCPU_STAT(instruction_chsc) },
90 	{ "instruction_essa", VCPU_STAT(instruction_essa) },
91 	{ "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 	{ "instruction_stfl", VCPU_STAT(instruction_stfl) },
93 	{ "instruction_tprot", VCPU_STAT(instruction_tprot) },
94 	{ "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
95 	{ "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
96 	{ "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
97 	{ "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
98 	{ "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 	{ "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
100 	{ "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
101 	{ "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 	{ "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
103 	{ "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
104 	{ "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 	{ "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 	{ "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
107 	{ "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 	{ "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 	{ "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
110 	{ "diagnose_10", VCPU_STAT(diagnose_10) },
111 	{ "diagnose_44", VCPU_STAT(diagnose_44) },
112 	{ "diagnose_9c", VCPU_STAT(diagnose_9c) },
113 	{ "diagnose_258", VCPU_STAT(diagnose_258) },
114 	{ "diagnose_308", VCPU_STAT(diagnose_308) },
115 	{ "diagnose_500", VCPU_STAT(diagnose_500) },
116 	{ NULL }
117 };
118 
119 /* upper facilities limit for kvm */
120 unsigned long kvm_s390_fac_list_mask[] = {
121 	0xffe6fffbfcfdfc40UL,
122 	0x005e800000000000UL,
123 };
124 
kvm_s390_fac_list_mask_size(void)125 unsigned long kvm_s390_fac_list_mask_size(void)
126 {
127 	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 	return ARRAY_SIZE(kvm_s390_fac_list_mask);
129 }
130 
131 static struct gmap_notifier gmap_notifier;
132 debug_info_t *kvm_s390_dbf;
133 
134 /* Section: not file related */
kvm_arch_hardware_enable(void)135 int kvm_arch_hardware_enable(void)
136 {
137 	/* every s390 is virtualization enabled ;-) */
138 	return 0;
139 }
140 
141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
142 
143 /*
144  * This callback is executed during stop_machine(). All CPUs are therefore
145  * temporarily stopped. In order not to change guest behavior, we have to
146  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147  * so a CPU won't be stopped while calculating with the epoch.
148  */
kvm_clock_sync(struct notifier_block * notifier,unsigned long val,void * v)149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
150 			  void *v)
151 {
152 	struct kvm *kvm;
153 	struct kvm_vcpu *vcpu;
154 	int i;
155 	unsigned long long *delta = v;
156 
157 	list_for_each_entry(kvm, &vm_list, vm_list) {
158 		kvm->arch.epoch -= *delta;
159 		kvm_for_each_vcpu(i, vcpu, kvm) {
160 			vcpu->arch.sie_block->epoch -= *delta;
161 		}
162 	}
163 	return NOTIFY_OK;
164 }
165 
166 static struct notifier_block kvm_clock_notifier = {
167 	.notifier_call = kvm_clock_sync,
168 };
169 
kvm_arch_hardware_setup(void)170 int kvm_arch_hardware_setup(void)
171 {
172 	gmap_notifier.notifier_call = kvm_gmap_notifier;
173 	gmap_register_ipte_notifier(&gmap_notifier);
174 	atomic_notifier_chain_register(&s390_epoch_delta_notifier,
175 				       &kvm_clock_notifier);
176 	return 0;
177 }
178 
kvm_arch_hardware_unsetup(void)179 void kvm_arch_hardware_unsetup(void)
180 {
181 	gmap_unregister_ipte_notifier(&gmap_notifier);
182 	atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
183 					 &kvm_clock_notifier);
184 }
185 
kvm_arch_init(void * opaque)186 int kvm_arch_init(void *opaque)
187 {
188 	kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
189 	if (!kvm_s390_dbf)
190 		return -ENOMEM;
191 
192 	if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
193 		debug_unregister(kvm_s390_dbf);
194 		return -ENOMEM;
195 	}
196 
197 	/* Register floating interrupt controller interface. */
198 	return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
199 }
200 
kvm_arch_exit(void)201 void kvm_arch_exit(void)
202 {
203 	debug_unregister(kvm_s390_dbf);
204 }
205 
206 /* Section: device related */
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)207 long kvm_arch_dev_ioctl(struct file *filp,
208 			unsigned int ioctl, unsigned long arg)
209 {
210 	if (ioctl == KVM_S390_ENABLE_SIE)
211 		return s390_enable_sie();
212 	return -EINVAL;
213 }
214 
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
216 {
217 	int r;
218 
219 	switch (ext) {
220 	case KVM_CAP_S390_PSW:
221 	case KVM_CAP_S390_GMAP:
222 	case KVM_CAP_SYNC_MMU:
223 #ifdef CONFIG_KVM_S390_UCONTROL
224 	case KVM_CAP_S390_UCONTROL:
225 #endif
226 	case KVM_CAP_ASYNC_PF:
227 	case KVM_CAP_SYNC_REGS:
228 	case KVM_CAP_ONE_REG:
229 	case KVM_CAP_ENABLE_CAP:
230 	case KVM_CAP_S390_CSS_SUPPORT:
231 	case KVM_CAP_IOEVENTFD:
232 	case KVM_CAP_DEVICE_CTRL:
233 	case KVM_CAP_ENABLE_CAP_VM:
234 	case KVM_CAP_S390_IRQCHIP:
235 	case KVM_CAP_VM_ATTRIBUTES:
236 	case KVM_CAP_MP_STATE:
237 	case KVM_CAP_S390_INJECT_IRQ:
238 	case KVM_CAP_S390_USER_SIGP:
239 	case KVM_CAP_S390_USER_STSI:
240 	case KVM_CAP_S390_SKEYS:
241 	case KVM_CAP_S390_IRQ_STATE:
242 		r = 1;
243 		break;
244 	case KVM_CAP_S390_MEM_OP:
245 		r = MEM_OP_MAX_SIZE;
246 		break;
247 	case KVM_CAP_NR_VCPUS:
248 	case KVM_CAP_MAX_VCPUS:
249 		r = KVM_MAX_VCPUS;
250 		break;
251 	case KVM_CAP_NR_MEMSLOTS:
252 		r = KVM_USER_MEM_SLOTS;
253 		break;
254 	case KVM_CAP_S390_COW:
255 		r = MACHINE_HAS_ESOP;
256 		break;
257 	case KVM_CAP_S390_VECTOR_REGISTERS:
258 		r = MACHINE_HAS_VX;
259 		break;
260 	default:
261 		r = 0;
262 	}
263 	return r;
264 }
265 
kvm_s390_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)266 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
267 					struct kvm_memory_slot *memslot)
268 {
269 	gfn_t cur_gfn, last_gfn;
270 	unsigned long address;
271 	struct gmap *gmap = kvm->arch.gmap;
272 
273 	down_read(&gmap->mm->mmap_sem);
274 	/* Loop over all guest pages */
275 	last_gfn = memslot->base_gfn + memslot->npages;
276 	for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
277 		address = gfn_to_hva_memslot(memslot, cur_gfn);
278 
279 		if (gmap_test_and_clear_dirty(address, gmap))
280 			mark_page_dirty(kvm, cur_gfn);
281 	}
282 	up_read(&gmap->mm->mmap_sem);
283 }
284 
285 /* Section: vm related */
286 /*
287  * Get (and clear) the dirty memory log for a memory slot.
288  */
kvm_vm_ioctl_get_dirty_log(struct kvm * kvm,struct kvm_dirty_log * log)289 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
290 			       struct kvm_dirty_log *log)
291 {
292 	int r;
293 	unsigned long n;
294 	struct kvm_memslots *slots;
295 	struct kvm_memory_slot *memslot;
296 	int is_dirty = 0;
297 
298 	mutex_lock(&kvm->slots_lock);
299 
300 	r = -EINVAL;
301 	if (log->slot >= KVM_USER_MEM_SLOTS)
302 		goto out;
303 
304 	slots = kvm_memslots(kvm);
305 	memslot = id_to_memslot(slots, log->slot);
306 	r = -ENOENT;
307 	if (!memslot->dirty_bitmap)
308 		goto out;
309 
310 	kvm_s390_sync_dirty_log(kvm, memslot);
311 	r = kvm_get_dirty_log(kvm, log, &is_dirty);
312 	if (r)
313 		goto out;
314 
315 	/* Clear the dirty log */
316 	if (is_dirty) {
317 		n = kvm_dirty_bitmap_bytes(memslot);
318 		memset(memslot->dirty_bitmap, 0, n);
319 	}
320 	r = 0;
321 out:
322 	mutex_unlock(&kvm->slots_lock);
323 	return r;
324 }
325 
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)326 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
327 {
328 	int r;
329 
330 	if (cap->flags)
331 		return -EINVAL;
332 
333 	switch (cap->cap) {
334 	case KVM_CAP_S390_IRQCHIP:
335 		VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
336 		kvm->arch.use_irqchip = 1;
337 		r = 0;
338 		break;
339 	case KVM_CAP_S390_USER_SIGP:
340 		VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
341 		kvm->arch.user_sigp = 1;
342 		r = 0;
343 		break;
344 	case KVM_CAP_S390_VECTOR_REGISTERS:
345 		mutex_lock(&kvm->lock);
346 		if (atomic_read(&kvm->online_vcpus)) {
347 			r = -EBUSY;
348 		} else if (MACHINE_HAS_VX) {
349 			set_kvm_facility(kvm->arch.model.fac->mask, 129);
350 			set_kvm_facility(kvm->arch.model.fac->list, 129);
351 			r = 0;
352 		} else
353 			r = -EINVAL;
354 		mutex_unlock(&kvm->lock);
355 		VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
356 			 r ? "(not available)" : "(success)");
357 		break;
358 	case KVM_CAP_S390_USER_STSI:
359 		VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
360 		kvm->arch.user_stsi = 1;
361 		r = 0;
362 		break;
363 	default:
364 		r = -EINVAL;
365 		break;
366 	}
367 	return r;
368 }
369 
kvm_s390_get_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)370 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
371 {
372 	int ret;
373 
374 	switch (attr->attr) {
375 	case KVM_S390_VM_MEM_LIMIT_SIZE:
376 		ret = 0;
377 		VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
378 			 kvm->arch.gmap->asce_end);
379 		if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
380 			ret = -EFAULT;
381 		break;
382 	default:
383 		ret = -ENXIO;
384 		break;
385 	}
386 	return ret;
387 }
388 
kvm_s390_set_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)389 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
390 {
391 	int ret;
392 	unsigned int idx;
393 	switch (attr->attr) {
394 	case KVM_S390_VM_MEM_ENABLE_CMMA:
395 		/* enable CMMA only for z10 and later (EDAT_1) */
396 		ret = -EINVAL;
397 		if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
398 			break;
399 
400 		ret = -EBUSY;
401 		VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
402 		mutex_lock(&kvm->lock);
403 		if (atomic_read(&kvm->online_vcpus) == 0) {
404 			kvm->arch.use_cmma = 1;
405 			ret = 0;
406 		}
407 		mutex_unlock(&kvm->lock);
408 		break;
409 	case KVM_S390_VM_MEM_CLR_CMMA:
410 		ret = -EINVAL;
411 		if (!kvm->arch.use_cmma)
412 			break;
413 
414 		VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
415 		mutex_lock(&kvm->lock);
416 		idx = srcu_read_lock(&kvm->srcu);
417 		s390_reset_cmma(kvm->arch.gmap->mm);
418 		srcu_read_unlock(&kvm->srcu, idx);
419 		mutex_unlock(&kvm->lock);
420 		ret = 0;
421 		break;
422 	case KVM_S390_VM_MEM_LIMIT_SIZE: {
423 		unsigned long new_limit;
424 
425 		if (kvm_is_ucontrol(kvm))
426 			return -EINVAL;
427 
428 		if (get_user(new_limit, (u64 __user *)attr->addr))
429 			return -EFAULT;
430 
431 		if (new_limit > kvm->arch.gmap->asce_end)
432 			return -E2BIG;
433 
434 		ret = -EBUSY;
435 		mutex_lock(&kvm->lock);
436 		if (atomic_read(&kvm->online_vcpus) == 0) {
437 			/* gmap_alloc will round the limit up */
438 			struct gmap *new = gmap_alloc(current->mm, new_limit);
439 
440 			if (!new) {
441 				ret = -ENOMEM;
442 			} else {
443 				gmap_free(kvm->arch.gmap);
444 				new->private = kvm;
445 				kvm->arch.gmap = new;
446 				ret = 0;
447 			}
448 		}
449 		mutex_unlock(&kvm->lock);
450 		VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
451 		break;
452 	}
453 	default:
454 		ret = -ENXIO;
455 		break;
456 	}
457 	return ret;
458 }
459 
460 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
461 
kvm_s390_vm_set_crypto(struct kvm * kvm,struct kvm_device_attr * attr)462 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
463 {
464 	struct kvm_vcpu *vcpu;
465 	int i;
466 
467 	if (!test_kvm_facility(kvm, 76))
468 		return -EINVAL;
469 
470 	mutex_lock(&kvm->lock);
471 	switch (attr->attr) {
472 	case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
473 		get_random_bytes(
474 			kvm->arch.crypto.crycb->aes_wrapping_key_mask,
475 			sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
476 		kvm->arch.crypto.aes_kw = 1;
477 		VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
478 		break;
479 	case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
480 		get_random_bytes(
481 			kvm->arch.crypto.crycb->dea_wrapping_key_mask,
482 			sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
483 		kvm->arch.crypto.dea_kw = 1;
484 		VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
485 		break;
486 	case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
487 		kvm->arch.crypto.aes_kw = 0;
488 		memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
489 			sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
490 		VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
491 		break;
492 	case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
493 		kvm->arch.crypto.dea_kw = 0;
494 		memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
495 			sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
496 		VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
497 		break;
498 	default:
499 		mutex_unlock(&kvm->lock);
500 		return -ENXIO;
501 	}
502 
503 	kvm_for_each_vcpu(i, vcpu, kvm) {
504 		kvm_s390_vcpu_crypto_setup(vcpu);
505 		exit_sie(vcpu);
506 	}
507 	mutex_unlock(&kvm->lock);
508 	return 0;
509 }
510 
kvm_s390_set_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)511 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
512 {
513 	u8 gtod_high;
514 
515 	if (copy_from_user(&gtod_high, (void __user *)attr->addr,
516 					   sizeof(gtod_high)))
517 		return -EFAULT;
518 
519 	if (gtod_high != 0)
520 		return -EINVAL;
521 	VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
522 
523 	return 0;
524 }
525 
kvm_s390_set_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)526 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
527 {
528 	u64 gtod;
529 
530 	if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
531 		return -EFAULT;
532 
533 	kvm_s390_set_tod_clock(kvm, gtod);
534 	VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
535 	return 0;
536 }
537 
kvm_s390_set_tod(struct kvm * kvm,struct kvm_device_attr * attr)538 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
539 {
540 	int ret;
541 
542 	if (attr->flags)
543 		return -EINVAL;
544 
545 	switch (attr->attr) {
546 	case KVM_S390_VM_TOD_HIGH:
547 		ret = kvm_s390_set_tod_high(kvm, attr);
548 		break;
549 	case KVM_S390_VM_TOD_LOW:
550 		ret = kvm_s390_set_tod_low(kvm, attr);
551 		break;
552 	default:
553 		ret = -ENXIO;
554 		break;
555 	}
556 	return ret;
557 }
558 
kvm_s390_get_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)559 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
560 {
561 	u8 gtod_high = 0;
562 
563 	if (copy_to_user((void __user *)attr->addr, &gtod_high,
564 					 sizeof(gtod_high)))
565 		return -EFAULT;
566 	VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
567 
568 	return 0;
569 }
570 
kvm_s390_get_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)571 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
572 {
573 	u64 gtod;
574 
575 	gtod = kvm_s390_get_tod_clock_fast(kvm);
576 	if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
577 		return -EFAULT;
578 	VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
579 
580 	return 0;
581 }
582 
kvm_s390_get_tod(struct kvm * kvm,struct kvm_device_attr * attr)583 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
584 {
585 	int ret;
586 
587 	if (attr->flags)
588 		return -EINVAL;
589 
590 	switch (attr->attr) {
591 	case KVM_S390_VM_TOD_HIGH:
592 		ret = kvm_s390_get_tod_high(kvm, attr);
593 		break;
594 	case KVM_S390_VM_TOD_LOW:
595 		ret = kvm_s390_get_tod_low(kvm, attr);
596 		break;
597 	default:
598 		ret = -ENXIO;
599 		break;
600 	}
601 	return ret;
602 }
603 
kvm_s390_set_processor(struct kvm * kvm,struct kvm_device_attr * attr)604 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
605 {
606 	struct kvm_s390_vm_cpu_processor *proc;
607 	int ret = 0;
608 
609 	mutex_lock(&kvm->lock);
610 	if (atomic_read(&kvm->online_vcpus)) {
611 		ret = -EBUSY;
612 		goto out;
613 	}
614 	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
615 	if (!proc) {
616 		ret = -ENOMEM;
617 		goto out;
618 	}
619 	if (!copy_from_user(proc, (void __user *)attr->addr,
620 			    sizeof(*proc))) {
621 		memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
622 		       sizeof(struct cpuid));
623 		kvm->arch.model.ibc = proc->ibc;
624 		memcpy(kvm->arch.model.fac->list, proc->fac_list,
625 		       S390_ARCH_FAC_LIST_SIZE_BYTE);
626 	} else
627 		ret = -EFAULT;
628 	kfree(proc);
629 out:
630 	mutex_unlock(&kvm->lock);
631 	return ret;
632 }
633 
kvm_s390_set_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)634 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
635 {
636 	int ret = -ENXIO;
637 
638 	switch (attr->attr) {
639 	case KVM_S390_VM_CPU_PROCESSOR:
640 		ret = kvm_s390_set_processor(kvm, attr);
641 		break;
642 	}
643 	return ret;
644 }
645 
kvm_s390_get_processor(struct kvm * kvm,struct kvm_device_attr * attr)646 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
647 {
648 	struct kvm_s390_vm_cpu_processor *proc;
649 	int ret = 0;
650 
651 	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
652 	if (!proc) {
653 		ret = -ENOMEM;
654 		goto out;
655 	}
656 	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
657 	proc->ibc = kvm->arch.model.ibc;
658 	memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
659 	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
660 		ret = -EFAULT;
661 	kfree(proc);
662 out:
663 	return ret;
664 }
665 
kvm_s390_get_machine(struct kvm * kvm,struct kvm_device_attr * attr)666 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
667 {
668 	struct kvm_s390_vm_cpu_machine *mach;
669 	int ret = 0;
670 
671 	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
672 	if (!mach) {
673 		ret = -ENOMEM;
674 		goto out;
675 	}
676 	get_cpu_id((struct cpuid *) &mach->cpuid);
677 	mach->ibc = sclp.ibc;
678 	memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
679 	       S390_ARCH_FAC_LIST_SIZE_BYTE);
680 	memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
681 	       S390_ARCH_FAC_LIST_SIZE_BYTE);
682 	if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
683 		ret = -EFAULT;
684 	kfree(mach);
685 out:
686 	return ret;
687 }
688 
kvm_s390_get_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)689 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
690 {
691 	int ret = -ENXIO;
692 
693 	switch (attr->attr) {
694 	case KVM_S390_VM_CPU_PROCESSOR:
695 		ret = kvm_s390_get_processor(kvm, attr);
696 		break;
697 	case KVM_S390_VM_CPU_MACHINE:
698 		ret = kvm_s390_get_machine(kvm, attr);
699 		break;
700 	}
701 	return ret;
702 }
703 
kvm_s390_vm_set_attr(struct kvm * kvm,struct kvm_device_attr * attr)704 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
705 {
706 	int ret;
707 
708 	switch (attr->group) {
709 	case KVM_S390_VM_MEM_CTRL:
710 		ret = kvm_s390_set_mem_control(kvm, attr);
711 		break;
712 	case KVM_S390_VM_TOD:
713 		ret = kvm_s390_set_tod(kvm, attr);
714 		break;
715 	case KVM_S390_VM_CPU_MODEL:
716 		ret = kvm_s390_set_cpu_model(kvm, attr);
717 		break;
718 	case KVM_S390_VM_CRYPTO:
719 		ret = kvm_s390_vm_set_crypto(kvm, attr);
720 		break;
721 	default:
722 		ret = -ENXIO;
723 		break;
724 	}
725 
726 	return ret;
727 }
728 
kvm_s390_vm_get_attr(struct kvm * kvm,struct kvm_device_attr * attr)729 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
730 {
731 	int ret;
732 
733 	switch (attr->group) {
734 	case KVM_S390_VM_MEM_CTRL:
735 		ret = kvm_s390_get_mem_control(kvm, attr);
736 		break;
737 	case KVM_S390_VM_TOD:
738 		ret = kvm_s390_get_tod(kvm, attr);
739 		break;
740 	case KVM_S390_VM_CPU_MODEL:
741 		ret = kvm_s390_get_cpu_model(kvm, attr);
742 		break;
743 	default:
744 		ret = -ENXIO;
745 		break;
746 	}
747 
748 	return ret;
749 }
750 
kvm_s390_vm_has_attr(struct kvm * kvm,struct kvm_device_attr * attr)751 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
752 {
753 	int ret;
754 
755 	switch (attr->group) {
756 	case KVM_S390_VM_MEM_CTRL:
757 		switch (attr->attr) {
758 		case KVM_S390_VM_MEM_ENABLE_CMMA:
759 		case KVM_S390_VM_MEM_CLR_CMMA:
760 		case KVM_S390_VM_MEM_LIMIT_SIZE:
761 			ret = 0;
762 			break;
763 		default:
764 			ret = -ENXIO;
765 			break;
766 		}
767 		break;
768 	case KVM_S390_VM_TOD:
769 		switch (attr->attr) {
770 		case KVM_S390_VM_TOD_LOW:
771 		case KVM_S390_VM_TOD_HIGH:
772 			ret = 0;
773 			break;
774 		default:
775 			ret = -ENXIO;
776 			break;
777 		}
778 		break;
779 	case KVM_S390_VM_CPU_MODEL:
780 		switch (attr->attr) {
781 		case KVM_S390_VM_CPU_PROCESSOR:
782 		case KVM_S390_VM_CPU_MACHINE:
783 			ret = 0;
784 			break;
785 		default:
786 			ret = -ENXIO;
787 			break;
788 		}
789 		break;
790 	case KVM_S390_VM_CRYPTO:
791 		switch (attr->attr) {
792 		case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
793 		case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
794 		case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
795 		case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
796 			ret = 0;
797 			break;
798 		default:
799 			ret = -ENXIO;
800 			break;
801 		}
802 		break;
803 	default:
804 		ret = -ENXIO;
805 		break;
806 	}
807 
808 	return ret;
809 }
810 
kvm_s390_get_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)811 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
812 {
813 	uint8_t *keys;
814 	uint64_t hva;
815 	unsigned long curkey;
816 	int i, r = 0;
817 
818 	if (args->flags != 0)
819 		return -EINVAL;
820 
821 	/* Is this guest using storage keys? */
822 	if (!mm_use_skey(current->mm))
823 		return KVM_S390_GET_SKEYS_NONE;
824 
825 	/* Enforce sane limit on memory allocation */
826 	if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
827 		return -EINVAL;
828 
829 	keys = kmalloc_array(args->count, sizeof(uint8_t),
830 			     GFP_KERNEL | __GFP_NOWARN);
831 	if (!keys)
832 		keys = vmalloc(sizeof(uint8_t) * args->count);
833 	if (!keys)
834 		return -ENOMEM;
835 
836 	for (i = 0; i < args->count; i++) {
837 		hva = gfn_to_hva(kvm, args->start_gfn + i);
838 		if (kvm_is_error_hva(hva)) {
839 			r = -EFAULT;
840 			goto out;
841 		}
842 
843 		curkey = get_guest_storage_key(current->mm, hva);
844 		if (IS_ERR_VALUE(curkey)) {
845 			r = curkey;
846 			goto out;
847 		}
848 		keys[i] = curkey;
849 	}
850 
851 	r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
852 			 sizeof(uint8_t) * args->count);
853 	if (r)
854 		r = -EFAULT;
855 out:
856 	kvfree(keys);
857 	return r;
858 }
859 
kvm_s390_set_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)860 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
861 {
862 	uint8_t *keys;
863 	uint64_t hva;
864 	int i, r = 0;
865 
866 	if (args->flags != 0)
867 		return -EINVAL;
868 
869 	/* Enforce sane limit on memory allocation */
870 	if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
871 		return -EINVAL;
872 
873 	keys = kmalloc_array(args->count, sizeof(uint8_t),
874 			     GFP_KERNEL | __GFP_NOWARN);
875 	if (!keys)
876 		keys = vmalloc(sizeof(uint8_t) * args->count);
877 	if (!keys)
878 		return -ENOMEM;
879 
880 	r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
881 			   sizeof(uint8_t) * args->count);
882 	if (r) {
883 		r = -EFAULT;
884 		goto out;
885 	}
886 
887 	/* Enable storage key handling for the guest */
888 	r = s390_enable_skey();
889 	if (r)
890 		goto out;
891 
892 	for (i = 0; i < args->count; i++) {
893 		hva = gfn_to_hva(kvm, args->start_gfn + i);
894 		if (kvm_is_error_hva(hva)) {
895 			r = -EFAULT;
896 			goto out;
897 		}
898 
899 		/* Lowest order bit is reserved */
900 		if (keys[i] & 0x01) {
901 			r = -EINVAL;
902 			goto out;
903 		}
904 
905 		r = set_guest_storage_key(current->mm, hva,
906 					  (unsigned long)keys[i], 0);
907 		if (r)
908 			goto out;
909 	}
910 out:
911 	kvfree(keys);
912 	return r;
913 }
914 
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)915 long kvm_arch_vm_ioctl(struct file *filp,
916 		       unsigned int ioctl, unsigned long arg)
917 {
918 	struct kvm *kvm = filp->private_data;
919 	void __user *argp = (void __user *)arg;
920 	struct kvm_device_attr attr;
921 	int r;
922 
923 	switch (ioctl) {
924 	case KVM_S390_INTERRUPT: {
925 		struct kvm_s390_interrupt s390int;
926 
927 		r = -EFAULT;
928 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
929 			break;
930 		r = kvm_s390_inject_vm(kvm, &s390int);
931 		break;
932 	}
933 	case KVM_ENABLE_CAP: {
934 		struct kvm_enable_cap cap;
935 		r = -EFAULT;
936 		if (copy_from_user(&cap, argp, sizeof(cap)))
937 			break;
938 		r = kvm_vm_ioctl_enable_cap(kvm, &cap);
939 		break;
940 	}
941 	case KVM_CREATE_IRQCHIP: {
942 		struct kvm_irq_routing_entry routing;
943 
944 		r = -EINVAL;
945 		if (kvm->arch.use_irqchip) {
946 			/* Set up dummy routing. */
947 			memset(&routing, 0, sizeof(routing));
948 			r = kvm_set_irq_routing(kvm, &routing, 0, 0);
949 		}
950 		break;
951 	}
952 	case KVM_SET_DEVICE_ATTR: {
953 		r = -EFAULT;
954 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
955 			break;
956 		r = kvm_s390_vm_set_attr(kvm, &attr);
957 		break;
958 	}
959 	case KVM_GET_DEVICE_ATTR: {
960 		r = -EFAULT;
961 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
962 			break;
963 		r = kvm_s390_vm_get_attr(kvm, &attr);
964 		break;
965 	}
966 	case KVM_HAS_DEVICE_ATTR: {
967 		r = -EFAULT;
968 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
969 			break;
970 		r = kvm_s390_vm_has_attr(kvm, &attr);
971 		break;
972 	}
973 	case KVM_S390_GET_SKEYS: {
974 		struct kvm_s390_skeys args;
975 
976 		r = -EFAULT;
977 		if (copy_from_user(&args, argp,
978 				   sizeof(struct kvm_s390_skeys)))
979 			break;
980 		r = kvm_s390_get_skeys(kvm, &args);
981 		break;
982 	}
983 	case KVM_S390_SET_SKEYS: {
984 		struct kvm_s390_skeys args;
985 
986 		r = -EFAULT;
987 		if (copy_from_user(&args, argp,
988 				   sizeof(struct kvm_s390_skeys)))
989 			break;
990 		r = kvm_s390_set_skeys(kvm, &args);
991 		break;
992 	}
993 	default:
994 		r = -ENOTTY;
995 	}
996 
997 	return r;
998 }
999 
kvm_s390_query_ap_config(u8 * config)1000 static int kvm_s390_query_ap_config(u8 *config)
1001 {
1002 	u32 fcn_code = 0x04000000UL;
1003 	u32 cc = 0;
1004 
1005 	memset(config, 0, 128);
1006 	asm volatile(
1007 		"lgr 0,%1\n"
1008 		"lgr 2,%2\n"
1009 		".long 0xb2af0000\n"		/* PQAP(QCI) */
1010 		"0: ipm %0\n"
1011 		"srl %0,28\n"
1012 		"1:\n"
1013 		EX_TABLE(0b, 1b)
1014 		: "+r" (cc)
1015 		: "r" (fcn_code), "r" (config)
1016 		: "cc", "0", "2", "memory"
1017 	);
1018 
1019 	return cc;
1020 }
1021 
kvm_s390_apxa_installed(void)1022 static int kvm_s390_apxa_installed(void)
1023 {
1024 	u8 config[128];
1025 	int cc;
1026 
1027 	if (test_facility(2) && test_facility(12)) {
1028 		cc = kvm_s390_query_ap_config(config);
1029 
1030 		if (cc)
1031 			pr_err("PQAP(QCI) failed with cc=%d", cc);
1032 		else
1033 			return config[0] & 0x40;
1034 	}
1035 
1036 	return 0;
1037 }
1038 
kvm_s390_set_crycb_format(struct kvm * kvm)1039 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1040 {
1041 	kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1042 
1043 	if (kvm_s390_apxa_installed())
1044 		kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1045 	else
1046 		kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1047 }
1048 
kvm_s390_get_cpu_id(struct cpuid * cpu_id)1049 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1050 {
1051 	get_cpu_id(cpu_id);
1052 	cpu_id->version = 0xff;
1053 }
1054 
kvm_s390_crypto_init(struct kvm * kvm)1055 static int kvm_s390_crypto_init(struct kvm *kvm)
1056 {
1057 	if (!test_kvm_facility(kvm, 76))
1058 		return 0;
1059 
1060 	kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1061 					 GFP_KERNEL | GFP_DMA);
1062 	if (!kvm->arch.crypto.crycb)
1063 		return -ENOMEM;
1064 
1065 	kvm_s390_set_crycb_format(kvm);
1066 
1067 	/* Enable AES/DEA protected key functions by default */
1068 	kvm->arch.crypto.aes_kw = 1;
1069 	kvm->arch.crypto.dea_kw = 1;
1070 	get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1071 			 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1072 	get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1073 			 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1074 
1075 	return 0;
1076 }
1077 
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)1078 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1079 {
1080 	int i, rc;
1081 	char debug_name[16];
1082 	static unsigned long sca_offset;
1083 
1084 	rc = -EINVAL;
1085 #ifdef CONFIG_KVM_S390_UCONTROL
1086 	if (type & ~KVM_VM_S390_UCONTROL)
1087 		goto out_err;
1088 	if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1089 		goto out_err;
1090 #else
1091 	if (type)
1092 		goto out_err;
1093 #endif
1094 
1095 	rc = s390_enable_sie();
1096 	if (rc)
1097 		goto out_err;
1098 
1099 	rc = -ENOMEM;
1100 
1101 	kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1102 	if (!kvm->arch.sca)
1103 		goto out_err;
1104 	spin_lock(&kvm_lock);
1105 	sca_offset += 16;
1106 	if (sca_offset + sizeof(struct sca_block) > PAGE_SIZE)
1107 		sca_offset = 0;
1108 	kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1109 	spin_unlock(&kvm_lock);
1110 
1111 	sprintf(debug_name, "kvm-%u", current->pid);
1112 
1113 	kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1114 	if (!kvm->arch.dbf)
1115 		goto out_err;
1116 
1117 	/*
1118 	 * The architectural maximum amount of facilities is 16 kbit. To store
1119 	 * this amount, 2 kbyte of memory is required. Thus we need a full
1120 	 * page to hold the guest facility list (arch.model.fac->list) and the
1121 	 * facility mask (arch.model.fac->mask). Its address size has to be
1122 	 * 31 bits and word aligned.
1123 	 */
1124 	kvm->arch.model.fac =
1125 		(struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1126 	if (!kvm->arch.model.fac)
1127 		goto out_err;
1128 
1129 	/* Populate the facility mask initially. */
1130 	memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1131 	       S390_ARCH_FAC_LIST_SIZE_BYTE);
1132 	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1133 		if (i < kvm_s390_fac_list_mask_size())
1134 			kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1135 		else
1136 			kvm->arch.model.fac->mask[i] = 0UL;
1137 	}
1138 
1139 	/* Populate the facility list initially. */
1140 	memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1141 	       S390_ARCH_FAC_LIST_SIZE_BYTE);
1142 
1143 	kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1144 	kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1145 
1146 	if (kvm_s390_crypto_init(kvm) < 0)
1147 		goto out_err;
1148 
1149 	spin_lock_init(&kvm->arch.float_int.lock);
1150 	for (i = 0; i < FIRQ_LIST_COUNT; i++)
1151 		INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1152 	init_waitqueue_head(&kvm->arch.ipte_wq);
1153 	mutex_init(&kvm->arch.ipte_mutex);
1154 
1155 	debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1156 	VM_EVENT(kvm, 3, "vm created with type %lu", type);
1157 
1158 	if (type & KVM_VM_S390_UCONTROL) {
1159 		kvm->arch.gmap = NULL;
1160 	} else {
1161 		kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1162 		if (!kvm->arch.gmap)
1163 			goto out_err;
1164 		kvm->arch.gmap->private = kvm;
1165 		kvm->arch.gmap->pfault_enabled = 0;
1166 	}
1167 
1168 	kvm->arch.css_support = 0;
1169 	kvm->arch.use_irqchip = 0;
1170 	kvm->arch.epoch = 0;
1171 
1172 	spin_lock_init(&kvm->arch.start_stop_lock);
1173 	KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
1174 
1175 	return 0;
1176 out_err:
1177 	kfree(kvm->arch.crypto.crycb);
1178 	free_page((unsigned long)kvm->arch.model.fac);
1179 	debug_unregister(kvm->arch.dbf);
1180 	free_page((unsigned long)(kvm->arch.sca));
1181 	KVM_EVENT(3, "creation of vm failed: %d", rc);
1182 	return rc;
1183 }
1184 
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)1185 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1186 {
1187 	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1188 	trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1189 	kvm_s390_clear_local_irqs(vcpu);
1190 	kvm_clear_async_pf_completion_queue(vcpu);
1191 	if (!kvm_is_ucontrol(vcpu->kvm)) {
1192 		clear_bit(63 - vcpu->vcpu_id,
1193 			  (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1194 		if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1195 		    (__u64) vcpu->arch.sie_block)
1196 			vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1197 	}
1198 	smp_mb();
1199 
1200 	if (kvm_is_ucontrol(vcpu->kvm))
1201 		gmap_free(vcpu->arch.gmap);
1202 
1203 	if (vcpu->kvm->arch.use_cmma)
1204 		kvm_s390_vcpu_unsetup_cmma(vcpu);
1205 	free_page((unsigned long)(vcpu->arch.sie_block));
1206 
1207 	kvm_vcpu_uninit(vcpu);
1208 	kmem_cache_free(kvm_vcpu_cache, vcpu);
1209 }
1210 
kvm_free_vcpus(struct kvm * kvm)1211 static void kvm_free_vcpus(struct kvm *kvm)
1212 {
1213 	unsigned int i;
1214 	struct kvm_vcpu *vcpu;
1215 
1216 	kvm_for_each_vcpu(i, vcpu, kvm)
1217 		kvm_arch_vcpu_destroy(vcpu);
1218 
1219 	mutex_lock(&kvm->lock);
1220 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1221 		kvm->vcpus[i] = NULL;
1222 
1223 	atomic_set(&kvm->online_vcpus, 0);
1224 	mutex_unlock(&kvm->lock);
1225 }
1226 
kvm_arch_destroy_vm(struct kvm * kvm)1227 void kvm_arch_destroy_vm(struct kvm *kvm)
1228 {
1229 	kvm_free_vcpus(kvm);
1230 	free_page((unsigned long)kvm->arch.model.fac);
1231 	free_page((unsigned long)(kvm->arch.sca));
1232 	debug_unregister(kvm->arch.dbf);
1233 	kfree(kvm->arch.crypto.crycb);
1234 	if (!kvm_is_ucontrol(kvm))
1235 		gmap_free(kvm->arch.gmap);
1236 	kvm_s390_destroy_adapters(kvm);
1237 	kvm_s390_clear_float_irqs(kvm);
1238 	KVM_EVENT(3, "vm 0x%p destroyed", kvm);
1239 }
1240 
1241 /* Section: vcpu related */
__kvm_ucontrol_vcpu_init(struct kvm_vcpu * vcpu)1242 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1243 {
1244 	vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1245 	if (!vcpu->arch.gmap)
1246 		return -ENOMEM;
1247 	vcpu->arch.gmap->private = vcpu->kvm;
1248 
1249 	return 0;
1250 }
1251 
kvm_arch_vcpu_init(struct kvm_vcpu * vcpu)1252 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1253 {
1254 	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1255 	kvm_clear_async_pf_completion_queue(vcpu);
1256 	vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1257 				    KVM_SYNC_GPRS |
1258 				    KVM_SYNC_ACRS |
1259 				    KVM_SYNC_CRS |
1260 				    KVM_SYNC_ARCH0 |
1261 				    KVM_SYNC_PFAULT;
1262 	if (test_kvm_facility(vcpu->kvm, 129))
1263 		vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1264 
1265 	if (kvm_is_ucontrol(vcpu->kvm))
1266 		return __kvm_ucontrol_vcpu_init(vcpu);
1267 
1268 	return 0;
1269 }
1270 
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)1271 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1272 {
1273 	/* Save host register state */
1274 	save_fpu_regs();
1275 	vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1276 	vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1277 
1278 	/* Depending on MACHINE_HAS_VX, data stored to vrs either
1279 	 * has vector register or floating point register format.
1280 	 */
1281 	current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1282 	current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1283 	if (test_fp_ctl(current->thread.fpu.fpc))
1284 		/* User space provided an invalid FPC, let's clear it */
1285 		current->thread.fpu.fpc = 0;
1286 
1287 	save_access_regs(vcpu->arch.host_acrs);
1288 	restore_access_regs(vcpu->run->s.regs.acrs);
1289 	gmap_enable(vcpu->arch.gmap);
1290 	atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1291 }
1292 
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)1293 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1294 {
1295 	atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1296 	gmap_disable(vcpu->arch.gmap);
1297 
1298 	/* Save guest register state */
1299 	save_fpu_regs();
1300 	vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1301 
1302 	/* Restore host register state */
1303 	current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1304 	current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1305 
1306 	save_access_regs(vcpu->run->s.regs.acrs);
1307 	restore_access_regs(vcpu->arch.host_acrs);
1308 }
1309 
kvm_s390_vcpu_initial_reset(struct kvm_vcpu * vcpu)1310 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1311 {
1312 	/* this equals initial cpu reset in pop, but we don't switch to ESA */
1313 	vcpu->arch.sie_block->gpsw.mask = 0UL;
1314 	vcpu->arch.sie_block->gpsw.addr = 0UL;
1315 	kvm_s390_set_prefix(vcpu, 0);
1316 	vcpu->arch.sie_block->cputm     = 0UL;
1317 	vcpu->arch.sie_block->ckc       = 0UL;
1318 	vcpu->arch.sie_block->todpr     = 0;
1319 	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1320 	vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1321 	vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1322 	/* make sure the new fpc will be lazily loaded */
1323 	save_fpu_regs();
1324 	current->thread.fpu.fpc = 0;
1325 	vcpu->arch.sie_block->gbea = 1;
1326 	vcpu->arch.sie_block->pp = 0;
1327 	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1328 	kvm_clear_async_pf_completion_queue(vcpu);
1329 	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1330 		kvm_s390_vcpu_stop(vcpu);
1331 	kvm_s390_clear_local_irqs(vcpu);
1332 }
1333 
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)1334 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1335 {
1336 	mutex_lock(&vcpu->kvm->lock);
1337 	preempt_disable();
1338 	vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1339 	preempt_enable();
1340 	mutex_unlock(&vcpu->kvm->lock);
1341 	if (!kvm_is_ucontrol(vcpu->kvm))
1342 		vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1343 }
1344 
kvm_s390_vcpu_crypto_setup(struct kvm_vcpu * vcpu)1345 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1346 {
1347 	if (!test_kvm_facility(vcpu->kvm, 76))
1348 		return;
1349 
1350 	vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1351 
1352 	if (vcpu->kvm->arch.crypto.aes_kw)
1353 		vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1354 	if (vcpu->kvm->arch.crypto.dea_kw)
1355 		vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1356 
1357 	vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1358 }
1359 
kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu * vcpu)1360 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1361 {
1362 	free_page(vcpu->arch.sie_block->cbrlo);
1363 	vcpu->arch.sie_block->cbrlo = 0;
1364 }
1365 
kvm_s390_vcpu_setup_cmma(struct kvm_vcpu * vcpu)1366 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1367 {
1368 	vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1369 	if (!vcpu->arch.sie_block->cbrlo)
1370 		return -ENOMEM;
1371 
1372 	vcpu->arch.sie_block->ecb2 |= 0x80;
1373 	vcpu->arch.sie_block->ecb2 &= ~0x08;
1374 	return 0;
1375 }
1376 
kvm_s390_vcpu_setup_model(struct kvm_vcpu * vcpu)1377 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1378 {
1379 	struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1380 
1381 	vcpu->arch.cpu_id = model->cpu_id;
1382 	vcpu->arch.sie_block->ibc = model->ibc;
1383 	vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1384 }
1385 
kvm_arch_vcpu_setup(struct kvm_vcpu * vcpu)1386 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1387 {
1388 	int rc = 0;
1389 
1390 	atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1391 						    CPUSTAT_SM |
1392 						    CPUSTAT_STOPPED);
1393 
1394 	if (test_kvm_facility(vcpu->kvm, 78))
1395 		atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1396 	else if (test_kvm_facility(vcpu->kvm, 8))
1397 		atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1398 
1399 	kvm_s390_vcpu_setup_model(vcpu);
1400 
1401 	vcpu->arch.sie_block->ecb   = 6;
1402 	if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1403 		vcpu->arch.sie_block->ecb |= 0x10;
1404 
1405 	vcpu->arch.sie_block->ecb2  = 8;
1406 	vcpu->arch.sie_block->eca   = 0xC1002000U;
1407 	if (sclp.has_siif)
1408 		vcpu->arch.sie_block->eca |= 1;
1409 	if (sclp.has_sigpif)
1410 		vcpu->arch.sie_block->eca |= 0x10000000U;
1411 	if (test_kvm_facility(vcpu->kvm, 129)) {
1412 		vcpu->arch.sie_block->eca |= 0x00020000;
1413 		vcpu->arch.sie_block->ecd |= 0x20000000;
1414 	}
1415 	vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1416 
1417 	if (vcpu->kvm->arch.use_cmma) {
1418 		rc = kvm_s390_vcpu_setup_cmma(vcpu);
1419 		if (rc)
1420 			return rc;
1421 	}
1422 	hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1423 	vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1424 
1425 	kvm_s390_vcpu_crypto_setup(vcpu);
1426 
1427 	return rc;
1428 }
1429 
kvm_arch_vcpu_create(struct kvm * kvm,unsigned int id)1430 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1431 				      unsigned int id)
1432 {
1433 	struct kvm_vcpu *vcpu;
1434 	struct sie_page *sie_page;
1435 	int rc = -EINVAL;
1436 
1437 	if (id >= KVM_MAX_VCPUS)
1438 		goto out;
1439 
1440 	rc = -ENOMEM;
1441 
1442 	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1443 	if (!vcpu)
1444 		goto out;
1445 
1446 	sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1447 	if (!sie_page)
1448 		goto out_free_cpu;
1449 
1450 	vcpu->arch.sie_block = &sie_page->sie_block;
1451 	vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1452 
1453 	vcpu->arch.sie_block->icpua = id;
1454 	if (!kvm_is_ucontrol(kvm)) {
1455 		if (!kvm->arch.sca) {
1456 			WARN_ON_ONCE(1);
1457 			goto out_free_cpu;
1458 		}
1459 		if (!kvm->arch.sca->cpu[id].sda)
1460 			kvm->arch.sca->cpu[id].sda =
1461 				(__u64) vcpu->arch.sie_block;
1462 		vcpu->arch.sie_block->scaoh =
1463 			(__u32)(((__u64)kvm->arch.sca) >> 32);
1464 		vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1465 		set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1466 	}
1467 
1468 	spin_lock_init(&vcpu->arch.local_int.lock);
1469 	vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1470 	vcpu->arch.local_int.wq = &vcpu->wq;
1471 	vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1472 
1473 	rc = kvm_vcpu_init(vcpu, kvm, id);
1474 	if (rc)
1475 		goto out_free_sie_block;
1476 	VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1477 		 vcpu->arch.sie_block);
1478 	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1479 
1480 	return vcpu;
1481 out_free_sie_block:
1482 	free_page((unsigned long)(vcpu->arch.sie_block));
1483 out_free_cpu:
1484 	kmem_cache_free(kvm_vcpu_cache, vcpu);
1485 out:
1486 	return ERR_PTR(rc);
1487 }
1488 
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)1489 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1490 {
1491 	return kvm_s390_vcpu_has_irq(vcpu, 0);
1492 }
1493 
kvm_s390_vcpu_block(struct kvm_vcpu * vcpu)1494 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1495 {
1496 	atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1497 	exit_sie(vcpu);
1498 }
1499 
kvm_s390_vcpu_unblock(struct kvm_vcpu * vcpu)1500 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1501 {
1502 	atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1503 }
1504 
kvm_s390_vcpu_request(struct kvm_vcpu * vcpu)1505 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1506 {
1507 	atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1508 	exit_sie(vcpu);
1509 }
1510 
kvm_s390_vcpu_request_handled(struct kvm_vcpu * vcpu)1511 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1512 {
1513 	atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1514 }
1515 
1516 /*
1517  * Kick a guest cpu out of SIE and wait until SIE is not running.
1518  * If the CPU is not running (e.g. waiting as idle) the function will
1519  * return immediately. */
exit_sie(struct kvm_vcpu * vcpu)1520 void exit_sie(struct kvm_vcpu *vcpu)
1521 {
1522 	atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1523 	while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1524 		cpu_relax();
1525 }
1526 
1527 /* Kick a guest cpu out of SIE to process a request synchronously */
kvm_s390_sync_request(int req,struct kvm_vcpu * vcpu)1528 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1529 {
1530 	kvm_make_request(req, vcpu);
1531 	kvm_s390_vcpu_request(vcpu);
1532 }
1533 
kvm_gmap_notifier(struct gmap * gmap,unsigned long address)1534 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1535 {
1536 	int i;
1537 	struct kvm *kvm = gmap->private;
1538 	struct kvm_vcpu *vcpu;
1539 
1540 	kvm_for_each_vcpu(i, vcpu, kvm) {
1541 		/* match against both prefix pages */
1542 		if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1543 			VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1544 			kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1545 		}
1546 	}
1547 }
1548 
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)1549 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1550 {
1551 	/* kvm common code refers to this, but never calls it */
1552 	BUG();
1553 	return 0;
1554 }
1555 
kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)1556 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1557 					   struct kvm_one_reg *reg)
1558 {
1559 	int r = -EINVAL;
1560 
1561 	switch (reg->id) {
1562 	case KVM_REG_S390_TODPR:
1563 		r = put_user(vcpu->arch.sie_block->todpr,
1564 			     (u32 __user *)reg->addr);
1565 		break;
1566 	case KVM_REG_S390_EPOCHDIFF:
1567 		r = put_user(vcpu->arch.sie_block->epoch,
1568 			     (u64 __user *)reg->addr);
1569 		break;
1570 	case KVM_REG_S390_CPU_TIMER:
1571 		r = put_user(vcpu->arch.sie_block->cputm,
1572 			     (u64 __user *)reg->addr);
1573 		break;
1574 	case KVM_REG_S390_CLOCK_COMP:
1575 		r = put_user(vcpu->arch.sie_block->ckc,
1576 			     (u64 __user *)reg->addr);
1577 		break;
1578 	case KVM_REG_S390_PFTOKEN:
1579 		r = put_user(vcpu->arch.pfault_token,
1580 			     (u64 __user *)reg->addr);
1581 		break;
1582 	case KVM_REG_S390_PFCOMPARE:
1583 		r = put_user(vcpu->arch.pfault_compare,
1584 			     (u64 __user *)reg->addr);
1585 		break;
1586 	case KVM_REG_S390_PFSELECT:
1587 		r = put_user(vcpu->arch.pfault_select,
1588 			     (u64 __user *)reg->addr);
1589 		break;
1590 	case KVM_REG_S390_PP:
1591 		r = put_user(vcpu->arch.sie_block->pp,
1592 			     (u64 __user *)reg->addr);
1593 		break;
1594 	case KVM_REG_S390_GBEA:
1595 		r = put_user(vcpu->arch.sie_block->gbea,
1596 			     (u64 __user *)reg->addr);
1597 		break;
1598 	default:
1599 		break;
1600 	}
1601 
1602 	return r;
1603 }
1604 
kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)1605 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1606 					   struct kvm_one_reg *reg)
1607 {
1608 	int r = -EINVAL;
1609 
1610 	switch (reg->id) {
1611 	case KVM_REG_S390_TODPR:
1612 		r = get_user(vcpu->arch.sie_block->todpr,
1613 			     (u32 __user *)reg->addr);
1614 		break;
1615 	case KVM_REG_S390_EPOCHDIFF:
1616 		r = get_user(vcpu->arch.sie_block->epoch,
1617 			     (u64 __user *)reg->addr);
1618 		break;
1619 	case KVM_REG_S390_CPU_TIMER:
1620 		r = get_user(vcpu->arch.sie_block->cputm,
1621 			     (u64 __user *)reg->addr);
1622 		break;
1623 	case KVM_REG_S390_CLOCK_COMP:
1624 		r = get_user(vcpu->arch.sie_block->ckc,
1625 			     (u64 __user *)reg->addr);
1626 		break;
1627 	case KVM_REG_S390_PFTOKEN:
1628 		r = get_user(vcpu->arch.pfault_token,
1629 			     (u64 __user *)reg->addr);
1630 		if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1631 			kvm_clear_async_pf_completion_queue(vcpu);
1632 		break;
1633 	case KVM_REG_S390_PFCOMPARE:
1634 		r = get_user(vcpu->arch.pfault_compare,
1635 			     (u64 __user *)reg->addr);
1636 		break;
1637 	case KVM_REG_S390_PFSELECT:
1638 		r = get_user(vcpu->arch.pfault_select,
1639 			     (u64 __user *)reg->addr);
1640 		break;
1641 	case KVM_REG_S390_PP:
1642 		r = get_user(vcpu->arch.sie_block->pp,
1643 			     (u64 __user *)reg->addr);
1644 		break;
1645 	case KVM_REG_S390_GBEA:
1646 		r = get_user(vcpu->arch.sie_block->gbea,
1647 			     (u64 __user *)reg->addr);
1648 		break;
1649 	default:
1650 		break;
1651 	}
1652 
1653 	return r;
1654 }
1655 
kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu * vcpu)1656 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1657 {
1658 	kvm_s390_vcpu_initial_reset(vcpu);
1659 	return 0;
1660 }
1661 
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)1662 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1663 {
1664 	memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1665 	return 0;
1666 }
1667 
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)1668 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1669 {
1670 	memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1671 	return 0;
1672 }
1673 
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)1674 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1675 				  struct kvm_sregs *sregs)
1676 {
1677 	memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1678 	memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1679 	restore_access_regs(vcpu->run->s.regs.acrs);
1680 	return 0;
1681 }
1682 
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)1683 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1684 				  struct kvm_sregs *sregs)
1685 {
1686 	memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1687 	memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1688 	return 0;
1689 }
1690 
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)1691 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1692 {
1693 	/* make sure the new values will be lazily loaded */
1694 	save_fpu_regs();
1695 	if (test_fp_ctl(fpu->fpc))
1696 		return -EINVAL;
1697 	current->thread.fpu.fpc = fpu->fpc;
1698 	if (MACHINE_HAS_VX)
1699 		convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
1700 	else
1701 		memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
1702 	return 0;
1703 }
1704 
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)1705 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1706 {
1707 	/* make sure we have the latest values */
1708 	save_fpu_regs();
1709 	if (MACHINE_HAS_VX)
1710 		convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
1711 	else
1712 		memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
1713 	fpu->fpc = current->thread.fpu.fpc;
1714 	return 0;
1715 }
1716 
kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu * vcpu,psw_t psw)1717 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1718 {
1719 	int rc = 0;
1720 
1721 	if (!is_vcpu_stopped(vcpu))
1722 		rc = -EBUSY;
1723 	else {
1724 		vcpu->run->psw_mask = psw.mask;
1725 		vcpu->run->psw_addr = psw.addr;
1726 	}
1727 	return rc;
1728 }
1729 
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)1730 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1731 				  struct kvm_translation *tr)
1732 {
1733 	return -EINVAL; /* not implemented yet */
1734 }
1735 
1736 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1737 			      KVM_GUESTDBG_USE_HW_BP | \
1738 			      KVM_GUESTDBG_ENABLE)
1739 
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)1740 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1741 					struct kvm_guest_debug *dbg)
1742 {
1743 	int rc = 0;
1744 
1745 	vcpu->guest_debug = 0;
1746 	kvm_s390_clear_bp_data(vcpu);
1747 
1748 	if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1749 		return -EINVAL;
1750 
1751 	if (dbg->control & KVM_GUESTDBG_ENABLE) {
1752 		vcpu->guest_debug = dbg->control;
1753 		/* enforce guest PER */
1754 		atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1755 
1756 		if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1757 			rc = kvm_s390_import_bp_data(vcpu, dbg);
1758 	} else {
1759 		atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1760 		vcpu->arch.guestdbg.last_bp = 0;
1761 	}
1762 
1763 	if (rc) {
1764 		vcpu->guest_debug = 0;
1765 		kvm_s390_clear_bp_data(vcpu);
1766 		atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1767 	}
1768 
1769 	return rc;
1770 }
1771 
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)1772 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1773 				    struct kvm_mp_state *mp_state)
1774 {
1775 	/* CHECK_STOP and LOAD are not supported yet */
1776 	return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1777 				       KVM_MP_STATE_OPERATING;
1778 }
1779 
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)1780 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1781 				    struct kvm_mp_state *mp_state)
1782 {
1783 	int rc = 0;
1784 
1785 	/* user space knows about this interface - let it control the state */
1786 	vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1787 
1788 	switch (mp_state->mp_state) {
1789 	case KVM_MP_STATE_STOPPED:
1790 		kvm_s390_vcpu_stop(vcpu);
1791 		break;
1792 	case KVM_MP_STATE_OPERATING:
1793 		kvm_s390_vcpu_start(vcpu);
1794 		break;
1795 	case KVM_MP_STATE_LOAD:
1796 	case KVM_MP_STATE_CHECK_STOP:
1797 		/* fall through - CHECK_STOP and LOAD are not supported yet */
1798 	default:
1799 		rc = -ENXIO;
1800 	}
1801 
1802 	return rc;
1803 }
1804 
ibs_enabled(struct kvm_vcpu * vcpu)1805 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1806 {
1807 	return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1808 }
1809 
kvm_s390_handle_requests(struct kvm_vcpu * vcpu)1810 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1811 {
1812 retry:
1813 	kvm_s390_vcpu_request_handled(vcpu);
1814 	if (!vcpu->requests)
1815 		return 0;
1816 	/*
1817 	 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1818 	 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1819 	 * This ensures that the ipte instruction for this request has
1820 	 * already finished. We might race against a second unmapper that
1821 	 * wants to set the blocking bit. Lets just retry the request loop.
1822 	 */
1823 	if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1824 		int rc;
1825 		rc = gmap_ipte_notify(vcpu->arch.gmap,
1826 				      kvm_s390_get_prefix(vcpu),
1827 				      PAGE_SIZE * 2);
1828 		if (rc)
1829 			return rc;
1830 		goto retry;
1831 	}
1832 
1833 	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1834 		vcpu->arch.sie_block->ihcpu = 0xffff;
1835 		goto retry;
1836 	}
1837 
1838 	if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1839 		if (!ibs_enabled(vcpu)) {
1840 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1841 			atomic_or(CPUSTAT_IBS,
1842 					&vcpu->arch.sie_block->cpuflags);
1843 		}
1844 		goto retry;
1845 	}
1846 
1847 	if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1848 		if (ibs_enabled(vcpu)) {
1849 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1850 			atomic_andnot(CPUSTAT_IBS,
1851 					  &vcpu->arch.sie_block->cpuflags);
1852 		}
1853 		goto retry;
1854 	}
1855 
1856 	/* nothing to do, just clear the request */
1857 	clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1858 
1859 	return 0;
1860 }
1861 
kvm_s390_set_tod_clock(struct kvm * kvm,u64 tod)1862 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
1863 {
1864 	struct kvm_vcpu *vcpu;
1865 	int i;
1866 
1867 	mutex_lock(&kvm->lock);
1868 	preempt_disable();
1869 	kvm->arch.epoch = tod - get_tod_clock();
1870 	kvm_s390_vcpu_block_all(kvm);
1871 	kvm_for_each_vcpu(i, vcpu, kvm)
1872 		vcpu->arch.sie_block->epoch = kvm->arch.epoch;
1873 	kvm_s390_vcpu_unblock_all(kvm);
1874 	preempt_enable();
1875 	mutex_unlock(&kvm->lock);
1876 }
1877 
1878 /**
1879  * kvm_arch_fault_in_page - fault-in guest page if necessary
1880  * @vcpu: The corresponding virtual cpu
1881  * @gpa: Guest physical address
1882  * @writable: Whether the page should be writable or not
1883  *
1884  * Make sure that a guest page has been faulted-in on the host.
1885  *
1886  * Return: Zero on success, negative error code otherwise.
1887  */
kvm_arch_fault_in_page(struct kvm_vcpu * vcpu,gpa_t gpa,int writable)1888 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1889 {
1890 	return gmap_fault(vcpu->arch.gmap, gpa,
1891 			  writable ? FAULT_FLAG_WRITE : 0);
1892 }
1893 
__kvm_inject_pfault_token(struct kvm_vcpu * vcpu,bool start_token,unsigned long token)1894 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1895 				      unsigned long token)
1896 {
1897 	struct kvm_s390_interrupt inti;
1898 	struct kvm_s390_irq irq;
1899 
1900 	if (start_token) {
1901 		irq.u.ext.ext_params2 = token;
1902 		irq.type = KVM_S390_INT_PFAULT_INIT;
1903 		WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1904 	} else {
1905 		inti.type = KVM_S390_INT_PFAULT_DONE;
1906 		inti.parm64 = token;
1907 		WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1908 	}
1909 }
1910 
kvm_arch_async_page_not_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)1911 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1912 				     struct kvm_async_pf *work)
1913 {
1914 	trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1915 	__kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1916 }
1917 
kvm_arch_async_page_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)1918 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1919 				 struct kvm_async_pf *work)
1920 {
1921 	trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1922 	__kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1923 }
1924 
kvm_arch_async_page_ready(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)1925 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1926 			       struct kvm_async_pf *work)
1927 {
1928 	/* s390 will always inject the page directly */
1929 }
1930 
kvm_arch_can_inject_async_page_present(struct kvm_vcpu * vcpu)1931 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1932 {
1933 	/*
1934 	 * s390 will always inject the page directly,
1935 	 * but we still want check_async_completion to cleanup
1936 	 */
1937 	return true;
1938 }
1939 
kvm_arch_setup_async_pf(struct kvm_vcpu * vcpu)1940 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1941 {
1942 	hva_t hva;
1943 	struct kvm_arch_async_pf arch;
1944 	int rc;
1945 
1946 	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1947 		return 0;
1948 	if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1949 	    vcpu->arch.pfault_compare)
1950 		return 0;
1951 	if (psw_extint_disabled(vcpu))
1952 		return 0;
1953 	if (kvm_s390_vcpu_has_irq(vcpu, 0))
1954 		return 0;
1955 	if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1956 		return 0;
1957 	if (!vcpu->arch.gmap->pfault_enabled)
1958 		return 0;
1959 
1960 	hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1961 	hva += current->thread.gmap_addr & ~PAGE_MASK;
1962 	if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1963 		return 0;
1964 
1965 	rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1966 	return rc;
1967 }
1968 
vcpu_pre_run(struct kvm_vcpu * vcpu)1969 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1970 {
1971 	int rc, cpuflags;
1972 
1973 	/*
1974 	 * On s390 notifications for arriving pages will be delivered directly
1975 	 * to the guest but the house keeping for completed pfaults is
1976 	 * handled outside the worker.
1977 	 */
1978 	kvm_check_async_pf_completion(vcpu);
1979 
1980 	memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1981 
1982 	if (need_resched())
1983 		schedule();
1984 
1985 	if (test_cpu_flag(CIF_MCCK_PENDING))
1986 		s390_handle_mcck();
1987 
1988 	if (!kvm_is_ucontrol(vcpu->kvm)) {
1989 		rc = kvm_s390_deliver_pending_interrupts(vcpu);
1990 		if (rc)
1991 			return rc;
1992 	}
1993 
1994 	rc = kvm_s390_handle_requests(vcpu);
1995 	if (rc)
1996 		return rc;
1997 
1998 	if (guestdbg_enabled(vcpu)) {
1999 		kvm_s390_backup_guest_per_regs(vcpu);
2000 		kvm_s390_patch_guest_per_regs(vcpu);
2001 	}
2002 
2003 	vcpu->arch.sie_block->icptcode = 0;
2004 	cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2005 	VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2006 	trace_kvm_s390_sie_enter(vcpu, cpuflags);
2007 
2008 	return 0;
2009 }
2010 
vcpu_post_run_fault_in_sie(struct kvm_vcpu * vcpu)2011 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2012 {
2013 	psw_t *psw = &vcpu->arch.sie_block->gpsw;
2014 	u8 opcode;
2015 	int rc;
2016 
2017 	VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2018 	trace_kvm_s390_sie_fault(vcpu);
2019 
2020 	/*
2021 	 * We want to inject an addressing exception, which is defined as a
2022 	 * suppressing or terminating exception. However, since we came here
2023 	 * by a DAT access exception, the PSW still points to the faulting
2024 	 * instruction since DAT exceptions are nullifying. So we've got
2025 	 * to look up the current opcode to get the length of the instruction
2026 	 * to be able to forward the PSW.
2027 	 */
2028 	rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2029 	if (rc)
2030 		return kvm_s390_inject_prog_cond(vcpu, rc);
2031 	psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2032 
2033 	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2034 }
2035 
vcpu_post_run(struct kvm_vcpu * vcpu,int exit_reason)2036 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2037 {
2038 	int rc = -1;
2039 
2040 	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2041 		   vcpu->arch.sie_block->icptcode);
2042 	trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2043 
2044 	if (guestdbg_enabled(vcpu))
2045 		kvm_s390_restore_guest_per_regs(vcpu);
2046 
2047 	if (exit_reason >= 0) {
2048 		rc = 0;
2049 	} else if (kvm_is_ucontrol(vcpu->kvm)) {
2050 		vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2051 		vcpu->run->s390_ucontrol.trans_exc_code =
2052 						current->thread.gmap_addr;
2053 		vcpu->run->s390_ucontrol.pgm_code = 0x10;
2054 		rc = -EREMOTE;
2055 
2056 	} else if (current->thread.gmap_pfault) {
2057 		trace_kvm_s390_major_guest_pfault(vcpu);
2058 		current->thread.gmap_pfault = 0;
2059 		if (kvm_arch_setup_async_pf(vcpu)) {
2060 			rc = 0;
2061 		} else {
2062 			gpa_t gpa = current->thread.gmap_addr;
2063 			rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
2064 		}
2065 	}
2066 
2067 	if (rc == -1)
2068 		rc = vcpu_post_run_fault_in_sie(vcpu);
2069 
2070 	memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2071 
2072 	if (rc == 0) {
2073 		if (kvm_is_ucontrol(vcpu->kvm))
2074 			/* Don't exit for host interrupts. */
2075 			rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
2076 		else
2077 			rc = kvm_handle_sie_intercept(vcpu);
2078 	}
2079 
2080 	return rc;
2081 }
2082 
__vcpu_run(struct kvm_vcpu * vcpu)2083 static int __vcpu_run(struct kvm_vcpu *vcpu)
2084 {
2085 	int rc, exit_reason;
2086 
2087 	/*
2088 	 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2089 	 * ning the guest), so that memslots (and other stuff) are protected
2090 	 */
2091 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2092 
2093 	do {
2094 		rc = vcpu_pre_run(vcpu);
2095 		if (rc)
2096 			break;
2097 
2098 		srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2099 		/*
2100 		 * As PF_VCPU will be used in fault handler, between
2101 		 * guest_enter and guest_exit should be no uaccess.
2102 		 */
2103 		local_irq_disable();
2104 		__kvm_guest_enter();
2105 		local_irq_enable();
2106 		exit_reason = sie64a(vcpu->arch.sie_block,
2107 				     vcpu->run->s.regs.gprs);
2108 		local_irq_disable();
2109 		__kvm_guest_exit();
2110 		local_irq_enable();
2111 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2112 
2113 		rc = vcpu_post_run(vcpu, exit_reason);
2114 	} while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2115 
2116 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2117 	return rc;
2118 }
2119 
sync_regs(struct kvm_vcpu * vcpu,struct kvm_run * kvm_run)2120 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2121 {
2122 	vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2123 	vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2124 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2125 		kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2126 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2127 		memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2128 		/* some control register changes require a tlb flush */
2129 		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2130 	}
2131 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2132 		vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2133 		vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2134 		vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2135 		vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2136 		vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2137 	}
2138 	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2139 		vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2140 		vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2141 		vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2142 		if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2143 			kvm_clear_async_pf_completion_queue(vcpu);
2144 	}
2145 	kvm_run->kvm_dirty_regs = 0;
2146 }
2147 
store_regs(struct kvm_vcpu * vcpu,struct kvm_run * kvm_run)2148 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2149 {
2150 	kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2151 	kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2152 	kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2153 	memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2154 	kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2155 	kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2156 	kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2157 	kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2158 	kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2159 	kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2160 	kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2161 	kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2162 }
2163 
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu,struct kvm_run * kvm_run)2164 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2165 {
2166 	int rc;
2167 	sigset_t sigsaved;
2168 
2169 	if (guestdbg_exit_pending(vcpu)) {
2170 		kvm_s390_prepare_debug_exit(vcpu);
2171 		return 0;
2172 	}
2173 
2174 	if (vcpu->sigset_active)
2175 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2176 
2177 	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2178 		kvm_s390_vcpu_start(vcpu);
2179 	} else if (is_vcpu_stopped(vcpu)) {
2180 		pr_err_ratelimited("can't run stopped vcpu %d\n",
2181 				   vcpu->vcpu_id);
2182 		return -EINVAL;
2183 	}
2184 
2185 	sync_regs(vcpu, kvm_run);
2186 
2187 	might_fault();
2188 	rc = __vcpu_run(vcpu);
2189 
2190 	if (signal_pending(current) && !rc) {
2191 		kvm_run->exit_reason = KVM_EXIT_INTR;
2192 		rc = -EINTR;
2193 	}
2194 
2195 	if (guestdbg_exit_pending(vcpu) && !rc)  {
2196 		kvm_s390_prepare_debug_exit(vcpu);
2197 		rc = 0;
2198 	}
2199 
2200 	if (rc == -EOPNOTSUPP) {
2201 		/* intercept cannot be handled in-kernel, prepare kvm-run */
2202 		kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
2203 		kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2204 		kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
2205 		kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
2206 		rc = 0;
2207 	}
2208 
2209 	if (rc == -EREMOTE) {
2210 		/* intercept was handled, but userspace support is needed
2211 		 * kvm_run has been prepared by the handler */
2212 		rc = 0;
2213 	}
2214 
2215 	store_regs(vcpu, kvm_run);
2216 
2217 	if (vcpu->sigset_active)
2218 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2219 
2220 	vcpu->stat.exit_userspace++;
2221 	return rc;
2222 }
2223 
2224 /*
2225  * store status at address
2226  * we use have two special cases:
2227  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2228  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2229  */
kvm_s390_store_status_unloaded(struct kvm_vcpu * vcpu,unsigned long gpa)2230 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2231 {
2232 	unsigned char archmode = 1;
2233 	freg_t fprs[NUM_FPRS];
2234 	unsigned int px;
2235 	u64 clkcomp;
2236 	int rc;
2237 
2238 	px = kvm_s390_get_prefix(vcpu);
2239 	if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2240 		if (write_guest_abs(vcpu, 163, &archmode, 1))
2241 			return -EFAULT;
2242 		gpa = 0;
2243 	} else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2244 		if (write_guest_real(vcpu, 163, &archmode, 1))
2245 			return -EFAULT;
2246 		gpa = px;
2247 	} else
2248 		gpa -= __LC_FPREGS_SAVE_AREA;
2249 
2250 	/* manually convert vector registers if necessary */
2251 	if (MACHINE_HAS_VX) {
2252 		convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2253 		rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2254 				     fprs, 128);
2255 	} else {
2256 		rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2257 				     vcpu->run->s.regs.vrs, 128);
2258 	}
2259 	rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2260 			      vcpu->run->s.regs.gprs, 128);
2261 	rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2262 			      &vcpu->arch.sie_block->gpsw, 16);
2263 	rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2264 			      &px, 4);
2265 	rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2266 			      &vcpu->run->s.regs.fpc, 4);
2267 	rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2268 			      &vcpu->arch.sie_block->todpr, 4);
2269 	rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2270 			      &vcpu->arch.sie_block->cputm, 8);
2271 	clkcomp = vcpu->arch.sie_block->ckc >> 8;
2272 	rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2273 			      &clkcomp, 8);
2274 	rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2275 			      &vcpu->run->s.regs.acrs, 64);
2276 	rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2277 			      &vcpu->arch.sie_block->gcr, 128);
2278 	return rc ? -EFAULT : 0;
2279 }
2280 
kvm_s390_vcpu_store_status(struct kvm_vcpu * vcpu,unsigned long addr)2281 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2282 {
2283 	/*
2284 	 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2285 	 * copying in vcpu load/put. Lets update our copies before we save
2286 	 * it into the save area
2287 	 */
2288 	save_fpu_regs();
2289 	vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2290 	save_access_regs(vcpu->run->s.regs.acrs);
2291 
2292 	return kvm_s390_store_status_unloaded(vcpu, addr);
2293 }
2294 
2295 /*
2296  * store additional status at address
2297  */
kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu * vcpu,unsigned long gpa)2298 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2299 					unsigned long gpa)
2300 {
2301 	/* Only bits 0-53 are used for address formation */
2302 	if (!(gpa & ~0x3ff))
2303 		return 0;
2304 
2305 	return write_guest_abs(vcpu, gpa & ~0x3ff,
2306 			       (void *)&vcpu->run->s.regs.vrs, 512);
2307 }
2308 
kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu * vcpu,unsigned long addr)2309 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2310 {
2311 	if (!test_kvm_facility(vcpu->kvm, 129))
2312 		return 0;
2313 
2314 	/*
2315 	 * The guest VXRS are in the host VXRs due to the lazy
2316 	 * copying in vcpu load/put. We can simply call save_fpu_regs()
2317 	 * to save the current register state because we are in the
2318 	 * middle of a load/put cycle.
2319 	 *
2320 	 * Let's update our copies before we save it into the save area.
2321 	 */
2322 	save_fpu_regs();
2323 
2324 	return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2325 }
2326 
__disable_ibs_on_vcpu(struct kvm_vcpu * vcpu)2327 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2328 {
2329 	kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2330 	kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2331 }
2332 
__disable_ibs_on_all_vcpus(struct kvm * kvm)2333 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2334 {
2335 	unsigned int i;
2336 	struct kvm_vcpu *vcpu;
2337 
2338 	kvm_for_each_vcpu(i, vcpu, kvm) {
2339 		__disable_ibs_on_vcpu(vcpu);
2340 	}
2341 }
2342 
__enable_ibs_on_vcpu(struct kvm_vcpu * vcpu)2343 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2344 {
2345 	kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2346 	kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2347 }
2348 
kvm_s390_vcpu_start(struct kvm_vcpu * vcpu)2349 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2350 {
2351 	int i, online_vcpus, started_vcpus = 0;
2352 
2353 	if (!is_vcpu_stopped(vcpu))
2354 		return;
2355 
2356 	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2357 	/* Only one cpu at a time may enter/leave the STOPPED state. */
2358 	spin_lock(&vcpu->kvm->arch.start_stop_lock);
2359 	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2360 
2361 	for (i = 0; i < online_vcpus; i++) {
2362 		if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2363 			started_vcpus++;
2364 	}
2365 
2366 	if (started_vcpus == 0) {
2367 		/* we're the only active VCPU -> speed it up */
2368 		__enable_ibs_on_vcpu(vcpu);
2369 	} else if (started_vcpus == 1) {
2370 		/*
2371 		 * As we are starting a second VCPU, we have to disable
2372 		 * the IBS facility on all VCPUs to remove potentially
2373 		 * oustanding ENABLE requests.
2374 		 */
2375 		__disable_ibs_on_all_vcpus(vcpu->kvm);
2376 	}
2377 
2378 	atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2379 	/*
2380 	 * Another VCPU might have used IBS while we were offline.
2381 	 * Let's play safe and flush the VCPU at startup.
2382 	 */
2383 	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2384 	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2385 	return;
2386 }
2387 
kvm_s390_vcpu_stop(struct kvm_vcpu * vcpu)2388 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2389 {
2390 	int i, online_vcpus, started_vcpus = 0;
2391 	struct kvm_vcpu *started_vcpu = NULL;
2392 
2393 	if (is_vcpu_stopped(vcpu))
2394 		return;
2395 
2396 	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2397 	/* Only one cpu at a time may enter/leave the STOPPED state. */
2398 	spin_lock(&vcpu->kvm->arch.start_stop_lock);
2399 	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2400 
2401 	/* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2402 	kvm_s390_clear_stop_irq(vcpu);
2403 
2404 	atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2405 	__disable_ibs_on_vcpu(vcpu);
2406 
2407 	for (i = 0; i < online_vcpus; i++) {
2408 		if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2409 			started_vcpus++;
2410 			started_vcpu = vcpu->kvm->vcpus[i];
2411 		}
2412 	}
2413 
2414 	if (started_vcpus == 1) {
2415 		/*
2416 		 * As we only have one VCPU left, we want to enable the
2417 		 * IBS facility for that VCPU to speed it up.
2418 		 */
2419 		__enable_ibs_on_vcpu(started_vcpu);
2420 	}
2421 
2422 	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2423 	return;
2424 }
2425 
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)2426 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2427 				     struct kvm_enable_cap *cap)
2428 {
2429 	int r;
2430 
2431 	if (cap->flags)
2432 		return -EINVAL;
2433 
2434 	switch (cap->cap) {
2435 	case KVM_CAP_S390_CSS_SUPPORT:
2436 		if (!vcpu->kvm->arch.css_support) {
2437 			vcpu->kvm->arch.css_support = 1;
2438 			VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2439 			trace_kvm_s390_enable_css(vcpu->kvm);
2440 		}
2441 		r = 0;
2442 		break;
2443 	default:
2444 		r = -EINVAL;
2445 		break;
2446 	}
2447 	return r;
2448 }
2449 
kvm_s390_guest_mem_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)2450 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2451 				  struct kvm_s390_mem_op *mop)
2452 {
2453 	void __user *uaddr = (void __user *)mop->buf;
2454 	void *tmpbuf = NULL;
2455 	int r, srcu_idx;
2456 	const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2457 				    | KVM_S390_MEMOP_F_CHECK_ONLY;
2458 
2459 	if (mop->flags & ~supported_flags)
2460 		return -EINVAL;
2461 
2462 	if (mop->size > MEM_OP_MAX_SIZE)
2463 		return -E2BIG;
2464 
2465 	if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2466 		tmpbuf = vmalloc(mop->size);
2467 		if (!tmpbuf)
2468 			return -ENOMEM;
2469 	}
2470 
2471 	srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2472 
2473 	switch (mop->op) {
2474 	case KVM_S390_MEMOP_LOGICAL_READ:
2475 		if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2476 			r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2477 			break;
2478 		}
2479 		r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2480 		if (r == 0) {
2481 			if (copy_to_user(uaddr, tmpbuf, mop->size))
2482 				r = -EFAULT;
2483 		}
2484 		break;
2485 	case KVM_S390_MEMOP_LOGICAL_WRITE:
2486 		if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2487 			r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2488 			break;
2489 		}
2490 		if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2491 			r = -EFAULT;
2492 			break;
2493 		}
2494 		r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2495 		break;
2496 	default:
2497 		r = -EINVAL;
2498 	}
2499 
2500 	srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2501 
2502 	if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2503 		kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2504 
2505 	vfree(tmpbuf);
2506 	return r;
2507 }
2508 
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2509 long kvm_arch_vcpu_ioctl(struct file *filp,
2510 			 unsigned int ioctl, unsigned long arg)
2511 {
2512 	struct kvm_vcpu *vcpu = filp->private_data;
2513 	void __user *argp = (void __user *)arg;
2514 	int idx;
2515 	long r;
2516 
2517 	switch (ioctl) {
2518 	case KVM_S390_IRQ: {
2519 		struct kvm_s390_irq s390irq;
2520 
2521 		r = -EFAULT;
2522 		if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2523 			break;
2524 		r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2525 		break;
2526 	}
2527 	case KVM_S390_INTERRUPT: {
2528 		struct kvm_s390_interrupt s390int;
2529 		struct kvm_s390_irq s390irq;
2530 
2531 		r = -EFAULT;
2532 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
2533 			break;
2534 		if (s390int_to_s390irq(&s390int, &s390irq))
2535 			return -EINVAL;
2536 		r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2537 		break;
2538 	}
2539 	case KVM_S390_STORE_STATUS:
2540 		idx = srcu_read_lock(&vcpu->kvm->srcu);
2541 		r = kvm_s390_vcpu_store_status(vcpu, arg);
2542 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
2543 		break;
2544 	case KVM_S390_SET_INITIAL_PSW: {
2545 		psw_t psw;
2546 
2547 		r = -EFAULT;
2548 		if (copy_from_user(&psw, argp, sizeof(psw)))
2549 			break;
2550 		r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2551 		break;
2552 	}
2553 	case KVM_S390_INITIAL_RESET:
2554 		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2555 		break;
2556 	case KVM_SET_ONE_REG:
2557 	case KVM_GET_ONE_REG: {
2558 		struct kvm_one_reg reg;
2559 		r = -EFAULT;
2560 		if (copy_from_user(&reg, argp, sizeof(reg)))
2561 			break;
2562 		if (ioctl == KVM_SET_ONE_REG)
2563 			r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2564 		else
2565 			r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2566 		break;
2567 	}
2568 #ifdef CONFIG_KVM_S390_UCONTROL
2569 	case KVM_S390_UCAS_MAP: {
2570 		struct kvm_s390_ucas_mapping ucasmap;
2571 
2572 		if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2573 			r = -EFAULT;
2574 			break;
2575 		}
2576 
2577 		if (!kvm_is_ucontrol(vcpu->kvm)) {
2578 			r = -EINVAL;
2579 			break;
2580 		}
2581 
2582 		r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2583 				     ucasmap.vcpu_addr, ucasmap.length);
2584 		break;
2585 	}
2586 	case KVM_S390_UCAS_UNMAP: {
2587 		struct kvm_s390_ucas_mapping ucasmap;
2588 
2589 		if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2590 			r = -EFAULT;
2591 			break;
2592 		}
2593 
2594 		if (!kvm_is_ucontrol(vcpu->kvm)) {
2595 			r = -EINVAL;
2596 			break;
2597 		}
2598 
2599 		r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2600 			ucasmap.length);
2601 		break;
2602 	}
2603 #endif
2604 	case KVM_S390_VCPU_FAULT: {
2605 		r = gmap_fault(vcpu->arch.gmap, arg, 0);
2606 		break;
2607 	}
2608 	case KVM_ENABLE_CAP:
2609 	{
2610 		struct kvm_enable_cap cap;
2611 		r = -EFAULT;
2612 		if (copy_from_user(&cap, argp, sizeof(cap)))
2613 			break;
2614 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2615 		break;
2616 	}
2617 	case KVM_S390_MEM_OP: {
2618 		struct kvm_s390_mem_op mem_op;
2619 
2620 		if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2621 			r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2622 		else
2623 			r = -EFAULT;
2624 		break;
2625 	}
2626 	case KVM_S390_SET_IRQ_STATE: {
2627 		struct kvm_s390_irq_state irq_state;
2628 
2629 		r = -EFAULT;
2630 		if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2631 			break;
2632 		if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2633 		    irq_state.len == 0 ||
2634 		    irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2635 			r = -EINVAL;
2636 			break;
2637 		}
2638 		r = kvm_s390_set_irq_state(vcpu,
2639 					   (void __user *) irq_state.buf,
2640 					   irq_state.len);
2641 		break;
2642 	}
2643 	case KVM_S390_GET_IRQ_STATE: {
2644 		struct kvm_s390_irq_state irq_state;
2645 
2646 		r = -EFAULT;
2647 		if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2648 			break;
2649 		if (irq_state.len == 0) {
2650 			r = -EINVAL;
2651 			break;
2652 		}
2653 		r = kvm_s390_get_irq_state(vcpu,
2654 					   (__u8 __user *)  irq_state.buf,
2655 					   irq_state.len);
2656 		break;
2657 	}
2658 	default:
2659 		r = -ENOTTY;
2660 	}
2661 	return r;
2662 }
2663 
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)2664 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2665 {
2666 #ifdef CONFIG_KVM_S390_UCONTROL
2667 	if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2668 		 && (kvm_is_ucontrol(vcpu->kvm))) {
2669 		vmf->page = virt_to_page(vcpu->arch.sie_block);
2670 		get_page(vmf->page);
2671 		return 0;
2672 	}
2673 #endif
2674 	return VM_FAULT_SIGBUS;
2675 }
2676 
kvm_arch_create_memslot(struct kvm * kvm,struct kvm_memory_slot * slot,unsigned long npages)2677 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2678 			    unsigned long npages)
2679 {
2680 	return 0;
2681 }
2682 
2683 /* Section: memory related */
kvm_arch_prepare_memory_region(struct kvm * kvm,struct kvm_memory_slot * memslot,const struct kvm_userspace_memory_region * mem,enum kvm_mr_change change)2684 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2685 				   struct kvm_memory_slot *memslot,
2686 				   const struct kvm_userspace_memory_region *mem,
2687 				   enum kvm_mr_change change)
2688 {
2689 	/* A few sanity checks. We can have memory slots which have to be
2690 	   located/ended at a segment boundary (1MB). The memory in userland is
2691 	   ok to be fragmented into various different vmas. It is okay to mmap()
2692 	   and munmap() stuff in this slot after doing this call at any time */
2693 
2694 	if (mem->userspace_addr & 0xffffful)
2695 		return -EINVAL;
2696 
2697 	if (mem->memory_size & 0xffffful)
2698 		return -EINVAL;
2699 
2700 	return 0;
2701 }
2702 
kvm_arch_commit_memory_region(struct kvm * kvm,const struct kvm_userspace_memory_region * mem,const struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)2703 void kvm_arch_commit_memory_region(struct kvm *kvm,
2704 				const struct kvm_userspace_memory_region *mem,
2705 				const struct kvm_memory_slot *old,
2706 				const struct kvm_memory_slot *new,
2707 				enum kvm_mr_change change)
2708 {
2709 	int rc;
2710 
2711 	/* If the basics of the memslot do not change, we do not want
2712 	 * to update the gmap. Every update causes several unnecessary
2713 	 * segment translation exceptions. This is usually handled just
2714 	 * fine by the normal fault handler + gmap, but it will also
2715 	 * cause faults on the prefix page of running guest CPUs.
2716 	 */
2717 	if (old->userspace_addr == mem->userspace_addr &&
2718 	    old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2719 	    old->npages * PAGE_SIZE == mem->memory_size)
2720 		return;
2721 
2722 	rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2723 		mem->guest_phys_addr, mem->memory_size);
2724 	if (rc)
2725 		pr_warn("failed to commit memory region\n");
2726 	return;
2727 }
2728 
kvm_s390_init(void)2729 static int __init kvm_s390_init(void)
2730 {
2731 	return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2732 }
2733 
kvm_s390_exit(void)2734 static void __exit kvm_s390_exit(void)
2735 {
2736 	kvm_exit();
2737 }
2738 
2739 module_init(kvm_s390_init);
2740 module_exit(kvm_s390_exit);
2741 
2742 /*
2743  * Enable autoloading of the kvm module.
2744  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2745  * since x86 takes a different approach.
2746  */
2747 #include <linux/miscdevice.h>
2748 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2749 MODULE_ALIAS("devname:kvm");
2750