This source file includes following definitions.
- kvm_vgic_early_init
- kvm_vgic_create
- kvm_vgic_dist_init
- kvm_vgic_vcpu_init
- kvm_vgic_vcpu_enable
- vgic_init
- kvm_vgic_dist_destroy
- kvm_vgic_vcpu_destroy
- __kvm_vgic_destroy
- kvm_vgic_destroy
- vgic_lazy_init
- kvm_vgic_map_resources
- vgic_init_cpu_starting
- vgic_init_cpu_dying
- vgic_maintenance_handler
- kvm_vgic_init_cpu_hardware
- kvm_vgic_hyp_init
1
2
3
4
5
6 #include <linux/uaccess.h>
7 #include <linux/interrupt.h>
8 #include <linux/cpu.h>
9 #include <linux/kvm_host.h>
10 #include <kvm/arm_vgic.h>
11 #include <asm/kvm_emulate.h>
12 #include <asm/kvm_mmu.h>
13 #include "vgic.h"
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 void kvm_vgic_early_init(struct kvm *kvm)
53 {
54 struct vgic_dist *dist = &kvm->arch.vgic;
55
56 INIT_LIST_HEAD(&dist->lpi_list_head);
57 INIT_LIST_HEAD(&dist->lpi_translation_cache);
58 raw_spin_lock_init(&dist->lpi_list_lock);
59 }
60
61
62
63
64
65
66
67
68
69
70
71 int kvm_vgic_create(struct kvm *kvm, u32 type)
72 {
73 int i, vcpu_lock_idx = -1, ret;
74 struct kvm_vcpu *vcpu;
75
76 if (irqchip_in_kernel(kvm))
77 return -EEXIST;
78
79
80
81
82
83
84
85 if (type == KVM_DEV_TYPE_ARM_VGIC_V2 &&
86 !kvm_vgic_global_state.can_emulate_gicv2)
87 return -ENODEV;
88
89
90
91
92
93
94 ret = -EBUSY;
95 kvm_for_each_vcpu(i, vcpu, kvm) {
96 if (!mutex_trylock(&vcpu->mutex))
97 goto out_unlock;
98 vcpu_lock_idx = i;
99 }
100
101 kvm_for_each_vcpu(i, vcpu, kvm) {
102 if (vcpu->arch.has_run_once)
103 goto out_unlock;
104 }
105 ret = 0;
106
107 if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
108 kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS;
109 else
110 kvm->arch.max_vcpus = VGIC_V3_MAX_CPUS;
111
112 if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus) {
113 ret = -E2BIG;
114 goto out_unlock;
115 }
116
117 kvm->arch.vgic.in_kernel = true;
118 kvm->arch.vgic.vgic_model = type;
119
120 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
121
122 if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
123 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
124 else
125 INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
126
127 out_unlock:
128 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
129 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
130 mutex_unlock(&vcpu->mutex);
131 }
132 return ret;
133 }
134
135
136
137
138
139
140
141
142 static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
143 {
144 struct vgic_dist *dist = &kvm->arch.vgic;
145 struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
146 int i;
147
148 dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL);
149 if (!dist->spis)
150 return -ENOMEM;
151
152
153
154
155
156
157
158
159
160 for (i = 0; i < nr_spis; i++) {
161 struct vgic_irq *irq = &dist->spis[i];
162
163 irq->intid = i + VGIC_NR_PRIVATE_IRQS;
164 INIT_LIST_HEAD(&irq->ap_list);
165 raw_spin_lock_init(&irq->irq_lock);
166 irq->vcpu = NULL;
167 irq->target_vcpu = vcpu0;
168 kref_init(&irq->refcount);
169 switch (dist->vgic_model) {
170 case KVM_DEV_TYPE_ARM_VGIC_V2:
171 irq->targets = 0;
172 irq->group = 0;
173 break;
174 case KVM_DEV_TYPE_ARM_VGIC_V3:
175 irq->mpidr = 0;
176 irq->group = 1;
177 break;
178 default:
179 kfree(dist->spis);
180 return -EINVAL;
181 }
182 }
183 return 0;
184 }
185
186
187
188
189
190
191
192
193
194
195 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
196 {
197 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
198 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
199 int ret = 0;
200 int i;
201
202 vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
203
204 INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
205 raw_spin_lock_init(&vgic_cpu->ap_list_lock);
206
207
208
209
210
211 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
212 struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
213
214 INIT_LIST_HEAD(&irq->ap_list);
215 raw_spin_lock_init(&irq->irq_lock);
216 irq->intid = i;
217 irq->vcpu = NULL;
218 irq->target_vcpu = vcpu;
219 kref_init(&irq->refcount);
220 if (vgic_irq_is_sgi(i)) {
221
222 irq->enabled = 1;
223 irq->config = VGIC_CONFIG_EDGE;
224 } else {
225
226 irq->config = VGIC_CONFIG_LEVEL;
227 }
228 }
229
230 if (!irqchip_in_kernel(vcpu->kvm))
231 return 0;
232
233
234
235
236
237 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
238 mutex_lock(&vcpu->kvm->lock);
239 ret = vgic_register_redist_iodev(vcpu);
240 mutex_unlock(&vcpu->kvm->lock);
241 }
242 return ret;
243 }
244
245 static void kvm_vgic_vcpu_enable(struct kvm_vcpu *vcpu)
246 {
247 if (kvm_vgic_global_state.type == VGIC_V2)
248 vgic_v2_enable(vcpu);
249 else
250 vgic_v3_enable(vcpu);
251 }
252
253
254
255
256
257
258
259
260
261
262
263 int vgic_init(struct kvm *kvm)
264 {
265 struct vgic_dist *dist = &kvm->arch.vgic;
266 struct kvm_vcpu *vcpu;
267 int ret = 0, i, idx;
268
269 if (vgic_initialized(kvm))
270 return 0;
271
272
273 if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
274 return -EBUSY;
275
276
277 if (!dist->nr_spis)
278 dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
279
280 ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
281 if (ret)
282 goto out;
283
284
285 kvm_for_each_vcpu(idx, vcpu, kvm) {
286 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
287
288 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
289 struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
290 switch (dist->vgic_model) {
291 case KVM_DEV_TYPE_ARM_VGIC_V3:
292 irq->group = 1;
293 irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
294 break;
295 case KVM_DEV_TYPE_ARM_VGIC_V2:
296 irq->group = 0;
297 irq->targets = 1U << idx;
298 break;
299 default:
300 ret = -EINVAL;
301 goto out;
302 }
303 }
304 }
305
306 if (vgic_has_its(kvm)) {
307 vgic_lpi_translation_cache_init(kvm);
308 ret = vgic_v4_init(kvm);
309 if (ret)
310 goto out;
311 }
312
313 kvm_for_each_vcpu(i, vcpu, kvm)
314 kvm_vgic_vcpu_enable(vcpu);
315
316 ret = kvm_vgic_setup_default_irq_routing(kvm);
317 if (ret)
318 goto out;
319
320 vgic_debug_init(kvm);
321
322 dist->implementation_rev = 2;
323 dist->initialized = true;
324
325 out:
326 return ret;
327 }
328
329 static void kvm_vgic_dist_destroy(struct kvm *kvm)
330 {
331 struct vgic_dist *dist = &kvm->arch.vgic;
332 struct vgic_redist_region *rdreg, *next;
333
334 dist->ready = false;
335 dist->initialized = false;
336
337 kfree(dist->spis);
338 dist->spis = NULL;
339 dist->nr_spis = 0;
340
341 if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
342 list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) {
343 list_del(&rdreg->list);
344 kfree(rdreg);
345 }
346 INIT_LIST_HEAD(&dist->rd_regions);
347 }
348
349 if (vgic_has_its(kvm))
350 vgic_lpi_translation_cache_destroy(kvm);
351
352 if (vgic_supports_direct_msis(kvm))
353 vgic_v4_teardown(kvm);
354 }
355
356 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
357 {
358 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
359
360 INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
361 }
362
363
364 static void __kvm_vgic_destroy(struct kvm *kvm)
365 {
366 struct kvm_vcpu *vcpu;
367 int i;
368
369 vgic_debug_destroy(kvm);
370
371 kvm_vgic_dist_destroy(kvm);
372
373 kvm_for_each_vcpu(i, vcpu, kvm)
374 kvm_vgic_vcpu_destroy(vcpu);
375 }
376
377 void kvm_vgic_destroy(struct kvm *kvm)
378 {
379 mutex_lock(&kvm->lock);
380 __kvm_vgic_destroy(kvm);
381 mutex_unlock(&kvm->lock);
382 }
383
384
385
386
387
388
389
390 int vgic_lazy_init(struct kvm *kvm)
391 {
392 int ret = 0;
393
394 if (unlikely(!vgic_initialized(kvm))) {
395
396
397
398
399
400
401 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
402 return -EBUSY;
403
404 mutex_lock(&kvm->lock);
405 ret = vgic_init(kvm);
406 mutex_unlock(&kvm->lock);
407 }
408
409 return ret;
410 }
411
412
413
414
415
416
417
418
419
420
421
422 int kvm_vgic_map_resources(struct kvm *kvm)
423 {
424 struct vgic_dist *dist = &kvm->arch.vgic;
425 int ret = 0;
426
427 mutex_lock(&kvm->lock);
428 if (!irqchip_in_kernel(kvm))
429 goto out;
430
431 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
432 ret = vgic_v2_map_resources(kvm);
433 else
434 ret = vgic_v3_map_resources(kvm);
435
436 if (ret)
437 __kvm_vgic_destroy(kvm);
438
439 out:
440 mutex_unlock(&kvm->lock);
441 return ret;
442 }
443
444
445
446 static int vgic_init_cpu_starting(unsigned int cpu)
447 {
448 enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0);
449 return 0;
450 }
451
452
453 static int vgic_init_cpu_dying(unsigned int cpu)
454 {
455 disable_percpu_irq(kvm_vgic_global_state.maint_irq);
456 return 0;
457 }
458
459 static irqreturn_t vgic_maintenance_handler(int irq, void *data)
460 {
461
462
463
464
465
466
467 return IRQ_HANDLED;
468 }
469
470
471
472
473
474
475 void kvm_vgic_init_cpu_hardware(void)
476 {
477 BUG_ON(preemptible());
478
479
480
481
482
483 if (kvm_vgic_global_state.type == VGIC_V2)
484 vgic_v2_init_lrs();
485 else
486 kvm_call_hyp(__vgic_v3_init_lrs);
487 }
488
489
490
491
492
493
494
495 int kvm_vgic_hyp_init(void)
496 {
497 const struct gic_kvm_info *gic_kvm_info;
498 int ret;
499
500 gic_kvm_info = gic_get_kvm_info();
501 if (!gic_kvm_info)
502 return -ENODEV;
503
504 if (!gic_kvm_info->maint_irq) {
505 kvm_err("No vgic maintenance irq\n");
506 return -ENXIO;
507 }
508
509 switch (gic_kvm_info->type) {
510 case GIC_V2:
511 ret = vgic_v2_probe(gic_kvm_info);
512 break;
513 case GIC_V3:
514 ret = vgic_v3_probe(gic_kvm_info);
515 if (!ret) {
516 static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
517 kvm_info("GIC system register CPU interface enabled\n");
518 }
519 break;
520 default:
521 ret = -ENODEV;
522 }
523
524 if (ret)
525 return ret;
526
527 kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq;
528 ret = request_percpu_irq(kvm_vgic_global_state.maint_irq,
529 vgic_maintenance_handler,
530 "vgic", kvm_get_running_vcpus());
531 if (ret) {
532 kvm_err("Cannot register interrupt %d\n",
533 kvm_vgic_global_state.maint_irq);
534 return ret;
535 }
536
537 ret = cpuhp_setup_state(CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING,
538 "kvm/arm/vgic:starting",
539 vgic_init_cpu_starting, vgic_init_cpu_dying);
540 if (ret) {
541 kvm_err("Cannot register vgic CPU notifier\n");
542 goto out_free_irq;
543 }
544
545 kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq);
546 return 0;
547
548 out_free_irq:
549 free_percpu_irq(kvm_vgic_global_state.maint_irq,
550 kvm_get_running_vcpus());
551 return ret;
552 }