121 	vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);  in add_sgi_source()
126 	return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);  in queue_sgi()
129 int kvm_vgic_map_resources(struct kvm *kvm)  in kvm_vgic_map_resources()  argument
131 	return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);  in kvm_vgic_map_resources()
276 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_edge()
285 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_enabled()
292 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_queued()
299 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_active()
306 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_set_queued()
313 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_clear_queued()
320 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_set_active()
327 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_clear_active()
334 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_get_level()
341 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_set_level()
348 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_clear_level()
355 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_soft_pend()
362 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_clear_soft_pend()
374 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_is_pending()
381 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_set_pending()
388 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_clear_pending()
485 bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,  in vgic_handle_enable_reg()  argument
490 	struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);  in vgic_handle_enable_reg()
492 	reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);  in vgic_handle_enable_reg()
500 		vgic_update_state(kvm);  in vgic_handle_enable_reg()
507 bool vgic_handle_set_pending_reg(struct kvm *kvm,  in vgic_handle_set_pending_reg()  argument
514 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_set_pending_reg()
537 		vgic_update_state(kvm);  in vgic_handle_set_pending_reg()
544 bool vgic_handle_clear_pending_reg(struct kvm *kvm,  in vgic_handle_clear_pending_reg()  argument
551 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_clear_pending_reg()
574 		vgic_update_state(kvm);  in vgic_handle_clear_pending_reg()
580 bool vgic_handle_set_active_reg(struct kvm *kvm,  in vgic_handle_set_active_reg()  argument
585 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_set_active_reg()
592 		vgic_update_state(kvm);  in vgic_handle_set_active_reg()
599 bool vgic_handle_clear_active_reg(struct kvm *kvm,  in vgic_handle_clear_active_reg()  argument
604 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_clear_active_reg()
611 		vgic_update_state(kvm);  in vgic_handle_clear_active_reg()
730 		vgic_update_state(vcpu->kvm);  in vgic_unqueue_irqs()
821 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_handle_mmio_access()
861 		vgic_kick_vcpus(vcpu->kvm);  in vgic_handle_mmio_access()
902 int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,  in vgic_register_kvm_io_dev()  argument
911 		vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);  in vgic_register_kvm_io_dev()
920 	mutex_lock(&kvm->slots_lock);  in vgic_register_kvm_io_dev()
922 	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,  in vgic_register_kvm_io_dev()
924 	mutex_unlock(&kvm->slots_lock);  in vgic_register_kvm_io_dev()
940 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in compute_active_for_cpu()
970 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in compute_pending_for_cpu()
1007 void vgic_update_state(struct kvm *kvm)  in vgic_update_state()  argument
1009 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_update_state()
1013 	kvm_for_each_vcpu(c, vcpu, kvm) {  in vgic_update_state()
1101 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in dist_active_irq()
1150 		vgic_update_state(vcpu->kvm);  in vgic_queue_irq_to_lr()
1188 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_queue_irq()
1252 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in __kvm_vgic_flush_hwstate()
1362 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_process_maintenance()
1363 	struct kvm *kvm = vcpu->kvm;  in vgic_process_maintenance()  local
1389 			kvm_notify_acked_irq(kvm, 0,  in vgic_process_maintenance()
1419 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_sync_hwirq()
1437 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in __kvm_vgic_sync_hwstate()
1463 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_flush_hwstate()
1465 	if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vgic_flush_hwstate()
1475 	if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vgic_sync_hwstate()
1483 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_vcpu_pending_irq()
1485 	if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vgic_vcpu_pending_irq()
1491 void vgic_kick_vcpus(struct kvm *kvm)  in vgic_kick_vcpus()  argument
1500 	kvm_for_each_vcpu(c, vcpu, kvm) {  in vgic_kick_vcpus()
1524 static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,  in vgic_update_irq_pending()  argument
1528 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_update_irq_pending()
1536 	if (irq_num >= min(kvm->arch.vgic.nr_irqs, 1020))  in vgic_update_irq_pending()
1541 	vcpu = kvm_get_vcpu(kvm, cpuid);  in vgic_update_irq_pending()
1557 		vcpu = kvm_get_vcpu(kvm, cpuid);  in vgic_update_irq_pending()
1607 		kvm_vcpu_kick(kvm_get_vcpu(kvm, cpuid));  in vgic_update_irq_pending()
1613 static int vgic_lazy_init(struct kvm *kvm)  in vgic_lazy_init()  argument
1617 	if (unlikely(!vgic_initialized(kvm))) {  in vgic_lazy_init()
1624 		if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)  in vgic_lazy_init()
1627 		mutex_lock(&kvm->lock);  in vgic_lazy_init()
1628 		ret = vgic_init(kvm);  in vgic_lazy_init()
1629 		mutex_unlock(&kvm->lock);  in vgic_lazy_init()
1650 int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,  in kvm_vgic_inject_irq()  argument
1656 	ret = vgic_lazy_init(kvm);  in kvm_vgic_inject_irq()
1660 	map = vgic_irq_map_search(kvm_get_vcpu(kvm, cpuid), irq_num);  in kvm_vgic_inject_irq()
1664 	return vgic_update_irq_pending(kvm, cpuid, NULL, irq_num, level);  in kvm_vgic_inject_irq()
1681 int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid,  in kvm_vgic_inject_mapped_irq()  argument
1686 	ret = vgic_lazy_init(kvm);  in kvm_vgic_inject_mapped_irq()
1690 	return vgic_update_irq_pending(kvm, cpuid, map, map->virt_irq, level);  in kvm_vgic_inject_mapped_irq()
1710 		return &vcpu->kvm->arch.vgic.irq_phys_map_list;  in vgic_get_irq_phys_map_list()
1729 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_map_phys_irq()
1823 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_unmap_phys_irq()
1847 static void vgic_destroy_irq_phys_map(struct kvm *kvm, struct list_head *root)  in vgic_destroy_irq_phys_map()  argument
1849 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_destroy_irq_phys_map()
1869 	vgic_destroy_irq_phys_map(vcpu->kvm, &vgic_cpu->irq_phys_map_list);  in kvm_vgic_vcpu_destroy()
1923 void kvm_vgic_destroy(struct kvm *kvm)  in kvm_vgic_destroy()  argument
1925 	struct vgic_dist *dist = &kvm->arch.vgic;  in kvm_vgic_destroy()
1929 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_vgic_destroy()
1949 	vgic_destroy_irq_phys_map(kvm, &dist->irq_phys_map_list);  in kvm_vgic_destroy()
1962 int vgic_init(struct kvm *kvm)  in vgic_init()  argument
1964 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_init()
1969 	if (vgic_initialized(kvm))  in vgic_init()
1972 	nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);  in vgic_init()
2021 	ret = kvm->arch.vgic.vm_ops.init_model(kvm);  in vgic_init()
2025 	kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {  in vgic_init()
2057 		kvm_vgic_destroy(kvm);  in vgic_init()
2062 static int init_vgic_model(struct kvm *kvm, int type)  in init_vgic_model()  argument
2066 		vgic_v2_init_emulation(kvm);  in init_vgic_model()
2070 		vgic_v3_init_emulation(kvm);  in init_vgic_model()
2077 	if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)  in init_vgic_model()
2088 void kvm_vgic_early_init(struct kvm *kvm)  in kvm_vgic_early_init()  argument
2090 	spin_lock_init(&kvm->arch.vgic.lock);  in kvm_vgic_early_init()
2091 	spin_lock_init(&kvm->arch.vgic.irq_phys_map_lock);  in kvm_vgic_early_init()
2092 	INIT_LIST_HEAD(&kvm->arch.vgic.irq_phys_map_list);  in kvm_vgic_early_init()
2095 int kvm_vgic_create(struct kvm *kvm, u32 type)  in kvm_vgic_create()  argument
2100 	mutex_lock(&kvm->lock);  in kvm_vgic_create()
2102 	if (irqchip_in_kernel(kvm)) {  in kvm_vgic_create()
2124 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_vgic_create()
2130 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_vgic_create()
2136 	ret = init_vgic_model(kvm, type);  in kvm_vgic_create()
2140 	kvm->arch.vgic.in_kernel = true;  in kvm_vgic_create()
2141 	kvm->arch.vgic.vgic_model = type;  in kvm_vgic_create()
2142 	kvm->arch.vgic.vctrl_base = vgic->vctrl_base;  in kvm_vgic_create()
2143 	kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;  in kvm_vgic_create()
2144 	kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;  in kvm_vgic_create()
2145 	kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;  in kvm_vgic_create()
2149 		vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);  in kvm_vgic_create()
2154 	mutex_unlock(&kvm->lock);  in kvm_vgic_create()
2158 static int vgic_ioaddr_overlap(struct kvm *kvm)  in vgic_ioaddr_overlap()  argument
2160 	phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;  in vgic_ioaddr_overlap()
2161 	phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;  in vgic_ioaddr_overlap()
2171 static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,  in vgic_ioaddr_assign()  argument
2188 	ret = vgic_ioaddr_overlap(kvm);  in vgic_ioaddr_assign()
2208 int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)  in kvm_vgic_addr()  argument
2211 	struct vgic_dist *vgic = &kvm->arch.vgic;  in kvm_vgic_addr()
2216 	mutex_lock(&kvm->lock);  in kvm_vgic_addr()
2258 			r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,  in kvm_vgic_addr()
2265 	mutex_unlock(&kvm->lock);  in kvm_vgic_addr()
2282 		r = kvm_vgic_addr(dev->kvm, type, &addr, true);  in vgic_set_common_attr()
2304 		mutex_lock(&dev->kvm->lock);  in vgic_set_common_attr()
2306 		if (vgic_ready(dev->kvm) || dev->kvm->arch.vgic.nr_irqs)  in vgic_set_common_attr()
2309 			dev->kvm->arch.vgic.nr_irqs = val;  in vgic_set_common_attr()
2311 		mutex_unlock(&dev->kvm->lock);  in vgic_set_common_attr()
2318 			r = vgic_init(dev->kvm);  in vgic_set_common_attr()
2338 		r = kvm_vgic_addr(dev->kvm, type, &addr, false);  in vgic_get_common_attr()
2349 		r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);  in vgic_get_common_attr()
2442 int kvm_irq_map_gsi(struct kvm *kvm,  in kvm_irq_map_gsi()  argument
2449 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_irq_map_chip_pin()  argument
2454 int kvm_set_irq(struct kvm *kvm, int irq_source_id,  in kvm_set_irq()  argument
2461 	BUG_ON(!vgic_initialized(kvm));  in kvm_set_irq()
2463 	return kvm_vgic_inject_irq(kvm, 0, spi, level);  in kvm_set_irq()
2468 		struct kvm *kvm, int irq_source_id,  in kvm_set_msi()  argument