1/* 2 * GICv3 distributor and redistributor emulation 3 * 4 * GICv3 emulation is currently only supported on a GICv3 host (because 5 * we rely on the hardware's CPU interface virtualization support), but 6 * supports both hardware with or without the optional GICv2 backwards 7 * compatibility features. 8 * 9 * Limitations of the emulation: 10 * (RAZ/WI: read as zero, write ignore, RAO/WI: read as one, write ignore) 11 * - We do not support LPIs (yet). TYPER.LPIS is reported as 0 and is RAZ/WI. 12 * - We do not support the message based interrupts (MBIs) triggered by 13 * writes to the GICD_{SET,CLR}SPI_* registers. TYPER.MBIS is reported as 0. 14 * - We do not support the (optional) backwards compatibility feature. 15 * GICD_CTLR.ARE resets to 1 and is RAO/WI. If the _host_ GIC supports 16 * the compatiblity feature, you can use a GICv2 in the guest, though. 17 * - We only support a single security state. GICD_CTLR.DS is 1 and is RAO/WI. 18 * - Priorities are not emulated (same as the GICv2 emulation). Linux 19 * as a guest is fine with this, because it does not use priorities. 20 * - We only support Group1 interrupts. Again Linux uses only those. 21 * 22 * Copyright (C) 2014 ARM Ltd. 23 * Author: Andre Przywara <andre.przywara@arm.com> 24 * 25 * This program is free software; you can redistribute it and/or modify 26 * it under the terms of the GNU General Public License version 2 as 27 * published by the Free Software Foundation. 28 * 29 * This program is distributed in the hope that it will be useful, 30 * but WITHOUT ANY WARRANTY; without even the implied warranty of 31 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 32 * GNU General Public License for more details. 33 * 34 * You should have received a copy of the GNU General Public License 35 * along with this program. If not, see <http://www.gnu.org/licenses/>. 36 */ 37 38#include <linux/cpu.h> 39#include <linux/kvm.h> 40#include <linux/kvm_host.h> 41#include <linux/interrupt.h> 42 43#include <linux/irqchip/arm-gic-v3.h> 44#include <kvm/arm_vgic.h> 45 46#include <asm/kvm_emulate.h> 47#include <asm/kvm_arm.h> 48#include <asm/kvm_mmu.h> 49 50#include "vgic.h" 51 52static bool handle_mmio_rao_wi(struct kvm_vcpu *vcpu, 53 struct kvm_exit_mmio *mmio, phys_addr_t offset) 54{ 55 u32 reg = 0xffffffff; 56 57 vgic_reg_access(mmio, ®, offset, 58 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED); 59 60 return false; 61} 62 63static bool handle_mmio_ctlr(struct kvm_vcpu *vcpu, 64 struct kvm_exit_mmio *mmio, phys_addr_t offset) 65{ 66 u32 reg = 0; 67 68 /* 69 * Force ARE and DS to 1, the guest cannot change this. 70 * For the time being we only support Group1 interrupts. 71 */ 72 if (vcpu->kvm->arch.vgic.enabled) 73 reg = GICD_CTLR_ENABLE_SS_G1; 74 reg |= GICD_CTLR_ARE_NS | GICD_CTLR_DS; 75 76 vgic_reg_access(mmio, ®, offset, 77 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE); 78 if (mmio->is_write) { 79 vcpu->kvm->arch.vgic.enabled = !!(reg & GICD_CTLR_ENABLE_SS_G1); 80 vgic_update_state(vcpu->kvm); 81 return true; 82 } 83 return false; 84} 85 86/* 87 * As this implementation does not provide compatibility 88 * with GICv2 (ARE==1), we report zero CPUs in bits [5..7]. 89 * Also LPIs and MBIs are not supported, so we set the respective bits to 0. 90 * Also we report at most 2**10=1024 interrupt IDs (to match 1024 SPIs). 91 */ 92#define INTERRUPT_ID_BITS 10 93static bool handle_mmio_typer(struct kvm_vcpu *vcpu, 94 struct kvm_exit_mmio *mmio, phys_addr_t offset) 95{ 96 u32 reg; 97 98 reg = (min(vcpu->kvm->arch.vgic.nr_irqs, 1024) >> 5) - 1; 99 100 reg |= (INTERRUPT_ID_BITS - 1) << 19; 101 102 vgic_reg_access(mmio, ®, offset, 103 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED); 104 105 return false; 106} 107 108static bool handle_mmio_iidr(struct kvm_vcpu *vcpu, 109 struct kvm_exit_mmio *mmio, phys_addr_t offset) 110{ 111 u32 reg; 112 113 reg = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); 114 vgic_reg_access(mmio, ®, offset, 115 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED); 116 117 return false; 118} 119 120static bool handle_mmio_set_enable_reg_dist(struct kvm_vcpu *vcpu, 121 struct kvm_exit_mmio *mmio, 122 phys_addr_t offset) 123{ 124 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8)) 125 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset, 126 vcpu->vcpu_id, 127 ACCESS_WRITE_SETBIT); 128 129 vgic_reg_access(mmio, NULL, offset, 130 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 131 return false; 132} 133 134static bool handle_mmio_clear_enable_reg_dist(struct kvm_vcpu *vcpu, 135 struct kvm_exit_mmio *mmio, 136 phys_addr_t offset) 137{ 138 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8)) 139 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset, 140 vcpu->vcpu_id, 141 ACCESS_WRITE_CLEARBIT); 142 143 vgic_reg_access(mmio, NULL, offset, 144 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 145 return false; 146} 147 148static bool handle_mmio_set_pending_reg_dist(struct kvm_vcpu *vcpu, 149 struct kvm_exit_mmio *mmio, 150 phys_addr_t offset) 151{ 152 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8)) 153 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset, 154 vcpu->vcpu_id); 155 156 vgic_reg_access(mmio, NULL, offset, 157 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 158 return false; 159} 160 161static bool handle_mmio_clear_pending_reg_dist(struct kvm_vcpu *vcpu, 162 struct kvm_exit_mmio *mmio, 163 phys_addr_t offset) 164{ 165 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8)) 166 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset, 167 vcpu->vcpu_id); 168 169 vgic_reg_access(mmio, NULL, offset, 170 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 171 return false; 172} 173 174static bool handle_mmio_set_active_reg_dist(struct kvm_vcpu *vcpu, 175 struct kvm_exit_mmio *mmio, 176 phys_addr_t offset) 177{ 178 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8)) 179 return vgic_handle_set_active_reg(vcpu->kvm, mmio, offset, 180 vcpu->vcpu_id); 181 182 vgic_reg_access(mmio, NULL, offset, 183 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 184 return false; 185} 186 187static bool handle_mmio_clear_active_reg_dist(struct kvm_vcpu *vcpu, 188 struct kvm_exit_mmio *mmio, 189 phys_addr_t offset) 190{ 191 if (likely(offset >= VGIC_NR_PRIVATE_IRQS / 8)) 192 return vgic_handle_clear_active_reg(vcpu->kvm, mmio, offset, 193 vcpu->vcpu_id); 194 195 vgic_reg_access(mmio, NULL, offset, 196 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 197 return false; 198} 199 200static bool handle_mmio_priority_reg_dist(struct kvm_vcpu *vcpu, 201 struct kvm_exit_mmio *mmio, 202 phys_addr_t offset) 203{ 204 u32 *reg; 205 206 if (unlikely(offset < VGIC_NR_PRIVATE_IRQS)) { 207 vgic_reg_access(mmio, NULL, offset, 208 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 209 return false; 210 } 211 212 reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority, 213 vcpu->vcpu_id, offset); 214 vgic_reg_access(mmio, reg, offset, 215 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE); 216 return false; 217} 218 219static bool handle_mmio_cfg_reg_dist(struct kvm_vcpu *vcpu, 220 struct kvm_exit_mmio *mmio, 221 phys_addr_t offset) 222{ 223 u32 *reg; 224 225 if (unlikely(offset < VGIC_NR_PRIVATE_IRQS / 4)) { 226 vgic_reg_access(mmio, NULL, offset, 227 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 228 return false; 229 } 230 231 reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg, 232 vcpu->vcpu_id, offset >> 1); 233 234 return vgic_handle_cfg_reg(reg, mmio, offset); 235} 236 237/* 238 * We use a compressed version of the MPIDR (all 32 bits in one 32-bit word) 239 * when we store the target MPIDR written by the guest. 240 */ 241static u32 compress_mpidr(unsigned long mpidr) 242{ 243 u32 ret; 244 245 ret = MPIDR_AFFINITY_LEVEL(mpidr, 0); 246 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8; 247 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16; 248 ret |= MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24; 249 250 return ret; 251} 252 253static unsigned long uncompress_mpidr(u32 value) 254{ 255 unsigned long mpidr; 256 257 mpidr = ((value >> 0) & 0xFF) << MPIDR_LEVEL_SHIFT(0); 258 mpidr |= ((value >> 8) & 0xFF) << MPIDR_LEVEL_SHIFT(1); 259 mpidr |= ((value >> 16) & 0xFF) << MPIDR_LEVEL_SHIFT(2); 260 mpidr |= (u64)((value >> 24) & 0xFF) << MPIDR_LEVEL_SHIFT(3); 261 262 return mpidr; 263} 264 265/* 266 * Lookup the given MPIDR value to get the vcpu_id (if there is one) 267 * and store that in the irq_spi_cpu[] array. 268 * This limits the number of VCPUs to 255 for now, extending the data 269 * type (or storing kvm_vcpu pointers) should lift the limit. 270 * Store the original MPIDR value in an extra array to support read-as-written. 271 * Unallocated MPIDRs are translated to a special value and caught 272 * before any array accesses. 273 */ 274static bool handle_mmio_route_reg(struct kvm_vcpu *vcpu, 275 struct kvm_exit_mmio *mmio, 276 phys_addr_t offset) 277{ 278 struct kvm *kvm = vcpu->kvm; 279 struct vgic_dist *dist = &kvm->arch.vgic; 280 int spi; 281 u32 reg; 282 int vcpu_id; 283 unsigned long *bmap, mpidr; 284 285 /* 286 * The upper 32 bits of each 64 bit register are zero, 287 * as we don't support Aff3. 288 */ 289 if ((offset & 4)) { 290 vgic_reg_access(mmio, NULL, offset, 291 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 292 return false; 293 } 294 295 /* This region only covers SPIs, so no handling of private IRQs here. */ 296 spi = offset / 8; 297 298 /* get the stored MPIDR for this IRQ */ 299 mpidr = uncompress_mpidr(dist->irq_spi_mpidr[spi]); 300 reg = mpidr; 301 302 vgic_reg_access(mmio, ®, offset, 303 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE); 304 305 if (!mmio->is_write) 306 return false; 307 308 /* 309 * Now clear the currently assigned vCPU from the map, making room 310 * for the new one to be written below 311 */ 312 vcpu = kvm_mpidr_to_vcpu(kvm, mpidr); 313 if (likely(vcpu)) { 314 vcpu_id = vcpu->vcpu_id; 315 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]); 316 __clear_bit(spi, bmap); 317 } 318 319 dist->irq_spi_mpidr[spi] = compress_mpidr(reg); 320 vcpu = kvm_mpidr_to_vcpu(kvm, reg & MPIDR_HWID_BITMASK); 321 322 /* 323 * The spec says that non-existent MPIDR values should not be 324 * forwarded to any existent (v)CPU, but should be able to become 325 * pending anyway. We simply keep the irq_spi_target[] array empty, so 326 * the interrupt will never be injected. 327 * irq_spi_cpu[irq] gets a magic value in this case. 328 */ 329 if (likely(vcpu)) { 330 vcpu_id = vcpu->vcpu_id; 331 dist->irq_spi_cpu[spi] = vcpu_id; 332 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]); 333 __set_bit(spi, bmap); 334 } else { 335 dist->irq_spi_cpu[spi] = VCPU_NOT_ALLOCATED; 336 } 337 338 vgic_update_state(kvm); 339 340 return true; 341} 342 343/* 344 * We should be careful about promising too much when a guest reads 345 * this register. Don't claim to be like any hardware implementation, 346 * but just report the GIC as version 3 - which is what a Linux guest 347 * would check. 348 */ 349static bool handle_mmio_idregs(struct kvm_vcpu *vcpu, 350 struct kvm_exit_mmio *mmio, 351 phys_addr_t offset) 352{ 353 u32 reg = 0; 354 355 switch (offset + GICD_IDREGS) { 356 case GICD_PIDR2: 357 reg = 0x3b; 358 break; 359 } 360 361 vgic_reg_access(mmio, ®, offset, 362 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED); 363 364 return false; 365} 366 367static const struct vgic_io_range vgic_v3_dist_ranges[] = { 368 { 369 .base = GICD_CTLR, 370 .len = 0x04, 371 .bits_per_irq = 0, 372 .handle_mmio = handle_mmio_ctlr, 373 }, 374 { 375 .base = GICD_TYPER, 376 .len = 0x04, 377 .bits_per_irq = 0, 378 .handle_mmio = handle_mmio_typer, 379 }, 380 { 381 .base = GICD_IIDR, 382 .len = 0x04, 383 .bits_per_irq = 0, 384 .handle_mmio = handle_mmio_iidr, 385 }, 386 { 387 /* this register is optional, it is RAZ/WI if not implemented */ 388 .base = GICD_STATUSR, 389 .len = 0x04, 390 .bits_per_irq = 0, 391 .handle_mmio = handle_mmio_raz_wi, 392 }, 393 { 394 /* this write only register is WI when TYPER.MBIS=0 */ 395 .base = GICD_SETSPI_NSR, 396 .len = 0x04, 397 .bits_per_irq = 0, 398 .handle_mmio = handle_mmio_raz_wi, 399 }, 400 { 401 /* this write only register is WI when TYPER.MBIS=0 */ 402 .base = GICD_CLRSPI_NSR, 403 .len = 0x04, 404 .bits_per_irq = 0, 405 .handle_mmio = handle_mmio_raz_wi, 406 }, 407 { 408 /* this is RAZ/WI when DS=1 */ 409 .base = GICD_SETSPI_SR, 410 .len = 0x04, 411 .bits_per_irq = 0, 412 .handle_mmio = handle_mmio_raz_wi, 413 }, 414 { 415 /* this is RAZ/WI when DS=1 */ 416 .base = GICD_CLRSPI_SR, 417 .len = 0x04, 418 .bits_per_irq = 0, 419 .handle_mmio = handle_mmio_raz_wi, 420 }, 421 { 422 .base = GICD_IGROUPR, 423 .len = 0x80, 424 .bits_per_irq = 1, 425 .handle_mmio = handle_mmio_rao_wi, 426 }, 427 { 428 .base = GICD_ISENABLER, 429 .len = 0x80, 430 .bits_per_irq = 1, 431 .handle_mmio = handle_mmio_set_enable_reg_dist, 432 }, 433 { 434 .base = GICD_ICENABLER, 435 .len = 0x80, 436 .bits_per_irq = 1, 437 .handle_mmio = handle_mmio_clear_enable_reg_dist, 438 }, 439 { 440 .base = GICD_ISPENDR, 441 .len = 0x80, 442 .bits_per_irq = 1, 443 .handle_mmio = handle_mmio_set_pending_reg_dist, 444 }, 445 { 446 .base = GICD_ICPENDR, 447 .len = 0x80, 448 .bits_per_irq = 1, 449 .handle_mmio = handle_mmio_clear_pending_reg_dist, 450 }, 451 { 452 .base = GICD_ISACTIVER, 453 .len = 0x80, 454 .bits_per_irq = 1, 455 .handle_mmio = handle_mmio_set_active_reg_dist, 456 }, 457 { 458 .base = GICD_ICACTIVER, 459 .len = 0x80, 460 .bits_per_irq = 1, 461 .handle_mmio = handle_mmio_clear_active_reg_dist, 462 }, 463 { 464 .base = GICD_IPRIORITYR, 465 .len = 0x400, 466 .bits_per_irq = 8, 467 .handle_mmio = handle_mmio_priority_reg_dist, 468 }, 469 { 470 /* TARGETSRn is RES0 when ARE=1 */ 471 .base = GICD_ITARGETSR, 472 .len = 0x400, 473 .bits_per_irq = 8, 474 .handle_mmio = handle_mmio_raz_wi, 475 }, 476 { 477 .base = GICD_ICFGR, 478 .len = 0x100, 479 .bits_per_irq = 2, 480 .handle_mmio = handle_mmio_cfg_reg_dist, 481 }, 482 { 483 /* this is RAZ/WI when DS=1 */ 484 .base = GICD_IGRPMODR, 485 .len = 0x80, 486 .bits_per_irq = 1, 487 .handle_mmio = handle_mmio_raz_wi, 488 }, 489 { 490 /* this is RAZ/WI when DS=1 */ 491 .base = GICD_NSACR, 492 .len = 0x100, 493 .bits_per_irq = 2, 494 .handle_mmio = handle_mmio_raz_wi, 495 }, 496 { 497 /* this is RAZ/WI when ARE=1 */ 498 .base = GICD_SGIR, 499 .len = 0x04, 500 .handle_mmio = handle_mmio_raz_wi, 501 }, 502 { 503 /* this is RAZ/WI when ARE=1 */ 504 .base = GICD_CPENDSGIR, 505 .len = 0x10, 506 .handle_mmio = handle_mmio_raz_wi, 507 }, 508 { 509 /* this is RAZ/WI when ARE=1 */ 510 .base = GICD_SPENDSGIR, 511 .len = 0x10, 512 .handle_mmio = handle_mmio_raz_wi, 513 }, 514 { 515 .base = GICD_IROUTER + 0x100, 516 .len = 0x1ee0, 517 .bits_per_irq = 64, 518 .handle_mmio = handle_mmio_route_reg, 519 }, 520 { 521 .base = GICD_IDREGS, 522 .len = 0x30, 523 .bits_per_irq = 0, 524 .handle_mmio = handle_mmio_idregs, 525 }, 526 {}, 527}; 528 529static bool handle_mmio_ctlr_redist(struct kvm_vcpu *vcpu, 530 struct kvm_exit_mmio *mmio, 531 phys_addr_t offset) 532{ 533 /* since we don't support LPIs, this register is zero for now */ 534 vgic_reg_access(mmio, NULL, offset, 535 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED); 536 return false; 537} 538 539static bool handle_mmio_typer_redist(struct kvm_vcpu *vcpu, 540 struct kvm_exit_mmio *mmio, 541 phys_addr_t offset) 542{ 543 u32 reg; 544 u64 mpidr; 545 struct kvm_vcpu *redist_vcpu = mmio->private; 546 int target_vcpu_id = redist_vcpu->vcpu_id; 547 548 /* the upper 32 bits contain the affinity value */ 549 if ((offset & ~3) == 4) { 550 mpidr = kvm_vcpu_get_mpidr_aff(redist_vcpu); 551 reg = compress_mpidr(mpidr); 552 553 vgic_reg_access(mmio, ®, offset, 554 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED); 555 return false; 556 } 557 558 reg = redist_vcpu->vcpu_id << 8; 559 if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1) 560 reg |= GICR_TYPER_LAST; 561 vgic_reg_access(mmio, ®, offset, 562 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED); 563 return false; 564} 565 566static bool handle_mmio_set_enable_reg_redist(struct kvm_vcpu *vcpu, 567 struct kvm_exit_mmio *mmio, 568 phys_addr_t offset) 569{ 570 struct kvm_vcpu *redist_vcpu = mmio->private; 571 572 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset, 573 redist_vcpu->vcpu_id, 574 ACCESS_WRITE_SETBIT); 575} 576 577static bool handle_mmio_clear_enable_reg_redist(struct kvm_vcpu *vcpu, 578 struct kvm_exit_mmio *mmio, 579 phys_addr_t offset) 580{ 581 struct kvm_vcpu *redist_vcpu = mmio->private; 582 583 return vgic_handle_enable_reg(vcpu->kvm, mmio, offset, 584 redist_vcpu->vcpu_id, 585 ACCESS_WRITE_CLEARBIT); 586} 587 588static bool handle_mmio_set_active_reg_redist(struct kvm_vcpu *vcpu, 589 struct kvm_exit_mmio *mmio, 590 phys_addr_t offset) 591{ 592 struct kvm_vcpu *redist_vcpu = mmio->private; 593 594 return vgic_handle_set_active_reg(vcpu->kvm, mmio, offset, 595 redist_vcpu->vcpu_id); 596} 597 598static bool handle_mmio_clear_active_reg_redist(struct kvm_vcpu *vcpu, 599 struct kvm_exit_mmio *mmio, 600 phys_addr_t offset) 601{ 602 struct kvm_vcpu *redist_vcpu = mmio->private; 603 604 return vgic_handle_clear_active_reg(vcpu->kvm, mmio, offset, 605 redist_vcpu->vcpu_id); 606} 607 608static bool handle_mmio_set_pending_reg_redist(struct kvm_vcpu *vcpu, 609 struct kvm_exit_mmio *mmio, 610 phys_addr_t offset) 611{ 612 struct kvm_vcpu *redist_vcpu = mmio->private; 613 614 return vgic_handle_set_pending_reg(vcpu->kvm, mmio, offset, 615 redist_vcpu->vcpu_id); 616} 617 618static bool handle_mmio_clear_pending_reg_redist(struct kvm_vcpu *vcpu, 619 struct kvm_exit_mmio *mmio, 620 phys_addr_t offset) 621{ 622 struct kvm_vcpu *redist_vcpu = mmio->private; 623 624 return vgic_handle_clear_pending_reg(vcpu->kvm, mmio, offset, 625 redist_vcpu->vcpu_id); 626} 627 628static bool handle_mmio_priority_reg_redist(struct kvm_vcpu *vcpu, 629 struct kvm_exit_mmio *mmio, 630 phys_addr_t offset) 631{ 632 struct kvm_vcpu *redist_vcpu = mmio->private; 633 u32 *reg; 634 635 reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority, 636 redist_vcpu->vcpu_id, offset); 637 vgic_reg_access(mmio, reg, offset, 638 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE); 639 return false; 640} 641 642static bool handle_mmio_cfg_reg_redist(struct kvm_vcpu *vcpu, 643 struct kvm_exit_mmio *mmio, 644 phys_addr_t offset) 645{ 646 struct kvm_vcpu *redist_vcpu = mmio->private; 647 648 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg, 649 redist_vcpu->vcpu_id, offset >> 1); 650 651 return vgic_handle_cfg_reg(reg, mmio, offset); 652} 653 654#define SGI_base(x) ((x) + SZ_64K) 655 656static const struct vgic_io_range vgic_redist_ranges[] = { 657 { 658 .base = GICR_CTLR, 659 .len = 0x04, 660 .bits_per_irq = 0, 661 .handle_mmio = handle_mmio_ctlr_redist, 662 }, 663 { 664 .base = GICR_TYPER, 665 .len = 0x08, 666 .bits_per_irq = 0, 667 .handle_mmio = handle_mmio_typer_redist, 668 }, 669 { 670 .base = GICR_IIDR, 671 .len = 0x04, 672 .bits_per_irq = 0, 673 .handle_mmio = handle_mmio_iidr, 674 }, 675 { 676 .base = GICR_WAKER, 677 .len = 0x04, 678 .bits_per_irq = 0, 679 .handle_mmio = handle_mmio_raz_wi, 680 }, 681 { 682 .base = GICR_IDREGS, 683 .len = 0x30, 684 .bits_per_irq = 0, 685 .handle_mmio = handle_mmio_idregs, 686 }, 687 { 688 .base = SGI_base(GICR_IGROUPR0), 689 .len = 0x04, 690 .bits_per_irq = 1, 691 .handle_mmio = handle_mmio_rao_wi, 692 }, 693 { 694 .base = SGI_base(GICR_ISENABLER0), 695 .len = 0x04, 696 .bits_per_irq = 1, 697 .handle_mmio = handle_mmio_set_enable_reg_redist, 698 }, 699 { 700 .base = SGI_base(GICR_ICENABLER0), 701 .len = 0x04, 702 .bits_per_irq = 1, 703 .handle_mmio = handle_mmio_clear_enable_reg_redist, 704 }, 705 { 706 .base = SGI_base(GICR_ISPENDR0), 707 .len = 0x04, 708 .bits_per_irq = 1, 709 .handle_mmio = handle_mmio_set_pending_reg_redist, 710 }, 711 { 712 .base = SGI_base(GICR_ICPENDR0), 713 .len = 0x04, 714 .bits_per_irq = 1, 715 .handle_mmio = handle_mmio_clear_pending_reg_redist, 716 }, 717 { 718 .base = SGI_base(GICR_ISACTIVER0), 719 .len = 0x04, 720 .bits_per_irq = 1, 721 .handle_mmio = handle_mmio_set_active_reg_redist, 722 }, 723 { 724 .base = SGI_base(GICR_ICACTIVER0), 725 .len = 0x04, 726 .bits_per_irq = 1, 727 .handle_mmio = handle_mmio_clear_active_reg_redist, 728 }, 729 { 730 .base = SGI_base(GICR_IPRIORITYR0), 731 .len = 0x20, 732 .bits_per_irq = 8, 733 .handle_mmio = handle_mmio_priority_reg_redist, 734 }, 735 { 736 .base = SGI_base(GICR_ICFGR0), 737 .len = 0x08, 738 .bits_per_irq = 2, 739 .handle_mmio = handle_mmio_cfg_reg_redist, 740 }, 741 { 742 .base = SGI_base(GICR_IGRPMODR0), 743 .len = 0x04, 744 .bits_per_irq = 1, 745 .handle_mmio = handle_mmio_raz_wi, 746 }, 747 { 748 .base = SGI_base(GICR_NSACR), 749 .len = 0x04, 750 .handle_mmio = handle_mmio_raz_wi, 751 }, 752 {}, 753}; 754 755static bool vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, int irq) 756{ 757 if (vgic_queue_irq(vcpu, 0, irq)) { 758 vgic_dist_irq_clear_pending(vcpu, irq); 759 vgic_cpu_irq_clear(vcpu, irq); 760 return true; 761 } 762 763 return false; 764} 765 766static int vgic_v3_map_resources(struct kvm *kvm, 767 const struct vgic_params *params) 768{ 769 int ret = 0; 770 struct vgic_dist *dist = &kvm->arch.vgic; 771 gpa_t rdbase = dist->vgic_redist_base; 772 struct vgic_io_device *iodevs = NULL; 773 int i; 774 775 if (!irqchip_in_kernel(kvm)) 776 return 0; 777 778 mutex_lock(&kvm->lock); 779 780 if (vgic_ready(kvm)) 781 goto out; 782 783 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) || 784 IS_VGIC_ADDR_UNDEF(dist->vgic_redist_base)) { 785 kvm_err("Need to set vgic distributor addresses first\n"); 786 ret = -ENXIO; 787 goto out; 788 } 789 790 /* 791 * For a VGICv3 we require the userland to explicitly initialize 792 * the VGIC before we need to use it. 793 */ 794 if (!vgic_initialized(kvm)) { 795 ret = -EBUSY; 796 goto out; 797 } 798 799 ret = vgic_register_kvm_io_dev(kvm, dist->vgic_dist_base, 800 GIC_V3_DIST_SIZE, vgic_v3_dist_ranges, 801 -1, &dist->dist_iodev); 802 if (ret) 803 goto out; 804 805 iodevs = kcalloc(dist->nr_cpus, sizeof(iodevs[0]), GFP_KERNEL); 806 if (!iodevs) { 807 ret = -ENOMEM; 808 goto out_unregister; 809 } 810 811 for (i = 0; i < dist->nr_cpus; i++) { 812 ret = vgic_register_kvm_io_dev(kvm, rdbase, 813 SZ_128K, vgic_redist_ranges, 814 i, &iodevs[i]); 815 if (ret) 816 goto out_unregister; 817 rdbase += GIC_V3_REDIST_SIZE; 818 } 819 820 dist->redist_iodevs = iodevs; 821 dist->ready = true; 822 goto out; 823 824out_unregister: 825 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &dist->dist_iodev.dev); 826 if (iodevs) { 827 for (i = 0; i < dist->nr_cpus; i++) { 828 if (iodevs[i].dev.ops) 829 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, 830 &iodevs[i].dev); 831 } 832 } 833 834out: 835 if (ret) 836 kvm_vgic_destroy(kvm); 837 mutex_unlock(&kvm->lock); 838 return ret; 839} 840 841static int vgic_v3_init_model(struct kvm *kvm) 842{ 843 int i; 844 u32 mpidr; 845 struct vgic_dist *dist = &kvm->arch.vgic; 846 int nr_spis = dist->nr_irqs - VGIC_NR_PRIVATE_IRQS; 847 848 dist->irq_spi_mpidr = kcalloc(nr_spis, sizeof(dist->irq_spi_mpidr[0]), 849 GFP_KERNEL); 850 851 if (!dist->irq_spi_mpidr) 852 return -ENOMEM; 853 854 /* Initialize the target VCPUs for each IRQ to VCPU 0 */ 855 mpidr = compress_mpidr(kvm_vcpu_get_mpidr_aff(kvm_get_vcpu(kvm, 0))); 856 for (i = VGIC_NR_PRIVATE_IRQS; i < dist->nr_irqs; i++) { 857 dist->irq_spi_cpu[i - VGIC_NR_PRIVATE_IRQS] = 0; 858 dist->irq_spi_mpidr[i - VGIC_NR_PRIVATE_IRQS] = mpidr; 859 vgic_bitmap_set_irq_val(dist->irq_spi_target, 0, i, 1); 860 } 861 862 return 0; 863} 864 865/* GICv3 does not keep track of SGI sources anymore. */ 866static void vgic_v3_add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source) 867{ 868} 869 870void vgic_v3_init_emulation(struct kvm *kvm) 871{ 872 struct vgic_dist *dist = &kvm->arch.vgic; 873 874 dist->vm_ops.queue_sgi = vgic_v3_queue_sgi; 875 dist->vm_ops.add_sgi_source = vgic_v3_add_sgi_source; 876 dist->vm_ops.init_model = vgic_v3_init_model; 877 dist->vm_ops.map_resources = vgic_v3_map_resources; 878 879 kvm->arch.max_vcpus = KVM_MAX_VCPUS; 880} 881 882/* 883 * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI 884 * generation register ICC_SGI1R_EL1) with a given VCPU. 885 * If the VCPU's MPIDR matches, return the level0 affinity, otherwise 886 * return -1. 887 */ 888static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu) 889{ 890 unsigned long affinity; 891 int level0; 892 893 /* 894 * Split the current VCPU's MPIDR into affinity level 0 and the 895 * rest as this is what we have to compare against. 896 */ 897 affinity = kvm_vcpu_get_mpidr_aff(vcpu); 898 level0 = MPIDR_AFFINITY_LEVEL(affinity, 0); 899 affinity &= ~MPIDR_LEVEL_MASK; 900 901 /* bail out if the upper three levels don't match */ 902 if (sgi_aff != affinity) 903 return -1; 904 905 /* Is this VCPU's bit set in the mask ? */ 906 if (!(sgi_cpu_mask & BIT(level0))) 907 return -1; 908 909 return level0; 910} 911 912#define SGI_AFFINITY_LEVEL(reg, level) \ 913 ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \ 914 >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level)) 915 916/** 917 * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs 918 * @vcpu: The VCPU requesting a SGI 919 * @reg: The value written into the ICC_SGI1R_EL1 register by that VCPU 920 * 921 * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register. 922 * This will trap in sys_regs.c and call this function. 923 * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the 924 * target processors as well as a bitmask of 16 Aff0 CPUs. 925 * If the interrupt routing mode bit is not set, we iterate over all VCPUs to 926 * check for matching ones. If this bit is set, we signal all, but not the 927 * calling VCPU. 928 */ 929void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg) 930{ 931 struct kvm *kvm = vcpu->kvm; 932 struct kvm_vcpu *c_vcpu; 933 struct vgic_dist *dist = &kvm->arch.vgic; 934 u16 target_cpus; 935 u64 mpidr; 936 int sgi, c; 937 int vcpu_id = vcpu->vcpu_id; 938 bool broadcast; 939 int updated = 0; 940 941 sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT; 942 broadcast = reg & BIT(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); 943 target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT; 944 mpidr = SGI_AFFINITY_LEVEL(reg, 3); 945 mpidr |= SGI_AFFINITY_LEVEL(reg, 2); 946 mpidr |= SGI_AFFINITY_LEVEL(reg, 1); 947 948 /* 949 * We take the dist lock here, because we come from the sysregs 950 * code path and not from the MMIO one (which already takes the lock). 951 */ 952 spin_lock(&dist->lock); 953 954 /* 955 * We iterate over all VCPUs to find the MPIDRs matching the request. 956 * If we have handled one CPU, we clear it's bit to detect early 957 * if we are already finished. This avoids iterating through all 958 * VCPUs when most of the times we just signal a single VCPU. 959 */ 960 kvm_for_each_vcpu(c, c_vcpu, kvm) { 961 962 /* Exit early if we have dealt with all requested CPUs */ 963 if (!broadcast && target_cpus == 0) 964 break; 965 966 /* Don't signal the calling VCPU */ 967 if (broadcast && c == vcpu_id) 968 continue; 969 970 if (!broadcast) { 971 int level0; 972 973 level0 = match_mpidr(mpidr, target_cpus, c_vcpu); 974 if (level0 == -1) 975 continue; 976 977 /* remove this matching VCPU from the mask */ 978 target_cpus &= ~BIT(level0); 979 } 980 981 /* Flag the SGI as pending */ 982 vgic_dist_irq_set_pending(c_vcpu, sgi); 983 updated = 1; 984 kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c); 985 } 986 if (updated) 987 vgic_update_state(vcpu->kvm); 988 spin_unlock(&dist->lock); 989 if (updated) 990 vgic_kick_vcpus(vcpu->kvm); 991} 992 993static int vgic_v3_create(struct kvm_device *dev, u32 type) 994{ 995 return kvm_vgic_create(dev->kvm, type); 996} 997 998static void vgic_v3_destroy(struct kvm_device *dev) 999{ 1000 kfree(dev); 1001} 1002 1003static int vgic_v3_set_attr(struct kvm_device *dev, 1004 struct kvm_device_attr *attr) 1005{ 1006 int ret; 1007 1008 ret = vgic_set_common_attr(dev, attr); 1009 if (ret != -ENXIO) 1010 return ret; 1011 1012 switch (attr->group) { 1013 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS: 1014 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: 1015 return -ENXIO; 1016 } 1017 1018 return -ENXIO; 1019} 1020 1021static int vgic_v3_get_attr(struct kvm_device *dev, 1022 struct kvm_device_attr *attr) 1023{ 1024 int ret; 1025 1026 ret = vgic_get_common_attr(dev, attr); 1027 if (ret != -ENXIO) 1028 return ret; 1029 1030 switch (attr->group) { 1031 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS: 1032 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: 1033 return -ENXIO; 1034 } 1035 1036 return -ENXIO; 1037} 1038 1039static int vgic_v3_has_attr(struct kvm_device *dev, 1040 struct kvm_device_attr *attr) 1041{ 1042 switch (attr->group) { 1043 case KVM_DEV_ARM_VGIC_GRP_ADDR: 1044 switch (attr->attr) { 1045 case KVM_VGIC_V2_ADDR_TYPE_DIST: 1046 case KVM_VGIC_V2_ADDR_TYPE_CPU: 1047 return -ENXIO; 1048 case KVM_VGIC_V3_ADDR_TYPE_DIST: 1049 case KVM_VGIC_V3_ADDR_TYPE_REDIST: 1050 return 0; 1051 } 1052 break; 1053 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS: 1054 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS: 1055 return -ENXIO; 1056 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: 1057 return 0; 1058 case KVM_DEV_ARM_VGIC_GRP_CTRL: 1059 switch (attr->attr) { 1060 case KVM_DEV_ARM_VGIC_CTRL_INIT: 1061 return 0; 1062 } 1063 } 1064 return -ENXIO; 1065} 1066 1067struct kvm_device_ops kvm_arm_vgic_v3_ops = { 1068 .name = "kvm-arm-vgic-v3", 1069 .create = vgic_v3_create, 1070 .destroy = vgic_v3_destroy, 1071 .set_attr = vgic_v3_set_attr, 1072 .get_attr = vgic_v3_get_attr, 1073 .has_attr = vgic_v3_has_attr, 1074}; 1075