1/* 2 * Copyright 2012 Michael Ellerman, IBM Corporation. 3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License, version 2, as 7 * published by the Free Software Foundation. 8 */ 9 10#include <linux/kernel.h> 11#include <linux/kvm_host.h> 12#include <linux/err.h> 13#include <linux/gfp.h> 14#include <linux/anon_inodes.h> 15#include <linux/spinlock.h> 16 17#include <asm/uaccess.h> 18#include <asm/kvm_book3s.h> 19#include <asm/kvm_ppc.h> 20#include <asm/hvcall.h> 21#include <asm/xics.h> 22#include <asm/debug.h> 23#include <asm/time.h> 24 25#include <linux/debugfs.h> 26#include <linux/seq_file.h> 27 28#include "book3s_xics.h" 29 30#if 1 31#define XICS_DBG(fmt...) do { } while (0) 32#else 33#define XICS_DBG(fmt...) trace_printk(fmt) 34#endif 35 36#define ENABLE_REALMODE true 37#define DEBUG_REALMODE false 38 39/* 40 * LOCKING 41 * ======= 42 * 43 * Each ICS has a spin lock protecting the information about the IRQ 44 * sources and avoiding simultaneous deliveries if the same interrupt. 45 * 46 * ICP operations are done via a single compare & swap transaction 47 * (most ICP state fits in the union kvmppc_icp_state) 48 */ 49 50/* 51 * TODO 52 * ==== 53 * 54 * - To speed up resends, keep a bitmap of "resend" set bits in the 55 * ICS 56 * 57 * - Speed up server# -> ICP lookup (array ? hash table ?) 58 * 59 * - Make ICS lockless as well, or at least a per-interrupt lock or hashed 60 * locks array to improve scalability 61 */ 62 63/* -- ICS routines -- */ 64 65static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 66 u32 new_irq); 67 68/* 69 * Return value ideally indicates how the interrupt was handled, but no 70 * callers look at it (given that we don't implement KVM_IRQ_LINE_STATUS), 71 * so just return 0. 72 */ 73static int ics_deliver_irq(struct kvmppc_xics *xics, u32 irq, u32 level) 74{ 75 struct ics_irq_state *state; 76 struct kvmppc_ics *ics; 77 u16 src; 78 79 XICS_DBG("ics deliver %#x (level: %d)\n", irq, level); 80 81 ics = kvmppc_xics_find_ics(xics, irq, &src); 82 if (!ics) { 83 XICS_DBG("ics_deliver_irq: IRQ 0x%06x not found !\n", irq); 84 return -EINVAL; 85 } 86 state = &ics->irq_state[src]; 87 if (!state->exists) 88 return -EINVAL; 89 90 /* 91 * We set state->asserted locklessly. This should be fine as 92 * we are the only setter, thus concurrent access is undefined 93 * to begin with. 94 */ 95 if (level == 1 || level == KVM_INTERRUPT_SET_LEVEL) 96 state->asserted = 1; 97 else if (level == 0 || level == KVM_INTERRUPT_UNSET) { 98 state->asserted = 0; 99 return 0; 100 } 101 102 /* Attempt delivery */ 103 icp_deliver_irq(xics, NULL, irq); 104 105 return 0; 106} 107 108static void ics_check_resend(struct kvmppc_xics *xics, struct kvmppc_ics *ics, 109 struct kvmppc_icp *icp) 110{ 111 int i; 112 113 unsigned long flags; 114 115 local_irq_save(flags); 116 arch_spin_lock(&ics->lock); 117 118 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 119 struct ics_irq_state *state = &ics->irq_state[i]; 120 121 if (!state->resend) 122 continue; 123 124 XICS_DBG("resend %#x prio %#x\n", state->number, 125 state->priority); 126 127 arch_spin_unlock(&ics->lock); 128 local_irq_restore(flags); 129 icp_deliver_irq(xics, icp, state->number); 130 local_irq_save(flags); 131 arch_spin_lock(&ics->lock); 132 } 133 134 arch_spin_unlock(&ics->lock); 135 local_irq_restore(flags); 136} 137 138static bool write_xive(struct kvmppc_xics *xics, struct kvmppc_ics *ics, 139 struct ics_irq_state *state, 140 u32 server, u32 priority, u32 saved_priority) 141{ 142 bool deliver; 143 unsigned long flags; 144 145 local_irq_save(flags); 146 arch_spin_lock(&ics->lock); 147 148 state->server = server; 149 state->priority = priority; 150 state->saved_priority = saved_priority; 151 deliver = false; 152 if ((state->masked_pending || state->resend) && priority != MASKED) { 153 state->masked_pending = 0; 154 deliver = true; 155 } 156 157 arch_spin_unlock(&ics->lock); 158 local_irq_restore(flags); 159 160 return deliver; 161} 162 163int kvmppc_xics_set_xive(struct kvm *kvm, u32 irq, u32 server, u32 priority) 164{ 165 struct kvmppc_xics *xics = kvm->arch.xics; 166 struct kvmppc_icp *icp; 167 struct kvmppc_ics *ics; 168 struct ics_irq_state *state; 169 u16 src; 170 171 if (!xics) 172 return -ENODEV; 173 174 ics = kvmppc_xics_find_ics(xics, irq, &src); 175 if (!ics) 176 return -EINVAL; 177 state = &ics->irq_state[src]; 178 179 icp = kvmppc_xics_find_server(kvm, server); 180 if (!icp) 181 return -EINVAL; 182 183 XICS_DBG("set_xive %#x server %#x prio %#x MP:%d RS:%d\n", 184 irq, server, priority, 185 state->masked_pending, state->resend); 186 187 if (write_xive(xics, ics, state, server, priority, priority)) 188 icp_deliver_irq(xics, icp, irq); 189 190 return 0; 191} 192 193int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority) 194{ 195 struct kvmppc_xics *xics = kvm->arch.xics; 196 struct kvmppc_ics *ics; 197 struct ics_irq_state *state; 198 u16 src; 199 unsigned long flags; 200 201 if (!xics) 202 return -ENODEV; 203 204 ics = kvmppc_xics_find_ics(xics, irq, &src); 205 if (!ics) 206 return -EINVAL; 207 state = &ics->irq_state[src]; 208 209 local_irq_save(flags); 210 arch_spin_lock(&ics->lock); 211 *server = state->server; 212 *priority = state->priority; 213 arch_spin_unlock(&ics->lock); 214 local_irq_restore(flags); 215 216 return 0; 217} 218 219int kvmppc_xics_int_on(struct kvm *kvm, u32 irq) 220{ 221 struct kvmppc_xics *xics = kvm->arch.xics; 222 struct kvmppc_icp *icp; 223 struct kvmppc_ics *ics; 224 struct ics_irq_state *state; 225 u16 src; 226 227 if (!xics) 228 return -ENODEV; 229 230 ics = kvmppc_xics_find_ics(xics, irq, &src); 231 if (!ics) 232 return -EINVAL; 233 state = &ics->irq_state[src]; 234 235 icp = kvmppc_xics_find_server(kvm, state->server); 236 if (!icp) 237 return -EINVAL; 238 239 if (write_xive(xics, ics, state, state->server, state->saved_priority, 240 state->saved_priority)) 241 icp_deliver_irq(xics, icp, irq); 242 243 return 0; 244} 245 246int kvmppc_xics_int_off(struct kvm *kvm, u32 irq) 247{ 248 struct kvmppc_xics *xics = kvm->arch.xics; 249 struct kvmppc_ics *ics; 250 struct ics_irq_state *state; 251 u16 src; 252 253 if (!xics) 254 return -ENODEV; 255 256 ics = kvmppc_xics_find_ics(xics, irq, &src); 257 if (!ics) 258 return -EINVAL; 259 state = &ics->irq_state[src]; 260 261 write_xive(xics, ics, state, state->server, MASKED, state->priority); 262 263 return 0; 264} 265 266/* -- ICP routines, including hcalls -- */ 267 268static inline bool icp_try_update(struct kvmppc_icp *icp, 269 union kvmppc_icp_state old, 270 union kvmppc_icp_state new, 271 bool change_self) 272{ 273 bool success; 274 275 /* Calculate new output value */ 276 new.out_ee = (new.xisr && (new.pending_pri < new.cppr)); 277 278 /* Attempt atomic update */ 279 success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw; 280 if (!success) 281 goto bail; 282 283 XICS_DBG("UPD [%04x] - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n", 284 icp->server_num, 285 old.cppr, old.mfrr, old.pending_pri, old.xisr, 286 old.need_resend, old.out_ee); 287 XICS_DBG("UPD - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n", 288 new.cppr, new.mfrr, new.pending_pri, new.xisr, 289 new.need_resend, new.out_ee); 290 /* 291 * Check for output state update 292 * 293 * Note that this is racy since another processor could be updating 294 * the state already. This is why we never clear the interrupt output 295 * here, we only ever set it. The clear only happens prior to doing 296 * an update and only by the processor itself. Currently we do it 297 * in Accept (H_XIRR) and Up_Cppr (H_XPPR). 298 * 299 * We also do not try to figure out whether the EE state has changed, 300 * we unconditionally set it if the new state calls for it. The reason 301 * for that is that we opportunistically remove the pending interrupt 302 * flag when raising CPPR, so we need to set it back here if an 303 * interrupt is still pending. 304 */ 305 if (new.out_ee) { 306 kvmppc_book3s_queue_irqprio(icp->vcpu, 307 BOOK3S_INTERRUPT_EXTERNAL_LEVEL); 308 if (!change_self) 309 kvmppc_fast_vcpu_kick(icp->vcpu); 310 } 311 bail: 312 return success; 313} 314 315static void icp_check_resend(struct kvmppc_xics *xics, 316 struct kvmppc_icp *icp) 317{ 318 u32 icsid; 319 320 /* Order this load with the test for need_resend in the caller */ 321 smp_rmb(); 322 for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) { 323 struct kvmppc_ics *ics = xics->ics[icsid]; 324 325 if (!test_and_clear_bit(icsid, icp->resend_map)) 326 continue; 327 if (!ics) 328 continue; 329 ics_check_resend(xics, ics, icp); 330 } 331} 332 333static bool icp_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority, 334 u32 *reject) 335{ 336 union kvmppc_icp_state old_state, new_state; 337 bool success; 338 339 XICS_DBG("try deliver %#x(P:%#x) to server %#x\n", irq, priority, 340 icp->server_num); 341 342 do { 343 old_state = new_state = READ_ONCE(icp->state); 344 345 *reject = 0; 346 347 /* See if we can deliver */ 348 success = new_state.cppr > priority && 349 new_state.mfrr > priority && 350 new_state.pending_pri > priority; 351 352 /* 353 * If we can, check for a rejection and perform the 354 * delivery 355 */ 356 if (success) { 357 *reject = new_state.xisr; 358 new_state.xisr = irq; 359 new_state.pending_pri = priority; 360 } else { 361 /* 362 * If we failed to deliver we set need_resend 363 * so a subsequent CPPR state change causes us 364 * to try a new delivery. 365 */ 366 new_state.need_resend = true; 367 } 368 369 } while (!icp_try_update(icp, old_state, new_state, false)); 370 371 return success; 372} 373 374static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 375 u32 new_irq) 376{ 377 struct ics_irq_state *state; 378 struct kvmppc_ics *ics; 379 u32 reject; 380 u16 src; 381 unsigned long flags; 382 383 /* 384 * This is used both for initial delivery of an interrupt and 385 * for subsequent rejection. 386 * 387 * Rejection can be racy vs. resends. We have evaluated the 388 * rejection in an atomic ICP transaction which is now complete, 389 * so potentially the ICP can already accept the interrupt again. 390 * 391 * So we need to retry the delivery. Essentially the reject path 392 * boils down to a failed delivery. Always. 393 * 394 * Now the interrupt could also have moved to a different target, 395 * thus we may need to re-do the ICP lookup as well 396 */ 397 398 again: 399 /* Get the ICS state and lock it */ 400 ics = kvmppc_xics_find_ics(xics, new_irq, &src); 401 if (!ics) { 402 XICS_DBG("icp_deliver_irq: IRQ 0x%06x not found !\n", new_irq); 403 return; 404 } 405 state = &ics->irq_state[src]; 406 407 /* Get a lock on the ICS */ 408 local_irq_save(flags); 409 arch_spin_lock(&ics->lock); 410 411 /* Get our server */ 412 if (!icp || state->server != icp->server_num) { 413 icp = kvmppc_xics_find_server(xics->kvm, state->server); 414 if (!icp) { 415 pr_warn("icp_deliver_irq: IRQ 0x%06x server 0x%x not found !\n", 416 new_irq, state->server); 417 goto out; 418 } 419 } 420 421 /* Clear the resend bit of that interrupt */ 422 state->resend = 0; 423 424 /* 425 * If masked, bail out 426 * 427 * Note: PAPR doesn't mention anything about masked pending 428 * when doing a resend, only when doing a delivery. 429 * 430 * However that would have the effect of losing a masked 431 * interrupt that was rejected and isn't consistent with 432 * the whole masked_pending business which is about not 433 * losing interrupts that occur while masked. 434 * 435 * I don't differenciate normal deliveries and resends, this 436 * implementation will differ from PAPR and not lose such 437 * interrupts. 438 */ 439 if (state->priority == MASKED) { 440 XICS_DBG("irq %#x masked pending\n", new_irq); 441 state->masked_pending = 1; 442 goto out; 443 } 444 445 /* 446 * Try the delivery, this will set the need_resend flag 447 * in the ICP as part of the atomic transaction if the 448 * delivery is not possible. 449 * 450 * Note that if successful, the new delivery might have itself 451 * rejected an interrupt that was "delivered" before we took the 452 * ics spin lock. 453 * 454 * In this case we do the whole sequence all over again for the 455 * new guy. We cannot assume that the rejected interrupt is less 456 * favored than the new one, and thus doesn't need to be delivered, 457 * because by the time we exit icp_try_to_deliver() the target 458 * processor may well have alrady consumed & completed it, and thus 459 * the rejected interrupt might actually be already acceptable. 460 */ 461 if (icp_try_to_deliver(icp, new_irq, state->priority, &reject)) { 462 /* 463 * Delivery was successful, did we reject somebody else ? 464 */ 465 if (reject && reject != XICS_IPI) { 466 arch_spin_unlock(&ics->lock); 467 local_irq_restore(flags); 468 new_irq = reject; 469 goto again; 470 } 471 } else { 472 /* 473 * We failed to deliver the interrupt we need to set the 474 * resend map bit and mark the ICS state as needing a resend 475 */ 476 set_bit(ics->icsid, icp->resend_map); 477 state->resend = 1; 478 479 /* 480 * If the need_resend flag got cleared in the ICP some time 481 * between icp_try_to_deliver() atomic update and now, then 482 * we know it might have missed the resend_map bit. So we 483 * retry 484 */ 485 smp_mb(); 486 if (!icp->state.need_resend) { 487 arch_spin_unlock(&ics->lock); 488 local_irq_restore(flags); 489 goto again; 490 } 491 } 492 out: 493 arch_spin_unlock(&ics->lock); 494 local_irq_restore(flags); 495} 496 497static void icp_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 498 u8 new_cppr) 499{ 500 union kvmppc_icp_state old_state, new_state; 501 bool resend; 502 503 /* 504 * This handles several related states in one operation: 505 * 506 * ICP State: Down_CPPR 507 * 508 * Load CPPR with new value and if the XISR is 0 509 * then check for resends: 510 * 511 * ICP State: Resend 512 * 513 * If MFRR is more favored than CPPR, check for IPIs 514 * and notify ICS of a potential resend. This is done 515 * asynchronously (when used in real mode, we will have 516 * to exit here). 517 * 518 * We do not handle the complete Check_IPI as documented 519 * here. In the PAPR, this state will be used for both 520 * Set_MFRR and Down_CPPR. However, we know that we aren't 521 * changing the MFRR state here so we don't need to handle 522 * the case of an MFRR causing a reject of a pending irq, 523 * this will have been handled when the MFRR was set in the 524 * first place. 525 * 526 * Thus we don't have to handle rejects, only resends. 527 * 528 * When implementing real mode for HV KVM, resend will lead to 529 * a H_TOO_HARD return and the whole transaction will be handled 530 * in virtual mode. 531 */ 532 do { 533 old_state = new_state = READ_ONCE(icp->state); 534 535 /* Down_CPPR */ 536 new_state.cppr = new_cppr; 537 538 /* 539 * Cut down Resend / Check_IPI / IPI 540 * 541 * The logic is that we cannot have a pending interrupt 542 * trumped by an IPI at this point (see above), so we 543 * know that either the pending interrupt is already an 544 * IPI (in which case we don't care to override it) or 545 * it's either more favored than us or non existent 546 */ 547 if (new_state.mfrr < new_cppr && 548 new_state.mfrr <= new_state.pending_pri) { 549 WARN_ON(new_state.xisr != XICS_IPI && 550 new_state.xisr != 0); 551 new_state.pending_pri = new_state.mfrr; 552 new_state.xisr = XICS_IPI; 553 } 554 555 /* Latch/clear resend bit */ 556 resend = new_state.need_resend; 557 new_state.need_resend = 0; 558 559 } while (!icp_try_update(icp, old_state, new_state, true)); 560 561 /* 562 * Now handle resend checks. Those are asynchronous to the ICP 563 * state update in HW (ie bus transactions) so we can handle them 564 * separately here too 565 */ 566 if (resend) 567 icp_check_resend(xics, icp); 568} 569 570static noinline unsigned long kvmppc_h_xirr(struct kvm_vcpu *vcpu) 571{ 572 union kvmppc_icp_state old_state, new_state; 573 struct kvmppc_icp *icp = vcpu->arch.icp; 574 u32 xirr; 575 576 /* First, remove EE from the processor */ 577 kvmppc_book3s_dequeue_irqprio(icp->vcpu, 578 BOOK3S_INTERRUPT_EXTERNAL_LEVEL); 579 580 /* 581 * ICP State: Accept_Interrupt 582 * 583 * Return the pending interrupt (if any) along with the 584 * current CPPR, then clear the XISR & set CPPR to the 585 * pending priority 586 */ 587 do { 588 old_state = new_state = READ_ONCE(icp->state); 589 590 xirr = old_state.xisr | (((u32)old_state.cppr) << 24); 591 if (!old_state.xisr) 592 break; 593 new_state.cppr = new_state.pending_pri; 594 new_state.pending_pri = 0xff; 595 new_state.xisr = 0; 596 597 } while (!icp_try_update(icp, old_state, new_state, true)); 598 599 XICS_DBG("h_xirr vcpu %d xirr %#x\n", vcpu->vcpu_id, xirr); 600 601 return xirr; 602} 603 604static noinline int kvmppc_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, 605 unsigned long mfrr) 606{ 607 union kvmppc_icp_state old_state, new_state; 608 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 609 struct kvmppc_icp *icp; 610 u32 reject; 611 bool resend; 612 bool local; 613 614 XICS_DBG("h_ipi vcpu %d to server %lu mfrr %#lx\n", 615 vcpu->vcpu_id, server, mfrr); 616 617 icp = vcpu->arch.icp; 618 local = icp->server_num == server; 619 if (!local) { 620 icp = kvmppc_xics_find_server(vcpu->kvm, server); 621 if (!icp) 622 return H_PARAMETER; 623 } 624 625 /* 626 * ICP state: Set_MFRR 627 * 628 * If the CPPR is more favored than the new MFRR, then 629 * nothing needs to be rejected as there can be no XISR to 630 * reject. If the MFRR is being made less favored then 631 * there might be a previously-rejected interrupt needing 632 * to be resent. 633 * 634 * ICP state: Check_IPI 635 * 636 * If the CPPR is less favored, then we might be replacing 637 * an interrupt, and thus need to possibly reject it. 638 * 639 * ICP State: IPI 640 * 641 * Besides rejecting any pending interrupts, we also 642 * update XISR and pending_pri to mark IPI as pending. 643 * 644 * PAPR does not describe this state, but if the MFRR is being 645 * made less favored than its earlier value, there might be 646 * a previously-rejected interrupt needing to be resent. 647 * Ideally, we would want to resend only if 648 * prio(pending_interrupt) < mfrr && 649 * prio(pending_interrupt) < cppr 650 * where pending interrupt is the one that was rejected. But 651 * we don't have that state, so we simply trigger a resend 652 * whenever the MFRR is made less favored. 653 */ 654 do { 655 old_state = new_state = READ_ONCE(icp->state); 656 657 /* Set_MFRR */ 658 new_state.mfrr = mfrr; 659 660 /* Check_IPI */ 661 reject = 0; 662 resend = false; 663 if (mfrr < new_state.cppr) { 664 /* Reject a pending interrupt if not an IPI */ 665 if (mfrr <= new_state.pending_pri) { 666 reject = new_state.xisr; 667 new_state.pending_pri = mfrr; 668 new_state.xisr = XICS_IPI; 669 } 670 } 671 672 if (mfrr > old_state.mfrr) { 673 resend = new_state.need_resend; 674 new_state.need_resend = 0; 675 } 676 } while (!icp_try_update(icp, old_state, new_state, local)); 677 678 /* Handle reject */ 679 if (reject && reject != XICS_IPI) 680 icp_deliver_irq(xics, icp, reject); 681 682 /* Handle resend */ 683 if (resend) 684 icp_check_resend(xics, icp); 685 686 return H_SUCCESS; 687} 688 689static int kvmppc_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server) 690{ 691 union kvmppc_icp_state state; 692 struct kvmppc_icp *icp; 693 694 icp = vcpu->arch.icp; 695 if (icp->server_num != server) { 696 icp = kvmppc_xics_find_server(vcpu->kvm, server); 697 if (!icp) 698 return H_PARAMETER; 699 } 700 state = READ_ONCE(icp->state); 701 kvmppc_set_gpr(vcpu, 4, ((u32)state.cppr << 24) | state.xisr); 702 kvmppc_set_gpr(vcpu, 5, state.mfrr); 703 return H_SUCCESS; 704} 705 706static noinline void kvmppc_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) 707{ 708 union kvmppc_icp_state old_state, new_state; 709 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 710 struct kvmppc_icp *icp = vcpu->arch.icp; 711 u32 reject; 712 713 XICS_DBG("h_cppr vcpu %d cppr %#lx\n", vcpu->vcpu_id, cppr); 714 715 /* 716 * ICP State: Set_CPPR 717 * 718 * We can safely compare the new value with the current 719 * value outside of the transaction as the CPPR is only 720 * ever changed by the processor on itself 721 */ 722 if (cppr > icp->state.cppr) 723 icp_down_cppr(xics, icp, cppr); 724 else if (cppr == icp->state.cppr) 725 return; 726 727 /* 728 * ICP State: Up_CPPR 729 * 730 * The processor is raising its priority, this can result 731 * in a rejection of a pending interrupt: 732 * 733 * ICP State: Reject_Current 734 * 735 * We can remove EE from the current processor, the update 736 * transaction will set it again if needed 737 */ 738 kvmppc_book3s_dequeue_irqprio(icp->vcpu, 739 BOOK3S_INTERRUPT_EXTERNAL_LEVEL); 740 741 do { 742 old_state = new_state = READ_ONCE(icp->state); 743 744 reject = 0; 745 new_state.cppr = cppr; 746 747 if (cppr <= new_state.pending_pri) { 748 reject = new_state.xisr; 749 new_state.xisr = 0; 750 new_state.pending_pri = 0xff; 751 } 752 753 } while (!icp_try_update(icp, old_state, new_state, true)); 754 755 /* 756 * Check for rejects. They are handled by doing a new delivery 757 * attempt (see comments in icp_deliver_irq). 758 */ 759 if (reject && reject != XICS_IPI) 760 icp_deliver_irq(xics, icp, reject); 761} 762 763static noinline int kvmppc_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) 764{ 765 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 766 struct kvmppc_icp *icp = vcpu->arch.icp; 767 struct kvmppc_ics *ics; 768 struct ics_irq_state *state; 769 u32 irq = xirr & 0x00ffffff; 770 u16 src; 771 772 XICS_DBG("h_eoi vcpu %d eoi %#lx\n", vcpu->vcpu_id, xirr); 773 774 /* 775 * ICP State: EOI 776 * 777 * Note: If EOI is incorrectly used by SW to lower the CPPR 778 * value (ie more favored), we do not check for rejection of 779 * a pending interrupt, this is a SW error and PAPR sepcifies 780 * that we don't have to deal with it. 781 * 782 * The sending of an EOI to the ICS is handled after the 783 * CPPR update 784 * 785 * ICP State: Down_CPPR which we handle 786 * in a separate function as it's shared with H_CPPR. 787 */ 788 icp_down_cppr(xics, icp, xirr >> 24); 789 790 /* IPIs have no EOI */ 791 if (irq == XICS_IPI) 792 return H_SUCCESS; 793 /* 794 * EOI handling: If the interrupt is still asserted, we need to 795 * resend it. We can take a lockless "peek" at the ICS state here. 796 * 797 * "Message" interrupts will never have "asserted" set 798 */ 799 ics = kvmppc_xics_find_ics(xics, irq, &src); 800 if (!ics) { 801 XICS_DBG("h_eoi: IRQ 0x%06x not found !\n", irq); 802 return H_PARAMETER; 803 } 804 state = &ics->irq_state[src]; 805 806 /* Still asserted, resend it */ 807 if (state->asserted) 808 icp_deliver_irq(xics, icp, irq); 809 810 kvm_notify_acked_irq(vcpu->kvm, 0, irq); 811 812 return H_SUCCESS; 813} 814 815static noinline int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall) 816{ 817 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 818 struct kvmppc_icp *icp = vcpu->arch.icp; 819 820 XICS_DBG("XICS_RM: H_%x completing, act: %x state: %lx tgt: %p\n", 821 hcall, icp->rm_action, icp->rm_dbgstate.raw, icp->rm_dbgtgt); 822 823 if (icp->rm_action & XICS_RM_KICK_VCPU) { 824 icp->n_rm_kick_vcpu++; 825 kvmppc_fast_vcpu_kick(icp->rm_kick_target); 826 } 827 if (icp->rm_action & XICS_RM_CHECK_RESEND) { 828 icp->n_rm_check_resend++; 829 icp_check_resend(xics, icp->rm_resend_icp); 830 } 831 if (icp->rm_action & XICS_RM_REJECT) { 832 icp->n_rm_reject++; 833 icp_deliver_irq(xics, icp, icp->rm_reject); 834 } 835 if (icp->rm_action & XICS_RM_NOTIFY_EOI) { 836 icp->n_rm_notify_eoi++; 837 kvm_notify_acked_irq(vcpu->kvm, 0, icp->rm_eoied_irq); 838 } 839 840 icp->rm_action = 0; 841 842 return H_SUCCESS; 843} 844 845int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 req) 846{ 847 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 848 unsigned long res; 849 int rc = H_SUCCESS; 850 851 /* Check if we have an ICP */ 852 if (!xics || !vcpu->arch.icp) 853 return H_HARDWARE; 854 855 /* These requests don't have real-mode implementations at present */ 856 switch (req) { 857 case H_XIRR_X: 858 res = kvmppc_h_xirr(vcpu); 859 kvmppc_set_gpr(vcpu, 4, res); 860 kvmppc_set_gpr(vcpu, 5, get_tb()); 861 return rc; 862 case H_IPOLL: 863 rc = kvmppc_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4)); 864 return rc; 865 } 866 867 /* Check for real mode returning too hard */ 868 if (xics->real_mode && is_kvmppc_hv_enabled(vcpu->kvm)) 869 return kvmppc_xics_rm_complete(vcpu, req); 870 871 switch (req) { 872 case H_XIRR: 873 res = kvmppc_h_xirr(vcpu); 874 kvmppc_set_gpr(vcpu, 4, res); 875 break; 876 case H_CPPR: 877 kvmppc_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4)); 878 break; 879 case H_EOI: 880 rc = kvmppc_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4)); 881 break; 882 case H_IPI: 883 rc = kvmppc_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4), 884 kvmppc_get_gpr(vcpu, 5)); 885 break; 886 } 887 888 return rc; 889} 890EXPORT_SYMBOL_GPL(kvmppc_xics_hcall); 891 892 893/* -- Initialisation code etc. -- */ 894 895static int xics_debug_show(struct seq_file *m, void *private) 896{ 897 struct kvmppc_xics *xics = m->private; 898 struct kvm *kvm = xics->kvm; 899 struct kvm_vcpu *vcpu; 900 int icsid, i; 901 unsigned long flags; 902 unsigned long t_rm_kick_vcpu, t_rm_check_resend; 903 unsigned long t_rm_reject, t_rm_notify_eoi; 904 unsigned long t_reject, t_check_resend; 905 906 if (!kvm) 907 return 0; 908 909 t_rm_kick_vcpu = 0; 910 t_rm_notify_eoi = 0; 911 t_rm_check_resend = 0; 912 t_rm_reject = 0; 913 t_check_resend = 0; 914 t_reject = 0; 915 916 seq_printf(m, "=========\nICP state\n=========\n"); 917 918 kvm_for_each_vcpu(i, vcpu, kvm) { 919 struct kvmppc_icp *icp = vcpu->arch.icp; 920 union kvmppc_icp_state state; 921 922 if (!icp) 923 continue; 924 925 state.raw = READ_ONCE(icp->state.raw); 926 seq_printf(m, "cpu server %#lx XIRR:%#x PPRI:%#x CPPR:%#x MFRR:%#x OUT:%d NR:%d\n", 927 icp->server_num, state.xisr, 928 state.pending_pri, state.cppr, state.mfrr, 929 state.out_ee, state.need_resend); 930 t_rm_kick_vcpu += icp->n_rm_kick_vcpu; 931 t_rm_notify_eoi += icp->n_rm_notify_eoi; 932 t_rm_check_resend += icp->n_rm_check_resend; 933 t_rm_reject += icp->n_rm_reject; 934 t_check_resend += icp->n_check_resend; 935 t_reject += icp->n_reject; 936 } 937 938 seq_printf(m, "ICP Guest->Host totals: kick_vcpu=%lu check_resend=%lu reject=%lu notify_eoi=%lu\n", 939 t_rm_kick_vcpu, t_rm_check_resend, 940 t_rm_reject, t_rm_notify_eoi); 941 seq_printf(m, "ICP Real Mode totals: check_resend=%lu resend=%lu\n", 942 t_check_resend, t_reject); 943 for (icsid = 0; icsid <= KVMPPC_XICS_MAX_ICS_ID; icsid++) { 944 struct kvmppc_ics *ics = xics->ics[icsid]; 945 946 if (!ics) 947 continue; 948 949 seq_printf(m, "=========\nICS state for ICS 0x%x\n=========\n", 950 icsid); 951 952 local_irq_save(flags); 953 arch_spin_lock(&ics->lock); 954 955 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 956 struct ics_irq_state *irq = &ics->irq_state[i]; 957 958 seq_printf(m, "irq 0x%06x: server %#x prio %#x save prio %#x asserted %d resend %d masked pending %d\n", 959 irq->number, irq->server, irq->priority, 960 irq->saved_priority, irq->asserted, 961 irq->resend, irq->masked_pending); 962 963 } 964 arch_spin_unlock(&ics->lock); 965 local_irq_restore(flags); 966 } 967 return 0; 968} 969 970static int xics_debug_open(struct inode *inode, struct file *file) 971{ 972 return single_open(file, xics_debug_show, inode->i_private); 973} 974 975static const struct file_operations xics_debug_fops = { 976 .open = xics_debug_open, 977 .read = seq_read, 978 .llseek = seq_lseek, 979 .release = single_release, 980}; 981 982static void xics_debugfs_init(struct kvmppc_xics *xics) 983{ 984 char *name; 985 986 name = kasprintf(GFP_KERNEL, "kvm-xics-%p", xics); 987 if (!name) { 988 pr_err("%s: no memory for name\n", __func__); 989 return; 990 } 991 992 xics->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root, 993 xics, &xics_debug_fops); 994 995 pr_debug("%s: created %s\n", __func__, name); 996 kfree(name); 997} 998 999static struct kvmppc_ics *kvmppc_xics_create_ics(struct kvm *kvm, 1000 struct kvmppc_xics *xics, int irq) 1001{ 1002 struct kvmppc_ics *ics; 1003 int i, icsid; 1004 1005 icsid = irq >> KVMPPC_XICS_ICS_SHIFT; 1006 1007 mutex_lock(&kvm->lock); 1008 1009 /* ICS already exists - somebody else got here first */ 1010 if (xics->ics[icsid]) 1011 goto out; 1012 1013 /* Create the ICS */ 1014 ics = kzalloc(sizeof(struct kvmppc_ics), GFP_KERNEL); 1015 if (!ics) 1016 goto out; 1017 1018 ics->icsid = icsid; 1019 1020 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 1021 ics->irq_state[i].number = (icsid << KVMPPC_XICS_ICS_SHIFT) | i; 1022 ics->irq_state[i].priority = MASKED; 1023 ics->irq_state[i].saved_priority = MASKED; 1024 } 1025 smp_wmb(); 1026 xics->ics[icsid] = ics; 1027 1028 if (icsid > xics->max_icsid) 1029 xics->max_icsid = icsid; 1030 1031 out: 1032 mutex_unlock(&kvm->lock); 1033 return xics->ics[icsid]; 1034} 1035 1036int kvmppc_xics_create_icp(struct kvm_vcpu *vcpu, unsigned long server_num) 1037{ 1038 struct kvmppc_icp *icp; 1039 1040 if (!vcpu->kvm->arch.xics) 1041 return -ENODEV; 1042 1043 if (kvmppc_xics_find_server(vcpu->kvm, server_num)) 1044 return -EEXIST; 1045 1046 icp = kzalloc(sizeof(struct kvmppc_icp), GFP_KERNEL); 1047 if (!icp) 1048 return -ENOMEM; 1049 1050 icp->vcpu = vcpu; 1051 icp->server_num = server_num; 1052 icp->state.mfrr = MASKED; 1053 icp->state.pending_pri = MASKED; 1054 vcpu->arch.icp = icp; 1055 1056 XICS_DBG("created server for vcpu %d\n", vcpu->vcpu_id); 1057 1058 return 0; 1059} 1060 1061u64 kvmppc_xics_get_icp(struct kvm_vcpu *vcpu) 1062{ 1063 struct kvmppc_icp *icp = vcpu->arch.icp; 1064 union kvmppc_icp_state state; 1065 1066 if (!icp) 1067 return 0; 1068 state = icp->state; 1069 return ((u64)state.cppr << KVM_REG_PPC_ICP_CPPR_SHIFT) | 1070 ((u64)state.xisr << KVM_REG_PPC_ICP_XISR_SHIFT) | 1071 ((u64)state.mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT) | 1072 ((u64)state.pending_pri << KVM_REG_PPC_ICP_PPRI_SHIFT); 1073} 1074 1075int kvmppc_xics_set_icp(struct kvm_vcpu *vcpu, u64 icpval) 1076{ 1077 struct kvmppc_icp *icp = vcpu->arch.icp; 1078 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 1079 union kvmppc_icp_state old_state, new_state; 1080 struct kvmppc_ics *ics; 1081 u8 cppr, mfrr, pending_pri; 1082 u32 xisr; 1083 u16 src; 1084 bool resend; 1085 1086 if (!icp || !xics) 1087 return -ENOENT; 1088 1089 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT; 1090 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) & 1091 KVM_REG_PPC_ICP_XISR_MASK; 1092 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT; 1093 pending_pri = icpval >> KVM_REG_PPC_ICP_PPRI_SHIFT; 1094 1095 /* Require the new state to be internally consistent */ 1096 if (xisr == 0) { 1097 if (pending_pri != 0xff) 1098 return -EINVAL; 1099 } else if (xisr == XICS_IPI) { 1100 if (pending_pri != mfrr || pending_pri >= cppr) 1101 return -EINVAL; 1102 } else { 1103 if (pending_pri >= mfrr || pending_pri >= cppr) 1104 return -EINVAL; 1105 ics = kvmppc_xics_find_ics(xics, xisr, &src); 1106 if (!ics) 1107 return -EINVAL; 1108 } 1109 1110 new_state.raw = 0; 1111 new_state.cppr = cppr; 1112 new_state.xisr = xisr; 1113 new_state.mfrr = mfrr; 1114 new_state.pending_pri = pending_pri; 1115 1116 /* 1117 * Deassert the CPU interrupt request. 1118 * icp_try_update will reassert it if necessary. 1119 */ 1120 kvmppc_book3s_dequeue_irqprio(icp->vcpu, 1121 BOOK3S_INTERRUPT_EXTERNAL_LEVEL); 1122 1123 /* 1124 * Note that if we displace an interrupt from old_state.xisr, 1125 * we don't mark it as rejected. We expect userspace to set 1126 * the state of the interrupt sources to be consistent with 1127 * the ICP states (either before or afterwards, which doesn't 1128 * matter). We do handle resends due to CPPR becoming less 1129 * favoured because that is necessary to end up with a 1130 * consistent state in the situation where userspace restores 1131 * the ICS states before the ICP states. 1132 */ 1133 do { 1134 old_state = READ_ONCE(icp->state); 1135 1136 if (new_state.mfrr <= old_state.mfrr) { 1137 resend = false; 1138 new_state.need_resend = old_state.need_resend; 1139 } else { 1140 resend = old_state.need_resend; 1141 new_state.need_resend = 0; 1142 } 1143 } while (!icp_try_update(icp, old_state, new_state, false)); 1144 1145 if (resend) 1146 icp_check_resend(xics, icp); 1147 1148 return 0; 1149} 1150 1151static int xics_get_source(struct kvmppc_xics *xics, long irq, u64 addr) 1152{ 1153 int ret; 1154 struct kvmppc_ics *ics; 1155 struct ics_irq_state *irqp; 1156 u64 __user *ubufp = (u64 __user *) addr; 1157 u16 idx; 1158 u64 val, prio; 1159 unsigned long flags; 1160 1161 ics = kvmppc_xics_find_ics(xics, irq, &idx); 1162 if (!ics) 1163 return -ENOENT; 1164 1165 irqp = &ics->irq_state[idx]; 1166 local_irq_save(flags); 1167 arch_spin_lock(&ics->lock); 1168 ret = -ENOENT; 1169 if (irqp->exists) { 1170 val = irqp->server; 1171 prio = irqp->priority; 1172 if (prio == MASKED) { 1173 val |= KVM_XICS_MASKED; 1174 prio = irqp->saved_priority; 1175 } 1176 val |= prio << KVM_XICS_PRIORITY_SHIFT; 1177 if (irqp->asserted) 1178 val |= KVM_XICS_LEVEL_SENSITIVE | KVM_XICS_PENDING; 1179 else if (irqp->masked_pending || irqp->resend) 1180 val |= KVM_XICS_PENDING; 1181 ret = 0; 1182 } 1183 arch_spin_unlock(&ics->lock); 1184 local_irq_restore(flags); 1185 1186 if (!ret && put_user(val, ubufp)) 1187 ret = -EFAULT; 1188 1189 return ret; 1190} 1191 1192static int xics_set_source(struct kvmppc_xics *xics, long irq, u64 addr) 1193{ 1194 struct kvmppc_ics *ics; 1195 struct ics_irq_state *irqp; 1196 u64 __user *ubufp = (u64 __user *) addr; 1197 u16 idx; 1198 u64 val; 1199 u8 prio; 1200 u32 server; 1201 unsigned long flags; 1202 1203 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS) 1204 return -ENOENT; 1205 1206 ics = kvmppc_xics_find_ics(xics, irq, &idx); 1207 if (!ics) { 1208 ics = kvmppc_xics_create_ics(xics->kvm, xics, irq); 1209 if (!ics) 1210 return -ENOMEM; 1211 } 1212 irqp = &ics->irq_state[idx]; 1213 if (get_user(val, ubufp)) 1214 return -EFAULT; 1215 1216 server = val & KVM_XICS_DESTINATION_MASK; 1217 prio = val >> KVM_XICS_PRIORITY_SHIFT; 1218 if (prio != MASKED && 1219 kvmppc_xics_find_server(xics->kvm, server) == NULL) 1220 return -EINVAL; 1221 1222 local_irq_save(flags); 1223 arch_spin_lock(&ics->lock); 1224 irqp->server = server; 1225 irqp->saved_priority = prio; 1226 if (val & KVM_XICS_MASKED) 1227 prio = MASKED; 1228 irqp->priority = prio; 1229 irqp->resend = 0; 1230 irqp->masked_pending = 0; 1231 irqp->asserted = 0; 1232 if ((val & KVM_XICS_PENDING) && (val & KVM_XICS_LEVEL_SENSITIVE)) 1233 irqp->asserted = 1; 1234 irqp->exists = 1; 1235 arch_spin_unlock(&ics->lock); 1236 local_irq_restore(flags); 1237 1238 if (val & KVM_XICS_PENDING) 1239 icp_deliver_irq(xics, NULL, irqp->number); 1240 1241 return 0; 1242} 1243 1244int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 1245 bool line_status) 1246{ 1247 struct kvmppc_xics *xics = kvm->arch.xics; 1248 1249 return ics_deliver_irq(xics, irq, level); 1250} 1251 1252int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm, 1253 int irq_source_id, int level, bool line_status) 1254{ 1255 if (!level) 1256 return -1; 1257 return kvm_set_irq(kvm, irq_source_id, irq_entry->gsi, 1258 level, line_status); 1259} 1260 1261static int xics_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1262{ 1263 struct kvmppc_xics *xics = dev->private; 1264 1265 switch (attr->group) { 1266 case KVM_DEV_XICS_GRP_SOURCES: 1267 return xics_set_source(xics, attr->attr, attr->addr); 1268 } 1269 return -ENXIO; 1270} 1271 1272static int xics_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1273{ 1274 struct kvmppc_xics *xics = dev->private; 1275 1276 switch (attr->group) { 1277 case KVM_DEV_XICS_GRP_SOURCES: 1278 return xics_get_source(xics, attr->attr, attr->addr); 1279 } 1280 return -ENXIO; 1281} 1282 1283static int xics_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1284{ 1285 switch (attr->group) { 1286 case KVM_DEV_XICS_GRP_SOURCES: 1287 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ && 1288 attr->attr < KVMPPC_XICS_NR_IRQS) 1289 return 0; 1290 break; 1291 } 1292 return -ENXIO; 1293} 1294 1295static void kvmppc_xics_free(struct kvm_device *dev) 1296{ 1297 struct kvmppc_xics *xics = dev->private; 1298 int i; 1299 struct kvm *kvm = xics->kvm; 1300 1301 debugfs_remove(xics->dentry); 1302 1303 if (kvm) 1304 kvm->arch.xics = NULL; 1305 1306 for (i = 0; i <= xics->max_icsid; i++) 1307 kfree(xics->ics[i]); 1308 kfree(xics); 1309 kfree(dev); 1310} 1311 1312static int kvmppc_xics_create(struct kvm_device *dev, u32 type) 1313{ 1314 struct kvmppc_xics *xics; 1315 struct kvm *kvm = dev->kvm; 1316 int ret = 0; 1317 1318 xics = kzalloc(sizeof(*xics), GFP_KERNEL); 1319 if (!xics) 1320 return -ENOMEM; 1321 1322 dev->private = xics; 1323 xics->dev = dev; 1324 xics->kvm = kvm; 1325 1326 /* Already there ? */ 1327 mutex_lock(&kvm->lock); 1328 if (kvm->arch.xics) 1329 ret = -EEXIST; 1330 else 1331 kvm->arch.xics = xics; 1332 mutex_unlock(&kvm->lock); 1333 1334 if (ret) { 1335 kfree(xics); 1336 return ret; 1337 } 1338 1339 xics_debugfs_init(xics); 1340 1341#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1342 if (cpu_has_feature(CPU_FTR_ARCH_206)) { 1343 /* Enable real mode support */ 1344 xics->real_mode = ENABLE_REALMODE; 1345 xics->real_mode_dbg = DEBUG_REALMODE; 1346 } 1347#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 1348 1349 return 0; 1350} 1351 1352struct kvm_device_ops kvm_xics_ops = { 1353 .name = "kvm-xics", 1354 .create = kvmppc_xics_create, 1355 .destroy = kvmppc_xics_free, 1356 .set_attr = xics_set_attr, 1357 .get_attr = xics_get_attr, 1358 .has_attr = xics_has_attr, 1359}; 1360 1361int kvmppc_xics_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu, 1362 u32 xcpu) 1363{ 1364 struct kvmppc_xics *xics = dev->private; 1365 int r = -EBUSY; 1366 1367 if (dev->ops != &kvm_xics_ops) 1368 return -EPERM; 1369 if (xics->kvm != vcpu->kvm) 1370 return -EPERM; 1371 if (vcpu->arch.irq_type) 1372 return -EBUSY; 1373 1374 r = kvmppc_xics_create_icp(vcpu, xcpu); 1375 if (!r) 1376 vcpu->arch.irq_type = KVMPPC_IRQ_XICS; 1377 1378 return r; 1379} 1380 1381void kvmppc_xics_free_icp(struct kvm_vcpu *vcpu) 1382{ 1383 if (!vcpu->arch.icp) 1384 return; 1385 kfree(vcpu->arch.icp); 1386 vcpu->arch.icp = NULL; 1387 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT; 1388} 1389 1390static int xics_set_irq(struct kvm_kernel_irq_routing_entry *e, 1391 struct kvm *kvm, int irq_source_id, int level, 1392 bool line_status) 1393{ 1394 return kvm_set_irq(kvm, irq_source_id, e->gsi, level, line_status); 1395} 1396 1397int kvm_irq_map_gsi(struct kvm *kvm, 1398 struct kvm_kernel_irq_routing_entry *entries, int gsi) 1399{ 1400 entries->gsi = gsi; 1401 entries->type = KVM_IRQ_ROUTING_IRQCHIP; 1402 entries->set = xics_set_irq; 1403 entries->irqchip.irqchip = 0; 1404 entries->irqchip.pin = gsi; 1405 return 1; 1406} 1407 1408int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin) 1409{ 1410 return pin; 1411} 1412