1/* 2 * Copyright (c) 2003-2012 Broadcom Corporation 3 * All Rights Reserved 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the Broadcom 9 * license below: 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in 19 * the documentation and/or other materials provided with the 20 * distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 24 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35#include <linux/types.h> 36#include <linux/pci.h> 37#include <linux/kernel.h> 38#include <linux/init.h> 39#include <linux/msi.h> 40#include <linux/mm.h> 41#include <linux/irq.h> 42#include <linux/irqdesc.h> 43#include <linux/console.h> 44 45#include <asm/io.h> 46 47#include <asm/netlogic/interrupt.h> 48#include <asm/netlogic/haldefs.h> 49#include <asm/netlogic/common.h> 50#include <asm/netlogic/mips-extns.h> 51 52#include <asm/netlogic/xlp-hal/iomap.h> 53#include <asm/netlogic/xlp-hal/xlp.h> 54#include <asm/netlogic/xlp-hal/pic.h> 55#include <asm/netlogic/xlp-hal/pcibus.h> 56#include <asm/netlogic/xlp-hal/bridge.h> 57 58#define XLP_MSIVEC_PER_LINK 32 59#define XLP_MSIXVEC_TOTAL (cpu_is_xlp9xx() ? 128 : 32) 60#define XLP_MSIXVEC_PER_LINK (cpu_is_xlp9xx() ? 32 : 8) 61 62/* 128 MSI irqs per node, mapped starting at NLM_MSI_VEC_BASE */ 63static inline int nlm_link_msiirq(int link, int msivec) 64{ 65 return NLM_MSI_VEC_BASE + link * XLP_MSIVEC_PER_LINK + msivec; 66} 67 68/* get the link MSI vector from irq number */ 69static inline int nlm_irq_msivec(int irq) 70{ 71 return (irq - NLM_MSI_VEC_BASE) % XLP_MSIVEC_PER_LINK; 72} 73 74/* get the link from the irq number */ 75static inline int nlm_irq_msilink(int irq) 76{ 77 int total_msivec = XLP_MSIVEC_PER_LINK * PCIE_NLINKS; 78 79 return ((irq - NLM_MSI_VEC_BASE) % total_msivec) / 80 XLP_MSIVEC_PER_LINK; 81} 82 83/* 84 * For XLP 8xx/4xx/3xx/2xx, only 32 MSI-X vectors are possible because 85 * there are only 32 PIC interrupts for MSI. We split them statically 86 * and use 8 MSI-X vectors per link - this keeps the allocation and 87 * lookup simple. 88 * On XLP 9xx, there are 32 vectors per link, and the interrupts are 89 * not routed thru PIC, so we can use all 128 MSI-X vectors. 90 */ 91static inline int nlm_link_msixirq(int link, int bit) 92{ 93 return NLM_MSIX_VEC_BASE + link * XLP_MSIXVEC_PER_LINK + bit; 94} 95 96/* get the link MSI vector from irq number */ 97static inline int nlm_irq_msixvec(int irq) 98{ 99 return (irq - NLM_MSIX_VEC_BASE) % XLP_MSIXVEC_TOTAL; 100} 101 102/* get the link from MSIX vec */ 103static inline int nlm_irq_msixlink(int msixvec) 104{ 105 return msixvec / XLP_MSIXVEC_PER_LINK; 106} 107 108/* 109 * Per link MSI and MSI-X information, set as IRQ handler data for 110 * MSI and MSI-X interrupts. 111 */ 112struct xlp_msi_data { 113 struct nlm_soc_info *node; 114 uint64_t lnkbase; 115 uint32_t msi_enabled_mask; 116 uint32_t msi_alloc_mask; 117 uint32_t msix_alloc_mask; 118 spinlock_t msi_lock; 119}; 120 121/* 122 * MSI Chip definitions 123 * 124 * On XLP, there is a PIC interrupt associated with each PCIe link on the 125 * chip (which appears as a PCI bridge to us). This gives us 32 MSI irqa 126 * per link and 128 overall. 127 * 128 * When a device connected to the link raises a MSI interrupt, we get a 129 * link interrupt and we then have to look at PCIE_MSI_STATUS register at 130 * the bridge to map it to the IRQ 131 */ 132static void xlp_msi_enable(struct irq_data *d) 133{ 134 struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); 135 unsigned long flags; 136 int vec; 137 138 vec = nlm_irq_msivec(d->irq); 139 spin_lock_irqsave(&md->msi_lock, flags); 140 md->msi_enabled_mask |= 1u << vec; 141 if (cpu_is_xlp9xx()) 142 nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN, 143 md->msi_enabled_mask); 144 else 145 nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask); 146 spin_unlock_irqrestore(&md->msi_lock, flags); 147} 148 149static void xlp_msi_disable(struct irq_data *d) 150{ 151 struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); 152 unsigned long flags; 153 int vec; 154 155 vec = nlm_irq_msivec(d->irq); 156 spin_lock_irqsave(&md->msi_lock, flags); 157 md->msi_enabled_mask &= ~(1u << vec); 158 if (cpu_is_xlp9xx()) 159 nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN, 160 md->msi_enabled_mask); 161 else 162 nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask); 163 spin_unlock_irqrestore(&md->msi_lock, flags); 164} 165 166static void xlp_msi_mask_ack(struct irq_data *d) 167{ 168 struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); 169 int link, vec; 170 171 link = nlm_irq_msilink(d->irq); 172 vec = nlm_irq_msivec(d->irq); 173 xlp_msi_disable(d); 174 175 /* Ack MSI on bridge */ 176 if (cpu_is_xlp9xx()) 177 nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_STATUS, 1u << vec); 178 else 179 nlm_write_reg(md->lnkbase, PCIE_MSI_STATUS, 1u << vec); 180 181} 182 183static struct irq_chip xlp_msi_chip = { 184 .name = "XLP-MSI", 185 .irq_enable = xlp_msi_enable, 186 .irq_disable = xlp_msi_disable, 187 .irq_mask_ack = xlp_msi_mask_ack, 188 .irq_unmask = xlp_msi_enable, 189}; 190 191/* 192 * XLP8XX/4XX/3XX/2XX: 193 * The MSI-X interrupt handling is different from MSI, there are 32 MSI-X 194 * interrupts generated by the PIC and each of these correspond to a MSI-X 195 * vector (0-31) that can be assigned. 196 * 197 * We divide the MSI-X vectors to 8 per link and do a per-link allocation 198 * 199 * XLP9XX: 200 * 32 MSI-X vectors are available per link, and the interrupts are not routed 201 * thru the PIC. PIC ack not needed. 202 * 203 * Enable and disable done using standard MSI functions. 204 */ 205static void xlp_msix_mask_ack(struct irq_data *d) 206{ 207 struct xlp_msi_data *md; 208 int link, msixvec; 209 uint32_t status_reg, bit; 210 211 msixvec = nlm_irq_msixvec(d->irq); 212 link = nlm_irq_msixlink(msixvec); 213 pci_msi_mask_irq(d); 214 md = irq_data_get_irq_handler_data(d); 215 216 /* Ack MSI on bridge */ 217 if (cpu_is_xlp9xx()) { 218 status_reg = PCIE_9XX_MSIX_STATUSX(link); 219 bit = msixvec % XLP_MSIXVEC_PER_LINK; 220 } else { 221 status_reg = PCIE_MSIX_STATUS; 222 bit = msixvec; 223 } 224 nlm_write_reg(md->lnkbase, status_reg, 1u << bit); 225 226 if (!cpu_is_xlp9xx()) 227 nlm_pic_ack(md->node->picbase, 228 PIC_IRT_PCIE_MSIX_INDEX(msixvec)); 229} 230 231static struct irq_chip xlp_msix_chip = { 232 .name = "XLP-MSIX", 233 .irq_enable = pci_msi_unmask_irq, 234 .irq_disable = pci_msi_mask_irq, 235 .irq_mask_ack = xlp_msix_mask_ack, 236 .irq_unmask = pci_msi_unmask_irq, 237}; 238 239void arch_teardown_msi_irq(unsigned int irq) 240{ 241} 242 243/* 244 * Setup a PCIe link for MSI. By default, the links are in 245 * legacy interrupt mode. We will switch them to MSI mode 246 * at the first MSI request. 247 */ 248static void xlp_config_link_msi(uint64_t lnkbase, int lirq, uint64_t msiaddr) 249{ 250 u32 val; 251 252 if (cpu_is_xlp9xx()) { 253 val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0); 254 if ((val & 0x200) == 0) { 255 val |= 0x200; /* MSI Interrupt enable */ 256 nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val); 257 } 258 } else { 259 val = nlm_read_reg(lnkbase, PCIE_INT_EN0); 260 if ((val & 0x200) == 0) { 261 val |= 0x200; 262 nlm_write_reg(lnkbase, PCIE_INT_EN0, val); 263 } 264 } 265 266 val = nlm_read_reg(lnkbase, 0x1); /* CMD */ 267 if ((val & 0x0400) == 0) { 268 val |= 0x0400; 269 nlm_write_reg(lnkbase, 0x1, val); 270 } 271 272 /* Update IRQ in the PCI irq reg */ 273 val = nlm_read_pci_reg(lnkbase, 0xf); 274 val &= ~0x1fu; 275 val |= (1 << 8) | lirq; 276 nlm_write_pci_reg(lnkbase, 0xf, val); 277 278 /* MSI addr */ 279 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRH, msiaddr >> 32); 280 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRL, msiaddr & 0xffffffff); 281 282 /* MSI cap for bridge */ 283 val = nlm_read_reg(lnkbase, PCIE_BRIDGE_MSI_CAP); 284 if ((val & (1 << 16)) == 0) { 285 val |= 0xb << 16; /* mmc32, msi enable */ 286 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_CAP, val); 287 } 288} 289 290/* 291 * Allocate a MSI vector on a link 292 */ 293static int xlp_setup_msi(uint64_t lnkbase, int node, int link, 294 struct msi_desc *desc) 295{ 296 struct xlp_msi_data *md; 297 struct msi_msg msg; 298 unsigned long flags; 299 int msivec, irt, lirq, xirq, ret; 300 uint64_t msiaddr; 301 302 /* Get MSI data for the link */ 303 lirq = PIC_PCIE_LINK_MSI_IRQ(link); 304 xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); 305 md = irq_get_handler_data(xirq); 306 msiaddr = MSI_LINK_ADDR(node, link); 307 308 spin_lock_irqsave(&md->msi_lock, flags); 309 if (md->msi_alloc_mask == 0) { 310 xlp_config_link_msi(lnkbase, lirq, msiaddr); 311 /* switch the link IRQ to MSI range */ 312 if (cpu_is_xlp9xx()) 313 irt = PIC_9XX_IRT_PCIE_LINK_INDEX(link); 314 else 315 irt = PIC_IRT_PCIE_LINK_INDEX(link); 316 nlm_setup_pic_irq(node, lirq, lirq, irt); 317 nlm_pic_init_irt(nlm_get_node(node)->picbase, irt, lirq, 318 node * nlm_threads_per_node(), 1 /*en */); 319 } 320 321 /* allocate a MSI vec, and tell the bridge about it */ 322 msivec = fls(md->msi_alloc_mask); 323 if (msivec == XLP_MSIVEC_PER_LINK) { 324 spin_unlock_irqrestore(&md->msi_lock, flags); 325 return -ENOMEM; 326 } 327 md->msi_alloc_mask |= (1u << msivec); 328 spin_unlock_irqrestore(&md->msi_lock, flags); 329 330 msg.address_hi = msiaddr >> 32; 331 msg.address_lo = msiaddr & 0xffffffff; 332 msg.data = 0xc00 | msivec; 333 334 xirq = xirq + msivec; /* msi mapped to global irq space */ 335 ret = irq_set_msi_desc(xirq, desc); 336 if (ret < 0) 337 return ret; 338 339 pci_write_msi_msg(xirq, &msg); 340 return 0; 341} 342 343/* 344 * Switch a link to MSI-X mode 345 */ 346static void xlp_config_link_msix(uint64_t lnkbase, int lirq, uint64_t msixaddr) 347{ 348 u32 val; 349 350 val = nlm_read_reg(lnkbase, 0x2C); 351 if ((val & 0x80000000U) == 0) { 352 val |= 0x80000000U; 353 nlm_write_reg(lnkbase, 0x2C, val); 354 } 355 356 if (cpu_is_xlp9xx()) { 357 val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0); 358 if ((val & 0x200) == 0) { 359 val |= 0x200; /* MSI Interrupt enable */ 360 nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val); 361 } 362 } else { 363 val = nlm_read_reg(lnkbase, PCIE_INT_EN0); 364 if ((val & 0x200) == 0) { 365 val |= 0x200; /* MSI Interrupt enable */ 366 nlm_write_reg(lnkbase, PCIE_INT_EN0, val); 367 } 368 } 369 370 val = nlm_read_reg(lnkbase, 0x1); /* CMD */ 371 if ((val & 0x0400) == 0) { 372 val |= 0x0400; 373 nlm_write_reg(lnkbase, 0x1, val); 374 } 375 376 /* Update IRQ in the PCI irq reg */ 377 val = nlm_read_pci_reg(lnkbase, 0xf); 378 val &= ~0x1fu; 379 val |= (1 << 8) | lirq; 380 nlm_write_pci_reg(lnkbase, 0xf, val); 381 382 if (cpu_is_xlp9xx()) { 383 /* MSI-X addresses */ 384 nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_BASE, 385 msixaddr >> 8); 386 nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_LIMIT, 387 (msixaddr + MSI_ADDR_SZ) >> 8); 388 } else { 389 /* MSI-X addresses */ 390 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_BASE, 391 msixaddr >> 8); 392 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_LIMIT, 393 (msixaddr + MSI_ADDR_SZ) >> 8); 394 } 395} 396 397/* 398 * Allocate a MSI-X vector 399 */ 400static int xlp_setup_msix(uint64_t lnkbase, int node, int link, 401 struct msi_desc *desc) 402{ 403 struct xlp_msi_data *md; 404 struct msi_msg msg; 405 unsigned long flags; 406 int t, msixvec, lirq, xirq, ret; 407 uint64_t msixaddr; 408 409 /* Get MSI data for the link */ 410 lirq = PIC_PCIE_MSIX_IRQ(link); 411 xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0)); 412 md = irq_get_handler_data(xirq); 413 msixaddr = MSIX_LINK_ADDR(node, link); 414 415 spin_lock_irqsave(&md->msi_lock, flags); 416 /* switch the PCIe link to MSI-X mode at the first alloc */ 417 if (md->msix_alloc_mask == 0) 418 xlp_config_link_msix(lnkbase, lirq, msixaddr); 419 420 /* allocate a MSI-X vec, and tell the bridge about it */ 421 t = fls(md->msix_alloc_mask); 422 if (t == XLP_MSIXVEC_PER_LINK) { 423 spin_unlock_irqrestore(&md->msi_lock, flags); 424 return -ENOMEM; 425 } 426 md->msix_alloc_mask |= (1u << t); 427 spin_unlock_irqrestore(&md->msi_lock, flags); 428 429 xirq += t; 430 msixvec = nlm_irq_msixvec(xirq); 431 432 msg.address_hi = msixaddr >> 32; 433 msg.address_lo = msixaddr & 0xffffffff; 434 msg.data = 0xc00 | msixvec; 435 436 ret = irq_set_msi_desc(xirq, desc); 437 if (ret < 0) 438 return ret; 439 440 pci_write_msi_msg(xirq, &msg); 441 return 0; 442} 443 444int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) 445{ 446 struct pci_dev *lnkdev; 447 uint64_t lnkbase; 448 int node, link, slot; 449 450 lnkdev = xlp_get_pcie_link(dev); 451 if (lnkdev == NULL) { 452 dev_err(&dev->dev, "Could not find bridge\n"); 453 return 1; 454 } 455 slot = PCI_SLOT(lnkdev->devfn); 456 link = PCI_FUNC(lnkdev->devfn); 457 node = slot / 8; 458 lnkbase = nlm_get_pcie_base(node, link); 459 460 if (desc->msi_attrib.is_msix) 461 return xlp_setup_msix(lnkbase, node, link, desc); 462 else 463 return xlp_setup_msi(lnkbase, node, link, desc); 464} 465 466void __init xlp_init_node_msi_irqs(int node, int link) 467{ 468 struct nlm_soc_info *nodep; 469 struct xlp_msi_data *md; 470 int irq, i, irt, msixvec, val; 471 472 pr_info("[%d %d] Init node PCI IRT\n", node, link); 473 nodep = nlm_get_node(node); 474 475 /* Alloc an MSI block for the link */ 476 md = kzalloc(sizeof(*md), GFP_KERNEL); 477 spin_lock_init(&md->msi_lock); 478 md->msi_enabled_mask = 0; 479 md->msi_alloc_mask = 0; 480 md->msix_alloc_mask = 0; 481 md->node = nodep; 482 md->lnkbase = nlm_get_pcie_base(node, link); 483 484 /* extended space for MSI interrupts */ 485 irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); 486 for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) { 487 irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq); 488 irq_set_handler_data(i, md); 489 } 490 491 for (i = 0; i < XLP_MSIXVEC_PER_LINK ; i++) { 492 if (cpu_is_xlp9xx()) { 493 val = ((node * nlm_threads_per_node()) << 7 | 494 PIC_PCIE_MSIX_IRQ(link) << 1 | 0 << 0); 495 nlm_write_pcie_reg(md->lnkbase, PCIE_9XX_MSIX_VECX(i + 496 (link * XLP_MSIXVEC_PER_LINK)), val); 497 } else { 498 /* Initialize MSI-X irts to generate one interrupt 499 * per link 500 */ 501 msixvec = link * XLP_MSIXVEC_PER_LINK + i; 502 irt = PIC_IRT_PCIE_MSIX_INDEX(msixvec); 503 nlm_pic_init_irt(nodep->picbase, irt, 504 PIC_PCIE_MSIX_IRQ(link), 505 node * nlm_threads_per_node(), 1); 506 } 507 508 /* Initialize MSI-X extended irq space for the link */ 509 irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i)); 510 irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq); 511 irq_set_handler_data(irq, md); 512 } 513} 514 515void nlm_dispatch_msi(int node, int lirq) 516{ 517 struct xlp_msi_data *md; 518 int link, i, irqbase; 519 u32 status; 520 521 link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE; 522 irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); 523 md = irq_get_handler_data(irqbase); 524 if (cpu_is_xlp9xx()) 525 status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSI_STATUS) & 526 md->msi_enabled_mask; 527 else 528 status = nlm_read_reg(md->lnkbase, PCIE_MSI_STATUS) & 529 md->msi_enabled_mask; 530 while (status) { 531 i = __ffs(status); 532 do_IRQ(irqbase + i); 533 status &= status - 1; 534 } 535 536 /* Ack at eirr and PIC */ 537 ack_c0_eirr(PIC_PCIE_LINK_MSI_IRQ(link)); 538 if (cpu_is_xlp9xx()) 539 nlm_pic_ack(md->node->picbase, 540 PIC_9XX_IRT_PCIE_LINK_INDEX(link)); 541 else 542 nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_LINK_INDEX(link)); 543} 544 545void nlm_dispatch_msix(int node, int lirq) 546{ 547 struct xlp_msi_data *md; 548 int link, i, irqbase; 549 u32 status; 550 551 link = lirq - PIC_PCIE_MSIX_IRQ_BASE; 552 irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0)); 553 md = irq_get_handler_data(irqbase); 554 if (cpu_is_xlp9xx()) 555 status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSIX_STATUSX(link)); 556 else 557 status = nlm_read_reg(md->lnkbase, PCIE_MSIX_STATUS); 558 559 /* narrow it down to the MSI-x vectors for our link */ 560 if (!cpu_is_xlp9xx()) 561 status = (status >> (link * XLP_MSIXVEC_PER_LINK)) & 562 ((1 << XLP_MSIXVEC_PER_LINK) - 1); 563 564 while (status) { 565 i = __ffs(status); 566 do_IRQ(irqbase + i); 567 status &= status - 1; 568 } 569 /* Ack at eirr and PIC */ 570 ack_c0_eirr(PIC_PCIE_MSIX_IRQ(link)); 571} 572