1/* 2 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation 3 * 4 * Rewrite, cleanup: 5 * 6 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation 7 * Copyright (C) 2006 Olof Johansson <olof@lixom.net> 8 * 9 * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR. 10 * 11 * 12 * This program is free software; you can redistribute it and/or modify 13 * it under the terms of the GNU General Public License as published by 14 * the Free Software Foundation; either version 2 of the License, or 15 * (at your option) any later version. 16 * 17 * This program is distributed in the hope that it will be useful, 18 * but WITHOUT ANY WARRANTY; without even the implied warranty of 19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 * GNU General Public License for more details. 21 * 22 * You should have received a copy of the GNU General Public License 23 * along with this program; if not, write to the Free Software 24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 25 */ 26 27#include <linux/init.h> 28#include <linux/types.h> 29#include <linux/slab.h> 30#include <linux/mm.h> 31#include <linux/memblock.h> 32#include <linux/spinlock.h> 33#include <linux/string.h> 34#include <linux/pci.h> 35#include <linux/dma-mapping.h> 36#include <linux/crash_dump.h> 37#include <linux/memory.h> 38#include <linux/of.h> 39#include <asm/io.h> 40#include <asm/prom.h> 41#include <asm/rtas.h> 42#include <asm/iommu.h> 43#include <asm/pci-bridge.h> 44#include <asm/machdep.h> 45#include <asm/firmware.h> 46#include <asm/tce.h> 47#include <asm/ppc-pci.h> 48#include <asm/udbg.h> 49#include <asm/mmzone.h> 50#include <asm/plpar_wrappers.h> 51 52#include "pseries.h" 53 54static void tce_invalidate_pSeries_sw(struct iommu_table *tbl, 55 __be64 *startp, __be64 *endp) 56{ 57 u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index; 58 unsigned long start, end, inc; 59 60 start = __pa(startp); 61 end = __pa(endp); 62 inc = L1_CACHE_BYTES; /* invalidate a cacheline of TCEs at a time */ 63 64 /* If this is non-zero, change the format. We shift the 65 * address and or in the magic from the device tree. */ 66 if (tbl->it_busno) { 67 start <<= 12; 68 end <<= 12; 69 inc <<= 12; 70 start |= tbl->it_busno; 71 end |= tbl->it_busno; 72 } 73 74 end |= inc - 1; /* round up end to be different than start */ 75 76 mb(); /* Make sure TCEs in memory are written */ 77 while (start <= end) { 78 out_be64(invalidate, start); 79 start += inc; 80 } 81} 82 83static int tce_build_pSeries(struct iommu_table *tbl, long index, 84 long npages, unsigned long uaddr, 85 enum dma_data_direction direction, 86 struct dma_attrs *attrs) 87{ 88 u64 proto_tce; 89 __be64 *tcep, *tces; 90 u64 rpn; 91 92 proto_tce = TCE_PCI_READ; // Read allowed 93 94 if (direction != DMA_TO_DEVICE) 95 proto_tce |= TCE_PCI_WRITE; 96 97 tces = tcep = ((__be64 *)tbl->it_base) + index; 98 99 while (npages--) { 100 /* can't move this out since we might cross MEMBLOCK boundary */ 101 rpn = __pa(uaddr) >> TCE_SHIFT; 102 *tcep = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT); 103 104 uaddr += TCE_PAGE_SIZE; 105 tcep++; 106 } 107 108 if (tbl->it_type & TCE_PCI_SWINV_CREATE) 109 tce_invalidate_pSeries_sw(tbl, tces, tcep - 1); 110 return 0; 111} 112 113 114static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages) 115{ 116 __be64 *tcep, *tces; 117 118 tces = tcep = ((__be64 *)tbl->it_base) + index; 119 120 while (npages--) 121 *(tcep++) = 0; 122 123 if (tbl->it_type & TCE_PCI_SWINV_FREE) 124 tce_invalidate_pSeries_sw(tbl, tces, tcep - 1); 125} 126 127static unsigned long tce_get_pseries(struct iommu_table *tbl, long index) 128{ 129 __be64 *tcep; 130 131 tcep = ((__be64 *)tbl->it_base) + index; 132 133 return be64_to_cpu(*tcep); 134} 135 136static void tce_free_pSeriesLP(struct iommu_table*, long, long); 137static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long); 138 139static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum, 140 long npages, unsigned long uaddr, 141 enum dma_data_direction direction, 142 struct dma_attrs *attrs) 143{ 144 u64 rc = 0; 145 u64 proto_tce, tce; 146 u64 rpn; 147 int ret = 0; 148 long tcenum_start = tcenum, npages_start = npages; 149 150 rpn = __pa(uaddr) >> TCE_SHIFT; 151 proto_tce = TCE_PCI_READ; 152 if (direction != DMA_TO_DEVICE) 153 proto_tce |= TCE_PCI_WRITE; 154 155 while (npages--) { 156 tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT; 157 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce); 158 159 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) { 160 ret = (int)rc; 161 tce_free_pSeriesLP(tbl, tcenum_start, 162 (npages_start - (npages + 1))); 163 break; 164 } 165 166 if (rc && printk_ratelimit()) { 167 printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 168 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 169 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 170 printk("\ttce val = 0x%llx\n", tce ); 171 dump_stack(); 172 } 173 174 tcenum++; 175 rpn++; 176 } 177 return ret; 178} 179 180static DEFINE_PER_CPU(__be64 *, tce_page); 181 182static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, 183 long npages, unsigned long uaddr, 184 enum dma_data_direction direction, 185 struct dma_attrs *attrs) 186{ 187 u64 rc = 0; 188 u64 proto_tce; 189 __be64 *tcep; 190 u64 rpn; 191 long l, limit; 192 long tcenum_start = tcenum, npages_start = npages; 193 int ret = 0; 194 unsigned long flags; 195 196 if (npages == 1) { 197 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr, 198 direction, attrs); 199 } 200 201 local_irq_save(flags); /* to protect tcep and the page behind it */ 202 203 tcep = __this_cpu_read(tce_page); 204 205 /* This is safe to do since interrupts are off when we're called 206 * from iommu_alloc{,_sg}() 207 */ 208 if (!tcep) { 209 tcep = (__be64 *)__get_free_page(GFP_ATOMIC); 210 /* If allocation fails, fall back to the loop implementation */ 211 if (!tcep) { 212 local_irq_restore(flags); 213 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr, 214 direction, attrs); 215 } 216 __this_cpu_write(tce_page, tcep); 217 } 218 219 rpn = __pa(uaddr) >> TCE_SHIFT; 220 proto_tce = TCE_PCI_READ; 221 if (direction != DMA_TO_DEVICE) 222 proto_tce |= TCE_PCI_WRITE; 223 224 /* We can map max one pageful of TCEs at a time */ 225 do { 226 /* 227 * Set up the page with TCE data, looping through and setting 228 * the values. 229 */ 230 limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE); 231 232 for (l = 0; l < limit; l++) { 233 tcep[l] = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT); 234 rpn++; 235 } 236 237 rc = plpar_tce_put_indirect((u64)tbl->it_index, 238 (u64)tcenum << 12, 239 (u64)__pa(tcep), 240 limit); 241 242 npages -= limit; 243 tcenum += limit; 244 } while (npages > 0 && !rc); 245 246 local_irq_restore(flags); 247 248 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) { 249 ret = (int)rc; 250 tce_freemulti_pSeriesLP(tbl, tcenum_start, 251 (npages_start - (npages + limit))); 252 return ret; 253 } 254 255 if (rc && printk_ratelimit()) { 256 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 257 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 258 printk("\tnpages = 0x%llx\n", (u64)npages); 259 printk("\ttce[0] val = 0x%llx\n", tcep[0]); 260 dump_stack(); 261 } 262 return ret; 263} 264 265static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages) 266{ 267 u64 rc; 268 269 while (npages--) { 270 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0); 271 272 if (rc && printk_ratelimit()) { 273 printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 274 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 275 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 276 dump_stack(); 277 } 278 279 tcenum++; 280 } 281} 282 283 284static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages) 285{ 286 u64 rc; 287 288 rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages); 289 290 if (rc && printk_ratelimit()) { 291 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n"); 292 printk("\trc = %lld\n", rc); 293 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 294 printk("\tnpages = 0x%llx\n", (u64)npages); 295 dump_stack(); 296 } 297} 298 299static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum) 300{ 301 u64 rc; 302 unsigned long tce_ret; 303 304 rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret); 305 306 if (rc && printk_ratelimit()) { 307 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc); 308 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 309 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 310 dump_stack(); 311 } 312 313 return tce_ret; 314} 315 316/* this is compatible with cells for the device tree property */ 317struct dynamic_dma_window_prop { 318 __be32 liobn; /* tce table number */ 319 __be64 dma_base; /* address hi,lo */ 320 __be32 tce_shift; /* ilog2(tce_page_size) */ 321 __be32 window_shift; /* ilog2(tce_window_size) */ 322}; 323 324struct direct_window { 325 struct device_node *device; 326 const struct dynamic_dma_window_prop *prop; 327 struct list_head list; 328}; 329 330/* Dynamic DMA Window support */ 331struct ddw_query_response { 332 u32 windows_available; 333 u32 largest_available_block; 334 u32 page_size; 335 u32 migration_capable; 336}; 337 338struct ddw_create_response { 339 u32 liobn; 340 u32 addr_hi; 341 u32 addr_lo; 342}; 343 344static LIST_HEAD(direct_window_list); 345/* prevents races between memory on/offline and window creation */ 346static DEFINE_SPINLOCK(direct_window_list_lock); 347/* protects initializing window twice for same device */ 348static DEFINE_MUTEX(direct_window_init_mutex); 349#define DIRECT64_PROPNAME "linux,direct64-ddr-window-info" 350 351static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn, 352 unsigned long num_pfn, const void *arg) 353{ 354 const struct dynamic_dma_window_prop *maprange = arg; 355 int rc; 356 u64 tce_size, num_tce, dma_offset, next; 357 u32 tce_shift; 358 long limit; 359 360 tce_shift = be32_to_cpu(maprange->tce_shift); 361 tce_size = 1ULL << tce_shift; 362 next = start_pfn << PAGE_SHIFT; 363 num_tce = num_pfn << PAGE_SHIFT; 364 365 /* round back to the beginning of the tce page size */ 366 num_tce += next & (tce_size - 1); 367 next &= ~(tce_size - 1); 368 369 /* covert to number of tces */ 370 num_tce |= tce_size - 1; 371 num_tce >>= tce_shift; 372 373 do { 374 /* 375 * Set up the page with TCE data, looping through and setting 376 * the values. 377 */ 378 limit = min_t(long, num_tce, 512); 379 dma_offset = next + be64_to_cpu(maprange->dma_base); 380 381 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn), 382 dma_offset, 383 0, limit); 384 next += limit * tce_size; 385 num_tce -= limit; 386 } while (num_tce > 0 && !rc); 387 388 return rc; 389} 390 391static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, 392 unsigned long num_pfn, const void *arg) 393{ 394 const struct dynamic_dma_window_prop *maprange = arg; 395 u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn; 396 __be64 *tcep; 397 u32 tce_shift; 398 u64 rc = 0; 399 long l, limit; 400 401 local_irq_disable(); /* to protect tcep and the page behind it */ 402 tcep = __this_cpu_read(tce_page); 403 404 if (!tcep) { 405 tcep = (__be64 *)__get_free_page(GFP_ATOMIC); 406 if (!tcep) { 407 local_irq_enable(); 408 return -ENOMEM; 409 } 410 __this_cpu_write(tce_page, tcep); 411 } 412 413 proto_tce = TCE_PCI_READ | TCE_PCI_WRITE; 414 415 liobn = (u64)be32_to_cpu(maprange->liobn); 416 tce_shift = be32_to_cpu(maprange->tce_shift); 417 tce_size = 1ULL << tce_shift; 418 next = start_pfn << PAGE_SHIFT; 419 num_tce = num_pfn << PAGE_SHIFT; 420 421 /* round back to the beginning of the tce page size */ 422 num_tce += next & (tce_size - 1); 423 next &= ~(tce_size - 1); 424 425 /* covert to number of tces */ 426 num_tce |= tce_size - 1; 427 num_tce >>= tce_shift; 428 429 /* We can map max one pageful of TCEs at a time */ 430 do { 431 /* 432 * Set up the page with TCE data, looping through and setting 433 * the values. 434 */ 435 limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE); 436 dma_offset = next + be64_to_cpu(maprange->dma_base); 437 438 for (l = 0; l < limit; l++) { 439 tcep[l] = cpu_to_be64(proto_tce | next); 440 next += tce_size; 441 } 442 443 rc = plpar_tce_put_indirect(liobn, 444 dma_offset, 445 (u64)__pa(tcep), 446 limit); 447 448 num_tce -= limit; 449 } while (num_tce > 0 && !rc); 450 451 /* error cleanup: caller will clear whole range */ 452 453 local_irq_enable(); 454 return rc; 455} 456 457static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn, 458 unsigned long num_pfn, void *arg) 459{ 460 return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg); 461} 462 463 464#ifdef CONFIG_PCI 465static void iommu_table_setparms(struct pci_controller *phb, 466 struct device_node *dn, 467 struct iommu_table *tbl) 468{ 469 struct device_node *node; 470 const unsigned long *basep, *sw_inval; 471 const u32 *sizep; 472 473 node = phb->dn; 474 475 basep = of_get_property(node, "linux,tce-base", NULL); 476 sizep = of_get_property(node, "linux,tce-size", NULL); 477 if (basep == NULL || sizep == NULL) { 478 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has " 479 "missing tce entries !\n", dn->full_name); 480 return; 481 } 482 483 tbl->it_base = (unsigned long)__va(*basep); 484 485 if (!is_kdump_kernel()) 486 memset((void *)tbl->it_base, 0, *sizep); 487 488 tbl->it_busno = phb->bus->number; 489 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K; 490 491 /* Units of tce entries */ 492 tbl->it_offset = phb->dma_window_base_cur >> tbl->it_page_shift; 493 494 /* Test if we are going over 2GB of DMA space */ 495 if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) { 496 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n"); 497 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n"); 498 } 499 500 phb->dma_window_base_cur += phb->dma_window_size; 501 502 /* Set the tce table size - measured in entries */ 503 tbl->it_size = phb->dma_window_size >> tbl->it_page_shift; 504 505 tbl->it_index = 0; 506 tbl->it_blocksize = 16; 507 tbl->it_type = TCE_PCI; 508 509 sw_inval = of_get_property(node, "linux,tce-sw-invalidate-info", NULL); 510 if (sw_inval) { 511 /* 512 * This property contains information on how to 513 * invalidate the TCE entry. The first property is 514 * the base MMIO address used to invalidate entries. 515 * The second property tells us the format of the TCE 516 * invalidate (whether it needs to be shifted) and 517 * some magic routing info to add to our invalidate 518 * command. 519 */ 520 tbl->it_index = (unsigned long) ioremap(sw_inval[0], 8); 521 tbl->it_busno = sw_inval[1]; /* overload this with magic */ 522 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE; 523 } 524} 525 526/* 527 * iommu_table_setparms_lpar 528 * 529 * Function: On pSeries LPAR systems, return TCE table info, given a pci bus. 530 */ 531static void iommu_table_setparms_lpar(struct pci_controller *phb, 532 struct device_node *dn, 533 struct iommu_table *tbl, 534 const __be32 *dma_window) 535{ 536 unsigned long offset, size; 537 538 of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size); 539 540 tbl->it_busno = phb->bus->number; 541 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K; 542 tbl->it_base = 0; 543 tbl->it_blocksize = 16; 544 tbl->it_type = TCE_PCI; 545 tbl->it_offset = offset >> tbl->it_page_shift; 546 tbl->it_size = size >> tbl->it_page_shift; 547} 548 549static void pci_dma_bus_setup_pSeries(struct pci_bus *bus) 550{ 551 struct device_node *dn; 552 struct iommu_table *tbl; 553 struct device_node *isa_dn, *isa_dn_orig; 554 struct device_node *tmp; 555 struct pci_dn *pci; 556 int children; 557 558 dn = pci_bus_to_OF_node(bus); 559 560 pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn->full_name); 561 562 if (bus->self) { 563 /* This is not a root bus, any setup will be done for the 564 * device-side of the bridge in iommu_dev_setup_pSeries(). 565 */ 566 return; 567 } 568 pci = PCI_DN(dn); 569 570 /* Check if the ISA bus on the system is under 571 * this PHB. 572 */ 573 isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa"); 574 575 while (isa_dn && isa_dn != dn) 576 isa_dn = isa_dn->parent; 577 578 of_node_put(isa_dn_orig); 579 580 /* Count number of direct PCI children of the PHB. */ 581 for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling) 582 children++; 583 584 pr_debug("Children: %d\n", children); 585 586 /* Calculate amount of DMA window per slot. Each window must be 587 * a power of two (due to pci_alloc_consistent requirements). 588 * 589 * Keep 256MB aside for PHBs with ISA. 590 */ 591 592 if (!isa_dn) { 593 /* No ISA/IDE - just set window size and return */ 594 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */ 595 596 while (pci->phb->dma_window_size * children > 0x80000000ul) 597 pci->phb->dma_window_size >>= 1; 598 pr_debug("No ISA/IDE, window size is 0x%llx\n", 599 pci->phb->dma_window_size); 600 pci->phb->dma_window_base_cur = 0; 601 602 return; 603 } 604 605 /* If we have ISA, then we probably have an IDE 606 * controller too. Allocate a 128MB table but 607 * skip the first 128MB to avoid stepping on ISA 608 * space. 609 */ 610 pci->phb->dma_window_size = 0x8000000ul; 611 pci->phb->dma_window_base_cur = 0x8000000ul; 612 613 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, 614 pci->phb->node); 615 616 iommu_table_setparms(pci->phb, dn, tbl); 617 pci->iommu_table = iommu_init_table(tbl, pci->phb->node); 618 iommu_register_group(tbl, pci_domain_nr(bus), 0); 619 620 /* Divide the rest (1.75GB) among the children */ 621 pci->phb->dma_window_size = 0x80000000ul; 622 while (pci->phb->dma_window_size * children > 0x70000000ul) 623 pci->phb->dma_window_size >>= 1; 624 625 pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size); 626} 627 628 629static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus) 630{ 631 struct iommu_table *tbl; 632 struct device_node *dn, *pdn; 633 struct pci_dn *ppci; 634 const __be32 *dma_window = NULL; 635 636 dn = pci_bus_to_OF_node(bus); 637 638 pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n", 639 dn->full_name); 640 641 /* Find nearest ibm,dma-window, walking up the device tree */ 642 for (pdn = dn; pdn != NULL; pdn = pdn->parent) { 643 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 644 if (dma_window != NULL) 645 break; 646 } 647 648 if (dma_window == NULL) { 649 pr_debug(" no ibm,dma-window property !\n"); 650 return; 651 } 652 653 ppci = PCI_DN(pdn); 654 655 pr_debug(" parent is %s, iommu_table: 0x%p\n", 656 pdn->full_name, ppci->iommu_table); 657 658 if (!ppci->iommu_table) { 659 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, 660 ppci->phb->node); 661 iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window); 662 ppci->iommu_table = iommu_init_table(tbl, ppci->phb->node); 663 iommu_register_group(tbl, pci_domain_nr(bus), 0); 664 pr_debug(" created table: %p\n", ppci->iommu_table); 665 } 666} 667 668 669static void pci_dma_dev_setup_pSeries(struct pci_dev *dev) 670{ 671 struct device_node *dn; 672 struct iommu_table *tbl; 673 674 pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev)); 675 676 dn = dev->dev.of_node; 677 678 /* If we're the direct child of a root bus, then we need to allocate 679 * an iommu table ourselves. The bus setup code should have setup 680 * the window sizes already. 681 */ 682 if (!dev->bus->self) { 683 struct pci_controller *phb = PCI_DN(dn)->phb; 684 685 pr_debug(" --> first child, no bridge. Allocating iommu table.\n"); 686 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, 687 phb->node); 688 iommu_table_setparms(phb, dn, tbl); 689 PCI_DN(dn)->iommu_table = iommu_init_table(tbl, phb->node); 690 iommu_register_group(tbl, pci_domain_nr(phb->bus), 0); 691 set_iommu_table_base_and_group(&dev->dev, 692 PCI_DN(dn)->iommu_table); 693 return; 694 } 695 696 /* If this device is further down the bus tree, search upwards until 697 * an already allocated iommu table is found and use that. 698 */ 699 700 while (dn && PCI_DN(dn) && PCI_DN(dn)->iommu_table == NULL) 701 dn = dn->parent; 702 703 if (dn && PCI_DN(dn)) 704 set_iommu_table_base_and_group(&dev->dev, 705 PCI_DN(dn)->iommu_table); 706 else 707 printk(KERN_WARNING "iommu: Device %s has no iommu table\n", 708 pci_name(dev)); 709} 710 711static int __read_mostly disable_ddw; 712 713static int __init disable_ddw_setup(char *str) 714{ 715 disable_ddw = 1; 716 printk(KERN_INFO "ppc iommu: disabling ddw.\n"); 717 718 return 0; 719} 720 721early_param("disable_ddw", disable_ddw_setup); 722 723static void remove_ddw(struct device_node *np, bool remove_prop) 724{ 725 struct dynamic_dma_window_prop *dwp; 726 struct property *win64; 727 u32 ddw_avail[3]; 728 u64 liobn; 729 int ret = 0; 730 731 ret = of_property_read_u32_array(np, "ibm,ddw-applicable", 732 &ddw_avail[0], 3); 733 734 win64 = of_find_property(np, DIRECT64_PROPNAME, NULL); 735 if (!win64) 736 return; 737 738 if (ret || win64->length < sizeof(*dwp)) 739 goto delprop; 740 741 dwp = win64->value; 742 liobn = (u64)be32_to_cpu(dwp->liobn); 743 744 /* clear the whole window, note the arg is in kernel pages */ 745 ret = tce_clearrange_multi_pSeriesLP(0, 746 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp); 747 if (ret) 748 pr_warning("%s failed to clear tces in window.\n", 749 np->full_name); 750 else 751 pr_debug("%s successfully cleared tces in window.\n", 752 np->full_name); 753 754 ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn); 755 if (ret) 756 pr_warning("%s: failed to remove direct window: rtas returned " 757 "%d to ibm,remove-pe-dma-window(%x) %llx\n", 758 np->full_name, ret, ddw_avail[2], liobn); 759 else 760 pr_debug("%s: successfully removed direct window: rtas returned " 761 "%d to ibm,remove-pe-dma-window(%x) %llx\n", 762 np->full_name, ret, ddw_avail[2], liobn); 763 764delprop: 765 if (remove_prop) 766 ret = of_remove_property(np, win64); 767 if (ret) 768 pr_warning("%s: failed to remove direct window property: %d\n", 769 np->full_name, ret); 770} 771 772static u64 find_existing_ddw(struct device_node *pdn) 773{ 774 struct direct_window *window; 775 const struct dynamic_dma_window_prop *direct64; 776 u64 dma_addr = 0; 777 778 spin_lock(&direct_window_list_lock); 779 /* check if we already created a window and dupe that config if so */ 780 list_for_each_entry(window, &direct_window_list, list) { 781 if (window->device == pdn) { 782 direct64 = window->prop; 783 dma_addr = be64_to_cpu(direct64->dma_base); 784 break; 785 } 786 } 787 spin_unlock(&direct_window_list_lock); 788 789 return dma_addr; 790} 791 792static int find_existing_ddw_windows(void) 793{ 794 int len; 795 struct device_node *pdn; 796 struct direct_window *window; 797 const struct dynamic_dma_window_prop *direct64; 798 799 if (!firmware_has_feature(FW_FEATURE_LPAR)) 800 return 0; 801 802 for_each_node_with_property(pdn, DIRECT64_PROPNAME) { 803 direct64 = of_get_property(pdn, DIRECT64_PROPNAME, &len); 804 if (!direct64) 805 continue; 806 807 window = kzalloc(sizeof(*window), GFP_KERNEL); 808 if (!window || len < sizeof(struct dynamic_dma_window_prop)) { 809 kfree(window); 810 remove_ddw(pdn, true); 811 continue; 812 } 813 814 window->device = pdn; 815 window->prop = direct64; 816 spin_lock(&direct_window_list_lock); 817 list_add(&window->list, &direct_window_list); 818 spin_unlock(&direct_window_list_lock); 819 } 820 821 return 0; 822} 823machine_arch_initcall(pseries, find_existing_ddw_windows); 824 825static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail, 826 struct ddw_query_response *query) 827{ 828 struct eeh_dev *edev; 829 u32 cfg_addr; 830 u64 buid; 831 int ret; 832 833 /* 834 * Get the config address and phb buid of the PE window. 835 * Rely on eeh to retrieve this for us. 836 * Retrieve them from the pci device, not the node with the 837 * dma-window property 838 */ 839 edev = pci_dev_to_eeh_dev(dev); 840 cfg_addr = edev->config_addr; 841 if (edev->pe_config_addr) 842 cfg_addr = edev->pe_config_addr; 843 buid = edev->phb->buid; 844 845 ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query, 846 cfg_addr, BUID_HI(buid), BUID_LO(buid)); 847 dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x" 848 " returned %d\n", ddw_avail[0], cfg_addr, BUID_HI(buid), 849 BUID_LO(buid), ret); 850 return ret; 851} 852 853static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail, 854 struct ddw_create_response *create, int page_shift, 855 int window_shift) 856{ 857 struct eeh_dev *edev; 858 u32 cfg_addr; 859 u64 buid; 860 int ret; 861 862 /* 863 * Get the config address and phb buid of the PE window. 864 * Rely on eeh to retrieve this for us. 865 * Retrieve them from the pci device, not the node with the 866 * dma-window property 867 */ 868 edev = pci_dev_to_eeh_dev(dev); 869 cfg_addr = edev->config_addr; 870 if (edev->pe_config_addr) 871 cfg_addr = edev->pe_config_addr; 872 buid = edev->phb->buid; 873 874 do { 875 /* extra outputs are LIOBN and dma-addr (hi, lo) */ 876 ret = rtas_call(ddw_avail[1], 5, 4, (u32 *)create, 877 cfg_addr, BUID_HI(buid), BUID_LO(buid), 878 page_shift, window_shift); 879 } while (rtas_busy_delay(ret)); 880 dev_info(&dev->dev, 881 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d " 882 "(liobn = 0x%x starting addr = %x %x)\n", ddw_avail[1], 883 cfg_addr, BUID_HI(buid), BUID_LO(buid), page_shift, 884 window_shift, ret, create->liobn, create->addr_hi, create->addr_lo); 885 886 return ret; 887} 888 889struct failed_ddw_pdn { 890 struct device_node *pdn; 891 struct list_head list; 892}; 893 894static LIST_HEAD(failed_ddw_pdn_list); 895 896/* 897 * If the PE supports dynamic dma windows, and there is space for a table 898 * that can map all pages in a linear offset, then setup such a table, 899 * and record the dma-offset in the struct device. 900 * 901 * dev: the pci device we are checking 902 * pdn: the parent pe node with the ibm,dma_window property 903 * Future: also check if we can remap the base window for our base page size 904 * 905 * returns the dma offset for use by dma_set_mask 906 */ 907static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn) 908{ 909 int len, ret; 910 struct ddw_query_response query; 911 struct ddw_create_response create; 912 int page_shift; 913 u64 dma_addr, max_addr; 914 struct device_node *dn; 915 u32 ddw_avail[3]; 916 struct direct_window *window; 917 struct property *win64; 918 struct dynamic_dma_window_prop *ddwprop; 919 struct failed_ddw_pdn *fpdn; 920 921 mutex_lock(&direct_window_init_mutex); 922 923 dma_addr = find_existing_ddw(pdn); 924 if (dma_addr != 0) 925 goto out_unlock; 926 927 /* 928 * If we already went through this for a previous function of 929 * the same device and failed, we don't want to muck with the 930 * DMA window again, as it will race with in-flight operations 931 * and can lead to EEHs. The above mutex protects access to the 932 * list. 933 */ 934 list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) { 935 if (!strcmp(fpdn->pdn->full_name, pdn->full_name)) 936 goto out_unlock; 937 } 938 939 /* 940 * the ibm,ddw-applicable property holds the tokens for: 941 * ibm,query-pe-dma-window 942 * ibm,create-pe-dma-window 943 * ibm,remove-pe-dma-window 944 * for the given node in that order. 945 * the property is actually in the parent, not the PE 946 */ 947 ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable", 948 &ddw_avail[0], 3); 949 if (ret) 950 goto out_failed; 951 952 /* 953 * Query if there is a second window of size to map the 954 * whole partition. Query returns number of windows, largest 955 * block assigned to PE (partition endpoint), and two bitmasks 956 * of page sizes: supported and supported for migrate-dma. 957 */ 958 dn = pci_device_to_OF_node(dev); 959 ret = query_ddw(dev, ddw_avail, &query); 960 if (ret != 0) 961 goto out_failed; 962 963 if (query.windows_available == 0) { 964 /* 965 * no additional windows are available for this device. 966 * We might be able to reallocate the existing window, 967 * trading in for a larger page size. 968 */ 969 dev_dbg(&dev->dev, "no free dynamic windows"); 970 goto out_failed; 971 } 972 if (query.page_size & 4) { 973 page_shift = 24; /* 16MB */ 974 } else if (query.page_size & 2) { 975 page_shift = 16; /* 64kB */ 976 } else if (query.page_size & 1) { 977 page_shift = 12; /* 4kB */ 978 } else { 979 dev_dbg(&dev->dev, "no supported direct page size in mask %x", 980 query.page_size); 981 goto out_failed; 982 } 983 /* verify the window * number of ptes will map the partition */ 984 /* check largest block * page size > max memory hotplug addr */ 985 max_addr = memory_hotplug_max(); 986 if (query.largest_available_block < (max_addr >> page_shift)) { 987 dev_dbg(&dev->dev, "can't map partiton max 0x%llx with %u " 988 "%llu-sized pages\n", max_addr, query.largest_available_block, 989 1ULL << page_shift); 990 goto out_failed; 991 } 992 len = order_base_2(max_addr); 993 win64 = kzalloc(sizeof(struct property), GFP_KERNEL); 994 if (!win64) { 995 dev_info(&dev->dev, 996 "couldn't allocate property for 64bit dma window\n"); 997 goto out_failed; 998 } 999 win64->name = kstrdup(DIRECT64_PROPNAME, GFP_KERNEL); 1000 win64->value = ddwprop = kmalloc(sizeof(*ddwprop), GFP_KERNEL); 1001 win64->length = sizeof(*ddwprop); 1002 if (!win64->name || !win64->value) { 1003 dev_info(&dev->dev, 1004 "couldn't allocate property name and value\n"); 1005 goto out_free_prop; 1006 } 1007 1008 ret = create_ddw(dev, ddw_avail, &create, page_shift, len); 1009 if (ret != 0) 1010 goto out_free_prop; 1011 1012 ddwprop->liobn = cpu_to_be32(create.liobn); 1013 ddwprop->dma_base = cpu_to_be64(((u64)create.addr_hi << 32) | 1014 create.addr_lo); 1015 ddwprop->tce_shift = cpu_to_be32(page_shift); 1016 ddwprop->window_shift = cpu_to_be32(len); 1017 1018 dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %s\n", 1019 create.liobn, dn->full_name); 1020 1021 window = kzalloc(sizeof(*window), GFP_KERNEL); 1022 if (!window) 1023 goto out_clear_window; 1024 1025 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT, 1026 win64->value, tce_setrange_multi_pSeriesLP_walk); 1027 if (ret) { 1028 dev_info(&dev->dev, "failed to map direct window for %s: %d\n", 1029 dn->full_name, ret); 1030 goto out_free_window; 1031 } 1032 1033 ret = of_add_property(pdn, win64); 1034 if (ret) { 1035 dev_err(&dev->dev, "unable to add dma window property for %s: %d", 1036 pdn->full_name, ret); 1037 goto out_free_window; 1038 } 1039 1040 window->device = pdn; 1041 window->prop = ddwprop; 1042 spin_lock(&direct_window_list_lock); 1043 list_add(&window->list, &direct_window_list); 1044 spin_unlock(&direct_window_list_lock); 1045 1046 dma_addr = be64_to_cpu(ddwprop->dma_base); 1047 goto out_unlock; 1048 1049out_free_window: 1050 kfree(window); 1051 1052out_clear_window: 1053 remove_ddw(pdn, true); 1054 1055out_free_prop: 1056 kfree(win64->name); 1057 kfree(win64->value); 1058 kfree(win64); 1059 1060out_failed: 1061 1062 fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL); 1063 if (!fpdn) 1064 goto out_unlock; 1065 fpdn->pdn = pdn; 1066 list_add(&fpdn->list, &failed_ddw_pdn_list); 1067 1068out_unlock: 1069 mutex_unlock(&direct_window_init_mutex); 1070 return dma_addr; 1071} 1072 1073static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev) 1074{ 1075 struct device_node *pdn, *dn; 1076 struct iommu_table *tbl; 1077 const __be32 *dma_window = NULL; 1078 struct pci_dn *pci; 1079 1080 pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev)); 1081 1082 /* dev setup for LPAR is a little tricky, since the device tree might 1083 * contain the dma-window properties per-device and not necessarily 1084 * for the bus. So we need to search upwards in the tree until we 1085 * either hit a dma-window property, OR find a parent with a table 1086 * already allocated. 1087 */ 1088 dn = pci_device_to_OF_node(dev); 1089 pr_debug(" node is %s\n", dn->full_name); 1090 1091 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table; 1092 pdn = pdn->parent) { 1093 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 1094 if (dma_window) 1095 break; 1096 } 1097 1098 if (!pdn || !PCI_DN(pdn)) { 1099 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: " 1100 "no DMA window found for pci dev=%s dn=%s\n", 1101 pci_name(dev), of_node_full_name(dn)); 1102 return; 1103 } 1104 pr_debug(" parent is %s\n", pdn->full_name); 1105 1106 pci = PCI_DN(pdn); 1107 if (!pci->iommu_table) { 1108 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, 1109 pci->phb->node); 1110 iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window); 1111 pci->iommu_table = iommu_init_table(tbl, pci->phb->node); 1112 iommu_register_group(tbl, pci_domain_nr(pci->phb->bus), 0); 1113 pr_debug(" created table: %p\n", pci->iommu_table); 1114 } else { 1115 pr_debug(" found DMA window, table: %p\n", pci->iommu_table); 1116 } 1117 1118 set_iommu_table_base_and_group(&dev->dev, pci->iommu_table); 1119} 1120 1121static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask) 1122{ 1123 bool ddw_enabled = false; 1124 struct device_node *pdn, *dn; 1125 struct pci_dev *pdev; 1126 const __be32 *dma_window = NULL; 1127 u64 dma_offset; 1128 1129 if (!dev->dma_mask) 1130 return -EIO; 1131 1132 if (!dev_is_pci(dev)) 1133 goto check_mask; 1134 1135 pdev = to_pci_dev(dev); 1136 1137 /* only attempt to use a new window if 64-bit DMA is requested */ 1138 if (!disable_ddw && dma_mask == DMA_BIT_MASK(64)) { 1139 dn = pci_device_to_OF_node(pdev); 1140 dev_dbg(dev, "node is %s\n", dn->full_name); 1141 1142 /* 1143 * the device tree might contain the dma-window properties 1144 * per-device and not necessarily for the bus. So we need to 1145 * search upwards in the tree until we either hit a dma-window 1146 * property, OR find a parent with a table already allocated. 1147 */ 1148 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table; 1149 pdn = pdn->parent) { 1150 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 1151 if (dma_window) 1152 break; 1153 } 1154 if (pdn && PCI_DN(pdn)) { 1155 dma_offset = enable_ddw(pdev, pdn); 1156 if (dma_offset != 0) { 1157 dev_info(dev, "Using 64-bit direct DMA at offset %llx\n", dma_offset); 1158 set_dma_offset(dev, dma_offset); 1159 set_dma_ops(dev, &dma_direct_ops); 1160 ddw_enabled = true; 1161 } 1162 } 1163 } 1164 1165 /* fall back on iommu ops, restore table pointer with ops */ 1166 if (!ddw_enabled && get_dma_ops(dev) != &dma_iommu_ops) { 1167 dev_info(dev, "Restoring 32-bit DMA via iommu\n"); 1168 set_dma_ops(dev, &dma_iommu_ops); 1169 pci_dma_dev_setup_pSeriesLP(pdev); 1170 } 1171 1172check_mask: 1173 if (!dma_supported(dev, dma_mask)) 1174 return -EIO; 1175 1176 *dev->dma_mask = dma_mask; 1177 return 0; 1178} 1179 1180static u64 dma_get_required_mask_pSeriesLP(struct device *dev) 1181{ 1182 if (!dev->dma_mask) 1183 return 0; 1184 1185 if (!disable_ddw && dev_is_pci(dev)) { 1186 struct pci_dev *pdev = to_pci_dev(dev); 1187 struct device_node *dn; 1188 1189 dn = pci_device_to_OF_node(pdev); 1190 1191 /* search upwards for ibm,dma-window */ 1192 for (; dn && PCI_DN(dn) && !PCI_DN(dn)->iommu_table; 1193 dn = dn->parent) 1194 if (of_get_property(dn, "ibm,dma-window", NULL)) 1195 break; 1196 /* if there is a ibm,ddw-applicable property require 64 bits */ 1197 if (dn && PCI_DN(dn) && 1198 of_get_property(dn, "ibm,ddw-applicable", NULL)) 1199 return DMA_BIT_MASK(64); 1200 } 1201 1202 return dma_iommu_ops.get_required_mask(dev); 1203} 1204 1205#else /* CONFIG_PCI */ 1206#define pci_dma_bus_setup_pSeries NULL 1207#define pci_dma_dev_setup_pSeries NULL 1208#define pci_dma_bus_setup_pSeriesLP NULL 1209#define pci_dma_dev_setup_pSeriesLP NULL 1210#define dma_set_mask_pSeriesLP NULL 1211#define dma_get_required_mask_pSeriesLP NULL 1212#endif /* !CONFIG_PCI */ 1213 1214static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, 1215 void *data) 1216{ 1217 struct direct_window *window; 1218 struct memory_notify *arg = data; 1219 int ret = 0; 1220 1221 switch (action) { 1222 case MEM_GOING_ONLINE: 1223 spin_lock(&direct_window_list_lock); 1224 list_for_each_entry(window, &direct_window_list, list) { 1225 ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, 1226 arg->nr_pages, window->prop); 1227 /* XXX log error */ 1228 } 1229 spin_unlock(&direct_window_list_lock); 1230 break; 1231 case MEM_CANCEL_ONLINE: 1232 case MEM_OFFLINE: 1233 spin_lock(&direct_window_list_lock); 1234 list_for_each_entry(window, &direct_window_list, list) { 1235 ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, 1236 arg->nr_pages, window->prop); 1237 /* XXX log error */ 1238 } 1239 spin_unlock(&direct_window_list_lock); 1240 break; 1241 default: 1242 break; 1243 } 1244 if (ret && action != MEM_CANCEL_ONLINE) 1245 return NOTIFY_BAD; 1246 1247 return NOTIFY_OK; 1248} 1249 1250static struct notifier_block iommu_mem_nb = { 1251 .notifier_call = iommu_mem_notifier, 1252}; 1253 1254static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data) 1255{ 1256 int err = NOTIFY_OK; 1257 struct of_reconfig_data *rd = data; 1258 struct device_node *np = rd->dn; 1259 struct pci_dn *pci = PCI_DN(np); 1260 struct direct_window *window; 1261 1262 switch (action) { 1263 case OF_RECONFIG_DETACH_NODE: 1264 /* 1265 * Removing the property will invoke the reconfig 1266 * notifier again, which causes dead-lock on the 1267 * read-write semaphore of the notifier chain. So 1268 * we have to remove the property when releasing 1269 * the device node. 1270 */ 1271 remove_ddw(np, false); 1272 if (pci && pci->iommu_table) 1273 iommu_free_table(pci->iommu_table, np->full_name); 1274 1275 spin_lock(&direct_window_list_lock); 1276 list_for_each_entry(window, &direct_window_list, list) { 1277 if (window->device == np) { 1278 list_del(&window->list); 1279 kfree(window); 1280 break; 1281 } 1282 } 1283 spin_unlock(&direct_window_list_lock); 1284 break; 1285 default: 1286 err = NOTIFY_DONE; 1287 break; 1288 } 1289 return err; 1290} 1291 1292static struct notifier_block iommu_reconfig_nb = { 1293 .notifier_call = iommu_reconfig_notifier, 1294}; 1295 1296/* These are called very early. */ 1297void iommu_init_early_pSeries(void) 1298{ 1299 if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL)) 1300 return; 1301 1302 if (firmware_has_feature(FW_FEATURE_LPAR)) { 1303 if (firmware_has_feature(FW_FEATURE_MULTITCE)) { 1304 ppc_md.tce_build = tce_buildmulti_pSeriesLP; 1305 ppc_md.tce_free = tce_freemulti_pSeriesLP; 1306 } else { 1307 ppc_md.tce_build = tce_build_pSeriesLP; 1308 ppc_md.tce_free = tce_free_pSeriesLP; 1309 } 1310 ppc_md.tce_get = tce_get_pSeriesLP; 1311 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP; 1312 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP; 1313 ppc_md.dma_set_mask = dma_set_mask_pSeriesLP; 1314 ppc_md.dma_get_required_mask = dma_get_required_mask_pSeriesLP; 1315 } else { 1316 ppc_md.tce_build = tce_build_pSeries; 1317 ppc_md.tce_free = tce_free_pSeries; 1318 ppc_md.tce_get = tce_get_pseries; 1319 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries; 1320 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries; 1321 } 1322 1323 1324 of_reconfig_notifier_register(&iommu_reconfig_nb); 1325 register_memory_notifier(&iommu_mem_nb); 1326 1327 set_pci_dma_ops(&dma_iommu_ops); 1328} 1329 1330static int __init disable_multitce(char *str) 1331{ 1332 if (strcmp(str, "off") == 0 && 1333 firmware_has_feature(FW_FEATURE_LPAR) && 1334 firmware_has_feature(FW_FEATURE_MULTITCE)) { 1335 printk(KERN_INFO "Disabling MULTITCE firmware feature\n"); 1336 ppc_md.tce_build = tce_build_pSeriesLP; 1337 ppc_md.tce_free = tce_free_pSeriesLP; 1338 powerpc_firmware_features &= ~FW_FEATURE_MULTITCE; 1339 } 1340 return 1; 1341} 1342 1343__setup("multitce=", disable_multitce); 1344 1345machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init); 1346