1/* 2 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation 3 * 4 * Rewrite, cleanup: 5 * 6 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation 7 * Copyright (C) 2006 Olof Johansson <olof@lixom.net> 8 * 9 * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR. 10 * 11 * 12 * This program is free software; you can redistribute it and/or modify 13 * it under the terms of the GNU General Public License as published by 14 * the Free Software Foundation; either version 2 of the License, or 15 * (at your option) any later version. 16 * 17 * This program is distributed in the hope that it will be useful, 18 * but WITHOUT ANY WARRANTY; without even the implied warranty of 19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 * GNU General Public License for more details. 21 * 22 * You should have received a copy of the GNU General Public License 23 * along with this program; if not, write to the Free Software 24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 25 */ 26 27#include <linux/init.h> 28#include <linux/types.h> 29#include <linux/slab.h> 30#include <linux/mm.h> 31#include <linux/memblock.h> 32#include <linux/spinlock.h> 33#include <linux/string.h> 34#include <linux/pci.h> 35#include <linux/dma-mapping.h> 36#include <linux/crash_dump.h> 37#include <linux/memory.h> 38#include <linux/of.h> 39#include <linux/iommu.h> 40#include <linux/rculist.h> 41#include <asm/io.h> 42#include <asm/prom.h> 43#include <asm/rtas.h> 44#include <asm/iommu.h> 45#include <asm/pci-bridge.h> 46#include <asm/machdep.h> 47#include <asm/firmware.h> 48#include <asm/tce.h> 49#include <asm/ppc-pci.h> 50#include <asm/udbg.h> 51#include <asm/mmzone.h> 52#include <asm/plpar_wrappers.h> 53 54#include "pseries.h" 55 56static struct iommu_table_group *iommu_pseries_alloc_group(int node) 57{ 58 struct iommu_table_group *table_group = NULL; 59 struct iommu_table *tbl = NULL; 60 struct iommu_table_group_link *tgl = NULL; 61 62 table_group = kzalloc_node(sizeof(struct iommu_table_group), GFP_KERNEL, 63 node); 64 if (!table_group) 65 goto fail_exit; 66 67 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node); 68 if (!tbl) 69 goto fail_exit; 70 71 tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL, 72 node); 73 if (!tgl) 74 goto fail_exit; 75 76 INIT_LIST_HEAD_RCU(&tbl->it_group_list); 77 tgl->table_group = table_group; 78 list_add_rcu(&tgl->next, &tbl->it_group_list); 79 80 table_group->tables[0] = tbl; 81 82 return table_group; 83 84fail_exit: 85 kfree(tgl); 86 kfree(table_group); 87 kfree(tbl); 88 89 return NULL; 90} 91 92static void iommu_pseries_free_group(struct iommu_table_group *table_group, 93 const char *node_name) 94{ 95 struct iommu_table *tbl; 96#ifdef CONFIG_IOMMU_API 97 struct iommu_table_group_link *tgl; 98#endif 99 100 if (!table_group) 101 return; 102 103 tbl = table_group->tables[0]; 104#ifdef CONFIG_IOMMU_API 105 tgl = list_first_entry_or_null(&tbl->it_group_list, 106 struct iommu_table_group_link, next); 107 108 WARN_ON_ONCE(!tgl); 109 if (tgl) { 110 list_del_rcu(&tgl->next); 111 kfree(tgl); 112 } 113 if (table_group->group) { 114 iommu_group_put(table_group->group); 115 BUG_ON(table_group->group); 116 } 117#endif 118 iommu_free_table(tbl, node_name); 119 120 kfree(table_group); 121} 122 123static void tce_invalidate_pSeries_sw(struct iommu_table *tbl, 124 __be64 *startp, __be64 *endp) 125{ 126 u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index; 127 unsigned long start, end, inc; 128 129 start = __pa(startp); 130 end = __pa(endp); 131 inc = L1_CACHE_BYTES; /* invalidate a cacheline of TCEs at a time */ 132 133 /* If this is non-zero, change the format. We shift the 134 * address and or in the magic from the device tree. */ 135 if (tbl->it_busno) { 136 start <<= 12; 137 end <<= 12; 138 inc <<= 12; 139 start |= tbl->it_busno; 140 end |= tbl->it_busno; 141 } 142 143 end |= inc - 1; /* round up end to be different than start */ 144 145 mb(); /* Make sure TCEs in memory are written */ 146 while (start <= end) { 147 out_be64(invalidate, start); 148 start += inc; 149 } 150} 151 152static int tce_build_pSeries(struct iommu_table *tbl, long index, 153 long npages, unsigned long uaddr, 154 enum dma_data_direction direction, 155 struct dma_attrs *attrs) 156{ 157 u64 proto_tce; 158 __be64 *tcep, *tces; 159 u64 rpn; 160 161 proto_tce = TCE_PCI_READ; // Read allowed 162 163 if (direction != DMA_TO_DEVICE) 164 proto_tce |= TCE_PCI_WRITE; 165 166 tces = tcep = ((__be64 *)tbl->it_base) + index; 167 168 while (npages--) { 169 /* can't move this out since we might cross MEMBLOCK boundary */ 170 rpn = __pa(uaddr) >> TCE_SHIFT; 171 *tcep = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT); 172 173 uaddr += TCE_PAGE_SIZE; 174 tcep++; 175 } 176 177 if (tbl->it_type & TCE_PCI_SWINV_CREATE) 178 tce_invalidate_pSeries_sw(tbl, tces, tcep - 1); 179 return 0; 180} 181 182 183static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages) 184{ 185 __be64 *tcep, *tces; 186 187 tces = tcep = ((__be64 *)tbl->it_base) + index; 188 189 while (npages--) 190 *(tcep++) = 0; 191 192 if (tbl->it_type & TCE_PCI_SWINV_FREE) 193 tce_invalidate_pSeries_sw(tbl, tces, tcep - 1); 194} 195 196static unsigned long tce_get_pseries(struct iommu_table *tbl, long index) 197{ 198 __be64 *tcep; 199 200 tcep = ((__be64 *)tbl->it_base) + index; 201 202 return be64_to_cpu(*tcep); 203} 204 205static void tce_free_pSeriesLP(struct iommu_table*, long, long); 206static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long); 207 208static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum, 209 long npages, unsigned long uaddr, 210 enum dma_data_direction direction, 211 struct dma_attrs *attrs) 212{ 213 u64 rc = 0; 214 u64 proto_tce, tce; 215 u64 rpn; 216 int ret = 0; 217 long tcenum_start = tcenum, npages_start = npages; 218 219 rpn = __pa(uaddr) >> TCE_SHIFT; 220 proto_tce = TCE_PCI_READ; 221 if (direction != DMA_TO_DEVICE) 222 proto_tce |= TCE_PCI_WRITE; 223 224 while (npages--) { 225 tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT; 226 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce); 227 228 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) { 229 ret = (int)rc; 230 tce_free_pSeriesLP(tbl, tcenum_start, 231 (npages_start - (npages + 1))); 232 break; 233 } 234 235 if (rc && printk_ratelimit()) { 236 printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 237 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 238 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 239 printk("\ttce val = 0x%llx\n", tce ); 240 dump_stack(); 241 } 242 243 tcenum++; 244 rpn++; 245 } 246 return ret; 247} 248 249static DEFINE_PER_CPU(__be64 *, tce_page); 250 251static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, 252 long npages, unsigned long uaddr, 253 enum dma_data_direction direction, 254 struct dma_attrs *attrs) 255{ 256 u64 rc = 0; 257 u64 proto_tce; 258 __be64 *tcep; 259 u64 rpn; 260 long l, limit; 261 long tcenum_start = tcenum, npages_start = npages; 262 int ret = 0; 263 unsigned long flags; 264 265 if ((npages == 1) || !firmware_has_feature(FW_FEATURE_MULTITCE)) { 266 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr, 267 direction, attrs); 268 } 269 270 local_irq_save(flags); /* to protect tcep and the page behind it */ 271 272 tcep = __this_cpu_read(tce_page); 273 274 /* This is safe to do since interrupts are off when we're called 275 * from iommu_alloc{,_sg}() 276 */ 277 if (!tcep) { 278 tcep = (__be64 *)__get_free_page(GFP_ATOMIC); 279 /* If allocation fails, fall back to the loop implementation */ 280 if (!tcep) { 281 local_irq_restore(flags); 282 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr, 283 direction, attrs); 284 } 285 __this_cpu_write(tce_page, tcep); 286 } 287 288 rpn = __pa(uaddr) >> TCE_SHIFT; 289 proto_tce = TCE_PCI_READ; 290 if (direction != DMA_TO_DEVICE) 291 proto_tce |= TCE_PCI_WRITE; 292 293 /* We can map max one pageful of TCEs at a time */ 294 do { 295 /* 296 * Set up the page with TCE data, looping through and setting 297 * the values. 298 */ 299 limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE); 300 301 for (l = 0; l < limit; l++) { 302 tcep[l] = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT); 303 rpn++; 304 } 305 306 rc = plpar_tce_put_indirect((u64)tbl->it_index, 307 (u64)tcenum << 12, 308 (u64)__pa(tcep), 309 limit); 310 311 npages -= limit; 312 tcenum += limit; 313 } while (npages > 0 && !rc); 314 315 local_irq_restore(flags); 316 317 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) { 318 ret = (int)rc; 319 tce_freemulti_pSeriesLP(tbl, tcenum_start, 320 (npages_start - (npages + limit))); 321 return ret; 322 } 323 324 if (rc && printk_ratelimit()) { 325 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 326 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 327 printk("\tnpages = 0x%llx\n", (u64)npages); 328 printk("\ttce[0] val = 0x%llx\n", tcep[0]); 329 dump_stack(); 330 } 331 return ret; 332} 333 334static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages) 335{ 336 u64 rc; 337 338 while (npages--) { 339 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0); 340 341 if (rc && printk_ratelimit()) { 342 printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 343 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 344 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 345 dump_stack(); 346 } 347 348 tcenum++; 349 } 350} 351 352 353static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages) 354{ 355 u64 rc; 356 357 if (!firmware_has_feature(FW_FEATURE_MULTITCE)) 358 return tce_free_pSeriesLP(tbl, tcenum, npages); 359 360 rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages); 361 362 if (rc && printk_ratelimit()) { 363 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n"); 364 printk("\trc = %lld\n", rc); 365 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 366 printk("\tnpages = 0x%llx\n", (u64)npages); 367 dump_stack(); 368 } 369} 370 371static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum) 372{ 373 u64 rc; 374 unsigned long tce_ret; 375 376 rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret); 377 378 if (rc && printk_ratelimit()) { 379 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc); 380 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 381 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 382 dump_stack(); 383 } 384 385 return tce_ret; 386} 387 388/* this is compatible with cells for the device tree property */ 389struct dynamic_dma_window_prop { 390 __be32 liobn; /* tce table number */ 391 __be64 dma_base; /* address hi,lo */ 392 __be32 tce_shift; /* ilog2(tce_page_size) */ 393 __be32 window_shift; /* ilog2(tce_window_size) */ 394}; 395 396struct direct_window { 397 struct device_node *device; 398 const struct dynamic_dma_window_prop *prop; 399 struct list_head list; 400}; 401 402/* Dynamic DMA Window support */ 403struct ddw_query_response { 404 u32 windows_available; 405 u32 largest_available_block; 406 u32 page_size; 407 u32 migration_capable; 408}; 409 410struct ddw_create_response { 411 u32 liobn; 412 u32 addr_hi; 413 u32 addr_lo; 414}; 415 416static LIST_HEAD(direct_window_list); 417/* prevents races between memory on/offline and window creation */ 418static DEFINE_SPINLOCK(direct_window_list_lock); 419/* protects initializing window twice for same device */ 420static DEFINE_MUTEX(direct_window_init_mutex); 421#define DIRECT64_PROPNAME "linux,direct64-ddr-window-info" 422 423static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn, 424 unsigned long num_pfn, const void *arg) 425{ 426 const struct dynamic_dma_window_prop *maprange = arg; 427 int rc; 428 u64 tce_size, num_tce, dma_offset, next; 429 u32 tce_shift; 430 long limit; 431 432 tce_shift = be32_to_cpu(maprange->tce_shift); 433 tce_size = 1ULL << tce_shift; 434 next = start_pfn << PAGE_SHIFT; 435 num_tce = num_pfn << PAGE_SHIFT; 436 437 /* round back to the beginning of the tce page size */ 438 num_tce += next & (tce_size - 1); 439 next &= ~(tce_size - 1); 440 441 /* covert to number of tces */ 442 num_tce |= tce_size - 1; 443 num_tce >>= tce_shift; 444 445 do { 446 /* 447 * Set up the page with TCE data, looping through and setting 448 * the values. 449 */ 450 limit = min_t(long, num_tce, 512); 451 dma_offset = next + be64_to_cpu(maprange->dma_base); 452 453 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn), 454 dma_offset, 455 0, limit); 456 next += limit * tce_size; 457 num_tce -= limit; 458 } while (num_tce > 0 && !rc); 459 460 return rc; 461} 462 463static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, 464 unsigned long num_pfn, const void *arg) 465{ 466 const struct dynamic_dma_window_prop *maprange = arg; 467 u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn; 468 __be64 *tcep; 469 u32 tce_shift; 470 u64 rc = 0; 471 long l, limit; 472 473 local_irq_disable(); /* to protect tcep and the page behind it */ 474 tcep = __this_cpu_read(tce_page); 475 476 if (!tcep) { 477 tcep = (__be64 *)__get_free_page(GFP_ATOMIC); 478 if (!tcep) { 479 local_irq_enable(); 480 return -ENOMEM; 481 } 482 __this_cpu_write(tce_page, tcep); 483 } 484 485 proto_tce = TCE_PCI_READ | TCE_PCI_WRITE; 486 487 liobn = (u64)be32_to_cpu(maprange->liobn); 488 tce_shift = be32_to_cpu(maprange->tce_shift); 489 tce_size = 1ULL << tce_shift; 490 next = start_pfn << PAGE_SHIFT; 491 num_tce = num_pfn << PAGE_SHIFT; 492 493 /* round back to the beginning of the tce page size */ 494 num_tce += next & (tce_size - 1); 495 next &= ~(tce_size - 1); 496 497 /* covert to number of tces */ 498 num_tce |= tce_size - 1; 499 num_tce >>= tce_shift; 500 501 /* We can map max one pageful of TCEs at a time */ 502 do { 503 /* 504 * Set up the page with TCE data, looping through and setting 505 * the values. 506 */ 507 limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE); 508 dma_offset = next + be64_to_cpu(maprange->dma_base); 509 510 for (l = 0; l < limit; l++) { 511 tcep[l] = cpu_to_be64(proto_tce | next); 512 next += tce_size; 513 } 514 515 rc = plpar_tce_put_indirect(liobn, 516 dma_offset, 517 (u64)__pa(tcep), 518 limit); 519 520 num_tce -= limit; 521 } while (num_tce > 0 && !rc); 522 523 /* error cleanup: caller will clear whole range */ 524 525 local_irq_enable(); 526 return rc; 527} 528 529static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn, 530 unsigned long num_pfn, void *arg) 531{ 532 return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg); 533} 534 535static void iommu_table_setparms(struct pci_controller *phb, 536 struct device_node *dn, 537 struct iommu_table *tbl) 538{ 539 struct device_node *node; 540 const unsigned long *basep, *sw_inval; 541 const u32 *sizep; 542 543 node = phb->dn; 544 545 basep = of_get_property(node, "linux,tce-base", NULL); 546 sizep = of_get_property(node, "linux,tce-size", NULL); 547 if (basep == NULL || sizep == NULL) { 548 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has " 549 "missing tce entries !\n", dn->full_name); 550 return; 551 } 552 553 tbl->it_base = (unsigned long)__va(*basep); 554 555 if (!is_kdump_kernel()) 556 memset((void *)tbl->it_base, 0, *sizep); 557 558 tbl->it_busno = phb->bus->number; 559 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K; 560 561 /* Units of tce entries */ 562 tbl->it_offset = phb->dma_window_base_cur >> tbl->it_page_shift; 563 564 /* Test if we are going over 2GB of DMA space */ 565 if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) { 566 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n"); 567 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n"); 568 } 569 570 phb->dma_window_base_cur += phb->dma_window_size; 571 572 /* Set the tce table size - measured in entries */ 573 tbl->it_size = phb->dma_window_size >> tbl->it_page_shift; 574 575 tbl->it_index = 0; 576 tbl->it_blocksize = 16; 577 tbl->it_type = TCE_PCI; 578 579 sw_inval = of_get_property(node, "linux,tce-sw-invalidate-info", NULL); 580 if (sw_inval) { 581 /* 582 * This property contains information on how to 583 * invalidate the TCE entry. The first property is 584 * the base MMIO address used to invalidate entries. 585 * The second property tells us the format of the TCE 586 * invalidate (whether it needs to be shifted) and 587 * some magic routing info to add to our invalidate 588 * command. 589 */ 590 tbl->it_index = (unsigned long) ioremap(sw_inval[0], 8); 591 tbl->it_busno = sw_inval[1]; /* overload this with magic */ 592 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE; 593 } 594} 595 596/* 597 * iommu_table_setparms_lpar 598 * 599 * Function: On pSeries LPAR systems, return TCE table info, given a pci bus. 600 */ 601static void iommu_table_setparms_lpar(struct pci_controller *phb, 602 struct device_node *dn, 603 struct iommu_table *tbl, 604 const __be32 *dma_window) 605{ 606 unsigned long offset, size; 607 608 of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size); 609 610 tbl->it_busno = phb->bus->number; 611 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K; 612 tbl->it_base = 0; 613 tbl->it_blocksize = 16; 614 tbl->it_type = TCE_PCI; 615 tbl->it_offset = offset >> tbl->it_page_shift; 616 tbl->it_size = size >> tbl->it_page_shift; 617} 618 619struct iommu_table_ops iommu_table_pseries_ops = { 620 .set = tce_build_pSeries, 621 .clear = tce_free_pSeries, 622 .get = tce_get_pseries 623}; 624 625static void pci_dma_bus_setup_pSeries(struct pci_bus *bus) 626{ 627 struct device_node *dn; 628 struct iommu_table *tbl; 629 struct device_node *isa_dn, *isa_dn_orig; 630 struct device_node *tmp; 631 struct pci_dn *pci; 632 int children; 633 634 dn = pci_bus_to_OF_node(bus); 635 636 pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn->full_name); 637 638 if (bus->self) { 639 /* This is not a root bus, any setup will be done for the 640 * device-side of the bridge in iommu_dev_setup_pSeries(). 641 */ 642 return; 643 } 644 pci = PCI_DN(dn); 645 646 /* Check if the ISA bus on the system is under 647 * this PHB. 648 */ 649 isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa"); 650 651 while (isa_dn && isa_dn != dn) 652 isa_dn = isa_dn->parent; 653 654 of_node_put(isa_dn_orig); 655 656 /* Count number of direct PCI children of the PHB. */ 657 for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling) 658 children++; 659 660 pr_debug("Children: %d\n", children); 661 662 /* Calculate amount of DMA window per slot. Each window must be 663 * a power of two (due to pci_alloc_consistent requirements). 664 * 665 * Keep 256MB aside for PHBs with ISA. 666 */ 667 668 if (!isa_dn) { 669 /* No ISA/IDE - just set window size and return */ 670 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */ 671 672 while (pci->phb->dma_window_size * children > 0x80000000ul) 673 pci->phb->dma_window_size >>= 1; 674 pr_debug("No ISA/IDE, window size is 0x%llx\n", 675 pci->phb->dma_window_size); 676 pci->phb->dma_window_base_cur = 0; 677 678 return; 679 } 680 681 /* If we have ISA, then we probably have an IDE 682 * controller too. Allocate a 128MB table but 683 * skip the first 128MB to avoid stepping on ISA 684 * space. 685 */ 686 pci->phb->dma_window_size = 0x8000000ul; 687 pci->phb->dma_window_base_cur = 0x8000000ul; 688 689 pci->table_group = iommu_pseries_alloc_group(pci->phb->node); 690 tbl = pci->table_group->tables[0]; 691 692 iommu_table_setparms(pci->phb, dn, tbl); 693 tbl->it_ops = &iommu_table_pseries_ops; 694 iommu_init_table(tbl, pci->phb->node); 695 iommu_register_group(pci->table_group, pci_domain_nr(bus), 0); 696 697 /* Divide the rest (1.75GB) among the children */ 698 pci->phb->dma_window_size = 0x80000000ul; 699 while (pci->phb->dma_window_size * children > 0x70000000ul) 700 pci->phb->dma_window_size >>= 1; 701 702 pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size); 703} 704 705struct iommu_table_ops iommu_table_lpar_multi_ops = { 706 .set = tce_buildmulti_pSeriesLP, 707 .clear = tce_freemulti_pSeriesLP, 708 .get = tce_get_pSeriesLP 709}; 710 711static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus) 712{ 713 struct iommu_table *tbl; 714 struct device_node *dn, *pdn; 715 struct pci_dn *ppci; 716 const __be32 *dma_window = NULL; 717 718 dn = pci_bus_to_OF_node(bus); 719 720 pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n", 721 dn->full_name); 722 723 /* Find nearest ibm,dma-window, walking up the device tree */ 724 for (pdn = dn; pdn != NULL; pdn = pdn->parent) { 725 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 726 if (dma_window != NULL) 727 break; 728 } 729 730 if (dma_window == NULL) { 731 pr_debug(" no ibm,dma-window property !\n"); 732 return; 733 } 734 735 ppci = PCI_DN(pdn); 736 737 pr_debug(" parent is %s, iommu_table: 0x%p\n", 738 pdn->full_name, ppci->table_group); 739 740 if (!ppci->table_group) { 741 ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node); 742 tbl = ppci->table_group->tables[0]; 743 iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window); 744 tbl->it_ops = &iommu_table_lpar_multi_ops; 745 iommu_init_table(tbl, ppci->phb->node); 746 iommu_register_group(ppci->table_group, 747 pci_domain_nr(bus), 0); 748 pr_debug(" created table: %p\n", ppci->table_group); 749 } 750} 751 752 753static void pci_dma_dev_setup_pSeries(struct pci_dev *dev) 754{ 755 struct device_node *dn; 756 struct iommu_table *tbl; 757 758 pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev)); 759 760 dn = dev->dev.of_node; 761 762 /* If we're the direct child of a root bus, then we need to allocate 763 * an iommu table ourselves. The bus setup code should have setup 764 * the window sizes already. 765 */ 766 if (!dev->bus->self) { 767 struct pci_controller *phb = PCI_DN(dn)->phb; 768 769 pr_debug(" --> first child, no bridge. Allocating iommu table.\n"); 770 PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node); 771 tbl = PCI_DN(dn)->table_group->tables[0]; 772 iommu_table_setparms(phb, dn, tbl); 773 tbl->it_ops = &iommu_table_pseries_ops; 774 iommu_init_table(tbl, phb->node); 775 iommu_register_group(PCI_DN(dn)->table_group, 776 pci_domain_nr(phb->bus), 0); 777 set_iommu_table_base(&dev->dev, tbl); 778 iommu_add_device(&dev->dev); 779 return; 780 } 781 782 /* If this device is further down the bus tree, search upwards until 783 * an already allocated iommu table is found and use that. 784 */ 785 786 while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL) 787 dn = dn->parent; 788 789 if (dn && PCI_DN(dn)) { 790 set_iommu_table_base(&dev->dev, 791 PCI_DN(dn)->table_group->tables[0]); 792 iommu_add_device(&dev->dev); 793 } else 794 printk(KERN_WARNING "iommu: Device %s has no iommu table\n", 795 pci_name(dev)); 796} 797 798static int __read_mostly disable_ddw; 799 800static int __init disable_ddw_setup(char *str) 801{ 802 disable_ddw = 1; 803 printk(KERN_INFO "ppc iommu: disabling ddw.\n"); 804 805 return 0; 806} 807 808early_param("disable_ddw", disable_ddw_setup); 809 810static void remove_ddw(struct device_node *np, bool remove_prop) 811{ 812 struct dynamic_dma_window_prop *dwp; 813 struct property *win64; 814 u32 ddw_avail[3]; 815 u64 liobn; 816 int ret = 0; 817 818 ret = of_property_read_u32_array(np, "ibm,ddw-applicable", 819 &ddw_avail[0], 3); 820 821 win64 = of_find_property(np, DIRECT64_PROPNAME, NULL); 822 if (!win64) 823 return; 824 825 if (ret || win64->length < sizeof(*dwp)) 826 goto delprop; 827 828 dwp = win64->value; 829 liobn = (u64)be32_to_cpu(dwp->liobn); 830 831 /* clear the whole window, note the arg is in kernel pages */ 832 ret = tce_clearrange_multi_pSeriesLP(0, 833 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp); 834 if (ret) 835 pr_warning("%s failed to clear tces in window.\n", 836 np->full_name); 837 else 838 pr_debug("%s successfully cleared tces in window.\n", 839 np->full_name); 840 841 ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn); 842 if (ret) 843 pr_warning("%s: failed to remove direct window: rtas returned " 844 "%d to ibm,remove-pe-dma-window(%x) %llx\n", 845 np->full_name, ret, ddw_avail[2], liobn); 846 else 847 pr_debug("%s: successfully removed direct window: rtas returned " 848 "%d to ibm,remove-pe-dma-window(%x) %llx\n", 849 np->full_name, ret, ddw_avail[2], liobn); 850 851delprop: 852 if (remove_prop) 853 ret = of_remove_property(np, win64); 854 if (ret) 855 pr_warning("%s: failed to remove direct window property: %d\n", 856 np->full_name, ret); 857} 858 859static u64 find_existing_ddw(struct device_node *pdn) 860{ 861 struct direct_window *window; 862 const struct dynamic_dma_window_prop *direct64; 863 u64 dma_addr = 0; 864 865 spin_lock(&direct_window_list_lock); 866 /* check if we already created a window and dupe that config if so */ 867 list_for_each_entry(window, &direct_window_list, list) { 868 if (window->device == pdn) { 869 direct64 = window->prop; 870 dma_addr = be64_to_cpu(direct64->dma_base); 871 break; 872 } 873 } 874 spin_unlock(&direct_window_list_lock); 875 876 return dma_addr; 877} 878 879static int find_existing_ddw_windows(void) 880{ 881 int len; 882 struct device_node *pdn; 883 struct direct_window *window; 884 const struct dynamic_dma_window_prop *direct64; 885 886 if (!firmware_has_feature(FW_FEATURE_LPAR)) 887 return 0; 888 889 for_each_node_with_property(pdn, DIRECT64_PROPNAME) { 890 direct64 = of_get_property(pdn, DIRECT64_PROPNAME, &len); 891 if (!direct64) 892 continue; 893 894 window = kzalloc(sizeof(*window), GFP_KERNEL); 895 if (!window || len < sizeof(struct dynamic_dma_window_prop)) { 896 kfree(window); 897 remove_ddw(pdn, true); 898 continue; 899 } 900 901 window->device = pdn; 902 window->prop = direct64; 903 spin_lock(&direct_window_list_lock); 904 list_add(&window->list, &direct_window_list); 905 spin_unlock(&direct_window_list_lock); 906 } 907 908 return 0; 909} 910machine_arch_initcall(pseries, find_existing_ddw_windows); 911 912static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail, 913 struct ddw_query_response *query) 914{ 915 struct eeh_dev *edev; 916 u32 cfg_addr; 917 u64 buid; 918 int ret; 919 920 /* 921 * Get the config address and phb buid of the PE window. 922 * Rely on eeh to retrieve this for us. 923 * Retrieve them from the pci device, not the node with the 924 * dma-window property 925 */ 926 edev = pci_dev_to_eeh_dev(dev); 927 cfg_addr = edev->config_addr; 928 if (edev->pe_config_addr) 929 cfg_addr = edev->pe_config_addr; 930 buid = edev->phb->buid; 931 932 ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query, 933 cfg_addr, BUID_HI(buid), BUID_LO(buid)); 934 dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x" 935 " returned %d\n", ddw_avail[0], cfg_addr, BUID_HI(buid), 936 BUID_LO(buid), ret); 937 return ret; 938} 939 940static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail, 941 struct ddw_create_response *create, int page_shift, 942 int window_shift) 943{ 944 struct eeh_dev *edev; 945 u32 cfg_addr; 946 u64 buid; 947 int ret; 948 949 /* 950 * Get the config address and phb buid of the PE window. 951 * Rely on eeh to retrieve this for us. 952 * Retrieve them from the pci device, not the node with the 953 * dma-window property 954 */ 955 edev = pci_dev_to_eeh_dev(dev); 956 cfg_addr = edev->config_addr; 957 if (edev->pe_config_addr) 958 cfg_addr = edev->pe_config_addr; 959 buid = edev->phb->buid; 960 961 do { 962 /* extra outputs are LIOBN and dma-addr (hi, lo) */ 963 ret = rtas_call(ddw_avail[1], 5, 4, (u32 *)create, 964 cfg_addr, BUID_HI(buid), BUID_LO(buid), 965 page_shift, window_shift); 966 } while (rtas_busy_delay(ret)); 967 dev_info(&dev->dev, 968 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d " 969 "(liobn = 0x%x starting addr = %x %x)\n", ddw_avail[1], 970 cfg_addr, BUID_HI(buid), BUID_LO(buid), page_shift, 971 window_shift, ret, create->liobn, create->addr_hi, create->addr_lo); 972 973 return ret; 974} 975 976struct failed_ddw_pdn { 977 struct device_node *pdn; 978 struct list_head list; 979}; 980 981static LIST_HEAD(failed_ddw_pdn_list); 982 983/* 984 * If the PE supports dynamic dma windows, and there is space for a table 985 * that can map all pages in a linear offset, then setup such a table, 986 * and record the dma-offset in the struct device. 987 * 988 * dev: the pci device we are checking 989 * pdn: the parent pe node with the ibm,dma_window property 990 * Future: also check if we can remap the base window for our base page size 991 * 992 * returns the dma offset for use by dma_set_mask 993 */ 994static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn) 995{ 996 int len, ret; 997 struct ddw_query_response query; 998 struct ddw_create_response create; 999 int page_shift; 1000 u64 dma_addr, max_addr; 1001 struct device_node *dn; 1002 u32 ddw_avail[3]; 1003 struct direct_window *window; 1004 struct property *win64; 1005 struct dynamic_dma_window_prop *ddwprop; 1006 struct failed_ddw_pdn *fpdn; 1007 1008 mutex_lock(&direct_window_init_mutex); 1009 1010 dma_addr = find_existing_ddw(pdn); 1011 if (dma_addr != 0) 1012 goto out_unlock; 1013 1014 /* 1015 * If we already went through this for a previous function of 1016 * the same device and failed, we don't want to muck with the 1017 * DMA window again, as it will race with in-flight operations 1018 * and can lead to EEHs. The above mutex protects access to the 1019 * list. 1020 */ 1021 list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) { 1022 if (!strcmp(fpdn->pdn->full_name, pdn->full_name)) 1023 goto out_unlock; 1024 } 1025 1026 /* 1027 * the ibm,ddw-applicable property holds the tokens for: 1028 * ibm,query-pe-dma-window 1029 * ibm,create-pe-dma-window 1030 * ibm,remove-pe-dma-window 1031 * for the given node in that order. 1032 * the property is actually in the parent, not the PE 1033 */ 1034 ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable", 1035 &ddw_avail[0], 3); 1036 if (ret) 1037 goto out_failed; 1038 1039 /* 1040 * Query if there is a second window of size to map the 1041 * whole partition. Query returns number of windows, largest 1042 * block assigned to PE (partition endpoint), and two bitmasks 1043 * of page sizes: supported and supported for migrate-dma. 1044 */ 1045 dn = pci_device_to_OF_node(dev); 1046 ret = query_ddw(dev, ddw_avail, &query); 1047 if (ret != 0) 1048 goto out_failed; 1049 1050 if (query.windows_available == 0) { 1051 /* 1052 * no additional windows are available for this device. 1053 * We might be able to reallocate the existing window, 1054 * trading in for a larger page size. 1055 */ 1056 dev_dbg(&dev->dev, "no free dynamic windows"); 1057 goto out_failed; 1058 } 1059 if (query.page_size & 4) { 1060 page_shift = 24; /* 16MB */ 1061 } else if (query.page_size & 2) { 1062 page_shift = 16; /* 64kB */ 1063 } else if (query.page_size & 1) { 1064 page_shift = 12; /* 4kB */ 1065 } else { 1066 dev_dbg(&dev->dev, "no supported direct page size in mask %x", 1067 query.page_size); 1068 goto out_failed; 1069 } 1070 /* verify the window * number of ptes will map the partition */ 1071 /* check largest block * page size > max memory hotplug addr */ 1072 max_addr = memory_hotplug_max(); 1073 if (query.largest_available_block < (max_addr >> page_shift)) { 1074 dev_dbg(&dev->dev, "can't map partiton max 0x%llx with %u " 1075 "%llu-sized pages\n", max_addr, query.largest_available_block, 1076 1ULL << page_shift); 1077 goto out_failed; 1078 } 1079 len = order_base_2(max_addr); 1080 win64 = kzalloc(sizeof(struct property), GFP_KERNEL); 1081 if (!win64) { 1082 dev_info(&dev->dev, 1083 "couldn't allocate property for 64bit dma window\n"); 1084 goto out_failed; 1085 } 1086 win64->name = kstrdup(DIRECT64_PROPNAME, GFP_KERNEL); 1087 win64->value = ddwprop = kmalloc(sizeof(*ddwprop), GFP_KERNEL); 1088 win64->length = sizeof(*ddwprop); 1089 if (!win64->name || !win64->value) { 1090 dev_info(&dev->dev, 1091 "couldn't allocate property name and value\n"); 1092 goto out_free_prop; 1093 } 1094 1095 ret = create_ddw(dev, ddw_avail, &create, page_shift, len); 1096 if (ret != 0) 1097 goto out_free_prop; 1098 1099 ddwprop->liobn = cpu_to_be32(create.liobn); 1100 ddwprop->dma_base = cpu_to_be64(((u64)create.addr_hi << 32) | 1101 create.addr_lo); 1102 ddwprop->tce_shift = cpu_to_be32(page_shift); 1103 ddwprop->window_shift = cpu_to_be32(len); 1104 1105 dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %s\n", 1106 create.liobn, dn->full_name); 1107 1108 window = kzalloc(sizeof(*window), GFP_KERNEL); 1109 if (!window) 1110 goto out_clear_window; 1111 1112 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT, 1113 win64->value, tce_setrange_multi_pSeriesLP_walk); 1114 if (ret) { 1115 dev_info(&dev->dev, "failed to map direct window for %s: %d\n", 1116 dn->full_name, ret); 1117 goto out_free_window; 1118 } 1119 1120 ret = of_add_property(pdn, win64); 1121 if (ret) { 1122 dev_err(&dev->dev, "unable to add dma window property for %s: %d", 1123 pdn->full_name, ret); 1124 goto out_free_window; 1125 } 1126 1127 window->device = pdn; 1128 window->prop = ddwprop; 1129 spin_lock(&direct_window_list_lock); 1130 list_add(&window->list, &direct_window_list); 1131 spin_unlock(&direct_window_list_lock); 1132 1133 dma_addr = be64_to_cpu(ddwprop->dma_base); 1134 goto out_unlock; 1135 1136out_free_window: 1137 kfree(window); 1138 1139out_clear_window: 1140 remove_ddw(pdn, true); 1141 1142out_free_prop: 1143 kfree(win64->name); 1144 kfree(win64->value); 1145 kfree(win64); 1146 1147out_failed: 1148 1149 fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL); 1150 if (!fpdn) 1151 goto out_unlock; 1152 fpdn->pdn = pdn; 1153 list_add(&fpdn->list, &failed_ddw_pdn_list); 1154 1155out_unlock: 1156 mutex_unlock(&direct_window_init_mutex); 1157 return dma_addr; 1158} 1159 1160static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev) 1161{ 1162 struct device_node *pdn, *dn; 1163 struct iommu_table *tbl; 1164 const __be32 *dma_window = NULL; 1165 struct pci_dn *pci; 1166 1167 pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev)); 1168 1169 /* dev setup for LPAR is a little tricky, since the device tree might 1170 * contain the dma-window properties per-device and not necessarily 1171 * for the bus. So we need to search upwards in the tree until we 1172 * either hit a dma-window property, OR find a parent with a table 1173 * already allocated. 1174 */ 1175 dn = pci_device_to_OF_node(dev); 1176 pr_debug(" node is %s\n", dn->full_name); 1177 1178 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group; 1179 pdn = pdn->parent) { 1180 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 1181 if (dma_window) 1182 break; 1183 } 1184 1185 if (!pdn || !PCI_DN(pdn)) { 1186 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: " 1187 "no DMA window found for pci dev=%s dn=%s\n", 1188 pci_name(dev), of_node_full_name(dn)); 1189 return; 1190 } 1191 pr_debug(" parent is %s\n", pdn->full_name); 1192 1193 pci = PCI_DN(pdn); 1194 if (!pci->table_group) { 1195 pci->table_group = iommu_pseries_alloc_group(pci->phb->node); 1196 tbl = pci->table_group->tables[0]; 1197 iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window); 1198 tbl->it_ops = &iommu_table_lpar_multi_ops; 1199 iommu_init_table(tbl, pci->phb->node); 1200 iommu_register_group(pci->table_group, 1201 pci_domain_nr(pci->phb->bus), 0); 1202 pr_debug(" created table: %p\n", pci->table_group); 1203 } else { 1204 pr_debug(" found DMA window, table: %p\n", pci->table_group); 1205 } 1206 1207 set_iommu_table_base(&dev->dev, pci->table_group->tables[0]); 1208 iommu_add_device(&dev->dev); 1209} 1210 1211static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask) 1212{ 1213 bool ddw_enabled = false; 1214 struct device_node *pdn, *dn; 1215 struct pci_dev *pdev; 1216 const __be32 *dma_window = NULL; 1217 u64 dma_offset; 1218 1219 if (!dev->dma_mask) 1220 return -EIO; 1221 1222 if (!dev_is_pci(dev)) 1223 goto check_mask; 1224 1225 pdev = to_pci_dev(dev); 1226 1227 /* only attempt to use a new window if 64-bit DMA is requested */ 1228 if (!disable_ddw && dma_mask == DMA_BIT_MASK(64)) { 1229 dn = pci_device_to_OF_node(pdev); 1230 dev_dbg(dev, "node is %s\n", dn->full_name); 1231 1232 /* 1233 * the device tree might contain the dma-window properties 1234 * per-device and not necessarily for the bus. So we need to 1235 * search upwards in the tree until we either hit a dma-window 1236 * property, OR find a parent with a table already allocated. 1237 */ 1238 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group; 1239 pdn = pdn->parent) { 1240 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 1241 if (dma_window) 1242 break; 1243 } 1244 if (pdn && PCI_DN(pdn)) { 1245 dma_offset = enable_ddw(pdev, pdn); 1246 if (dma_offset != 0) { 1247 dev_info(dev, "Using 64-bit direct DMA at offset %llx\n", dma_offset); 1248 set_dma_offset(dev, dma_offset); 1249 set_dma_ops(dev, &dma_direct_ops); 1250 ddw_enabled = true; 1251 } 1252 } 1253 } 1254 1255 /* fall back on iommu ops */ 1256 if (!ddw_enabled && get_dma_ops(dev) != &dma_iommu_ops) { 1257 dev_info(dev, "Restoring 32-bit DMA via iommu\n"); 1258 set_dma_ops(dev, &dma_iommu_ops); 1259 } 1260 1261check_mask: 1262 if (!dma_supported(dev, dma_mask)) 1263 return -EIO; 1264 1265 *dev->dma_mask = dma_mask; 1266 return 0; 1267} 1268 1269static u64 dma_get_required_mask_pSeriesLP(struct device *dev) 1270{ 1271 if (!dev->dma_mask) 1272 return 0; 1273 1274 if (!disable_ddw && dev_is_pci(dev)) { 1275 struct pci_dev *pdev = to_pci_dev(dev); 1276 struct device_node *dn; 1277 1278 dn = pci_device_to_OF_node(pdev); 1279 1280 /* search upwards for ibm,dma-window */ 1281 for (; dn && PCI_DN(dn) && !PCI_DN(dn)->table_group; 1282 dn = dn->parent) 1283 if (of_get_property(dn, "ibm,dma-window", NULL)) 1284 break; 1285 /* if there is a ibm,ddw-applicable property require 64 bits */ 1286 if (dn && PCI_DN(dn) && 1287 of_get_property(dn, "ibm,ddw-applicable", NULL)) 1288 return DMA_BIT_MASK(64); 1289 } 1290 1291 return dma_iommu_ops.get_required_mask(dev); 1292} 1293 1294static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, 1295 void *data) 1296{ 1297 struct direct_window *window; 1298 struct memory_notify *arg = data; 1299 int ret = 0; 1300 1301 switch (action) { 1302 case MEM_GOING_ONLINE: 1303 spin_lock(&direct_window_list_lock); 1304 list_for_each_entry(window, &direct_window_list, list) { 1305 ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, 1306 arg->nr_pages, window->prop); 1307 /* XXX log error */ 1308 } 1309 spin_unlock(&direct_window_list_lock); 1310 break; 1311 case MEM_CANCEL_ONLINE: 1312 case MEM_OFFLINE: 1313 spin_lock(&direct_window_list_lock); 1314 list_for_each_entry(window, &direct_window_list, list) { 1315 ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, 1316 arg->nr_pages, window->prop); 1317 /* XXX log error */ 1318 } 1319 spin_unlock(&direct_window_list_lock); 1320 break; 1321 default: 1322 break; 1323 } 1324 if (ret && action != MEM_CANCEL_ONLINE) 1325 return NOTIFY_BAD; 1326 1327 return NOTIFY_OK; 1328} 1329 1330static struct notifier_block iommu_mem_nb = { 1331 .notifier_call = iommu_mem_notifier, 1332}; 1333 1334static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data) 1335{ 1336 int err = NOTIFY_OK; 1337 struct of_reconfig_data *rd = data; 1338 struct device_node *np = rd->dn; 1339 struct pci_dn *pci = PCI_DN(np); 1340 struct direct_window *window; 1341 1342 switch (action) { 1343 case OF_RECONFIG_DETACH_NODE: 1344 /* 1345 * Removing the property will invoke the reconfig 1346 * notifier again, which causes dead-lock on the 1347 * read-write semaphore of the notifier chain. So 1348 * we have to remove the property when releasing 1349 * the device node. 1350 */ 1351 remove_ddw(np, false); 1352 if (pci && pci->table_group) 1353 iommu_pseries_free_group(pci->table_group, 1354 np->full_name); 1355 1356 spin_lock(&direct_window_list_lock); 1357 list_for_each_entry(window, &direct_window_list, list) { 1358 if (window->device == np) { 1359 list_del(&window->list); 1360 kfree(window); 1361 break; 1362 } 1363 } 1364 spin_unlock(&direct_window_list_lock); 1365 break; 1366 default: 1367 err = NOTIFY_DONE; 1368 break; 1369 } 1370 return err; 1371} 1372 1373static struct notifier_block iommu_reconfig_nb = { 1374 .notifier_call = iommu_reconfig_notifier, 1375}; 1376 1377/* These are called very early. */ 1378void iommu_init_early_pSeries(void) 1379{ 1380 if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL)) 1381 return; 1382 1383 if (firmware_has_feature(FW_FEATURE_LPAR)) { 1384 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP; 1385 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP; 1386 ppc_md.dma_set_mask = dma_set_mask_pSeriesLP; 1387 ppc_md.dma_get_required_mask = dma_get_required_mask_pSeriesLP; 1388 } else { 1389 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries; 1390 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries; 1391 } 1392 1393 1394 of_reconfig_notifier_register(&iommu_reconfig_nb); 1395 register_memory_notifier(&iommu_mem_nb); 1396 1397 set_pci_dma_ops(&dma_iommu_ops); 1398} 1399 1400static int __init disable_multitce(char *str) 1401{ 1402 if (strcmp(str, "off") == 0 && 1403 firmware_has_feature(FW_FEATURE_LPAR) && 1404 firmware_has_feature(FW_FEATURE_MULTITCE)) { 1405 printk(KERN_INFO "Disabling MULTITCE firmware feature\n"); 1406 powerpc_firmware_features &= ~FW_FEATURE_MULTITCE; 1407 } 1408 return 1; 1409} 1410 1411__setup("multitce=", disable_multitce); 1412 1413machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init); 1414