1/* 2 * File: drivers/pci/pcie/aspm.c 3 * Enabling PCIe link L0s/L1 state and Clock Power Management 4 * 5 * Copyright (C) 2007 Intel 6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 7 * Copyright (C) Shaohua Li (shaohua.li@intel.com) 8 */ 9 10#include <linux/kernel.h> 11#include <linux/module.h> 12#include <linux/moduleparam.h> 13#include <linux/pci.h> 14#include <linux/pci_regs.h> 15#include <linux/errno.h> 16#include <linux/pm.h> 17#include <linux/init.h> 18#include <linux/slab.h> 19#include <linux/jiffies.h> 20#include <linux/delay.h> 21#include <linux/pci-aspm.h> 22#include "../pci.h" 23 24#ifdef MODULE_PARAM_PREFIX 25#undef MODULE_PARAM_PREFIX 26#endif 27#define MODULE_PARAM_PREFIX "pcie_aspm." 28 29/* Note: those are not register definitions */ 30#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 31#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 32#define ASPM_STATE_L1 (4) /* L1 state */ 33#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 34#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1) 35 36struct aspm_latency { 37 u32 l0s; /* L0s latency (nsec) */ 38 u32 l1; /* L1 latency (nsec) */ 39}; 40 41struct pcie_link_state { 42 struct pci_dev *pdev; /* Upstream component of the Link */ 43 struct pcie_link_state *root; /* pointer to the root port link */ 44 struct pcie_link_state *parent; /* pointer to the parent Link state */ 45 struct list_head sibling; /* node in link_list */ 46 struct list_head children; /* list of child link states */ 47 struct list_head link; /* node in parent's children list */ 48 49 /* ASPM state */ 50 u32 aspm_support:3; /* Supported ASPM state */ 51 u32 aspm_enabled:3; /* Enabled ASPM state */ 52 u32 aspm_capable:3; /* Capable ASPM state with latency */ 53 u32 aspm_default:3; /* Default ASPM state by BIOS */ 54 u32 aspm_disable:3; /* Disabled ASPM state */ 55 56 /* Clock PM state */ 57 u32 clkpm_capable:1; /* Clock PM capable? */ 58 u32 clkpm_enabled:1; /* Current Clock PM state */ 59 u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 60 61 /* Exit latencies */ 62 struct aspm_latency latency_up; /* Upstream direction exit latency */ 63 struct aspm_latency latency_dw; /* Downstream direction exit latency */ 64 /* 65 * Endpoint acceptable latencies. A pcie downstream port only 66 * has one slot under it, so at most there are 8 functions. 67 */ 68 struct aspm_latency acceptable[8]; 69}; 70 71static int aspm_disabled, aspm_force; 72static bool aspm_support_enabled = true; 73static DEFINE_MUTEX(aspm_lock); 74static LIST_HEAD(link_list); 75 76#define POLICY_DEFAULT 0 /* BIOS default setting */ 77#define POLICY_PERFORMANCE 1 /* high performance */ 78#define POLICY_POWERSAVE 2 /* high power saving */ 79 80#ifdef CONFIG_PCIEASPM_PERFORMANCE 81static int aspm_policy = POLICY_PERFORMANCE; 82#elif defined CONFIG_PCIEASPM_POWERSAVE 83static int aspm_policy = POLICY_POWERSAVE; 84#else 85static int aspm_policy; 86#endif 87 88static const char *policy_str[] = { 89 [POLICY_DEFAULT] = "default", 90 [POLICY_PERFORMANCE] = "performance", 91 [POLICY_POWERSAVE] = "powersave" 92}; 93 94#define LINK_RETRAIN_TIMEOUT HZ 95 96static int policy_to_aspm_state(struct pcie_link_state *link) 97{ 98 switch (aspm_policy) { 99 case POLICY_PERFORMANCE: 100 /* Disable ASPM and Clock PM */ 101 return 0; 102 case POLICY_POWERSAVE: 103 /* Enable ASPM L0s/L1 */ 104 return ASPM_STATE_ALL; 105 case POLICY_DEFAULT: 106 return link->aspm_default; 107 } 108 return 0; 109} 110 111static int policy_to_clkpm_state(struct pcie_link_state *link) 112{ 113 switch (aspm_policy) { 114 case POLICY_PERFORMANCE: 115 /* Disable ASPM and Clock PM */ 116 return 0; 117 case POLICY_POWERSAVE: 118 /* Disable Clock PM */ 119 return 1; 120 case POLICY_DEFAULT: 121 return link->clkpm_default; 122 } 123 return 0; 124} 125 126static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 127{ 128 struct pci_dev *child; 129 struct pci_bus *linkbus = link->pdev->subordinate; 130 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0; 131 132 list_for_each_entry(child, &linkbus->devices, bus_list) 133 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 134 PCI_EXP_LNKCTL_CLKREQ_EN, 135 val); 136 link->clkpm_enabled = !!enable; 137} 138 139static void pcie_set_clkpm(struct pcie_link_state *link, int enable) 140{ 141 /* Don't enable Clock PM if the link is not Clock PM capable */ 142 if (!link->clkpm_capable && enable) 143 enable = 0; 144 /* Need nothing if the specified equals to current state */ 145 if (link->clkpm_enabled == enable) 146 return; 147 pcie_set_clkpm_nocheck(link, enable); 148} 149 150static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 151{ 152 int capable = 1, enabled = 1; 153 u32 reg32; 154 u16 reg16; 155 struct pci_dev *child; 156 struct pci_bus *linkbus = link->pdev->subordinate; 157 158 /* All functions should have the same cap and state, take the worst */ 159 list_for_each_entry(child, &linkbus->devices, bus_list) { 160 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); 161 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 162 capable = 0; 163 enabled = 0; 164 break; 165 } 166 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 167 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 168 enabled = 0; 169 } 170 link->clkpm_enabled = enabled; 171 link->clkpm_default = enabled; 172 link->clkpm_capable = (blacklist) ? 0 : capable; 173} 174 175/* 176 * pcie_aspm_configure_common_clock: check if the 2 ends of a link 177 * could use common clock. If they are, configure them to use the 178 * common clock. That will reduce the ASPM state exit latency. 179 */ 180static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 181{ 182 int same_clock = 1; 183 u16 reg16, parent_reg, child_reg[8]; 184 unsigned long start_jiffies; 185 struct pci_dev *child, *parent = link->pdev; 186 struct pci_bus *linkbus = parent->subordinate; 187 /* 188 * All functions of a slot should have the same Slot Clock 189 * Configuration, so just check one function 190 */ 191 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 192 BUG_ON(!pci_is_pcie(child)); 193 194 /* Check downstream component if bit Slot Clock Configuration is 1 */ 195 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); 196 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 197 same_clock = 0; 198 199 /* Check upstream component if bit Slot Clock Configuration is 1 */ 200 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 201 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 202 same_clock = 0; 203 204 /* Configure downstream component, all functions */ 205 list_for_each_entry(child, &linkbus->devices, bus_list) { 206 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 207 child_reg[PCI_FUNC(child->devfn)] = reg16; 208 if (same_clock) 209 reg16 |= PCI_EXP_LNKCTL_CCC; 210 else 211 reg16 &= ~PCI_EXP_LNKCTL_CCC; 212 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); 213 } 214 215 /* Configure upstream component */ 216 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); 217 parent_reg = reg16; 218 if (same_clock) 219 reg16 |= PCI_EXP_LNKCTL_CCC; 220 else 221 reg16 &= ~PCI_EXP_LNKCTL_CCC; 222 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 223 224 /* Retrain link */ 225 reg16 |= PCI_EXP_LNKCTL_RL; 226 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 227 228 /* Wait for link training end. Break out after waiting for timeout */ 229 start_jiffies = jiffies; 230 for (;;) { 231 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 232 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 233 break; 234 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 235 break; 236 msleep(1); 237 } 238 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 239 return; 240 241 /* Training failed. Restore common clock configurations */ 242 dev_err(&parent->dev, "ASPM: Could not configure common clock\n"); 243 list_for_each_entry(child, &linkbus->devices, bus_list) 244 pcie_capability_write_word(child, PCI_EXP_LNKCTL, 245 child_reg[PCI_FUNC(child->devfn)]); 246 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); 247} 248 249/* Convert L0s latency encoding to ns */ 250static u32 calc_l0s_latency(u32 encoding) 251{ 252 if (encoding == 0x7) 253 return (5 * 1000); /* > 4us */ 254 return (64 << encoding); 255} 256 257/* Convert L0s acceptable latency encoding to ns */ 258static u32 calc_l0s_acceptable(u32 encoding) 259{ 260 if (encoding == 0x7) 261 return -1U; 262 return (64 << encoding); 263} 264 265/* Convert L1 latency encoding to ns */ 266static u32 calc_l1_latency(u32 encoding) 267{ 268 if (encoding == 0x7) 269 return (65 * 1000); /* > 64us */ 270 return (1000 << encoding); 271} 272 273/* Convert L1 acceptable latency encoding to ns */ 274static u32 calc_l1_acceptable(u32 encoding) 275{ 276 if (encoding == 0x7) 277 return -1U; 278 return (1000 << encoding); 279} 280 281struct aspm_register_info { 282 u32 support:2; 283 u32 enabled:2; 284 u32 latency_encoding_l0s; 285 u32 latency_encoding_l1; 286}; 287 288static void pcie_get_aspm_reg(struct pci_dev *pdev, 289 struct aspm_register_info *info) 290{ 291 u16 reg16; 292 u32 reg32; 293 294 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); 295 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 296 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 297 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 298 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); 299 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; 300} 301 302static void pcie_aspm_check_latency(struct pci_dev *endpoint) 303{ 304 u32 latency, l1_switch_latency = 0; 305 struct aspm_latency *acceptable; 306 struct pcie_link_state *link; 307 308 /* Device not in D0 doesn't need latency check */ 309 if ((endpoint->current_state != PCI_D0) && 310 (endpoint->current_state != PCI_UNKNOWN)) 311 return; 312 313 link = endpoint->bus->self->link_state; 314 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; 315 316 while (link) { 317 /* Check upstream direction L0s latency */ 318 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 319 (link->latency_up.l0s > acceptable->l0s)) 320 link->aspm_capable &= ~ASPM_STATE_L0S_UP; 321 322 /* Check downstream direction L0s latency */ 323 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 324 (link->latency_dw.l0s > acceptable->l0s)) 325 link->aspm_capable &= ~ASPM_STATE_L0S_DW; 326 /* 327 * Check L1 latency. 328 * Every switch on the path to root complex need 1 329 * more microsecond for L1. Spec doesn't mention L0s. 330 */ 331 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); 332 if ((link->aspm_capable & ASPM_STATE_L1) && 333 (latency + l1_switch_latency > acceptable->l1)) 334 link->aspm_capable &= ~ASPM_STATE_L1; 335 l1_switch_latency += 1000; 336 337 link = link->parent; 338 } 339} 340 341static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 342{ 343 struct pci_dev *child, *parent = link->pdev; 344 struct pci_bus *linkbus = parent->subordinate; 345 struct aspm_register_info upreg, dwreg; 346 347 if (blacklist) { 348 /* Set enabled/disable so that we will disable ASPM later */ 349 link->aspm_enabled = ASPM_STATE_ALL; 350 link->aspm_disable = ASPM_STATE_ALL; 351 return; 352 } 353 354 /* Configure common clock before checking latencies */ 355 pcie_aspm_configure_common_clock(link); 356 357 /* Get upstream/downstream components' register state */ 358 pcie_get_aspm_reg(parent, &upreg); 359 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 360 pcie_get_aspm_reg(child, &dwreg); 361 362 /* 363 * Setup L0s state 364 * 365 * Note that we must not enable L0s in either direction on a 366 * given link unless components on both sides of the link each 367 * support L0s. 368 */ 369 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) 370 link->aspm_support |= ASPM_STATE_L0S; 371 if (dwreg.enabled & PCIE_LINK_STATE_L0S) 372 link->aspm_enabled |= ASPM_STATE_L0S_UP; 373 if (upreg.enabled & PCIE_LINK_STATE_L0S) 374 link->aspm_enabled |= ASPM_STATE_L0S_DW; 375 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); 376 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); 377 378 /* Setup L1 state */ 379 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) 380 link->aspm_support |= ASPM_STATE_L1; 381 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) 382 link->aspm_enabled |= ASPM_STATE_L1; 383 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); 384 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); 385 386 /* Save default state */ 387 link->aspm_default = link->aspm_enabled; 388 389 /* Setup initial capable state. Will be updated later */ 390 link->aspm_capable = link->aspm_support; 391 /* 392 * If the downstream component has pci bridge function, don't 393 * do ASPM for now. 394 */ 395 list_for_each_entry(child, &linkbus->devices, bus_list) { 396 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { 397 link->aspm_disable = ASPM_STATE_ALL; 398 break; 399 } 400 } 401 402 /* Get and check endpoint acceptable latencies */ 403 list_for_each_entry(child, &linkbus->devices, bus_list) { 404 u32 reg32, encoding; 405 struct aspm_latency *acceptable = 406 &link->acceptable[PCI_FUNC(child->devfn)]; 407 408 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && 409 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) 410 continue; 411 412 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 413 /* Calculate endpoint L0s acceptable latency */ 414 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 415 acceptable->l0s = calc_l0s_acceptable(encoding); 416 /* Calculate endpoint L1 acceptable latency */ 417 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 418 acceptable->l1 = calc_l1_acceptable(encoding); 419 420 pcie_aspm_check_latency(child); 421 } 422} 423 424static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 425{ 426 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, 427 PCI_EXP_LNKCTL_ASPMC, val); 428} 429 430static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 431{ 432 u32 upstream = 0, dwstream = 0; 433 struct pci_dev *child, *parent = link->pdev; 434 struct pci_bus *linkbus = parent->subordinate; 435 436 /* Nothing to do if the link is already in the requested state */ 437 state &= (link->aspm_capable & ~link->aspm_disable); 438 if (link->aspm_enabled == state) 439 return; 440 /* Convert ASPM state to upstream/downstream ASPM register state */ 441 if (state & ASPM_STATE_L0S_UP) 442 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; 443 if (state & ASPM_STATE_L0S_DW) 444 upstream |= PCI_EXP_LNKCTL_ASPM_L0S; 445 if (state & ASPM_STATE_L1) { 446 upstream |= PCI_EXP_LNKCTL_ASPM_L1; 447 dwstream |= PCI_EXP_LNKCTL_ASPM_L1; 448 } 449 /* 450 * Spec 2.0 suggests all functions should be configured the 451 * same setting for ASPM. Enabling ASPM L1 should be done in 452 * upstream component first and then downstream, and vice 453 * versa for disabling ASPM L1. Spec doesn't mention L0S. 454 */ 455 if (state & ASPM_STATE_L1) 456 pcie_config_aspm_dev(parent, upstream); 457 list_for_each_entry(child, &linkbus->devices, bus_list) 458 pcie_config_aspm_dev(child, dwstream); 459 if (!(state & ASPM_STATE_L1)) 460 pcie_config_aspm_dev(parent, upstream); 461 462 link->aspm_enabled = state; 463} 464 465static void pcie_config_aspm_path(struct pcie_link_state *link) 466{ 467 while (link) { 468 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 469 link = link->parent; 470 } 471} 472 473static void free_link_state(struct pcie_link_state *link) 474{ 475 link->pdev->link_state = NULL; 476 kfree(link); 477} 478 479static int pcie_aspm_sanity_check(struct pci_dev *pdev) 480{ 481 struct pci_dev *child; 482 u32 reg32; 483 484 /* 485 * Some functions in a slot might not all be PCIe functions, 486 * very strange. Disable ASPM for the whole slot 487 */ 488 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 489 if (!pci_is_pcie(child)) 490 return -EINVAL; 491 492 /* 493 * If ASPM is disabled then we're not going to change 494 * the BIOS state. It's safe to continue even if it's a 495 * pre-1.1 device 496 */ 497 498 if (aspm_disabled) 499 continue; 500 501 /* 502 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 503 * RBER bit to determine if a function is 1.1 version device 504 */ 505 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 506 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 507 dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); 508 return -EINVAL; 509 } 510 } 511 return 0; 512} 513 514static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 515{ 516 struct pcie_link_state *link; 517 518 link = kzalloc(sizeof(*link), GFP_KERNEL); 519 if (!link) 520 return NULL; 521 INIT_LIST_HEAD(&link->sibling); 522 INIT_LIST_HEAD(&link->children); 523 INIT_LIST_HEAD(&link->link); 524 link->pdev = pdev; 525 if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) { 526 struct pcie_link_state *parent; 527 parent = pdev->bus->parent->self->link_state; 528 if (!parent) { 529 kfree(link); 530 return NULL; 531 } 532 link->parent = parent; 533 list_add(&link->link, &parent->children); 534 } 535 /* Setup a pointer to the root port link */ 536 if (!link->parent) 537 link->root = link; 538 else 539 link->root = link->parent->root; 540 541 list_add(&link->sibling, &link_list); 542 pdev->link_state = link; 543 return link; 544} 545 546/* 547 * pcie_aspm_init_link_state: Initiate PCI express link state. 548 * It is called after the pcie and its children devices are scanned. 549 * @pdev: the root port or switch downstream port 550 */ 551void pcie_aspm_init_link_state(struct pci_dev *pdev) 552{ 553 struct pcie_link_state *link; 554 int blacklist = !!pcie_aspm_sanity_check(pdev); 555 556 if (!aspm_support_enabled) 557 return; 558 559 if (pdev->link_state) 560 return; 561 562 /* 563 * We allocate pcie_link_state for the component on the upstream 564 * end of a Link, so there's nothing to do unless this device has a 565 * Link on its secondary side. 566 */ 567 if (!pdev->has_secondary_link) 568 return; 569 570 /* VIA has a strange chipset, root port is under a bridge */ 571 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && 572 pdev->bus->self) 573 return; 574 575 down_read(&pci_bus_sem); 576 if (list_empty(&pdev->subordinate->devices)) 577 goto out; 578 579 mutex_lock(&aspm_lock); 580 link = alloc_pcie_link_state(pdev); 581 if (!link) 582 goto unlock; 583 /* 584 * Setup initial ASPM state. Note that we need to configure 585 * upstream links also because capable state of them can be 586 * update through pcie_aspm_cap_init(). 587 */ 588 pcie_aspm_cap_init(link, blacklist); 589 590 /* Setup initial Clock PM state */ 591 pcie_clkpm_cap_init(link, blacklist); 592 593 /* 594 * At this stage drivers haven't had an opportunity to change the 595 * link policy setting. Enabling ASPM on broken hardware can cripple 596 * it even before the driver has had a chance to disable ASPM, so 597 * default to a safe level right now. If we're enabling ASPM beyond 598 * the BIOS's expectation, we'll do so once pci_enable_device() is 599 * called. 600 */ 601 if (aspm_policy != POLICY_POWERSAVE) { 602 pcie_config_aspm_path(link); 603 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 604 } 605 606unlock: 607 mutex_unlock(&aspm_lock); 608out: 609 up_read(&pci_bus_sem); 610} 611 612/* Recheck latencies and update aspm_capable for links under the root */ 613static void pcie_update_aspm_capable(struct pcie_link_state *root) 614{ 615 struct pcie_link_state *link; 616 BUG_ON(root->parent); 617 list_for_each_entry(link, &link_list, sibling) { 618 if (link->root != root) 619 continue; 620 link->aspm_capable = link->aspm_support; 621 } 622 list_for_each_entry(link, &link_list, sibling) { 623 struct pci_dev *child; 624 struct pci_bus *linkbus = link->pdev->subordinate; 625 if (link->root != root) 626 continue; 627 list_for_each_entry(child, &linkbus->devices, bus_list) { 628 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && 629 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) 630 continue; 631 pcie_aspm_check_latency(child); 632 } 633 } 634} 635 636/* @pdev: the endpoint device */ 637void pcie_aspm_exit_link_state(struct pci_dev *pdev) 638{ 639 struct pci_dev *parent = pdev->bus->self; 640 struct pcie_link_state *link, *root, *parent_link; 641 642 if (!parent || !parent->link_state) 643 return; 644 645 down_read(&pci_bus_sem); 646 mutex_lock(&aspm_lock); 647 /* 648 * All PCIe functions are in one slot, remove one function will remove 649 * the whole slot, so just wait until we are the last function left. 650 */ 651 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) 652 goto out; 653 654 link = parent->link_state; 655 root = link->root; 656 parent_link = link->parent; 657 658 /* All functions are removed, so just disable ASPM for the link */ 659 pcie_config_aspm_link(link, 0); 660 list_del(&link->sibling); 661 list_del(&link->link); 662 /* Clock PM is for endpoint device */ 663 free_link_state(link); 664 665 /* Recheck latencies and configure upstream links */ 666 if (parent_link) { 667 pcie_update_aspm_capable(root); 668 pcie_config_aspm_path(parent_link); 669 } 670out: 671 mutex_unlock(&aspm_lock); 672 up_read(&pci_bus_sem); 673} 674 675/* @pdev: the root port or switch downstream port */ 676void pcie_aspm_pm_state_change(struct pci_dev *pdev) 677{ 678 struct pcie_link_state *link = pdev->link_state; 679 680 if (aspm_disabled || !link) 681 return; 682 /* 683 * Devices changed PM state, we should recheck if latency 684 * meets all functions' requirement 685 */ 686 down_read(&pci_bus_sem); 687 mutex_lock(&aspm_lock); 688 pcie_update_aspm_capable(link->root); 689 pcie_config_aspm_path(link); 690 mutex_unlock(&aspm_lock); 691 up_read(&pci_bus_sem); 692} 693 694void pcie_aspm_powersave_config_link(struct pci_dev *pdev) 695{ 696 struct pcie_link_state *link = pdev->link_state; 697 698 if (aspm_disabled || !link) 699 return; 700 701 if (aspm_policy != POLICY_POWERSAVE) 702 return; 703 704 down_read(&pci_bus_sem); 705 mutex_lock(&aspm_lock); 706 pcie_config_aspm_path(link); 707 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 708 mutex_unlock(&aspm_lock); 709 up_read(&pci_bus_sem); 710} 711 712static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem) 713{ 714 struct pci_dev *parent = pdev->bus->self; 715 struct pcie_link_state *link; 716 717 if (!pci_is_pcie(pdev)) 718 return; 719 720 if (pdev->has_secondary_link) 721 parent = pdev; 722 if (!parent || !parent->link_state) 723 return; 724 725 /* 726 * A driver requested that ASPM be disabled on this device, but 727 * if we don't have permission to manage ASPM (e.g., on ACPI 728 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and 729 * the _OSC method), we can't honor that request. Windows has 730 * a similar mechanism using "PciASPMOptOut", which is also 731 * ignored in this situation. 732 */ 733 if (aspm_disabled) { 734 dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n"); 735 return; 736 } 737 738 if (sem) 739 down_read(&pci_bus_sem); 740 mutex_lock(&aspm_lock); 741 link = parent->link_state; 742 if (state & PCIE_LINK_STATE_L0S) 743 link->aspm_disable |= ASPM_STATE_L0S; 744 if (state & PCIE_LINK_STATE_L1) 745 link->aspm_disable |= ASPM_STATE_L1; 746 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 747 748 if (state & PCIE_LINK_STATE_CLKPM) { 749 link->clkpm_capable = 0; 750 pcie_set_clkpm(link, 0); 751 } 752 mutex_unlock(&aspm_lock); 753 if (sem) 754 up_read(&pci_bus_sem); 755} 756 757void pci_disable_link_state_locked(struct pci_dev *pdev, int state) 758{ 759 __pci_disable_link_state(pdev, state, false); 760} 761EXPORT_SYMBOL(pci_disable_link_state_locked); 762 763/** 764 * pci_disable_link_state - Disable device's link state, so the link will 765 * never enter specific states. Note that if the BIOS didn't grant ASPM 766 * control to the OS, this does nothing because we can't touch the LNKCTL 767 * register. 768 * 769 * @pdev: PCI device 770 * @state: ASPM link state to disable 771 */ 772void pci_disable_link_state(struct pci_dev *pdev, int state) 773{ 774 __pci_disable_link_state(pdev, state, true); 775} 776EXPORT_SYMBOL(pci_disable_link_state); 777 778static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) 779{ 780 int i; 781 struct pcie_link_state *link; 782 783 if (aspm_disabled) 784 return -EPERM; 785 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 786 if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) 787 break; 788 if (i >= ARRAY_SIZE(policy_str)) 789 return -EINVAL; 790 if (i == aspm_policy) 791 return 0; 792 793 down_read(&pci_bus_sem); 794 mutex_lock(&aspm_lock); 795 aspm_policy = i; 796 list_for_each_entry(link, &link_list, sibling) { 797 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 798 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 799 } 800 mutex_unlock(&aspm_lock); 801 up_read(&pci_bus_sem); 802 return 0; 803} 804 805static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) 806{ 807 int i, cnt = 0; 808 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 809 if (i == aspm_policy) 810 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 811 else 812 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 813 return cnt; 814} 815 816module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 817 NULL, 0644); 818 819#ifdef CONFIG_PCIEASPM_DEBUG 820static ssize_t link_state_show(struct device *dev, 821 struct device_attribute *attr, 822 char *buf) 823{ 824 struct pci_dev *pci_device = to_pci_dev(dev); 825 struct pcie_link_state *link_state = pci_device->link_state; 826 827 return sprintf(buf, "%d\n", link_state->aspm_enabled); 828} 829 830static ssize_t link_state_store(struct device *dev, 831 struct device_attribute *attr, 832 const char *buf, 833 size_t n) 834{ 835 struct pci_dev *pdev = to_pci_dev(dev); 836 struct pcie_link_state *link, *root = pdev->link_state->root; 837 u32 val, state = 0; 838 839 if (kstrtouint(buf, 10, &val)) 840 return -EINVAL; 841 842 if (aspm_disabled) 843 return -EPERM; 844 if (n < 1 || val > 3) 845 return -EINVAL; 846 847 /* Convert requested state to ASPM state */ 848 if (val & PCIE_LINK_STATE_L0S) 849 state |= ASPM_STATE_L0S; 850 if (val & PCIE_LINK_STATE_L1) 851 state |= ASPM_STATE_L1; 852 853 down_read(&pci_bus_sem); 854 mutex_lock(&aspm_lock); 855 list_for_each_entry(link, &link_list, sibling) { 856 if (link->root != root) 857 continue; 858 pcie_config_aspm_link(link, state); 859 } 860 mutex_unlock(&aspm_lock); 861 up_read(&pci_bus_sem); 862 return n; 863} 864 865static ssize_t clk_ctl_show(struct device *dev, 866 struct device_attribute *attr, 867 char *buf) 868{ 869 struct pci_dev *pci_device = to_pci_dev(dev); 870 struct pcie_link_state *link_state = pci_device->link_state; 871 872 return sprintf(buf, "%d\n", link_state->clkpm_enabled); 873} 874 875static ssize_t clk_ctl_store(struct device *dev, 876 struct device_attribute *attr, 877 const char *buf, 878 size_t n) 879{ 880 struct pci_dev *pdev = to_pci_dev(dev); 881 bool state; 882 883 if (strtobool(buf, &state)) 884 return -EINVAL; 885 886 down_read(&pci_bus_sem); 887 mutex_lock(&aspm_lock); 888 pcie_set_clkpm_nocheck(pdev->link_state, state); 889 mutex_unlock(&aspm_lock); 890 up_read(&pci_bus_sem); 891 892 return n; 893} 894 895static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store); 896static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store); 897 898static char power_group[] = "power"; 899void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) 900{ 901 struct pcie_link_state *link_state = pdev->link_state; 902 903 if (!link_state) 904 return; 905 906 if (link_state->aspm_support) 907 sysfs_add_file_to_group(&pdev->dev.kobj, 908 &dev_attr_link_state.attr, power_group); 909 if (link_state->clkpm_capable) 910 sysfs_add_file_to_group(&pdev->dev.kobj, 911 &dev_attr_clk_ctl.attr, power_group); 912} 913 914void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) 915{ 916 struct pcie_link_state *link_state = pdev->link_state; 917 918 if (!link_state) 919 return; 920 921 if (link_state->aspm_support) 922 sysfs_remove_file_from_group(&pdev->dev.kobj, 923 &dev_attr_link_state.attr, power_group); 924 if (link_state->clkpm_capable) 925 sysfs_remove_file_from_group(&pdev->dev.kobj, 926 &dev_attr_clk_ctl.attr, power_group); 927} 928#endif 929 930static int __init pcie_aspm_disable(char *str) 931{ 932 if (!strcmp(str, "off")) { 933 aspm_policy = POLICY_DEFAULT; 934 aspm_disabled = 1; 935 aspm_support_enabled = false; 936 printk(KERN_INFO "PCIe ASPM is disabled\n"); 937 } else if (!strcmp(str, "force")) { 938 aspm_force = 1; 939 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 940 } 941 return 1; 942} 943 944__setup("pcie_aspm=", pcie_aspm_disable); 945 946void pcie_no_aspm(void) 947{ 948 /* 949 * Disabling ASPM is intended to prevent the kernel from modifying 950 * existing hardware state, not to clear existing state. To that end: 951 * (a) set policy to POLICY_DEFAULT in order to avoid changing state 952 * (b) prevent userspace from changing policy 953 */ 954 if (!aspm_force) { 955 aspm_policy = POLICY_DEFAULT; 956 aspm_disabled = 1; 957 } 958} 959 960bool pcie_aspm_support_enabled(void) 961{ 962 return aspm_support_enabled; 963} 964EXPORT_SYMBOL(pcie_aspm_support_enabled); 965