1/* Intel Ethernet Switch Host Interface Driver 2 * Copyright(c) 2013 - 2015 Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * The full GNU General Public License is included in this distribution in 14 * the file called "COPYING". 15 * 16 * Contact Information: 17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 19 */ 20 21#include <linux/module.h> 22#include <linux/aer.h> 23 24#include "fm10k.h" 25 26static const struct fm10k_info *fm10k_info_tbl[] = { 27 [fm10k_device_pf] = &fm10k_pf_info, 28 [fm10k_device_vf] = &fm10k_vf_info, 29}; 30 31/** 32 * fm10k_pci_tbl - PCI Device ID Table 33 * 34 * Wildcard entries (PCI_ANY_ID) should come last 35 * Last entry must be all 0s 36 * 37 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 38 * Class, Class Mask, private data (not used) } 39 */ 40static const struct pci_device_id fm10k_pci_tbl[] = { 41 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), fm10k_device_pf }, 42 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), fm10k_device_vf }, 43 /* required last entry */ 44 { 0, } 45}; 46MODULE_DEVICE_TABLE(pci, fm10k_pci_tbl); 47 48u16 fm10k_read_pci_cfg_word(struct fm10k_hw *hw, u32 reg) 49{ 50 struct fm10k_intfc *interface = hw->back; 51 u16 value = 0; 52 53 if (FM10K_REMOVED(hw->hw_addr)) 54 return ~value; 55 56 pci_read_config_word(interface->pdev, reg, &value); 57 if (value == 0xFFFF) 58 fm10k_write_flush(hw); 59 60 return value; 61} 62 63u32 fm10k_read_reg(struct fm10k_hw *hw, int reg) 64{ 65 u32 __iomem *hw_addr = ACCESS_ONCE(hw->hw_addr); 66 u32 value = 0; 67 68 if (FM10K_REMOVED(hw_addr)) 69 return ~value; 70 71 value = readl(&hw_addr[reg]); 72 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 73 struct fm10k_intfc *interface = hw->back; 74 struct net_device *netdev = interface->netdev; 75 76 hw->hw_addr = NULL; 77 netif_device_detach(netdev); 78 netdev_err(netdev, "PCIe link lost, device now detached\n"); 79 } 80 81 return value; 82} 83 84static int fm10k_hw_ready(struct fm10k_intfc *interface) 85{ 86 struct fm10k_hw *hw = &interface->hw; 87 88 fm10k_write_flush(hw); 89 90 return FM10K_REMOVED(hw->hw_addr) ? -ENODEV : 0; 91} 92 93void fm10k_service_event_schedule(struct fm10k_intfc *interface) 94{ 95 if (!test_bit(__FM10K_SERVICE_DISABLE, &interface->state) && 96 !test_and_set_bit(__FM10K_SERVICE_SCHED, &interface->state)) 97 queue_work(fm10k_workqueue, &interface->service_task); 98} 99 100static void fm10k_service_event_complete(struct fm10k_intfc *interface) 101{ 102 BUG_ON(!test_bit(__FM10K_SERVICE_SCHED, &interface->state)); 103 104 /* flush memory to make sure state is correct before next watchog */ 105 smp_mb__before_atomic(); 106 clear_bit(__FM10K_SERVICE_SCHED, &interface->state); 107} 108 109/** 110 * fm10k_service_timer - Timer Call-back 111 * @data: pointer to interface cast into an unsigned long 112 **/ 113static void fm10k_service_timer(unsigned long data) 114{ 115 struct fm10k_intfc *interface = (struct fm10k_intfc *)data; 116 117 /* Reset the timer */ 118 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 119 120 fm10k_service_event_schedule(interface); 121} 122 123static void fm10k_detach_subtask(struct fm10k_intfc *interface) 124{ 125 struct net_device *netdev = interface->netdev; 126 127 /* do nothing if device is still present or hw_addr is set */ 128 if (netif_device_present(netdev) || interface->hw.hw_addr) 129 return; 130 131 rtnl_lock(); 132 133 if (netif_running(netdev)) 134 dev_close(netdev); 135 136 rtnl_unlock(); 137} 138 139static void fm10k_reinit(struct fm10k_intfc *interface) 140{ 141 struct net_device *netdev = interface->netdev; 142 struct fm10k_hw *hw = &interface->hw; 143 int err; 144 145 WARN_ON(in_interrupt()); 146 147 /* put off any impending NetWatchDogTimeout */ 148 netdev->trans_start = jiffies; 149 150 while (test_and_set_bit(__FM10K_RESETTING, &interface->state)) 151 usleep_range(1000, 2000); 152 153 rtnl_lock(); 154 155 fm10k_iov_suspend(interface->pdev); 156 157 if (netif_running(netdev)) 158 fm10k_close(netdev); 159 160 fm10k_mbx_free_irq(interface); 161 162 /* delay any future reset requests */ 163 interface->last_reset = jiffies + (10 * HZ); 164 165 /* reset and initialize the hardware so it is in a known state */ 166 err = hw->mac.ops.reset_hw(hw) ? : hw->mac.ops.init_hw(hw); 167 if (err) 168 dev_err(&interface->pdev->dev, "init_hw failed: %d\n", err); 169 170 /* reassociate interrupts */ 171 fm10k_mbx_request_irq(interface); 172 173 /* reset clock */ 174 fm10k_ts_reset(interface); 175 176 if (netif_running(netdev)) 177 fm10k_open(netdev); 178 179 fm10k_iov_resume(interface->pdev); 180 181 rtnl_unlock(); 182 183 clear_bit(__FM10K_RESETTING, &interface->state); 184} 185 186static void fm10k_reset_subtask(struct fm10k_intfc *interface) 187{ 188 if (!(interface->flags & FM10K_FLAG_RESET_REQUESTED)) 189 return; 190 191 interface->flags &= ~FM10K_FLAG_RESET_REQUESTED; 192 193 netdev_err(interface->netdev, "Reset interface\n"); 194 195 fm10k_reinit(interface); 196} 197 198/** 199 * fm10k_configure_swpri_map - Configure Receive SWPRI to PC mapping 200 * @interface: board private structure 201 * 202 * Configure the SWPRI to PC mapping for the port. 203 **/ 204static void fm10k_configure_swpri_map(struct fm10k_intfc *interface) 205{ 206 struct net_device *netdev = interface->netdev; 207 struct fm10k_hw *hw = &interface->hw; 208 int i; 209 210 /* clear flag indicating update is needed */ 211 interface->flags &= ~FM10K_FLAG_SWPRI_CONFIG; 212 213 /* these registers are only available on the PF */ 214 if (hw->mac.type != fm10k_mac_pf) 215 return; 216 217 /* configure SWPRI to PC map */ 218 for (i = 0; i < FM10K_SWPRI_MAX; i++) 219 fm10k_write_reg(hw, FM10K_SWPRI_MAP(i), 220 netdev_get_prio_tc_map(netdev, i)); 221} 222 223/** 224 * fm10k_watchdog_update_host_state - Update the link status based on host. 225 * @interface: board private structure 226 **/ 227static void fm10k_watchdog_update_host_state(struct fm10k_intfc *interface) 228{ 229 struct fm10k_hw *hw = &interface->hw; 230 s32 err; 231 232 if (test_bit(__FM10K_LINK_DOWN, &interface->state)) { 233 interface->host_ready = false; 234 if (time_is_after_jiffies(interface->link_down_event)) 235 return; 236 clear_bit(__FM10K_LINK_DOWN, &interface->state); 237 } 238 239 if (interface->flags & FM10K_FLAG_SWPRI_CONFIG) { 240 if (rtnl_trylock()) { 241 fm10k_configure_swpri_map(interface); 242 rtnl_unlock(); 243 } 244 } 245 246 /* lock the mailbox for transmit and receive */ 247 fm10k_mbx_lock(interface); 248 249 err = hw->mac.ops.get_host_state(hw, &interface->host_ready); 250 if (err && time_is_before_jiffies(interface->last_reset)) 251 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 252 253 /* free the lock */ 254 fm10k_mbx_unlock(interface); 255} 256 257/** 258 * fm10k_mbx_subtask - Process upstream and downstream mailboxes 259 * @interface: board private structure 260 * 261 * This function will process both the upstream and downstream mailboxes. 262 * It is necessary for us to hold the rtnl_lock while doing this as the 263 * mailbox accesses are protected by this lock. 264 **/ 265static void fm10k_mbx_subtask(struct fm10k_intfc *interface) 266{ 267 /* process upstream mailbox and update device state */ 268 fm10k_watchdog_update_host_state(interface); 269 270 /* process downstream mailboxes */ 271 fm10k_iov_mbx(interface); 272} 273 274/** 275 * fm10k_watchdog_host_is_ready - Update netdev status based on host ready 276 * @interface: board private structure 277 **/ 278static void fm10k_watchdog_host_is_ready(struct fm10k_intfc *interface) 279{ 280 struct net_device *netdev = interface->netdev; 281 282 /* only continue if link state is currently down */ 283 if (netif_carrier_ok(netdev)) 284 return; 285 286 netif_info(interface, drv, netdev, "NIC Link is up\n"); 287 288 netif_carrier_on(netdev); 289 netif_tx_wake_all_queues(netdev); 290} 291 292/** 293 * fm10k_watchdog_host_not_ready - Update netdev status based on host not ready 294 * @interface: board private structure 295 **/ 296static void fm10k_watchdog_host_not_ready(struct fm10k_intfc *interface) 297{ 298 struct net_device *netdev = interface->netdev; 299 300 /* only continue if link state is currently up */ 301 if (!netif_carrier_ok(netdev)) 302 return; 303 304 netif_info(interface, drv, netdev, "NIC Link is down\n"); 305 306 netif_carrier_off(netdev); 307 netif_tx_stop_all_queues(netdev); 308} 309 310/** 311 * fm10k_update_stats - Update the board statistics counters. 312 * @interface: board private structure 313 **/ 314void fm10k_update_stats(struct fm10k_intfc *interface) 315{ 316 struct net_device_stats *net_stats = &interface->netdev->stats; 317 struct fm10k_hw *hw = &interface->hw; 318 u64 rx_errors = 0, rx_csum_errors = 0, tx_csum_errors = 0; 319 u64 restart_queue = 0, tx_busy = 0, alloc_failed = 0; 320 u64 rx_bytes_nic = 0, rx_pkts_nic = 0, rx_drops_nic = 0; 321 u64 tx_bytes_nic = 0, tx_pkts_nic = 0; 322 u64 bytes, pkts; 323 int i; 324 325 /* do not allow stats update via service task for next second */ 326 interface->next_stats_update = jiffies + HZ; 327 328 /* gather some stats to the interface struct that are per queue */ 329 for (bytes = 0, pkts = 0, i = 0; i < interface->num_tx_queues; i++) { 330 struct fm10k_ring *tx_ring = interface->tx_ring[i]; 331 332 restart_queue += tx_ring->tx_stats.restart_queue; 333 tx_busy += tx_ring->tx_stats.tx_busy; 334 tx_csum_errors += tx_ring->tx_stats.csum_err; 335 bytes += tx_ring->stats.bytes; 336 pkts += tx_ring->stats.packets; 337 } 338 339 interface->restart_queue = restart_queue; 340 interface->tx_busy = tx_busy; 341 net_stats->tx_bytes = bytes; 342 net_stats->tx_packets = pkts; 343 interface->tx_csum_errors = tx_csum_errors; 344 /* gather some stats to the interface struct that are per queue */ 345 for (bytes = 0, pkts = 0, i = 0; i < interface->num_rx_queues; i++) { 346 struct fm10k_ring *rx_ring = interface->rx_ring[i]; 347 348 bytes += rx_ring->stats.bytes; 349 pkts += rx_ring->stats.packets; 350 alloc_failed += rx_ring->rx_stats.alloc_failed; 351 rx_csum_errors += rx_ring->rx_stats.csum_err; 352 rx_errors += rx_ring->rx_stats.errors; 353 } 354 355 net_stats->rx_bytes = bytes; 356 net_stats->rx_packets = pkts; 357 interface->alloc_failed = alloc_failed; 358 interface->rx_csum_errors = rx_csum_errors; 359 360 hw->mac.ops.update_hw_stats(hw, &interface->stats); 361 362 for (i = 0; i < hw->mac.max_queues; i++) { 363 struct fm10k_hw_stats_q *q = &interface->stats.q[i]; 364 365 tx_bytes_nic += q->tx_bytes.count; 366 tx_pkts_nic += q->tx_packets.count; 367 rx_bytes_nic += q->rx_bytes.count; 368 rx_pkts_nic += q->rx_packets.count; 369 rx_drops_nic += q->rx_drops.count; 370 } 371 372 interface->tx_bytes_nic = tx_bytes_nic; 373 interface->tx_packets_nic = tx_pkts_nic; 374 interface->rx_bytes_nic = rx_bytes_nic; 375 interface->rx_packets_nic = rx_pkts_nic; 376 interface->rx_drops_nic = rx_drops_nic; 377 378 /* Fill out the OS statistics structure */ 379 net_stats->rx_errors = rx_errors; 380 net_stats->rx_dropped = interface->stats.nodesc_drop.count; 381} 382 383/** 384 * fm10k_watchdog_flush_tx - flush queues on host not ready 385 * @interface - pointer to the device interface structure 386 **/ 387static void fm10k_watchdog_flush_tx(struct fm10k_intfc *interface) 388{ 389 int some_tx_pending = 0; 390 int i; 391 392 /* nothing to do if carrier is up */ 393 if (netif_carrier_ok(interface->netdev)) 394 return; 395 396 for (i = 0; i < interface->num_tx_queues; i++) { 397 struct fm10k_ring *tx_ring = interface->tx_ring[i]; 398 399 if (tx_ring->next_to_use != tx_ring->next_to_clean) { 400 some_tx_pending = 1; 401 break; 402 } 403 } 404 405 /* We've lost link, so the controller stops DMA, but we've got 406 * queued Tx work that's never going to get done, so reset 407 * controller to flush Tx. 408 */ 409 if (some_tx_pending) 410 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 411} 412 413/** 414 * fm10k_watchdog_subtask - check and bring link up 415 * @interface - pointer to the device interface structure 416 **/ 417static void fm10k_watchdog_subtask(struct fm10k_intfc *interface) 418{ 419 /* if interface is down do nothing */ 420 if (test_bit(__FM10K_DOWN, &interface->state) || 421 test_bit(__FM10K_RESETTING, &interface->state)) 422 return; 423 424 if (interface->host_ready) 425 fm10k_watchdog_host_is_ready(interface); 426 else 427 fm10k_watchdog_host_not_ready(interface); 428 429 /* update stats only once every second */ 430 if (time_is_before_jiffies(interface->next_stats_update)) 431 fm10k_update_stats(interface); 432 433 /* flush any uncompleted work */ 434 fm10k_watchdog_flush_tx(interface); 435} 436 437/** 438 * fm10k_check_hang_subtask - check for hung queues and dropped interrupts 439 * @interface - pointer to the device interface structure 440 * 441 * This function serves two purposes. First it strobes the interrupt lines 442 * in order to make certain interrupts are occurring. Secondly it sets the 443 * bits needed to check for TX hangs. As a result we should immediately 444 * determine if a hang has occurred. 445 */ 446static void fm10k_check_hang_subtask(struct fm10k_intfc *interface) 447{ 448 int i; 449 450 /* If we're down or resetting, just bail */ 451 if (test_bit(__FM10K_DOWN, &interface->state) || 452 test_bit(__FM10K_RESETTING, &interface->state)) 453 return; 454 455 /* rate limit tx hang checks to only once every 2 seconds */ 456 if (time_is_after_eq_jiffies(interface->next_tx_hang_check)) 457 return; 458 interface->next_tx_hang_check = jiffies + (2 * HZ); 459 460 if (netif_carrier_ok(interface->netdev)) { 461 /* Force detection of hung controller */ 462 for (i = 0; i < interface->num_tx_queues; i++) 463 set_check_for_tx_hang(interface->tx_ring[i]); 464 465 /* Rearm all in-use q_vectors for immediate firing */ 466 for (i = 0; i < interface->num_q_vectors; i++) { 467 struct fm10k_q_vector *qv = interface->q_vector[i]; 468 469 if (!qv->tx.count && !qv->rx.count) 470 continue; 471 writel(FM10K_ITR_ENABLE | FM10K_ITR_PENDING2, qv->itr); 472 } 473 } 474} 475 476/** 477 * fm10k_service_task - manages and runs subtasks 478 * @work: pointer to work_struct containing our data 479 **/ 480static void fm10k_service_task(struct work_struct *work) 481{ 482 struct fm10k_intfc *interface; 483 484 interface = container_of(work, struct fm10k_intfc, service_task); 485 486 /* tasks always capable of running, but must be rtnl protected */ 487 fm10k_mbx_subtask(interface); 488 fm10k_detach_subtask(interface); 489 fm10k_reset_subtask(interface); 490 491 /* tasks only run when interface is up */ 492 fm10k_watchdog_subtask(interface); 493 fm10k_check_hang_subtask(interface); 494 fm10k_ts_tx_subtask(interface); 495 496 /* release lock on service events to allow scheduling next event */ 497 fm10k_service_event_complete(interface); 498} 499 500/** 501 * fm10k_configure_tx_ring - Configure Tx ring after Reset 502 * @interface: board private structure 503 * @ring: structure containing ring specific data 504 * 505 * Configure the Tx descriptor ring after a reset. 506 **/ 507static void fm10k_configure_tx_ring(struct fm10k_intfc *interface, 508 struct fm10k_ring *ring) 509{ 510 struct fm10k_hw *hw = &interface->hw; 511 u64 tdba = ring->dma; 512 u32 size = ring->count * sizeof(struct fm10k_tx_desc); 513 u32 txint = FM10K_INT_MAP_DISABLE; 514 u32 txdctl = FM10K_TXDCTL_ENABLE | (1 << FM10K_TXDCTL_MAX_TIME_SHIFT); 515 u8 reg_idx = ring->reg_idx; 516 517 /* disable queue to avoid issues while updating state */ 518 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), 0); 519 fm10k_write_flush(hw); 520 521 /* possible poll here to verify ring resources have been cleaned */ 522 523 /* set location and size for descriptor ring */ 524 fm10k_write_reg(hw, FM10K_TDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 525 fm10k_write_reg(hw, FM10K_TDBAH(reg_idx), tdba >> 32); 526 fm10k_write_reg(hw, FM10K_TDLEN(reg_idx), size); 527 528 /* reset head and tail pointers */ 529 fm10k_write_reg(hw, FM10K_TDH(reg_idx), 0); 530 fm10k_write_reg(hw, FM10K_TDT(reg_idx), 0); 531 532 /* store tail pointer */ 533 ring->tail = &interface->uc_addr[FM10K_TDT(reg_idx)]; 534 535 /* reset ntu and ntc to place SW in sync with hardwdare */ 536 ring->next_to_clean = 0; 537 ring->next_to_use = 0; 538 539 /* Map interrupt */ 540 if (ring->q_vector) { 541 txint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 542 txint |= FM10K_INT_MAP_TIMER0; 543 } 544 545 fm10k_write_reg(hw, FM10K_TXINT(reg_idx), txint); 546 547 /* enable use of FTAG bit in Tx descriptor, register is RO for VF */ 548 fm10k_write_reg(hw, FM10K_PFVTCTL(reg_idx), 549 FM10K_PFVTCTL_FTAG_DESC_ENABLE); 550 551 /* enable queue */ 552 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), txdctl); 553} 554 555/** 556 * fm10k_enable_tx_ring - Verify Tx ring is enabled after configuration 557 * @interface: board private structure 558 * @ring: structure containing ring specific data 559 * 560 * Verify the Tx descriptor ring is ready for transmit. 561 **/ 562static void fm10k_enable_tx_ring(struct fm10k_intfc *interface, 563 struct fm10k_ring *ring) 564{ 565 struct fm10k_hw *hw = &interface->hw; 566 int wait_loop = 10; 567 u32 txdctl; 568 u8 reg_idx = ring->reg_idx; 569 570 /* if we are already enabled just exit */ 571 if (fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)) & FM10K_TXDCTL_ENABLE) 572 return; 573 574 /* poll to verify queue is enabled */ 575 do { 576 usleep_range(1000, 2000); 577 txdctl = fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)); 578 } while (!(txdctl & FM10K_TXDCTL_ENABLE) && --wait_loop); 579 if (!wait_loop) 580 netif_err(interface, drv, interface->netdev, 581 "Could not enable Tx Queue %d\n", reg_idx); 582} 583 584/** 585 * fm10k_configure_tx - Configure Transmit Unit after Reset 586 * @interface: board private structure 587 * 588 * Configure the Tx unit of the MAC after a reset. 589 **/ 590static void fm10k_configure_tx(struct fm10k_intfc *interface) 591{ 592 int i; 593 594 /* Setup the HW Tx Head and Tail descriptor pointers */ 595 for (i = 0; i < interface->num_tx_queues; i++) 596 fm10k_configure_tx_ring(interface, interface->tx_ring[i]); 597 598 /* poll here to verify that Tx rings are now enabled */ 599 for (i = 0; i < interface->num_tx_queues; i++) 600 fm10k_enable_tx_ring(interface, interface->tx_ring[i]); 601} 602 603/** 604 * fm10k_configure_rx_ring - Configure Rx ring after Reset 605 * @interface: board private structure 606 * @ring: structure containing ring specific data 607 * 608 * Configure the Rx descriptor ring after a reset. 609 **/ 610static void fm10k_configure_rx_ring(struct fm10k_intfc *interface, 611 struct fm10k_ring *ring) 612{ 613 u64 rdba = ring->dma; 614 struct fm10k_hw *hw = &interface->hw; 615 u32 size = ring->count * sizeof(union fm10k_rx_desc); 616 u32 rxqctl = FM10K_RXQCTL_ENABLE | FM10K_RXQCTL_PF; 617 u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 618 u32 srrctl = FM10K_SRRCTL_BUFFER_CHAINING_EN; 619 u32 rxint = FM10K_INT_MAP_DISABLE; 620 u8 rx_pause = interface->rx_pause; 621 u8 reg_idx = ring->reg_idx; 622 623 /* disable queue to avoid issues while updating state */ 624 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), 0); 625 fm10k_write_flush(hw); 626 627 /* possible poll here to verify ring resources have been cleaned */ 628 629 /* set location and size for descriptor ring */ 630 fm10k_write_reg(hw, FM10K_RDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 631 fm10k_write_reg(hw, FM10K_RDBAH(reg_idx), rdba >> 32); 632 fm10k_write_reg(hw, FM10K_RDLEN(reg_idx), size); 633 634 /* reset head and tail pointers */ 635 fm10k_write_reg(hw, FM10K_RDH(reg_idx), 0); 636 fm10k_write_reg(hw, FM10K_RDT(reg_idx), 0); 637 638 /* store tail pointer */ 639 ring->tail = &interface->uc_addr[FM10K_RDT(reg_idx)]; 640 641 /* reset ntu and ntc to place SW in sync with hardwdare */ 642 ring->next_to_clean = 0; 643 ring->next_to_use = 0; 644 ring->next_to_alloc = 0; 645 646 /* Configure the Rx buffer size for one buff without split */ 647 srrctl |= FM10K_RX_BUFSZ >> FM10K_SRRCTL_BSIZEPKT_SHIFT; 648 649 /* Configure the Rx ring to suppress loopback packets */ 650 srrctl |= FM10K_SRRCTL_LOOPBACK_SUPPRESS; 651 fm10k_write_reg(hw, FM10K_SRRCTL(reg_idx), srrctl); 652 653 /* Enable drop on empty */ 654#ifdef CONFIG_DCB 655 if (interface->pfc_en) 656 rx_pause = interface->pfc_en; 657#endif 658 if (!(rx_pause & (1 << ring->qos_pc))) 659 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 660 661 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 662 663 /* assign default VLAN to queue */ 664 ring->vid = hw->mac.default_vid; 665 666 /* Map interrupt */ 667 if (ring->q_vector) { 668 rxint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 669 rxint |= FM10K_INT_MAP_TIMER1; 670 } 671 672 fm10k_write_reg(hw, FM10K_RXINT(reg_idx), rxint); 673 674 /* enable queue */ 675 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl); 676 677 /* place buffers on ring for receive data */ 678 fm10k_alloc_rx_buffers(ring, fm10k_desc_unused(ring)); 679} 680 681/** 682 * fm10k_update_rx_drop_en - Configures the drop enable bits for Rx rings 683 * @interface: board private structure 684 * 685 * Configure the drop enable bits for the Rx rings. 686 **/ 687void fm10k_update_rx_drop_en(struct fm10k_intfc *interface) 688{ 689 struct fm10k_hw *hw = &interface->hw; 690 u8 rx_pause = interface->rx_pause; 691 int i; 692 693#ifdef CONFIG_DCB 694 if (interface->pfc_en) 695 rx_pause = interface->pfc_en; 696 697#endif 698 for (i = 0; i < interface->num_rx_queues; i++) { 699 struct fm10k_ring *ring = interface->rx_ring[i]; 700 u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 701 u8 reg_idx = ring->reg_idx; 702 703 if (!(rx_pause & (1 << ring->qos_pc))) 704 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 705 706 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 707 } 708} 709 710/** 711 * fm10k_configure_dglort - Configure Receive DGLORT after reset 712 * @interface: board private structure 713 * 714 * Configure the DGLORT description and RSS tables. 715 **/ 716static void fm10k_configure_dglort(struct fm10k_intfc *interface) 717{ 718 struct fm10k_dglort_cfg dglort = { 0 }; 719 struct fm10k_hw *hw = &interface->hw; 720 int i; 721 u32 mrqc; 722 723 /* Fill out hash function seeds */ 724 for (i = 0; i < FM10K_RSSRK_SIZE; i++) 725 fm10k_write_reg(hw, FM10K_RSSRK(0, i), interface->rssrk[i]); 726 727 /* Write RETA table to hardware */ 728 for (i = 0; i < FM10K_RETA_SIZE; i++) 729 fm10k_write_reg(hw, FM10K_RETA(0, i), interface->reta[i]); 730 731 /* Generate RSS hash based on packet types, TCP/UDP 732 * port numbers and/or IPv4/v6 src and dst addresses 733 */ 734 mrqc = FM10K_MRQC_IPV4 | 735 FM10K_MRQC_TCP_IPV4 | 736 FM10K_MRQC_IPV6 | 737 FM10K_MRQC_TCP_IPV6; 738 739 if (interface->flags & FM10K_FLAG_RSS_FIELD_IPV4_UDP) 740 mrqc |= FM10K_MRQC_UDP_IPV4; 741 if (interface->flags & FM10K_FLAG_RSS_FIELD_IPV6_UDP) 742 mrqc |= FM10K_MRQC_UDP_IPV6; 743 744 fm10k_write_reg(hw, FM10K_MRQC(0), mrqc); 745 746 /* configure default DGLORT mapping for RSS/DCB */ 747 dglort.inner_rss = 1; 748 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 749 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 750 hw->mac.ops.configure_dglort_map(hw, &dglort); 751 752 /* assign GLORT per queue for queue mapped testing */ 753 if (interface->glort_count > 64) { 754 memset(&dglort, 0, sizeof(dglort)); 755 dglort.inner_rss = 1; 756 dglort.glort = interface->glort + 64; 757 dglort.idx = fm10k_dglort_pf_queue; 758 dglort.queue_l = fls(interface->num_rx_queues - 1); 759 hw->mac.ops.configure_dglort_map(hw, &dglort); 760 } 761 762 /* assign glort value for RSS/DCB specific to this interface */ 763 memset(&dglort, 0, sizeof(dglort)); 764 dglort.inner_rss = 1; 765 dglort.glort = interface->glort; 766 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 767 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 768 /* configure DGLORT mapping for RSS/DCB */ 769 dglort.idx = fm10k_dglort_pf_rss; 770 if (interface->l2_accel) 771 dglort.shared_l = fls(interface->l2_accel->size); 772 hw->mac.ops.configure_dglort_map(hw, &dglort); 773} 774 775/** 776 * fm10k_configure_rx - Configure Receive Unit after Reset 777 * @interface: board private structure 778 * 779 * Configure the Rx unit of the MAC after a reset. 780 **/ 781static void fm10k_configure_rx(struct fm10k_intfc *interface) 782{ 783 int i; 784 785 /* Configure SWPRI to PC map */ 786 fm10k_configure_swpri_map(interface); 787 788 /* Configure RSS and DGLORT map */ 789 fm10k_configure_dglort(interface); 790 791 /* Setup the HW Rx Head and Tail descriptor pointers */ 792 for (i = 0; i < interface->num_rx_queues; i++) 793 fm10k_configure_rx_ring(interface, interface->rx_ring[i]); 794 795 /* possible poll here to verify that Rx rings are now enabled */ 796} 797 798static void fm10k_napi_enable_all(struct fm10k_intfc *interface) 799{ 800 struct fm10k_q_vector *q_vector; 801 int q_idx; 802 803 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 804 q_vector = interface->q_vector[q_idx]; 805 napi_enable(&q_vector->napi); 806 } 807} 808 809static irqreturn_t fm10k_msix_clean_rings(int __always_unused irq, void *data) 810{ 811 struct fm10k_q_vector *q_vector = data; 812 813 if (q_vector->rx.count || q_vector->tx.count) 814 napi_schedule(&q_vector->napi); 815 816 return IRQ_HANDLED; 817} 818 819static irqreturn_t fm10k_msix_mbx_vf(int __always_unused irq, void *data) 820{ 821 struct fm10k_intfc *interface = data; 822 struct fm10k_hw *hw = &interface->hw; 823 struct fm10k_mbx_info *mbx = &hw->mbx; 824 825 /* re-enable mailbox interrupt and indicate 20us delay */ 826 fm10k_write_reg(hw, FM10K_VFITR(FM10K_MBX_VECTOR), 827 FM10K_ITR_ENABLE | FM10K_MBX_INT_DELAY); 828 829 /* service upstream mailbox */ 830 if (fm10k_mbx_trylock(interface)) { 831 mbx->ops.process(hw, mbx); 832 fm10k_mbx_unlock(interface); 833 } 834 835 hw->mac.get_host_state = 1; 836 fm10k_service_event_schedule(interface); 837 838 return IRQ_HANDLED; 839} 840 841#ifdef CONFIG_NET_POLL_CONTROLLER 842/** 843 * fm10k_netpoll - A Polling 'interrupt' handler 844 * @netdev: network interface device structure 845 * 846 * This is used by netconsole to send skbs without having to re-enable 847 * interrupts. It's not called while the normal interrupt routine is executing. 848 **/ 849void fm10k_netpoll(struct net_device *netdev) 850{ 851 struct fm10k_intfc *interface = netdev_priv(netdev); 852 int i; 853 854 /* if interface is down do nothing */ 855 if (test_bit(__FM10K_DOWN, &interface->state)) 856 return; 857 858 for (i = 0; i < interface->num_q_vectors; i++) 859 fm10k_msix_clean_rings(0, interface->q_vector[i]); 860} 861 862#endif 863#define FM10K_ERR_MSG(type) case (type): error = #type; break 864static void fm10k_print_fault(struct fm10k_intfc *interface, int type, 865 struct fm10k_fault *fault) 866{ 867 struct pci_dev *pdev = interface->pdev; 868 char *error; 869 870 switch (type) { 871 case FM10K_PCA_FAULT: 872 switch (fault->type) { 873 default: 874 error = "Unknown PCA error"; 875 break; 876 FM10K_ERR_MSG(PCA_NO_FAULT); 877 FM10K_ERR_MSG(PCA_UNMAPPED_ADDR); 878 FM10K_ERR_MSG(PCA_BAD_QACCESS_PF); 879 FM10K_ERR_MSG(PCA_BAD_QACCESS_VF); 880 FM10K_ERR_MSG(PCA_MALICIOUS_REQ); 881 FM10K_ERR_MSG(PCA_POISONED_TLP); 882 FM10K_ERR_MSG(PCA_TLP_ABORT); 883 } 884 break; 885 case FM10K_THI_FAULT: 886 switch (fault->type) { 887 default: 888 error = "Unknown THI error"; 889 break; 890 FM10K_ERR_MSG(THI_NO_FAULT); 891 FM10K_ERR_MSG(THI_MAL_DIS_Q_FAULT); 892 } 893 break; 894 case FM10K_FUM_FAULT: 895 switch (fault->type) { 896 default: 897 error = "Unknown FUM error"; 898 break; 899 FM10K_ERR_MSG(FUM_NO_FAULT); 900 FM10K_ERR_MSG(FUM_UNMAPPED_ADDR); 901 FM10K_ERR_MSG(FUM_BAD_VF_QACCESS); 902 FM10K_ERR_MSG(FUM_ADD_DECODE_ERR); 903 FM10K_ERR_MSG(FUM_RO_ERROR); 904 FM10K_ERR_MSG(FUM_QPRC_CRC_ERROR); 905 FM10K_ERR_MSG(FUM_CSR_TIMEOUT); 906 FM10K_ERR_MSG(FUM_INVALID_TYPE); 907 FM10K_ERR_MSG(FUM_INVALID_LENGTH); 908 FM10K_ERR_MSG(FUM_INVALID_BE); 909 FM10K_ERR_MSG(FUM_INVALID_ALIGN); 910 } 911 break; 912 default: 913 error = "Undocumented fault"; 914 break; 915 } 916 917 dev_warn(&pdev->dev, 918 "%s Address: 0x%llx SpecInfo: 0x%x Func: %02x.%0x\n", 919 error, fault->address, fault->specinfo, 920 PCI_SLOT(fault->func), PCI_FUNC(fault->func)); 921} 922 923static void fm10k_report_fault(struct fm10k_intfc *interface, u32 eicr) 924{ 925 struct fm10k_hw *hw = &interface->hw; 926 struct fm10k_fault fault = { 0 }; 927 int type, err; 928 929 for (eicr &= FM10K_EICR_FAULT_MASK, type = FM10K_PCA_FAULT; 930 eicr; 931 eicr >>= 1, type += FM10K_FAULT_SIZE) { 932 /* only check if there is an error reported */ 933 if (!(eicr & 0x1)) 934 continue; 935 936 /* retrieve fault info */ 937 err = hw->mac.ops.get_fault(hw, type, &fault); 938 if (err) { 939 dev_err(&interface->pdev->dev, 940 "error reading fault\n"); 941 continue; 942 } 943 944 fm10k_print_fault(interface, type, &fault); 945 } 946} 947 948static void fm10k_reset_drop_on_empty(struct fm10k_intfc *interface, u32 eicr) 949{ 950 struct fm10k_hw *hw = &interface->hw; 951 const u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 952 u32 maxholdq; 953 int q; 954 955 if (!(eicr & FM10K_EICR_MAXHOLDTIME)) 956 return; 957 958 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(7)); 959 if (maxholdq) 960 fm10k_write_reg(hw, FM10K_MAXHOLDQ(7), maxholdq); 961 for (q = 255;;) { 962 if (maxholdq & (1 << 31)) { 963 if (q < FM10K_MAX_QUEUES_PF) { 964 interface->rx_overrun_pf++; 965 fm10k_write_reg(hw, FM10K_RXDCTL(q), rxdctl); 966 } else { 967 interface->rx_overrun_vf++; 968 } 969 } 970 971 maxholdq *= 2; 972 if (!maxholdq) 973 q &= ~(32 - 1); 974 975 if (!q) 976 break; 977 978 if (q-- % 32) 979 continue; 980 981 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(q / 32)); 982 if (maxholdq) 983 fm10k_write_reg(hw, FM10K_MAXHOLDQ(q / 32), maxholdq); 984 } 985} 986 987static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data) 988{ 989 struct fm10k_intfc *interface = data; 990 struct fm10k_hw *hw = &interface->hw; 991 struct fm10k_mbx_info *mbx = &hw->mbx; 992 u32 eicr; 993 994 /* unmask any set bits related to this interrupt */ 995 eicr = fm10k_read_reg(hw, FM10K_EICR); 996 fm10k_write_reg(hw, FM10K_EICR, eicr & (FM10K_EICR_MAILBOX | 997 FM10K_EICR_SWITCHREADY | 998 FM10K_EICR_SWITCHNOTREADY)); 999 1000 /* report any faults found to the message log */ 1001 fm10k_report_fault(interface, eicr); 1002 1003 /* reset any queues disabled due to receiver overrun */ 1004 fm10k_reset_drop_on_empty(interface, eicr); 1005 1006 /* service mailboxes */ 1007 if (fm10k_mbx_trylock(interface)) { 1008 mbx->ops.process(hw, mbx); 1009 /* handle VFLRE events */ 1010 fm10k_iov_event(interface); 1011 fm10k_mbx_unlock(interface); 1012 } 1013 1014 /* if switch toggled state we should reset GLORTs */ 1015 if (eicr & FM10K_EICR_SWITCHNOTREADY) { 1016 /* force link down for at least 4 seconds */ 1017 interface->link_down_event = jiffies + (4 * HZ); 1018 set_bit(__FM10K_LINK_DOWN, &interface->state); 1019 1020 /* reset dglort_map back to no config */ 1021 hw->mac.dglort_map = FM10K_DGLORTMAP_NONE; 1022 } 1023 1024 /* we should validate host state after interrupt event */ 1025 hw->mac.get_host_state = 1; 1026 1027 /* validate host state, and handle VF mailboxes in the service task */ 1028 fm10k_service_event_schedule(interface); 1029 1030 /* re-enable mailbox interrupt and indicate 20us delay */ 1031 fm10k_write_reg(hw, FM10K_ITR(FM10K_MBX_VECTOR), 1032 FM10K_ITR_ENABLE | FM10K_MBX_INT_DELAY); 1033 1034 return IRQ_HANDLED; 1035} 1036 1037void fm10k_mbx_free_irq(struct fm10k_intfc *interface) 1038{ 1039 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1040 struct fm10k_hw *hw = &interface->hw; 1041 int itr_reg; 1042 1043 /* disconnect the mailbox */ 1044 hw->mbx.ops.disconnect(hw, &hw->mbx); 1045 1046 /* disable Mailbox cause */ 1047 if (hw->mac.type == fm10k_mac_pf) { 1048 fm10k_write_reg(hw, FM10K_EIMR, 1049 FM10K_EIMR_DISABLE(PCA_FAULT) | 1050 FM10K_EIMR_DISABLE(FUM_FAULT) | 1051 FM10K_EIMR_DISABLE(MAILBOX) | 1052 FM10K_EIMR_DISABLE(SWITCHREADY) | 1053 FM10K_EIMR_DISABLE(SWITCHNOTREADY) | 1054 FM10K_EIMR_DISABLE(SRAMERROR) | 1055 FM10K_EIMR_DISABLE(VFLR) | 1056 FM10K_EIMR_DISABLE(MAXHOLDTIME)); 1057 itr_reg = FM10K_ITR(FM10K_MBX_VECTOR); 1058 } else { 1059 itr_reg = FM10K_VFITR(FM10K_MBX_VECTOR); 1060 } 1061 1062 fm10k_write_reg(hw, itr_reg, FM10K_ITR_MASK_SET); 1063 1064 free_irq(entry->vector, interface); 1065} 1066 1067static s32 fm10k_mbx_mac_addr(struct fm10k_hw *hw, u32 **results, 1068 struct fm10k_mbx_info *mbx) 1069{ 1070 bool vlan_override = hw->mac.vlan_override; 1071 u16 default_vid = hw->mac.default_vid; 1072 struct fm10k_intfc *interface; 1073 s32 err; 1074 1075 err = fm10k_msg_mac_vlan_vf(hw, results, mbx); 1076 if (err) 1077 return err; 1078 1079 interface = container_of(hw, struct fm10k_intfc, hw); 1080 1081 /* MAC was changed so we need reset */ 1082 if (is_valid_ether_addr(hw->mac.perm_addr) && 1083 memcmp(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN)) 1084 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1085 1086 /* VLAN override was changed, or default VLAN changed */ 1087 if ((vlan_override != hw->mac.vlan_override) || 1088 (default_vid != hw->mac.default_vid)) 1089 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1090 1091 return 0; 1092} 1093 1094static s32 fm10k_1588_msg_vf(struct fm10k_hw *hw, u32 **results, 1095 struct fm10k_mbx_info __always_unused *mbx) 1096{ 1097 struct fm10k_intfc *interface; 1098 u64 timestamp; 1099 s32 err; 1100 1101 err = fm10k_tlv_attr_get_u64(results[FM10K_1588_MSG_TIMESTAMP], 1102 ×tamp); 1103 if (err) 1104 return err; 1105 1106 interface = container_of(hw, struct fm10k_intfc, hw); 1107 1108 fm10k_ts_tx_hwtstamp(interface, 0, timestamp); 1109 1110 return 0; 1111} 1112 1113/* generic error handler for mailbox issues */ 1114static s32 fm10k_mbx_error(struct fm10k_hw *hw, u32 **results, 1115 struct fm10k_mbx_info __always_unused *mbx) 1116{ 1117 struct fm10k_intfc *interface; 1118 struct pci_dev *pdev; 1119 1120 interface = container_of(hw, struct fm10k_intfc, hw); 1121 pdev = interface->pdev; 1122 1123 dev_err(&pdev->dev, "Unknown message ID %u\n", 1124 **results & FM10K_TLV_ID_MASK); 1125 1126 return 0; 1127} 1128 1129static const struct fm10k_msg_data vf_mbx_data[] = { 1130 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test), 1131 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_mbx_mac_addr), 1132 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf), 1133 FM10K_VF_MSG_1588_HANDLER(fm10k_1588_msg_vf), 1134 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1135}; 1136 1137static int fm10k_mbx_request_irq_vf(struct fm10k_intfc *interface) 1138{ 1139 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1140 struct net_device *dev = interface->netdev; 1141 struct fm10k_hw *hw = &interface->hw; 1142 int err; 1143 1144 /* Use timer0 for interrupt moderation on the mailbox */ 1145 u32 itr = FM10K_INT_MAP_TIMER0 | entry->entry; 1146 1147 /* register mailbox handlers */ 1148 err = hw->mbx.ops.register_handlers(&hw->mbx, vf_mbx_data); 1149 if (err) 1150 return err; 1151 1152 /* request the IRQ */ 1153 err = request_irq(entry->vector, fm10k_msix_mbx_vf, 0, 1154 dev->name, interface); 1155 if (err) { 1156 netif_err(interface, probe, dev, 1157 "request_irq for msix_mbx failed: %d\n", err); 1158 return err; 1159 } 1160 1161 /* map all of the interrupt sources */ 1162 fm10k_write_reg(hw, FM10K_VFINT_MAP, itr); 1163 1164 /* enable interrupt */ 1165 fm10k_write_reg(hw, FM10K_VFITR(entry->entry), FM10K_ITR_ENABLE); 1166 1167 return 0; 1168} 1169 1170static s32 fm10k_lport_map(struct fm10k_hw *hw, u32 **results, 1171 struct fm10k_mbx_info *mbx) 1172{ 1173 struct fm10k_intfc *interface; 1174 u32 dglort_map = hw->mac.dglort_map; 1175 s32 err; 1176 1177 err = fm10k_msg_lport_map_pf(hw, results, mbx); 1178 if (err) 1179 return err; 1180 1181 interface = container_of(hw, struct fm10k_intfc, hw); 1182 1183 /* we need to reset if port count was just updated */ 1184 if (dglort_map != hw->mac.dglort_map) 1185 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1186 1187 return 0; 1188} 1189 1190static s32 fm10k_update_pvid(struct fm10k_hw *hw, u32 **results, 1191 struct fm10k_mbx_info __always_unused *mbx) 1192{ 1193 struct fm10k_intfc *interface; 1194 u16 glort, pvid; 1195 u32 pvid_update; 1196 s32 err; 1197 1198 err = fm10k_tlv_attr_get_u32(results[FM10K_PF_ATTR_ID_UPDATE_PVID], 1199 &pvid_update); 1200 if (err) 1201 return err; 1202 1203 /* extract values from the pvid update */ 1204 glort = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_GLORT); 1205 pvid = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_PVID); 1206 1207 /* if glort is not valid return error */ 1208 if (!fm10k_glort_valid_pf(hw, glort)) 1209 return FM10K_ERR_PARAM; 1210 1211 /* verify VID is valid */ 1212 if (pvid >= FM10K_VLAN_TABLE_VID_MAX) 1213 return FM10K_ERR_PARAM; 1214 1215 interface = container_of(hw, struct fm10k_intfc, hw); 1216 1217 /* check to see if this belongs to one of the VFs */ 1218 err = fm10k_iov_update_pvid(interface, glort, pvid); 1219 if (!err) 1220 return 0; 1221 1222 /* we need to reset if default VLAN was just updated */ 1223 if (pvid != hw->mac.default_vid) 1224 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1225 1226 hw->mac.default_vid = pvid; 1227 1228 return 0; 1229} 1230 1231static s32 fm10k_1588_msg_pf(struct fm10k_hw *hw, u32 **results, 1232 struct fm10k_mbx_info __always_unused *mbx) 1233{ 1234 struct fm10k_swapi_1588_timestamp timestamp; 1235 struct fm10k_iov_data *iov_data; 1236 struct fm10k_intfc *interface; 1237 u16 sglort, vf_idx; 1238 s32 err; 1239 1240 err = fm10k_tlv_attr_get_le_struct( 1241 results[FM10K_PF_ATTR_ID_1588_TIMESTAMP], 1242 ×tamp, sizeof(timestamp)); 1243 if (err) 1244 return err; 1245 1246 interface = container_of(hw, struct fm10k_intfc, hw); 1247 1248 if (timestamp.dglort) { 1249 fm10k_ts_tx_hwtstamp(interface, timestamp.dglort, 1250 le64_to_cpu(timestamp.egress)); 1251 return 0; 1252 } 1253 1254 /* either dglort or sglort must be set */ 1255 if (!timestamp.sglort) 1256 return FM10K_ERR_PARAM; 1257 1258 /* verify GLORT is at least one of the ones we own */ 1259 sglort = le16_to_cpu(timestamp.sglort); 1260 if (!fm10k_glort_valid_pf(hw, sglort)) 1261 return FM10K_ERR_PARAM; 1262 1263 if (sglort == interface->glort) { 1264 fm10k_ts_tx_hwtstamp(interface, 0, 1265 le64_to_cpu(timestamp.ingress)); 1266 return 0; 1267 } 1268 1269 /* if there is no iov_data then there is no mailboxes to process */ 1270 if (!ACCESS_ONCE(interface->iov_data)) 1271 return FM10K_ERR_PARAM; 1272 1273 rcu_read_lock(); 1274 1275 /* notify VF if this timestamp belongs to it */ 1276 iov_data = interface->iov_data; 1277 vf_idx = (hw->mac.dglort_map & FM10K_DGLORTMAP_NONE) - sglort; 1278 1279 if (!iov_data || vf_idx >= iov_data->num_vfs) { 1280 err = FM10K_ERR_PARAM; 1281 goto err_unlock; 1282 } 1283 1284 err = hw->iov.ops.report_timestamp(hw, &iov_data->vf_info[vf_idx], 1285 le64_to_cpu(timestamp.ingress)); 1286 1287err_unlock: 1288 rcu_read_unlock(); 1289 1290 return err; 1291} 1292 1293static const struct fm10k_msg_data pf_mbx_data[] = { 1294 FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf), 1295 FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf), 1296 FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_lport_map), 1297 FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf), 1298 FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf), 1299 FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_update_pvid), 1300 FM10K_PF_MSG_1588_TIMESTAMP_HANDLER(fm10k_1588_msg_pf), 1301 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1302}; 1303 1304static int fm10k_mbx_request_irq_pf(struct fm10k_intfc *interface) 1305{ 1306 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1307 struct net_device *dev = interface->netdev; 1308 struct fm10k_hw *hw = &interface->hw; 1309 int err; 1310 1311 /* Use timer0 for interrupt moderation on the mailbox */ 1312 u32 mbx_itr = FM10K_INT_MAP_TIMER0 | entry->entry; 1313 u32 other_itr = FM10K_INT_MAP_IMMEDIATE | entry->entry; 1314 1315 /* register mailbox handlers */ 1316 err = hw->mbx.ops.register_handlers(&hw->mbx, pf_mbx_data); 1317 if (err) 1318 return err; 1319 1320 /* request the IRQ */ 1321 err = request_irq(entry->vector, fm10k_msix_mbx_pf, 0, 1322 dev->name, interface); 1323 if (err) { 1324 netif_err(interface, probe, dev, 1325 "request_irq for msix_mbx failed: %d\n", err); 1326 return err; 1327 } 1328 1329 /* Enable interrupts w/ no moderation for "other" interrupts */ 1330 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), other_itr); 1331 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), other_itr); 1332 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_SRAM), other_itr); 1333 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_MaxHoldTime), other_itr); 1334 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_VFLR), other_itr); 1335 1336 /* Enable interrupts w/ moderation for mailbox */ 1337 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_Mailbox), mbx_itr); 1338 1339 /* Enable individual interrupt causes */ 1340 fm10k_write_reg(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) | 1341 FM10K_EIMR_ENABLE(FUM_FAULT) | 1342 FM10K_EIMR_ENABLE(MAILBOX) | 1343 FM10K_EIMR_ENABLE(SWITCHREADY) | 1344 FM10K_EIMR_ENABLE(SWITCHNOTREADY) | 1345 FM10K_EIMR_ENABLE(SRAMERROR) | 1346 FM10K_EIMR_ENABLE(VFLR) | 1347 FM10K_EIMR_ENABLE(MAXHOLDTIME)); 1348 1349 /* enable interrupt */ 1350 fm10k_write_reg(hw, FM10K_ITR(entry->entry), FM10K_ITR_ENABLE); 1351 1352 return 0; 1353} 1354 1355int fm10k_mbx_request_irq(struct fm10k_intfc *interface) 1356{ 1357 struct fm10k_hw *hw = &interface->hw; 1358 int err; 1359 1360 /* enable Mailbox cause */ 1361 if (hw->mac.type == fm10k_mac_pf) 1362 err = fm10k_mbx_request_irq_pf(interface); 1363 else 1364 err = fm10k_mbx_request_irq_vf(interface); 1365 1366 /* connect mailbox */ 1367 if (!err) 1368 err = hw->mbx.ops.connect(hw, &hw->mbx); 1369 1370 return err; 1371} 1372 1373/** 1374 * fm10k_qv_free_irq - release interrupts associated with queue vectors 1375 * @interface: board private structure 1376 * 1377 * Release all interrupts associated with this interface 1378 **/ 1379void fm10k_qv_free_irq(struct fm10k_intfc *interface) 1380{ 1381 int vector = interface->num_q_vectors; 1382 struct fm10k_hw *hw = &interface->hw; 1383 struct msix_entry *entry; 1384 1385 entry = &interface->msix_entries[NON_Q_VECTORS(hw) + vector]; 1386 1387 while (vector) { 1388 struct fm10k_q_vector *q_vector; 1389 1390 vector--; 1391 entry--; 1392 q_vector = interface->q_vector[vector]; 1393 1394 if (!q_vector->tx.count && !q_vector->rx.count) 1395 continue; 1396 1397 /* disable interrupts */ 1398 1399 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1400 1401 free_irq(entry->vector, q_vector); 1402 } 1403} 1404 1405/** 1406 * fm10k_qv_request_irq - initialize interrupts for queue vectors 1407 * @interface: board private structure 1408 * 1409 * Attempts to configure interrupts using the best available 1410 * capabilities of the hardware and kernel. 1411 **/ 1412int fm10k_qv_request_irq(struct fm10k_intfc *interface) 1413{ 1414 struct net_device *dev = interface->netdev; 1415 struct fm10k_hw *hw = &interface->hw; 1416 struct msix_entry *entry; 1417 int ri = 0, ti = 0; 1418 int vector, err; 1419 1420 entry = &interface->msix_entries[NON_Q_VECTORS(hw)]; 1421 1422 for (vector = 0; vector < interface->num_q_vectors; vector++) { 1423 struct fm10k_q_vector *q_vector = interface->q_vector[vector]; 1424 1425 /* name the vector */ 1426 if (q_vector->tx.count && q_vector->rx.count) { 1427 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1428 "%s-TxRx-%d", dev->name, ri++); 1429 ti++; 1430 } else if (q_vector->rx.count) { 1431 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1432 "%s-rx-%d", dev->name, ri++); 1433 } else if (q_vector->tx.count) { 1434 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1435 "%s-tx-%d", dev->name, ti++); 1436 } else { 1437 /* skip this unused q_vector */ 1438 continue; 1439 } 1440 1441 /* Assign ITR register to q_vector */ 1442 q_vector->itr = (hw->mac.type == fm10k_mac_pf) ? 1443 &interface->uc_addr[FM10K_ITR(entry->entry)] : 1444 &interface->uc_addr[FM10K_VFITR(entry->entry)]; 1445 1446 /* request the IRQ */ 1447 err = request_irq(entry->vector, &fm10k_msix_clean_rings, 0, 1448 q_vector->name, q_vector); 1449 if (err) { 1450 netif_err(interface, probe, dev, 1451 "request_irq failed for MSIX interrupt Error: %d\n", 1452 err); 1453 goto err_out; 1454 } 1455 1456 /* Enable q_vector */ 1457 writel(FM10K_ITR_ENABLE, q_vector->itr); 1458 1459 entry++; 1460 } 1461 1462 return 0; 1463 1464err_out: 1465 /* wind through the ring freeing all entries and vectors */ 1466 while (vector) { 1467 struct fm10k_q_vector *q_vector; 1468 1469 entry--; 1470 vector--; 1471 q_vector = interface->q_vector[vector]; 1472 1473 if (!q_vector->tx.count && !q_vector->rx.count) 1474 continue; 1475 1476 /* disable interrupts */ 1477 1478 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1479 1480 free_irq(entry->vector, q_vector); 1481 } 1482 1483 return err; 1484} 1485 1486void fm10k_up(struct fm10k_intfc *interface) 1487{ 1488 struct fm10k_hw *hw = &interface->hw; 1489 1490 /* Enable Tx/Rx DMA */ 1491 hw->mac.ops.start_hw(hw); 1492 1493 /* configure Tx descriptor rings */ 1494 fm10k_configure_tx(interface); 1495 1496 /* configure Rx descriptor rings */ 1497 fm10k_configure_rx(interface); 1498 1499 /* configure interrupts */ 1500 hw->mac.ops.update_int_moderator(hw); 1501 1502 /* clear down bit to indicate we are ready to go */ 1503 clear_bit(__FM10K_DOWN, &interface->state); 1504 1505 /* enable polling cleanups */ 1506 fm10k_napi_enable_all(interface); 1507 1508 /* re-establish Rx filters */ 1509 fm10k_restore_rx_state(interface); 1510 1511 /* enable transmits */ 1512 netif_tx_start_all_queues(interface->netdev); 1513 1514 /* kick off the service timer now */ 1515 hw->mac.get_host_state = 1; 1516 mod_timer(&interface->service_timer, jiffies); 1517} 1518 1519static void fm10k_napi_disable_all(struct fm10k_intfc *interface) 1520{ 1521 struct fm10k_q_vector *q_vector; 1522 int q_idx; 1523 1524 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 1525 q_vector = interface->q_vector[q_idx]; 1526 napi_disable(&q_vector->napi); 1527 } 1528} 1529 1530void fm10k_down(struct fm10k_intfc *interface) 1531{ 1532 struct net_device *netdev = interface->netdev; 1533 struct fm10k_hw *hw = &interface->hw; 1534 1535 /* signal that we are down to the interrupt handler and service task */ 1536 set_bit(__FM10K_DOWN, &interface->state); 1537 1538 /* call carrier off first to avoid false dev_watchdog timeouts */ 1539 netif_carrier_off(netdev); 1540 1541 /* disable transmits */ 1542 netif_tx_stop_all_queues(netdev); 1543 netif_tx_disable(netdev); 1544 1545 /* reset Rx filters */ 1546 fm10k_reset_rx_state(interface); 1547 1548 /* allow 10ms for device to quiesce */ 1549 usleep_range(10000, 20000); 1550 1551 /* disable polling routines */ 1552 fm10k_napi_disable_all(interface); 1553 1554 /* capture stats one last time before stopping interface */ 1555 fm10k_update_stats(interface); 1556 1557 /* Disable DMA engine for Tx/Rx */ 1558 hw->mac.ops.stop_hw(hw); 1559 1560 /* free any buffers still on the rings */ 1561 fm10k_clean_all_tx_rings(interface); 1562} 1563 1564/** 1565 * fm10k_sw_init - Initialize general software structures 1566 * @interface: host interface private structure to initialize 1567 * 1568 * fm10k_sw_init initializes the interface private data structure. 1569 * Fields are initialized based on PCI device information and 1570 * OS network device settings (MTU size). 1571 **/ 1572static int fm10k_sw_init(struct fm10k_intfc *interface, 1573 const struct pci_device_id *ent) 1574{ 1575 const struct fm10k_info *fi = fm10k_info_tbl[ent->driver_data]; 1576 struct fm10k_hw *hw = &interface->hw; 1577 struct pci_dev *pdev = interface->pdev; 1578 struct net_device *netdev = interface->netdev; 1579 u32 rss_key[FM10K_RSSRK_SIZE]; 1580 unsigned int rss; 1581 int err; 1582 1583 /* initialize back pointer */ 1584 hw->back = interface; 1585 hw->hw_addr = interface->uc_addr; 1586 1587 /* PCI config space info */ 1588 hw->vendor_id = pdev->vendor; 1589 hw->device_id = pdev->device; 1590 hw->revision_id = pdev->revision; 1591 hw->subsystem_vendor_id = pdev->subsystem_vendor; 1592 hw->subsystem_device_id = pdev->subsystem_device; 1593 1594 /* Setup hw api */ 1595 memcpy(&hw->mac.ops, fi->mac_ops, sizeof(hw->mac.ops)); 1596 hw->mac.type = fi->mac; 1597 1598 /* Setup IOV handlers */ 1599 if (fi->iov_ops) 1600 memcpy(&hw->iov.ops, fi->iov_ops, sizeof(hw->iov.ops)); 1601 1602 /* Set common capability flags and settings */ 1603 rss = min_t(int, FM10K_MAX_RSS_INDICES, num_online_cpus()); 1604 interface->ring_feature[RING_F_RSS].limit = rss; 1605 fi->get_invariants(hw); 1606 1607 /* pick up the PCIe bus settings for reporting later */ 1608 if (hw->mac.ops.get_bus_info) 1609 hw->mac.ops.get_bus_info(hw); 1610 1611 /* limit the usable DMA range */ 1612 if (hw->mac.ops.set_dma_mask) 1613 hw->mac.ops.set_dma_mask(hw, dma_get_mask(&pdev->dev)); 1614 1615 /* update netdev with DMA restrictions */ 1616 if (dma_get_mask(&pdev->dev) > DMA_BIT_MASK(32)) { 1617 netdev->features |= NETIF_F_HIGHDMA; 1618 netdev->vlan_features |= NETIF_F_HIGHDMA; 1619 } 1620 1621 /* delay any future reset requests */ 1622 interface->last_reset = jiffies + (10 * HZ); 1623 1624 /* reset and initialize the hardware so it is in a known state */ 1625 err = hw->mac.ops.reset_hw(hw) ? : hw->mac.ops.init_hw(hw); 1626 if (err) { 1627 dev_err(&pdev->dev, "init_hw failed: %d\n", err); 1628 return err; 1629 } 1630 1631 /* initialize hardware statistics */ 1632 hw->mac.ops.update_hw_stats(hw, &interface->stats); 1633 1634 /* Set upper limit on IOV VFs that can be allocated */ 1635 pci_sriov_set_totalvfs(pdev, hw->iov.total_vfs); 1636 1637 /* Start with random Ethernet address */ 1638 eth_random_addr(hw->mac.addr); 1639 1640 /* Initialize MAC address from hardware */ 1641 err = hw->mac.ops.read_mac_addr(hw); 1642 if (err) { 1643 dev_warn(&pdev->dev, 1644 "Failed to obtain MAC address defaulting to random\n"); 1645 /* tag address assignment as random */ 1646 netdev->addr_assign_type |= NET_ADDR_RANDOM; 1647 } 1648 1649 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 1650 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len); 1651 1652 if (!is_valid_ether_addr(netdev->perm_addr)) { 1653 dev_err(&pdev->dev, "Invalid MAC Address\n"); 1654 return -EIO; 1655 } 1656 1657 /* assign BAR 4 resources for use with PTP */ 1658 if (fm10k_read_reg(hw, FM10K_CTRL) & FM10K_CTRL_BAR4_ALLOWED) 1659 interface->sw_addr = ioremap(pci_resource_start(pdev, 4), 1660 pci_resource_len(pdev, 4)); 1661 hw->sw_addr = interface->sw_addr; 1662 1663 /* Only the PF can support VXLAN and NVGRE offloads */ 1664 if (hw->mac.type != fm10k_mac_pf) { 1665 netdev->hw_enc_features = 0; 1666 netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; 1667 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 1668 } 1669 1670 /* initialize DCBNL interface */ 1671 fm10k_dcbnl_set_ops(netdev); 1672 1673 /* Initialize service timer and service task */ 1674 set_bit(__FM10K_SERVICE_DISABLE, &interface->state); 1675 setup_timer(&interface->service_timer, &fm10k_service_timer, 1676 (unsigned long)interface); 1677 INIT_WORK(&interface->service_task, fm10k_service_task); 1678 1679 /* kick off service timer now, even when interface is down */ 1680 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 1681 1682 /* Intitialize timestamp data */ 1683 fm10k_ts_init(interface); 1684 1685 /* set default ring sizes */ 1686 interface->tx_ring_count = FM10K_DEFAULT_TXD; 1687 interface->rx_ring_count = FM10K_DEFAULT_RXD; 1688 1689 /* set default interrupt moderation */ 1690 interface->tx_itr = FM10K_ITR_10K; 1691 interface->rx_itr = FM10K_ITR_ADAPTIVE | FM10K_ITR_20K; 1692 1693 /* initialize vxlan_port list */ 1694 INIT_LIST_HEAD(&interface->vxlan_port); 1695 1696 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 1697 memcpy(interface->rssrk, rss_key, sizeof(rss_key)); 1698 1699 /* Start off interface as being down */ 1700 set_bit(__FM10K_DOWN, &interface->state); 1701 1702 return 0; 1703} 1704 1705static void fm10k_slot_warn(struct fm10k_intfc *interface) 1706{ 1707 struct device *dev = &interface->pdev->dev; 1708 struct fm10k_hw *hw = &interface->hw; 1709 1710 if (hw->mac.ops.is_slot_appropriate(hw)) 1711 return; 1712 1713 dev_warn(dev, 1714 "For optimal performance, a %s %s slot is recommended.\n", 1715 (hw->bus_caps.width == fm10k_bus_width_pcie_x1 ? "x1" : 1716 hw->bus_caps.width == fm10k_bus_width_pcie_x4 ? "x4" : 1717 "x8"), 1718 (hw->bus_caps.speed == fm10k_bus_speed_2500 ? "2.5GT/s" : 1719 hw->bus_caps.speed == fm10k_bus_speed_5000 ? "5.0GT/s" : 1720 "8.0GT/s")); 1721 dev_warn(dev, 1722 "A slot with more lanes and/or higher speed is suggested.\n"); 1723} 1724 1725/** 1726 * fm10k_probe - Device Initialization Routine 1727 * @pdev: PCI device information struct 1728 * @ent: entry in fm10k_pci_tbl 1729 * 1730 * Returns 0 on success, negative on failure 1731 * 1732 * fm10k_probe initializes an interface identified by a pci_dev structure. 1733 * The OS initialization, configuring of the interface private structure, 1734 * and a hardware reset occur. 1735 **/ 1736static int fm10k_probe(struct pci_dev *pdev, 1737 const struct pci_device_id *ent) 1738{ 1739 struct net_device *netdev; 1740 struct fm10k_intfc *interface; 1741 struct fm10k_hw *hw; 1742 int err; 1743 u64 dma_mask; 1744 1745 err = pci_enable_device_mem(pdev); 1746 if (err) 1747 return err; 1748 1749 /* By default fm10k only supports a 48 bit DMA mask */ 1750 dma_mask = DMA_BIT_MASK(48) | dma_get_required_mask(&pdev->dev); 1751 1752 if ((dma_mask <= DMA_BIT_MASK(32)) || 1753 dma_set_mask_and_coherent(&pdev->dev, dma_mask)) { 1754 dma_mask &= DMA_BIT_MASK(32); 1755 1756 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1757 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 1758 if (err) { 1759 err = dma_set_coherent_mask(&pdev->dev, 1760 DMA_BIT_MASK(32)); 1761 if (err) { 1762 dev_err(&pdev->dev, 1763 "No usable DMA configuration, aborting\n"); 1764 goto err_dma; 1765 } 1766 } 1767 } 1768 1769 err = pci_request_selected_regions(pdev, 1770 pci_select_bars(pdev, 1771 IORESOURCE_MEM), 1772 fm10k_driver_name); 1773 if (err) { 1774 dev_err(&pdev->dev, 1775 "pci_request_selected_regions failed 0x%x\n", err); 1776 goto err_pci_reg; 1777 } 1778 1779 pci_enable_pcie_error_reporting(pdev); 1780 1781 pci_set_master(pdev); 1782 pci_save_state(pdev); 1783 1784 netdev = fm10k_alloc_netdev(); 1785 if (!netdev) { 1786 err = -ENOMEM; 1787 goto err_alloc_netdev; 1788 } 1789 1790 SET_NETDEV_DEV(netdev, &pdev->dev); 1791 1792 interface = netdev_priv(netdev); 1793 pci_set_drvdata(pdev, interface); 1794 1795 interface->netdev = netdev; 1796 interface->pdev = pdev; 1797 hw = &interface->hw; 1798 1799 interface->uc_addr = ioremap(pci_resource_start(pdev, 0), 1800 FM10K_UC_ADDR_SIZE); 1801 if (!interface->uc_addr) { 1802 err = -EIO; 1803 goto err_ioremap; 1804 } 1805 1806 err = fm10k_sw_init(interface, ent); 1807 if (err) 1808 goto err_sw_init; 1809 1810 /* enable debugfs support */ 1811 fm10k_dbg_intfc_init(interface); 1812 1813 err = fm10k_init_queueing_scheme(interface); 1814 if (err) 1815 goto err_sw_init; 1816 1817 err = fm10k_mbx_request_irq(interface); 1818 if (err) 1819 goto err_mbx_interrupt; 1820 1821 /* final check of hardware state before registering the interface */ 1822 err = fm10k_hw_ready(interface); 1823 if (err) 1824 goto err_register; 1825 1826 err = register_netdev(netdev); 1827 if (err) 1828 goto err_register; 1829 1830 /* carrier off reporting is important to ethtool even BEFORE open */ 1831 netif_carrier_off(netdev); 1832 1833 /* stop all the transmit queues from transmitting until link is up */ 1834 netif_tx_stop_all_queues(netdev); 1835 1836 /* Register PTP interface */ 1837 fm10k_ptp_register(interface); 1838 1839 /* print bus type/speed/width info */ 1840 dev_info(&pdev->dev, "(PCI Express:%s Width: %s Payload: %s)\n", 1841 (hw->bus.speed == fm10k_bus_speed_8000 ? "8.0GT/s" : 1842 hw->bus.speed == fm10k_bus_speed_5000 ? "5.0GT/s" : 1843 hw->bus.speed == fm10k_bus_speed_2500 ? "2.5GT/s" : 1844 "Unknown"), 1845 (hw->bus.width == fm10k_bus_width_pcie_x8 ? "x8" : 1846 hw->bus.width == fm10k_bus_width_pcie_x4 ? "x4" : 1847 hw->bus.width == fm10k_bus_width_pcie_x1 ? "x1" : 1848 "Unknown"), 1849 (hw->bus.payload == fm10k_bus_payload_128 ? "128B" : 1850 hw->bus.payload == fm10k_bus_payload_256 ? "256B" : 1851 hw->bus.payload == fm10k_bus_payload_512 ? "512B" : 1852 "Unknown")); 1853 1854 /* print warning for non-optimal configurations */ 1855 fm10k_slot_warn(interface); 1856 1857 /* enable SR-IOV after registering netdev to enforce PF/VF ordering */ 1858 fm10k_iov_configure(pdev, 0); 1859 1860 /* clear the service task disable bit to allow service task to start */ 1861 clear_bit(__FM10K_SERVICE_DISABLE, &interface->state); 1862 1863 return 0; 1864 1865err_register: 1866 fm10k_mbx_free_irq(interface); 1867err_mbx_interrupt: 1868 fm10k_clear_queueing_scheme(interface); 1869err_sw_init: 1870 if (interface->sw_addr) 1871 iounmap(interface->sw_addr); 1872 iounmap(interface->uc_addr); 1873err_ioremap: 1874 free_netdev(netdev); 1875err_alloc_netdev: 1876 pci_release_selected_regions(pdev, 1877 pci_select_bars(pdev, IORESOURCE_MEM)); 1878err_pci_reg: 1879err_dma: 1880 pci_disable_device(pdev); 1881 return err; 1882} 1883 1884/** 1885 * fm10k_remove - Device Removal Routine 1886 * @pdev: PCI device information struct 1887 * 1888 * fm10k_remove is called by the PCI subsystem to alert the driver 1889 * that it should release a PCI device. The could be caused by a 1890 * Hot-Plug event, or because the driver is going to be removed from 1891 * memory. 1892 **/ 1893static void fm10k_remove(struct pci_dev *pdev) 1894{ 1895 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 1896 struct net_device *netdev = interface->netdev; 1897 1898 del_timer_sync(&interface->service_timer); 1899 1900 set_bit(__FM10K_SERVICE_DISABLE, &interface->state); 1901 cancel_work_sync(&interface->service_task); 1902 1903 /* free netdev, this may bounce the interrupts due to setup_tc */ 1904 if (netdev->reg_state == NETREG_REGISTERED) 1905 unregister_netdev(netdev); 1906 1907 /* cleanup timestamp handling */ 1908 fm10k_ptp_unregister(interface); 1909 1910 /* release VFs */ 1911 fm10k_iov_disable(pdev); 1912 1913 /* disable mailbox interrupt */ 1914 fm10k_mbx_free_irq(interface); 1915 1916 /* free interrupts */ 1917 fm10k_clear_queueing_scheme(interface); 1918 1919 /* remove any debugfs interfaces */ 1920 fm10k_dbg_intfc_exit(interface); 1921 1922 if (interface->sw_addr) 1923 iounmap(interface->sw_addr); 1924 iounmap(interface->uc_addr); 1925 1926 free_netdev(netdev); 1927 1928 pci_release_selected_regions(pdev, 1929 pci_select_bars(pdev, IORESOURCE_MEM)); 1930 1931 pci_disable_pcie_error_reporting(pdev); 1932 1933 pci_disable_device(pdev); 1934} 1935 1936#ifdef CONFIG_PM 1937/** 1938 * fm10k_resume - Restore device to pre-sleep state 1939 * @pdev: PCI device information struct 1940 * 1941 * fm10k_resume is called after the system has powered back up from a sleep 1942 * state and is ready to resume operation. This function is meant to restore 1943 * the device back to its pre-sleep state. 1944 **/ 1945static int fm10k_resume(struct pci_dev *pdev) 1946{ 1947 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 1948 struct net_device *netdev = interface->netdev; 1949 struct fm10k_hw *hw = &interface->hw; 1950 u32 err; 1951 1952 pci_set_power_state(pdev, PCI_D0); 1953 pci_restore_state(pdev); 1954 1955 /* pci_restore_state clears dev->state_saved so call 1956 * pci_save_state to restore it. 1957 */ 1958 pci_save_state(pdev); 1959 1960 err = pci_enable_device_mem(pdev); 1961 if (err) { 1962 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 1963 return err; 1964 } 1965 pci_set_master(pdev); 1966 1967 pci_wake_from_d3(pdev, false); 1968 1969 /* refresh hw_addr in case it was dropped */ 1970 hw->hw_addr = interface->uc_addr; 1971 1972 /* reset hardware to known state */ 1973 err = hw->mac.ops.init_hw(&interface->hw); 1974 if (err) 1975 return err; 1976 1977 /* reset statistics starting values */ 1978 hw->mac.ops.rebind_hw_stats(hw, &interface->stats); 1979 1980 /* reset clock */ 1981 fm10k_ts_reset(interface); 1982 1983 rtnl_lock(); 1984 1985 err = fm10k_init_queueing_scheme(interface); 1986 if (!err) { 1987 fm10k_mbx_request_irq(interface); 1988 if (netif_running(netdev)) 1989 err = fm10k_open(netdev); 1990 } 1991 1992 rtnl_unlock(); 1993 1994 if (err) 1995 return err; 1996 1997 /* restore SR-IOV interface */ 1998 fm10k_iov_resume(pdev); 1999 2000 netif_device_attach(netdev); 2001 2002 return 0; 2003} 2004 2005/** 2006 * fm10k_suspend - Prepare the device for a system sleep state 2007 * @pdev: PCI device information struct 2008 * 2009 * fm10k_suspend is meant to shutdown the device prior to the system entering 2010 * a sleep state. The fm10k hardware does not support wake on lan so the 2011 * driver simply needs to shut down the device so it is in a low power state. 2012 **/ 2013static int fm10k_suspend(struct pci_dev *pdev, 2014 pm_message_t __always_unused state) 2015{ 2016 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2017 struct net_device *netdev = interface->netdev; 2018 int err = 0; 2019 2020 netif_device_detach(netdev); 2021 2022 fm10k_iov_suspend(pdev); 2023 2024 rtnl_lock(); 2025 2026 if (netif_running(netdev)) 2027 fm10k_close(netdev); 2028 2029 fm10k_mbx_free_irq(interface); 2030 2031 fm10k_clear_queueing_scheme(interface); 2032 2033 rtnl_unlock(); 2034 2035 err = pci_save_state(pdev); 2036 if (err) 2037 return err; 2038 2039 pci_disable_device(pdev); 2040 pci_wake_from_d3(pdev, false); 2041 pci_set_power_state(pdev, PCI_D3hot); 2042 2043 return 0; 2044} 2045 2046#endif /* CONFIG_PM */ 2047/** 2048 * fm10k_io_error_detected - called when PCI error is detected 2049 * @pdev: Pointer to PCI device 2050 * @state: The current pci connection state 2051 * 2052 * This function is called after a PCI bus error affecting 2053 * this device has been detected. 2054 */ 2055static pci_ers_result_t fm10k_io_error_detected(struct pci_dev *pdev, 2056 pci_channel_state_t state) 2057{ 2058 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2059 struct net_device *netdev = interface->netdev; 2060 2061 netif_device_detach(netdev); 2062 2063 if (state == pci_channel_io_perm_failure) 2064 return PCI_ERS_RESULT_DISCONNECT; 2065 2066 if (netif_running(netdev)) 2067 fm10k_close(netdev); 2068 2069 fm10k_mbx_free_irq(interface); 2070 2071 pci_disable_device(pdev); 2072 2073 /* Request a slot reset. */ 2074 return PCI_ERS_RESULT_NEED_RESET; 2075} 2076 2077/** 2078 * fm10k_io_slot_reset - called after the pci bus has been reset. 2079 * @pdev: Pointer to PCI device 2080 * 2081 * Restart the card from scratch, as if from a cold-boot. 2082 */ 2083static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev) 2084{ 2085 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2086 pci_ers_result_t result; 2087 2088 if (pci_enable_device_mem(pdev)) { 2089 dev_err(&pdev->dev, 2090 "Cannot re-enable PCI device after reset.\n"); 2091 result = PCI_ERS_RESULT_DISCONNECT; 2092 } else { 2093 pci_set_master(pdev); 2094 pci_restore_state(pdev); 2095 2096 /* After second error pci->state_saved is false, this 2097 * resets it so EEH doesn't break. 2098 */ 2099 pci_save_state(pdev); 2100 2101 pci_wake_from_d3(pdev, false); 2102 2103 /* refresh hw_addr in case it was dropped */ 2104 interface->hw.hw_addr = interface->uc_addr; 2105 2106 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 2107 fm10k_service_event_schedule(interface); 2108 2109 result = PCI_ERS_RESULT_RECOVERED; 2110 } 2111 2112 pci_cleanup_aer_uncorrect_error_status(pdev); 2113 2114 return result; 2115} 2116 2117/** 2118 * fm10k_io_resume - called when traffic can start flowing again. 2119 * @pdev: Pointer to PCI device 2120 * 2121 * This callback is called when the error recovery driver tells us that 2122 * its OK to resume normal operation. 2123 */ 2124static void fm10k_io_resume(struct pci_dev *pdev) 2125{ 2126 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2127 struct net_device *netdev = interface->netdev; 2128 struct fm10k_hw *hw = &interface->hw; 2129 int err = 0; 2130 2131 /* reset hardware to known state */ 2132 hw->mac.ops.init_hw(&interface->hw); 2133 2134 /* reset statistics starting values */ 2135 hw->mac.ops.rebind_hw_stats(hw, &interface->stats); 2136 2137 /* reassociate interrupts */ 2138 fm10k_mbx_request_irq(interface); 2139 2140 /* reset clock */ 2141 fm10k_ts_reset(interface); 2142 2143 if (netif_running(netdev)) 2144 err = fm10k_open(netdev); 2145 2146 /* final check of hardware state before registering the interface */ 2147 err = err ? : fm10k_hw_ready(interface); 2148 2149 if (!err) 2150 netif_device_attach(netdev); 2151} 2152 2153static const struct pci_error_handlers fm10k_err_handler = { 2154 .error_detected = fm10k_io_error_detected, 2155 .slot_reset = fm10k_io_slot_reset, 2156 .resume = fm10k_io_resume, 2157}; 2158 2159static struct pci_driver fm10k_driver = { 2160 .name = fm10k_driver_name, 2161 .id_table = fm10k_pci_tbl, 2162 .probe = fm10k_probe, 2163 .remove = fm10k_remove, 2164#ifdef CONFIG_PM 2165 .suspend = fm10k_suspend, 2166 .resume = fm10k_resume, 2167#endif 2168 .sriov_configure = fm10k_iov_configure, 2169 .err_handler = &fm10k_err_handler 2170}; 2171 2172/** 2173 * fm10k_register_pci_driver - register driver interface 2174 * 2175 * This funciton is called on module load in order to register the driver. 2176 **/ 2177int fm10k_register_pci_driver(void) 2178{ 2179 return pci_register_driver(&fm10k_driver); 2180} 2181 2182/** 2183 * fm10k_unregister_pci_driver - unregister driver interface 2184 * 2185 * This funciton is called on module unload in order to remove the driver. 2186 **/ 2187void fm10k_unregister_pci_driver(void) 2188{ 2189 pci_unregister_driver(&fm10k_driver); 2190} 2191