drivers/net/ethernet/intel/igc/igc

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
igc_reset
igc_power_up_link
igc_power_down_link
igc_release_hw_control
igc_get_hw_control
igc_free_tx_resources
igc_free_all_tx_resources
igc_clean_tx_ring
igc_clean_all_tx_rings
igc_setup_tx_resources
igc_setup_all_tx_resources
igc_clean_rx_ring
igc_clean_all_rx_rings
igc_free_rx_resources
igc_free_all_rx_resources
igc_setup_rx_resources
igc_setup_all_rx_resources
igc_configure_rx_ring
igc_configure_rx
igc_configure_tx_ring
igc_configure_tx
igc_setup_mrqc
igc_setup_rctl
igc_setup_tctl
igc_set_mac
igc_tx_ctxtdesc
igc_ipv6_csum_is_sctp
igc_tx_csum
__igc_maybe_stop_tx
igc_maybe_stop_tx
igc_tx_cmd_type
igc_tx_olinfo_status
igc_tx_map
igc_xmit_frame_ring
igc_tx_queue_mapping
igc_xmit_frame
igc_rx_hash
igc_process_skb_fields
igc_get_rx_buffer
igc_add_rx_frag
igc_build_skb
igc_construct_skb
igc_reuse_rx_page
igc_page_is_reserved
igc_can_reuse_rx_page
igc_is_non_eop
igc_cleanup_headers
igc_put_rx_buffer
igc_alloc_rx_buffers
igc_clean_rx_irq
igc_rx_offset
igc_alloc_mapped_page
igc_clean_tx_irq
igc_up
igc_update_stats
igc_nfc_filter_exit
igc_nfc_filter_restore
igc_down
igc_reinit_locked
igc_reset_task
igc_change_mtu
igc_get_stats
igc_fix_features
igc_set_features
igc_features_check
igc_configure
igc_rar_set_index
igc_set_default_mac_filter
igc_mac_entry_can_be_used
igc_add_mac_filter_flags
igc_add_mac_steering_filter
igc_del_mac_filter_flags
igc_del_mac_steering_filter
igc_set_rx_mode
igc_msix_other
igc_write_ivar
igc_assign_vector
igc_configure_msix
igc_msix_ring
igc_request_msix
igc_reset_q_vector
igc_reset_interrupt_capability
igc_clear_interrupt_scheme
igc_free_q_vectors
igc_free_q_vector
igc_update_phy_info
igc_has_link
igc_watchdog
igc_watchdog_task
igc_update_ring_itr
igc_update_itr
igc_intr_msi
igc_intr
igc_set_itr
igc_ring_irq_enable
igc_poll
igc_set_interrupt_capability
igc_add_ring
igc_alloc_q_vector
igc_alloc_q_vectors
igc_cache_ring_register
igc_init_interrupt_scheme
igc_free_irq
igc_irq_disable
igc_irq_enable
igc_request_irq
igc_write_itr
__igc_open
igc_open
__igc_close
igc_close
igc_read_pci_cfg
igc_write_pci_cfg
igc_read_pcie_cap_reg
igc_write_pcie_cap_reg
igc_rd32
igc_set_spd_dplx
igc_probe
igc_remove
igc_set_flag_queue_pairs
igc_get_max_rss_queues
igc_init_queue_configuration
igc_sw_init
igc_reinit_queues
igc_get_hw_dev
igc_init_module
igc_exit_module
   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2018 Intel Corporation */
   3 
   4 #include <linux/module.h>
   5 #include <linux/types.h>
   6 #include <linux/if_vlan.h>
   7 #include <linux/aer.h>
   8 #include <linux/tcp.h>
   9 #include <linux/udp.h>
  10 #include <linux/ip.h>
  11 
  12 #include <net/ipv6.h>
  13 
  14 #include "igc.h"
  15 #include "igc_hw.h"
  16 
  17 #define DRV_VERSION     "0.0.1-k"
  18 #define DRV_SUMMARY     "Intel(R) 2.5G Ethernet Linux Driver"
  19 
  20 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  21 
  22 static int debug = -1;
  23 
  24 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  25 MODULE_DESCRIPTION(DRV_SUMMARY);
  26 MODULE_LICENSE("GPL v2");
  27 MODULE_VERSION(DRV_VERSION);
  28 module_param(debug, int, 0);
  29 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  30 
  31 char igc_driver_name[] = "igc";
  32 char igc_driver_version[] = DRV_VERSION;
  33 static const char igc_driver_string[] = DRV_SUMMARY;
  34 static const char igc_copyright[] =
  35         "Copyright(c) 2018 Intel Corporation.";
  36 
  37 static const struct igc_info *igc_info_tbl[] = {
  38         [board_base] = &igc_base_info,
  39 };
  40 
  41 static const struct pci_device_id igc_pci_tbl[] = {
  42         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
  43         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
  44         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
  45         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
  46         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
  47         /* required last entry */
  48         {0, }
  49 };
  50 
  51 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
  52 
  53 /* forward declaration */
  54 static void igc_clean_tx_ring(struct igc_ring *tx_ring);
  55 static int igc_sw_init(struct igc_adapter *);
  56 static void igc_configure(struct igc_adapter *adapter);
  57 static void igc_power_down_link(struct igc_adapter *adapter);
  58 static void igc_set_default_mac_filter(struct igc_adapter *adapter);
  59 static void igc_set_rx_mode(struct net_device *netdev);
  60 static void igc_write_itr(struct igc_q_vector *q_vector);
  61 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector);
  62 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx);
  63 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
  64                                          bool msix);
  65 static void igc_free_q_vectors(struct igc_adapter *adapter);
  66 static void igc_irq_disable(struct igc_adapter *adapter);
  67 static void igc_irq_enable(struct igc_adapter *adapter);
  68 static void igc_configure_msix(struct igc_adapter *adapter);
  69 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
  70                                   struct igc_rx_buffer *bi);
  71 
  72 enum latency_range {
  73         lowest_latency = 0,
  74         low_latency = 1,
  75         bulk_latency = 2,
  76         latency_invalid = 255
  77 };
  78 
  79 void igc_reset(struct igc_adapter *adapter)
  80 {
  81         struct pci_dev *pdev = adapter->pdev;
  82         struct igc_hw *hw = &adapter->hw;
  83         struct igc_fc_info *fc = &hw->fc;
  84         u32 pba, hwm;
  85 
  86         /* Repartition PBA for greater than 9k MTU if required */
  87         pba = IGC_PBA_34K;
  88 
  89         /* flow control settings
  90          * The high water mark must be low enough to fit one full frame
  91          * after transmitting the pause frame.  As such we must have enough
  92          * space to allow for us to complete our current transmit and then
  93          * receive the frame that is in progress from the link partner.
  94          * Set it to:
  95          * - the full Rx FIFO size minus one full Tx plus one full Rx frame
  96          */
  97         hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
  98 
  99         fc->high_water = hwm & 0xFFFFFFF0;      /* 16-byte granularity */
 100         fc->low_water = fc->high_water - 16;
 101         fc->pause_time = 0xFFFF;
 102         fc->send_xon = 1;
 103         fc->current_mode = fc->requested_mode;
 104 
 105         hw->mac.ops.reset_hw(hw);
 106 
 107         if (hw->mac.ops.init_hw(hw))
 108                 dev_err(&pdev->dev, "Hardware Error\n");
 109 
 110         if (!netif_running(adapter->netdev))
 111                 igc_power_down_link(adapter);
 112 
 113         igc_get_phy_info(hw);
 114 }
 115 
 116 /**
 117  * igc_power_up_link - Power up the phy/serdes link
 118  * @adapter: address of board private structure
 119  */
 120 static void igc_power_up_link(struct igc_adapter *adapter)
 121 {
 122         igc_reset_phy(&adapter->hw);
 123 
 124         if (adapter->hw.phy.media_type == igc_media_type_copper)
 125                 igc_power_up_phy_copper(&adapter->hw);
 126 
 127         igc_setup_link(&adapter->hw);
 128 }
 129 
 130 /**
 131  * igc_power_down_link - Power down the phy/serdes link
 132  * @adapter: address of board private structure
 133  */
 134 static void igc_power_down_link(struct igc_adapter *adapter)
 135 {
 136         if (adapter->hw.phy.media_type == igc_media_type_copper)
 137                 igc_power_down_phy_copper_base(&adapter->hw);
 138 }
 139 
 140 /**
 141  * igc_release_hw_control - release control of the h/w to f/w
 142  * @adapter: address of board private structure
 143  *
 144  * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 145  * For ASF and Pass Through versions of f/w this means that the
 146  * driver is no longer loaded.
 147  */
 148 static void igc_release_hw_control(struct igc_adapter *adapter)
 149 {
 150         struct igc_hw *hw = &adapter->hw;
 151         u32 ctrl_ext;
 152 
 153         /* Let firmware take over control of h/w */
 154         ctrl_ext = rd32(IGC_CTRL_EXT);
 155         wr32(IGC_CTRL_EXT,
 156              ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 157 }
 158 
 159 /**
 160  * igc_get_hw_control - get control of the h/w from f/w
 161  * @adapter: address of board private structure
 162  *
 163  * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 164  * For ASF and Pass Through versions of f/w this means that
 165  * the driver is loaded.
 166  */
 167 static void igc_get_hw_control(struct igc_adapter *adapter)
 168 {
 169         struct igc_hw *hw = &adapter->hw;
 170         u32 ctrl_ext;
 171 
 172         /* Let firmware know the driver has taken over */
 173         ctrl_ext = rd32(IGC_CTRL_EXT);
 174         wr32(IGC_CTRL_EXT,
 175              ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 176 }
 177 
 178 /**
 179  * igc_free_tx_resources - Free Tx Resources per Queue
 180  * @tx_ring: Tx descriptor ring for a specific queue
 181  *
 182  * Free all transmit software resources
 183  */
 184 void igc_free_tx_resources(struct igc_ring *tx_ring)
 185 {
 186         igc_clean_tx_ring(tx_ring);
 187 
 188         vfree(tx_ring->tx_buffer_info);
 189         tx_ring->tx_buffer_info = NULL;
 190 
 191         /* if not set, then don't free */
 192         if (!tx_ring->desc)
 193                 return;
 194 
 195         dma_free_coherent(tx_ring->dev, tx_ring->size,
 196                           tx_ring->desc, tx_ring->dma);
 197 
 198         tx_ring->desc = NULL;
 199 }
 200 
 201 /**
 202  * igc_free_all_tx_resources - Free Tx Resources for All Queues
 203  * @adapter: board private structure
 204  *
 205  * Free all transmit software resources
 206  */
 207 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
 208 {
 209         int i;
 210 
 211         for (i = 0; i < adapter->num_tx_queues; i++)
 212                 igc_free_tx_resources(adapter->tx_ring[i]);
 213 }
 214 
 215 /**
 216  * igc_clean_tx_ring - Free Tx Buffers
 217  * @tx_ring: ring to be cleaned
 218  */
 219 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
 220 {
 221         u16 i = tx_ring->next_to_clean;
 222         struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
 223 
 224         while (i != tx_ring->next_to_use) {
 225                 union igc_adv_tx_desc *eop_desc, *tx_desc;
 226 
 227                 /* Free all the Tx ring sk_buffs */
 228                 dev_kfree_skb_any(tx_buffer->skb);
 229 
 230                 /* unmap skb header data */
 231                 dma_unmap_single(tx_ring->dev,
 232                                  dma_unmap_addr(tx_buffer, dma),
 233                                  dma_unmap_len(tx_buffer, len),
 234                                  DMA_TO_DEVICE);
 235 
 236                 /* check for eop_desc to determine the end of the packet */
 237                 eop_desc = tx_buffer->next_to_watch;
 238                 tx_desc = IGC_TX_DESC(tx_ring, i);
 239 
 240                 /* unmap remaining buffers */
 241                 while (tx_desc != eop_desc) {
 242                         tx_buffer++;
 243                         tx_desc++;
 244                         i++;
 245                         if (unlikely(i == tx_ring->count)) {
 246                                 i = 0;
 247                                 tx_buffer = tx_ring->tx_buffer_info;
 248                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
 249                         }
 250 
 251                         /* unmap any remaining paged data */
 252                         if (dma_unmap_len(tx_buffer, len))
 253                                 dma_unmap_page(tx_ring->dev,
 254                                                dma_unmap_addr(tx_buffer, dma),
 255                                                dma_unmap_len(tx_buffer, len),
 256                                                DMA_TO_DEVICE);
 257                 }
 258 
 259                 /* move us one more past the eop_desc for start of next pkt */
 260                 tx_buffer++;
 261                 i++;
 262                 if (unlikely(i == tx_ring->count)) {
 263                         i = 0;
 264                         tx_buffer = tx_ring->tx_buffer_info;
 265                 }
 266         }
 267 
 268         /* reset BQL for queue */
 269         netdev_tx_reset_queue(txring_txq(tx_ring));
 270 
 271         /* reset next_to_use and next_to_clean */
 272         tx_ring->next_to_use = 0;
 273         tx_ring->next_to_clean = 0;
 274 }
 275 
 276 /**
 277  * igc_clean_all_tx_rings - Free Tx Buffers for all queues
 278  * @adapter: board private structure
 279  */
 280 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
 281 {
 282         int i;
 283 
 284         for (i = 0; i < adapter->num_tx_queues; i++)
 285                 if (adapter->tx_ring[i])
 286                         igc_clean_tx_ring(adapter->tx_ring[i]);
 287 }
 288 
 289 /**
 290  * igc_setup_tx_resources - allocate Tx resources (Descriptors)
 291  * @tx_ring: tx descriptor ring (for a specific queue) to setup
 292  *
 293  * Return 0 on success, negative on failure
 294  */
 295 int igc_setup_tx_resources(struct igc_ring *tx_ring)
 296 {
 297         struct device *dev = tx_ring->dev;
 298         int size = 0;
 299 
 300         size = sizeof(struct igc_tx_buffer) * tx_ring->count;
 301         tx_ring->tx_buffer_info = vzalloc(size);
 302         if (!tx_ring->tx_buffer_info)
 303                 goto err;
 304 
 305         /* round up to nearest 4K */
 306         tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
 307         tx_ring->size = ALIGN(tx_ring->size, 4096);
 308 
 309         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 310                                            &tx_ring->dma, GFP_KERNEL);
 311 
 312         if (!tx_ring->desc)
 313                 goto err;
 314 
 315         tx_ring->next_to_use = 0;
 316         tx_ring->next_to_clean = 0;
 317 
 318         return 0;
 319 
 320 err:
 321         vfree(tx_ring->tx_buffer_info);
 322         dev_err(dev,
 323                 "Unable to allocate memory for the transmit descriptor ring\n");
 324         return -ENOMEM;
 325 }
 326 
 327 /**
 328  * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
 329  * @adapter: board private structure
 330  *
 331  * Return 0 on success, negative on failure
 332  */
 333 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
 334 {
 335         struct pci_dev *pdev = adapter->pdev;
 336         int i, err = 0;
 337 
 338         for (i = 0; i < adapter->num_tx_queues; i++) {
 339                 err = igc_setup_tx_resources(adapter->tx_ring[i]);
 340                 if (err) {
 341                         dev_err(&pdev->dev,
 342                                 "Allocation for Tx Queue %u failed\n", i);
 343                         for (i--; i >= 0; i--)
 344                                 igc_free_tx_resources(adapter->tx_ring[i]);
 345                         break;
 346                 }
 347         }
 348 
 349         return err;
 350 }
 351 
 352 /**
 353  * igc_clean_rx_ring - Free Rx Buffers per Queue
 354  * @rx_ring: ring to free buffers from
 355  */
 356 static void igc_clean_rx_ring(struct igc_ring *rx_ring)
 357 {
 358         u16 i = rx_ring->next_to_clean;
 359 
 360         dev_kfree_skb(rx_ring->skb);
 361         rx_ring->skb = NULL;
 362 
 363         /* Free all the Rx ring sk_buffs */
 364         while (i != rx_ring->next_to_alloc) {
 365                 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
 366 
 367                 /* Invalidate cache lines that may have been written to by
 368                  * device so that we avoid corrupting memory.
 369                  */
 370                 dma_sync_single_range_for_cpu(rx_ring->dev,
 371                                               buffer_info->dma,
 372                                               buffer_info->page_offset,
 373                                               igc_rx_bufsz(rx_ring),
 374                                               DMA_FROM_DEVICE);
 375 
 376                 /* free resources associated with mapping */
 377                 dma_unmap_page_attrs(rx_ring->dev,
 378                                      buffer_info->dma,
 379                                      igc_rx_pg_size(rx_ring),
 380                                      DMA_FROM_DEVICE,
 381                                      IGC_RX_DMA_ATTR);
 382                 __page_frag_cache_drain(buffer_info->page,
 383                                         buffer_info->pagecnt_bias);
 384 
 385                 i++;
 386                 if (i == rx_ring->count)
 387                         i = 0;
 388         }
 389 
 390         rx_ring->next_to_alloc = 0;
 391         rx_ring->next_to_clean = 0;
 392         rx_ring->next_to_use = 0;
 393 }
 394 
 395 /**
 396  * igc_clean_all_rx_rings - Free Rx Buffers for all queues
 397  * @adapter: board private structure
 398  */
 399 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
 400 {
 401         int i;
 402 
 403         for (i = 0; i < adapter->num_rx_queues; i++)
 404                 if (adapter->rx_ring[i])
 405                         igc_clean_rx_ring(adapter->rx_ring[i]);
 406 }
 407 
 408 /**
 409  * igc_free_rx_resources - Free Rx Resources
 410  * @rx_ring: ring to clean the resources from
 411  *
 412  * Free all receive software resources
 413  */
 414 void igc_free_rx_resources(struct igc_ring *rx_ring)
 415 {
 416         igc_clean_rx_ring(rx_ring);
 417 
 418         vfree(rx_ring->rx_buffer_info);
 419         rx_ring->rx_buffer_info = NULL;
 420 
 421         /* if not set, then don't free */
 422         if (!rx_ring->desc)
 423                 return;
 424 
 425         dma_free_coherent(rx_ring->dev, rx_ring->size,
 426                           rx_ring->desc, rx_ring->dma);
 427 
 428         rx_ring->desc = NULL;
 429 }
 430 
 431 /**
 432  * igc_free_all_rx_resources - Free Rx Resources for All Queues
 433  * @adapter: board private structure
 434  *
 435  * Free all receive software resources
 436  */
 437 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
 438 {
 439         int i;
 440 
 441         for (i = 0; i < adapter->num_rx_queues; i++)
 442                 igc_free_rx_resources(adapter->rx_ring[i]);
 443 }
 444 
 445 /**
 446  * igc_setup_rx_resources - allocate Rx resources (Descriptors)
 447  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
 448  *
 449  * Returns 0 on success, negative on failure
 450  */
 451 int igc_setup_rx_resources(struct igc_ring *rx_ring)
 452 {
 453         struct device *dev = rx_ring->dev;
 454         int size, desc_len;
 455 
 456         size = sizeof(struct igc_rx_buffer) * rx_ring->count;
 457         rx_ring->rx_buffer_info = vzalloc(size);
 458         if (!rx_ring->rx_buffer_info)
 459                 goto err;
 460 
 461         desc_len = sizeof(union igc_adv_rx_desc);
 462 
 463         /* Round up to nearest 4K */
 464         rx_ring->size = rx_ring->count * desc_len;
 465         rx_ring->size = ALIGN(rx_ring->size, 4096);
 466 
 467         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
 468                                            &rx_ring->dma, GFP_KERNEL);
 469 
 470         if (!rx_ring->desc)
 471                 goto err;
 472 
 473         rx_ring->next_to_alloc = 0;
 474         rx_ring->next_to_clean = 0;
 475         rx_ring->next_to_use = 0;
 476 
 477         return 0;
 478 
 479 err:
 480         vfree(rx_ring->rx_buffer_info);
 481         rx_ring->rx_buffer_info = NULL;
 482         dev_err(dev,
 483                 "Unable to allocate memory for the receive descriptor ring\n");
 484         return -ENOMEM;
 485 }
 486 
 487 /**
 488  * igc_setup_all_rx_resources - wrapper to allocate Rx resources
 489  *                                (Descriptors) for all queues
 490  * @adapter: board private structure
 491  *
 492  * Return 0 on success, negative on failure
 493  */
 494 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
 495 {
 496         struct pci_dev *pdev = adapter->pdev;
 497         int i, err = 0;
 498 
 499         for (i = 0; i < adapter->num_rx_queues; i++) {
 500                 err = igc_setup_rx_resources(adapter->rx_ring[i]);
 501                 if (err) {
 502                         dev_err(&pdev->dev,
 503                                 "Allocation for Rx Queue %u failed\n", i);
 504                         for (i--; i >= 0; i--)
 505                                 igc_free_rx_resources(adapter->rx_ring[i]);
 506                         break;
 507                 }
 508         }
 509 
 510         return err;
 511 }
 512 
 513 /**
 514  * igc_configure_rx_ring - Configure a receive ring after Reset
 515  * @adapter: board private structure
 516  * @ring: receive ring to be configured
 517  *
 518  * Configure the Rx unit of the MAC after a reset.
 519  */
 520 static void igc_configure_rx_ring(struct igc_adapter *adapter,
 521                                   struct igc_ring *ring)
 522 {
 523         struct igc_hw *hw = &adapter->hw;
 524         union igc_adv_rx_desc *rx_desc;
 525         int reg_idx = ring->reg_idx;
 526         u32 srrctl = 0, rxdctl = 0;
 527         u64 rdba = ring->dma;
 528 
 529         /* disable the queue */
 530         wr32(IGC_RXDCTL(reg_idx), 0);
 531 
 532         /* Set DMA base address registers */
 533         wr32(IGC_RDBAL(reg_idx),
 534              rdba & 0x00000000ffffffffULL);
 535         wr32(IGC_RDBAH(reg_idx), rdba >> 32);
 536         wr32(IGC_RDLEN(reg_idx),
 537              ring->count * sizeof(union igc_adv_rx_desc));
 538 
 539         /* initialize head and tail */
 540         ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
 541         wr32(IGC_RDH(reg_idx), 0);
 542         writel(0, ring->tail);
 543 
 544         /* reset next-to- use/clean to place SW in sync with hardware */
 545         ring->next_to_clean = 0;
 546         ring->next_to_use = 0;
 547 
 548         /* set descriptor configuration */
 549         srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT;
 550         if (ring_uses_large_buffer(ring))
 551                 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
 552         else
 553                 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
 554         srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
 555 
 556         wr32(IGC_SRRCTL(reg_idx), srrctl);
 557 
 558         rxdctl |= IGC_RX_PTHRESH;
 559         rxdctl |= IGC_RX_HTHRESH << 8;
 560         rxdctl |= IGC_RX_WTHRESH << 16;
 561 
 562         /* initialize rx_buffer_info */
 563         memset(ring->rx_buffer_info, 0,
 564                sizeof(struct igc_rx_buffer) * ring->count);
 565 
 566         /* initialize Rx descriptor 0 */
 567         rx_desc = IGC_RX_DESC(ring, 0);
 568         rx_desc->wb.upper.length = 0;
 569 
 570         /* enable receive descriptor fetching */
 571         rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
 572 
 573         wr32(IGC_RXDCTL(reg_idx), rxdctl);
 574 }
 575 
 576 /**
 577  * igc_configure_rx - Configure receive Unit after Reset
 578  * @adapter: board private structure
 579  *
 580  * Configure the Rx unit of the MAC after a reset.
 581  */
 582 static void igc_configure_rx(struct igc_adapter *adapter)
 583 {
 584         int i;
 585 
 586         /* Setup the HW Rx Head and Tail Descriptor Pointers and
 587          * the Base and Length of the Rx Descriptor Ring
 588          */
 589         for (i = 0; i < adapter->num_rx_queues; i++)
 590                 igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
 591 }
 592 
 593 /**
 594  * igc_configure_tx_ring - Configure transmit ring after Reset
 595  * @adapter: board private structure
 596  * @ring: tx ring to configure
 597  *
 598  * Configure a transmit ring after a reset.
 599  */
 600 static void igc_configure_tx_ring(struct igc_adapter *adapter,
 601                                   struct igc_ring *ring)
 602 {
 603         struct igc_hw *hw = &adapter->hw;
 604         int reg_idx = ring->reg_idx;
 605         u64 tdba = ring->dma;
 606         u32 txdctl = 0;
 607 
 608         /* disable the queue */
 609         wr32(IGC_TXDCTL(reg_idx), 0);
 610         wrfl();
 611         mdelay(10);
 612 
 613         wr32(IGC_TDLEN(reg_idx),
 614              ring->count * sizeof(union igc_adv_tx_desc));
 615         wr32(IGC_TDBAL(reg_idx),
 616              tdba & 0x00000000ffffffffULL);
 617         wr32(IGC_TDBAH(reg_idx), tdba >> 32);
 618 
 619         ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
 620         wr32(IGC_TDH(reg_idx), 0);
 621         writel(0, ring->tail);
 622 
 623         txdctl |= IGC_TX_PTHRESH;
 624         txdctl |= IGC_TX_HTHRESH << 8;
 625         txdctl |= IGC_TX_WTHRESH << 16;
 626 
 627         txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
 628         wr32(IGC_TXDCTL(reg_idx), txdctl);
 629 }
 630 
 631 /**
 632  * igc_configure_tx - Configure transmit Unit after Reset
 633  * @adapter: board private structure
 634  *
 635  * Configure the Tx unit of the MAC after a reset.
 636  */
 637 static void igc_configure_tx(struct igc_adapter *adapter)
 638 {
 639         int i;
 640 
 641         for (i = 0; i < adapter->num_tx_queues; i++)
 642                 igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
 643 }
 644 
 645 /**
 646  * igc_setup_mrqc - configure the multiple receive queue control registers
 647  * @adapter: Board private structure
 648  */
 649 static void igc_setup_mrqc(struct igc_adapter *adapter)
 650 {
 651         struct igc_hw *hw = &adapter->hw;
 652         u32 j, num_rx_queues;
 653         u32 mrqc, rxcsum;
 654         u32 rss_key[10];
 655 
 656         netdev_rss_key_fill(rss_key, sizeof(rss_key));
 657         for (j = 0; j < 10; j++)
 658                 wr32(IGC_RSSRK(j), rss_key[j]);
 659 
 660         num_rx_queues = adapter->rss_queues;
 661 
 662         if (adapter->rss_indir_tbl_init != num_rx_queues) {
 663                 for (j = 0; j < IGC_RETA_SIZE; j++)
 664                         adapter->rss_indir_tbl[j] =
 665                         (j * num_rx_queues) / IGC_RETA_SIZE;
 666                 adapter->rss_indir_tbl_init = num_rx_queues;
 667         }
 668         igc_write_rss_indir_tbl(adapter);
 669 
 670         /* Disable raw packet checksumming so that RSS hash is placed in
 671          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
 672          * offloads as they are enabled by default
 673          */
 674         rxcsum = rd32(IGC_RXCSUM);
 675         rxcsum |= IGC_RXCSUM_PCSD;
 676 
 677         /* Enable Receive Checksum Offload for SCTP */
 678         rxcsum |= IGC_RXCSUM_CRCOFL;
 679 
 680         /* Don't need to set TUOFL or IPOFL, they default to 1 */
 681         wr32(IGC_RXCSUM, rxcsum);
 682 
 683         /* Generate RSS hash based on packet types, TCP/UDP
 684          * port numbers and/or IPv4/v6 src and dst addresses
 685          */
 686         mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
 687                IGC_MRQC_RSS_FIELD_IPV4_TCP |
 688                IGC_MRQC_RSS_FIELD_IPV6 |
 689                IGC_MRQC_RSS_FIELD_IPV6_TCP |
 690                IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
 691 
 692         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
 693                 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
 694         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
 695                 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
 696 
 697         mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
 698 
 699         wr32(IGC_MRQC, mrqc);
 700 }
 701 
 702 /**
 703  * igc_setup_rctl - configure the receive control registers
 704  * @adapter: Board private structure
 705  */
 706 static void igc_setup_rctl(struct igc_adapter *adapter)
 707 {
 708         struct igc_hw *hw = &adapter->hw;
 709         u32 rctl;
 710 
 711         rctl = rd32(IGC_RCTL);
 712 
 713         rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
 714         rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
 715 
 716         rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
 717                 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
 718 
 719         /* enable stripping of CRC. Newer features require
 720          * that the HW strips the CRC.
 721          */
 722         rctl |= IGC_RCTL_SECRC;
 723 
 724         /* disable store bad packets and clear size bits. */
 725         rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
 726 
 727         /* enable LPE to allow for reception of jumbo frames */
 728         rctl |= IGC_RCTL_LPE;
 729 
 730         /* disable queue 0 to prevent tail write w/o re-config */
 731         wr32(IGC_RXDCTL(0), 0);
 732 
 733         /* This is useful for sniffing bad packets. */
 734         if (adapter->netdev->features & NETIF_F_RXALL) {
 735                 /* UPE and MPE will be handled by normal PROMISC logic
 736                  * in set_rx_mode
 737                  */
 738                 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
 739                          IGC_RCTL_BAM | /* RX All Bcast Pkts */
 740                          IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
 741 
 742                 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
 743                           IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
 744         }
 745 
 746         wr32(IGC_RCTL, rctl);
 747 }
 748 
 749 /**
 750  * igc_setup_tctl - configure the transmit control registers
 751  * @adapter: Board private structure
 752  */
 753 static void igc_setup_tctl(struct igc_adapter *adapter)
 754 {
 755         struct igc_hw *hw = &adapter->hw;
 756         u32 tctl;
 757 
 758         /* disable queue 0 which icould be enabled by default */
 759         wr32(IGC_TXDCTL(0), 0);
 760 
 761         /* Program the Transmit Control Register */
 762         tctl = rd32(IGC_TCTL);
 763         tctl &= ~IGC_TCTL_CT;
 764         tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
 765                 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
 766 
 767         /* Enable transmits */
 768         tctl |= IGC_TCTL_EN;
 769 
 770         wr32(IGC_TCTL, tctl);
 771 }
 772 
 773 /**
 774  * igc_set_mac - Change the Ethernet Address of the NIC
 775  * @netdev: network interface device structure
 776  * @p: pointer to an address structure
 777  *
 778  * Returns 0 on success, negative on failure
 779  */
 780 static int igc_set_mac(struct net_device *netdev, void *p)
 781 {
 782         struct igc_adapter *adapter = netdev_priv(netdev);
 783         struct igc_hw *hw = &adapter->hw;
 784         struct sockaddr *addr = p;
 785 
 786         if (!is_valid_ether_addr(addr->sa_data))
 787                 return -EADDRNOTAVAIL;
 788 
 789         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 790         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 791 
 792         /* set the correct pool for the new PF MAC address in entry 0 */
 793         igc_set_default_mac_filter(adapter);
 794 
 795         return 0;
 796 }
 797 
 798 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
 799                             struct igc_tx_buffer *first,
 800                             u32 vlan_macip_lens, u32 type_tucmd,
 801                             u32 mss_l4len_idx)
 802 {
 803         struct igc_adv_tx_context_desc *context_desc;
 804         u16 i = tx_ring->next_to_use;
 805         struct timespec64 ts;
 806 
 807         context_desc = IGC_TX_CTXTDESC(tx_ring, i);
 808 
 809         i++;
 810         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
 811 
 812         /* set bits to identify this as an advanced context descriptor */
 813         type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
 814 
 815         /* For 82575, context index must be unique per ring. */
 816         if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
 817                 mss_l4len_idx |= tx_ring->reg_idx << 4;
 818 
 819         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
 820         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
 821         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
 822 
 823         /* We assume there is always a valid Tx time available. Invalid times
 824          * should have been handled by the upper layers.
 825          */
 826         if (tx_ring->launchtime_enable) {
 827                 ts = ktime_to_timespec64(first->skb->tstamp);
 828                 first->skb->tstamp = ktime_set(0, 0);
 829                 context_desc->launch_time = cpu_to_le32(ts.tv_nsec / 32);
 830         } else {
 831                 context_desc->launch_time = 0;
 832         }
 833 }
 834 
 835 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb)
 836 {
 837         unsigned int offset = 0;
 838 
 839         ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL);
 840 
 841         return offset == skb_checksum_start_offset(skb);
 842 }
 843 
 844 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first)
 845 {
 846         struct sk_buff *skb = first->skb;
 847         u32 vlan_macip_lens = 0;
 848         u32 type_tucmd = 0;
 849 
 850         if (skb->ip_summed != CHECKSUM_PARTIAL) {
 851 csum_failed:
 852                 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
 853                     !tx_ring->launchtime_enable)
 854                         return;
 855                 goto no_csum;
 856         }
 857 
 858         switch (skb->csum_offset) {
 859         case offsetof(struct tcphdr, check):
 860                 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
 861                 /* fall through */
 862         case offsetof(struct udphdr, check):
 863                 break;
 864         case offsetof(struct sctphdr, checksum):
 865                 /* validate that this is actually an SCTP request */
 866                 if ((first->protocol == htons(ETH_P_IP) &&
 867                      (ip_hdr(skb)->protocol == IPPROTO_SCTP)) ||
 868                     (first->protocol == htons(ETH_P_IPV6) &&
 869                      igc_ipv6_csum_is_sctp(skb))) {
 870                         type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
 871                         break;
 872                 }
 873                 /* fall through */
 874         default:
 875                 skb_checksum_help(skb);
 876                 goto csum_failed;
 877         }
 878 
 879         /* update TX checksum flag */
 880         first->tx_flags |= IGC_TX_FLAGS_CSUM;
 881         vlan_macip_lens = skb_checksum_start_offset(skb) -
 882                           skb_network_offset(skb);
 883 no_csum:
 884         vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
 885         vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
 886 
 887         igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
 888 }
 889 
 890 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
 891 {
 892         struct net_device *netdev = tx_ring->netdev;
 893 
 894         netif_stop_subqueue(netdev, tx_ring->queue_index);
 895 
 896         /* memory barriier comment */
 897         smp_mb();
 898 
 899         /* We need to check again in a case another CPU has just
 900          * made room available.
 901          */
 902         if (igc_desc_unused(tx_ring) < size)
 903                 return -EBUSY;
 904 
 905         /* A reprieve! */
 906         netif_wake_subqueue(netdev, tx_ring->queue_index);
 907 
 908         u64_stats_update_begin(&tx_ring->tx_syncp2);
 909         tx_ring->tx_stats.restart_queue2++;
 910         u64_stats_update_end(&tx_ring->tx_syncp2);
 911 
 912         return 0;
 913 }
 914 
 915 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
 916 {
 917         if (igc_desc_unused(tx_ring) >= size)
 918                 return 0;
 919         return __igc_maybe_stop_tx(tx_ring, size);
 920 }
 921 
 922 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
 923 {
 924         /* set type for advanced descriptor with frame checksum insertion */
 925         u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
 926                        IGC_ADVTXD_DCMD_DEXT |
 927                        IGC_ADVTXD_DCMD_IFCS;
 928 
 929         return cmd_type;
 930 }
 931 
 932 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
 933                                  union igc_adv_tx_desc *tx_desc,
 934                                  u32 tx_flags, unsigned int paylen)
 935 {
 936         u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
 937 
 938         /* insert L4 checksum */
 939         olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
 940                           ((IGC_TXD_POPTS_TXSM << 8) /
 941                           IGC_TX_FLAGS_CSUM);
 942 
 943         /* insert IPv4 checksum */
 944         olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
 945                           (((IGC_TXD_POPTS_IXSM << 8)) /
 946                           IGC_TX_FLAGS_IPV4);
 947 
 948         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
 949 }
 950 
 951 static int igc_tx_map(struct igc_ring *tx_ring,
 952                       struct igc_tx_buffer *first,
 953                       const u8 hdr_len)
 954 {
 955         struct sk_buff *skb = first->skb;
 956         struct igc_tx_buffer *tx_buffer;
 957         union igc_adv_tx_desc *tx_desc;
 958         u32 tx_flags = first->tx_flags;
 959         skb_frag_t *frag;
 960         u16 i = tx_ring->next_to_use;
 961         unsigned int data_len, size;
 962         dma_addr_t dma;
 963         u32 cmd_type = igc_tx_cmd_type(skb, tx_flags);
 964 
 965         tx_desc = IGC_TX_DESC(tx_ring, i);
 966 
 967         igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
 968 
 969         size = skb_headlen(skb);
 970         data_len = skb->data_len;
 971 
 972         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
 973 
 974         tx_buffer = first;
 975 
 976         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
 977                 if (dma_mapping_error(tx_ring->dev, dma))
 978                         goto dma_error;
 979 
 980                 /* record length, and DMA address */
 981                 dma_unmap_len_set(tx_buffer, len, size);
 982                 dma_unmap_addr_set(tx_buffer, dma, dma);
 983 
 984                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
 985 
 986                 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
 987                         tx_desc->read.cmd_type_len =
 988                                 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
 989 
 990                         i++;
 991                         tx_desc++;
 992                         if (i == tx_ring->count) {
 993                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
 994                                 i = 0;
 995                         }
 996                         tx_desc->read.olinfo_status = 0;
 997 
 998                         dma += IGC_MAX_DATA_PER_TXD;
 999                         size -= IGC_MAX_DATA_PER_TXD;
1000 
1001                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
1002                 }
1003 
1004                 if (likely(!data_len))
1005                         break;
1006 
1007                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1008 
1009                 i++;
1010                 tx_desc++;
1011                 if (i == tx_ring->count) {
1012                         tx_desc = IGC_TX_DESC(tx_ring, 0);
1013                         i = 0;
1014                 }
1015                 tx_desc->read.olinfo_status = 0;
1016 
1017                 size = skb_frag_size(frag);
1018                 data_len -= size;
1019 
1020                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
1021                                        size, DMA_TO_DEVICE);
1022 
1023                 tx_buffer = &tx_ring->tx_buffer_info[i];
1024         }
1025 
1026         /* write last descriptor with RS and EOP bits */
1027         cmd_type |= size | IGC_TXD_DCMD;
1028         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1029 
1030         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1031 
1032         /* set the timestamp */
1033         first->time_stamp = jiffies;
1034 
1035         skb_tx_timestamp(skb);
1036 
1037         /* Force memory writes to complete before letting h/w know there
1038          * are new descriptors to fetch.  (Only applicable for weak-ordered
1039          * memory model archs, such as IA-64).
1040          *
1041          * We also need this memory barrier to make certain all of the
1042          * status bits have been updated before next_to_watch is written.
1043          */
1044         wmb();
1045 
1046         /* set next_to_watch value indicating a packet is present */
1047         first->next_to_watch = tx_desc;
1048 
1049         i++;
1050         if (i == tx_ring->count)
1051                 i = 0;
1052 
1053         tx_ring->next_to_use = i;
1054 
1055         /* Make sure there is space in the ring for the next send. */
1056         igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
1057 
1058         if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1059                 writel(i, tx_ring->tail);
1060         }
1061 
1062         return 0;
1063 dma_error:
1064         dev_err(tx_ring->dev, "TX DMA map failed\n");
1065         tx_buffer = &tx_ring->tx_buffer_info[i];
1066 
1067         /* clear dma mappings for failed tx_buffer_info map */
1068         while (tx_buffer != first) {
1069                 if (dma_unmap_len(tx_buffer, len))
1070                         dma_unmap_page(tx_ring->dev,
1071                                        dma_unmap_addr(tx_buffer, dma),
1072                                        dma_unmap_len(tx_buffer, len),
1073                                        DMA_TO_DEVICE);
1074                 dma_unmap_len_set(tx_buffer, len, 0);
1075 
1076                 if (i-- == 0)
1077                         i += tx_ring->count;
1078                 tx_buffer = &tx_ring->tx_buffer_info[i];
1079         }
1080 
1081         if (dma_unmap_len(tx_buffer, len))
1082                 dma_unmap_single(tx_ring->dev,
1083                                  dma_unmap_addr(tx_buffer, dma),
1084                                  dma_unmap_len(tx_buffer, len),
1085                                  DMA_TO_DEVICE);
1086         dma_unmap_len_set(tx_buffer, len, 0);
1087 
1088         dev_kfree_skb_any(tx_buffer->skb);
1089         tx_buffer->skb = NULL;
1090 
1091         tx_ring->next_to_use = i;
1092 
1093         return -1;
1094 }
1095 
1096 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1097                                        struct igc_ring *tx_ring)
1098 {
1099         u16 count = TXD_USE_COUNT(skb_headlen(skb));
1100         __be16 protocol = vlan_get_protocol(skb);
1101         struct igc_tx_buffer *first;
1102         u32 tx_flags = 0;
1103         unsigned short f;
1104         u8 hdr_len = 0;
1105 
1106         /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1107          *      + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1108          *      + 2 desc gap to keep tail from touching head,
1109          *      + 1 desc for context descriptor,
1110          * otherwise try next time
1111          */
1112         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1113                 count += TXD_USE_COUNT(skb_frag_size(
1114                                                 &skb_shinfo(skb)->frags[f]));
1115 
1116         if (igc_maybe_stop_tx(tx_ring, count + 3)) {
1117                 /* this is a hard error */
1118                 return NETDEV_TX_BUSY;
1119         }
1120 
1121         /* record the location of the first descriptor for this packet */
1122         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1123         first->skb = skb;
1124         first->bytecount = skb->len;
1125         first->gso_segs = 1;
1126 
1127         /* record initial flags and protocol */
1128         first->tx_flags = tx_flags;
1129         first->protocol = protocol;
1130 
1131         igc_tx_csum(tx_ring, first);
1132 
1133         igc_tx_map(tx_ring, first, hdr_len);
1134 
1135         return NETDEV_TX_OK;
1136 }
1137 
1138 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1139                                                     struct sk_buff *skb)
1140 {
1141         unsigned int r_idx = skb->queue_mapping;
1142 
1143         if (r_idx >= adapter->num_tx_queues)
1144                 r_idx = r_idx % adapter->num_tx_queues;
1145 
1146         return adapter->tx_ring[r_idx];
1147 }
1148 
1149 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1150                                   struct net_device *netdev)
1151 {
1152         struct igc_adapter *adapter = netdev_priv(netdev);
1153 
1154         /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1155          * in order to meet this minimum size requirement.
1156          */
1157         if (skb->len < 17) {
1158                 if (skb_padto(skb, 17))
1159                         return NETDEV_TX_OK;
1160                 skb->len = 17;
1161         }
1162 
1163         return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1164 }
1165 
1166 static inline void igc_rx_hash(struct igc_ring *ring,
1167                                union igc_adv_rx_desc *rx_desc,
1168                                struct sk_buff *skb)
1169 {
1170         if (ring->netdev->features & NETIF_F_RXHASH)
1171                 skb_set_hash(skb,
1172                              le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1173                              PKT_HASH_TYPE_L3);
1174 }
1175 
1176 /**
1177  * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1178  * @rx_ring: rx descriptor ring packet is being transacted on
1179  * @rx_desc: pointer to the EOP Rx descriptor
1180  * @skb: pointer to current skb being populated
1181  *
1182  * This function checks the ring, descriptor, and packet information in
1183  * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1184  * other fields within the skb.
1185  */
1186 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1187                                    union igc_adv_rx_desc *rx_desc,
1188                                    struct sk_buff *skb)
1189 {
1190         igc_rx_hash(rx_ring, rx_desc, skb);
1191 
1192         skb_record_rx_queue(skb, rx_ring->queue_index);
1193 
1194         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1195 }
1196 
1197 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1198                                                const unsigned int size)
1199 {
1200         struct igc_rx_buffer *rx_buffer;
1201 
1202         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1203         prefetchw(rx_buffer->page);
1204 
1205         /* we are reusing so sync this buffer for CPU use */
1206         dma_sync_single_range_for_cpu(rx_ring->dev,
1207                                       rx_buffer->dma,
1208                                       rx_buffer->page_offset,
1209                                       size,
1210                                       DMA_FROM_DEVICE);
1211 
1212         rx_buffer->pagecnt_bias--;
1213 
1214         return rx_buffer;
1215 }
1216 
1217 /**
1218  * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1219  * @rx_ring: rx descriptor ring to transact packets on
1220  * @rx_buffer: buffer containing page to add
1221  * @skb: sk_buff to place the data into
1222  * @size: size of buffer to be added
1223  *
1224  * This function will add the data contained in rx_buffer->page to the skb.
1225  */
1226 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1227                             struct igc_rx_buffer *rx_buffer,
1228                             struct sk_buff *skb,
1229                             unsigned int size)
1230 {
1231 #if (PAGE_SIZE < 8192)
1232         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1233 
1234         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1235                         rx_buffer->page_offset, size, truesize);
1236         rx_buffer->page_offset ^= truesize;
1237 #else
1238         unsigned int truesize = ring_uses_build_skb(rx_ring) ?
1239                                 SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1240                                 SKB_DATA_ALIGN(size);
1241         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1242                         rx_buffer->page_offset, size, truesize);
1243         rx_buffer->page_offset += truesize;
1244 #endif
1245 }
1246 
1247 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1248                                      struct igc_rx_buffer *rx_buffer,
1249                                      union igc_adv_rx_desc *rx_desc,
1250                                      unsigned int size)
1251 {
1252         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1253 #if (PAGE_SIZE < 8192)
1254         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1255 #else
1256         unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1257                                 SKB_DATA_ALIGN(IGC_SKB_PAD + size);
1258 #endif
1259         struct sk_buff *skb;
1260 
1261         /* prefetch first cache line of first page */
1262         prefetch(va);
1263 #if L1_CACHE_BYTES < 128
1264         prefetch(va + L1_CACHE_BYTES);
1265 #endif
1266 
1267         /* build an skb around the page buffer */
1268         skb = build_skb(va - IGC_SKB_PAD, truesize);
1269         if (unlikely(!skb))
1270                 return NULL;
1271 
1272         /* update pointers within the skb to store the data */
1273         skb_reserve(skb, IGC_SKB_PAD);
1274         __skb_put(skb, size);
1275 
1276         /* update buffer offset */
1277 #if (PAGE_SIZE < 8192)
1278         rx_buffer->page_offset ^= truesize;
1279 #else
1280         rx_buffer->page_offset += truesize;
1281 #endif
1282 
1283         return skb;
1284 }
1285 
1286 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1287                                          struct igc_rx_buffer *rx_buffer,
1288                                          union igc_adv_rx_desc *rx_desc,
1289                                          unsigned int size)
1290 {
1291         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1292 #if (PAGE_SIZE < 8192)
1293         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1294 #else
1295         unsigned int truesize = SKB_DATA_ALIGN(size);
1296 #endif
1297         unsigned int headlen;
1298         struct sk_buff *skb;
1299 
1300         /* prefetch first cache line of first page */
1301         prefetch(va);
1302 #if L1_CACHE_BYTES < 128
1303         prefetch(va + L1_CACHE_BYTES);
1304 #endif
1305 
1306         /* allocate a skb to store the frags */
1307         skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);
1308         if (unlikely(!skb))
1309                 return NULL;
1310 
1311         /* Determine available headroom for copy */
1312         headlen = size;
1313         if (headlen > IGC_RX_HDR_LEN)
1314                 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1315 
1316         /* align pull length to size of long to optimize memcpy performance */
1317         memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1318 
1319         /* update all of the pointers */
1320         size -= headlen;
1321         if (size) {
1322                 skb_add_rx_frag(skb, 0, rx_buffer->page,
1323                                 (va + headlen) - page_address(rx_buffer->page),
1324                                 size, truesize);
1325 #if (PAGE_SIZE < 8192)
1326                 rx_buffer->page_offset ^= truesize;
1327 #else
1328                 rx_buffer->page_offset += truesize;
1329 #endif
1330         } else {
1331                 rx_buffer->pagecnt_bias++;
1332         }
1333 
1334         return skb;
1335 }
1336 
1337 /**
1338  * igc_reuse_rx_page - page flip buffer and store it back on the ring
1339  * @rx_ring: rx descriptor ring to store buffers on
1340  * @old_buff: donor buffer to have page reused
1341  *
1342  * Synchronizes page for reuse by the adapter
1343  */
1344 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
1345                               struct igc_rx_buffer *old_buff)
1346 {
1347         u16 nta = rx_ring->next_to_alloc;
1348         struct igc_rx_buffer *new_buff;
1349 
1350         new_buff = &rx_ring->rx_buffer_info[nta];
1351 
1352         /* update, and store next to alloc */
1353         nta++;
1354         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1355 
1356         /* Transfer page from old buffer to new buffer.
1357          * Move each member individually to avoid possible store
1358          * forwarding stalls.
1359          */
1360         new_buff->dma           = old_buff->dma;
1361         new_buff->page          = old_buff->page;
1362         new_buff->page_offset   = old_buff->page_offset;
1363         new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
1364 }
1365 
1366 static inline bool igc_page_is_reserved(struct page *page)
1367 {
1368         return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
1369 }
1370 
1371 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer)
1372 {
1373         unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1374         struct page *page = rx_buffer->page;
1375 
1376         /* avoid re-using remote pages */
1377         if (unlikely(igc_page_is_reserved(page)))
1378                 return false;
1379 
1380 #if (PAGE_SIZE < 8192)
1381         /* if we are only owner of page we can reuse it */
1382         if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
1383                 return false;
1384 #else
1385 #define IGC_LAST_OFFSET \
1386         (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1387 
1388         if (rx_buffer->page_offset > IGC_LAST_OFFSET)
1389                 return false;
1390 #endif
1391 
1392         /* If we have drained the page fragment pool we need to update
1393          * the pagecnt_bias and page count so that we fully restock the
1394          * number of references the driver holds.
1395          */
1396         if (unlikely(!pagecnt_bias)) {
1397                 page_ref_add(page, USHRT_MAX);
1398                 rx_buffer->pagecnt_bias = USHRT_MAX;
1399         }
1400 
1401         return true;
1402 }
1403 
1404 /**
1405  * igc_is_non_eop - process handling of non-EOP buffers
1406  * @rx_ring: Rx ring being processed
1407  * @rx_desc: Rx descriptor for current buffer
1408  * @skb: current socket buffer containing buffer in progress
1409  *
1410  * This function updates next to clean.  If the buffer is an EOP buffer
1411  * this function exits returning false, otherwise it will place the
1412  * sk_buff in the next buffer to be chained and return true indicating
1413  * that this is in fact a non-EOP buffer.
1414  */
1415 static bool igc_is_non_eop(struct igc_ring *rx_ring,
1416                            union igc_adv_rx_desc *rx_desc)
1417 {
1418         u32 ntc = rx_ring->next_to_clean + 1;
1419 
1420         /* fetch, update, and store next to clean */
1421         ntc = (ntc < rx_ring->count) ? ntc : 0;
1422         rx_ring->next_to_clean = ntc;
1423 
1424         prefetch(IGC_RX_DESC(rx_ring, ntc));
1425 
1426         if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
1427                 return false;
1428 
1429         return true;
1430 }
1431 
1432 /**
1433  * igc_cleanup_headers - Correct corrupted or empty headers
1434  * @rx_ring: rx descriptor ring packet is being transacted on
1435  * @rx_desc: pointer to the EOP Rx descriptor
1436  * @skb: pointer to current skb being fixed
1437  *
1438  * Address the case where we are pulling data in on pages only
1439  * and as such no data is present in the skb header.
1440  *
1441  * In addition if skb is not at least 60 bytes we need to pad it so that
1442  * it is large enough to qualify as a valid Ethernet frame.
1443  *
1444  * Returns true if an error was encountered and skb was freed.
1445  */
1446 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
1447                                 union igc_adv_rx_desc *rx_desc,
1448                                 struct sk_buff *skb)
1449 {
1450         if (unlikely((igc_test_staterr(rx_desc,
1451                                        IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) {
1452                 struct net_device *netdev = rx_ring->netdev;
1453 
1454                 if (!(netdev->features & NETIF_F_RXALL)) {
1455                         dev_kfree_skb_any(skb);
1456                         return true;
1457                 }
1458         }
1459 
1460         /* if eth_skb_pad returns an error the skb was freed */
1461         if (eth_skb_pad(skb))
1462                 return true;
1463 
1464         return false;
1465 }
1466 
1467 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
1468                               struct igc_rx_buffer *rx_buffer)
1469 {
1470         if (igc_can_reuse_rx_page(rx_buffer)) {
1471                 /* hand second half of page back to the ring */
1472                 igc_reuse_rx_page(rx_ring, rx_buffer);
1473         } else {
1474                 /* We are not reusing the buffer so unmap it and free
1475                  * any references we are holding to it
1476                  */
1477                 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1478                                      igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
1479                                      IGC_RX_DMA_ATTR);
1480                 __page_frag_cache_drain(rx_buffer->page,
1481                                         rx_buffer->pagecnt_bias);
1482         }
1483 
1484         /* clear contents of rx_buffer */
1485         rx_buffer->page = NULL;
1486 }
1487 
1488 /**
1489  * igc_alloc_rx_buffers - Replace used receive buffers; packet split
1490  * @adapter: address of board private structure
1491  */
1492 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
1493 {
1494         union igc_adv_rx_desc *rx_desc;
1495         u16 i = rx_ring->next_to_use;
1496         struct igc_rx_buffer *bi;
1497         u16 bufsz;
1498 
1499         /* nothing to do */
1500         if (!cleaned_count)
1501                 return;
1502 
1503         rx_desc = IGC_RX_DESC(rx_ring, i);
1504         bi = &rx_ring->rx_buffer_info[i];
1505         i -= rx_ring->count;
1506 
1507         bufsz = igc_rx_bufsz(rx_ring);
1508 
1509         do {
1510                 if (!igc_alloc_mapped_page(rx_ring, bi))
1511                         break;
1512 
1513                 /* sync the buffer for use by the device */
1514                 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
1515                                                  bi->page_offset, bufsz,
1516                                                  DMA_FROM_DEVICE);
1517 
1518                 /* Refresh the desc even if buffer_addrs didn't change
1519                  * because each write-back erases this info.
1520                  */
1521                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1522 
1523                 rx_desc++;
1524                 bi++;
1525                 i++;
1526                 if (unlikely(!i)) {
1527                         rx_desc = IGC_RX_DESC(rx_ring, 0);
1528                         bi = rx_ring->rx_buffer_info;
1529                         i -= rx_ring->count;
1530                 }
1531 
1532                 /* clear the length for the next_to_use descriptor */
1533                 rx_desc->wb.upper.length = 0;
1534 
1535                 cleaned_count--;
1536         } while (cleaned_count);
1537 
1538         i += rx_ring->count;
1539 
1540         if (rx_ring->next_to_use != i) {
1541                 /* record the next descriptor to use */
1542                 rx_ring->next_to_use = i;
1543 
1544                 /* update next to alloc since we have filled the ring */
1545                 rx_ring->next_to_alloc = i;
1546 
1547                 /* Force memory writes to complete before letting h/w
1548                  * know there are new descriptors to fetch.  (Only
1549                  * applicable for weak-ordered memory model archs,
1550                  * such as IA-64).
1551                  */
1552                 wmb();
1553                 writel(i, rx_ring->tail);
1554         }
1555 }
1556 
1557 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
1558 {
1559         unsigned int total_bytes = 0, total_packets = 0;
1560         struct igc_ring *rx_ring = q_vector->rx.ring;
1561         struct sk_buff *skb = rx_ring->skb;
1562         u16 cleaned_count = igc_desc_unused(rx_ring);
1563 
1564         while (likely(total_packets < budget)) {
1565                 union igc_adv_rx_desc *rx_desc;
1566                 struct igc_rx_buffer *rx_buffer;
1567                 unsigned int size;
1568 
1569                 /* return some buffers to hardware, one at a time is too slow */
1570                 if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
1571                         igc_alloc_rx_buffers(rx_ring, cleaned_count);
1572                         cleaned_count = 0;
1573                 }
1574 
1575                 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
1576                 size = le16_to_cpu(rx_desc->wb.upper.length);
1577                 if (!size)
1578                         break;
1579 
1580                 /* This memory barrier is needed to keep us from reading
1581                  * any other fields out of the rx_desc until we know the
1582                  * descriptor has been written back
1583                  */
1584                 dma_rmb();
1585 
1586                 rx_buffer = igc_get_rx_buffer(rx_ring, size);
1587 
1588                 /* retrieve a buffer from the ring */
1589                 if (skb)
1590                         igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
1591                 else if (ring_uses_build_skb(rx_ring))
1592                         skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size);
1593                 else
1594                         skb = igc_construct_skb(rx_ring, rx_buffer,
1595                                                 rx_desc, size);
1596 
1597                 /* exit if we failed to retrieve a buffer */
1598                 if (!skb) {
1599                         rx_ring->rx_stats.alloc_failed++;
1600                         rx_buffer->pagecnt_bias++;
1601                         break;
1602                 }
1603 
1604                 igc_put_rx_buffer(rx_ring, rx_buffer);
1605                 cleaned_count++;
1606 
1607                 /* fetch next buffer in frame if non-eop */
1608                 if (igc_is_non_eop(rx_ring, rx_desc))
1609                         continue;
1610 
1611                 /* verify the packet layout is correct */
1612                 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
1613                         skb = NULL;
1614                         continue;
1615                 }
1616 
1617                 /* probably a little skewed due to removing CRC */
1618                 total_bytes += skb->len;
1619 
1620                 /* populate checksum, timestamp, VLAN, and protocol */
1621                 igc_process_skb_fields(rx_ring, rx_desc, skb);
1622 
1623                 napi_gro_receive(&q_vector->napi, skb);
1624 
1625                 /* reset skb pointer */
1626                 skb = NULL;
1627 
1628                 /* update budget accounting */
1629                 total_packets++;
1630         }
1631 
1632         /* place incomplete frames back on ring for completion */
1633         rx_ring->skb = skb;
1634 
1635         u64_stats_update_begin(&rx_ring->rx_syncp);
1636         rx_ring->rx_stats.packets += total_packets;
1637         rx_ring->rx_stats.bytes += total_bytes;
1638         u64_stats_update_end(&rx_ring->rx_syncp);
1639         q_vector->rx.total_packets += total_packets;
1640         q_vector->rx.total_bytes += total_bytes;
1641 
1642         if (cleaned_count)
1643                 igc_alloc_rx_buffers(rx_ring, cleaned_count);
1644 
1645         return total_packets;
1646 }
1647 
1648 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
1649 {
1650         return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0;
1651 }
1652 
1653 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
1654                                   struct igc_rx_buffer *bi)
1655 {
1656         struct page *page = bi->page;
1657         dma_addr_t dma;
1658 
1659         /* since we are recycling buffers we should seldom need to alloc */
1660         if (likely(page))
1661                 return true;
1662 
1663         /* alloc new page for storage */
1664         page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
1665         if (unlikely(!page)) {
1666                 rx_ring->rx_stats.alloc_failed++;
1667                 return false;
1668         }
1669 
1670         /* map page for use */
1671         dma = dma_map_page_attrs(rx_ring->dev, page, 0,
1672                                  igc_rx_pg_size(rx_ring),
1673                                  DMA_FROM_DEVICE,
1674                                  IGC_RX_DMA_ATTR);
1675 
1676         /* if mapping failed free memory back to system since
1677          * there isn't much point in holding memory we can't use
1678          */
1679         if (dma_mapping_error(rx_ring->dev, dma)) {
1680                 __free_page(page);
1681 
1682                 rx_ring->rx_stats.alloc_failed++;
1683                 return false;
1684         }
1685 
1686         bi->dma = dma;
1687         bi->page = page;
1688         bi->page_offset = igc_rx_offset(rx_ring);
1689         bi->pagecnt_bias = 1;
1690 
1691         return true;
1692 }
1693 
1694 /**
1695  * igc_clean_tx_irq - Reclaim resources after transmit completes
1696  * @q_vector: pointer to q_vector containing needed info
1697  * @napi_budget: Used to determine if we are in netpoll
1698  *
1699  * returns true if ring is completely cleaned
1700  */
1701 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
1702 {
1703         struct igc_adapter *adapter = q_vector->adapter;
1704         unsigned int total_bytes = 0, total_packets = 0;
1705         unsigned int budget = q_vector->tx.work_limit;
1706         struct igc_ring *tx_ring = q_vector->tx.ring;
1707         unsigned int i = tx_ring->next_to_clean;
1708         struct igc_tx_buffer *tx_buffer;
1709         union igc_adv_tx_desc *tx_desc;
1710 
1711         if (test_bit(__IGC_DOWN, &adapter->state))
1712                 return true;
1713 
1714         tx_buffer = &tx_ring->tx_buffer_info[i];
1715         tx_desc = IGC_TX_DESC(tx_ring, i);
1716         i -= tx_ring->count;
1717 
1718         do {
1719                 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
1720 
1721                 /* if next_to_watch is not set then there is no work pending */
1722                 if (!eop_desc)
1723                         break;
1724 
1725                 /* prevent any other reads prior to eop_desc */
1726                 smp_rmb();
1727 
1728                 /* if DD is not set pending work has not been completed */
1729                 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
1730                         break;
1731 
1732                 /* clear next_to_watch to prevent false hangs */
1733                 tx_buffer->next_to_watch = NULL;
1734 
1735                 /* update the statistics for this packet */
1736                 total_bytes += tx_buffer->bytecount;
1737                 total_packets += tx_buffer->gso_segs;
1738 
1739                 /* free the skb */
1740                 napi_consume_skb(tx_buffer->skb, napi_budget);
1741 
1742                 /* unmap skb header data */
1743                 dma_unmap_single(tx_ring->dev,
1744                                  dma_unmap_addr(tx_buffer, dma),
1745                                  dma_unmap_len(tx_buffer, len),
1746                                  DMA_TO_DEVICE);
1747 
1748                 /* clear tx_buffer data */
1749                 dma_unmap_len_set(tx_buffer, len, 0);
1750 
1751                 /* clear last DMA location and unmap remaining buffers */
1752                 while (tx_desc != eop_desc) {
1753                         tx_buffer++;
1754                         tx_desc++;
1755                         i++;
1756                         if (unlikely(!i)) {
1757                                 i -= tx_ring->count;
1758                                 tx_buffer = tx_ring->tx_buffer_info;
1759                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
1760                         }
1761 
1762                         /* unmap any remaining paged data */
1763                         if (dma_unmap_len(tx_buffer, len)) {
1764                                 dma_unmap_page(tx_ring->dev,
1765                                                dma_unmap_addr(tx_buffer, dma),
1766                                                dma_unmap_len(tx_buffer, len),
1767                                                DMA_TO_DEVICE);
1768                                 dma_unmap_len_set(tx_buffer, len, 0);
1769                         }
1770                 }
1771 
1772                 /* move us one more past the eop_desc for start of next pkt */
1773                 tx_buffer++;
1774                 tx_desc++;
1775                 i++;
1776                 if (unlikely(!i)) {
1777                         i -= tx_ring->count;
1778                         tx_buffer = tx_ring->tx_buffer_info;
1779                         tx_desc = IGC_TX_DESC(tx_ring, 0);
1780                 }
1781 
1782                 /* issue prefetch for next Tx descriptor */
1783                 prefetch(tx_desc);
1784 
1785                 /* update budget accounting */
1786                 budget--;
1787         } while (likely(budget));
1788 
1789         netdev_tx_completed_queue(txring_txq(tx_ring),
1790                                   total_packets, total_bytes);
1791 
1792         i += tx_ring->count;
1793         tx_ring->next_to_clean = i;
1794         u64_stats_update_begin(&tx_ring->tx_syncp);
1795         tx_ring->tx_stats.bytes += total_bytes;
1796         tx_ring->tx_stats.packets += total_packets;
1797         u64_stats_update_end(&tx_ring->tx_syncp);
1798         q_vector->tx.total_bytes += total_bytes;
1799         q_vector->tx.total_packets += total_packets;
1800 
1801         if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
1802                 struct igc_hw *hw = &adapter->hw;
1803 
1804                 /* Detect a transmit hang in hardware, this serializes the
1805                  * check with the clearing of time_stamp and movement of i
1806                  */
1807                 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
1808                 if (tx_buffer->next_to_watch &&
1809                     time_after(jiffies, tx_buffer->time_stamp +
1810                     (adapter->tx_timeout_factor * HZ)) &&
1811                     !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) {
1812                         /* detected Tx unit hang */
1813                         dev_err(tx_ring->dev,
1814                                 "Detected Tx Unit Hang\n"
1815                                 "  Tx Queue             <%d>\n"
1816                                 "  TDH                  <%x>\n"
1817                                 "  TDT                  <%x>\n"
1818                                 "  next_to_use          <%x>\n"
1819                                 "  next_to_clean        <%x>\n"
1820                                 "buffer_info[next_to_clean]\n"
1821                                 "  time_stamp           <%lx>\n"
1822                                 "  next_to_watch        <%p>\n"
1823                                 "  jiffies              <%lx>\n"
1824                                 "  desc.status          <%x>\n",
1825                                 tx_ring->queue_index,
1826                                 rd32(IGC_TDH(tx_ring->reg_idx)),
1827                                 readl(tx_ring->tail),
1828                                 tx_ring->next_to_use,
1829                                 tx_ring->next_to_clean,
1830                                 tx_buffer->time_stamp,
1831                                 tx_buffer->next_to_watch,
1832                                 jiffies,
1833                                 tx_buffer->next_to_watch->wb.status);
1834                         netif_stop_subqueue(tx_ring->netdev,
1835                                             tx_ring->queue_index);
1836 
1837                         /* we are about to reset, no point in enabling stuff */
1838                         return true;
1839                 }
1840         }
1841 
1842 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
1843         if (unlikely(total_packets &&
1844                      netif_carrier_ok(tx_ring->netdev) &&
1845                      igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
1846                 /* Make sure that anybody stopping the queue after this
1847                  * sees the new next_to_clean.
1848                  */
1849                 smp_mb();
1850                 if (__netif_subqueue_stopped(tx_ring->netdev,
1851                                              tx_ring->queue_index) &&
1852                     !(test_bit(__IGC_DOWN, &adapter->state))) {
1853                         netif_wake_subqueue(tx_ring->netdev,
1854                                             tx_ring->queue_index);
1855 
1856                         u64_stats_update_begin(&tx_ring->tx_syncp);
1857                         tx_ring->tx_stats.restart_queue++;
1858                         u64_stats_update_end(&tx_ring->tx_syncp);
1859                 }
1860         }
1861 
1862         return !!budget;
1863 }
1864 
1865 /**
1866  * igc_up - Open the interface and prepare it to handle traffic
1867  * @adapter: board private structure
1868  */
1869 void igc_up(struct igc_adapter *adapter)
1870 {
1871         struct igc_hw *hw = &adapter->hw;
1872         int i = 0;
1873 
1874         /* hardware has been reset, we need to reload some things */
1875         igc_configure(adapter);
1876 
1877         clear_bit(__IGC_DOWN, &adapter->state);
1878 
1879         for (i = 0; i < adapter->num_q_vectors; i++)
1880                 napi_enable(&adapter->q_vector[i]->napi);
1881 
1882         if (adapter->msix_entries)
1883                 igc_configure_msix(adapter);
1884         else
1885                 igc_assign_vector(adapter->q_vector[0], 0);
1886 
1887         /* Clear any pending interrupts. */
1888         rd32(IGC_ICR);
1889         igc_irq_enable(adapter);
1890 
1891         netif_tx_start_all_queues(adapter->netdev);
1892 
1893         /* start the watchdog. */
1894         hw->mac.get_link_status = 1;
1895         schedule_work(&adapter->watchdog_task);
1896 }
1897 
1898 /**
1899  * igc_update_stats - Update the board statistics counters
1900  * @adapter: board private structure
1901  */
1902 void igc_update_stats(struct igc_adapter *adapter)
1903 {
1904         struct rtnl_link_stats64 *net_stats = &adapter->stats64;
1905         struct pci_dev *pdev = adapter->pdev;
1906         struct igc_hw *hw = &adapter->hw;
1907         u64 _bytes, _packets;
1908         u64 bytes, packets;
1909         unsigned int start;
1910         u32 mpc;
1911         int i;
1912 
1913         /* Prevent stats update while adapter is being reset, or if the pci
1914          * connection is down.
1915          */
1916         if (adapter->link_speed == 0)
1917                 return;
1918         if (pci_channel_offline(pdev))
1919                 return;
1920 
1921         packets = 0;
1922         bytes = 0;
1923 
1924         rcu_read_lock();
1925         for (i = 0; i < adapter->num_rx_queues; i++) {
1926                 struct igc_ring *ring = adapter->rx_ring[i];
1927                 u32 rqdpc = rd32(IGC_RQDPC(i));
1928 
1929                 if (hw->mac.type >= igc_i225)
1930                         wr32(IGC_RQDPC(i), 0);
1931 
1932                 if (rqdpc) {
1933                         ring->rx_stats.drops += rqdpc;
1934                         net_stats->rx_fifo_errors += rqdpc;
1935                 }
1936 
1937                 do {
1938                         start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
1939                         _bytes = ring->rx_stats.bytes;
1940                         _packets = ring->rx_stats.packets;
1941                 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
1942                 bytes += _bytes;
1943                 packets += _packets;
1944         }
1945 
1946         net_stats->rx_bytes = bytes;
1947         net_stats->rx_packets = packets;
1948 
1949         packets = 0;
1950         bytes = 0;
1951         for (i = 0; i < adapter->num_tx_queues; i++) {
1952                 struct igc_ring *ring = adapter->tx_ring[i];
1953 
1954                 do {
1955                         start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
1956                         _bytes = ring->tx_stats.bytes;
1957                         _packets = ring->tx_stats.packets;
1958                 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
1959                 bytes += _bytes;
1960                 packets += _packets;
1961         }
1962         net_stats->tx_bytes = bytes;
1963         net_stats->tx_packets = packets;
1964         rcu_read_unlock();
1965 
1966         /* read stats registers */
1967         adapter->stats.crcerrs += rd32(IGC_CRCERRS);
1968         adapter->stats.gprc += rd32(IGC_GPRC);
1969         adapter->stats.gorc += rd32(IGC_GORCL);
1970         rd32(IGC_GORCH); /* clear GORCL */
1971         adapter->stats.bprc += rd32(IGC_BPRC);
1972         adapter->stats.mprc += rd32(IGC_MPRC);
1973         adapter->stats.roc += rd32(IGC_ROC);
1974 
1975         adapter->stats.prc64 += rd32(IGC_PRC64);
1976         adapter->stats.prc127 += rd32(IGC_PRC127);
1977         adapter->stats.prc255 += rd32(IGC_PRC255);
1978         adapter->stats.prc511 += rd32(IGC_PRC511);
1979         adapter->stats.prc1023 += rd32(IGC_PRC1023);
1980         adapter->stats.prc1522 += rd32(IGC_PRC1522);
1981         adapter->stats.symerrs += rd32(IGC_SYMERRS);
1982         adapter->stats.sec += rd32(IGC_SEC);
1983 
1984         mpc = rd32(IGC_MPC);
1985         adapter->stats.mpc += mpc;
1986         net_stats->rx_fifo_errors += mpc;
1987         adapter->stats.scc += rd32(IGC_SCC);
1988         adapter->stats.ecol += rd32(IGC_ECOL);
1989         adapter->stats.mcc += rd32(IGC_MCC);
1990         adapter->stats.latecol += rd32(IGC_LATECOL);
1991         adapter->stats.dc += rd32(IGC_DC);
1992         adapter->stats.rlec += rd32(IGC_RLEC);
1993         adapter->stats.xonrxc += rd32(IGC_XONRXC);
1994         adapter->stats.xontxc += rd32(IGC_XONTXC);
1995         adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
1996         adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
1997         adapter->stats.fcruc += rd32(IGC_FCRUC);
1998         adapter->stats.gptc += rd32(IGC_GPTC);
1999         adapter->stats.gotc += rd32(IGC_GOTCL);
2000         rd32(IGC_GOTCH); /* clear GOTCL */
2001         adapter->stats.rnbc += rd32(IGC_RNBC);
2002         adapter->stats.ruc += rd32(IGC_RUC);
2003         adapter->stats.rfc += rd32(IGC_RFC);
2004         adapter->stats.rjc += rd32(IGC_RJC);
2005         adapter->stats.tor += rd32(IGC_TORH);
2006         adapter->stats.tot += rd32(IGC_TOTH);
2007         adapter->stats.tpr += rd32(IGC_TPR);
2008 
2009         adapter->stats.ptc64 += rd32(IGC_PTC64);
2010         adapter->stats.ptc127 += rd32(IGC_PTC127);
2011         adapter->stats.ptc255 += rd32(IGC_PTC255);
2012         adapter->stats.ptc511 += rd32(IGC_PTC511);
2013         adapter->stats.ptc1023 += rd32(IGC_PTC1023);
2014         adapter->stats.ptc1522 += rd32(IGC_PTC1522);
2015 
2016         adapter->stats.mptc += rd32(IGC_MPTC);
2017         adapter->stats.bptc += rd32(IGC_BPTC);
2018 
2019         adapter->stats.tpt += rd32(IGC_TPT);
2020         adapter->stats.colc += rd32(IGC_COLC);
2021 
2022         adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
2023 
2024         adapter->stats.tsctc += rd32(IGC_TSCTC);
2025         adapter->stats.tsctfc += rd32(IGC_TSCTFC);
2026 
2027         adapter->stats.iac += rd32(IGC_IAC);
2028         adapter->stats.icrxoc += rd32(IGC_ICRXOC);
2029         adapter->stats.icrxptc += rd32(IGC_ICRXPTC);
2030         adapter->stats.icrxatc += rd32(IGC_ICRXATC);
2031         adapter->stats.ictxptc += rd32(IGC_ICTXPTC);
2032         adapter->stats.ictxatc += rd32(IGC_ICTXATC);
2033         adapter->stats.ictxqec += rd32(IGC_ICTXQEC);
2034         adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC);
2035         adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC);
2036 
2037         /* Fill out the OS statistics structure */
2038         net_stats->multicast = adapter->stats.mprc;
2039         net_stats->collisions = adapter->stats.colc;
2040 
2041         /* Rx Errors */
2042 
2043         /* RLEC on some newer hardware can be incorrect so build
2044          * our own version based on RUC and ROC
2045          */
2046         net_stats->rx_errors = adapter->stats.rxerrc +
2047                 adapter->stats.crcerrs + adapter->stats.algnerrc +
2048                 adapter->stats.ruc + adapter->stats.roc +
2049                 adapter->stats.cexterr;
2050         net_stats->rx_length_errors = adapter->stats.ruc +
2051                                       adapter->stats.roc;
2052         net_stats->rx_crc_errors = adapter->stats.crcerrs;
2053         net_stats->rx_frame_errors = adapter->stats.algnerrc;
2054         net_stats->rx_missed_errors = adapter->stats.mpc;
2055 
2056         /* Tx Errors */
2057         net_stats->tx_errors = adapter->stats.ecol +
2058                                adapter->stats.latecol;
2059         net_stats->tx_aborted_errors = adapter->stats.ecol;
2060         net_stats->tx_window_errors = adapter->stats.latecol;
2061         net_stats->tx_carrier_errors = adapter->stats.tncrs;
2062 
2063         /* Tx Dropped needs to be maintained elsewhere */
2064 
2065         /* Management Stats */
2066         adapter->stats.mgptc += rd32(IGC_MGTPTC);
2067         adapter->stats.mgprc += rd32(IGC_MGTPRC);
2068         adapter->stats.mgpdc += rd32(IGC_MGTPDC);
2069 }
2070 
2071 static void igc_nfc_filter_exit(struct igc_adapter *adapter)
2072 {
2073         struct igc_nfc_filter *rule;
2074 
2075         spin_lock(&adapter->nfc_lock);
2076 
2077         hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
2078                 igc_erase_filter(adapter, rule);
2079 
2080         hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
2081                 igc_erase_filter(adapter, rule);
2082 
2083         spin_unlock(&adapter->nfc_lock);
2084 }
2085 
2086 static void igc_nfc_filter_restore(struct igc_adapter *adapter)
2087 {
2088         struct igc_nfc_filter *rule;
2089 
2090         spin_lock(&adapter->nfc_lock);
2091 
2092         hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
2093                 igc_add_filter(adapter, rule);
2094 
2095         spin_unlock(&adapter->nfc_lock);
2096 }
2097 
2098 /**
2099  * igc_down - Close the interface
2100  * @adapter: board private structure
2101  */
2102 void igc_down(struct igc_adapter *adapter)
2103 {
2104         struct net_device *netdev = adapter->netdev;
2105         struct igc_hw *hw = &adapter->hw;
2106         u32 tctl, rctl;
2107         int i = 0;
2108 
2109         set_bit(__IGC_DOWN, &adapter->state);
2110 
2111         /* disable receives in the hardware */
2112         rctl = rd32(IGC_RCTL);
2113         wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
2114         /* flush and sleep below */
2115 
2116         igc_nfc_filter_exit(adapter);
2117 
2118         /* set trans_start so we don't get spurious watchdogs during reset */
2119         netif_trans_update(netdev);
2120 
2121         netif_carrier_off(netdev);
2122         netif_tx_stop_all_queues(netdev);
2123 
2124         /* disable transmits in the hardware */
2125         tctl = rd32(IGC_TCTL);
2126         tctl &= ~IGC_TCTL_EN;
2127         wr32(IGC_TCTL, tctl);
2128         /* flush both disables and wait for them to finish */
2129         wrfl();
2130         usleep_range(10000, 20000);
2131 
2132         igc_irq_disable(adapter);
2133 
2134         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2135 
2136         for (i = 0; i < adapter->num_q_vectors; i++) {
2137                 if (adapter->q_vector[i]) {
2138                         napi_synchronize(&adapter->q_vector[i]->napi);
2139                         napi_disable(&adapter->q_vector[i]->napi);
2140                 }
2141         }
2142 
2143         del_timer_sync(&adapter->watchdog_timer);
2144         del_timer_sync(&adapter->phy_info_timer);
2145 
2146         /* record the stats before reset*/
2147         spin_lock(&adapter->stats64_lock);
2148         igc_update_stats(adapter);
2149         spin_unlock(&adapter->stats64_lock);
2150 
2151         adapter->link_speed = 0;
2152         adapter->link_duplex = 0;
2153 
2154         if (!pci_channel_offline(adapter->pdev))
2155                 igc_reset(adapter);
2156 
2157         /* clear VLAN promisc flag so VFTA will be updated if necessary */
2158         adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
2159 
2160         igc_clean_all_tx_rings(adapter);
2161         igc_clean_all_rx_rings(adapter);
2162 }
2163 
2164 void igc_reinit_locked(struct igc_adapter *adapter)
2165 {
2166         WARN_ON(in_interrupt());
2167         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
2168                 usleep_range(1000, 2000);
2169         igc_down(adapter);
2170         igc_up(adapter);
2171         clear_bit(__IGC_RESETTING, &adapter->state);
2172 }
2173 
2174 static void igc_reset_task(struct work_struct *work)
2175 {
2176         struct igc_adapter *adapter;
2177 
2178         adapter = container_of(work, struct igc_adapter, reset_task);
2179 
2180         netdev_err(adapter->netdev, "Reset adapter\n");
2181         igc_reinit_locked(adapter);
2182 }
2183 
2184 /**
2185  * igc_change_mtu - Change the Maximum Transfer Unit
2186  * @netdev: network interface device structure
2187  * @new_mtu: new value for maximum frame size
2188  *
2189  * Returns 0 on success, negative on failure
2190  */
2191 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
2192 {
2193         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2194         struct igc_adapter *adapter = netdev_priv(netdev);
2195         struct pci_dev *pdev = adapter->pdev;
2196 
2197         /* adjust max frame to be at least the size of a standard frame */
2198         if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
2199                 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
2200 
2201         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
2202                 usleep_range(1000, 2000);
2203 
2204         /* igc_down has a dependency on max_frame_size */
2205         adapter->max_frame_size = max_frame;
2206 
2207         if (netif_running(netdev))
2208                 igc_down(adapter);
2209 
2210         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
2211                  netdev->mtu, new_mtu);
2212         netdev->mtu = new_mtu;
2213 
2214         if (netif_running(netdev))
2215                 igc_up(adapter);
2216         else
2217                 igc_reset(adapter);
2218 
2219         clear_bit(__IGC_RESETTING, &adapter->state);
2220 
2221         return 0;
2222 }
2223 
2224 /**
2225  * igc_get_stats - Get System Network Statistics
2226  * @netdev: network interface device structure
2227  *
2228  * Returns the address of the device statistics structure.
2229  * The statistics are updated here and also from the timer callback.
2230  */
2231 static struct net_device_stats *igc_get_stats(struct net_device *netdev)
2232 {
2233         struct igc_adapter *adapter = netdev_priv(netdev);
2234 
2235         if (!test_bit(__IGC_RESETTING, &adapter->state))
2236                 igc_update_stats(adapter);
2237 
2238         /* only return the current stats */
2239         return &netdev->stats;
2240 }
2241 
2242 static netdev_features_t igc_fix_features(struct net_device *netdev,
2243                                           netdev_features_t features)
2244 {
2245         /* Since there is no support for separate Rx/Tx vlan accel
2246          * enable/disable make sure Tx flag is always in same state as Rx.
2247          */
2248         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2249                 features |= NETIF_F_HW_VLAN_CTAG_TX;
2250         else
2251                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2252 
2253         return features;
2254 }
2255 
2256 static int igc_set_features(struct net_device *netdev,
2257                             netdev_features_t features)
2258 {
2259         netdev_features_t changed = netdev->features ^ features;
2260         struct igc_adapter *adapter = netdev_priv(netdev);
2261 
2262         /* Add VLAN support */
2263         if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2264                 return 0;
2265 
2266         if (!(features & NETIF_F_NTUPLE)) {
2267                 struct hlist_node *node2;
2268                 struct igc_nfc_filter *rule;
2269 
2270                 spin_lock(&adapter->nfc_lock);
2271                 hlist_for_each_entry_safe(rule, node2,
2272                                           &adapter->nfc_filter_list, nfc_node) {
2273                         igc_erase_filter(adapter, rule);
2274                         hlist_del(&rule->nfc_node);
2275                         kfree(rule);
2276                 }
2277                 spin_unlock(&adapter->nfc_lock);
2278                 adapter->nfc_filter_count = 0;
2279         }
2280 
2281         netdev->features = features;
2282 
2283         if (netif_running(netdev))
2284                 igc_reinit_locked(adapter);
2285         else
2286                 igc_reset(adapter);
2287 
2288         return 1;
2289 }
2290 
2291 static netdev_features_t
2292 igc_features_check(struct sk_buff *skb, struct net_device *dev,
2293                    netdev_features_t features)
2294 {
2295         unsigned int network_hdr_len, mac_hdr_len;
2296 
2297         /* Make certain the headers can be described by a context descriptor */
2298         mac_hdr_len = skb_network_header(skb) - skb->data;
2299         if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
2300                 return features & ~(NETIF_F_HW_CSUM |
2301                                     NETIF_F_SCTP_CRC |
2302                                     NETIF_F_HW_VLAN_CTAG_TX |
2303                                     NETIF_F_TSO |
2304                                     NETIF_F_TSO6);
2305 
2306         network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2307         if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
2308                 return features & ~(NETIF_F_HW_CSUM |
2309                                     NETIF_F_SCTP_CRC |
2310                                     NETIF_F_TSO |
2311                                     NETIF_F_TSO6);
2312 
2313         /* We can only support IPv4 TSO in tunnels if we can mangle the
2314          * inner IP ID field, so strip TSO if MANGLEID is not supported.
2315          */
2316         if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2317                 features &= ~NETIF_F_TSO;
2318 
2319         return features;
2320 }
2321 
2322 /**
2323  * igc_configure - configure the hardware for RX and TX
2324  * @adapter: private board structure
2325  */
2326 static void igc_configure(struct igc_adapter *adapter)
2327 {
2328         struct net_device *netdev = adapter->netdev;
2329         int i = 0;
2330 
2331         igc_get_hw_control(adapter);
2332         igc_set_rx_mode(netdev);
2333 
2334         igc_setup_tctl(adapter);
2335         igc_setup_mrqc(adapter);
2336         igc_setup_rctl(adapter);
2337 
2338         igc_nfc_filter_restore(adapter);
2339         igc_configure_tx(adapter);
2340         igc_configure_rx(adapter);
2341 
2342         igc_rx_fifo_flush_base(&adapter->hw);
2343 
2344         /* call igc_desc_unused which always leaves
2345          * at least 1 descriptor unused to make sure
2346          * next_to_use != next_to_clean
2347          */
2348         for (i = 0; i < adapter->num_rx_queues; i++) {
2349                 struct igc_ring *ring = adapter->rx_ring[i];
2350 
2351                 igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
2352         }
2353 }
2354 
2355 /**
2356  * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
2357  * @adapter: address of board private structure
2358  * @index: Index of the RAR entry which need to be synced with MAC table
2359  */
2360 static void igc_rar_set_index(struct igc_adapter *adapter, u32 index)
2361 {
2362         u8 *addr = adapter->mac_table[index].addr;
2363         struct igc_hw *hw = &adapter->hw;
2364         u32 rar_low, rar_high;
2365 
2366         /* HW expects these to be in network order when they are plugged
2367          * into the registers which are little endian.  In order to guarantee
2368          * that ordering we need to do an leXX_to_cpup here in order to be
2369          * ready for the byteswap that occurs with writel
2370          */
2371         rar_low = le32_to_cpup((__le32 *)(addr));
2372         rar_high = le16_to_cpup((__le16 *)(addr + 4));
2373 
2374         /* Indicate to hardware the Address is Valid. */
2375         if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) {
2376                 if (is_valid_ether_addr(addr))
2377                         rar_high |= IGC_RAH_AV;
2378 
2379                 rar_high |= IGC_RAH_POOL_1 <<
2380                         adapter->mac_table[index].queue;
2381         }
2382 
2383         wr32(IGC_RAL(index), rar_low);
2384         wrfl();
2385         wr32(IGC_RAH(index), rar_high);
2386         wrfl();
2387 }
2388 
2389 /* Set default MAC address for the PF in the first RAR entry */
2390 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
2391 {
2392         struct igc_mac_addr *mac_table = &adapter->mac_table[0];
2393 
2394         ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
2395         mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE;
2396 
2397         igc_rar_set_index(adapter, 0);
2398 }
2399 
2400 /* If the filter to be added and an already existing filter express
2401  * the same address and address type, it should be possible to only
2402  * override the other configurations, for example the queue to steer
2403  * traffic.
2404  */
2405 static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry,
2406                                       const u8 *addr, const u8 flags)
2407 {
2408         if (!(entry->state & IGC_MAC_STATE_IN_USE))
2409                 return true;
2410 
2411         if ((entry->state & IGC_MAC_STATE_SRC_ADDR) !=
2412             (flags & IGC_MAC_STATE_SRC_ADDR))
2413                 return false;
2414 
2415         if (!ether_addr_equal(addr, entry->addr))
2416                 return false;
2417 
2418         return true;
2419 }
2420 
2421 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
2422  * 'flags' is used to indicate what kind of match is made, match is by
2423  * default for the destination address, if matching by source address
2424  * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used.
2425  */
2426 static int igc_add_mac_filter_flags(struct igc_adapter *adapter,
2427                                     const u8 *addr, const u8 queue,
2428                                     const u8 flags)
2429 {
2430         struct igc_hw *hw = &adapter->hw;
2431         int rar_entries = hw->mac.rar_entry_count;
2432         int i;
2433 
2434         if (is_zero_ether_addr(addr))
2435                 return -EINVAL;
2436 
2437         /* Search for the first empty entry in the MAC table.
2438          * Do not touch entries at the end of the table reserved for the VF MAC
2439          * addresses.
2440          */
2441         for (i = 0; i < rar_entries; i++) {
2442                 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i],
2443                                                addr, flags))
2444                         continue;
2445 
2446                 ether_addr_copy(adapter->mac_table[i].addr, addr);
2447                 adapter->mac_table[i].queue = queue;
2448                 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags;
2449 
2450                 igc_rar_set_index(adapter, i);
2451                 return i;
2452         }
2453 
2454         return -ENOSPC;
2455 }
2456 
2457 int igc_add_mac_steering_filter(struct igc_adapter *adapter,
2458                                 const u8 *addr, u8 queue, u8 flags)
2459 {
2460         return igc_add_mac_filter_flags(adapter, addr, queue,
2461                                         IGC_MAC_STATE_QUEUE_STEERING | flags);
2462 }
2463 
2464 /* Remove a MAC filter for 'addr' directing matching traffic to
2465  * 'queue', 'flags' is used to indicate what kind of match need to be
2466  * removed, match is by default for the destination address, if
2467  * matching by source address is to be removed the flag
2468  * IGC_MAC_STATE_SRC_ADDR can be used.
2469  */
2470 static int igc_del_mac_filter_flags(struct igc_adapter *adapter,
2471                                     const u8 *addr, const u8 queue,
2472                                     const u8 flags)
2473 {
2474         struct igc_hw *hw = &adapter->hw;
2475         int rar_entries = hw->mac.rar_entry_count;
2476         int i;
2477 
2478         if (is_zero_ether_addr(addr))
2479                 return -EINVAL;
2480 
2481         /* Search for matching entry in the MAC table based on given address
2482          * and queue. Do not touch entries at the end of the table reserved
2483          * for the VF MAC addresses.
2484          */
2485         for (i = 0; i < rar_entries; i++) {
2486                 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE))
2487                         continue;
2488                 if ((adapter->mac_table[i].state & flags) != flags)
2489                         continue;
2490                 if (adapter->mac_table[i].queue != queue)
2491                         continue;
2492                 if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
2493                         continue;
2494 
2495                 /* When a filter for the default address is "deleted",
2496                  * we return it to its initial configuration
2497                  */
2498                 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) {
2499                         adapter->mac_table[i].state =
2500                                 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE;
2501                 } else {
2502                         adapter->mac_table[i].state = 0;
2503                         adapter->mac_table[i].queue = 0;
2504                         memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
2505                 }
2506 
2507                 igc_rar_set_index(adapter, i);
2508                 return 0;
2509         }
2510 
2511         return -ENOENT;
2512 }
2513 
2514 int igc_del_mac_steering_filter(struct igc_adapter *adapter,
2515                                 const u8 *addr, u8 queue, u8 flags)
2516 {
2517         return igc_del_mac_filter_flags(adapter, addr, queue,
2518                                         IGC_MAC_STATE_QUEUE_STEERING | flags);
2519 }
2520 
2521 /**
2522  * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2523  * @netdev: network interface device structure
2524  *
2525  * The set_rx_mode entry point is called whenever the unicast or multicast
2526  * address lists or the network interface flags are updated.  This routine is
2527  * responsible for configuring the hardware for proper unicast, multicast,
2528  * promiscuous mode, and all-multi behavior.
2529  */
2530 static void igc_set_rx_mode(struct net_device *netdev)
2531 {
2532 }
2533 
2534 /**
2535  * igc_msix_other - msix other interrupt handler
2536  * @irq: interrupt number
2537  * @data: pointer to a q_vector
2538  */
2539 static irqreturn_t igc_msix_other(int irq, void *data)
2540 {
2541         struct igc_adapter *adapter = data;
2542         struct igc_hw *hw = &adapter->hw;
2543         u32 icr = rd32(IGC_ICR);
2544 
2545         /* reading ICR causes bit 31 of EICR to be cleared */
2546         if (icr & IGC_ICR_DRSTA)
2547                 schedule_work(&adapter->reset_task);
2548 
2549         if (icr & IGC_ICR_DOUTSYNC) {
2550                 /* HW is reporting DMA is out of sync */
2551                 adapter->stats.doosync++;
2552         }
2553 
2554         if (icr & IGC_ICR_LSC) {
2555                 hw->mac.get_link_status = 1;
2556                 /* guard against interrupt when we're going down */
2557                 if (!test_bit(__IGC_DOWN, &adapter->state))
2558                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
2559         }
2560 
2561         wr32(IGC_EIMS, adapter->eims_other);
2562 
2563         return IRQ_HANDLED;
2564 }
2565 
2566 /**
2567  * igc_write_ivar - configure ivar for given MSI-X vector
2568  * @hw: pointer to the HW structure
2569  * @msix_vector: vector number we are allocating to a given ring
2570  * @index: row index of IVAR register to write within IVAR table
2571  * @offset: column offset of in IVAR, should be multiple of 8
2572  *
2573  * The IVAR table consists of 2 columns,
2574  * each containing an cause allocation for an Rx and Tx ring, and a
2575  * variable number of rows depending on the number of queues supported.
2576  */
2577 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
2578                            int index, int offset)
2579 {
2580         u32 ivar = array_rd32(IGC_IVAR0, index);
2581 
2582         /* clear any bits that are currently set */
2583         ivar &= ~((u32)0xFF << offset);
2584 
2585         /* write vector and valid bit */
2586         ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
2587 
2588         array_wr32(IGC_IVAR0, index, ivar);
2589 }
2590 
2591 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
2592 {
2593         struct igc_adapter *adapter = q_vector->adapter;
2594         struct igc_hw *hw = &adapter->hw;
2595         int rx_queue = IGC_N0_QUEUE;
2596         int tx_queue = IGC_N0_QUEUE;
2597 
2598         if (q_vector->rx.ring)
2599                 rx_queue = q_vector->rx.ring->reg_idx;
2600         if (q_vector->tx.ring)
2601                 tx_queue = q_vector->tx.ring->reg_idx;
2602 
2603         switch (hw->mac.type) {
2604         case igc_i225:
2605                 if (rx_queue > IGC_N0_QUEUE)
2606                         igc_write_ivar(hw, msix_vector,
2607                                        rx_queue >> 1,
2608                                        (rx_queue & 0x1) << 4);
2609                 if (tx_queue > IGC_N0_QUEUE)
2610                         igc_write_ivar(hw, msix_vector,
2611                                        tx_queue >> 1,
2612                                        ((tx_queue & 0x1) << 4) + 8);
2613                 q_vector->eims_value = BIT(msix_vector);
2614                 break;
2615         default:
2616                 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
2617                 break;
2618         }
2619 
2620         /* add q_vector eims value to global eims_enable_mask */
2621         adapter->eims_enable_mask |= q_vector->eims_value;
2622 
2623         /* configure q_vector to set itr on first interrupt */
2624         q_vector->set_itr = 1;
2625 }
2626 
2627 /**
2628  * igc_configure_msix - Configure MSI-X hardware
2629  * @adapter: Pointer to adapter structure
2630  *
2631  * igc_configure_msix sets up the hardware to properly
2632  * generate MSI-X interrupts.
2633  */
2634 static void igc_configure_msix(struct igc_adapter *adapter)
2635 {
2636         struct igc_hw *hw = &adapter->hw;
2637         int i, vector = 0;
2638         u32 tmp;
2639 
2640         adapter->eims_enable_mask = 0;
2641 
2642         /* set vector for other causes, i.e. link changes */
2643         switch (hw->mac.type) {
2644         case igc_i225:
2645                 /* Turn on MSI-X capability first, or our settings
2646                  * won't stick.  And it will take days to debug.
2647                  */
2648                 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
2649                      IGC_GPIE_PBA | IGC_GPIE_EIAME |
2650                      IGC_GPIE_NSICR);
2651 
2652                 /* enable msix_other interrupt */
2653                 adapter->eims_other = BIT(vector);
2654                 tmp = (vector++ | IGC_IVAR_VALID) << 8;
2655 
2656                 wr32(IGC_IVAR_MISC, tmp);
2657                 break;
2658         default:
2659                 /* do nothing, since nothing else supports MSI-X */
2660                 break;
2661         } /* switch (hw->mac.type) */
2662 
2663         adapter->eims_enable_mask |= adapter->eims_other;
2664 
2665         for (i = 0; i < adapter->num_q_vectors; i++)
2666                 igc_assign_vector(adapter->q_vector[i], vector++);
2667 
2668         wrfl();
2669 }
2670 
2671 static irqreturn_t igc_msix_ring(int irq, void *data)
2672 {
2673         struct igc_q_vector *q_vector = data;
2674 
2675         /* Write the ITR value calculated from the previous interrupt. */
2676         igc_write_itr(q_vector);
2677 
2678         napi_schedule(&q_vector->napi);
2679 
2680         return IRQ_HANDLED;
2681 }
2682 
2683 /**
2684  * igc_request_msix - Initialize MSI-X interrupts
2685  * @adapter: Pointer to adapter structure
2686  *
2687  * igc_request_msix allocates MSI-X vectors and requests interrupts from the
2688  * kernel.
2689  */
2690 static int igc_request_msix(struct igc_adapter *adapter)
2691 {
2692         int i = 0, err = 0, vector = 0, free_vector = 0;
2693         struct net_device *netdev = adapter->netdev;
2694 
2695         err = request_irq(adapter->msix_entries[vector].vector,
2696                           &igc_msix_other, 0, netdev->name, adapter);
2697         if (err)
2698                 goto err_out;
2699 
2700         for (i = 0; i < adapter->num_q_vectors; i++) {
2701                 struct igc_q_vector *q_vector = adapter->q_vector[i];
2702 
2703                 vector++;
2704 
2705                 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
2706 
2707                 if (q_vector->rx.ring && q_vector->tx.ring)
2708                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
2709                                 q_vector->rx.ring->queue_index);
2710                 else if (q_vector->tx.ring)
2711                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
2712                                 q_vector->tx.ring->queue_index);
2713                 else if (q_vector->rx.ring)
2714                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
2715                                 q_vector->rx.ring->queue_index);
2716                 else
2717                         sprintf(q_vector->name, "%s-unused", netdev->name);
2718 
2719                 err = request_irq(adapter->msix_entries[vector].vector,
2720                                   igc_msix_ring, 0, q_vector->name,
2721                                   q_vector);
2722                 if (err)
2723                         goto err_free;
2724         }
2725 
2726         igc_configure_msix(adapter);
2727         return 0;
2728 
2729 err_free:
2730         /* free already assigned IRQs */
2731         free_irq(adapter->msix_entries[free_vector++].vector, adapter);
2732 
2733         vector--;
2734         for (i = 0; i < vector; i++) {
2735                 free_irq(adapter->msix_entries[free_vector++].vector,
2736                          adapter->q_vector[i]);
2737         }
2738 err_out:
2739         return err;
2740 }
2741 
2742 /**
2743  * igc_reset_q_vector - Reset config for interrupt vector
2744  * @adapter: board private structure to initialize
2745  * @v_idx: Index of vector to be reset
2746  *
2747  * If NAPI is enabled it will delete any references to the
2748  * NAPI struct. This is preparation for igc_free_q_vector.
2749  */
2750 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
2751 {
2752         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
2753 
2754         /* if we're coming from igc_set_interrupt_capability, the vectors are
2755          * not yet allocated
2756          */
2757         if (!q_vector)
2758                 return;
2759 
2760         if (q_vector->tx.ring)
2761                 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
2762 
2763         if (q_vector->rx.ring)
2764                 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
2765 
2766         netif_napi_del(&q_vector->napi);
2767 }
2768 
2769 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
2770 {
2771         int v_idx = adapter->num_q_vectors;
2772 
2773         if (adapter->msix_entries) {
2774                 pci_disable_msix(adapter->pdev);
2775                 kfree(adapter->msix_entries);
2776                 adapter->msix_entries = NULL;
2777         } else if (adapter->flags & IGC_FLAG_HAS_MSI) {
2778                 pci_disable_msi(adapter->pdev);
2779         }
2780 
2781         while (v_idx--)
2782                 igc_reset_q_vector(adapter, v_idx);
2783 }
2784 
2785 /**
2786  * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
2787  * @adapter: Pointer to adapter structure
2788  *
2789  * This function resets the device so that it has 0 rx queues, tx queues, and
2790  * MSI-X interrupts allocated.
2791  */
2792 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
2793 {
2794         igc_free_q_vectors(adapter);
2795         igc_reset_interrupt_capability(adapter);
2796 }
2797 
2798 /**
2799  * igc_free_q_vectors - Free memory allocated for interrupt vectors
2800  * @adapter: board private structure to initialize
2801  *
2802  * This function frees the memory allocated to the q_vectors.  In addition if
2803  * NAPI is enabled it will delete any references to the NAPI struct prior
2804  * to freeing the q_vector.
2805  */
2806 static void igc_free_q_vectors(struct igc_adapter *adapter)
2807 {
2808         int v_idx = adapter->num_q_vectors;
2809 
2810         adapter->num_tx_queues = 0;
2811         adapter->num_rx_queues = 0;
2812         adapter->num_q_vectors = 0;
2813 
2814         while (v_idx--) {
2815                 igc_reset_q_vector(adapter, v_idx);
2816                 igc_free_q_vector(adapter, v_idx);
2817         }
2818 }
2819 
2820 /**
2821  * igc_free_q_vector - Free memory allocated for specific interrupt vector
2822  * @adapter: board private structure to initialize
2823  * @v_idx: Index of vector to be freed
2824  *
2825  * This function frees the memory allocated to the q_vector.
2826  */
2827 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
2828 {
2829         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
2830 
2831         adapter->q_vector[v_idx] = NULL;
2832 
2833         /* igc_get_stats64() might access the rings on this vector,
2834          * we must wait a grace period before freeing it.
2835          */
2836         if (q_vector)
2837                 kfree_rcu(q_vector, rcu);
2838 }
2839 
2840 /* Need to wait a few seconds after link up to get diagnostic information from
2841  * the phy
2842  */
2843 static void igc_update_phy_info(struct timer_list *t)
2844 {
2845         struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
2846 
2847         igc_get_phy_info(&adapter->hw);
2848 }
2849 
2850 /**
2851  * igc_has_link - check shared code for link and determine up/down
2852  * @adapter: pointer to driver private info
2853  */
2854 bool igc_has_link(struct igc_adapter *adapter)
2855 {
2856         struct igc_hw *hw = &adapter->hw;
2857         bool link_active = false;
2858 
2859         /* get_link_status is set on LSC (link status) interrupt or
2860          * rx sequence error interrupt.  get_link_status will stay
2861          * false until the igc_check_for_link establishes link
2862          * for copper adapters ONLY
2863          */
2864         switch (hw->phy.media_type) {
2865         case igc_media_type_copper:
2866                 if (!hw->mac.get_link_status)
2867                         return true;
2868                 hw->mac.ops.check_for_link(hw);
2869                 link_active = !hw->mac.get_link_status;
2870                 break;
2871         default:
2872         case igc_media_type_unknown:
2873                 break;
2874         }
2875 
2876         if (hw->mac.type == igc_i225 &&
2877             hw->phy.id == I225_I_PHY_ID) {
2878                 if (!netif_carrier_ok(adapter->netdev)) {
2879                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2880                 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
2881                         adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
2882                         adapter->link_check_timeout = jiffies;
2883                 }
2884         }
2885 
2886         return link_active;
2887 }
2888 
2889 /**
2890  * igc_watchdog - Timer Call-back
2891  * @data: pointer to adapter cast into an unsigned long
2892  */
2893 static void igc_watchdog(struct timer_list *t)
2894 {
2895         struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
2896         /* Do the rest outside of interrupt context */
2897         schedule_work(&adapter->watchdog_task);
2898 }
2899 
2900 static void igc_watchdog_task(struct work_struct *work)
2901 {
2902         struct igc_adapter *adapter = container_of(work,
2903                                                    struct igc_adapter,
2904                                                    watchdog_task);
2905         struct net_device *netdev = adapter->netdev;
2906         struct igc_hw *hw = &adapter->hw;
2907         struct igc_phy_info *phy = &hw->phy;
2908         u16 phy_data, retry_count = 20;
2909         u32 connsw;
2910         u32 link;
2911         int i;
2912 
2913         link = igc_has_link(adapter);
2914 
2915         if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
2916                 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
2917                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2918                 else
2919                         link = false;
2920         }
2921 
2922         /* Force link down if we have fiber to swap to */
2923         if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
2924                 if (hw->phy.media_type == igc_media_type_copper) {
2925                         connsw = rd32(IGC_CONNSW);
2926                         if (!(connsw & IGC_CONNSW_AUTOSENSE_EN))
2927                                 link = 0;
2928                 }
2929         }
2930         if (link) {
2931                 if (!netif_carrier_ok(netdev)) {
2932                         u32 ctrl;
2933 
2934                         hw->mac.ops.get_speed_and_duplex(hw,
2935                                                          &adapter->link_speed,
2936                                                          &adapter->link_duplex);
2937 
2938                         ctrl = rd32(IGC_CTRL);
2939                         /* Link status message must follow this format */
2940                         netdev_info(netdev,
2941                                     "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
2942                                     netdev->name,
2943                                     adapter->link_speed,
2944                                     adapter->link_duplex == FULL_DUPLEX ?
2945                                     "Full" : "Half",
2946                                     (ctrl & IGC_CTRL_TFCE) &&
2947                                     (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
2948                                     (ctrl & IGC_CTRL_RFCE) ?  "RX" :
2949                                     (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
2950 
2951                         /* check if SmartSpeed worked */
2952                         igc_check_downshift(hw);
2953                         if (phy->speed_downgraded)
2954                                 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
2955 
2956                         /* adjust timeout factor according to speed/duplex */
2957                         adapter->tx_timeout_factor = 1;
2958                         switch (adapter->link_speed) {
2959                         case SPEED_10:
2960                                 adapter->tx_timeout_factor = 14;
2961                                 break;
2962                         case SPEED_100:
2963                                 /* maybe add some timeout factor ? */
2964                                 break;
2965                         }
2966 
2967                         if (adapter->link_speed != SPEED_1000)
2968                                 goto no_wait;
2969 
2970                         /* wait for Remote receiver status OK */
2971 retry_read_status:
2972                         if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
2973                                               &phy_data)) {
2974                                 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
2975                                     retry_count) {
2976                                         msleep(100);
2977                                         retry_count--;
2978                                         goto retry_read_status;
2979                                 } else if (!retry_count) {
2980                                         dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
2981                                 }
2982                         } else {
2983                                 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
2984                         }
2985 no_wait:
2986                         netif_carrier_on(netdev);
2987 
2988                         /* link state has changed, schedule phy info update */
2989                         if (!test_bit(__IGC_DOWN, &adapter->state))
2990                                 mod_timer(&adapter->phy_info_timer,
2991                                           round_jiffies(jiffies + 2 * HZ));
2992                 }
2993         } else {
2994                 if (netif_carrier_ok(netdev)) {
2995                         adapter->link_speed = 0;
2996                         adapter->link_duplex = 0;
2997 
2998                         /* Links status message must follow this format */
2999                         netdev_info(netdev, "igc: %s NIC Link is Down\n",
3000                                     netdev->name);
3001                         netif_carrier_off(netdev);
3002 
3003                         /* link state has changed, schedule phy info update */
3004                         if (!test_bit(__IGC_DOWN, &adapter->state))
3005                                 mod_timer(&adapter->phy_info_timer,
3006                                           round_jiffies(jiffies + 2 * HZ));
3007 
3008                         /* link is down, time to check for alternate media */
3009                         if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
3010                                 if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
3011                                         schedule_work(&adapter->reset_task);
3012                                         /* return immediately */
3013                                         return;
3014                                 }
3015                         }
3016 
3017                 /* also check for alternate media here */
3018                 } else if (!netif_carrier_ok(netdev) &&
3019                            (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
3020                         if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
3021                                 schedule_work(&adapter->reset_task);
3022                                 /* return immediately */
3023                                 return;
3024                         }
3025                 }
3026         }
3027 
3028         spin_lock(&adapter->stats64_lock);
3029         igc_update_stats(adapter);
3030         spin_unlock(&adapter->stats64_lock);
3031 
3032         for (i = 0; i < adapter->num_tx_queues; i++) {
3033                 struct igc_ring *tx_ring = adapter->tx_ring[i];
3034 
3035                 if (!netif_carrier_ok(netdev)) {
3036                         /* We've lost link, so the controller stops DMA,
3037                          * but we've got queued Tx work that's never going
3038                          * to get done, so reset controller to flush Tx.
3039                          * (Do the reset outside of interrupt context).
3040                          */
3041                         if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
3042                                 adapter->tx_timeout_count++;
3043                                 schedule_work(&adapter->reset_task);
3044                                 /* return immediately since reset is imminent */
3045                                 return;
3046                         }
3047                 }
3048 
3049                 /* Force detection of hung controller every watchdog period */
3050                 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3051         }
3052 
3053         /* Cause software interrupt to ensure Rx ring is cleaned */
3054         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
3055                 u32 eics = 0;
3056 
3057                 for (i = 0; i < adapter->num_q_vectors; i++)
3058                         eics |= adapter->q_vector[i]->eims_value;
3059                 wr32(IGC_EICS, eics);
3060         } else {
3061                 wr32(IGC_ICS, IGC_ICS_RXDMT0);
3062         }
3063 
3064         /* Reset the timer */
3065         if (!test_bit(__IGC_DOWN, &adapter->state)) {
3066                 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
3067                         mod_timer(&adapter->watchdog_timer,
3068                                   round_jiffies(jiffies +  HZ));
3069                 else
3070                         mod_timer(&adapter->watchdog_timer,
3071                                   round_jiffies(jiffies + 2 * HZ));
3072         }
3073 }
3074 
3075 /**
3076  * igc_update_ring_itr - update the dynamic ITR value based on packet size
3077  * @q_vector: pointer to q_vector
3078  *
3079  * Stores a new ITR value based on strictly on packet size.  This
3080  * algorithm is less sophisticated than that used in igc_update_itr,
3081  * due to the difficulty of synchronizing statistics across multiple
3082  * receive rings.  The divisors and thresholds used by this function
3083  * were determined based on theoretical maximum wire speed and testing
3084  * data, in order to minimize response time while increasing bulk
3085  * throughput.
3086  * NOTE: This function is called only when operating in a multiqueue
3087  * receive environment.
3088  */
3089 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
3090 {
3091         struct igc_adapter *adapter = q_vector->adapter;
3092         int new_val = q_vector->itr_val;
3093         int avg_wire_size = 0;
3094         unsigned int packets;
3095 
3096         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3097          * ints/sec - ITR timer value of 120 ticks.
3098          */
3099         switch (adapter->link_speed) {
3100         case SPEED_10:
3101         case SPEED_100:
3102                 new_val = IGC_4K_ITR;
3103                 goto set_itr_val;
3104         default:
3105                 break;
3106         }
3107 
3108         packets = q_vector->rx.total_packets;
3109         if (packets)
3110                 avg_wire_size = q_vector->rx.total_bytes / packets;
3111 
3112         packets = q_vector->tx.total_packets;
3113         if (packets)
3114                 avg_wire_size = max_t(u32, avg_wire_size,
3115                                       q_vector->tx.total_bytes / packets);
3116 
3117         /* if avg_wire_size isn't set no work was done */
3118         if (!avg_wire_size)
3119                 goto clear_counts;
3120 
3121         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3122         avg_wire_size += 24;
3123 
3124         /* Don't starve jumbo frames */
3125         avg_wire_size = min(avg_wire_size, 3000);
3126 
3127         /* Give a little boost to mid-size frames */
3128         if (avg_wire_size > 300 && avg_wire_size < 1200)
3129                 new_val = avg_wire_size / 3;
3130         else
3131                 new_val = avg_wire_size / 2;
3132 
3133         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3134         if (new_val < IGC_20K_ITR &&
3135             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3136             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3137                 new_val = IGC_20K_ITR;
3138 
3139 set_itr_val:
3140         if (new_val != q_vector->itr_val) {
3141                 q_vector->itr_val = new_val;
3142                 q_vector->set_itr = 1;
3143         }
3144 clear_counts:
3145         q_vector->rx.total_bytes = 0;
3146         q_vector->rx.total_packets = 0;
3147         q_vector->tx.total_bytes = 0;
3148         q_vector->tx.total_packets = 0;
3149 }
3150 
3151 /**
3152  * igc_update_itr - update the dynamic ITR value based on statistics
3153  * @q_vector: pointer to q_vector
3154  * @ring_container: ring info to update the itr for
3155  *
3156  * Stores a new ITR value based on packets and byte
3157  * counts during the last interrupt.  The advantage of per interrupt
3158  * computation is faster updates and more accurate ITR for the current
3159  * traffic pattern.  Constants in this function were computed
3160  * based on theoretical maximum wire speed and thresholds were set based
3161  * on testing data as well as attempting to minimize response time
3162  * while increasing bulk throughput.
3163  * NOTE: These calculations are only valid when operating in a single-
3164  * queue environment.
3165  */
3166 static void igc_update_itr(struct igc_q_vector *q_vector,
3167                            struct igc_ring_container *ring_container)
3168 {
3169         unsigned int packets = ring_container->total_packets;
3170         unsigned int bytes = ring_container->total_bytes;
3171         u8 itrval = ring_container->itr;
3172 
3173         /* no packets, exit with status unchanged */
3174         if (packets == 0)
3175                 return;
3176 
3177         switch (itrval) {
3178         case lowest_latency:
3179                 /* handle TSO and jumbo frames */
3180                 if (bytes / packets > 8000)
3181                         itrval = bulk_latency;
3182                 else if ((packets < 5) && (bytes > 512))
3183                         itrval = low_latency;
3184                 break;
3185         case low_latency:  /* 50 usec aka 20000 ints/s */
3186                 if (bytes > 10000) {
3187                         /* this if handles the TSO accounting */
3188                         if (bytes / packets > 8000)
3189                                 itrval = bulk_latency;
3190                         else if ((packets < 10) || ((bytes / packets) > 1200))
3191                                 itrval = bulk_latency;
3192                         else if ((packets > 35))
3193                                 itrval = lowest_latency;
3194                 } else if (bytes / packets > 2000) {
3195                         itrval = bulk_latency;
3196                 } else if (packets <= 2 && bytes < 512) {
3197                         itrval = lowest_latency;
3198                 }
3199                 break;
3200         case bulk_latency: /* 250 usec aka 4000 ints/s */
3201                 if (bytes > 25000) {
3202                         if (packets > 35)
3203                                 itrval = low_latency;
3204                 } else if (bytes < 1500) {
3205                         itrval = low_latency;
3206                 }
3207                 break;
3208         }
3209 
3210         /* clear work counters since we have the values we need */
3211         ring_container->total_bytes = 0;
3212         ring_container->total_packets = 0;
3213 
3214         /* write updated itr to ring container */
3215         ring_container->itr = itrval;
3216 }
3217 
3218 /**
3219  * igc_intr_msi - Interrupt Handler
3220  * @irq: interrupt number
3221  * @data: pointer to a network interface device structure
3222  */
3223 static irqreturn_t igc_intr_msi(int irq, void *data)
3224 {
3225         struct igc_adapter *adapter = data;
3226         struct igc_q_vector *q_vector = adapter->q_vector[0];
3227         struct igc_hw *hw = &adapter->hw;
3228         /* read ICR disables interrupts using IAM */
3229         u32 icr = rd32(IGC_ICR);
3230 
3231         igc_write_itr(q_vector);
3232 
3233         if (icr & IGC_ICR_DRSTA)
3234                 schedule_work(&adapter->reset_task);
3235 
3236         if (icr & IGC_ICR_DOUTSYNC) {
3237                 /* HW is reporting DMA is out of sync */
3238                 adapter->stats.doosync++;
3239         }
3240 
3241         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
3242                 hw->mac.get_link_status = 1;
3243                 if (!test_bit(__IGC_DOWN, &adapter->state))
3244                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
3245         }
3246 
3247         napi_schedule(&q_vector->napi);
3248 
3249         return IRQ_HANDLED;
3250 }
3251 
3252 /**
3253  * igc_intr - Legacy Interrupt Handler
3254  * @irq: interrupt number
3255  * @data: pointer to a network interface device structure
3256  */
3257 static irqreturn_t igc_intr(int irq, void *data)
3258 {
3259         struct igc_adapter *adapter = data;
3260         struct igc_q_vector *q_vector = adapter->q_vector[0];
3261         struct igc_hw *hw = &adapter->hw;
3262         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
3263          * need for the IMC write
3264          */
3265         u32 icr = rd32(IGC_ICR);
3266 
3267         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
3268          * not set, then the adapter didn't send an interrupt
3269          */
3270         if (!(icr & IGC_ICR_INT_ASSERTED))
3271                 return IRQ_NONE;
3272 
3273         igc_write_itr(q_vector);
3274 
3275         if (icr & IGC_ICR_DRSTA)
3276                 schedule_work(&adapter->reset_task);
3277 
3278         if (icr & IGC_ICR_DOUTSYNC) {
3279                 /* HW is reporting DMA is out of sync */
3280                 adapter->stats.doosync++;
3281         }
3282 
3283         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
3284                 hw->mac.get_link_status = 1;
3285                 /* guard against interrupt when we're going down */
3286                 if (!test_bit(__IGC_DOWN, &adapter->state))
3287                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
3288         }
3289 
3290         napi_schedule(&q_vector->napi);
3291 
3292         return IRQ_HANDLED;
3293 }
3294 
3295 static void igc_set_itr(struct igc_q_vector *q_vector)
3296 {
3297         struct igc_adapter *adapter = q_vector->adapter;
3298         u32 new_itr = q_vector->itr_val;
3299         u8 current_itr = 0;
3300 
3301         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3302         switch (adapter->link_speed) {
3303         case SPEED_10:
3304         case SPEED_100:
3305                 current_itr = 0;
3306                 new_itr = IGC_4K_ITR;
3307                 goto set_itr_now;
3308         default:
3309                 break;
3310         }
3311 
3312         igc_update_itr(q_vector, &q_vector->tx);
3313         igc_update_itr(q_vector, &q_vector->rx);
3314 
3315         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
3316 
3317         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3318         if (current_itr == lowest_latency &&
3319             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3320             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3321                 current_itr = low_latency;
3322 
3323         switch (current_itr) {
3324         /* counts and packets in update_itr are dependent on these numbers */
3325         case lowest_latency:
3326                 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
3327                 break;
3328         case low_latency:
3329                 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
3330                 break;
3331         case bulk_latency:
3332                 new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
3333                 break;
3334         default:
3335                 break;
3336         }
3337 
3338 set_itr_now:
3339         if (new_itr != q_vector->itr_val) {
3340                 /* this attempts to bias the interrupt rate towards Bulk
3341                  * by adding intermediate steps when interrupt rate is
3342                  * increasing
3343                  */
3344                 new_itr = new_itr > q_vector->itr_val ?
3345                           max((new_itr * q_vector->itr_val) /
3346                           (new_itr + (q_vector->itr_val >> 2)),
3347                           new_itr) : new_itr;
3348                 /* Don't write the value here; it resets the adapter's
3349                  * internal timer, and causes us to delay far longer than
3350                  * we should between interrupts.  Instead, we write the ITR
3351                  * value at the beginning of the next interrupt so the timing
3352                  * ends up being correct.
3353                  */
3354                 q_vector->itr_val = new_itr;
3355                 q_vector->set_itr = 1;
3356         }
3357 }
3358 
3359 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
3360 {
3361         struct igc_adapter *adapter = q_vector->adapter;
3362         struct igc_hw *hw = &adapter->hw;
3363 
3364         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
3365             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
3366                 if (adapter->num_q_vectors == 1)
3367                         igc_set_itr(q_vector);
3368                 else
3369                         igc_update_ring_itr(q_vector);
3370         }
3371 
3372         if (!test_bit(__IGC_DOWN, &adapter->state)) {
3373                 if (adapter->msix_entries)
3374                         wr32(IGC_EIMS, q_vector->eims_value);
3375                 else
3376                         igc_irq_enable(adapter);
3377         }
3378 }
3379 
3380 /**
3381  * igc_poll - NAPI Rx polling callback
3382  * @napi: napi polling structure
3383  * @budget: count of how many packets we should handle
3384  */
3385 static int igc_poll(struct napi_struct *napi, int budget)
3386 {
3387         struct igc_q_vector *q_vector = container_of(napi,
3388                                                      struct igc_q_vector,
3389                                                      napi);
3390         bool clean_complete = true;
3391         int work_done = 0;
3392 
3393         if (q_vector->tx.ring)
3394                 clean_complete = igc_clean_tx_irq(q_vector, budget);
3395 
3396         if (q_vector->rx.ring) {
3397                 int cleaned = igc_clean_rx_irq(q_vector, budget);
3398 
3399                 work_done += cleaned;
3400                 if (cleaned >= budget)
3401                         clean_complete = false;
3402         }
3403 
3404         /* If all work not completed, return budget and keep polling */
3405         if (!clean_complete)
3406                 return budget;
3407 
3408         /* Exit the polling mode, but don't re-enable interrupts if stack might
3409          * poll us due to busy-polling
3410          */
3411         if (likely(napi_complete_done(napi, work_done)))
3412                 igc_ring_irq_enable(q_vector);
3413 
3414         return min(work_done, budget - 1);
3415 }
3416 
3417 /**
3418  * igc_set_interrupt_capability - set MSI or MSI-X if supported
3419  * @adapter: Pointer to adapter structure
3420  *
3421  * Attempt to configure interrupts using the best available
3422  * capabilities of the hardware and kernel.
3423  */
3424 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
3425                                          bool msix)
3426 {
3427         int numvecs, i;
3428         int err;
3429 
3430         if (!msix)
3431                 goto msi_only;
3432         adapter->flags |= IGC_FLAG_HAS_MSIX;
3433 
3434         /* Number of supported queues. */
3435         adapter->num_rx_queues = adapter->rss_queues;
3436 
3437         adapter->num_tx_queues = adapter->rss_queues;
3438 
3439         /* start with one vector for every Rx queue */
3440         numvecs = adapter->num_rx_queues;
3441 
3442         /* if Tx handler is separate add 1 for every Tx queue */
3443         if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
3444                 numvecs += adapter->num_tx_queues;
3445 
3446         /* store the number of vectors reserved for queues */
3447         adapter->num_q_vectors = numvecs;
3448 
3449         /* add 1 vector for link status interrupts */
3450         numvecs++;
3451 
3452         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
3453                                         GFP_KERNEL);
3454 
3455         if (!adapter->msix_entries)
3456                 return;
3457 
3458         /* populate entry values */
3459         for (i = 0; i < numvecs; i++)
3460                 adapter->msix_entries[i].entry = i;
3461 
3462         err = pci_enable_msix_range(adapter->pdev,
3463                                     adapter->msix_entries,
3464                                     numvecs,
3465                                     numvecs);
3466         if (err > 0)
3467                 return;
3468 
3469         kfree(adapter->msix_entries);
3470         adapter->msix_entries = NULL;
3471 
3472         igc_reset_interrupt_capability(adapter);
3473 
3474 msi_only:
3475         adapter->flags &= ~IGC_FLAG_HAS_MSIX;
3476 
3477         adapter->rss_queues = 1;
3478         adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
3479         adapter->num_rx_queues = 1;
3480         adapter->num_tx_queues = 1;
3481         adapter->num_q_vectors = 1;
3482         if (!pci_enable_msi(adapter->pdev))
3483                 adapter->flags |= IGC_FLAG_HAS_MSI;
3484 }
3485 
3486 static void igc_add_ring(struct igc_ring *ring,
3487                          struct igc_ring_container *head)
3488 {
3489         head->ring = ring;
3490         head->count++;
3491 }
3492 
3493 /**
3494  * igc_alloc_q_vector - Allocate memory for a single interrupt vector
3495  * @adapter: board private structure to initialize
3496  * @v_count: q_vectors allocated on adapter, used for ring interleaving
3497  * @v_idx: index of vector in adapter struct
3498  * @txr_count: total number of Tx rings to allocate
3499  * @txr_idx: index of first Tx ring to allocate
3500  * @rxr_count: total number of Rx rings to allocate
3501  * @rxr_idx: index of first Rx ring to allocate
3502  *
3503  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
3504  */
3505 static int igc_alloc_q_vector(struct igc_adapter *adapter,
3506                               unsigned int v_count, unsigned int v_idx,
3507                               unsigned int txr_count, unsigned int txr_idx,
3508                               unsigned int rxr_count, unsigned int rxr_idx)
3509 {
3510         struct igc_q_vector *q_vector;
3511         struct igc_ring *ring;
3512         int ring_count;
3513 
3514         /* igc only supports 1 Tx and/or 1 Rx queue per vector */
3515         if (txr_count > 1 || rxr_count > 1)
3516                 return -ENOMEM;
3517 
3518         ring_count = txr_count + rxr_count;
3519 
3520         /* allocate q_vector and rings */
3521         q_vector = adapter->q_vector[v_idx];
3522         if (!q_vector)
3523                 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
3524                                    GFP_KERNEL);
3525         else
3526                 memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
3527         if (!q_vector)
3528                 return -ENOMEM;
3529 
3530         /* initialize NAPI */
3531         netif_napi_add(adapter->netdev, &q_vector->napi,
3532                        igc_poll, 64);
3533 
3534         /* tie q_vector and adapter together */
3535         adapter->q_vector[v_idx] = q_vector;
3536         q_vector->adapter = adapter;
3537 
3538         /* initialize work limits */
3539         q_vector->tx.work_limit = adapter->tx_work_limit;
3540 
3541         /* initialize ITR configuration */
3542         q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
3543         q_vector->itr_val = IGC_START_ITR;
3544 
3545         /* initialize pointer to rings */
3546         ring = q_vector->ring;
3547 
3548         /* initialize ITR */
3549         if (rxr_count) {
3550                 /* rx or rx/tx vector */
3551                 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
3552                         q_vector->itr_val = adapter->rx_itr_setting;
3553         } else {
3554                 /* tx only vector */
3555                 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
3556                         q_vector->itr_val = adapter->tx_itr_setting;
3557         }
3558 
3559         if (txr_count) {
3560                 /* assign generic ring traits */
3561                 ring->dev = &adapter->pdev->dev;
3562                 ring->netdev = adapter->netdev;
3563 
3564                 /* configure backlink on ring */
3565                 ring->q_vector = q_vector;
3566 
3567                 /* update q_vector Tx values */
3568                 igc_add_ring(ring, &q_vector->tx);
3569 
3570                 /* apply Tx specific ring traits */
3571                 ring->count = adapter->tx_ring_count;
3572                 ring->queue_index = txr_idx;
3573 
3574                 /* assign ring to adapter */
3575                 adapter->tx_ring[txr_idx] = ring;
3576 
3577                 /* push pointer to next ring */
3578                 ring++;
3579         }
3580 
3581         if (rxr_count) {
3582                 /* assign generic ring traits */
3583                 ring->dev = &adapter->pdev->dev;
3584                 ring->netdev = adapter->netdev;
3585 
3586                 /* configure backlink on ring */
3587                 ring->q_vector = q_vector;
3588 
3589                 /* update q_vector Rx values */
3590                 igc_add_ring(ring, &q_vector->rx);
3591 
3592                 /* apply Rx specific ring traits */
3593                 ring->count = adapter->rx_ring_count;
3594                 ring->queue_index = rxr_idx;
3595 
3596                 /* assign ring to adapter */
3597                 adapter->rx_ring[rxr_idx] = ring;
3598         }
3599 
3600         return 0;
3601 }
3602 
3603 /**
3604  * igc_alloc_q_vectors - Allocate memory for interrupt vectors
3605  * @adapter: board private structure to initialize
3606  *
3607  * We allocate one q_vector per queue interrupt.  If allocation fails we
3608  * return -ENOMEM.
3609  */
3610 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
3611 {
3612         int rxr_remaining = adapter->num_rx_queues;
3613         int txr_remaining = adapter->num_tx_queues;
3614         int rxr_idx = 0, txr_idx = 0, v_idx = 0;
3615         int q_vectors = adapter->num_q_vectors;
3616         int err;
3617 
3618         if (q_vectors >= (rxr_remaining + txr_remaining)) {
3619                 for (; rxr_remaining; v_idx++) {
3620                         err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3621                                                  0, 0, 1, rxr_idx);
3622 
3623                         if (err)
3624                                 goto err_out;
3625 
3626                         /* update counts and index */
3627                         rxr_remaining--;
3628                         rxr_idx++;
3629                 }
3630         }
3631 
3632         for (; v_idx < q_vectors; v_idx++) {
3633                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
3634                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
3635 
3636                 err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3637                                          tqpv, txr_idx, rqpv, rxr_idx);
3638 
3639                 if (err)
3640                         goto err_out;
3641 
3642                 /* update counts and index */
3643                 rxr_remaining -= rqpv;
3644                 txr_remaining -= tqpv;
3645                 rxr_idx++;
3646                 txr_idx++;
3647         }
3648 
3649         return 0;
3650 
3651 err_out:
3652         adapter->num_tx_queues = 0;
3653         adapter->num_rx_queues = 0;
3654         adapter->num_q_vectors = 0;
3655 
3656         while (v_idx--)
3657                 igc_free_q_vector(adapter, v_idx);
3658 
3659         return -ENOMEM;
3660 }
3661 
3662 /**
3663  * igc_cache_ring_register - Descriptor ring to register mapping
3664  * @adapter: board private structure to initialize
3665  *
3666  * Once we know the feature-set enabled for the device, we'll cache
3667  * the register offset the descriptor ring is assigned to.
3668  */
3669 static void igc_cache_ring_register(struct igc_adapter *adapter)
3670 {
3671         int i = 0, j = 0;
3672 
3673         switch (adapter->hw.mac.type) {
3674         case igc_i225:
3675         /* Fall through */
3676         default:
3677                 for (; i < adapter->num_rx_queues; i++)
3678                         adapter->rx_ring[i]->reg_idx = i;
3679                 for (; j < adapter->num_tx_queues; j++)
3680                         adapter->tx_ring[j]->reg_idx = j;
3681                 break;
3682         }
3683 }
3684 
3685 /**
3686  * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
3687  * @adapter: Pointer to adapter structure
3688  *
3689  * This function initializes the interrupts and allocates all of the queues.
3690  */
3691 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
3692 {
3693         struct pci_dev *pdev = adapter->pdev;
3694         int err = 0;
3695 
3696         igc_set_interrupt_capability(adapter, msix);
3697 
3698         err = igc_alloc_q_vectors(adapter);
3699         if (err) {
3700                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
3701                 goto err_alloc_q_vectors;
3702         }
3703 
3704         igc_cache_ring_register(adapter);
3705 
3706         return 0;
3707 
3708 err_alloc_q_vectors:
3709         igc_reset_interrupt_capability(adapter);
3710         return err;
3711 }
3712 
3713 static void igc_free_irq(struct igc_adapter *adapter)
3714 {
3715         if (adapter->msix_entries) {
3716                 int vector = 0, i;
3717 
3718                 free_irq(adapter->msix_entries[vector++].vector, adapter);
3719 
3720                 for (i = 0; i < adapter->num_q_vectors; i++)
3721                         free_irq(adapter->msix_entries[vector++].vector,
3722                                  adapter->q_vector[i]);
3723         } else {
3724                 free_irq(adapter->pdev->irq, adapter);
3725         }
3726 }
3727 
3728 /**
3729  * igc_irq_disable - Mask off interrupt generation on the NIC
3730  * @adapter: board private structure
3731  */
3732 static void igc_irq_disable(struct igc_adapter *adapter)
3733 {
3734         struct igc_hw *hw = &adapter->hw;
3735 
3736         if (adapter->msix_entries) {
3737                 u32 regval = rd32(IGC_EIAM);
3738 
3739                 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
3740                 wr32(IGC_EIMC, adapter->eims_enable_mask);
3741                 regval = rd32(IGC_EIAC);
3742                 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
3743         }
3744 
3745         wr32(IGC_IAM, 0);
3746         wr32(IGC_IMC, ~0);
3747         wrfl();
3748 
3749         if (adapter->msix_entries) {
3750                 int vector = 0, i;
3751 
3752                 synchronize_irq(adapter->msix_entries[vector++].vector);
3753 
3754                 for (i = 0; i < adapter->num_q_vectors; i++)
3755                         synchronize_irq(adapter->msix_entries[vector++].vector);
3756         } else {
3757                 synchronize_irq(adapter->pdev->irq);
3758         }
3759 }
3760 
3761 /**
3762  * igc_irq_enable - Enable default interrupt generation settings
3763  * @adapter: board private structure
3764  */
3765 static void igc_irq_enable(struct igc_adapter *adapter)
3766 {
3767         struct igc_hw *hw = &adapter->hw;
3768 
3769         if (adapter->msix_entries) {
3770                 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
3771                 u32 regval = rd32(IGC_EIAC);
3772 
3773                 wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
3774                 regval = rd32(IGC_EIAM);
3775                 wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
3776                 wr32(IGC_EIMS, adapter->eims_enable_mask);
3777                 wr32(IGC_IMS, ims);
3778         } else {
3779                 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3780                 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3781         }
3782 }
3783 
3784 /**
3785  * igc_request_irq - initialize interrupts
3786  * @adapter: Pointer to adapter structure
3787  *
3788  * Attempts to configure interrupts using the best available
3789  * capabilities of the hardware and kernel.
3790  */
3791 static int igc_request_irq(struct igc_adapter *adapter)
3792 {
3793         struct net_device *netdev = adapter->netdev;
3794         struct pci_dev *pdev = adapter->pdev;
3795         int err = 0;
3796 
3797         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
3798                 err = igc_request_msix(adapter);
3799                 if (!err)
3800                         goto request_done;
3801                 /* fall back to MSI */
3802                 igc_free_all_tx_resources(adapter);
3803                 igc_free_all_rx_resources(adapter);
3804 
3805                 igc_clear_interrupt_scheme(adapter);
3806                 err = igc_init_interrupt_scheme(adapter, false);
3807                 if (err)
3808                         goto request_done;
3809                 igc_setup_all_tx_resources(adapter);
3810                 igc_setup_all_rx_resources(adapter);
3811                 igc_configure(adapter);
3812         }
3813 
3814         igc_assign_vector(adapter->q_vector[0], 0);
3815 
3816         if (adapter->flags & IGC_FLAG_HAS_MSI) {
3817                 err = request_irq(pdev->irq, &igc_intr_msi, 0,
3818                                   netdev->name, adapter);
3819                 if (!err)
3820                         goto request_done;
3821 
3822                 /* fall back to legacy interrupts */
3823                 igc_reset_interrupt_capability(adapter);
3824                 adapter->flags &= ~IGC_FLAG_HAS_MSI;
3825         }
3826 
3827         err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
3828                           netdev->name, adapter);
3829 
3830         if (err)
3831                 dev_err(&pdev->dev, "Error %d getting interrupt\n",
3832                         err);
3833 
3834 request_done:
3835         return err;
3836 }
3837 
3838 static void igc_write_itr(struct igc_q_vector *q_vector)
3839 {
3840         u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
3841 
3842         if (!q_vector->set_itr)
3843                 return;
3844 
3845         if (!itr_val)
3846                 itr_val = IGC_ITR_VAL_MASK;
3847 
3848         itr_val |= IGC_EITR_CNT_IGNR;
3849 
3850         writel(itr_val, q_vector->itr_register);
3851         q_vector->set_itr = 0;
3852 }
3853 
3854 /**
3855  * igc_open - Called when a network interface is made active
3856  * @netdev: network interface device structure
3857  *
3858  * Returns 0 on success, negative value on failure
3859  *
3860  * The open entry point is called when a network interface is made
3861  * active by the system (IFF_UP).  At this point all resources needed
3862  * for transmit and receive operations are allocated, the interrupt
3863  * handler is registered with the OS, the watchdog timer is started,
3864  * and the stack is notified that the interface is ready.
3865  */
3866 static int __igc_open(struct net_device *netdev, bool resuming)
3867 {
3868         struct igc_adapter *adapter = netdev_priv(netdev);
3869         struct igc_hw *hw = &adapter->hw;
3870         int err = 0;
3871         int i = 0;
3872 
3873         /* disallow open during test */
3874 
3875         if (test_bit(__IGC_TESTING, &adapter->state)) {
3876                 WARN_ON(resuming);
3877                 return -EBUSY;
3878         }
3879 
3880         netif_carrier_off(netdev);
3881 
3882         /* allocate transmit descriptors */
3883         err = igc_setup_all_tx_resources(adapter);
3884         if (err)
3885                 goto err_setup_tx;
3886 
3887         /* allocate receive descriptors */
3888         err = igc_setup_all_rx_resources(adapter);
3889         if (err)
3890                 goto err_setup_rx;
3891 
3892         igc_power_up_link(adapter);
3893 
3894         igc_configure(adapter);
3895 
3896         err = igc_request_irq(adapter);
3897         if (err)
3898                 goto err_req_irq;
3899 
3900         /* Notify the stack of the actual queue counts. */
3901         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
3902         if (err)
3903                 goto err_set_queues;
3904 
3905         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
3906         if (err)
3907                 goto err_set_queues;
3908 
3909         clear_bit(__IGC_DOWN, &adapter->state);
3910 
3911         for (i = 0; i < adapter->num_q_vectors; i++)
3912                 napi_enable(&adapter->q_vector[i]->napi);
3913 
3914         /* Clear any pending interrupts. */
3915         rd32(IGC_ICR);
3916         igc_irq_enable(adapter);
3917 
3918         netif_tx_start_all_queues(netdev);
3919 
3920         /* start the watchdog. */
3921         hw->mac.get_link_status = 1;
3922         schedule_work(&adapter->watchdog_task);
3923 
3924         return IGC_SUCCESS;
3925 
3926 err_set_queues:
3927         igc_free_irq(adapter);
3928 err_req_irq:
3929         igc_release_hw_control(adapter);
3930         igc_power_down_link(adapter);
3931         igc_free_all_rx_resources(adapter);
3932 err_setup_rx:
3933         igc_free_all_tx_resources(adapter);
3934 err_setup_tx:
3935         igc_reset(adapter);
3936 
3937         return err;
3938 }
3939 
3940 static int igc_open(struct net_device *netdev)
3941 {
3942         return __igc_open(netdev, false);
3943 }
3944 
3945 /**
3946  * igc_close - Disables a network interface
3947  * @netdev: network interface device structure
3948  *
3949  * Returns 0, this is not allowed to fail
3950  *
3951  * The close entry point is called when an interface is de-activated
3952  * by the OS.  The hardware is still under the driver's control, but
3953  * needs to be disabled.  A global MAC reset is issued to stop the
3954  * hardware, and all transmit and receive resources are freed.
3955  */
3956 static int __igc_close(struct net_device *netdev, bool suspending)
3957 {
3958         struct igc_adapter *adapter = netdev_priv(netdev);
3959 
3960         WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
3961 
3962         igc_down(adapter);
3963 
3964         igc_release_hw_control(adapter);
3965 
3966         igc_free_irq(adapter);
3967 
3968         igc_free_all_tx_resources(adapter);
3969         igc_free_all_rx_resources(adapter);
3970 
3971         return 0;
3972 }
3973 
3974 static int igc_close(struct net_device *netdev)
3975 {
3976         if (netif_device_present(netdev) || netdev->dismantle)
3977                 return __igc_close(netdev, false);
3978         return 0;
3979 }
3980 
3981 static const struct net_device_ops igc_netdev_ops = {
3982         .ndo_open               = igc_open,
3983         .ndo_stop               = igc_close,
3984         .ndo_start_xmit         = igc_xmit_frame,
3985         .ndo_set_mac_address    = igc_set_mac,
3986         .ndo_change_mtu         = igc_change_mtu,
3987         .ndo_get_stats          = igc_get_stats,
3988         .ndo_fix_features       = igc_fix_features,
3989         .ndo_set_features       = igc_set_features,
3990         .ndo_features_check     = igc_features_check,
3991 };
3992 
3993 /* PCIe configuration access */
3994 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
3995 {
3996         struct igc_adapter *adapter = hw->back;
3997 
3998         pci_read_config_word(adapter->pdev, reg, value);
3999 }
4000 
4001 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
4002 {
4003         struct igc_adapter *adapter = hw->back;
4004 
4005         pci_write_config_word(adapter->pdev, reg, *value);
4006 }
4007 
4008 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
4009 {
4010         struct igc_adapter *adapter = hw->back;
4011 
4012         if (!pci_is_pcie(adapter->pdev))
4013                 return -IGC_ERR_CONFIG;
4014 
4015         pcie_capability_read_word(adapter->pdev, reg, value);
4016 
4017         return IGC_SUCCESS;
4018 }
4019 
4020 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
4021 {
4022         struct igc_adapter *adapter = hw->back;
4023 
4024         if (!pci_is_pcie(adapter->pdev))
4025                 return -IGC_ERR_CONFIG;
4026 
4027         pcie_capability_write_word(adapter->pdev, reg, *value);
4028 
4029         return IGC_SUCCESS;
4030 }
4031 
4032 u32 igc_rd32(struct igc_hw *hw, u32 reg)
4033 {
4034         struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
4035         u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
4036         u32 value = 0;
4037 
4038         if (IGC_REMOVED(hw_addr))
4039                 return ~value;
4040 
4041         value = readl(&hw_addr[reg]);
4042 
4043         /* reads should not return all F's */
4044         if (!(~value) && (!reg || !(~readl(hw_addr)))) {
4045                 struct net_device *netdev = igc->netdev;
4046 
4047                 hw->hw_addr = NULL;
4048                 netif_device_detach(netdev);
4049                 netdev_err(netdev, "PCIe link lost, device now detached\n");
4050                 WARN(pci_device_is_present(igc->pdev),
4051                      "igc: Failed to read reg 0x%x!\n", reg);
4052         }
4053 
4054         return value;
4055 }
4056 
4057 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx)
4058 {
4059         struct pci_dev *pdev = adapter->pdev;
4060         struct igc_mac_info *mac = &adapter->hw.mac;
4061 
4062         mac->autoneg = 0;
4063 
4064         /* Make sure dplx is at most 1 bit and lsb of speed is not set
4065          * for the switch() below to work
4066          */
4067         if ((spd & 1) || (dplx & ~1))
4068                 goto err_inval;
4069 
4070         switch (spd + dplx) {
4071         case SPEED_10 + DUPLEX_HALF:
4072                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
4073                 break;
4074         case SPEED_10 + DUPLEX_FULL:
4075                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
4076                 break;
4077         case SPEED_100 + DUPLEX_HALF:
4078                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
4079                 break;
4080         case SPEED_100 + DUPLEX_FULL:
4081                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
4082                 break;
4083         case SPEED_1000 + DUPLEX_FULL:
4084                 mac->autoneg = 1;
4085                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
4086                 break;
4087         case SPEED_1000 + DUPLEX_HALF: /* not supported */
4088                 goto err_inval;
4089         case SPEED_2500 + DUPLEX_FULL:
4090                 mac->autoneg = 1;
4091                 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL;
4092                 break;
4093         case SPEED_2500 + DUPLEX_HALF: /* not supported */
4094         default:
4095                 goto err_inval;
4096         }
4097 
4098         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
4099         adapter->hw.phy.mdix = AUTO_ALL_MODES;
4100 
4101         return 0;
4102 
4103 err_inval:
4104         dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
4105         return -EINVAL;
4106 }
4107 
4108 /**
4109  * igc_probe - Device Initialization Routine
4110  * @pdev: PCI device information struct
4111  * @ent: entry in igc_pci_tbl
4112  *
4113  * Returns 0 on success, negative on failure
4114  *
4115  * igc_probe initializes an adapter identified by a pci_dev structure.
4116  * The OS initialization, configuring the adapter private structure,
4117  * and a hardware reset occur.
4118  */
4119 static int igc_probe(struct pci_dev *pdev,
4120                      const struct pci_device_id *ent)
4121 {
4122         struct igc_adapter *adapter;
4123         struct net_device *netdev;
4124         struct igc_hw *hw;
4125         const struct igc_info *ei = igc_info_tbl[ent->driver_data];
4126         int err;
4127 
4128         err = pci_enable_device_mem(pdev);
4129         if (err)
4130                 return err;
4131 
4132         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
4133         if (!err) {
4134                 err = dma_set_coherent_mask(&pdev->dev,
4135                                             DMA_BIT_MASK(64));
4136         } else {
4137                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
4138                 if (err) {
4139                         err = dma_set_coherent_mask(&pdev->dev,
4140                                                     DMA_BIT_MASK(32));
4141                         if (err) {
4142                                 dev_err(&pdev->dev, "igc: Wrong DMA config\n");
4143                                 goto err_dma;
4144                         }
4145                 }
4146         }
4147 
4148         err = pci_request_selected_regions(pdev,
4149                                            pci_select_bars(pdev,
4150                                                            IORESOURCE_MEM),
4151                                            igc_driver_name);
4152         if (err)
4153                 goto err_pci_reg;
4154 
4155         pci_enable_pcie_error_reporting(pdev);
4156 
4157         pci_set_master(pdev);
4158 
4159         err = -ENOMEM;
4160         netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
4161                                    IGC_MAX_TX_QUEUES);
4162 
4163         if (!netdev)
4164                 goto err_alloc_etherdev;
4165 
4166         SET_NETDEV_DEV(netdev, &pdev->dev);
4167 
4168         pci_set_drvdata(pdev, netdev);
4169         adapter = netdev_priv(netdev);
4170         adapter->netdev = netdev;
4171         adapter->pdev = pdev;
4172         hw = &adapter->hw;
4173         hw->back = adapter;
4174         adapter->port_num = hw->bus.func;
4175         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
4176 
4177         err = pci_save_state(pdev);
4178         if (err)
4179                 goto err_ioremap;
4180 
4181         err = -EIO;
4182         adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
4183                                    pci_resource_len(pdev, 0));
4184         if (!adapter->io_addr)
4185                 goto err_ioremap;
4186 
4187         /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
4188         hw->hw_addr = adapter->io_addr;
4189 
4190         netdev->netdev_ops = &igc_netdev_ops;
4191         igc_set_ethtool_ops(netdev);
4192         netdev->watchdog_timeo = 5 * HZ;
4193 
4194         netdev->mem_start = pci_resource_start(pdev, 0);
4195         netdev->mem_end = pci_resource_end(pdev, 0);
4196 
4197         /* PCI config space info */
4198         hw->vendor_id = pdev->vendor;
4199         hw->device_id = pdev->device;
4200         hw->revision_id = pdev->revision;
4201         hw->subsystem_vendor_id = pdev->subsystem_vendor;
4202         hw->subsystem_device_id = pdev->subsystem_device;
4203 
4204         /* Copy the default MAC and PHY function pointers */
4205         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
4206         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
4207 
4208         /* Initialize skew-specific constants */
4209         err = ei->get_invariants(hw);
4210         if (err)
4211                 goto err_sw_init;
4212 
4213         /* Add supported features to the features list*/
4214         netdev->features |= NETIF_F_HW_CSUM;
4215 
4216         /* setup the private structure */
4217         err = igc_sw_init(adapter);
4218         if (err)
4219                 goto err_sw_init;
4220 
4221         /* copy netdev features into list of user selectable features */
4222         netdev->hw_features |= NETIF_F_NTUPLE;
4223         netdev->hw_features |= netdev->features;
4224 
4225         /* MTU range: 68 - 9216 */
4226         netdev->min_mtu = ETH_MIN_MTU;
4227         netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
4228 
4229         /* before reading the NVM, reset the controller to put the device in a
4230          * known good starting state
4231          */
4232         hw->mac.ops.reset_hw(hw);
4233 
4234         if (igc_get_flash_presence_i225(hw)) {
4235                 if (hw->nvm.ops.validate(hw) < 0) {
4236                         dev_err(&pdev->dev,
4237                                 "The NVM Checksum Is Not Valid\n");
4238                         err = -EIO;
4239                         goto err_eeprom;
4240                 }
4241         }
4242 
4243         if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
4244                 /* copy the MAC address out of the NVM */
4245                 if (hw->mac.ops.read_mac_addr(hw))
4246                         dev_err(&pdev->dev, "NVM Read Error\n");
4247         }
4248 
4249         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
4250 
4251         if (!is_valid_ether_addr(netdev->dev_addr)) {
4252                 dev_err(&pdev->dev, "Invalid MAC Address\n");
4253                 err = -EIO;
4254                 goto err_eeprom;
4255         }
4256 
4257         /* configure RXPBSIZE and TXPBSIZE */
4258         wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
4259         wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
4260 
4261         timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
4262         timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
4263 
4264         INIT_WORK(&adapter->reset_task, igc_reset_task);
4265         INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
4266 
4267         /* Initialize link properties that are user-changeable */
4268         adapter->fc_autoneg = true;
4269         hw->mac.autoneg = true;
4270         hw->phy.autoneg_advertised = 0xaf;
4271 
4272         hw->fc.requested_mode = igc_fc_default;
4273         hw->fc.current_mode = igc_fc_default;
4274 
4275         /* reset the hardware with the new settings */
4276         igc_reset(adapter);
4277 
4278         /* let the f/w know that the h/w is now under the control of the
4279          * driver.
4280          */
4281         igc_get_hw_control(adapter);
4282 
4283         strncpy(netdev->name, "eth%d", IFNAMSIZ);
4284         err = register_netdev(netdev);
4285         if (err)
4286                 goto err_register;
4287 
4288          /* carrier off reporting is important to ethtool even BEFORE open */
4289         netif_carrier_off(netdev);
4290 
4291         /* Check if Media Autosense is enabled */
4292         adapter->ei = *ei;
4293 
4294         /* print pcie link status and MAC address */
4295         pcie_print_link_status(pdev);
4296         netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
4297 
4298         return 0;
4299 
4300 err_register:
4301         igc_release_hw_control(adapter);
4302 err_eeprom:
4303         if (!igc_check_reset_block(hw))
4304                 igc_reset_phy(hw);
4305 err_sw_init:
4306         igc_clear_interrupt_scheme(adapter);
4307         iounmap(adapter->io_addr);
4308 err_ioremap:
4309         free_netdev(netdev);
4310 err_alloc_etherdev:
4311         pci_release_selected_regions(pdev,
4312                                      pci_select_bars(pdev, IORESOURCE_MEM));
4313 err_pci_reg:
4314 err_dma:
4315         pci_disable_device(pdev);
4316         return err;
4317 }
4318 
4319 /**
4320  * igc_remove - Device Removal Routine
4321  * @pdev: PCI device information struct
4322  *
4323  * igc_remove is called by the PCI subsystem to alert the driver
4324  * that it should release a PCI device.  This could be caused by a
4325  * Hot-Plug event, or because the driver is going to be removed from
4326  * memory.
4327  */
4328 static void igc_remove(struct pci_dev *pdev)
4329 {
4330         struct net_device *netdev = pci_get_drvdata(pdev);
4331         struct igc_adapter *adapter = netdev_priv(netdev);
4332 
4333         set_bit(__IGC_DOWN, &adapter->state);
4334 
4335         del_timer_sync(&adapter->watchdog_timer);
4336         del_timer_sync(&adapter->phy_info_timer);
4337 
4338         cancel_work_sync(&adapter->reset_task);
4339         cancel_work_sync(&adapter->watchdog_task);
4340 
4341         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
4342          * would have already happened in close and is redundant.
4343          */
4344         igc_release_hw_control(adapter);
4345         unregister_netdev(netdev);
4346 
4347         igc_clear_interrupt_scheme(adapter);
4348         pci_iounmap(pdev, adapter->io_addr);
4349         pci_release_mem_regions(pdev);
4350 
4351         kfree(adapter->mac_table);
4352         kfree(adapter->shadow_vfta);
4353         free_netdev(netdev);
4354 
4355         pci_disable_pcie_error_reporting(pdev);
4356 
4357         pci_disable_device(pdev);
4358 }
4359 
4360 static struct pci_driver igc_driver = {
4361         .name     = igc_driver_name,
4362         .id_table = igc_pci_tbl,
4363         .probe    = igc_probe,
4364         .remove   = igc_remove,
4365 };
4366 
4367 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
4368                               const u32 max_rss_queues)
4369 {
4370         /* Determine if we need to pair queues. */
4371         /* If rss_queues > half of max_rss_queues, pair the queues in
4372          * order to conserve interrupts due to limited supply.
4373          */
4374         if (adapter->rss_queues > (max_rss_queues / 2))
4375                 adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4376         else
4377                 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
4378 }
4379 
4380 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
4381 {
4382         unsigned int max_rss_queues;
4383 
4384         /* Determine the maximum number of RSS queues supported. */
4385         max_rss_queues = IGC_MAX_RX_QUEUES;
4386 
4387         return max_rss_queues;
4388 }
4389 
4390 static void igc_init_queue_configuration(struct igc_adapter *adapter)
4391 {
4392         u32 max_rss_queues;
4393 
4394         max_rss_queues = igc_get_max_rss_queues(adapter);
4395         adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4396 
4397         igc_set_flag_queue_pairs(adapter, max_rss_queues);
4398 }
4399 
4400 /**
4401  * igc_sw_init - Initialize general software structures (struct igc_adapter)
4402  * @adapter: board private structure to initialize
4403  *
4404  * igc_sw_init initializes the Adapter private data structure.
4405  * Fields are initialized based on PCI device information and
4406  * OS network device settings (MTU size).
4407  */
4408 static int igc_sw_init(struct igc_adapter *adapter)
4409 {
4410         struct net_device *netdev = adapter->netdev;
4411         struct pci_dev *pdev = adapter->pdev;
4412         struct igc_hw *hw = &adapter->hw;
4413 
4414         int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count;
4415 
4416         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4417 
4418         /* set default ring sizes */
4419         adapter->tx_ring_count = IGC_DEFAULT_TXD;
4420         adapter->rx_ring_count = IGC_DEFAULT_RXD;
4421 
4422         /* set default ITR values */
4423         adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4424         adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4425 
4426         /* set default work limits */
4427         adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4428 
4429         /* adjust max frame to be at least the size of a standard frame */
4430         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4431                                 VLAN_HLEN;
4432         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4433 
4434         spin_lock_init(&adapter->nfc_lock);
4435         spin_lock_init(&adapter->stats64_lock);
4436         /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4437         adapter->flags |= IGC_FLAG_HAS_MSIX;
4438 
4439         adapter->mac_table = kzalloc(size, GFP_ATOMIC);
4440         if (!adapter->mac_table)
4441                 return -ENOMEM;
4442 
4443         igc_init_queue_configuration(adapter);
4444 
4445         /* This call may decrease the number of queues */
4446         if (igc_init_interrupt_scheme(adapter, true)) {
4447                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4448                 return -ENOMEM;
4449         }
4450 
4451         /* Explicitly disable IRQ since the NIC can be in any state. */
4452         igc_irq_disable(adapter);
4453 
4454         set_bit(__IGC_DOWN, &adapter->state);
4455 
4456         return 0;
4457 }
4458 
4459 /**
4460  * igc_reinit_queues - return error
4461  * @adapter: pointer to adapter structure
4462  */
4463 int igc_reinit_queues(struct igc_adapter *adapter)
4464 {
4465         struct net_device *netdev = adapter->netdev;
4466         struct pci_dev *pdev = adapter->pdev;
4467         int err = 0;
4468 
4469         if (netif_running(netdev))
4470                 igc_close(netdev);
4471 
4472         igc_reset_interrupt_capability(adapter);
4473 
4474         if (igc_init_interrupt_scheme(adapter, true)) {
4475                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4476                 return -ENOMEM;
4477         }
4478 
4479         if (netif_running(netdev))
4480                 err = igc_open(netdev);
4481 
4482         return err;
4483 }
4484 
4485 /**
4486  * igc_get_hw_dev - return device
4487  * @hw: pointer to hardware structure
4488  *
4489  * used by hardware layer to print debugging information
4490  */
4491 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
4492 {
4493         struct igc_adapter *adapter = hw->back;
4494 
4495         return adapter->netdev;
4496 }
4497 
4498 /**
4499  * igc_init_module - Driver Registration Routine
4500  *
4501  * igc_init_module is the first routine called when the driver is
4502  * loaded. All it does is register with the PCI subsystem.
4503  */
4504 static int __init igc_init_module(void)
4505 {
4506         int ret;
4507 
4508         pr_info("%s - version %s\n",
4509                 igc_driver_string, igc_driver_version);
4510 
4511         pr_info("%s\n", igc_copyright);
4512 
4513         ret = pci_register_driver(&igc_driver);
4514         return ret;
4515 }
4516 
4517 module_init(igc_init_module);
4518 
4519 /**
4520  * igc_exit_module - Driver Exit Cleanup Routine
4521  *
4522  * igc_exit_module is called just before the driver is removed
4523  * from memory.
4524  */
4525 static void __exit igc_exit_module(void)
4526 {
4527         pci_unregister_driver(&igc_driver);
4528 }
4529 
4530 module_exit(igc_exit_module);
4531 /* igc_main.c */
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/net/ethernet/intel/igc/igc_main.c

DEFINITIONS
