drivers/net/ethernet/cavium/liquidio/lio

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
octeon_console_debug_enabled
octeon_droq_bh
lio_wait_for_oq_pkts
force_io_queues_off
pcierror_quiesce_device
cleanup_aer_uncorrect_error_status
stop_pci_io
liquidio_pcie_error_detected
liquidio_pcie_mmio_enabled
liquidio_pcie_slot_reset
liquidio_pcie_resume
liquidio_suspend
liquidio_resume
liquidio_init_pci
liquidio_deinit_pci
check_txq_status
print_link_info
octnet_link_status_change
setup_link_status_change_wq
cleanup_link_status_change_wq
update_link_status
lio_sync_octeon_time
setup_sync_octeon_time_wq
cleanup_sync_octeon_time_wq
get_other_octeon_device
disable_all_vf_links
liquidio_watchdog
liquidio_probe
fw_type_is_auto
octeon_pci_flr
octeon_destroy_resources
send_rx_ctrl_cmd
liquidio_destroy_nic_device
liquidio_stop_nic_module
liquidio_remove
octeon_chip_specific_setup
octeon_pci_os_setup
free_netbuf
free_netsgbuf
free_netsgbuf_with_resp
liquidio_ptp_adjfreq
liquidio_ptp_adjtime
liquidio_ptp_gettime
liquidio_ptp_settime
liquidio_ptp_enable
oct_ptp_open
liquidio_ptp_init
load_firmware
octnet_poll_check_txq_status
setup_tx_poll_fn
cleanup_tx_poll_fn
liquidio_open
liquidio_stop
get_new_flags
liquidio_set_mcast_list
liquidio_set_mac
liquidio_get_stats64
hwtstamp_ioctl
liquidio_ioctl
handle_timestamp
send_nic_timestamp_pkt
liquidio_xmit
liquidio_tx_timeout
liquidio_vlan_rx_add_vid
liquidio_vlan_rx_kill_vid
liquidio_set_rxcsum_command
liquidio_vxlan_port_command
liquidio_fix_features
liquidio_set_features
liquidio_add_vxlan_port
liquidio_del_vxlan_port
__liquidio_set_vf_mac
liquidio_set_vf_mac
liquidio_set_vf_spoofchk
liquidio_set_vf_vlan
liquidio_get_vf_config
liquidio_send_vf_trust_cmd
liquidio_set_vf_trust
liquidio_set_vf_link_state
liquidio_eswitch_mode_get
liquidio_eswitch_mode_set
liquidio_get_port_parent_id
liquidio_get_vf_stats
liquidio_init
lio_nic_info
setup_nic_devices
octeon_enable_sriov
lio_pci_sriov_disable
liquidio_enable_sriov
liquidio_init_nic_module
nic_starter
octeon_recv_vf_drv_notice
octeon_device_init
octeon_dbg_console_print
liquidio_exit
   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/module.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/pci.h>
  21 #include <linux/firmware.h>
  22 #include <net/vxlan.h>
  23 #include <linux/kthread.h>
  24 #include "liquidio_common.h"
  25 #include "octeon_droq.h"
  26 #include "octeon_iq.h"
  27 #include "response_manager.h"
  28 #include "octeon_device.h"
  29 #include "octeon_nic.h"
  30 #include "octeon_main.h"
  31 #include "octeon_network.h"
  32 #include "cn66xx_regs.h"
  33 #include "cn66xx_device.h"
  34 #include "cn68xx_device.h"
  35 #include "cn23xx_pf_device.h"
  36 #include "liquidio_image.h"
  37 #include "lio_vf_rep.h"
  38 
  39 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  40 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  41 MODULE_LICENSE("GPL");
  42 MODULE_VERSION(LIQUIDIO_VERSION);
  43 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
  44                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
  46                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  47 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
  48                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  49 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
  50                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  51 
  52 static int ddr_timeout = 10000;
  53 module_param(ddr_timeout, int, 0644);
  54 MODULE_PARM_DESC(ddr_timeout,
  55                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  56 
  57 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  58 
  59 static int debug = -1;
  60 module_param(debug, int, 0644);
  61 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  62 
  63 static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;
  64 module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
  65 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");
  66 
  67 static u32 console_bitmask;
  68 module_param(console_bitmask, int, 0644);
  69 MODULE_PARM_DESC(console_bitmask,
  70                  "Bitmask indicating which consoles have debug output redirected to syslog.");
  71 
  72 /**
  73  * \brief determines if a given console has debug enabled.
  74  * @param console console to check
  75  * @returns  1 = enabled. 0 otherwise
  76  */
  77 static int octeon_console_debug_enabled(u32 console)
  78 {
  79         return (console_bitmask >> (console)) & 0x1;
  80 }
  81 
  82 /* Polling interval for determining when NIC application is alive */
  83 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  84 
  85 /* runtime link query interval */
  86 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  87 /* update localtime to octeon firmware every 60 seconds.
  88  * make firmware to use same time reference, so that it will be easy to
  89  * correlate firmware logged events/errors with host events, for debugging.
  90  */
  91 #define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 60000
  92 
  93 /* time to wait for possible in-flight requests in milliseconds */
  94 #define WAIT_INFLIGHT_REQUEST   msecs_to_jiffies(1000)
  95 
  96 struct lio_trusted_vf_ctx {
  97         struct completion complete;
  98         int status;
  99 };
 100 
 101 struct oct_link_status_resp {
 102         u64 rh;
 103         struct oct_link_info link_info;
 104         u64 status;
 105 };
 106 
 107 struct oct_timestamp_resp {
 108         u64 rh;
 109         u64 timestamp;
 110         u64 status;
 111 };
 112 
 113 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 114 
 115 union tx_info {
 116         u64 u64;
 117         struct {
 118 #ifdef __BIG_ENDIAN_BITFIELD
 119                 u16 gso_size;
 120                 u16 gso_segs;
 121                 u32 reserved;
 122 #else
 123                 u32 reserved;
 124                 u16 gso_segs;
 125                 u16 gso_size;
 126 #endif
 127         } s;
 128 };
 129 
 130 /** Octeon device properties to be used by the NIC module.
 131  * Each octeon device in the system will be represented
 132  * by this structure in the NIC module.
 133  */
 134 
 135 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 136 #define OCTNIC_GSO_MAX_SIZE                                                    \
 137         (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 138 
 139 struct handshake {
 140         struct completion init;
 141         struct completion started;
 142         struct pci_dev *pci_dev;
 143         int init_ok;
 144         int started_ok;
 145 };
 146 
 147 #ifdef CONFIG_PCI_IOV
 148 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
 149 #endif
 150 
 151 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
 152                                     char *prefix, char *suffix);
 153 
 154 static int octeon_device_init(struct octeon_device *);
 155 static int liquidio_stop(struct net_device *netdev);
 156 static void liquidio_remove(struct pci_dev *pdev);
 157 static int liquidio_probe(struct pci_dev *pdev,
 158                           const struct pci_device_id *ent);
 159 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
 160                                       int linkstate);
 161 
 162 static struct handshake handshake[MAX_OCTEON_DEVICES];
 163 static struct completion first_stage;
 164 
 165 static void octeon_droq_bh(unsigned long pdev)
 166 {
 167         int q_no;
 168         int reschedule = 0;
 169         struct octeon_device *oct = (struct octeon_device *)pdev;
 170         struct octeon_device_priv *oct_priv =
 171                 (struct octeon_device_priv *)oct->priv;
 172 
 173         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
 174                 if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
 175                         continue;
 176                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 177                                                           MAX_PACKET_BUDGET);
 178                 lio_enable_irq(oct->droq[q_no], NULL);
 179 
 180                 if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
 181                         /* set time and cnt interrupt thresholds for this DROQ
 182                          * for NAPI
 183                          */
 184                         int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
 185 
 186                         octeon_write_csr64(
 187                             oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
 188                             0x5700000040ULL);
 189                         octeon_write_csr64(
 190                             oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
 191                 }
 192         }
 193 
 194         if (reschedule)
 195                 tasklet_schedule(&oct_priv->droq_tasklet);
 196 }
 197 
 198 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 199 {
 200         struct octeon_device_priv *oct_priv =
 201                 (struct octeon_device_priv *)oct->priv;
 202         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 203         int i;
 204 
 205         do {
 206                 pending_pkts = 0;
 207 
 208                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
 209                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
 210                                 continue;
 211                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
 212                 }
 213                 if (pkt_cnt > 0) {
 214                         pending_pkts += pkt_cnt;
 215                         tasklet_schedule(&oct_priv->droq_tasklet);
 216                 }
 217                 pkt_cnt = 0;
 218                 schedule_timeout_uninterruptible(1);
 219 
 220         } while (retry-- && pending_pkts);
 221 
 222         return pkt_cnt;
 223 }
 224 
 225 /**
 226  * \brief Forces all IO queues off on a given device
 227  * @param oct Pointer to Octeon device
 228  */
 229 static void force_io_queues_off(struct octeon_device *oct)
 230 {
 231         if ((oct->chip_id == OCTEON_CN66XX) ||
 232             (oct->chip_id == OCTEON_CN68XX)) {
 233                 /* Reset the Enable bits for Input Queues. */
 234                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 235 
 236                 /* Reset the Enable bits for Output Queues. */
 237                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 238         }
 239 }
 240 
 241 /**
 242  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 243  * @param oct Pointer to Octeon device
 244  */
 245 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 246 {
 247         int i;
 248 
 249         /* Disable the input and output queues now. No more packets will
 250          * arrive from Octeon, but we should wait for all packet processing
 251          * to finish.
 252          */
 253         force_io_queues_off(oct);
 254 
 255         /* To allow for in-flight requests */
 256         schedule_timeout_uninterruptible(WAIT_INFLIGHT_REQUEST);
 257 
 258         if (wait_for_pending_requests(oct))
 259                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 260 
 261         /* Force all requests waiting to be fetched by OCTEON to complete. */
 262         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 263                 struct octeon_instr_queue *iq;
 264 
 265                 if (!(oct->io_qmask.iq & BIT_ULL(i)))
 266                         continue;
 267                 iq = oct->instr_queue[i];
 268 
 269                 if (atomic_read(&iq->instr_pending)) {
 270                         spin_lock_bh(&iq->lock);
 271                         iq->fill_cnt = 0;
 272                         iq->octeon_read_index = iq->host_write_index;
 273                         iq->stats.instr_processed +=
 274                                 atomic_read(&iq->instr_pending);
 275                         lio_process_iq_request_list(oct, iq, 0);
 276                         spin_unlock_bh(&iq->lock);
 277                 }
 278         }
 279 
 280         /* Force all pending ordered list requests to time out. */
 281         lio_process_ordered_list(oct, 1);
 282 
 283         /* We do not need to wait for output queue packets to be processed. */
 284 }
 285 
 286 /**
 287  * \brief Cleanup PCI AER uncorrectable error status
 288  * @param dev Pointer to PCI device
 289  */
 290 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 291 {
 292         int pos = 0x100;
 293         u32 status, mask;
 294 
 295         pr_info("%s :\n", __func__);
 296 
 297         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 298         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 299         if (dev->error_state == pci_channel_io_normal)
 300                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 301         else
 302                 status &= mask;         /* Clear corresponding fatal bits */
 303         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 304 }
 305 
 306 /**
 307  * \brief Stop all PCI IO to a given device
 308  * @param dev Pointer to Octeon device
 309  */
 310 static void stop_pci_io(struct octeon_device *oct)
 311 {
 312         /* No more instructions will be forwarded. */
 313         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 314 
 315         pci_disable_device(oct->pci_dev);
 316 
 317         /* Disable interrupts  */
 318         oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
 319 
 320         pcierror_quiesce_device(oct);
 321 
 322         /* Release the interrupt line */
 323         free_irq(oct->pci_dev->irq, oct);
 324 
 325         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 326                 pci_disable_msi(oct->pci_dev);
 327 
 328         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 329                 lio_get_state_string(&oct->status));
 330 
 331         /* making it a common function for all OCTEON models */
 332         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 333 }
 334 
 335 /**
 336  * \brief called when PCI error is detected
 337  * @param pdev Pointer to PCI device
 338  * @param state The current pci connection state
 339  *
 340  * This function is called after a PCI bus error affecting
 341  * this device has been detected.
 342  */
 343 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 344                                                      pci_channel_state_t state)
 345 {
 346         struct octeon_device *oct = pci_get_drvdata(pdev);
 347 
 348         /* Non-correctable Non-fatal errors */
 349         if (state == pci_channel_io_normal) {
 350                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 351                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 352                 return PCI_ERS_RESULT_CAN_RECOVER;
 353         }
 354 
 355         /* Non-correctable Fatal errors */
 356         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 357         stop_pci_io(oct);
 358 
 359         /* Always return a DISCONNECT. There is no support for recovery but only
 360          * for a clean shutdown.
 361          */
 362         return PCI_ERS_RESULT_DISCONNECT;
 363 }
 364 
 365 /**
 366  * \brief mmio handler
 367  * @param pdev Pointer to PCI device
 368  */
 369 static pci_ers_result_t liquidio_pcie_mmio_enabled(
 370                                 struct pci_dev *pdev __attribute__((unused)))
 371 {
 372         /* We should never hit this since we never ask for a reset for a Fatal
 373          * Error. We always return DISCONNECT in io_error above.
 374          * But play safe and return RECOVERED for now.
 375          */
 376         return PCI_ERS_RESULT_RECOVERED;
 377 }
 378 
 379 /**
 380  * \brief called after the pci bus has been reset.
 381  * @param pdev Pointer to PCI device
 382  *
 383  * Restart the card from scratch, as if from a cold-boot. Implementation
 384  * resembles the first-half of the octeon_resume routine.
 385  */
 386 static pci_ers_result_t liquidio_pcie_slot_reset(
 387                                 struct pci_dev *pdev __attribute__((unused)))
 388 {
 389         /* We should never hit this since we never ask for a reset for a Fatal
 390          * Error. We always return DISCONNECT in io_error above.
 391          * But play safe and return RECOVERED for now.
 392          */
 393         return PCI_ERS_RESULT_RECOVERED;
 394 }
 395 
 396 /**
 397  * \brief called when traffic can start flowing again.
 398  * @param pdev Pointer to PCI device
 399  *
 400  * This callback is called when the error recovery driver tells us that
 401  * its OK to resume normal operation. Implementation resembles the
 402  * second-half of the octeon_resume routine.
 403  */
 404 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
 405 {
 406         /* Nothing to be done here. */
 407 }
 408 
 409 #ifdef CONFIG_PM
 410 /**
 411  * \brief called when suspending
 412  * @param pdev Pointer to PCI device
 413  * @param state state to suspend to
 414  */
 415 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
 416                             pm_message_t state __attribute__((unused)))
 417 {
 418         return 0;
 419 }
 420 
 421 /**
 422  * \brief called when resuming
 423  * @param pdev Pointer to PCI device
 424  */
 425 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
 426 {
 427         return 0;
 428 }
 429 #endif
 430 
 431 /* For PCI-E Advanced Error Recovery (AER) Interface */
 432 static const struct pci_error_handlers liquidio_err_handler = {
 433         .error_detected = liquidio_pcie_error_detected,
 434         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 435         .slot_reset     = liquidio_pcie_slot_reset,
 436         .resume         = liquidio_pcie_resume,
 437 };
 438 
 439 static const struct pci_device_id liquidio_pci_tbl[] = {
 440         {       /* 68xx */
 441                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 442         },
 443         {       /* 66xx */
 444                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 445         },
 446         {       /* 23xx pf */
 447                 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 448         },
 449         {
 450                 0, 0, 0, 0, 0, 0, 0
 451         }
 452 };
 453 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 454 
 455 static struct pci_driver liquidio_pci_driver = {
 456         .name           = "LiquidIO",
 457         .id_table       = liquidio_pci_tbl,
 458         .probe          = liquidio_probe,
 459         .remove         = liquidio_remove,
 460         .err_handler    = &liquidio_err_handler,    /* For AER */
 461 
 462 #ifdef CONFIG_PM
 463         .suspend        = liquidio_suspend,
 464         .resume         = liquidio_resume,
 465 #endif
 466 #ifdef CONFIG_PCI_IOV
 467         .sriov_configure = liquidio_enable_sriov,
 468 #endif
 469 };
 470 
 471 /**
 472  * \brief register PCI driver
 473  */
 474 static int liquidio_init_pci(void)
 475 {
 476         return pci_register_driver(&liquidio_pci_driver);
 477 }
 478 
 479 /**
 480  * \brief unregister PCI driver
 481  */
 482 static void liquidio_deinit_pci(void)
 483 {
 484         pci_unregister_driver(&liquidio_pci_driver);
 485 }
 486 
 487 /**
 488  * \brief Check Tx queue status, and take appropriate action
 489  * @param lio per-network private data
 490  * @returns 0 if full, number of queues woken up otherwise
 491  */
 492 static inline int check_txq_status(struct lio *lio)
 493 {
 494         int numqs = lio->netdev->real_num_tx_queues;
 495         int ret_val = 0;
 496         int q, iq;
 497 
 498         /* check each sub-queue state */
 499         for (q = 0; q < numqs; q++) {
 500                 iq = lio->linfo.txpciq[q %
 501                         lio->oct_dev->num_iqs].s.q_no;
 502                 if (octnet_iq_is_full(lio->oct_dev, iq))
 503                         continue;
 504                 if (__netif_subqueue_stopped(lio->netdev, q)) {
 505                         netif_wake_subqueue(lio->netdev, q);
 506                         INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
 507                                                   tx_restart, 1);
 508                         ret_val++;
 509                 }
 510         }
 511 
 512         return ret_val;
 513 }
 514 
 515 /**
 516  * \brief Print link information
 517  * @param netdev network device
 518  */
 519 static void print_link_info(struct net_device *netdev)
 520 {
 521         struct lio *lio = GET_LIO(netdev);
 522 
 523         if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
 524             ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
 525                 struct oct_link_info *linfo = &lio->linfo;
 526 
 527                 if (linfo->link.s.link_up) {
 528                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 529                                    linfo->link.s.speed,
 530                                    (linfo->link.s.duplex) ? "Full" : "Half");
 531                 } else {
 532                         netif_info(lio, link, lio->netdev, "Link Down\n");
 533                 }
 534         }
 535 }
 536 
 537 /**
 538  * \brief Routine to notify MTU change
 539  * @param work work_struct data structure
 540  */
 541 static void octnet_link_status_change(struct work_struct *work)
 542 {
 543         struct cavium_wk *wk = (struct cavium_wk *)work;
 544         struct lio *lio = (struct lio *)wk->ctxptr;
 545 
 546         /* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
 547          * this API is invoked only when new max-MTU of the interface is
 548          * less than current MTU.
 549          */
 550         rtnl_lock();
 551         dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
 552         rtnl_unlock();
 553 }
 554 
 555 /**
 556  * \brief Sets up the mtu status change work
 557  * @param netdev network device
 558  */
 559 static inline int setup_link_status_change_wq(struct net_device *netdev)
 560 {
 561         struct lio *lio = GET_LIO(netdev);
 562         struct octeon_device *oct = lio->oct_dev;
 563 
 564         lio->link_status_wq.wq = alloc_workqueue("link-status",
 565                                                  WQ_MEM_RECLAIM, 0);
 566         if (!lio->link_status_wq.wq) {
 567                 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
 568                 return -1;
 569         }
 570         INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
 571                           octnet_link_status_change);
 572         lio->link_status_wq.wk.ctxptr = lio;
 573 
 574         return 0;
 575 }
 576 
 577 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
 578 {
 579         struct lio *lio = GET_LIO(netdev);
 580 
 581         if (lio->link_status_wq.wq) {
 582                 cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
 583                 destroy_workqueue(lio->link_status_wq.wq);
 584         }
 585 }
 586 
 587 /**
 588  * \brief Update link status
 589  * @param netdev network device
 590  * @param ls link status structure
 591  *
 592  * Called on receipt of a link status response from the core application to
 593  * update each interface's link status.
 594  */
 595 static inline void update_link_status(struct net_device *netdev,
 596                                       union oct_link_status *ls)
 597 {
 598         struct lio *lio = GET_LIO(netdev);
 599         int changed = (lio->linfo.link.u64 != ls->u64);
 600         int current_max_mtu = lio->linfo.link.s.mtu;
 601         struct octeon_device *oct = lio->oct_dev;
 602 
 603         dev_dbg(&oct->pci_dev->dev, "%s: lio->linfo.link.u64=%llx, ls->u64=%llx\n",
 604                 __func__, lio->linfo.link.u64, ls->u64);
 605         lio->linfo.link.u64 = ls->u64;
 606 
 607         if ((lio->intf_open) && (changed)) {
 608                 print_link_info(netdev);
 609                 lio->link_changes++;
 610 
 611                 if (lio->linfo.link.s.link_up) {
 612                         dev_dbg(&oct->pci_dev->dev, "%s: link_up", __func__);
 613                         netif_carrier_on(netdev);
 614                         wake_txqs(netdev);
 615                 } else {
 616                         dev_dbg(&oct->pci_dev->dev, "%s: link_off", __func__);
 617                         netif_carrier_off(netdev);
 618                         stop_txqs(netdev);
 619                 }
 620                 if (lio->linfo.link.s.mtu != current_max_mtu) {
 621                         netif_info(lio, probe, lio->netdev, "Max MTU changed from %d to %d\n",
 622                                    current_max_mtu, lio->linfo.link.s.mtu);
 623                         netdev->max_mtu = lio->linfo.link.s.mtu;
 624                 }
 625                 if (lio->linfo.link.s.mtu < netdev->mtu) {
 626                         dev_warn(&oct->pci_dev->dev,
 627                                  "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
 628                                      netdev->mtu, lio->linfo.link.s.mtu);
 629                         queue_delayed_work(lio->link_status_wq.wq,
 630                                            &lio->link_status_wq.wk.work, 0);
 631                 }
 632         }
 633 }
 634 
 635 /**
 636  * lio_sync_octeon_time - send latest localtime to octeon firmware so that
 637  * firmware will correct it's time, in case there is a time skew
 638  *
 639  * @work: work scheduled to send time update to octeon firmware
 640  **/
 641 static void lio_sync_octeon_time(struct work_struct *work)
 642 {
 643         struct cavium_wk *wk = (struct cavium_wk *)work;
 644         struct lio *lio = (struct lio *)wk->ctxptr;
 645         struct octeon_device *oct = lio->oct_dev;
 646         struct octeon_soft_command *sc;
 647         struct timespec64 ts;
 648         struct lio_time *lt;
 649         int ret;
 650 
 651         sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 16, 0);
 652         if (!sc) {
 653                 dev_err(&oct->pci_dev->dev,
 654                         "Failed to sync time to octeon: soft command allocation failed\n");
 655                 return;
 656         }
 657 
 658         lt = (struct lio_time *)sc->virtdptr;
 659 
 660         /* Get time of the day */
 661         ktime_get_real_ts64(&ts);
 662         lt->sec = ts.tv_sec;
 663         lt->nsec = ts.tv_nsec;
 664         octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);
 665 
 666         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
 667         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
 668                                     OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);
 669 
 670         init_completion(&sc->complete);
 671         sc->sc_status = OCTEON_REQUEST_PENDING;
 672 
 673         ret = octeon_send_soft_command(oct, sc);
 674         if (ret == IQ_SEND_FAILED) {
 675                 dev_err(&oct->pci_dev->dev,
 676                         "Failed to sync time to octeon: failed to send soft command\n");
 677                 octeon_free_soft_command(oct, sc);
 678         } else {
 679                 WRITE_ONCE(sc->caller_is_done, true);
 680         }
 681 
 682         queue_delayed_work(lio->sync_octeon_time_wq.wq,
 683                            &lio->sync_octeon_time_wq.wk.work,
 684                            msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
 685 }
 686 
 687 /**
 688  * setup_sync_octeon_time_wq - Sets up the work to periodically update
 689  * local time to octeon firmware
 690  *
 691  * @netdev - network device which should send time update to firmware
 692  **/
 693 static inline int setup_sync_octeon_time_wq(struct net_device *netdev)
 694 {
 695         struct lio *lio = GET_LIO(netdev);
 696         struct octeon_device *oct = lio->oct_dev;
 697 
 698         lio->sync_octeon_time_wq.wq =
 699                 alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);
 700         if (!lio->sync_octeon_time_wq.wq) {
 701                 dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");
 702                 return -1;
 703         }
 704         INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,
 705                           lio_sync_octeon_time);
 706         lio->sync_octeon_time_wq.wk.ctxptr = lio;
 707         queue_delayed_work(lio->sync_octeon_time_wq.wq,
 708                            &lio->sync_octeon_time_wq.wk.work,
 709                            msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
 710 
 711         return 0;
 712 }
 713 
 714 /**
 715  * cleanup_sync_octeon_time_wq - stop scheduling and destroy the work created
 716  * to periodically update local time to octeon firmware
 717  *
 718  * @netdev - network device which should send time update to firmware
 719  **/
 720 static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)
 721 {
 722         struct lio *lio = GET_LIO(netdev);
 723         struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;
 724 
 725         if (time_wq->wq) {
 726                 cancel_delayed_work_sync(&time_wq->wk.work);
 727                 destroy_workqueue(time_wq->wq);
 728         }
 729 }
 730 
 731 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
 732 {
 733         struct octeon_device *other_oct;
 734 
 735         other_oct = lio_get_device(oct->octeon_id + 1);
 736 
 737         if (other_oct && other_oct->pci_dev) {
 738                 int oct_busnum, other_oct_busnum;
 739 
 740                 oct_busnum = oct->pci_dev->bus->number;
 741                 other_oct_busnum = other_oct->pci_dev->bus->number;
 742 
 743                 if (oct_busnum == other_oct_busnum) {
 744                         int oct_slot, other_oct_slot;
 745 
 746                         oct_slot = PCI_SLOT(oct->pci_dev->devfn);
 747                         other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
 748 
 749                         if (oct_slot == other_oct_slot)
 750                                 return other_oct;
 751                 }
 752         }
 753 
 754         return NULL;
 755 }
 756 
 757 static void disable_all_vf_links(struct octeon_device *oct)
 758 {
 759         struct net_device *netdev;
 760         int max_vfs, vf, i;
 761 
 762         if (!oct)
 763                 return;
 764 
 765         max_vfs = oct->sriov_info.max_vfs;
 766 
 767         for (i = 0; i < oct->ifcount; i++) {
 768                 netdev = oct->props[i].netdev;
 769                 if (!netdev)
 770                         continue;
 771 
 772                 for (vf = 0; vf < max_vfs; vf++)
 773                         liquidio_set_vf_link_state(netdev, vf,
 774                                                    IFLA_VF_LINK_STATE_DISABLE);
 775         }
 776 }
 777 
 778 static int liquidio_watchdog(void *param)
 779 {
 780         bool err_msg_was_printed[LIO_MAX_CORES];
 781         u16 mask_of_crashed_or_stuck_cores = 0;
 782         bool all_vf_links_are_disabled = false;
 783         struct octeon_device *oct = param;
 784         struct octeon_device *other_oct;
 785 #ifdef CONFIG_MODULE_UNLOAD
 786         long refcount, vfs_referencing_pf;
 787         u64 vfs_mask1, vfs_mask2;
 788 #endif
 789         int core;
 790 
 791         memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
 792 
 793         while (!kthread_should_stop()) {
 794                 /* sleep for a couple of seconds so that we don't hog the CPU */
 795                 set_current_state(TASK_INTERRUPTIBLE);
 796                 schedule_timeout(msecs_to_jiffies(2000));
 797 
 798                 mask_of_crashed_or_stuck_cores =
 799                     (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
 800 
 801                 if (!mask_of_crashed_or_stuck_cores)
 802                         continue;
 803 
 804                 WRITE_ONCE(oct->cores_crashed, true);
 805                 other_oct = get_other_octeon_device(oct);
 806                 if (other_oct)
 807                         WRITE_ONCE(other_oct->cores_crashed, true);
 808 
 809                 for (core = 0; core < LIO_MAX_CORES; core++) {
 810                         bool core_crashed_or_got_stuck;
 811 
 812                         core_crashed_or_got_stuck =
 813                                                 (mask_of_crashed_or_stuck_cores
 814                                                  >> core) & 1;
 815 
 816                         if (core_crashed_or_got_stuck &&
 817                             !err_msg_was_printed[core]) {
 818                                 dev_err(&oct->pci_dev->dev,
 819                                         "ERROR: Octeon core %d crashed or got stuck!  See oct-fwdump for details.\n",
 820                                         core);
 821                                 err_msg_was_printed[core] = true;
 822                         }
 823                 }
 824 
 825                 if (all_vf_links_are_disabled)
 826                         continue;
 827 
 828                 disable_all_vf_links(oct);
 829                 disable_all_vf_links(other_oct);
 830                 all_vf_links_are_disabled = true;
 831 
 832 #ifdef CONFIG_MODULE_UNLOAD
 833                 vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
 834                 vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
 835 
 836                 vfs_referencing_pf  = hweight64(vfs_mask1);
 837                 vfs_referencing_pf += hweight64(vfs_mask2);
 838 
 839                 refcount = module_refcount(THIS_MODULE);
 840                 if (refcount >= vfs_referencing_pf) {
 841                         while (vfs_referencing_pf) {
 842                                 module_put(THIS_MODULE);
 843                                 vfs_referencing_pf--;
 844                         }
 845                 }
 846 #endif
 847         }
 848 
 849         return 0;
 850 }
 851 
 852 /**
 853  * \brief PCI probe handler
 854  * @param pdev PCI device structure
 855  * @param ent unused
 856  */
 857 static int
 858 liquidio_probe(struct pci_dev *pdev,
 859                const struct pci_device_id *ent __attribute__((unused)))
 860 {
 861         struct octeon_device *oct_dev = NULL;
 862         struct handshake *hs;
 863 
 864         oct_dev = octeon_allocate_device(pdev->device,
 865                                          sizeof(struct octeon_device_priv));
 866         if (!oct_dev) {
 867                 dev_err(&pdev->dev, "Unable to allocate device\n");
 868                 return -ENOMEM;
 869         }
 870 
 871         if (pdev->device == OCTEON_CN23XX_PF_VID)
 872                 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
 873 
 874         /* Enable PTP for 6XXX Device */
 875         if (((pdev->device == OCTEON_CN66XX) ||
 876              (pdev->device == OCTEON_CN68XX)))
 877                 oct_dev->ptp_enable = true;
 878         else
 879                 oct_dev->ptp_enable = false;
 880 
 881         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
 882                  (u32)pdev->vendor, (u32)pdev->device);
 883 
 884         /* Assign octeon_device for this device to the private data area. */
 885         pci_set_drvdata(pdev, oct_dev);
 886 
 887         /* set linux specific device pointer */
 888         oct_dev->pci_dev = (void *)pdev;
 889 
 890         oct_dev->subsystem_id = pdev->subsystem_vendor |
 891                 (pdev->subsystem_device << 16);
 892 
 893         hs = &handshake[oct_dev->octeon_id];
 894         init_completion(&hs->init);
 895         init_completion(&hs->started);
 896         hs->pci_dev = pdev;
 897 
 898         if (oct_dev->octeon_id == 0)
 899                 /* first LiquidIO NIC is detected */
 900                 complete(&first_stage);
 901 
 902         if (octeon_device_init(oct_dev)) {
 903                 complete(&hs->init);
 904                 liquidio_remove(pdev);
 905                 return -ENOMEM;
 906         }
 907 
 908         if (OCTEON_CN23XX_PF(oct_dev)) {
 909                 u8 bus, device, function;
 910 
 911                 if (atomic_read(oct_dev->adapter_refcount) == 1) {
 912                         /* Each NIC gets one watchdog kernel thread.  The first
 913                          * PF (of each NIC) that gets pci_driver->probe()'d
 914                          * creates that thread.
 915                          */
 916                         bus = pdev->bus->number;
 917                         device = PCI_SLOT(pdev->devfn);
 918                         function = PCI_FUNC(pdev->devfn);
 919                         oct_dev->watchdog_task = kthread_create(
 920                             liquidio_watchdog, oct_dev,
 921                             "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
 922                         if (!IS_ERR(oct_dev->watchdog_task)) {
 923                                 wake_up_process(oct_dev->watchdog_task);
 924                         } else {
 925                                 oct_dev->watchdog_task = NULL;
 926                                 dev_err(&oct_dev->pci_dev->dev,
 927                                         "failed to create kernel_thread\n");
 928                                 liquidio_remove(pdev);
 929                                 return -1;
 930                         }
 931                 }
 932         }
 933 
 934         oct_dev->rx_pause = 1;
 935         oct_dev->tx_pause = 1;
 936 
 937         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
 938 
 939         return 0;
 940 }
 941 
 942 static bool fw_type_is_auto(void)
 943 {
 944         return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,
 945                        sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;
 946 }
 947 
 948 /**
 949  * \brief PCI FLR for each Octeon device.
 950  * @param oct octeon device
 951  */
 952 static void octeon_pci_flr(struct octeon_device *oct)
 953 {
 954         int rc;
 955 
 956         pci_save_state(oct->pci_dev);
 957 
 958         pci_cfg_access_lock(oct->pci_dev);
 959 
 960         /* Quiesce the device completely */
 961         pci_write_config_word(oct->pci_dev, PCI_COMMAND,
 962                               PCI_COMMAND_INTX_DISABLE);
 963 
 964         rc = __pci_reset_function_locked(oct->pci_dev);
 965 
 966         if (rc != 0)
 967                 dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
 968                         rc, oct->pf_num);
 969 
 970         pci_cfg_access_unlock(oct->pci_dev);
 971 
 972         pci_restore_state(oct->pci_dev);
 973 }
 974 
 975 /**
 976  *\brief Destroy resources associated with octeon device
 977  * @param pdev PCI device structure
 978  * @param ent unused
 979  */
 980 static void octeon_destroy_resources(struct octeon_device *oct)
 981 {
 982         int i, refcount;
 983         struct msix_entry *msix_entries;
 984         struct octeon_device_priv *oct_priv =
 985                 (struct octeon_device_priv *)oct->priv;
 986 
 987         struct handshake *hs;
 988 
 989         switch (atomic_read(&oct->status)) {
 990         case OCT_DEV_RUNNING:
 991         case OCT_DEV_CORE_OK:
 992 
 993                 /* No more instructions will be forwarded. */
 994                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
 995 
 996                 oct->app_mode = CVM_DRV_INVALID_APP;
 997                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 998                         lio_get_state_string(&oct->status));
 999 
1000                 schedule_timeout_uninterruptible(HZ / 10);
1001 
1002                 /* fallthrough */
1003         case OCT_DEV_HOST_OK:
1004 
1005                 /* fallthrough */
1006         case OCT_DEV_CONSOLE_INIT_DONE:
1007                 /* Remove any consoles */
1008                 octeon_remove_consoles(oct);
1009 
1010                 /* fallthrough */
1011         case OCT_DEV_IO_QUEUES_DONE:
1012                 if (lio_wait_for_instr_fetch(oct))
1013                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1014 
1015                 if (wait_for_pending_requests(oct))
1016                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1017 
1018                 /* Disable the input and output queues now. No more packets will
1019                  * arrive from Octeon, but we should wait for all packet
1020                  * processing to finish.
1021                  */
1022                 oct->fn_list.disable_io_queues(oct);
1023 
1024                 if (lio_wait_for_oq_pkts(oct))
1025                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1026 
1027                 /* Force all requests waiting to be fetched by OCTEON to
1028                  * complete.
1029                  */
1030                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1031                         struct octeon_instr_queue *iq;
1032 
1033                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
1034                                 continue;
1035                         iq = oct->instr_queue[i];
1036 
1037                         if (atomic_read(&iq->instr_pending)) {
1038                                 spin_lock_bh(&iq->lock);
1039                                 iq->fill_cnt = 0;
1040                                 iq->octeon_read_index = iq->host_write_index;
1041                                 iq->stats.instr_processed +=
1042                                         atomic_read(&iq->instr_pending);
1043                                 lio_process_iq_request_list(oct, iq, 0);
1044                                 spin_unlock_bh(&iq->lock);
1045                         }
1046                 }
1047 
1048                 lio_process_ordered_list(oct, 1);
1049                 octeon_free_sc_done_list(oct);
1050                 octeon_free_sc_zombie_list(oct);
1051 
1052         /* fallthrough */
1053         case OCT_DEV_INTR_SET_DONE:
1054                 /* Disable interrupts  */
1055                 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1056 
1057                 if (oct->msix_on) {
1058                         msix_entries = (struct msix_entry *)oct->msix_entries;
1059                         for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1060                                 if (oct->ioq_vector[i].vector) {
1061                                         /* clear the affinity_cpumask */
1062                                         irq_set_affinity_hint(
1063                                                         msix_entries[i].vector,
1064                                                         NULL);
1065                                         free_irq(msix_entries[i].vector,
1066                                                  &oct->ioq_vector[i]);
1067                                         oct->ioq_vector[i].vector = 0;
1068                                 }
1069                         }
1070                         /* non-iov vector's argument is oct struct */
1071                         free_irq(msix_entries[i].vector, oct);
1072 
1073                         pci_disable_msix(oct->pci_dev);
1074                         kfree(oct->msix_entries);
1075                         oct->msix_entries = NULL;
1076                 } else {
1077                         /* Release the interrupt line */
1078                         free_irq(oct->pci_dev->irq, oct);
1079 
1080                         if (oct->flags & LIO_FLAG_MSI_ENABLED)
1081                                 pci_disable_msi(oct->pci_dev);
1082                 }
1083 
1084                 kfree(oct->irq_name_storage);
1085                 oct->irq_name_storage = NULL;
1086 
1087         /* fallthrough */
1088         case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
1089                 if (OCTEON_CN23XX_PF(oct))
1090                         octeon_free_ioq_vector(oct);
1091 
1092         /* fallthrough */
1093         case OCT_DEV_MBOX_SETUP_DONE:
1094                 if (OCTEON_CN23XX_PF(oct))
1095                         oct->fn_list.free_mbox(oct);
1096 
1097         /* fallthrough */
1098         case OCT_DEV_IN_RESET:
1099         case OCT_DEV_DROQ_INIT_DONE:
1100                 /* Wait for any pending operations */
1101                 mdelay(100);
1102                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1103                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
1104                                 continue;
1105                         octeon_delete_droq(oct, i);
1106                 }
1107 
1108                 /* Force any pending handshakes to complete */
1109                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1110                         hs = &handshake[i];
1111 
1112                         if (hs->pci_dev) {
1113                                 handshake[oct->octeon_id].init_ok = 0;
1114                                 complete(&handshake[oct->octeon_id].init);
1115                                 handshake[oct->octeon_id].started_ok = 0;
1116                                 complete(&handshake[oct->octeon_id].started);
1117                         }
1118                 }
1119 
1120                 /* fallthrough */
1121         case OCT_DEV_RESP_LIST_INIT_DONE:
1122                 octeon_delete_response_list(oct);
1123 
1124                 /* fallthrough */
1125         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1126                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1127                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
1128                                 continue;
1129                         octeon_delete_instr_queue(oct, i);
1130                 }
1131 #ifdef CONFIG_PCI_IOV
1132                 if (oct->sriov_info.sriov_enabled)
1133                         pci_disable_sriov(oct->pci_dev);
1134 #endif
1135                 /* fallthrough */
1136         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1137                 octeon_free_sc_buffer_pool(oct);
1138 
1139                 /* fallthrough */
1140         case OCT_DEV_DISPATCH_INIT_DONE:
1141                 octeon_delete_dispatch_list(oct);
1142                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1143 
1144                 /* fallthrough */
1145         case OCT_DEV_PCI_MAP_DONE:
1146                 refcount = octeon_deregister_device(oct);
1147 
1148                 /* Soft reset the octeon device before exiting.
1149                  * However, if fw was loaded from card (i.e. autoboot),
1150                  * perform an FLR instead.
1151                  * Implementation note: only soft-reset the device
1152                  * if it is a CN6XXX OR the LAST CN23XX device.
1153                  */
1154                 if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)
1155                         octeon_pci_flr(oct);
1156                 else if (OCTEON_CN6XXX(oct) || !refcount)
1157                         oct->fn_list.soft_reset(oct);
1158 
1159                 octeon_unmap_pci_barx(oct, 0);
1160                 octeon_unmap_pci_barx(oct, 1);
1161 
1162                 /* fallthrough */
1163         case OCT_DEV_PCI_ENABLE_DONE:
1164                 pci_clear_master(oct->pci_dev);
1165                 /* Disable the device, releasing the PCI INT */
1166                 pci_disable_device(oct->pci_dev);
1167 
1168                 /* fallthrough */
1169         case OCT_DEV_BEGIN_STATE:
1170                 /* Nothing to be done here either */
1171                 break;
1172         }                       /* end switch (oct->status) */
1173 
1174         tasklet_kill(&oct_priv->droq_tasklet);
1175 }
1176 
1177 /**
1178  * \brief Send Rx control command
1179  * @param lio per-network private data
1180  * @param start_stop whether to start or stop
1181  */
1182 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1183 {
1184         struct octeon_soft_command *sc;
1185         union octnet_cmd *ncmd;
1186         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1187         int retval;
1188 
1189         if (oct->props[lio->ifidx].rx_on == start_stop)
1190                 return;
1191 
1192         sc = (struct octeon_soft_command *)
1193                 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1194                                           16, 0);
1195         if (!sc) {
1196                 netif_info(lio, rx_err, lio->netdev,
1197                            "Failed to allocate octeon_soft_command\n");
1198                 return;
1199         }
1200 
1201         ncmd = (union octnet_cmd *)sc->virtdptr;
1202 
1203         ncmd->u64 = 0;
1204         ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1205         ncmd->s.param1 = start_stop;
1206 
1207         octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1208 
1209         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1210 
1211         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1212                                     OPCODE_NIC_CMD, 0, 0, 0);
1213 
1214         init_completion(&sc->complete);
1215         sc->sc_status = OCTEON_REQUEST_PENDING;
1216 
1217         retval = octeon_send_soft_command(oct, sc);
1218         if (retval == IQ_SEND_FAILED) {
1219                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1220                 octeon_free_soft_command(oct, sc);
1221                 return;
1222         } else {
1223                 /* Sleep on a wait queue till the cond flag indicates that the
1224                  * response arrived or timed-out.
1225                  */
1226                 retval = wait_for_sc_completion_timeout(oct, sc, 0);
1227                 if (retval)
1228                         return;
1229 
1230                 oct->props[lio->ifidx].rx_on = start_stop;
1231                 WRITE_ONCE(sc->caller_is_done, true);
1232         }
1233 }
1234 
1235 /**
1236  * \brief Destroy NIC device interface
1237  * @param oct octeon device
1238  * @param ifidx which interface to destroy
1239  *
1240  * Cleanup associated with each interface for an Octeon device  when NIC
1241  * module is being unloaded or if initialization fails during load.
1242  */
1243 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1244 {
1245         struct net_device *netdev = oct->props[ifidx].netdev;
1246         struct octeon_device_priv *oct_priv =
1247                 (struct octeon_device_priv *)oct->priv;
1248         struct napi_struct *napi, *n;
1249         struct lio *lio;
1250 
1251         if (!netdev) {
1252                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1253                         __func__, ifidx);
1254                 return;
1255         }
1256 
1257         lio = GET_LIO(netdev);
1258 
1259         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1260 
1261         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1262                 liquidio_stop(netdev);
1263 
1264         if (oct->props[lio->ifidx].napi_enabled == 1) {
1265                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1266                         napi_disable(napi);
1267 
1268                 oct->props[lio->ifidx].napi_enabled = 0;
1269 
1270                 if (OCTEON_CN23XX_PF(oct))
1271                         oct->droq[0]->ops.poll_mode = 0;
1272         }
1273 
1274         /* Delete NAPI */
1275         list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1276                 netif_napi_del(napi);
1277 
1278         tasklet_enable(&oct_priv->droq_tasklet);
1279 
1280         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1281                 unregister_netdev(netdev);
1282 
1283         cleanup_sync_octeon_time_wq(netdev);
1284         cleanup_link_status_change_wq(netdev);
1285 
1286         cleanup_rx_oom_poll_fn(netdev);
1287 
1288         lio_delete_glists(lio);
1289 
1290         free_netdev(netdev);
1291 
1292         oct->props[ifidx].gmxport = -1;
1293 
1294         oct->props[ifidx].netdev = NULL;
1295 }
1296 
1297 /**
1298  * \brief Stop complete NIC functionality
1299  * @param oct octeon device
1300  */
1301 static int liquidio_stop_nic_module(struct octeon_device *oct)
1302 {
1303         int i, j;
1304         struct lio *lio;
1305 
1306         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1307         if (!oct->ifcount) {
1308                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1309                 return 1;
1310         }
1311 
1312         spin_lock_bh(&oct->cmd_resp_wqlock);
1313         oct->cmd_resp_state = OCT_DRV_OFFLINE;
1314         spin_unlock_bh(&oct->cmd_resp_wqlock);
1315 
1316         lio_vf_rep_destroy(oct);
1317 
1318         for (i = 0; i < oct->ifcount; i++) {
1319                 lio = GET_LIO(oct->props[i].netdev);
1320                 for (j = 0; j < oct->num_oqs; j++)
1321                         octeon_unregister_droq_ops(oct,
1322                                                    lio->linfo.rxpciq[j].s.q_no);
1323         }
1324 
1325         for (i = 0; i < oct->ifcount; i++)
1326                 liquidio_destroy_nic_device(oct, i);
1327 
1328         if (oct->devlink) {
1329                 devlink_unregister(oct->devlink);
1330                 devlink_free(oct->devlink);
1331                 oct->devlink = NULL;
1332         }
1333 
1334         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1335         return 0;
1336 }
1337 
1338 /**
1339  * \brief Cleans up resources at unload time
1340  * @param pdev PCI device structure
1341  */
1342 static void liquidio_remove(struct pci_dev *pdev)
1343 {
1344         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1345 
1346         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1347 
1348         if (oct_dev->watchdog_task)
1349                 kthread_stop(oct_dev->watchdog_task);
1350 
1351         if (!oct_dev->octeon_id &&
1352             oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)
1353                 lio_vf_rep_modexit();
1354 
1355         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1356                 liquidio_stop_nic_module(oct_dev);
1357 
1358         /* Reset the octeon device and cleanup all memory allocated for
1359          * the octeon device by driver.
1360          */
1361         octeon_destroy_resources(oct_dev);
1362 
1363         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1364 
1365         /* This octeon device has been removed. Update the global
1366          * data structure to reflect this. Free the device structure.
1367          */
1368         octeon_free_device_mem(oct_dev);
1369 }
1370 
1371 /**
1372  * \brief Identify the Octeon device and to map the BAR address space
1373  * @param oct octeon device
1374  */
1375 static int octeon_chip_specific_setup(struct octeon_device *oct)
1376 {
1377         u32 dev_id, rev_id;
1378         int ret = 1;
1379         char *s;
1380 
1381         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1382         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1383         oct->rev_id = rev_id & 0xff;
1384 
1385         switch (dev_id) {
1386         case OCTEON_CN68XX_PCIID:
1387                 oct->chip_id = OCTEON_CN68XX;
1388                 ret = lio_setup_cn68xx_octeon_device(oct);
1389                 s = "CN68XX";
1390                 break;
1391 
1392         case OCTEON_CN66XX_PCIID:
1393                 oct->chip_id = OCTEON_CN66XX;
1394                 ret = lio_setup_cn66xx_octeon_device(oct);
1395                 s = "CN66XX";
1396                 break;
1397 
1398         case OCTEON_CN23XX_PCIID_PF:
1399                 oct->chip_id = OCTEON_CN23XX_PF_VID;
1400                 ret = setup_cn23xx_octeon_pf_device(oct);
1401                 if (ret)
1402                         break;
1403 #ifdef CONFIG_PCI_IOV
1404                 if (!ret)
1405                         pci_sriov_set_totalvfs(oct->pci_dev,
1406                                                oct->sriov_info.max_vfs);
1407 #endif
1408                 s = "CN23XX";
1409                 break;
1410 
1411         default:
1412                 s = "?";
1413                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1414                         dev_id);
1415         }
1416 
1417         if (!ret)
1418                 dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1419                          OCTEON_MAJOR_REV(oct),
1420                          OCTEON_MINOR_REV(oct),
1421                          octeon_get_conf(oct)->card_name,
1422                          LIQUIDIO_VERSION);
1423 
1424         return ret;
1425 }
1426 
1427 /**
1428  * \brief PCI initialization for each Octeon device.
1429  * @param oct octeon device
1430  */
1431 static int octeon_pci_os_setup(struct octeon_device *oct)
1432 {
1433         /* setup PCI stuff first */
1434         if (pci_enable_device(oct->pci_dev)) {
1435                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1436                 return 1;
1437         }
1438 
1439         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1440                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1441                 pci_disable_device(oct->pci_dev);
1442                 return 1;
1443         }
1444 
1445         /* Enable PCI DMA Master. */
1446         pci_set_master(oct->pci_dev);
1447 
1448         return 0;
1449 }
1450 
1451 /**
1452  * \brief Unmap and free network buffer
1453  * @param buf buffer
1454  */
1455 static void free_netbuf(void *buf)
1456 {
1457         struct sk_buff *skb;
1458         struct octnet_buf_free_info *finfo;
1459         struct lio *lio;
1460 
1461         finfo = (struct octnet_buf_free_info *)buf;
1462         skb = finfo->skb;
1463         lio = finfo->lio;
1464 
1465         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1466                          DMA_TO_DEVICE);
1467 
1468         tx_buffer_free(skb);
1469 }
1470 
1471 /**
1472  * \brief Unmap and free gather buffer
1473  * @param buf buffer
1474  */
1475 static void free_netsgbuf(void *buf)
1476 {
1477         struct octnet_buf_free_info *finfo;
1478         struct sk_buff *skb;
1479         struct lio *lio;
1480         struct octnic_gather *g;
1481         int i, frags, iq;
1482 
1483         finfo = (struct octnet_buf_free_info *)buf;
1484         skb = finfo->skb;
1485         lio = finfo->lio;
1486         g = finfo->g;
1487         frags = skb_shinfo(skb)->nr_frags;
1488 
1489         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1490                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1491                          DMA_TO_DEVICE);
1492 
1493         i = 1;
1494         while (frags--) {
1495                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1496 
1497                 pci_unmap_page((lio->oct_dev)->pci_dev,
1498                                g->sg[(i >> 2)].ptr[(i & 3)],
1499                                skb_frag_size(frag), DMA_TO_DEVICE);
1500                 i++;
1501         }
1502 
1503         iq = skb_iq(lio->oct_dev, skb);
1504         spin_lock(&lio->glist_lock[iq]);
1505         list_add_tail(&g->list, &lio->glist[iq]);
1506         spin_unlock(&lio->glist_lock[iq]);
1507 
1508         tx_buffer_free(skb);
1509 }
1510 
1511 /**
1512  * \brief Unmap and free gather buffer with response
1513  * @param buf buffer
1514  */
1515 static void free_netsgbuf_with_resp(void *buf)
1516 {
1517         struct octeon_soft_command *sc;
1518         struct octnet_buf_free_info *finfo;
1519         struct sk_buff *skb;
1520         struct lio *lio;
1521         struct octnic_gather *g;
1522         int i, frags, iq;
1523 
1524         sc = (struct octeon_soft_command *)buf;
1525         skb = (struct sk_buff *)sc->callback_arg;
1526         finfo = (struct octnet_buf_free_info *)&skb->cb;
1527 
1528         lio = finfo->lio;
1529         g = finfo->g;
1530         frags = skb_shinfo(skb)->nr_frags;
1531 
1532         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1533                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1534                          DMA_TO_DEVICE);
1535 
1536         i = 1;
1537         while (frags--) {
1538                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1539 
1540                 pci_unmap_page((lio->oct_dev)->pci_dev,
1541                                g->sg[(i >> 2)].ptr[(i & 3)],
1542                                skb_frag_size(frag), DMA_TO_DEVICE);
1543                 i++;
1544         }
1545 
1546         iq = skb_iq(lio->oct_dev, skb);
1547 
1548         spin_lock(&lio->glist_lock[iq]);
1549         list_add_tail(&g->list, &lio->glist[iq]);
1550         spin_unlock(&lio->glist_lock[iq]);
1551 
1552         /* Don't free the skb yet */
1553 }
1554 
1555 /**
1556  * \brief Adjust ptp frequency
1557  * @param ptp PTP clock info
1558  * @param ppb how much to adjust by, in parts-per-billion
1559  */
1560 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1561 {
1562         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1563         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1564         u64 comp, delta;
1565         unsigned long flags;
1566         bool neg_adj = false;
1567 
1568         if (ppb < 0) {
1569                 neg_adj = true;
1570                 ppb = -ppb;
1571         }
1572 
1573         /* The hardware adds the clock compensation value to the
1574          * PTP clock on every coprocessor clock cycle, so we
1575          * compute the delta in terms of coprocessor clocks.
1576          */
1577         delta = (u64)ppb << 32;
1578         do_div(delta, oct->coproc_clock_rate);
1579 
1580         spin_lock_irqsave(&lio->ptp_lock, flags);
1581         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1582         if (neg_adj)
1583                 comp -= delta;
1584         else
1585                 comp += delta;
1586         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1587         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1588 
1589         return 0;
1590 }
1591 
1592 /**
1593  * \brief Adjust ptp time
1594  * @param ptp PTP clock info
1595  * @param delta how much to adjust by, in nanosecs
1596  */
1597 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1598 {
1599         unsigned long flags;
1600         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1601 
1602         spin_lock_irqsave(&lio->ptp_lock, flags);
1603         lio->ptp_adjust += delta;
1604         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1605 
1606         return 0;
1607 }
1608 
1609 /**
1610  * \brief Get hardware clock time, including any adjustment
1611  * @param ptp PTP clock info
1612  * @param ts timespec
1613  */
1614 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1615                                 struct timespec64 *ts)
1616 {
1617         u64 ns;
1618         unsigned long flags;
1619         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1620         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1621 
1622         spin_lock_irqsave(&lio->ptp_lock, flags);
1623         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1624         ns += lio->ptp_adjust;
1625         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1626 
1627         *ts = ns_to_timespec64(ns);
1628 
1629         return 0;
1630 }
1631 
1632 /**
1633  * \brief Set hardware clock time. Reset adjustment
1634  * @param ptp PTP clock info
1635  * @param ts timespec
1636  */
1637 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1638                                 const struct timespec64 *ts)
1639 {
1640         u64 ns;
1641         unsigned long flags;
1642         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1643         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1644 
1645         ns = timespec64_to_ns(ts);
1646 
1647         spin_lock_irqsave(&lio->ptp_lock, flags);
1648         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1649         lio->ptp_adjust = 0;
1650         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1651 
1652         return 0;
1653 }
1654 
1655 /**
1656  * \brief Check if PTP is enabled
1657  * @param ptp PTP clock info
1658  * @param rq request
1659  * @param on is it on
1660  */
1661 static int
1662 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1663                     struct ptp_clock_request *rq __attribute__((unused)),
1664                     int on __attribute__((unused)))
1665 {
1666         return -EOPNOTSUPP;
1667 }
1668 
1669 /**
1670  * \brief Open PTP clock source
1671  * @param netdev network device
1672  */
1673 static void oct_ptp_open(struct net_device *netdev)
1674 {
1675         struct lio *lio = GET_LIO(netdev);
1676         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1677 
1678         spin_lock_init(&lio->ptp_lock);
1679 
1680         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1681         lio->ptp_info.owner = THIS_MODULE;
1682         lio->ptp_info.max_adj = 250000000;
1683         lio->ptp_info.n_alarm = 0;
1684         lio->ptp_info.n_ext_ts = 0;
1685         lio->ptp_info.n_per_out = 0;
1686         lio->ptp_info.pps = 0;
1687         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1688         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1689         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1690         lio->ptp_info.settime64 = liquidio_ptp_settime;
1691         lio->ptp_info.enable = liquidio_ptp_enable;
1692 
1693         lio->ptp_adjust = 0;
1694 
1695         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1696                                              &oct->pci_dev->dev);
1697 
1698         if (IS_ERR(lio->ptp_clock))
1699                 lio->ptp_clock = NULL;
1700 }
1701 
1702 /**
1703  * \brief Init PTP clock
1704  * @param oct octeon device
1705  */
1706 static void liquidio_ptp_init(struct octeon_device *oct)
1707 {
1708         u64 clock_comp, cfg;
1709 
1710         clock_comp = (u64)NSEC_PER_SEC << 32;
1711         do_div(clock_comp, oct->coproc_clock_rate);
1712         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1713 
1714         /* Enable */
1715         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1716         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1717 }
1718 
1719 /**
1720  * \brief Load firmware to device
1721  * @param oct octeon device
1722  *
1723  * Maps device to firmware filename, requests firmware, and downloads it
1724  */
1725 static int load_firmware(struct octeon_device *oct)
1726 {
1727         int ret = 0;
1728         const struct firmware *fw;
1729         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1730         char *tmp_fw_type;
1731 
1732         if (fw_type_is_auto()) {
1733                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1734                 strncpy(fw_type, tmp_fw_type, sizeof(fw_type));
1735         } else {
1736                 tmp_fw_type = fw_type;
1737         }
1738 
1739         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1740                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1741                 LIO_FW_NAME_SUFFIX);
1742 
1743         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1744         if (ret) {
1745                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n",
1746                         fw_name);
1747                 release_firmware(fw);
1748                 return ret;
1749         }
1750 
1751         ret = octeon_download_firmware(oct, fw->data, fw->size);
1752 
1753         release_firmware(fw);
1754 
1755         return ret;
1756 }
1757 
1758 /**
1759  * \brief Poll routine for checking transmit queue status
1760  * @param work work_struct data structure
1761  */
1762 static void octnet_poll_check_txq_status(struct work_struct *work)
1763 {
1764         struct cavium_wk *wk = (struct cavium_wk *)work;
1765         struct lio *lio = (struct lio *)wk->ctxptr;
1766 
1767         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
1768                 return;
1769 
1770         check_txq_status(lio);
1771         queue_delayed_work(lio->txq_status_wq.wq,
1772                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1773 }
1774 
1775 /**
1776  * \brief Sets up the txq poll check
1777  * @param netdev network device
1778  */
1779 static inline int setup_tx_poll_fn(struct net_device *netdev)
1780 {
1781         struct lio *lio = GET_LIO(netdev);
1782         struct octeon_device *oct = lio->oct_dev;
1783 
1784         lio->txq_status_wq.wq = alloc_workqueue("txq-status",
1785                                                 WQ_MEM_RECLAIM, 0);
1786         if (!lio->txq_status_wq.wq) {
1787                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
1788                 return -1;
1789         }
1790         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
1791                           octnet_poll_check_txq_status);
1792         lio->txq_status_wq.wk.ctxptr = lio;
1793         queue_delayed_work(lio->txq_status_wq.wq,
1794                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1795         return 0;
1796 }
1797 
1798 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
1799 {
1800         struct lio *lio = GET_LIO(netdev);
1801 
1802         if (lio->txq_status_wq.wq) {
1803                 cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
1804                 destroy_workqueue(lio->txq_status_wq.wq);
1805         }
1806 }
1807 
1808 /**
1809  * \brief Net device open for LiquidIO
1810  * @param netdev network device
1811  */
1812 static int liquidio_open(struct net_device *netdev)
1813 {
1814         struct lio *lio = GET_LIO(netdev);
1815         struct octeon_device *oct = lio->oct_dev;
1816         struct octeon_device_priv *oct_priv =
1817                 (struct octeon_device_priv *)oct->priv;
1818         struct napi_struct *napi, *n;
1819 
1820         if (oct->props[lio->ifidx].napi_enabled == 0) {
1821                 tasklet_disable(&oct_priv->droq_tasklet);
1822 
1823                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1824                         napi_enable(napi);
1825 
1826                 oct->props[lio->ifidx].napi_enabled = 1;
1827 
1828                 if (OCTEON_CN23XX_PF(oct))
1829                         oct->droq[0]->ops.poll_mode = 1;
1830         }
1831 
1832         if (oct->ptp_enable)
1833                 oct_ptp_open(netdev);
1834 
1835         ifstate_set(lio, LIO_IFSTATE_RUNNING);
1836 
1837         if (OCTEON_CN23XX_PF(oct)) {
1838                 if (!oct->msix_on)
1839                         if (setup_tx_poll_fn(netdev))
1840                                 return -1;
1841         } else {
1842                 if (setup_tx_poll_fn(netdev))
1843                         return -1;
1844         }
1845 
1846         netif_tx_start_all_queues(netdev);
1847 
1848         /* Ready for link status updates */
1849         lio->intf_open = 1;
1850 
1851         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
1852 
1853         /* tell Octeon to start forwarding packets to host */
1854         send_rx_ctrl_cmd(lio, 1);
1855 
1856         /* start periodical statistics fetch */
1857         INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
1858         lio->stats_wk.ctxptr = lio;
1859         schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
1860                                         (LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
1861 
1862         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
1863                  netdev->name);
1864 
1865         return 0;
1866 }
1867 
1868 /**
1869  * \brief Net device stop for LiquidIO
1870  * @param netdev network device
1871  */
1872 static int liquidio_stop(struct net_device *netdev)
1873 {
1874         struct lio *lio = GET_LIO(netdev);
1875         struct octeon_device *oct = lio->oct_dev;
1876         struct octeon_device_priv *oct_priv =
1877                 (struct octeon_device_priv *)oct->priv;
1878         struct napi_struct *napi, *n;
1879 
1880         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1881 
1882         /* Stop any link updates */
1883         lio->intf_open = 0;
1884 
1885         stop_txqs(netdev);
1886 
1887         /* Inform that netif carrier is down */
1888         netif_carrier_off(netdev);
1889         netif_tx_disable(netdev);
1890 
1891         lio->linfo.link.s.link_up = 0;
1892         lio->link_changes++;
1893 
1894         /* Tell Octeon that nic interface is down. */
1895         send_rx_ctrl_cmd(lio, 0);
1896 
1897         if (OCTEON_CN23XX_PF(oct)) {
1898                 if (!oct->msix_on)
1899                         cleanup_tx_poll_fn(netdev);
1900         } else {
1901                 cleanup_tx_poll_fn(netdev);
1902         }
1903 
1904         cancel_delayed_work_sync(&lio->stats_wk.work);
1905 
1906         if (lio->ptp_clock) {
1907                 ptp_clock_unregister(lio->ptp_clock);
1908                 lio->ptp_clock = NULL;
1909         }
1910 
1911         /* Wait for any pending Rx descriptors */
1912         if (lio_wait_for_clean_oq(oct))
1913                 netif_info(lio, rx_err, lio->netdev,
1914                            "Proceeding with stop interface after partial RX desc processing\n");
1915 
1916         if (oct->props[lio->ifidx].napi_enabled == 1) {
1917                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1918                         napi_disable(napi);
1919 
1920                 oct->props[lio->ifidx].napi_enabled = 0;
1921 
1922                 if (OCTEON_CN23XX_PF(oct))
1923                         oct->droq[0]->ops.poll_mode = 0;
1924 
1925                 tasklet_enable(&oct_priv->droq_tasklet);
1926         }
1927 
1928         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
1929 
1930         return 0;
1931 }
1932 
1933 /**
1934  * \brief Converts a mask based on net device flags
1935  * @param netdev network device
1936  *
1937  * This routine generates a octnet_ifflags mask from the net device flags
1938  * received from the OS.
1939  */
1940 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
1941 {
1942         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1943 
1944         if (netdev->flags & IFF_PROMISC)
1945                 f |= OCTNET_IFFLAG_PROMISC;
1946 
1947         if (netdev->flags & IFF_ALLMULTI)
1948                 f |= OCTNET_IFFLAG_ALLMULTI;
1949 
1950         if (netdev->flags & IFF_MULTICAST) {
1951                 f |= OCTNET_IFFLAG_MULTICAST;
1952 
1953                 /* Accept all multicast addresses if there are more than we
1954                  * can handle
1955                  */
1956                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1957                         f |= OCTNET_IFFLAG_ALLMULTI;
1958         }
1959 
1960         if (netdev->flags & IFF_BROADCAST)
1961                 f |= OCTNET_IFFLAG_BROADCAST;
1962 
1963         return f;
1964 }
1965 
1966 /**
1967  * \brief Net device set_multicast_list
1968  * @param netdev network device
1969  */
1970 static void liquidio_set_mcast_list(struct net_device *netdev)
1971 {
1972         struct lio *lio = GET_LIO(netdev);
1973         struct octeon_device *oct = lio->oct_dev;
1974         struct octnic_ctrl_pkt nctrl;
1975         struct netdev_hw_addr *ha;
1976         u64 *mc;
1977         int ret;
1978         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1979 
1980         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1981 
1982         /* Create a ctrl pkt command to be sent to core app. */
1983         nctrl.ncmd.u64 = 0;
1984         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1985         nctrl.ncmd.s.param1 = get_new_flags(netdev);
1986         nctrl.ncmd.s.param2 = mc_count;
1987         nctrl.ncmd.s.more = mc_count;
1988         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1989         nctrl.netpndev = (u64)netdev;
1990         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1991 
1992         /* copy all the addresses into the udd */
1993         mc = &nctrl.udd[0];
1994         netdev_for_each_mc_addr(ha, netdev) {
1995                 *mc = 0;
1996                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
1997                 /* no need to swap bytes */
1998 
1999                 if (++mc > &nctrl.udd[mc_count])
2000                         break;
2001         }
2002 
2003         /* Apparently, any activity in this call from the kernel has to
2004          * be atomic. So we won't wait for response.
2005          */
2006 
2007         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2008         if (ret) {
2009                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2010                         ret);
2011         }
2012 }
2013 
2014 /**
2015  * \brief Net device set_mac_address
2016  * @param netdev network device
2017  */
2018 static int liquidio_set_mac(struct net_device *netdev, void *p)
2019 {
2020         int ret = 0;
2021         struct lio *lio = GET_LIO(netdev);
2022         struct octeon_device *oct = lio->oct_dev;
2023         struct sockaddr *addr = (struct sockaddr *)p;
2024         struct octnic_ctrl_pkt nctrl;
2025 
2026         if (!is_valid_ether_addr(addr->sa_data))
2027                 return -EADDRNOTAVAIL;
2028 
2029         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2030 
2031         nctrl.ncmd.u64 = 0;
2032         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2033         nctrl.ncmd.s.param1 = 0;
2034         nctrl.ncmd.s.more = 1;
2035         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2036         nctrl.netpndev = (u64)netdev;
2037 
2038         nctrl.udd[0] = 0;
2039         /* The MAC Address is presented in network byte order. */
2040         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2041 
2042         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2043         if (ret < 0) {
2044                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2045                 return -ENOMEM;
2046         }
2047 
2048         if (nctrl.sc_status) {
2049                 dev_err(&oct->pci_dev->dev,
2050                         "%s: MAC Address change failed. sc return=%x\n",
2051                          __func__, nctrl.sc_status);
2052                 return -EIO;
2053         }
2054 
2055         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2056         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2057 
2058         return 0;
2059 }
2060 
2061 static void
2062 liquidio_get_stats64(struct net_device *netdev,
2063                      struct rtnl_link_stats64 *lstats)
2064 {
2065         struct lio *lio = GET_LIO(netdev);
2066         struct octeon_device *oct;
2067         u64 pkts = 0, drop = 0, bytes = 0;
2068         struct oct_droq_stats *oq_stats;
2069         struct oct_iq_stats *iq_stats;
2070         int i, iq_no, oq_no;
2071 
2072         oct = lio->oct_dev;
2073 
2074         if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
2075                 return;
2076 
2077         for (i = 0; i < oct->num_iqs; i++) {
2078                 iq_no = lio->linfo.txpciq[i].s.q_no;
2079                 iq_stats = &oct->instr_queue[iq_no]->stats;
2080                 pkts += iq_stats->tx_done;
2081                 drop += iq_stats->tx_dropped;
2082                 bytes += iq_stats->tx_tot_bytes;
2083         }
2084 
2085         lstats->tx_packets = pkts;
2086         lstats->tx_bytes = bytes;
2087         lstats->tx_dropped = drop;
2088 
2089         pkts = 0;
2090         drop = 0;
2091         bytes = 0;
2092 
2093         for (i = 0; i < oct->num_oqs; i++) {
2094                 oq_no = lio->linfo.rxpciq[i].s.q_no;
2095                 oq_stats = &oct->droq[oq_no]->stats;
2096                 pkts += oq_stats->rx_pkts_received;
2097                 drop += (oq_stats->rx_dropped +
2098                          oq_stats->dropped_nodispatch +
2099                          oq_stats->dropped_toomany +
2100                          oq_stats->dropped_nomem);
2101                 bytes += oq_stats->rx_bytes_received;
2102         }
2103 
2104         lstats->rx_bytes = bytes;
2105         lstats->rx_packets = pkts;
2106         lstats->rx_dropped = drop;
2107 
2108         lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
2109         lstats->collisions = oct->link_stats.fromhost.total_collisions;
2110 
2111         /* detailed rx_errors: */
2112         lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
2113         /* recved pkt with crc error    */
2114         lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
2115         /* recv'd frame alignment error */
2116         lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
2117         /* recv'r fifo overrun */
2118         lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;
2119 
2120         lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
2121                 lstats->rx_frame_errors + lstats->rx_fifo_errors;
2122 
2123         /* detailed tx_errors */
2124         lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
2125         lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
2126         lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;
2127 
2128         lstats->tx_errors = lstats->tx_aborted_errors +
2129                 lstats->tx_carrier_errors +
2130                 lstats->tx_fifo_errors;
2131 }
2132 
2133 /**
2134  * \brief Handler for SIOCSHWTSTAMP ioctl
2135  * @param netdev network device
2136  * @param ifr interface request
2137  * @param cmd command
2138  */
2139 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2140 {
2141         struct hwtstamp_config conf;
2142         struct lio *lio = GET_LIO(netdev);
2143 
2144         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2145                 return -EFAULT;
2146 
2147         if (conf.flags)
2148                 return -EINVAL;
2149 
2150         switch (conf.tx_type) {
2151         case HWTSTAMP_TX_ON:
2152         case HWTSTAMP_TX_OFF:
2153                 break;
2154         default:
2155                 return -ERANGE;
2156         }
2157 
2158         switch (conf.rx_filter) {
2159         case HWTSTAMP_FILTER_NONE:
2160                 break;
2161         case HWTSTAMP_FILTER_ALL:
2162         case HWTSTAMP_FILTER_SOME:
2163         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2164         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2165         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2166         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2167         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2168         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2169         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2170         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2171         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2172         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2173         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2174         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2175         case HWTSTAMP_FILTER_NTP_ALL:
2176                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2177                 break;
2178         default:
2179                 return -ERANGE;
2180         }
2181 
2182         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2183                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2184 
2185         else
2186                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2187 
2188         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2189 }
2190 
2191 /**
2192  * \brief ioctl handler
2193  * @param netdev network device
2194  * @param ifr interface request
2195  * @param cmd command
2196  */
2197 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2198 {
2199         struct lio *lio = GET_LIO(netdev);
2200 
2201         switch (cmd) {
2202         case SIOCSHWTSTAMP:
2203                 if (lio->oct_dev->ptp_enable)
2204                         return hwtstamp_ioctl(netdev, ifr);
2205                 /* fall through */
2206         default:
2207                 return -EOPNOTSUPP;
2208         }
2209 }
2210 
2211 /**
2212  * \brief handle a Tx timestamp response
2213  * @param status response status
2214  * @param buf pointer to skb
2215  */
2216 static void handle_timestamp(struct octeon_device *oct,
2217                              u32 status,
2218                              void *buf)
2219 {
2220         struct octnet_buf_free_info *finfo;
2221         struct octeon_soft_command *sc;
2222         struct oct_timestamp_resp *resp;
2223         struct lio *lio;
2224         struct sk_buff *skb = (struct sk_buff *)buf;
2225 
2226         finfo = (struct octnet_buf_free_info *)skb->cb;
2227         lio = finfo->lio;
2228         sc = finfo->sc;
2229         oct = lio->oct_dev;
2230         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2231 
2232         if (status != OCTEON_REQUEST_DONE) {
2233                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2234                         CVM_CAST64(status));
2235                 resp->timestamp = 0;
2236         }
2237 
2238         octeon_swap_8B_data(&resp->timestamp, 1);
2239 
2240         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2241                 struct skb_shared_hwtstamps ts;
2242                 u64 ns = resp->timestamp;
2243 
2244                 netif_info(lio, tx_done, lio->netdev,
2245                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2246                            skb, (unsigned long long)ns);
2247                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2248                 skb_tstamp_tx(skb, &ts);
2249         }
2250 
2251         octeon_free_soft_command(oct, sc);
2252         tx_buffer_free(skb);
2253 }
2254 
2255 /* \brief Send a data packet that will be timestamped
2256  * @param oct octeon device
2257  * @param ndata pointer to network data
2258  * @param finfo pointer to private network data
2259  */
2260 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2261                                          struct octnic_data_pkt *ndata,
2262                                          struct octnet_buf_free_info *finfo,
2263                                          int xmit_more)
2264 {
2265         int retval;
2266         struct octeon_soft_command *sc;
2267         struct lio *lio;
2268         int ring_doorbell;
2269         u32 len;
2270 
2271         lio = finfo->lio;
2272 
2273         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2274                                             sizeof(struct oct_timestamp_resp));
2275         finfo->sc = sc;
2276 
2277         if (!sc) {
2278                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2279                 return IQ_SEND_FAILED;
2280         }
2281 
2282         if (ndata->reqtype == REQTYPE_NORESP_NET)
2283                 ndata->reqtype = REQTYPE_RESP_NET;
2284         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2285                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2286 
2287         sc->callback = handle_timestamp;
2288         sc->callback_arg = finfo->skb;
2289         sc->iq_no = ndata->q_no;
2290 
2291         if (OCTEON_CN23XX_PF(oct))
2292                 len = (u32)((struct octeon_instr_ih3 *)
2293                             (&sc->cmd.cmd3.ih3))->dlengsz;
2294         else
2295                 len = (u32)((struct octeon_instr_ih2 *)
2296                             (&sc->cmd.cmd2.ih2))->dlengsz;
2297 
2298         ring_doorbell = !xmit_more;
2299 
2300         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2301                                      sc, len, ndata->reqtype);
2302 
2303         if (retval == IQ_SEND_FAILED) {
2304                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2305                         retval);
2306                 octeon_free_soft_command(oct, sc);
2307         } else {
2308                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2309         }
2310 
2311         return retval;
2312 }
2313 
2314 /** \brief Transmit networks packets to the Octeon interface
2315  * @param skbuff   skbuff struct to be passed to network layer.
2316  * @param netdev    pointer to network device
2317  * @returns whether the packet was transmitted to the device okay or not
2318  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2319  */
2320 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2321 {
2322         struct lio *lio;
2323         struct octnet_buf_free_info *finfo;
2324         union octnic_cmd_setup cmdsetup;
2325         struct octnic_data_pkt ndata;
2326         struct octeon_device *oct;
2327         struct oct_iq_stats *stats;
2328         struct octeon_instr_irh *irh;
2329         union tx_info *tx_info;
2330         int status = 0;
2331         int q_idx = 0, iq_no = 0;
2332         int j, xmit_more = 0;
2333         u64 dptr = 0;
2334         u32 tag = 0;
2335 
2336         lio = GET_LIO(netdev);
2337         oct = lio->oct_dev;
2338 
2339         q_idx = skb_iq(oct, skb);
2340         tag = q_idx;
2341         iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2342 
2343         stats = &oct->instr_queue[iq_no]->stats;
2344 
2345         /* Check for all conditions in which the current packet cannot be
2346          * transmitted.
2347          */
2348         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2349             (!lio->linfo.link.s.link_up) ||
2350             (skb->len <= 0)) {
2351                 netif_info(lio, tx_err, lio->netdev,
2352                            "Transmit failed link_status : %d\n",
2353                            lio->linfo.link.s.link_up);
2354                 goto lio_xmit_failed;
2355         }
2356 
2357         /* Use space in skb->cb to store info used to unmap and
2358          * free the buffers.
2359          */
2360         finfo = (struct octnet_buf_free_info *)skb->cb;
2361         finfo->lio = lio;
2362         finfo->skb = skb;
2363         finfo->sc = NULL;
2364 
2365         /* Prepare the attributes for the data to be passed to OSI. */
2366         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2367 
2368         ndata.buf = (void *)finfo;
2369 
2370         ndata.q_no = iq_no;
2371 
2372         if (octnet_iq_is_full(oct, ndata.q_no)) {
2373                 /* defer sending if queue is full */
2374                 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2375                            ndata.q_no);
2376                 stats->tx_iq_busy++;
2377                 return NETDEV_TX_BUSY;
2378         }
2379 
2380         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2381          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2382          */
2383 
2384         ndata.datasize = skb->len;
2385 
2386         cmdsetup.u64 = 0;
2387         cmdsetup.s.iq_no = iq_no;
2388 
2389         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2390                 if (skb->encapsulation) {
2391                         cmdsetup.s.tnl_csum = 1;
2392                         stats->tx_vxlan++;
2393                 } else {
2394                         cmdsetup.s.transport_csum = 1;
2395                 }
2396         }
2397         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2398                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2399                 cmdsetup.s.timestamp = 1;
2400         }
2401 
2402         if (skb_shinfo(skb)->nr_frags == 0) {
2403                 cmdsetup.s.u.datasize = skb->len;
2404                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2405 
2406                 /* Offload checksum calculation for TCP/UDP packets */
2407                 dptr = dma_map_single(&oct->pci_dev->dev,
2408                                       skb->data,
2409                                       skb->len,
2410                                       DMA_TO_DEVICE);
2411                 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2412                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2413                                 __func__);
2414                         stats->tx_dmamap_fail++;
2415                         return NETDEV_TX_BUSY;
2416                 }
2417 
2418                 if (OCTEON_CN23XX_PF(oct))
2419                         ndata.cmd.cmd3.dptr = dptr;
2420                 else
2421                         ndata.cmd.cmd2.dptr = dptr;
2422                 finfo->dptr = dptr;
2423                 ndata.reqtype = REQTYPE_NORESP_NET;
2424 
2425         } else {
2426                 int i, frags;
2427                 skb_frag_t *frag;
2428                 struct octnic_gather *g;
2429 
2430                 spin_lock(&lio->glist_lock[q_idx]);
2431                 g = (struct octnic_gather *)
2432                         lio_list_delete_head(&lio->glist[q_idx]);
2433                 spin_unlock(&lio->glist_lock[q_idx]);
2434 
2435                 if (!g) {
2436                         netif_info(lio, tx_err, lio->netdev,
2437                                    "Transmit scatter gather: glist null!\n");
2438                         goto lio_xmit_failed;
2439                 }
2440 
2441                 cmdsetup.s.gather = 1;
2442                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2443                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2444 
2445                 memset(g->sg, 0, g->sg_size);
2446 
2447                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2448                                                  skb->data,
2449                                                  (skb->len - skb->data_len),
2450                                                  DMA_TO_DEVICE);
2451                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2452                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2453                                 __func__);
2454                         stats->tx_dmamap_fail++;
2455                         return NETDEV_TX_BUSY;
2456                 }
2457                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2458 
2459                 frags = skb_shinfo(skb)->nr_frags;
2460                 i = 1;
2461                 while (frags--) {
2462                         frag = &skb_shinfo(skb)->frags[i - 1];
2463 
2464                         g->sg[(i >> 2)].ptr[(i & 3)] =
2465                                 skb_frag_dma_map(&oct->pci_dev->dev,
2466                                                  frag, 0, skb_frag_size(frag),
2467                                                  DMA_TO_DEVICE);
2468 
2469                         if (dma_mapping_error(&oct->pci_dev->dev,
2470                                               g->sg[i >> 2].ptr[i & 3])) {
2471                                 dma_unmap_single(&oct->pci_dev->dev,
2472                                                  g->sg[0].ptr[0],
2473                                                  skb->len - skb->data_len,
2474                                                  DMA_TO_DEVICE);
2475                                 for (j = 1; j < i; j++) {
2476                                         frag = &skb_shinfo(skb)->frags[j - 1];
2477                                         dma_unmap_page(&oct->pci_dev->dev,
2478                                                        g->sg[j >> 2].ptr[j & 3],
2479                                                        skb_frag_size(frag),
2480                                                        DMA_TO_DEVICE);
2481                                 }
2482                                 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2483                                         __func__);
2484                                 return NETDEV_TX_BUSY;
2485                         }
2486 
2487                         add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),
2488                                     (i & 3));
2489                         i++;
2490                 }
2491 
2492                 dptr = g->sg_dma_ptr;
2493 
2494                 if (OCTEON_CN23XX_PF(oct))
2495                         ndata.cmd.cmd3.dptr = dptr;
2496                 else
2497                         ndata.cmd.cmd2.dptr = dptr;
2498                 finfo->dptr = dptr;
2499                 finfo->g = g;
2500 
2501                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2502         }
2503 
2504         if (OCTEON_CN23XX_PF(oct)) {
2505                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
2506                 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
2507         } else {
2508                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2509                 tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2510         }
2511 
2512         if (skb_shinfo(skb)->gso_size) {
2513                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2514                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2515                 stats->tx_gso++;
2516         }
2517 
2518         /* HW insert VLAN tag */
2519         if (skb_vlan_tag_present(skb)) {
2520                 irh->priority = skb_vlan_tag_get(skb) >> 13;
2521                 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2522         }
2523 
2524         xmit_more = netdev_xmit_more();
2525 
2526         if (unlikely(cmdsetup.s.timestamp))
2527                 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2528         else
2529                 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2530         if (status == IQ_SEND_FAILED)
2531                 goto lio_xmit_failed;
2532 
2533         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2534 
2535         if (status == IQ_SEND_STOP)
2536                 netif_stop_subqueue(netdev, q_idx);
2537 
2538         netif_trans_update(netdev);
2539 
2540         if (tx_info->s.gso_segs)
2541                 stats->tx_done += tx_info->s.gso_segs;
2542         else
2543                 stats->tx_done++;
2544         stats->tx_tot_bytes += ndata.datasize;
2545 
2546         return NETDEV_TX_OK;
2547 
2548 lio_xmit_failed:
2549         stats->tx_dropped++;
2550         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2551                    iq_no, stats->tx_dropped);
2552         if (dptr)
2553                 dma_unmap_single(&oct->pci_dev->dev, dptr,
2554                                  ndata.datasize, DMA_TO_DEVICE);
2555 
2556         octeon_ring_doorbell_locked(oct, iq_no);
2557 
2558         tx_buffer_free(skb);
2559         return NETDEV_TX_OK;
2560 }
2561 
2562 /** \brief Network device Tx timeout
2563  * @param netdev    pointer to network device
2564  */
2565 static void liquidio_tx_timeout(struct net_device *netdev)
2566 {
2567         struct lio *lio;
2568 
2569         lio = GET_LIO(netdev);
2570 
2571         netif_info(lio, tx_err, lio->netdev,
2572                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2573                    netdev->stats.tx_dropped);
2574         netif_trans_update(netdev);
2575         wake_txqs(netdev);
2576 }
2577 
2578 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2579                                     __be16 proto __attribute__((unused)),
2580                                     u16 vid)
2581 {
2582         struct lio *lio = GET_LIO(netdev);
2583         struct octeon_device *oct = lio->oct_dev;
2584         struct octnic_ctrl_pkt nctrl;
2585         int ret = 0;
2586 
2587         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2588 
2589         nctrl.ncmd.u64 = 0;
2590         nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2591         nctrl.ncmd.s.param1 = vid;
2592         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2593         nctrl.netpndev = (u64)netdev;
2594         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2595 
2596         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2597         if (ret) {
2598                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2599                         ret);
2600                 if (ret > 0)
2601                         ret = -EIO;
2602         }
2603 
2604         return ret;
2605 }
2606 
2607 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2608                                      __be16 proto __attribute__((unused)),
2609                                      u16 vid)
2610 {
2611         struct lio *lio = GET_LIO(netdev);
2612         struct octeon_device *oct = lio->oct_dev;
2613         struct octnic_ctrl_pkt nctrl;
2614         int ret = 0;
2615 
2616         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2617 
2618         nctrl.ncmd.u64 = 0;
2619         nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2620         nctrl.ncmd.s.param1 = vid;
2621         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2622         nctrl.netpndev = (u64)netdev;
2623         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2624 
2625         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2626         if (ret) {
2627                 dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
2628                         ret);
2629                 if (ret > 0)
2630                         ret = -EIO;
2631         }
2632         return ret;
2633 }
2634 
2635 /** Sending command to enable/disable RX checksum offload
2636  * @param netdev                pointer to network device
2637  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
2638  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
2639  *                              OCTNET_CMD_RXCSUM_DISABLE
2640  * @returns                     SUCCESS or FAILURE
2641  */
2642 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2643                                        u8 rx_cmd)
2644 {
2645         struct lio *lio = GET_LIO(netdev);
2646         struct octeon_device *oct = lio->oct_dev;
2647         struct octnic_ctrl_pkt nctrl;
2648         int ret = 0;
2649 
2650         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2651 
2652         nctrl.ncmd.u64 = 0;
2653         nctrl.ncmd.s.cmd = command;
2654         nctrl.ncmd.s.param1 = rx_cmd;
2655         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2656         nctrl.netpndev = (u64)netdev;
2657         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2658 
2659         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2660         if (ret) {
2661                 dev_err(&oct->pci_dev->dev,
2662                         "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2663                         ret);
2664                 if (ret > 0)
2665                         ret = -EIO;
2666         }
2667         return ret;
2668 }
2669 
2670 /** Sending command to add/delete VxLAN UDP port to firmware
2671  * @param netdev                pointer to network device
2672  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
2673  * @param vxlan_port            VxLAN port to be added or deleted
2674  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
2675  *                              OCTNET_CMD_VXLAN_PORT_DEL
2676  * @returns                     SUCCESS or FAILURE
2677  */
2678 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2679                                        u16 vxlan_port, u8 vxlan_cmd_bit)
2680 {
2681         struct lio *lio = GET_LIO(netdev);
2682         struct octeon_device *oct = lio->oct_dev;
2683         struct octnic_ctrl_pkt nctrl;
2684         int ret = 0;
2685 
2686         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2687 
2688         nctrl.ncmd.u64 = 0;
2689         nctrl.ncmd.s.cmd = command;
2690         nctrl.ncmd.s.more = vxlan_cmd_bit;
2691         nctrl.ncmd.s.param1 = vxlan_port;
2692         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2693         nctrl.netpndev = (u64)netdev;
2694         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2695 
2696         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2697         if (ret) {
2698                 dev_err(&oct->pci_dev->dev,
2699                         "VxLAN port add/delete failed in core (ret:0x%x)\n",
2700                         ret);
2701                 if (ret > 0)
2702                         ret = -EIO;
2703         }
2704         return ret;
2705 }
2706 
2707 /** \brief Net device fix features
2708  * @param netdev  pointer to network device
2709  * @param request features requested
2710  * @returns updated features list
2711  */
2712 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2713                                                netdev_features_t request)
2714 {
2715         struct lio *lio = netdev_priv(netdev);
2716 
2717         if ((request & NETIF_F_RXCSUM) &&
2718             !(lio->dev_capability & NETIF_F_RXCSUM))
2719                 request &= ~NETIF_F_RXCSUM;
2720 
2721         if ((request & NETIF_F_HW_CSUM) &&
2722             !(lio->dev_capability & NETIF_F_HW_CSUM))
2723                 request &= ~NETIF_F_HW_CSUM;
2724 
2725         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2726                 request &= ~NETIF_F_TSO;
2727 
2728         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2729                 request &= ~NETIF_F_TSO6;
2730 
2731         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2732                 request &= ~NETIF_F_LRO;
2733 
2734         /*Disable LRO if RXCSUM is off */
2735         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2736             (lio->dev_capability & NETIF_F_LRO))
2737                 request &= ~NETIF_F_LRO;
2738 
2739         if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2740             !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
2741                 request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
2742 
2743         return request;
2744 }
2745 
2746 /** \brief Net device set features
2747  * @param netdev  pointer to network device
2748  * @param features features to enable/disable
2749  */
2750 static int liquidio_set_features(struct net_device *netdev,
2751                                  netdev_features_t features)
2752 {
2753         struct lio *lio = netdev_priv(netdev);
2754 
2755         if ((features & NETIF_F_LRO) &&
2756             (lio->dev_capability & NETIF_F_LRO) &&
2757             !(netdev->features & NETIF_F_LRO))
2758                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2759                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2760         else if (!(features & NETIF_F_LRO) &&
2761                  (lio->dev_capability & NETIF_F_LRO) &&
2762                  (netdev->features & NETIF_F_LRO))
2763                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
2764                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2765 
2766         /* Sending command to firmware to enable/disable RX checksum
2767          * offload settings using ethtool
2768          */
2769         if (!(netdev->features & NETIF_F_RXCSUM) &&
2770             (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2771             (features & NETIF_F_RXCSUM))
2772                 liquidio_set_rxcsum_command(netdev,
2773                                             OCTNET_CMD_TNL_RX_CSUM_CTL,
2774                                             OCTNET_CMD_RXCSUM_ENABLE);
2775         else if ((netdev->features & NETIF_F_RXCSUM) &&
2776                  (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2777                  !(features & NETIF_F_RXCSUM))
2778                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2779                                             OCTNET_CMD_RXCSUM_DISABLE);
2780 
2781         if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2782             (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2783             !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2784                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2785                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
2786         else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2787                  (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2788                  (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2789                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2790                                      OCTNET_CMD_VLAN_FILTER_DISABLE);
2791 
2792         return 0;
2793 }
2794 
2795 static void liquidio_add_vxlan_port(struct net_device *netdev,
2796                                     struct udp_tunnel_info *ti)
2797 {
2798         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2799                 return;
2800 
2801         liquidio_vxlan_port_command(netdev,
2802                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
2803                                     htons(ti->port),
2804                                     OCTNET_CMD_VXLAN_PORT_ADD);
2805 }
2806 
2807 static void liquidio_del_vxlan_port(struct net_device *netdev,
2808                                     struct udp_tunnel_info *ti)
2809 {
2810         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2811                 return;
2812 
2813         liquidio_vxlan_port_command(netdev,
2814                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
2815                                     htons(ti->port),
2816                                     OCTNET_CMD_VXLAN_PORT_DEL);
2817 }
2818 
2819 static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
2820                                  u8 *mac, bool is_admin_assigned)
2821 {
2822         struct lio *lio = GET_LIO(netdev);
2823         struct octeon_device *oct = lio->oct_dev;
2824         struct octnic_ctrl_pkt nctrl;
2825         int ret = 0;
2826 
2827         if (!is_valid_ether_addr(mac))
2828                 return -EINVAL;
2829 
2830         if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
2831                 return -EINVAL;
2832 
2833         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2834 
2835         nctrl.ncmd.u64 = 0;
2836         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2837         /* vfidx is 0 based, but vf_num (param1) is 1 based */
2838         nctrl.ncmd.s.param1 = vfidx + 1;
2839         nctrl.ncmd.s.more = 1;
2840         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2841         nctrl.netpndev = (u64)netdev;
2842         if (is_admin_assigned) {
2843                 nctrl.ncmd.s.param2 = true;
2844                 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2845         }
2846 
2847         nctrl.udd[0] = 0;
2848         /* The MAC Address is presented in network byte order. */
2849         ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
2850 
2851         oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
2852 
2853         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2854         if (ret > 0)
2855                 ret = -EIO;
2856 
2857         return ret;
2858 }
2859 
2860 static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
2861 {
2862         struct lio *lio = GET_LIO(netdev);
2863         struct octeon_device *oct = lio->oct_dev;
2864         int retval;
2865 
2866         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2867                 return -EINVAL;
2868 
2869         retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
2870         if (!retval)
2871                 cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
2872 
2873         return retval;
2874 }
2875 
2876 static int liquidio_set_vf_spoofchk(struct net_device *netdev, int vfidx,
2877                                     bool enable)
2878 {
2879         struct lio *lio = GET_LIO(netdev);
2880         struct octeon_device *oct = lio->oct_dev;
2881         struct octnic_ctrl_pkt nctrl;
2882         int retval;
2883 
2884         if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SPOOFCHK_CAP)) {
2885                 netif_info(lio, drv, lio->netdev,
2886                            "firmware does not support spoofchk\n");
2887                 return -EOPNOTSUPP;
2888         }
2889 
2890         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
2891                 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
2892                 return -EINVAL;
2893         }
2894 
2895         if (enable) {
2896                 if (oct->sriov_info.vf_spoofchk[vfidx])
2897                         return 0;
2898         } else {
2899                 /* Clear */
2900                 if (!oct->sriov_info.vf_spoofchk[vfidx])
2901                         return 0;
2902         }
2903 
2904         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2905         nctrl.ncmd.s.cmdgroup = OCTNET_CMD_GROUP1;
2906         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_SPOOFCHK;
2907         nctrl.ncmd.s.param1 =
2908                 vfidx + 1; /* vfidx is 0 based,
2909                             * but vf_num (param1) is 1 based
2910                             */
2911         nctrl.ncmd.s.param2 = enable;
2912         nctrl.ncmd.s.more = 0;
2913         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2914         nctrl.cb_fn = NULL;
2915 
2916         retval = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2917 
2918         if (retval) {
2919                 netif_info(lio, drv, lio->netdev,
2920                            "Failed to set VF %d spoofchk %s\n", vfidx,
2921                         enable ? "on" : "off");
2922                 return -1;
2923         }
2924 
2925         oct->sriov_info.vf_spoofchk[vfidx] = enable;
2926         netif_info(lio, drv, lio->netdev, "VF %u spoofchk is %s\n", vfidx,
2927                    enable ? "on" : "off");
2928 
2929         return 0;
2930 }
2931 
2932 static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
2933                                 u16 vlan, u8 qos, __be16 vlan_proto)
2934 {
2935         struct lio *lio = GET_LIO(netdev);
2936         struct octeon_device *oct = lio->oct_dev;
2937         struct octnic_ctrl_pkt nctrl;
2938         u16 vlantci;
2939         int ret = 0;
2940 
2941         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2942                 return -EINVAL;
2943 
2944         if (vlan_proto != htons(ETH_P_8021Q))
2945                 return -EPROTONOSUPPORT;
2946 
2947         if (vlan >= VLAN_N_VID || qos > 7)
2948                 return -EINVAL;
2949 
2950         if (vlan)
2951                 vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
2952         else
2953                 vlantci = 0;
2954 
2955         if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
2956                 return 0;
2957 
2958         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2959 
2960         if (vlan)
2961                 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2962         else
2963                 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2964 
2965         nctrl.ncmd.s.param1 = vlantci;
2966         nctrl.ncmd.s.param2 =
2967             vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
2968         nctrl.ncmd.s.more = 0;
2969         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2970         nctrl.cb_fn = NULL;
2971 
2972         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2973         if (ret) {
2974                 if (ret > 0)
2975                         ret = -EIO;
2976                 return ret;
2977         }
2978 
2979         oct->sriov_info.vf_vlantci[vfidx] = vlantci;
2980 
2981         return ret;
2982 }
2983 
2984 static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
2985                                   struct ifla_vf_info *ivi)
2986 {
2987         struct lio *lio = GET_LIO(netdev);
2988         struct octeon_device *oct = lio->oct_dev;
2989         u8 *macaddr;
2990 
2991         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2992                 return -EINVAL;
2993 
2994         memset(ivi, 0, sizeof(struct ifla_vf_info));
2995 
2996         ivi->vf = vfidx;
2997         macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
2998         ether_addr_copy(&ivi->mac[0], macaddr);
2999         ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
3000         ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
3001         if (oct->sriov_info.trusted_vf.active &&
3002             oct->sriov_info.trusted_vf.id == vfidx)
3003                 ivi->trusted = true;
3004         else
3005                 ivi->trusted = false;
3006         ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
3007         ivi->spoofchk = oct->sriov_info.vf_spoofchk[vfidx];
3008         ivi->max_tx_rate = lio->linfo.link.s.speed;
3009         ivi->min_tx_rate = 0;
3010 
3011         return 0;
3012 }
3013 
3014 static int liquidio_send_vf_trust_cmd(struct lio *lio, int vfidx, bool trusted)
3015 {
3016         struct octeon_device *oct = lio->oct_dev;
3017         struct octeon_soft_command *sc;
3018         int retval;
3019 
3020         sc = octeon_alloc_soft_command(oct, 0, 16, 0);
3021         if (!sc)
3022                 return -ENOMEM;
3023 
3024         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
3025 
3026         /* vfidx is 0 based, but vf_num (param1) is 1 based */
3027         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
3028                                     OPCODE_NIC_SET_TRUSTED_VF, 0, vfidx + 1,
3029                                     trusted);
3030 
3031         init_completion(&sc->complete);
3032         sc->sc_status = OCTEON_REQUEST_PENDING;
3033 
3034         retval = octeon_send_soft_command(oct, sc);
3035         if (retval == IQ_SEND_FAILED) {
3036                 octeon_free_soft_command(oct, sc);
3037                 retval = -1;
3038         } else {
3039                 /* Wait for response or timeout */
3040                 retval = wait_for_sc_completion_timeout(oct, sc, 0);
3041                 if (retval)
3042                         return (retval);
3043 
3044                 WRITE_ONCE(sc->caller_is_done, true);
3045         }
3046 
3047         return retval;
3048 }
3049 
3050 static int liquidio_set_vf_trust(struct net_device *netdev, int vfidx,
3051                                  bool setting)
3052 {
3053         struct lio *lio = GET_LIO(netdev);
3054         struct octeon_device *oct = lio->oct_dev;
3055 
3056         if (strcmp(oct->fw_info.liquidio_firmware_version, "1.7.1") < 0) {
3057                 /* trusted vf is not supported by firmware older than 1.7.1 */
3058                 return -EOPNOTSUPP;
3059         }
3060 
3061         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
3062                 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
3063                 return -EINVAL;
3064         }
3065 
3066         if (setting) {
3067                 /* Set */
3068 
3069                 if (oct->sriov_info.trusted_vf.active &&
3070                     oct->sriov_info.trusted_vf.id == vfidx)
3071                         return 0;
3072 
3073                 if (oct->sriov_info.trusted_vf.active) {
3074                         netif_info(lio, drv, lio->netdev, "More than one trusted VF is not allowed\n");
3075                         return -EPERM;
3076                 }
3077         } else {
3078                 /* Clear */
3079 
3080                 if (!oct->sriov_info.trusted_vf.active)
3081                         return 0;
3082         }
3083 
3084         if (!liquidio_send_vf_trust_cmd(lio, vfidx, setting)) {
3085                 if (setting) {
3086                         oct->sriov_info.trusted_vf.id = vfidx;
3087                         oct->sriov_info.trusted_vf.active = true;
3088                 } else {
3089                         oct->sriov_info.trusted_vf.active = false;
3090                 }
3091 
3092                 netif_info(lio, drv, lio->netdev, "VF %u is %strusted\n", vfidx,
3093                            setting ? "" : "not ");
3094         } else {
3095                 netif_info(lio, drv, lio->netdev, "Failed to set VF trusted\n");
3096                 return -1;
3097         }
3098 
3099         return 0;
3100 }
3101 
3102 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3103                                       int linkstate)
3104 {
3105         struct lio *lio = GET_LIO(netdev);
3106         struct octeon_device *oct = lio->oct_dev;
3107         struct octnic_ctrl_pkt nctrl;
3108         int ret = 0;
3109 
3110         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3111                 return -EINVAL;
3112 
3113         if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3114                 return 0;
3115 
3116         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3117         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3118         nctrl.ncmd.s.param1 =
3119             vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3120         nctrl.ncmd.s.param2 = linkstate;
3121         nctrl.ncmd.s.more = 0;
3122         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3123         nctrl.cb_fn = NULL;
3124 
3125         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
3126 
3127         if (!ret)
3128                 oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3129         else if (ret > 0)
3130                 ret = -EIO;
3131 
3132         return ret;
3133 }
3134 
3135 static int
3136 liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3137 {
3138         struct lio_devlink_priv *priv;
3139         struct octeon_device *oct;
3140 
3141         priv = devlink_priv(devlink);
3142         oct = priv->oct;
3143 
3144         *mode = oct->eswitch_mode;
3145 
3146         return 0;
3147 }
3148 
3149 static int
3150 liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode,
3151                           struct netlink_ext_ack *extack)
3152 {
3153         struct lio_devlink_priv *priv;
3154         struct octeon_device *oct;
3155         int ret = 0;
3156 
3157         priv = devlink_priv(devlink);
3158         oct = priv->oct;
3159 
3160         if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))
3161                 return -EINVAL;
3162 
3163         if (oct->eswitch_mode == mode)
3164                 return 0;
3165 
3166         switch (mode) {
3167         case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3168                 oct->eswitch_mode = mode;
3169                 ret = lio_vf_rep_create(oct);
3170                 break;
3171 
3172         case DEVLINK_ESWITCH_MODE_LEGACY:
3173                 lio_vf_rep_destroy(oct);
3174                 oct->eswitch_mode = mode;
3175                 break;
3176 
3177         default:
3178                 ret = -EINVAL;
3179         }
3180 
3181         return ret;
3182 }
3183 
3184 static const struct devlink_ops liquidio_devlink_ops = {
3185         .eswitch_mode_get = liquidio_eswitch_mode_get,
3186         .eswitch_mode_set = liquidio_eswitch_mode_set,
3187 };
3188 
3189 static int
3190 liquidio_get_port_parent_id(struct net_device *dev,
3191                             struct netdev_phys_item_id *ppid)
3192 {
3193         struct lio *lio = GET_LIO(dev);
3194         struct octeon_device *oct = lio->oct_dev;
3195 
3196         if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
3197                 return -EOPNOTSUPP;
3198 
3199         ppid->id_len = ETH_ALEN;
3200         ether_addr_copy(ppid->id, (void *)&lio->linfo.hw_addr + 2);
3201 
3202         return 0;
3203 }
3204 
3205 static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,
3206                                  struct ifla_vf_stats *vf_stats)
3207 {
3208         struct lio *lio = GET_LIO(netdev);
3209         struct octeon_device *oct = lio->oct_dev;
3210         struct oct_vf_stats stats;
3211         int ret;
3212 
3213         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3214                 return -EINVAL;
3215 
3216         memset(&stats, 0, sizeof(struct oct_vf_stats));
3217         ret = cn23xx_get_vf_stats(oct, vfidx, &stats);
3218         if (!ret) {
3219                 vf_stats->rx_packets = stats.rx_packets;
3220                 vf_stats->tx_packets = stats.tx_packets;
3221                 vf_stats->rx_bytes = stats.rx_bytes;
3222                 vf_stats->tx_bytes = stats.tx_bytes;
3223                 vf_stats->broadcast = stats.broadcast;
3224                 vf_stats->multicast = stats.multicast;
3225         }
3226 
3227         return ret;
3228 }
3229 
3230 static const struct net_device_ops lionetdevops = {
3231         .ndo_open               = liquidio_open,
3232         .ndo_stop               = liquidio_stop,
3233         .ndo_start_xmit         = liquidio_xmit,
3234         .ndo_get_stats64        = liquidio_get_stats64,
3235         .ndo_set_mac_address    = liquidio_set_mac,
3236         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3237         .ndo_tx_timeout         = liquidio_tx_timeout,
3238 
3239         .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3240         .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3241         .ndo_change_mtu         = liquidio_change_mtu,
3242         .ndo_do_ioctl           = liquidio_ioctl,
3243         .ndo_fix_features       = liquidio_fix_features,
3244         .ndo_set_features       = liquidio_set_features,
3245         .ndo_udp_tunnel_add     = liquidio_add_vxlan_port,
3246         .ndo_udp_tunnel_del     = liquidio_del_vxlan_port,
3247         .ndo_set_vf_mac         = liquidio_set_vf_mac,
3248         .ndo_set_vf_vlan        = liquidio_set_vf_vlan,
3249         .ndo_get_vf_config      = liquidio_get_vf_config,
3250         .ndo_set_vf_spoofchk    = liquidio_set_vf_spoofchk,
3251         .ndo_set_vf_trust       = liquidio_set_vf_trust,
3252         .ndo_set_vf_link_state  = liquidio_set_vf_link_state,
3253         .ndo_get_vf_stats       = liquidio_get_vf_stats,
3254         .ndo_get_port_parent_id = liquidio_get_port_parent_id,
3255 };
3256 
3257 /** \brief Entry point for the liquidio module
3258  */
3259 static int __init liquidio_init(void)
3260 {
3261         int i;
3262         struct handshake *hs;
3263 
3264         init_completion(&first_stage);
3265 
3266         octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
3267 
3268         if (liquidio_init_pci())
3269                 return -EINVAL;
3270 
3271         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3272 
3273         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3274                 hs = &handshake[i];
3275                 if (hs->pci_dev) {
3276                         wait_for_completion(&hs->init);
3277                         if (!hs->init_ok) {
3278                                 /* init handshake failed */
3279                                 dev_err(&hs->pci_dev->dev,
3280                                         "Failed to init device\n");
3281                                 liquidio_deinit_pci();
3282                                 return -EIO;
3283                         }
3284                 }
3285         }
3286 
3287         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3288                 hs = &handshake[i];
3289                 if (hs->pci_dev) {
3290                         wait_for_completion_timeout(&hs->started,
3291                                                     msecs_to_jiffies(30000));
3292                         if (!hs->started_ok) {
3293                                 /* starter handshake failed */
3294                                 dev_err(&hs->pci_dev->dev,
3295                                         "Firmware failed to start\n");
3296                                 liquidio_deinit_pci();
3297                                 return -EIO;
3298                         }
3299                 }
3300         }
3301 
3302         return 0;
3303 }
3304 
3305 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3306 {
3307         struct octeon_device *oct = (struct octeon_device *)buf;
3308         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3309         int gmxport = 0;
3310         union oct_link_status *ls;
3311         int i;
3312 
3313         if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
3314                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3315                         recv_pkt->buffer_size[0],
3316                         recv_pkt->rh.r_nic_info.gmxport);
3317                 goto nic_info_err;
3318         }
3319 
3320         gmxport = recv_pkt->rh.r_nic_info.gmxport;
3321         ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3322                 OCT_DROQ_INFO_SIZE);
3323 
3324         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3325         for (i = 0; i < oct->ifcount; i++) {
3326                 if (oct->props[i].gmxport == gmxport) {
3327                         update_link_status(oct->props[i].netdev, ls);
3328                         break;
3329                 }
3330         }
3331 
3332 nic_info_err:
3333         for (i = 0; i < recv_pkt->buffer_count; i++)
3334                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3335         octeon_free_recv_info(recv_info);
3336         return 0;
3337 }
3338 
3339 /**
3340  * \brief Setup network interfaces
3341  * @param octeon_dev  octeon device
3342  *
3343  * Called during init time for each device. It assumes the NIC
3344  * is already up and running.  The link information for each
3345  * interface is passed in link_info.
3346  */
3347 static int setup_nic_devices(struct octeon_device *octeon_dev)
3348 {
3349         struct lio *lio = NULL;
3350         struct net_device *netdev;
3351         u8 mac[6], i, j, *fw_ver, *micro_ver;
3352         unsigned long micro;
3353         u32 cur_ver;
3354         struct octeon_soft_command *sc;
3355         struct liquidio_if_cfg_resp *resp;
3356         struct octdev_props *props;
3357         int retval, num_iqueues, num_oqueues;
3358         int max_num_queues = 0;
3359         union oct_nic_if_cfg if_cfg;
3360         unsigned int base_queue;
3361         unsigned int gmx_port_id;
3362         u32 resp_size, data_size;
3363         u32 ifidx_or_pfnum;
3364         struct lio_version *vdata;
3365         struct devlink *devlink;
3366         struct lio_devlink_priv *lio_devlink;
3367 
3368         /* This is to handle link status changes */
3369         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3370                                     OPCODE_NIC_INFO,
3371                                     lio_nic_info, octeon_dev);
3372 
3373         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3374          * They are handled directly.
3375          */
3376         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3377                                         free_netbuf);
3378 
3379         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3380                                         free_netsgbuf);
3381 
3382         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3383                                         free_netsgbuf_with_resp);
3384 
3385         for (i = 0; i < octeon_dev->ifcount; i++) {
3386                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3387                 data_size = sizeof(struct lio_version);
3388                 sc = (struct octeon_soft_command *)
3389                         octeon_alloc_soft_command(octeon_dev, data_size,
3390                                                   resp_size, 0);
3391                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3392                 vdata = (struct lio_version *)sc->virtdptr;
3393 
3394                 *((u64 *)vdata) = 0;
3395                 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3396                 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3397                 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3398 
3399                 if (OCTEON_CN23XX_PF(octeon_dev)) {
3400                         num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3401                         num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3402                         base_queue = octeon_dev->sriov_info.pf_srn;
3403 
3404                         gmx_port_id = octeon_dev->pf_num;
3405                         ifidx_or_pfnum = octeon_dev->pf_num;
3406                 } else {
3407                         num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3408                                                 octeon_get_conf(octeon_dev), i);
3409                         num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3410                                                 octeon_get_conf(octeon_dev), i);
3411                         base_queue = CFG_GET_BASE_QUE_NIC_IF(
3412                                                 octeon_get_conf(octeon_dev), i);
3413                         gmx_port_id = CFG_GET_GMXID_NIC_IF(
3414                                                 octeon_get_conf(octeon_dev), i);
3415                         ifidx_or_pfnum = i;
3416                 }
3417 
3418                 dev_dbg(&octeon_dev->pci_dev->dev,
3419                         "requesting config for interface %d, iqs %d, oqs %d\n",
3420                         ifidx_or_pfnum, num_iqueues, num_oqueues);
3421 
3422                 if_cfg.u64 = 0;
3423                 if_cfg.s.num_iqueues = num_iqueues;
3424                 if_cfg.s.num_oqueues = num_oqueues;
3425                 if_cfg.s.base_queue = base_queue;
3426                 if_cfg.s.gmx_port_id = gmx_port_id;
3427 
3428                 sc->iq_no = 0;
3429 
3430                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3431                                             OPCODE_NIC_IF_CFG, 0,
3432                                             if_cfg.u64, 0);
3433 
3434                 init_completion(&sc->complete);
3435                 sc->sc_status = OCTEON_REQUEST_PENDING;
3436 
3437                 retval = octeon_send_soft_command(octeon_dev, sc);
3438                 if (retval == IQ_SEND_FAILED) {
3439                         dev_err(&octeon_dev->pci_dev->dev,
3440                                 "iq/oq config failed status: %x\n",
3441                                 retval);
3442                         /* Soft instr is freed by driver in case of failure. */
3443                         octeon_free_soft_command(octeon_dev, sc);
3444                         return(-EIO);
3445                 }
3446 
3447                 /* Sleep on a wait queue till the cond flag indicates that the
3448                  * response arrived or timed-out.
3449                  */
3450                 retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
3451                 if (retval)
3452                         return retval;
3453 
3454                 retval = resp->status;
3455                 if (retval) {
3456                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3457                         WRITE_ONCE(sc->caller_is_done, true);
3458                         goto setup_nic_dev_done;
3459                 }
3460                 snprintf(octeon_dev->fw_info.liquidio_firmware_version,
3461                          32, "%s",
3462                          resp->cfg_info.liquidio_firmware_version);
3463 
3464                 /* Verify f/w version (in case of 'auto' loading from flash) */
3465                 fw_ver = octeon_dev->fw_info.liquidio_firmware_version;
3466                 if (memcmp(LIQUIDIO_BASE_VERSION,
3467                            fw_ver,
3468                            strlen(LIQUIDIO_BASE_VERSION))) {
3469                         dev_err(&octeon_dev->pci_dev->dev,
3470                                 "Unmatched firmware version. Expected %s.x, got %s.\n",
3471                                 LIQUIDIO_BASE_VERSION, fw_ver);
3472                         WRITE_ONCE(sc->caller_is_done, true);
3473                         goto setup_nic_dev_done;
3474                 } else if (atomic_read(octeon_dev->adapter_fw_state) ==
3475                            FW_IS_PRELOADED) {
3476                         dev_info(&octeon_dev->pci_dev->dev,
3477                                  "Using auto-loaded firmware version %s.\n",
3478                                  fw_ver);
3479                 }
3480 
3481                 /* extract micro version field; point past '<maj>.<min>.' */
3482                 micro_ver = fw_ver + strlen(LIQUIDIO_BASE_VERSION) + 1;
3483                 if (kstrtoul(micro_ver, 10, &micro) != 0)
3484                         micro = 0;
3485                 octeon_dev->fw_info.ver.maj = LIQUIDIO_BASE_MAJOR_VERSION;
3486                 octeon_dev->fw_info.ver.min = LIQUIDIO_BASE_MINOR_VERSION;
3487                 octeon_dev->fw_info.ver.rev = micro;
3488 
3489                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3490                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3491 
3492                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3493                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3494 
3495                 if (!(num_iqueues) || !(num_oqueues)) {
3496                         dev_err(&octeon_dev->pci_dev->dev,
3497                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3498                                 resp->cfg_info.iqmask,
3499                                 resp->cfg_info.oqmask);
3500                         WRITE_ONCE(sc->caller_is_done, true);
3501                         goto setup_nic_dev_done;
3502                 }
3503 
3504                 if (OCTEON_CN6XXX(octeon_dev)) {
3505                         max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3506                                                                     cn6xxx));
3507                 } else if (OCTEON_CN23XX_PF(octeon_dev)) {
3508                         max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3509                                                                     cn23xx_pf));
3510                 }
3511 
3512                 dev_dbg(&octeon_dev->pci_dev->dev,
3513                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",
3514                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3515                         num_iqueues, num_oqueues, max_num_queues);
3516                 netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);
3517 
3518                 if (!netdev) {
3519                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3520                         WRITE_ONCE(sc->caller_is_done, true);
3521                         goto setup_nic_dev_done;
3522                 }
3523 
3524                 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3525 
3526                 /* Associate the routines that will handle different
3527                  * netdev tasks.
3528                  */
3529                 netdev->netdev_ops = &lionetdevops;
3530 
3531                 retval = netif_set_real_num_rx_queues(netdev, num_oqueues);
3532                 if (retval) {
3533                         dev_err(&octeon_dev->pci_dev->dev,
3534                                 "setting real number rx failed\n");
3535                         WRITE_ONCE(sc->caller_is_done, true);
3536                         goto setup_nic_dev_free;
3537                 }
3538 
3539                 retval = netif_set_real_num_tx_queues(netdev, num_iqueues);
3540                 if (retval) {
3541                         dev_err(&octeon_dev->pci_dev->dev,
3542                                 "setting real number tx failed\n");
3543                         WRITE_ONCE(sc->caller_is_done, true);
3544                         goto setup_nic_dev_free;
3545                 }
3546 
3547                 lio = GET_LIO(netdev);
3548 
3549                 memset(lio, 0, sizeof(struct lio));
3550 
3551                 lio->ifidx = ifidx_or_pfnum;
3552 
3553                 props = &octeon_dev->props[i];
3554                 props->gmxport = resp->cfg_info.linfo.gmxport;
3555                 props->netdev = netdev;
3556 
3557                 lio->linfo.num_rxpciq = num_oqueues;
3558                 lio->linfo.num_txpciq = num_iqueues;
3559                 for (j = 0; j < num_oqueues; j++) {
3560                         lio->linfo.rxpciq[j].u64 =
3561                                 resp->cfg_info.linfo.rxpciq[j].u64;
3562                 }
3563                 for (j = 0; j < num_iqueues; j++) {
3564                         lio->linfo.txpciq[j].u64 =
3565                                 resp->cfg_info.linfo.txpciq[j].u64;
3566                 }
3567                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3568                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3569                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3570 
3571                 WRITE_ONCE(sc->caller_is_done, true);
3572 
3573                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3574 
3575                 if (OCTEON_CN23XX_PF(octeon_dev) ||
3576                     OCTEON_CN6XXX(octeon_dev)) {
3577                         lio->dev_capability = NETIF_F_HIGHDMA
3578                                               | NETIF_F_IP_CSUM
3579                                               | NETIF_F_IPV6_CSUM
3580                                               | NETIF_F_SG | NETIF_F_RXCSUM
3581                                               | NETIF_F_GRO
3582                                               | NETIF_F_TSO | NETIF_F_TSO6
3583                                               | NETIF_F_LRO;
3584                 }
3585                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3586 
3587                 /*  Copy of transmit encapsulation capabilities:
3588                  *  TSO, TSO6, Checksums for this device
3589                  */
3590                 lio->enc_dev_capability = NETIF_F_IP_CSUM
3591                                           | NETIF_F_IPV6_CSUM
3592                                           | NETIF_F_GSO_UDP_TUNNEL
3593                                           | NETIF_F_HW_CSUM | NETIF_F_SG
3594                                           | NETIF_F_RXCSUM
3595                                           | NETIF_F_TSO | NETIF_F_TSO6
3596                                           | NETIF_F_LRO;
3597 
3598                 netdev->hw_enc_features = (lio->enc_dev_capability &
3599                                            ~NETIF_F_LRO);
3600 
3601                 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3602 
3603                 netdev->vlan_features = lio->dev_capability;
3604                 /* Add any unchangeable hw features */
3605                 lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3606                                         NETIF_F_HW_VLAN_CTAG_RX |
3607                                         NETIF_F_HW_VLAN_CTAG_TX;
3608 
3609                 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3610 
3611                 netdev->hw_features = lio->dev_capability;
3612                 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3613                 netdev->hw_features = netdev->hw_features &
3614                         ~NETIF_F_HW_VLAN_CTAG_RX;
3615 
3616                 /* MTU range: 68 - 16000 */
3617                 netdev->min_mtu = LIO_MIN_MTU_SIZE;
3618                 netdev->max_mtu = LIO_MAX_MTU_SIZE;
3619 
3620                 /* Point to the  properties for octeon device to which this
3621                  * interface belongs.
3622                  */
3623                 lio->oct_dev = octeon_dev;
3624                 lio->octprops = props;
3625                 lio->netdev = netdev;
3626 
3627                 dev_dbg(&octeon_dev->pci_dev->dev,
3628                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3629                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3630 
3631                 for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
3632                         u8 vfmac[ETH_ALEN];
3633 
3634                         eth_random_addr(vfmac);
3635                         if (__liquidio_set_vf_mac(netdev, j, vfmac, false)) {
3636                                 dev_err(&octeon_dev->pci_dev->dev,
3637                                         "Error setting VF%d MAC address\n",
3638                                         j);
3639                                 goto setup_nic_dev_free;
3640                         }
3641                 }
3642 
3643                 /* 64-bit swap required on LE machines */
3644                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3645                 for (j = 0; j < 6; j++)
3646                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3647 
3648                 /* Copy MAC Address to OS network device structure */
3649 
3650                 ether_addr_copy(netdev->dev_addr, mac);
3651 
3652                 /* By default all interfaces on a single Octeon uses the same
3653                  * tx and rx queues
3654                  */
3655                 lio->txq = lio->linfo.txpciq[0].s.q_no;
3656                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3657                 if (liquidio_setup_io_queues(octeon_dev, i,
3658                                              lio->linfo.num_txpciq,
3659                                              lio->linfo.num_rxpciq)) {
3660                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3661                         goto setup_nic_dev_free;
3662                 }
3663 
3664                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3665 
3666                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3667                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3668 
3669                 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
3670                         dev_err(&octeon_dev->pci_dev->dev,
3671                                 "Gather list allocation failed\n");
3672                         goto setup_nic_dev_free;
3673                 }
3674 
3675                 /* Register ethtool support */
3676                 liquidio_set_ethtool_ops(netdev);
3677                 if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3678                         octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3679                 else
3680                         octeon_dev->priv_flags = 0x0;
3681 
3682                 if (netdev->features & NETIF_F_LRO)
3683                         liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3684                                              OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3685 
3686                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3687                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
3688 
3689                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3690                         liquidio_set_feature(netdev,
3691                                              OCTNET_CMD_VERBOSE_ENABLE, 0);
3692 
3693                 if (setup_link_status_change_wq(netdev))
3694                         goto setup_nic_dev_free;
3695 
3696                 if ((octeon_dev->fw_info.app_cap_flags &
3697                      LIQUIDIO_TIME_SYNC_CAP) &&
3698                     setup_sync_octeon_time_wq(netdev))
3699                         goto setup_nic_dev_free;
3700 
3701                 if (setup_rx_oom_poll_fn(netdev))
3702                         goto setup_nic_dev_free;
3703 
3704                 /* Register the network device with the OS */
3705                 if (register_netdev(netdev)) {
3706                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3707                         goto setup_nic_dev_free;
3708                 }
3709 
3710                 dev_dbg(&octeon_dev->pci_dev->dev,
3711                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3712                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3713                 netif_carrier_off(netdev);
3714                 lio->link_changes++;
3715 
3716                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3717 
3718                 /* Sending command to firmware to enable Rx checksum offload
3719                  * by default at the time of setup of Liquidio driver for
3720                  * this device
3721                  */
3722                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3723                                             OCTNET_CMD_RXCSUM_ENABLE);
3724                 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3725                                      OCTNET_CMD_TXCSUM_ENABLE);
3726 
3727                 dev_dbg(&octeon_dev->pci_dev->dev,
3728                         "NIC ifidx:%d Setup successful\n", i);
3729 
3730                 if (octeon_dev->subsystem_id ==
3731                         OCTEON_CN2350_25GB_SUBSYS_ID ||
3732                     octeon_dev->subsystem_id ==
3733                         OCTEON_CN2360_25GB_SUBSYS_ID) {
3734                         cur_ver = OCT_FW_VER(octeon_dev->fw_info.ver.maj,
3735                                              octeon_dev->fw_info.ver.min,
3736                                              octeon_dev->fw_info.ver.rev);
3737 
3738                         /* speed control unsupported in f/w older than 1.7.2 */
3739                         if (cur_ver < OCT_FW_VER(1, 7, 2)) {
3740                                 dev_info(&octeon_dev->pci_dev->dev,
3741                                          "speed setting not supported by f/w.");
3742                                 octeon_dev->speed_setting = 25;
3743                                 octeon_dev->no_speed_setting = 1;
3744                         } else {
3745                                 liquidio_get_speed(lio);
3746                         }
3747 
3748                         if (octeon_dev->speed_setting == 0) {
3749                                 octeon_dev->speed_setting = 25;
3750                                 octeon_dev->no_speed_setting = 1;
3751                         }
3752                 } else {
3753                         octeon_dev->no_speed_setting = 1;
3754                         octeon_dev->speed_setting = 10;
3755                 }
3756                 octeon_dev->speed_boot = octeon_dev->speed_setting;
3757 
3758                 /* don't read FEC setting if unsupported by f/w (see above) */
3759                 if (octeon_dev->speed_boot == 25 &&
3760                     !octeon_dev->no_speed_setting) {
3761                         liquidio_get_fec(lio);
3762                         octeon_dev->props[lio->ifidx].fec_boot =
3763                                 octeon_dev->props[lio->ifidx].fec;
3764                 }
3765         }
3766 
3767         devlink = devlink_alloc(&liquidio_devlink_ops,
3768                                 sizeof(struct lio_devlink_priv));
3769         if (!devlink) {
3770                 dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");
3771                 goto setup_nic_dev_free;
3772         }
3773 
3774         lio_devlink = devlink_priv(devlink);
3775         lio_devlink->oct = octeon_dev;
3776 
3777         if (devlink_register(devlink, &octeon_dev->pci_dev->dev)) {
3778                 devlink_free(devlink);
3779                 dev_err(&octeon_dev->pci_dev->dev,
3780                         "devlink registration failed\n");
3781                 goto setup_nic_dev_free;
3782         }
3783 
3784         octeon_dev->devlink = devlink;
3785         octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
3786 
3787         return 0;
3788 
3789 setup_nic_dev_free:
3790 
3791         while (i--) {
3792                 dev_err(&octeon_dev->pci_dev->dev,
3793                         "NIC ifidx:%d Setup failed\n", i);
3794                 liquidio_destroy_nic_device(octeon_dev, i);
3795         }
3796 
3797 setup_nic_dev_done:
3798 
3799         return -ENODEV;
3800 }
3801 
3802 #ifdef CONFIG_PCI_IOV
3803 static int octeon_enable_sriov(struct octeon_device *oct)
3804 {
3805         unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
3806         struct pci_dev *vfdev;
3807         int err;
3808         u32 u;
3809 
3810         if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
3811                 err = pci_enable_sriov(oct->pci_dev,
3812                                        oct->sriov_info.num_vfs_alloced);
3813                 if (err) {
3814                         dev_err(&oct->pci_dev->dev,
3815                                 "OCTEON: Failed to enable PCI sriov: %d\n",
3816                                 err);
3817                         oct->sriov_info.num_vfs_alloced = 0;
3818                         return err;
3819                 }
3820                 oct->sriov_info.sriov_enabled = 1;
3821 
3822                 /* init lookup table that maps DPI ring number to VF pci_dev
3823                  * struct pointer
3824                  */
3825                 u = 0;
3826                 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3827                                        OCTEON_CN23XX_VF_VID, NULL);
3828                 while (vfdev) {
3829                         if (vfdev->is_virtfn &&
3830                             (vfdev->physfn == oct->pci_dev)) {
3831                                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
3832                                         vfdev;
3833                                 u += oct->sriov_info.rings_per_vf;
3834                         }
3835                         vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3836                                                OCTEON_CN23XX_VF_VID, vfdev);
3837                 }
3838         }
3839 
3840         return num_vfs_alloced;
3841 }
3842 
3843 static int lio_pci_sriov_disable(struct octeon_device *oct)
3844 {
3845         int u;
3846 
3847         if (pci_vfs_assigned(oct->pci_dev)) {
3848                 dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
3849                 return -EPERM;
3850         }
3851 
3852         pci_disable_sriov(oct->pci_dev);
3853 
3854         u = 0;
3855         while (u < MAX_POSSIBLE_VFS) {
3856                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
3857                 u += oct->sriov_info.rings_per_vf;
3858         }
3859 
3860         oct->sriov_info.num_vfs_alloced = 0;
3861         dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
3862                  oct->pf_num);
3863 
3864         return 0;
3865 }
3866 
3867 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
3868 {
3869         struct octeon_device *oct = pci_get_drvdata(dev);
3870         int ret = 0;
3871 
3872         if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
3873             (oct->sriov_info.sriov_enabled)) {
3874                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
3875                          oct->pf_num, num_vfs);
3876                 return 0;
3877         }
3878 
3879         if (!num_vfs) {
3880                 lio_vf_rep_destroy(oct);
3881                 ret = lio_pci_sriov_disable(oct);
3882         } else if (num_vfs > oct->sriov_info.max_vfs) {
3883                 dev_err(&oct->pci_dev->dev,
3884                         "OCTEON: Max allowed VFs:%d user requested:%d",
3885                         oct->sriov_info.max_vfs, num_vfs);
3886                 ret = -EPERM;
3887         } else {
3888                 oct->sriov_info.num_vfs_alloced = num_vfs;
3889                 ret = octeon_enable_sriov(oct);
3890                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
3891                          oct->pf_num, num_vfs);
3892                 ret = lio_vf_rep_create(oct);
3893                 if (ret)
3894                         dev_info(&oct->pci_dev->dev,
3895                                  "vf representor create failed");
3896         }
3897 
3898         return ret;
3899 }
3900 #endif
3901 
3902 /**
3903  * \brief initialize the NIC
3904  * @param oct octeon device
3905  *
3906  * This initialization routine is called once the Octeon device application is
3907  * up and running
3908  */
3909 static int liquidio_init_nic_module(struct octeon_device *oct)
3910 {
3911         int i, retval = 0;
3912         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3913 
3914         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3915 
3916         /* only default iq and oq were initialized
3917          * initialize the rest as well
3918          */
3919         /* run port_config command for each port */
3920         oct->ifcount = num_nic_ports;
3921 
3922         memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
3923 
3924         for (i = 0; i < MAX_OCTEON_LINKS; i++)
3925                 oct->props[i].gmxport = -1;
3926 
3927         retval = setup_nic_devices(oct);
3928         if (retval) {
3929                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3930                 goto octnet_init_failure;
3931         }
3932 
3933         /* Call vf_rep_modinit if the firmware is switchdev capable
3934          * and do it from the first liquidio function probed.
3935          */
3936         if (!oct->octeon_id &&
3937             oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {
3938                 retval = lio_vf_rep_modinit();
3939                 if (retval) {
3940                         liquidio_stop_nic_module(oct);
3941                         goto octnet_init_failure;
3942                 }
3943         }
3944 
3945         liquidio_ptp_init(oct);
3946 
3947         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3948 
3949         return retval;
3950 
3951 octnet_init_failure:
3952 
3953         oct->ifcount = 0;
3954 
3955         return retval;
3956 }
3957 
3958 /**
3959  * \brief starter callback that invokes the remaining initialization work after
3960  * the NIC is up and running.
3961  * @param octptr  work struct work_struct
3962  */
3963 static void nic_starter(struct work_struct *work)
3964 {
3965         struct octeon_device *oct;
3966         struct cavium_wk *wk = (struct cavium_wk *)work;
3967 
3968         oct = (struct octeon_device *)wk->ctxptr;
3969 
3970         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3971                 return;
3972 
3973         /* If the status of the device is CORE_OK, the core
3974          * application has reported its application type. Call
3975          * any registered handlers now and move to the RUNNING
3976          * state.
3977          */
3978         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3979                 schedule_delayed_work(&oct->nic_poll_work.work,
3980                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3981                 return;
3982         }
3983 
3984         atomic_set(&oct->status, OCT_DEV_RUNNING);
3985 
3986         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3987                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3988 
3989                 if (liquidio_init_nic_module(oct))
3990                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3991                 else
3992                         handshake[oct->octeon_id].started_ok = 1;
3993         } else {
3994                 dev_err(&oct->pci_dev->dev,
3995                         "Unexpected application running on NIC (%d). Check firmware.\n",
3996                         oct->app_mode);
3997         }
3998 
3999         complete(&handshake[oct->octeon_id].started);
4000 }
4001 
4002 static int
4003 octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
4004 {
4005         struct octeon_device *oct = (struct octeon_device *)buf;
4006         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
4007         int i, notice, vf_idx;
4008         bool cores_crashed;
4009         u64 *data, vf_num;
4010 
4011         notice = recv_pkt->rh.r.ossp;
4012         data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
4013 
4014         /* the first 64-bit word of data is the vf_num */
4015         vf_num = data[0];
4016         octeon_swap_8B_data(&vf_num, 1);
4017         vf_idx = (int)vf_num - 1;
4018 
4019         cores_crashed = READ_ONCE(oct->cores_crashed);
4020 
4021         if (notice == VF_DRV_LOADED) {
4022                 if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
4023                         oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
4024                         dev_info(&oct->pci_dev->dev,
4025                                  "driver for VF%d was loaded\n", vf_idx);
4026                         if (!cores_crashed)
4027                                 try_module_get(THIS_MODULE);
4028                 }
4029         } else if (notice == VF_DRV_REMOVED) {
4030                 if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4031                         oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4032                         dev_info(&oct->pci_dev->dev,
4033                                  "driver for VF%d was removed\n", vf_idx);
4034                         if (!cores_crashed)
4035                                 module_put(THIS_MODULE);
4036                 }
4037         } else if (notice == VF_DRV_MACADDR_CHANGED) {
4038                 u8 *b = (u8 *)&data[1];
4039 
4040                 oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4041                 dev_info(&oct->pci_dev->dev,
4042                          "VF driver changed VF%d's MAC address to %pM\n",
4043                          vf_idx, b + 2);
4044         }
4045 
4046         for (i = 0; i < recv_pkt->buffer_count; i++)
4047                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
4048         octeon_free_recv_info(recv_info);
4049 
4050         return 0;
4051 }
4052 
4053 /**
4054  * \brief Device initialization for each Octeon device that is probed
4055  * @param octeon_dev  octeon device
4056  */
4057 static int octeon_device_init(struct octeon_device *octeon_dev)
4058 {
4059         int j, ret;
4060         char bootcmd[] = "\n";
4061         char *dbg_enb = NULL;
4062         enum lio_fw_state fw_state;
4063         struct octeon_device_priv *oct_priv =
4064                 (struct octeon_device_priv *)octeon_dev->priv;
4065         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4066 
4067         /* Enable access to the octeon device and make its DMA capability
4068          * known to the OS.
4069          */
4070         if (octeon_pci_os_setup(octeon_dev))
4071                 return 1;
4072 
4073         atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4074 
4075         /* Identify the Octeon type and map the BAR address space. */
4076         if (octeon_chip_specific_setup(octeon_dev)) {
4077                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4078                 return 1;
4079         }
4080 
4081         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4082 
4083         /* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4084          * since that is what is required for the reference to be removed
4085          * during de-initialization (see 'octeon_destroy_resources').
4086          */
4087         octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4088                                PCI_SLOT(octeon_dev->pci_dev->devfn),
4089                                PCI_FUNC(octeon_dev->pci_dev->devfn),
4090                                true);
4091 
4092         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4093 
4094         /* CN23XX supports preloaded firmware if the following is true:
4095          *
4096          * The adapter indicates that firmware is currently running AND
4097          * 'fw_type' is 'auto'.
4098          *
4099          * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).
4100          */
4101         if (OCTEON_CN23XX_PF(octeon_dev) &&
4102             cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {
4103                 atomic_cmpxchg(octeon_dev->adapter_fw_state,
4104                                FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);
4105         }
4106 
4107         /* If loading firmware, only first device of adapter needs to do so. */
4108         fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,
4109                                   FW_NEEDS_TO_BE_LOADED,
4110                                   FW_IS_BEING_LOADED);
4111 
4112         /* Here, [local variable] 'fw_state' is set to one of:
4113          *
4114          *   FW_IS_PRELOADED:       No firmware is to be loaded (see above)
4115          *   FW_NEEDS_TO_BE_LOADED: The driver's first instance will load
4116          *                          firmware to the adapter.
4117          *   FW_IS_BEING_LOADED:    The driver's second instance will not load
4118          *                          firmware to the adapter.
4119          */
4120 
4121         /* Prior to f/w load, perform a soft reset of the Octeon device;
4122          * if error resetting, return w/error.
4123          */
4124         if (fw_state == FW_NEEDS_TO_BE_LOADED)
4125                 if (octeon_dev->fn_list.soft_reset(octeon_dev))
4126                         return 1;
4127 
4128         /* Initialize the dispatch mechanism used to push packets arriving on
4129          * Octeon Output queues.
4130          */
4131         if (octeon_init_dispatch_list(octeon_dev))
4132                 return 1;
4133 
4134         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4135                                     OPCODE_NIC_CORE_DRV_ACTIVE,
4136                                     octeon_core_drv_init,
4137                                     octeon_dev);
4138 
4139         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4140                                     OPCODE_NIC_VF_DRV_NOTICE,
4141                                     octeon_recv_vf_drv_notice, octeon_dev);
4142         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4143         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4144         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4145                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4146 
4147         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4148 
4149         if (octeon_set_io_queues_off(octeon_dev)) {
4150                 dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4151                 return 1;
4152         }
4153 
4154         if (OCTEON_CN23XX_PF(octeon_dev)) {
4155                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4156                 if (ret) {
4157                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4158                         return ret;
4159                 }
4160         }
4161 
4162         /* Initialize soft command buffer pool
4163          */
4164         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4165                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4166                 return 1;
4167         }
4168         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4169 
4170         /*  Setup the data structures that manage this Octeon's Input queues. */
4171         if (octeon_setup_instr_queues(octeon_dev)) {
4172                 dev_err(&octeon_dev->pci_dev->dev,
4173                         "instruction queue initialization failed\n");
4174                 return 1;
4175         }
4176         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4177 
4178         /* Initialize lists to manage the requests of different types that
4179          * arrive from user & kernel applications for this octeon device.
4180          */
4181         if (octeon_setup_response_list(octeon_dev)) {
4182                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4183                 return 1;
4184         }
4185         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4186 
4187         if (octeon_setup_output_queues(octeon_dev)) {
4188                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4189                 return 1;
4190         }
4191 
4192         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4193 
4194         if (OCTEON_CN23XX_PF(octeon_dev)) {
4195                 if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4196                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4197                         return 1;
4198                 }
4199                 atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4200 
4201                 if (octeon_allocate_ioq_vector
4202                                 (octeon_dev,
4203                                  octeon_dev->sriov_info.num_pf_rings)) {
4204                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4205                         return 1;
4206                 }
4207                 atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
4208 
4209         } else {
4210                 /* The input and output queue registers were setup earlier (the
4211                  * queues were not enabled). Any additional registers
4212                  * that need to be programmed should be done now.
4213                  */
4214                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4215                 if (ret) {
4216                         dev_err(&octeon_dev->pci_dev->dev,
4217                                 "Failed to configure device registers\n");
4218                         return ret;
4219                 }
4220         }
4221 
4222         /* Initialize the tasklet that handles output queue packet processing.*/
4223         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4224         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
4225                      (unsigned long)octeon_dev);
4226 
4227         /* Setup the interrupt handler and record the INT SUM register address
4228          */
4229         if (octeon_setup_interrupt(octeon_dev,
4230                                    octeon_dev->sriov_info.num_pf_rings))
4231                 return 1;
4232 
4233         /* Enable Octeon device interrupts */
4234         octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4235 
4236         atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4237 
4238         /* Send Credit for Octeon Output queues. Credits are always sent BEFORE
4239          * the output queue is enabled.
4240          * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
4241          * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
4242          * Otherwise, it is possible that the DRV_ACTIVE message will be sent
4243          * before any credits have been issued, causing the ring to be reset
4244          * (and the f/w appear to never have started).
4245          */
4246         for (j = 0; j < octeon_dev->num_oqs; j++)
4247                 writel(octeon_dev->droq[j]->max_count,
4248                        octeon_dev->droq[j]->pkts_credit_reg);
4249 
4250         /* Enable the input and output queues for this Octeon device */
4251         ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4252         if (ret) {
4253                 dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4254                 return ret;
4255         }
4256 
4257         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4258 
4259         if (fw_state == FW_NEEDS_TO_BE_LOADED) {
4260                 dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4261                 if (!ddr_timeout) {
4262                         dev_info(&octeon_dev->pci_dev->dev,
4263                                  "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4264                 }
4265 
4266                 schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4267 
4268                 /* Wait for the octeon to initialize DDR after the soft-reset.*/
4269                 while (!ddr_timeout) {
4270                         set_current_state(TASK_INTERRUPTIBLE);
4271                         if (schedule_timeout(HZ / 10)) {
4272                                 /* user probably pressed Control-C */
4273                                 return 1;
4274                         }
4275                 }
4276                 ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4277                 if (ret) {
4278                         dev_err(&octeon_dev->pci_dev->dev,
4279                                 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4280                                 ret);
4281                         return 1;
4282                 }
4283 
4284                 if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4285                         dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4286                         return 1;
4287                 }
4288 
4289                 /* Divert uboot to take commands from host instead. */
4290                 ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4291 
4292                 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4293                 ret = octeon_init_consoles(octeon_dev);
4294                 if (ret) {
4295                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4296                         return 1;
4297                 }
4298                 /* If console debug enabled, specify empty string to use default
4299                  * enablement ELSE specify NULL string for 'disabled'.
4300                  */
4301                 dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
4302                 ret = octeon_add_console(octeon_dev, 0, dbg_enb);
4303                 if (ret) {
4304                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4305                         return 1;
4306                 } else if (octeon_console_debug_enabled(0)) {
4307                         /* If console was added AND we're logging console output
4308                          * then set our console print function.
4309                          */
4310                         octeon_dev->console[0].print = octeon_dbg_console_print;
4311                 }
4312 
4313                 atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4314 
4315                 dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4316                 ret = load_firmware(octeon_dev);
4317                 if (ret) {
4318                         dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4319                         return 1;
4320                 }
4321 
4322                 atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);
4323         }
4324 
4325         handshake[octeon_dev->octeon_id].init_ok = 1;
4326         complete(&handshake[octeon_dev->octeon_id].init);
4327 
4328         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4329 
4330         return 0;
4331 }
4332 
4333 /**
4334  * \brief Debug console print function
4335  * @param octeon_dev  octeon device
4336  * @param console_num console number
4337  * @param prefix      first portion of line to display
4338  * @param suffix      second portion of line to display
4339  *
4340  * The OCTEON debug console outputs entire lines (excluding '\n').
4341  * Normally, the line will be passed in the 'prefix' parameter.
4342  * However, due to buffering, it is possible for a line to be split into two
4343  * parts, in which case they will be passed as the 'prefix' parameter and
4344  * 'suffix' parameter.
4345  */
4346 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
4347                                     char *prefix, char *suffix)
4348 {
4349         if (prefix && suffix)
4350                 dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
4351                          suffix);
4352         else if (prefix)
4353                 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
4354         else if (suffix)
4355                 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
4356 
4357         return 0;
4358 }
4359 
4360 /**
4361  * \brief Exits the module
4362  */
4363 static void __exit liquidio_exit(void)
4364 {
4365         liquidio_deinit_pci();
4366 
4367         pr_info("LiquidIO network module is now unloaded\n");
4368 }
4369 
4370 module_init(liquidio_init);
4371 module_exit(liquidio_exit);
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/net/ethernet/cavium/liquidio/lio_main.c

DEFINITIONS
