root/drivers/net/ethernet/socionext/netsec.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. netsec_write
  2. netsec_read
  3. netsec_clk_type
  4. netsec_wait_while_busy
  5. netsec_mac_write
  6. netsec_mac_read
  7. netsec_mac_wait_while_busy
  8. netsec_mac_update_to_phy_state
  9. netsec_phy_write
  10. netsec_phy_read
  11. netsec_et_get_drvinfo
  12. netsec_et_get_coalesce
  13. netsec_et_set_coalesce
  14. netsec_et_get_msglevel
  15. netsec_et_set_msglevel
  16. netsec_set_rx_de
  17. netsec_clean_tx_dring
  18. netsec_process_tx
  19. netsec_alloc_rx_data
  20. netsec_rx_fill
  21. netsec_xdp_ring_tx_db
  22. netsec_finalize_xdp_rx
  23. netsec_set_tx_de
  24. netsec_xdp_queue_one
  25. netsec_xdp_xmit_back
  26. netsec_run_xdp
  27. netsec_process_rx
  28. netsec_napi_poll
  29. netsec_desc_used
  30. netsec_check_stop_tx
  31. netsec_netdev_start_xmit
  32. netsec_uninit_pkt_dring
  33. netsec_free_dring
  34. netsec_alloc_dring
  35. netsec_setup_tx_dring
  36. netsec_setup_rx_dring
  37. netsec_netdev_load_ucode_region
  38. netsec_netdev_load_microcode
  39. netsec_reset_hardware
  40. netsec_start_gmac
  41. netsec_stop_gmac
  42. netsec_phy_adjust_link
  43. netsec_irq_handler
  44. netsec_netdev_open
  45. netsec_netdev_stop
  46. netsec_netdev_init
  47. netsec_netdev_uninit
  48. netsec_netdev_set_features
  49. netsec_netdev_ioctl
  50. netsec_xdp_xmit
  51. netsec_xdp_setup
  52. netsec_xdp
  53. netsec_of_probe
  54. netsec_acpi_probe
  55. netsec_unregister_mdio
  56. netsec_register_mdio
  57. netsec_probe
  58. netsec_remove
  59. netsec_runtime_suspend
  60. netsec_runtime_resume

   1 // SPDX-License-Identifier: GPL-2.0+
   2 
   3 #include <linux/types.h>
   4 #include <linux/clk.h>
   5 #include <linux/platform_device.h>
   6 #include <linux/pm_runtime.h>
   7 #include <linux/acpi.h>
   8 #include <linux/of_mdio.h>
   9 #include <linux/etherdevice.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/io.h>
  12 #include <linux/netlink.h>
  13 #include <linux/bpf.h>
  14 #include <linux/bpf_trace.h>
  15 
  16 #include <net/tcp.h>
  17 #include <net/page_pool.h>
  18 #include <net/ip6_checksum.h>
  19 
  20 #define NETSEC_REG_SOFT_RST                     0x104
  21 #define NETSEC_REG_COM_INIT                     0x120
  22 
  23 #define NETSEC_REG_TOP_STATUS                   0x200
  24 #define NETSEC_IRQ_RX                           BIT(1)
  25 #define NETSEC_IRQ_TX                           BIT(0)
  26 
  27 #define NETSEC_REG_TOP_INTEN                    0x204
  28 #define NETSEC_REG_INTEN_SET                    0x234
  29 #define NETSEC_REG_INTEN_CLR                    0x238
  30 
  31 #define NETSEC_REG_NRM_TX_STATUS                0x400
  32 #define NETSEC_REG_NRM_TX_INTEN                 0x404
  33 #define NETSEC_REG_NRM_TX_INTEN_SET             0x428
  34 #define NETSEC_REG_NRM_TX_INTEN_CLR             0x42c
  35 #define NRM_TX_ST_NTOWNR        BIT(17)
  36 #define NRM_TX_ST_TR_ERR        BIT(16)
  37 #define NRM_TX_ST_TXDONE        BIT(15)
  38 #define NRM_TX_ST_TMREXP        BIT(14)
  39 
  40 #define NETSEC_REG_NRM_RX_STATUS                0x440
  41 #define NETSEC_REG_NRM_RX_INTEN                 0x444
  42 #define NETSEC_REG_NRM_RX_INTEN_SET             0x468
  43 #define NETSEC_REG_NRM_RX_INTEN_CLR             0x46c
  44 #define NRM_RX_ST_RC_ERR        BIT(16)
  45 #define NRM_RX_ST_PKTCNT        BIT(15)
  46 #define NRM_RX_ST_TMREXP        BIT(14)
  47 
  48 #define NETSEC_REG_PKT_CMD_BUF                  0xd0
  49 
  50 #define NETSEC_REG_CLK_EN                       0x100
  51 
  52 #define NETSEC_REG_PKT_CTRL                     0x140
  53 
  54 #define NETSEC_REG_DMA_TMR_CTRL                 0x20c
  55 #define NETSEC_REG_F_TAIKI_MC_VER               0x22c
  56 #define NETSEC_REG_F_TAIKI_VER                  0x230
  57 #define NETSEC_REG_DMA_HM_CTRL                  0x214
  58 #define NETSEC_REG_DMA_MH_CTRL                  0x220
  59 #define NETSEC_REG_ADDR_DIS_CORE                0x218
  60 #define NETSEC_REG_DMAC_HM_CMD_BUF              0x210
  61 #define NETSEC_REG_DMAC_MH_CMD_BUF              0x21c
  62 
  63 #define NETSEC_REG_NRM_TX_PKTCNT                0x410
  64 
  65 #define NETSEC_REG_NRM_TX_DONE_PKTCNT           0x414
  66 #define NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT     0x418
  67 
  68 #define NETSEC_REG_NRM_TX_TMR                   0x41c
  69 
  70 #define NETSEC_REG_NRM_RX_PKTCNT                0x454
  71 #define NETSEC_REG_NRM_RX_RXINT_PKTCNT          0x458
  72 #define NETSEC_REG_NRM_TX_TXINT_TMR             0x420
  73 #define NETSEC_REG_NRM_RX_RXINT_TMR             0x460
  74 
  75 #define NETSEC_REG_NRM_RX_TMR                   0x45c
  76 
  77 #define NETSEC_REG_NRM_TX_DESC_START_UP         0x434
  78 #define NETSEC_REG_NRM_TX_DESC_START_LW         0x408
  79 #define NETSEC_REG_NRM_RX_DESC_START_UP         0x474
  80 #define NETSEC_REG_NRM_RX_DESC_START_LW         0x448
  81 
  82 #define NETSEC_REG_NRM_TX_CONFIG                0x430
  83 #define NETSEC_REG_NRM_RX_CONFIG                0x470
  84 
  85 #define MAC_REG_STATUS                          0x1024
  86 #define MAC_REG_DATA                            0x11c0
  87 #define MAC_REG_CMD                             0x11c4
  88 #define MAC_REG_FLOW_TH                         0x11cc
  89 #define MAC_REG_INTF_SEL                        0x11d4
  90 #define MAC_REG_DESC_INIT                       0x11fc
  91 #define MAC_REG_DESC_SOFT_RST                   0x1204
  92 #define NETSEC_REG_MODE_TRANS_COMP_STATUS       0x500
  93 
  94 #define GMAC_REG_MCR                            0x0000
  95 #define GMAC_REG_MFFR                           0x0004
  96 #define GMAC_REG_GAR                            0x0010
  97 #define GMAC_REG_GDR                            0x0014
  98 #define GMAC_REG_FCR                            0x0018
  99 #define GMAC_REG_BMR                            0x1000
 100 #define GMAC_REG_RDLAR                          0x100c
 101 #define GMAC_REG_TDLAR                          0x1010
 102 #define GMAC_REG_OMR                            0x1018
 103 
 104 #define MHZ(n)          ((n) * 1000 * 1000)
 105 
 106 #define NETSEC_TX_SHIFT_OWN_FIELD               31
 107 #define NETSEC_TX_SHIFT_LD_FIELD                30
 108 #define NETSEC_TX_SHIFT_DRID_FIELD              24
 109 #define NETSEC_TX_SHIFT_PT_FIELD                21
 110 #define NETSEC_TX_SHIFT_TDRID_FIELD             16
 111 #define NETSEC_TX_SHIFT_CC_FIELD                15
 112 #define NETSEC_TX_SHIFT_FS_FIELD                9
 113 #define NETSEC_TX_LAST                          8
 114 #define NETSEC_TX_SHIFT_CO                      7
 115 #define NETSEC_TX_SHIFT_SO                      6
 116 #define NETSEC_TX_SHIFT_TRS_FIELD               4
 117 
 118 #define NETSEC_RX_PKT_OWN_FIELD                 31
 119 #define NETSEC_RX_PKT_LD_FIELD                  30
 120 #define NETSEC_RX_PKT_SDRID_FIELD               24
 121 #define NETSEC_RX_PKT_FR_FIELD                  23
 122 #define NETSEC_RX_PKT_ER_FIELD                  21
 123 #define NETSEC_RX_PKT_ERR_FIELD                 16
 124 #define NETSEC_RX_PKT_TDRID_FIELD               12
 125 #define NETSEC_RX_PKT_FS_FIELD                  9
 126 #define NETSEC_RX_PKT_LS_FIELD                  8
 127 #define NETSEC_RX_PKT_CO_FIELD                  6
 128 
 129 #define NETSEC_RX_PKT_ERR_MASK                  3
 130 
 131 #define NETSEC_MAX_TX_PKT_LEN                   1518
 132 #define NETSEC_MAX_TX_JUMBO_PKT_LEN             9018
 133 
 134 #define NETSEC_RING_GMAC                        15
 135 #define NETSEC_RING_MAX                         2
 136 
 137 #define NETSEC_TCP_SEG_LEN_MAX                  1460
 138 #define NETSEC_TCP_JUMBO_SEG_LEN_MAX            8960
 139 
 140 #define NETSEC_RX_CKSUM_NOTAVAIL                0
 141 #define NETSEC_RX_CKSUM_OK                      1
 142 #define NETSEC_RX_CKSUM_NG                      2
 143 
 144 #define NETSEC_TOP_IRQ_REG_CODE_LOAD_END        BIT(20)
 145 #define NETSEC_IRQ_TRANSITION_COMPLETE          BIT(4)
 146 
 147 #define NETSEC_MODE_TRANS_COMP_IRQ_N2T          BIT(20)
 148 #define NETSEC_MODE_TRANS_COMP_IRQ_T2N          BIT(19)
 149 
 150 #define NETSEC_INT_PKTCNT_MAX                   2047
 151 
 152 #define NETSEC_FLOW_START_TH_MAX                95
 153 #define NETSEC_FLOW_STOP_TH_MAX                 95
 154 #define NETSEC_FLOW_PAUSE_TIME_MIN              5
 155 
 156 #define NETSEC_CLK_EN_REG_DOM_ALL               0x3f
 157 
 158 #define NETSEC_PKT_CTRL_REG_MODE_NRM            BIT(28)
 159 #define NETSEC_PKT_CTRL_REG_EN_JUMBO            BIT(27)
 160 #define NETSEC_PKT_CTRL_REG_LOG_CHKSUM_ER       BIT(3)
 161 #define NETSEC_PKT_CTRL_REG_LOG_HD_INCOMPLETE   BIT(2)
 162 #define NETSEC_PKT_CTRL_REG_LOG_HD_ER           BIT(1)
 163 #define NETSEC_PKT_CTRL_REG_DRP_NO_MATCH        BIT(0)
 164 
 165 #define NETSEC_CLK_EN_REG_DOM_G                 BIT(5)
 166 #define NETSEC_CLK_EN_REG_DOM_C                 BIT(1)
 167 #define NETSEC_CLK_EN_REG_DOM_D                 BIT(0)
 168 
 169 #define NETSEC_COM_INIT_REG_DB                  BIT(2)
 170 #define NETSEC_COM_INIT_REG_CLS                 BIT(1)
 171 #define NETSEC_COM_INIT_REG_ALL                 (NETSEC_COM_INIT_REG_CLS | \
 172                                                  NETSEC_COM_INIT_REG_DB)
 173 
 174 #define NETSEC_SOFT_RST_REG_RESET               0
 175 #define NETSEC_SOFT_RST_REG_RUN                 BIT(31)
 176 
 177 #define NETSEC_DMA_CTRL_REG_STOP                1
 178 #define MH_CTRL__MODE_TRANS                     BIT(20)
 179 
 180 #define NETSEC_GMAC_CMD_ST_READ                 0
 181 #define NETSEC_GMAC_CMD_ST_WRITE                BIT(28)
 182 #define NETSEC_GMAC_CMD_ST_BUSY                 BIT(31)
 183 
 184 #define NETSEC_GMAC_BMR_REG_COMMON              0x00412080
 185 #define NETSEC_GMAC_BMR_REG_RESET               0x00020181
 186 #define NETSEC_GMAC_BMR_REG_SWR                 0x00000001
 187 
 188 #define NETSEC_GMAC_OMR_REG_ST                  BIT(13)
 189 #define NETSEC_GMAC_OMR_REG_SR                  BIT(1)
 190 
 191 #define NETSEC_GMAC_MCR_REG_IBN                 BIT(30)
 192 #define NETSEC_GMAC_MCR_REG_CST                 BIT(25)
 193 #define NETSEC_GMAC_MCR_REG_JE                  BIT(20)
 194 #define NETSEC_MCR_PS                           BIT(15)
 195 #define NETSEC_GMAC_MCR_REG_FES                 BIT(14)
 196 #define NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON  0x0000280c
 197 #define NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON  0x0001a00c
 198 
 199 #define NETSEC_FCR_RFE                          BIT(2)
 200 #define NETSEC_FCR_TFE                          BIT(1)
 201 
 202 #define NETSEC_GMAC_GAR_REG_GW                  BIT(1)
 203 #define NETSEC_GMAC_GAR_REG_GB                  BIT(0)
 204 
 205 #define NETSEC_GMAC_GAR_REG_SHIFT_PA            11
 206 #define NETSEC_GMAC_GAR_REG_SHIFT_GR            6
 207 #define GMAC_REG_SHIFT_CR_GAR                   2
 208 
 209 #define NETSEC_GMAC_GAR_REG_CR_25_35_MHZ        2
 210 #define NETSEC_GMAC_GAR_REG_CR_35_60_MHZ        3
 211 #define NETSEC_GMAC_GAR_REG_CR_60_100_MHZ       0
 212 #define NETSEC_GMAC_GAR_REG_CR_100_150_MHZ      1
 213 #define NETSEC_GMAC_GAR_REG_CR_150_250_MHZ      4
 214 #define NETSEC_GMAC_GAR_REG_CR_250_300_MHZ      5
 215 
 216 #define NETSEC_GMAC_RDLAR_REG_COMMON            0x18000
 217 #define NETSEC_GMAC_TDLAR_REG_COMMON            0x1c000
 218 
 219 #define NETSEC_REG_NETSEC_VER_F_TAIKI           0x50000
 220 
 221 #define NETSEC_REG_DESC_RING_CONFIG_CFG_UP      BIT(31)
 222 #define NETSEC_REG_DESC_RING_CONFIG_CH_RST      BIT(30)
 223 #define NETSEC_REG_DESC_TMR_MODE                4
 224 #define NETSEC_REG_DESC_ENDIAN                  0
 225 
 226 #define NETSEC_MAC_DESC_SOFT_RST_SOFT_RST       1
 227 #define NETSEC_MAC_DESC_INIT_REG_INIT           1
 228 
 229 #define NETSEC_EEPROM_MAC_ADDRESS               0x00
 230 #define NETSEC_EEPROM_HM_ME_ADDRESS_H           0x08
 231 #define NETSEC_EEPROM_HM_ME_ADDRESS_L           0x0C
 232 #define NETSEC_EEPROM_HM_ME_SIZE                0x10
 233 #define NETSEC_EEPROM_MH_ME_ADDRESS_H           0x14
 234 #define NETSEC_EEPROM_MH_ME_ADDRESS_L           0x18
 235 #define NETSEC_EEPROM_MH_ME_SIZE                0x1C
 236 #define NETSEC_EEPROM_PKT_ME_ADDRESS            0x20
 237 #define NETSEC_EEPROM_PKT_ME_SIZE               0x24
 238 
 239 #define DESC_NUM        256
 240 
 241 #define NETSEC_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN)
 242 #define NETSEC_RXBUF_HEADROOM (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \
 243                                NET_IP_ALIGN)
 244 #define NETSEC_RX_BUF_NON_DATA (NETSEC_RXBUF_HEADROOM + \
 245                                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 246 
 247 #define DESC_SZ sizeof(struct netsec_de)
 248 
 249 #define NETSEC_F_NETSEC_VER_MAJOR_NUM(x)        ((x) & 0xffff0000)
 250 
 251 #define NETSEC_XDP_PASS          0
 252 #define NETSEC_XDP_CONSUMED      BIT(0)
 253 #define NETSEC_XDP_TX            BIT(1)
 254 #define NETSEC_XDP_REDIR         BIT(2)
 255 #define NETSEC_XDP_RX_OK (NETSEC_XDP_PASS | NETSEC_XDP_TX | NETSEC_XDP_REDIR)
 256 
 257 enum ring_id {
 258         NETSEC_RING_TX = 0,
 259         NETSEC_RING_RX
 260 };
 261 
 262 enum buf_type {
 263         TYPE_NETSEC_SKB = 0,
 264         TYPE_NETSEC_XDP_TX,
 265         TYPE_NETSEC_XDP_NDO,
 266 };
 267 
 268 struct netsec_desc {
 269         union {
 270                 struct sk_buff *skb;
 271                 struct xdp_frame *xdpf;
 272         };
 273         dma_addr_t dma_addr;
 274         void *addr;
 275         u16 len;
 276         u8 buf_type;
 277 };
 278 
 279 struct netsec_desc_ring {
 280         dma_addr_t desc_dma;
 281         struct netsec_desc *desc;
 282         void *vaddr;
 283         u16 head, tail;
 284         u16 xdp_xmit; /* netsec_xdp_xmit packets */
 285         struct page_pool *page_pool;
 286         struct xdp_rxq_info xdp_rxq;
 287         spinlock_t lock; /* XDP tx queue locking */
 288 };
 289 
 290 struct netsec_priv {
 291         struct netsec_desc_ring desc_ring[NETSEC_RING_MAX];
 292         struct ethtool_coalesce et_coalesce;
 293         struct bpf_prog *xdp_prog;
 294         spinlock_t reglock; /* protect reg access */
 295         struct napi_struct napi;
 296         phy_interface_t phy_interface;
 297         struct net_device *ndev;
 298         struct device_node *phy_np;
 299         struct phy_device *phydev;
 300         struct mii_bus *mii_bus;
 301         void __iomem *ioaddr;
 302         void __iomem *eeprom_base;
 303         struct device *dev;
 304         struct clk *clk;
 305         u32 msg_enable;
 306         u32 freq;
 307         u32 phy_addr;
 308         bool rx_cksum_offload_flag;
 309 };
 310 
 311 struct netsec_de { /* Netsec Descriptor layout */
 312         u32 attr;
 313         u32 data_buf_addr_up;
 314         u32 data_buf_addr_lw;
 315         u32 buf_len_info;
 316 };
 317 
 318 struct netsec_tx_pkt_ctrl {
 319         u16 tcp_seg_len;
 320         bool tcp_seg_offload_flag;
 321         bool cksum_offload_flag;
 322 };
 323 
 324 struct netsec_rx_pkt_info {
 325         int rx_cksum_result;
 326         int err_code;
 327         bool err_flag;
 328 };
 329 
 330 static void netsec_write(struct netsec_priv *priv, u32 reg_addr, u32 val)
 331 {
 332         writel(val, priv->ioaddr + reg_addr);
 333 }
 334 
 335 static u32 netsec_read(struct netsec_priv *priv, u32 reg_addr)
 336 {
 337         return readl(priv->ioaddr + reg_addr);
 338 }
 339 
 340 /************* MDIO BUS OPS FOLLOW *************/
 341 
 342 #define TIMEOUT_SPINS_MAC               1000
 343 #define TIMEOUT_SECONDARY_MS_MAC        100
 344 
 345 static u32 netsec_clk_type(u32 freq)
 346 {
 347         if (freq < MHZ(35))
 348                 return NETSEC_GMAC_GAR_REG_CR_25_35_MHZ;
 349         if (freq < MHZ(60))
 350                 return NETSEC_GMAC_GAR_REG_CR_35_60_MHZ;
 351         if (freq < MHZ(100))
 352                 return NETSEC_GMAC_GAR_REG_CR_60_100_MHZ;
 353         if (freq < MHZ(150))
 354                 return NETSEC_GMAC_GAR_REG_CR_100_150_MHZ;
 355         if (freq < MHZ(250))
 356                 return NETSEC_GMAC_GAR_REG_CR_150_250_MHZ;
 357 
 358         return NETSEC_GMAC_GAR_REG_CR_250_300_MHZ;
 359 }
 360 
 361 static int netsec_wait_while_busy(struct netsec_priv *priv, u32 addr, u32 mask)
 362 {
 363         u32 timeout = TIMEOUT_SPINS_MAC;
 364 
 365         while (--timeout && netsec_read(priv, addr) & mask)
 366                 cpu_relax();
 367         if (timeout)
 368                 return 0;
 369 
 370         timeout = TIMEOUT_SECONDARY_MS_MAC;
 371         while (--timeout && netsec_read(priv, addr) & mask)
 372                 usleep_range(1000, 2000);
 373 
 374         if (timeout)
 375                 return 0;
 376 
 377         netdev_WARN(priv->ndev, "%s: timeout\n", __func__);
 378 
 379         return -ETIMEDOUT;
 380 }
 381 
 382 static int netsec_mac_write(struct netsec_priv *priv, u32 addr, u32 value)
 383 {
 384         netsec_write(priv, MAC_REG_DATA, value);
 385         netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_WRITE);
 386         return netsec_wait_while_busy(priv,
 387                                       MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY);
 388 }
 389 
 390 static int netsec_mac_read(struct netsec_priv *priv, u32 addr, u32 *read)
 391 {
 392         int ret;
 393 
 394         netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_READ);
 395         ret = netsec_wait_while_busy(priv,
 396                                      MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY);
 397         if (ret)
 398                 return ret;
 399 
 400         *read = netsec_read(priv, MAC_REG_DATA);
 401 
 402         return 0;
 403 }
 404 
 405 static int netsec_mac_wait_while_busy(struct netsec_priv *priv,
 406                                       u32 addr, u32 mask)
 407 {
 408         u32 timeout = TIMEOUT_SPINS_MAC;
 409         int ret, data;
 410 
 411         do {
 412                 ret = netsec_mac_read(priv, addr, &data);
 413                 if (ret)
 414                         break;
 415                 cpu_relax();
 416         } while (--timeout && (data & mask));
 417 
 418         if (timeout)
 419                 return 0;
 420 
 421         timeout = TIMEOUT_SECONDARY_MS_MAC;
 422         do {
 423                 usleep_range(1000, 2000);
 424 
 425                 ret = netsec_mac_read(priv, addr, &data);
 426                 if (ret)
 427                         break;
 428                 cpu_relax();
 429         } while (--timeout && (data & mask));
 430 
 431         if (timeout && !ret)
 432                 return 0;
 433 
 434         netdev_WARN(priv->ndev, "%s: timeout\n", __func__);
 435 
 436         return -ETIMEDOUT;
 437 }
 438 
 439 static int netsec_mac_update_to_phy_state(struct netsec_priv *priv)
 440 {
 441         struct phy_device *phydev = priv->ndev->phydev;
 442         u32 value = 0;
 443 
 444         value = phydev->duplex ? NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON :
 445                                  NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON;
 446 
 447         if (phydev->speed != SPEED_1000)
 448                 value |= NETSEC_MCR_PS;
 449 
 450         if (priv->phy_interface != PHY_INTERFACE_MODE_GMII &&
 451             phydev->speed == SPEED_100)
 452                 value |= NETSEC_GMAC_MCR_REG_FES;
 453 
 454         value |= NETSEC_GMAC_MCR_REG_CST | NETSEC_GMAC_MCR_REG_JE;
 455 
 456         if (phy_interface_mode_is_rgmii(priv->phy_interface))
 457                 value |= NETSEC_GMAC_MCR_REG_IBN;
 458 
 459         if (netsec_mac_write(priv, GMAC_REG_MCR, value))
 460                 return -ETIMEDOUT;
 461 
 462         return 0;
 463 }
 464 
 465 static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr);
 466 
 467 static int netsec_phy_write(struct mii_bus *bus,
 468                             int phy_addr, int reg, u16 val)
 469 {
 470         int status;
 471         struct netsec_priv *priv = bus->priv;
 472 
 473         if (netsec_mac_write(priv, GMAC_REG_GDR, val))
 474                 return -ETIMEDOUT;
 475         if (netsec_mac_write(priv, GMAC_REG_GAR,
 476                              phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA |
 477                              reg << NETSEC_GMAC_GAR_REG_SHIFT_GR |
 478                              NETSEC_GMAC_GAR_REG_GW | NETSEC_GMAC_GAR_REG_GB |
 479                              (netsec_clk_type(priv->freq) <<
 480                               GMAC_REG_SHIFT_CR_GAR)))
 481                 return -ETIMEDOUT;
 482 
 483         status = netsec_mac_wait_while_busy(priv, GMAC_REG_GAR,
 484                                             NETSEC_GMAC_GAR_REG_GB);
 485 
 486         /* Developerbox implements RTL8211E PHY and there is
 487          * a compatibility problem with F_GMAC4.
 488          * RTL8211E expects MDC clock must be kept toggling for several
 489          * clock cycle with MDIO high before entering the IDLE state.
 490          * To meet this requirement, netsec driver needs to issue dummy
 491          * read(e.g. read PHYID1(offset 0x2) register) right after write.
 492          */
 493         netsec_phy_read(bus, phy_addr, MII_PHYSID1);
 494 
 495         return status;
 496 }
 497 
 498 static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr)
 499 {
 500         struct netsec_priv *priv = bus->priv;
 501         u32 data;
 502         int ret;
 503 
 504         if (netsec_mac_write(priv, GMAC_REG_GAR, NETSEC_GMAC_GAR_REG_GB |
 505                              phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA |
 506                              reg_addr << NETSEC_GMAC_GAR_REG_SHIFT_GR |
 507                              (netsec_clk_type(priv->freq) <<
 508                               GMAC_REG_SHIFT_CR_GAR)))
 509                 return -ETIMEDOUT;
 510 
 511         ret = netsec_mac_wait_while_busy(priv, GMAC_REG_GAR,
 512                                          NETSEC_GMAC_GAR_REG_GB);
 513         if (ret)
 514                 return ret;
 515 
 516         ret = netsec_mac_read(priv, GMAC_REG_GDR, &data);
 517         if (ret)
 518                 return ret;
 519 
 520         return data;
 521 }
 522 
 523 /************* ETHTOOL_OPS FOLLOW *************/
 524 
 525 static void netsec_et_get_drvinfo(struct net_device *net_device,
 526                                   struct ethtool_drvinfo *info)
 527 {
 528         strlcpy(info->driver, "netsec", sizeof(info->driver));
 529         strlcpy(info->bus_info, dev_name(net_device->dev.parent),
 530                 sizeof(info->bus_info));
 531 }
 532 
 533 static int netsec_et_get_coalesce(struct net_device *net_device,
 534                                   struct ethtool_coalesce *et_coalesce)
 535 {
 536         struct netsec_priv *priv = netdev_priv(net_device);
 537 
 538         *et_coalesce = priv->et_coalesce;
 539 
 540         return 0;
 541 }
 542 
 543 static int netsec_et_set_coalesce(struct net_device *net_device,
 544                                   struct ethtool_coalesce *et_coalesce)
 545 {
 546         struct netsec_priv *priv = netdev_priv(net_device);
 547 
 548         priv->et_coalesce = *et_coalesce;
 549 
 550         if (priv->et_coalesce.tx_coalesce_usecs < 50)
 551                 priv->et_coalesce.tx_coalesce_usecs = 50;
 552         if (priv->et_coalesce.tx_max_coalesced_frames < 1)
 553                 priv->et_coalesce.tx_max_coalesced_frames = 1;
 554 
 555         netsec_write(priv, NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT,
 556                      priv->et_coalesce.tx_max_coalesced_frames);
 557         netsec_write(priv, NETSEC_REG_NRM_TX_TXINT_TMR,
 558                      priv->et_coalesce.tx_coalesce_usecs);
 559         netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TXDONE);
 560         netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TMREXP);
 561 
 562         if (priv->et_coalesce.rx_coalesce_usecs < 50)
 563                 priv->et_coalesce.rx_coalesce_usecs = 50;
 564         if (priv->et_coalesce.rx_max_coalesced_frames < 1)
 565                 priv->et_coalesce.rx_max_coalesced_frames = 1;
 566 
 567         netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_PKTCNT,
 568                      priv->et_coalesce.rx_max_coalesced_frames);
 569         netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_TMR,
 570                      priv->et_coalesce.rx_coalesce_usecs);
 571         netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_PKTCNT);
 572         netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_TMREXP);
 573 
 574         return 0;
 575 }
 576 
 577 static u32 netsec_et_get_msglevel(struct net_device *dev)
 578 {
 579         struct netsec_priv *priv = netdev_priv(dev);
 580 
 581         return priv->msg_enable;
 582 }
 583 
 584 static void netsec_et_set_msglevel(struct net_device *dev, u32 datum)
 585 {
 586         struct netsec_priv *priv = netdev_priv(dev);
 587 
 588         priv->msg_enable = datum;
 589 }
 590 
 591 static const struct ethtool_ops netsec_ethtool_ops = {
 592         .get_drvinfo            = netsec_et_get_drvinfo,
 593         .get_link_ksettings     = phy_ethtool_get_link_ksettings,
 594         .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 595         .get_link               = ethtool_op_get_link,
 596         .get_coalesce           = netsec_et_get_coalesce,
 597         .set_coalesce           = netsec_et_set_coalesce,
 598         .get_msglevel           = netsec_et_get_msglevel,
 599         .set_msglevel           = netsec_et_set_msglevel,
 600 };
 601 
 602 /************* NETDEV_OPS FOLLOW *************/
 603 
 604 
 605 static void netsec_set_rx_de(struct netsec_priv *priv,
 606                              struct netsec_desc_ring *dring, u16 idx,
 607                              const struct netsec_desc *desc)
 608 {
 609         struct netsec_de *de = dring->vaddr + DESC_SZ * idx;
 610         u32 attr = (1 << NETSEC_RX_PKT_OWN_FIELD) |
 611                    (1 << NETSEC_RX_PKT_FS_FIELD) |
 612                    (1 << NETSEC_RX_PKT_LS_FIELD);
 613 
 614         if (idx == DESC_NUM - 1)
 615                 attr |= (1 << NETSEC_RX_PKT_LD_FIELD);
 616 
 617         de->data_buf_addr_up = upper_32_bits(desc->dma_addr);
 618         de->data_buf_addr_lw = lower_32_bits(desc->dma_addr);
 619         de->buf_len_info = desc->len;
 620         de->attr = attr;
 621         dma_wmb();
 622 
 623         dring->desc[idx].dma_addr = desc->dma_addr;
 624         dring->desc[idx].addr = desc->addr;
 625         dring->desc[idx].len = desc->len;
 626 }
 627 
 628 static bool netsec_clean_tx_dring(struct netsec_priv *priv)
 629 {
 630         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
 631         struct netsec_de *entry;
 632         int tail = dring->tail;
 633         unsigned int bytes;
 634         int cnt = 0;
 635 
 636         spin_lock(&dring->lock);
 637 
 638         bytes = 0;
 639         entry = dring->vaddr + DESC_SZ * tail;
 640 
 641         while (!(entry->attr & (1U << NETSEC_TX_SHIFT_OWN_FIELD)) &&
 642                cnt < DESC_NUM) {
 643                 struct netsec_desc *desc;
 644                 int eop;
 645 
 646                 desc = &dring->desc[tail];
 647                 eop = (entry->attr >> NETSEC_TX_LAST) & 1;
 648                 dma_rmb();
 649 
 650                 /* if buf_type is either TYPE_NETSEC_SKB or
 651                  * TYPE_NETSEC_XDP_NDO we mapped it
 652                  */
 653                 if (desc->buf_type != TYPE_NETSEC_XDP_TX)
 654                         dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
 655                                          DMA_TO_DEVICE);
 656 
 657                 if (!eop)
 658                         goto next;
 659 
 660                 if (desc->buf_type == TYPE_NETSEC_SKB) {
 661                         bytes += desc->skb->len;
 662                         dev_kfree_skb(desc->skb);
 663                 } else {
 664                         xdp_return_frame(desc->xdpf);
 665                 }
 666 next:
 667                 /* clean up so netsec_uninit_pkt_dring() won't free the skb
 668                  * again
 669                  */
 670                 *desc = (struct netsec_desc){};
 671 
 672                 /* entry->attr is not going to be accessed by the NIC until
 673                  * netsec_set_tx_de() is called. No need for a dma_wmb() here
 674                  */
 675                 entry->attr = 1U << NETSEC_TX_SHIFT_OWN_FIELD;
 676                 /* move tail ahead */
 677                 dring->tail = (tail + 1) % DESC_NUM;
 678 
 679                 tail = dring->tail;
 680                 entry = dring->vaddr + DESC_SZ * tail;
 681                 cnt++;
 682         }
 683 
 684         spin_unlock(&dring->lock);
 685 
 686         if (!cnt)
 687                 return false;
 688 
 689         /* reading the register clears the irq */
 690         netsec_read(priv, NETSEC_REG_NRM_TX_DONE_PKTCNT);
 691 
 692         priv->ndev->stats.tx_packets += cnt;
 693         priv->ndev->stats.tx_bytes += bytes;
 694 
 695         netdev_completed_queue(priv->ndev, cnt, bytes);
 696 
 697         return true;
 698 }
 699 
 700 static void netsec_process_tx(struct netsec_priv *priv)
 701 {
 702         struct net_device *ndev = priv->ndev;
 703         bool cleaned;
 704 
 705         cleaned = netsec_clean_tx_dring(priv);
 706 
 707         if (cleaned && netif_queue_stopped(ndev)) {
 708                 /* Make sure we update the value, anyone stopping the queue
 709                  * after this will read the proper consumer idx
 710                  */
 711                 smp_wmb();
 712                 netif_wake_queue(ndev);
 713         }
 714 }
 715 
 716 static void *netsec_alloc_rx_data(struct netsec_priv *priv,
 717                                   dma_addr_t *dma_handle, u16 *desc_len)
 718 
 719 {
 720 
 721         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
 722         enum dma_data_direction dma_dir;
 723         struct page *page;
 724 
 725         page = page_pool_dev_alloc_pages(dring->page_pool);
 726         if (!page)
 727                 return NULL;
 728 
 729         /* We allocate the same buffer length for XDP and non-XDP cases.
 730          * page_pool API will map the whole page, skip what's needed for
 731          * network payloads and/or XDP
 732          */
 733         *dma_handle = page_pool_get_dma_addr(page) + NETSEC_RXBUF_HEADROOM;
 734         /* Make sure the incoming payload fits in the page for XDP and non-XDP
 735          * cases and reserve enough space for headroom + skb_shared_info
 736          */
 737         *desc_len = PAGE_SIZE - NETSEC_RX_BUF_NON_DATA;
 738         dma_dir = page_pool_get_dma_dir(dring->page_pool);
 739         dma_sync_single_for_device(priv->dev, *dma_handle, *desc_len, dma_dir);
 740 
 741         return page_address(page);
 742 }
 743 
 744 static void netsec_rx_fill(struct netsec_priv *priv, u16 from, u16 num)
 745 {
 746         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
 747         u16 idx = from;
 748 
 749         while (num) {
 750                 netsec_set_rx_de(priv, dring, idx, &dring->desc[idx]);
 751                 idx++;
 752                 if (idx >= DESC_NUM)
 753                         idx = 0;
 754                 num--;
 755         }
 756 }
 757 
 758 static void netsec_xdp_ring_tx_db(struct netsec_priv *priv, u16 pkts)
 759 {
 760         if (likely(pkts))
 761                 netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, pkts);
 762 }
 763 
 764 static void netsec_finalize_xdp_rx(struct netsec_priv *priv, u32 xdp_res,
 765                                    u16 pkts)
 766 {
 767         if (xdp_res & NETSEC_XDP_REDIR)
 768                 xdp_do_flush_map();
 769 
 770         if (xdp_res & NETSEC_XDP_TX)
 771                 netsec_xdp_ring_tx_db(priv, pkts);
 772 }
 773 
 774 static void netsec_set_tx_de(struct netsec_priv *priv,
 775                              struct netsec_desc_ring *dring,
 776                              const struct netsec_tx_pkt_ctrl *tx_ctrl,
 777                              const struct netsec_desc *desc, void *buf)
 778 {
 779         int idx = dring->head;
 780         struct netsec_de *de;
 781         u32 attr;
 782 
 783         de = dring->vaddr + (DESC_SZ * idx);
 784 
 785         attr = (1 << NETSEC_TX_SHIFT_OWN_FIELD) |
 786                (1 << NETSEC_TX_SHIFT_PT_FIELD) |
 787                (NETSEC_RING_GMAC << NETSEC_TX_SHIFT_TDRID_FIELD) |
 788                (1 << NETSEC_TX_SHIFT_FS_FIELD) |
 789                (1 << NETSEC_TX_LAST) |
 790                (tx_ctrl->cksum_offload_flag << NETSEC_TX_SHIFT_CO) |
 791                (tx_ctrl->tcp_seg_offload_flag << NETSEC_TX_SHIFT_SO) |
 792                (1 << NETSEC_TX_SHIFT_TRS_FIELD);
 793         if (idx == DESC_NUM - 1)
 794                 attr |= (1 << NETSEC_TX_SHIFT_LD_FIELD);
 795 
 796         de->data_buf_addr_up = upper_32_bits(desc->dma_addr);
 797         de->data_buf_addr_lw = lower_32_bits(desc->dma_addr);
 798         de->buf_len_info = (tx_ctrl->tcp_seg_len << 16) | desc->len;
 799         de->attr = attr;
 800 
 801         dring->desc[idx] = *desc;
 802         if (desc->buf_type == TYPE_NETSEC_SKB)
 803                 dring->desc[idx].skb = buf;
 804         else if (desc->buf_type == TYPE_NETSEC_XDP_TX ||
 805                  desc->buf_type == TYPE_NETSEC_XDP_NDO)
 806                 dring->desc[idx].xdpf = buf;
 807 
 808         /* move head ahead */
 809         dring->head = (dring->head + 1) % DESC_NUM;
 810 }
 811 
 812 /* The current driver only supports 1 Txq, this should run under spin_lock() */
 813 static u32 netsec_xdp_queue_one(struct netsec_priv *priv,
 814                                 struct xdp_frame *xdpf, bool is_ndo)
 815 
 816 {
 817         struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX];
 818         struct page *page = virt_to_page(xdpf->data);
 819         struct netsec_tx_pkt_ctrl tx_ctrl = {};
 820         struct netsec_desc tx_desc;
 821         dma_addr_t dma_handle;
 822         u16 filled;
 823 
 824         if (tx_ring->head >= tx_ring->tail)
 825                 filled = tx_ring->head - tx_ring->tail;
 826         else
 827                 filled = tx_ring->head + DESC_NUM - tx_ring->tail;
 828 
 829         if (DESC_NUM - filled <= 1)
 830                 return NETSEC_XDP_CONSUMED;
 831 
 832         if (is_ndo) {
 833                 /* this is for ndo_xdp_xmit, the buffer needs mapping before
 834                  * sending
 835                  */
 836                 dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len,
 837                                             DMA_TO_DEVICE);
 838                 if (dma_mapping_error(priv->dev, dma_handle))
 839                         return NETSEC_XDP_CONSUMED;
 840                 tx_desc.buf_type = TYPE_NETSEC_XDP_NDO;
 841         } else {
 842                 /* This is the device Rx buffer from page_pool. No need to remap
 843                  * just sync and send it
 844                  */
 845                 struct netsec_desc_ring *rx_ring =
 846                         &priv->desc_ring[NETSEC_RING_RX];
 847                 enum dma_data_direction dma_dir =
 848                         page_pool_get_dma_dir(rx_ring->page_pool);
 849 
 850                 dma_handle = page_pool_get_dma_addr(page) + xdpf->headroom +
 851                         sizeof(*xdpf);
 852                 dma_sync_single_for_device(priv->dev, dma_handle, xdpf->len,
 853                                            dma_dir);
 854                 tx_desc.buf_type = TYPE_NETSEC_XDP_TX;
 855         }
 856 
 857         tx_desc.dma_addr = dma_handle;
 858         tx_desc.addr = xdpf->data;
 859         tx_desc.len = xdpf->len;
 860 
 861         netsec_set_tx_de(priv, tx_ring, &tx_ctrl, &tx_desc, xdpf);
 862 
 863         return NETSEC_XDP_TX;
 864 }
 865 
 866 static u32 netsec_xdp_xmit_back(struct netsec_priv *priv, struct xdp_buff *xdp)
 867 {
 868         struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX];
 869         struct xdp_frame *xdpf = convert_to_xdp_frame(xdp);
 870         u32 ret;
 871 
 872         if (unlikely(!xdpf))
 873                 return NETSEC_XDP_CONSUMED;
 874 
 875         spin_lock(&tx_ring->lock);
 876         ret = netsec_xdp_queue_one(priv, xdpf, false);
 877         spin_unlock(&tx_ring->lock);
 878 
 879         return ret;
 880 }
 881 
 882 static u32 netsec_run_xdp(struct netsec_priv *priv, struct bpf_prog *prog,
 883                           struct xdp_buff *xdp)
 884 {
 885         u32 ret = NETSEC_XDP_PASS;
 886         int err;
 887         u32 act;
 888 
 889         act = bpf_prog_run_xdp(prog, xdp);
 890 
 891         switch (act) {
 892         case XDP_PASS:
 893                 ret = NETSEC_XDP_PASS;
 894                 break;
 895         case XDP_TX:
 896                 ret = netsec_xdp_xmit_back(priv, xdp);
 897                 if (ret != NETSEC_XDP_TX)
 898                         xdp_return_buff(xdp);
 899                 break;
 900         case XDP_REDIRECT:
 901                 err = xdp_do_redirect(priv->ndev, xdp, prog);
 902                 if (!err) {
 903                         ret = NETSEC_XDP_REDIR;
 904                 } else {
 905                         ret = NETSEC_XDP_CONSUMED;
 906                         xdp_return_buff(xdp);
 907                 }
 908                 break;
 909         default:
 910                 bpf_warn_invalid_xdp_action(act);
 911                 /* fall through */
 912         case XDP_ABORTED:
 913                 trace_xdp_exception(priv->ndev, prog, act);
 914                 /* fall through -- handle aborts by dropping packet */
 915         case XDP_DROP:
 916                 ret = NETSEC_XDP_CONSUMED;
 917                 xdp_return_buff(xdp);
 918                 break;
 919         }
 920 
 921         return ret;
 922 }
 923 
 924 static int netsec_process_rx(struct netsec_priv *priv, int budget)
 925 {
 926         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
 927         struct net_device *ndev = priv->ndev;
 928         struct netsec_rx_pkt_info rx_info;
 929         enum dma_data_direction dma_dir;
 930         struct bpf_prog *xdp_prog;
 931         u16 xdp_xmit = 0;
 932         u32 xdp_act = 0;
 933         int done = 0;
 934 
 935         rcu_read_lock();
 936         xdp_prog = READ_ONCE(priv->xdp_prog);
 937         dma_dir = page_pool_get_dma_dir(dring->page_pool);
 938 
 939         while (done < budget) {
 940                 u16 idx = dring->tail;
 941                 struct netsec_de *de = dring->vaddr + (DESC_SZ * idx);
 942                 struct netsec_desc *desc = &dring->desc[idx];
 943                 struct page *page = virt_to_page(desc->addr);
 944                 u32 xdp_result = NETSEC_XDP_PASS;
 945                 struct sk_buff *skb = NULL;
 946                 u16 pkt_len, desc_len;
 947                 dma_addr_t dma_handle;
 948                 struct xdp_buff xdp;
 949                 void *buf_addr;
 950 
 951                 if (de->attr & (1U << NETSEC_RX_PKT_OWN_FIELD)) {
 952                         /* reading the register clears the irq */
 953                         netsec_read(priv, NETSEC_REG_NRM_RX_PKTCNT);
 954                         break;
 955                 }
 956 
 957                 /* This  barrier is needed to keep us from reading
 958                  * any other fields out of the netsec_de until we have
 959                  * verified the descriptor has been written back
 960                  */
 961                 dma_rmb();
 962                 done++;
 963 
 964                 pkt_len = de->buf_len_info >> 16;
 965                 rx_info.err_code = (de->attr >> NETSEC_RX_PKT_ERR_FIELD) &
 966                         NETSEC_RX_PKT_ERR_MASK;
 967                 rx_info.err_flag = (de->attr >> NETSEC_RX_PKT_ER_FIELD) & 1;
 968                 if (rx_info.err_flag) {
 969                         netif_err(priv, drv, priv->ndev,
 970                                   "%s: rx fail err(%d)\n", __func__,
 971                                   rx_info.err_code);
 972                         ndev->stats.rx_dropped++;
 973                         dring->tail = (dring->tail + 1) % DESC_NUM;
 974                         /* reuse buffer page frag */
 975                         netsec_rx_fill(priv, idx, 1);
 976                         continue;
 977                 }
 978                 rx_info.rx_cksum_result =
 979                         (de->attr >> NETSEC_RX_PKT_CO_FIELD) & 3;
 980 
 981                 /* allocate a fresh buffer and map it to the hardware.
 982                  * This will eventually replace the old buffer in the hardware
 983                  */
 984                 buf_addr = netsec_alloc_rx_data(priv, &dma_handle, &desc_len);
 985 
 986                 if (unlikely(!buf_addr))
 987                         break;
 988 
 989                 dma_sync_single_for_cpu(priv->dev, desc->dma_addr, pkt_len,
 990                                         dma_dir);
 991                 prefetch(desc->addr);
 992 
 993                 xdp.data_hard_start = desc->addr;
 994                 xdp.data = desc->addr + NETSEC_RXBUF_HEADROOM;
 995                 xdp_set_data_meta_invalid(&xdp);
 996                 xdp.data_end = xdp.data + pkt_len;
 997                 xdp.rxq = &dring->xdp_rxq;
 998 
 999                 if (xdp_prog) {
1000                         xdp_result = netsec_run_xdp(priv, xdp_prog, &xdp);
1001                         if (xdp_result != NETSEC_XDP_PASS) {
1002                                 xdp_act |= xdp_result;
1003                                 if (xdp_result == NETSEC_XDP_TX)
1004                                         xdp_xmit++;
1005                                 goto next;
1006                         }
1007                 }
1008                 skb = build_skb(desc->addr, desc->len + NETSEC_RX_BUF_NON_DATA);
1009 
1010                 if (unlikely(!skb)) {
1011                         /* If skb fails recycle_direct will either unmap and
1012                          * free the page or refill the cache depending on the
1013                          * cache state. Since we paid the allocation cost if
1014                          * building an skb fails try to put the page into cache
1015                          */
1016                         page_pool_recycle_direct(dring->page_pool, page);
1017                         netif_err(priv, drv, priv->ndev,
1018                                   "rx failed to build skb\n");
1019                         break;
1020                 }
1021                 page_pool_release_page(dring->page_pool, page);
1022 
1023                 skb_reserve(skb, xdp.data - xdp.data_hard_start);
1024                 skb_put(skb, xdp.data_end - xdp.data);
1025                 skb->protocol = eth_type_trans(skb, priv->ndev);
1026 
1027                 if (priv->rx_cksum_offload_flag &&
1028                     rx_info.rx_cksum_result == NETSEC_RX_CKSUM_OK)
1029                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1030 
1031 next:
1032                 if ((skb && napi_gro_receive(&priv->napi, skb) != GRO_DROP) ||
1033                     xdp_result & NETSEC_XDP_RX_OK) {
1034                         ndev->stats.rx_packets++;
1035                         ndev->stats.rx_bytes += xdp.data_end - xdp.data;
1036                 }
1037 
1038                 /* Update the descriptor with fresh buffers */
1039                 desc->len = desc_len;
1040                 desc->dma_addr = dma_handle;
1041                 desc->addr = buf_addr;
1042 
1043                 netsec_rx_fill(priv, idx, 1);
1044                 dring->tail = (dring->tail + 1) % DESC_NUM;
1045         }
1046         netsec_finalize_xdp_rx(priv, xdp_act, xdp_xmit);
1047 
1048         rcu_read_unlock();
1049 
1050         return done;
1051 }
1052 
1053 static int netsec_napi_poll(struct napi_struct *napi, int budget)
1054 {
1055         struct netsec_priv *priv;
1056         int done;
1057 
1058         priv = container_of(napi, struct netsec_priv, napi);
1059 
1060         netsec_process_tx(priv);
1061         done = netsec_process_rx(priv, budget);
1062 
1063         if (done < budget && napi_complete_done(napi, done)) {
1064                 unsigned long flags;
1065 
1066                 spin_lock_irqsave(&priv->reglock, flags);
1067                 netsec_write(priv, NETSEC_REG_INTEN_SET,
1068                              NETSEC_IRQ_RX | NETSEC_IRQ_TX);
1069                 spin_unlock_irqrestore(&priv->reglock, flags);
1070         }
1071 
1072         return done;
1073 }
1074 
1075 
1076 static int netsec_desc_used(struct netsec_desc_ring *dring)
1077 {
1078         int used;
1079 
1080         if (dring->head >= dring->tail)
1081                 used = dring->head - dring->tail;
1082         else
1083                 used = dring->head + DESC_NUM - dring->tail;
1084 
1085         return used;
1086 }
1087 
1088 static int netsec_check_stop_tx(struct netsec_priv *priv, int used)
1089 {
1090         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
1091 
1092         /* keep tail from touching the queue */
1093         if (DESC_NUM - used < 2) {
1094                 netif_stop_queue(priv->ndev);
1095 
1096                 /* Make sure we read the updated value in case
1097                  * descriptors got freed
1098                  */
1099                 smp_rmb();
1100 
1101                 used = netsec_desc_used(dring);
1102                 if (DESC_NUM - used < 2)
1103                         return NETDEV_TX_BUSY;
1104 
1105                 netif_wake_queue(priv->ndev);
1106         }
1107 
1108         return 0;
1109 }
1110 
1111 static netdev_tx_t netsec_netdev_start_xmit(struct sk_buff *skb,
1112                                             struct net_device *ndev)
1113 {
1114         struct netsec_priv *priv = netdev_priv(ndev);
1115         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
1116         struct netsec_tx_pkt_ctrl tx_ctrl = {};
1117         struct netsec_desc tx_desc;
1118         u16 tso_seg_len = 0;
1119         int filled;
1120 
1121         spin_lock_bh(&dring->lock);
1122         filled = netsec_desc_used(dring);
1123         if (netsec_check_stop_tx(priv, filled)) {
1124                 spin_unlock_bh(&dring->lock);
1125                 net_warn_ratelimited("%s %s Tx queue full\n",
1126                                      dev_name(priv->dev), ndev->name);
1127                 return NETDEV_TX_BUSY;
1128         }
1129 
1130         if (skb->ip_summed == CHECKSUM_PARTIAL)
1131                 tx_ctrl.cksum_offload_flag = true;
1132 
1133         if (skb_is_gso(skb))
1134                 tso_seg_len = skb_shinfo(skb)->gso_size;
1135 
1136         if (tso_seg_len > 0) {
1137                 if (skb->protocol == htons(ETH_P_IP)) {
1138                         ip_hdr(skb)->tot_len = 0;
1139                         tcp_hdr(skb)->check =
1140                                 ~tcp_v4_check(0, ip_hdr(skb)->saddr,
1141                                               ip_hdr(skb)->daddr, 0);
1142                 } else {
1143                         ipv6_hdr(skb)->payload_len = 0;
1144                         tcp_hdr(skb)->check =
1145                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1146                                                  &ipv6_hdr(skb)->daddr,
1147                                                  0, IPPROTO_TCP, 0);
1148                 }
1149 
1150                 tx_ctrl.tcp_seg_offload_flag = true;
1151                 tx_ctrl.tcp_seg_len = tso_seg_len;
1152         }
1153 
1154         tx_desc.dma_addr = dma_map_single(priv->dev, skb->data,
1155                                           skb_headlen(skb), DMA_TO_DEVICE);
1156         if (dma_mapping_error(priv->dev, tx_desc.dma_addr)) {
1157                 spin_unlock_bh(&dring->lock);
1158                 netif_err(priv, drv, priv->ndev,
1159                           "%s: DMA mapping failed\n", __func__);
1160                 ndev->stats.tx_dropped++;
1161                 dev_kfree_skb_any(skb);
1162                 return NETDEV_TX_OK;
1163         }
1164         tx_desc.addr = skb->data;
1165         tx_desc.len = skb_headlen(skb);
1166         tx_desc.buf_type = TYPE_NETSEC_SKB;
1167 
1168         skb_tx_timestamp(skb);
1169         netdev_sent_queue(priv->ndev, skb->len);
1170 
1171         netsec_set_tx_de(priv, dring, &tx_ctrl, &tx_desc, skb);
1172         spin_unlock_bh(&dring->lock);
1173         netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, 1); /* submit another tx */
1174 
1175         return NETDEV_TX_OK;
1176 }
1177 
1178 static void netsec_uninit_pkt_dring(struct netsec_priv *priv, int id)
1179 {
1180         struct netsec_desc_ring *dring = &priv->desc_ring[id];
1181         struct netsec_desc *desc;
1182         u16 idx;
1183 
1184         if (!dring->vaddr || !dring->desc)
1185                 return;
1186         for (idx = 0; idx < DESC_NUM; idx++) {
1187                 desc = &dring->desc[idx];
1188                 if (!desc->addr)
1189                         continue;
1190 
1191                 if (id == NETSEC_RING_RX) {
1192                         struct page *page = virt_to_page(desc->addr);
1193 
1194                         page_pool_put_page(dring->page_pool, page, false);
1195                 } else if (id == NETSEC_RING_TX) {
1196                         dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
1197                                          DMA_TO_DEVICE);
1198                         dev_kfree_skb(desc->skb);
1199                 }
1200         }
1201 
1202         /* Rx is currently using page_pool */
1203         if (id == NETSEC_RING_RX) {
1204                 if (xdp_rxq_info_is_reg(&dring->xdp_rxq))
1205                         xdp_rxq_info_unreg(&dring->xdp_rxq);
1206                 page_pool_destroy(dring->page_pool);
1207         }
1208 
1209         memset(dring->desc, 0, sizeof(struct netsec_desc) * DESC_NUM);
1210         memset(dring->vaddr, 0, DESC_SZ * DESC_NUM);
1211 
1212         dring->head = 0;
1213         dring->tail = 0;
1214 
1215         if (id == NETSEC_RING_TX)
1216                 netdev_reset_queue(priv->ndev);
1217 }
1218 
1219 static void netsec_free_dring(struct netsec_priv *priv, int id)
1220 {
1221         struct netsec_desc_ring *dring = &priv->desc_ring[id];
1222 
1223         if (dring->vaddr) {
1224                 dma_free_coherent(priv->dev, DESC_SZ * DESC_NUM,
1225                                   dring->vaddr, dring->desc_dma);
1226                 dring->vaddr = NULL;
1227         }
1228 
1229         kfree(dring->desc);
1230         dring->desc = NULL;
1231 }
1232 
1233 static int netsec_alloc_dring(struct netsec_priv *priv, enum ring_id id)
1234 {
1235         struct netsec_desc_ring *dring = &priv->desc_ring[id];
1236 
1237         dring->vaddr = dma_alloc_coherent(priv->dev, DESC_SZ * DESC_NUM,
1238                                           &dring->desc_dma, GFP_KERNEL);
1239         if (!dring->vaddr)
1240                 goto err;
1241 
1242         dring->desc = kcalloc(DESC_NUM, sizeof(*dring->desc), GFP_KERNEL);
1243         if (!dring->desc)
1244                 goto err;
1245 
1246         return 0;
1247 err:
1248         netsec_free_dring(priv, id);
1249 
1250         return -ENOMEM;
1251 }
1252 
1253 static void netsec_setup_tx_dring(struct netsec_priv *priv)
1254 {
1255         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
1256         int i;
1257 
1258         for (i = 0; i < DESC_NUM; i++) {
1259                 struct netsec_de *de;
1260 
1261                 de = dring->vaddr + (DESC_SZ * i);
1262                 /* de->attr is not going to be accessed by the NIC
1263                  * until netsec_set_tx_de() is called.
1264                  * No need for a dma_wmb() here
1265                  */
1266                 de->attr = 1U << NETSEC_TX_SHIFT_OWN_FIELD;
1267         }
1268 }
1269 
1270 static int netsec_setup_rx_dring(struct netsec_priv *priv)
1271 {
1272         struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
1273         struct bpf_prog *xdp_prog = READ_ONCE(priv->xdp_prog);
1274         struct page_pool_params pp_params = { 0 };
1275         int i, err;
1276 
1277         pp_params.order = 0;
1278         /* internal DMA mapping in page_pool */
1279         pp_params.flags = PP_FLAG_DMA_MAP;
1280         pp_params.pool_size = DESC_NUM;
1281         pp_params.nid = cpu_to_node(0);
1282         pp_params.dev = priv->dev;
1283         pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
1284 
1285         dring->page_pool = page_pool_create(&pp_params);
1286         if (IS_ERR(dring->page_pool)) {
1287                 err = PTR_ERR(dring->page_pool);
1288                 dring->page_pool = NULL;
1289                 goto err_out;
1290         }
1291 
1292         err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0);
1293         if (err)
1294                 goto err_out;
1295 
1296         err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_POOL,
1297                                          dring->page_pool);
1298         if (err)
1299                 goto err_out;
1300 
1301         for (i = 0; i < DESC_NUM; i++) {
1302                 struct netsec_desc *desc = &dring->desc[i];
1303                 dma_addr_t dma_handle;
1304                 void *buf;
1305                 u16 len;
1306 
1307                 buf = netsec_alloc_rx_data(priv, &dma_handle, &len);
1308 
1309                 if (!buf) {
1310                         err = -ENOMEM;
1311                         goto err_out;
1312                 }
1313                 desc->dma_addr = dma_handle;
1314                 desc->addr = buf;
1315                 desc->len = len;
1316         }
1317 
1318         netsec_rx_fill(priv, 0, DESC_NUM);
1319 
1320         return 0;
1321 
1322 err_out:
1323         netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
1324         return err;
1325 }
1326 
1327 static int netsec_netdev_load_ucode_region(struct netsec_priv *priv, u32 reg,
1328                                            u32 addr_h, u32 addr_l, u32 size)
1329 {
1330         u64 base = (u64)addr_h << 32 | addr_l;
1331         void __iomem *ucode;
1332         u32 i;
1333 
1334         ucode = ioremap(base, size * sizeof(u32));
1335         if (!ucode)
1336                 return -ENOMEM;
1337 
1338         for (i = 0; i < size; i++)
1339                 netsec_write(priv, reg, readl(ucode + i * 4));
1340 
1341         iounmap(ucode);
1342         return 0;
1343 }
1344 
1345 static int netsec_netdev_load_microcode(struct netsec_priv *priv)
1346 {
1347         u32 addr_h, addr_l, size;
1348         int err;
1349 
1350         addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_H);
1351         addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_L);
1352         size = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_SIZE);
1353         err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_HM_CMD_BUF,
1354                                               addr_h, addr_l, size);
1355         if (err)
1356                 return err;
1357 
1358         addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_H);
1359         addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_L);
1360         size = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_SIZE);
1361         err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_MH_CMD_BUF,
1362                                               addr_h, addr_l, size);
1363         if (err)
1364                 return err;
1365 
1366         addr_h = 0;
1367         addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_ADDRESS);
1368         size = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_SIZE);
1369         err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_PKT_CMD_BUF,
1370                                               addr_h, addr_l, size);
1371         if (err)
1372                 return err;
1373 
1374         return 0;
1375 }
1376 
1377 static int netsec_reset_hardware(struct netsec_priv *priv,
1378                                  bool load_ucode)
1379 {
1380         u32 value;
1381         int err;
1382 
1383         /* stop DMA engines */
1384         if (!netsec_read(priv, NETSEC_REG_ADDR_DIS_CORE)) {
1385                 netsec_write(priv, NETSEC_REG_DMA_HM_CTRL,
1386                              NETSEC_DMA_CTRL_REG_STOP);
1387                 netsec_write(priv, NETSEC_REG_DMA_MH_CTRL,
1388                              NETSEC_DMA_CTRL_REG_STOP);
1389 
1390                 while (netsec_read(priv, NETSEC_REG_DMA_HM_CTRL) &
1391                        NETSEC_DMA_CTRL_REG_STOP)
1392                         cpu_relax();
1393 
1394                 while (netsec_read(priv, NETSEC_REG_DMA_MH_CTRL) &
1395                        NETSEC_DMA_CTRL_REG_STOP)
1396                         cpu_relax();
1397         }
1398 
1399         netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RESET);
1400         netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RUN);
1401         netsec_write(priv, NETSEC_REG_COM_INIT, NETSEC_COM_INIT_REG_ALL);
1402 
1403         while (netsec_read(priv, NETSEC_REG_COM_INIT) != 0)
1404                 cpu_relax();
1405 
1406         /* set desc_start addr */
1407         netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_UP,
1408                      upper_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma));
1409         netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_LW,
1410                      lower_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma));
1411 
1412         netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_UP,
1413                      upper_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma));
1414         netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_LW,
1415                      lower_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma));
1416 
1417         /* set normal tx dring ring config */
1418         netsec_write(priv, NETSEC_REG_NRM_TX_CONFIG,
1419                      1 << NETSEC_REG_DESC_ENDIAN);
1420         netsec_write(priv, NETSEC_REG_NRM_RX_CONFIG,
1421                      1 << NETSEC_REG_DESC_ENDIAN);
1422 
1423         if (load_ucode) {
1424                 err = netsec_netdev_load_microcode(priv);
1425                 if (err) {
1426                         netif_err(priv, probe, priv->ndev,
1427                                   "%s: failed to load microcode (%d)\n",
1428                                   __func__, err);
1429                         return err;
1430                 }
1431         }
1432 
1433         /* start DMA engines */
1434         netsec_write(priv, NETSEC_REG_DMA_TMR_CTRL, priv->freq / 1000000 - 1);
1435         netsec_write(priv, NETSEC_REG_ADDR_DIS_CORE, 0);
1436 
1437         usleep_range(1000, 2000);
1438 
1439         if (!(netsec_read(priv, NETSEC_REG_TOP_STATUS) &
1440               NETSEC_TOP_IRQ_REG_CODE_LOAD_END)) {
1441                 netif_err(priv, probe, priv->ndev,
1442                           "microengine start failed\n");
1443                 return -ENXIO;
1444         }
1445         netsec_write(priv, NETSEC_REG_TOP_STATUS,
1446                      NETSEC_TOP_IRQ_REG_CODE_LOAD_END);
1447 
1448         value = NETSEC_PKT_CTRL_REG_MODE_NRM;
1449         if (priv->ndev->mtu > ETH_DATA_LEN)
1450                 value |= NETSEC_PKT_CTRL_REG_EN_JUMBO;
1451 
1452         /* change to normal mode */
1453         netsec_write(priv, NETSEC_REG_DMA_MH_CTRL, MH_CTRL__MODE_TRANS);
1454         netsec_write(priv, NETSEC_REG_PKT_CTRL, value);
1455 
1456         while ((netsec_read(priv, NETSEC_REG_MODE_TRANS_COMP_STATUS) &
1457                 NETSEC_MODE_TRANS_COMP_IRQ_T2N) == 0)
1458                 cpu_relax();
1459 
1460         /* clear any pending EMPTY/ERR irq status */
1461         netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, ~0);
1462 
1463         /* Disable TX & RX intr */
1464         netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0);
1465 
1466         return 0;
1467 }
1468 
1469 static int netsec_start_gmac(struct netsec_priv *priv)
1470 {
1471         struct phy_device *phydev = priv->ndev->phydev;
1472         u32 value = 0;
1473         int ret;
1474 
1475         if (phydev->speed != SPEED_1000)
1476                 value = (NETSEC_GMAC_MCR_REG_CST |
1477                          NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON);
1478 
1479         if (netsec_mac_write(priv, GMAC_REG_MCR, value))
1480                 return -ETIMEDOUT;
1481         if (netsec_mac_write(priv, GMAC_REG_BMR,
1482                              NETSEC_GMAC_BMR_REG_RESET))
1483                 return -ETIMEDOUT;
1484 
1485         /* Wait soft reset */
1486         usleep_range(1000, 5000);
1487 
1488         ret = netsec_mac_read(priv, GMAC_REG_BMR, &value);
1489         if (ret)
1490                 return ret;
1491         if (value & NETSEC_GMAC_BMR_REG_SWR)
1492                 return -EAGAIN;
1493 
1494         netsec_write(priv, MAC_REG_DESC_SOFT_RST, 1);
1495         if (netsec_wait_while_busy(priv, MAC_REG_DESC_SOFT_RST, 1))
1496                 return -ETIMEDOUT;
1497 
1498         netsec_write(priv, MAC_REG_DESC_INIT, 1);
1499         if (netsec_wait_while_busy(priv, MAC_REG_DESC_INIT, 1))
1500                 return -ETIMEDOUT;
1501 
1502         if (netsec_mac_write(priv, GMAC_REG_BMR,
1503                              NETSEC_GMAC_BMR_REG_COMMON))
1504                 return -ETIMEDOUT;
1505         if (netsec_mac_write(priv, GMAC_REG_RDLAR,
1506                              NETSEC_GMAC_RDLAR_REG_COMMON))
1507                 return -ETIMEDOUT;
1508         if (netsec_mac_write(priv, GMAC_REG_TDLAR,
1509                              NETSEC_GMAC_TDLAR_REG_COMMON))
1510                 return -ETIMEDOUT;
1511         if (netsec_mac_write(priv, GMAC_REG_MFFR, 0x80000001))
1512                 return -ETIMEDOUT;
1513 
1514         ret = netsec_mac_update_to_phy_state(priv);
1515         if (ret)
1516                 return ret;
1517 
1518         ret = netsec_mac_read(priv, GMAC_REG_OMR, &value);
1519         if (ret)
1520                 return ret;
1521 
1522         value |= NETSEC_GMAC_OMR_REG_SR;
1523         value |= NETSEC_GMAC_OMR_REG_ST;
1524 
1525         netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0);
1526         netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0);
1527 
1528         netsec_et_set_coalesce(priv->ndev, &priv->et_coalesce);
1529 
1530         if (netsec_mac_write(priv, GMAC_REG_OMR, value))
1531                 return -ETIMEDOUT;
1532 
1533         return 0;
1534 }
1535 
1536 static int netsec_stop_gmac(struct netsec_priv *priv)
1537 {
1538         u32 value;
1539         int ret;
1540 
1541         ret = netsec_mac_read(priv, GMAC_REG_OMR, &value);
1542         if (ret)
1543                 return ret;
1544         value &= ~NETSEC_GMAC_OMR_REG_SR;
1545         value &= ~NETSEC_GMAC_OMR_REG_ST;
1546 
1547         /* disable all interrupts */
1548         netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0);
1549         netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0);
1550 
1551         return netsec_mac_write(priv, GMAC_REG_OMR, value);
1552 }
1553 
1554 static void netsec_phy_adjust_link(struct net_device *ndev)
1555 {
1556         struct netsec_priv *priv = netdev_priv(ndev);
1557 
1558         if (ndev->phydev->link)
1559                 netsec_start_gmac(priv);
1560         else
1561                 netsec_stop_gmac(priv);
1562 
1563         phy_print_status(ndev->phydev);
1564 }
1565 
1566 static irqreturn_t netsec_irq_handler(int irq, void *dev_id)
1567 {
1568         struct netsec_priv *priv = dev_id;
1569         u32 val, status = netsec_read(priv, NETSEC_REG_TOP_STATUS);
1570         unsigned long flags;
1571 
1572         /* Disable interrupts */
1573         if (status & NETSEC_IRQ_TX) {
1574                 val = netsec_read(priv, NETSEC_REG_NRM_TX_STATUS);
1575                 netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, val);
1576         }
1577         if (status & NETSEC_IRQ_RX) {
1578                 val = netsec_read(priv, NETSEC_REG_NRM_RX_STATUS);
1579                 netsec_write(priv, NETSEC_REG_NRM_RX_STATUS, val);
1580         }
1581 
1582         spin_lock_irqsave(&priv->reglock, flags);
1583         netsec_write(priv, NETSEC_REG_INTEN_CLR, NETSEC_IRQ_RX | NETSEC_IRQ_TX);
1584         spin_unlock_irqrestore(&priv->reglock, flags);
1585 
1586         napi_schedule(&priv->napi);
1587 
1588         return IRQ_HANDLED;
1589 }
1590 
1591 static int netsec_netdev_open(struct net_device *ndev)
1592 {
1593         struct netsec_priv *priv = netdev_priv(ndev);
1594         int ret;
1595 
1596         pm_runtime_get_sync(priv->dev);
1597 
1598         netsec_setup_tx_dring(priv);
1599         ret = netsec_setup_rx_dring(priv);
1600         if (ret) {
1601                 netif_err(priv, probe, priv->ndev,
1602                           "%s: fail setup ring\n", __func__);
1603                 goto err1;
1604         }
1605 
1606         ret = request_irq(priv->ndev->irq, netsec_irq_handler,
1607                           IRQF_SHARED, "netsec", priv);
1608         if (ret) {
1609                 netif_err(priv, drv, priv->ndev, "request_irq failed\n");
1610                 goto err2;
1611         }
1612 
1613         if (dev_of_node(priv->dev)) {
1614                 if (!of_phy_connect(priv->ndev, priv->phy_np,
1615                                     netsec_phy_adjust_link, 0,
1616                                     priv->phy_interface)) {
1617                         netif_err(priv, link, priv->ndev, "missing PHY\n");
1618                         ret = -ENODEV;
1619                         goto err3;
1620                 }
1621         } else {
1622                 ret = phy_connect_direct(priv->ndev, priv->phydev,
1623                                          netsec_phy_adjust_link,
1624                                          priv->phy_interface);
1625                 if (ret) {
1626                         netif_err(priv, link, priv->ndev,
1627                                   "phy_connect_direct() failed (%d)\n", ret);
1628                         goto err3;
1629                 }
1630         }
1631 
1632         phy_start(ndev->phydev);
1633 
1634         netsec_start_gmac(priv);
1635         napi_enable(&priv->napi);
1636         netif_start_queue(ndev);
1637 
1638         /* Enable TX+RX intr. */
1639         netsec_write(priv, NETSEC_REG_INTEN_SET, NETSEC_IRQ_RX | NETSEC_IRQ_TX);
1640 
1641         return 0;
1642 err3:
1643         free_irq(priv->ndev->irq, priv);
1644 err2:
1645         netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
1646 err1:
1647         pm_runtime_put_sync(priv->dev);
1648         return ret;
1649 }
1650 
1651 static int netsec_netdev_stop(struct net_device *ndev)
1652 {
1653         int ret;
1654         struct netsec_priv *priv = netdev_priv(ndev);
1655 
1656         netif_stop_queue(priv->ndev);
1657         dma_wmb();
1658 
1659         napi_disable(&priv->napi);
1660 
1661         netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0);
1662         netsec_stop_gmac(priv);
1663 
1664         free_irq(priv->ndev->irq, priv);
1665 
1666         netsec_uninit_pkt_dring(priv, NETSEC_RING_TX);
1667         netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
1668 
1669         phy_stop(ndev->phydev);
1670         phy_disconnect(ndev->phydev);
1671 
1672         ret = netsec_reset_hardware(priv, false);
1673 
1674         pm_runtime_put_sync(priv->dev);
1675 
1676         return ret;
1677 }
1678 
1679 static int netsec_netdev_init(struct net_device *ndev)
1680 {
1681         struct netsec_priv *priv = netdev_priv(ndev);
1682         int ret;
1683         u16 data;
1684 
1685         BUILD_BUG_ON_NOT_POWER_OF_2(DESC_NUM);
1686 
1687         ret = netsec_alloc_dring(priv, NETSEC_RING_TX);
1688         if (ret)
1689                 return ret;
1690 
1691         ret = netsec_alloc_dring(priv, NETSEC_RING_RX);
1692         if (ret)
1693                 goto err1;
1694 
1695         /* set phy power down */
1696         data = netsec_phy_read(priv->mii_bus, priv->phy_addr, MII_BMCR) |
1697                 BMCR_PDOWN;
1698         netsec_phy_write(priv->mii_bus, priv->phy_addr, MII_BMCR, data);
1699 
1700         ret = netsec_reset_hardware(priv, true);
1701         if (ret)
1702                 goto err2;
1703 
1704         spin_lock_init(&priv->desc_ring[NETSEC_RING_TX].lock);
1705         spin_lock_init(&priv->desc_ring[NETSEC_RING_RX].lock);
1706 
1707         return 0;
1708 err2:
1709         netsec_free_dring(priv, NETSEC_RING_RX);
1710 err1:
1711         netsec_free_dring(priv, NETSEC_RING_TX);
1712         return ret;
1713 }
1714 
1715 static void netsec_netdev_uninit(struct net_device *ndev)
1716 {
1717         struct netsec_priv *priv = netdev_priv(ndev);
1718 
1719         netsec_free_dring(priv, NETSEC_RING_RX);
1720         netsec_free_dring(priv, NETSEC_RING_TX);
1721 }
1722 
1723 static int netsec_netdev_set_features(struct net_device *ndev,
1724                                       netdev_features_t features)
1725 {
1726         struct netsec_priv *priv = netdev_priv(ndev);
1727 
1728         priv->rx_cksum_offload_flag = !!(features & NETIF_F_RXCSUM);
1729 
1730         return 0;
1731 }
1732 
1733 static int netsec_netdev_ioctl(struct net_device *ndev, struct ifreq *ifr,
1734                                int cmd)
1735 {
1736         return phy_mii_ioctl(ndev->phydev, ifr, cmd);
1737 }
1738 
1739 static int netsec_xdp_xmit(struct net_device *ndev, int n,
1740                            struct xdp_frame **frames, u32 flags)
1741 {
1742         struct netsec_priv *priv = netdev_priv(ndev);
1743         struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX];
1744         int drops = 0;
1745         int i;
1746 
1747         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1748                 return -EINVAL;
1749 
1750         spin_lock(&tx_ring->lock);
1751         for (i = 0; i < n; i++) {
1752                 struct xdp_frame *xdpf = frames[i];
1753                 int err;
1754 
1755                 err = netsec_xdp_queue_one(priv, xdpf, true);
1756                 if (err != NETSEC_XDP_TX) {
1757                         xdp_return_frame_rx_napi(xdpf);
1758                         drops++;
1759                 } else {
1760                         tx_ring->xdp_xmit++;
1761                 }
1762         }
1763         spin_unlock(&tx_ring->lock);
1764 
1765         if (unlikely(flags & XDP_XMIT_FLUSH)) {
1766                 netsec_xdp_ring_tx_db(priv, tx_ring->xdp_xmit);
1767                 tx_ring->xdp_xmit = 0;
1768         }
1769 
1770         return n - drops;
1771 }
1772 
1773 static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
1774                             struct netlink_ext_ack *extack)
1775 {
1776         struct net_device *dev = priv->ndev;
1777         struct bpf_prog *old_prog;
1778 
1779         /* For now just support only the usual MTU sized frames */
1780         if (prog && dev->mtu > 1500) {
1781                 NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
1782                 return -EOPNOTSUPP;
1783         }
1784 
1785         if (netif_running(dev))
1786                 netsec_netdev_stop(dev);
1787 
1788         /* Detach old prog, if any */
1789         old_prog = xchg(&priv->xdp_prog, prog);
1790         if (old_prog)
1791                 bpf_prog_put(old_prog);
1792 
1793         if (netif_running(dev))
1794                 netsec_netdev_open(dev);
1795 
1796         return 0;
1797 }
1798 
1799 static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp)
1800 {
1801         struct netsec_priv *priv = netdev_priv(ndev);
1802 
1803         switch (xdp->command) {
1804         case XDP_SETUP_PROG:
1805                 return netsec_xdp_setup(priv, xdp->prog, xdp->extack);
1806         case XDP_QUERY_PROG:
1807                 xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
1808                 return 0;
1809         default:
1810                 return -EINVAL;
1811         }
1812 }
1813 
1814 static const struct net_device_ops netsec_netdev_ops = {
1815         .ndo_init               = netsec_netdev_init,
1816         .ndo_uninit             = netsec_netdev_uninit,
1817         .ndo_open               = netsec_netdev_open,
1818         .ndo_stop               = netsec_netdev_stop,
1819         .ndo_start_xmit         = netsec_netdev_start_xmit,
1820         .ndo_set_features       = netsec_netdev_set_features,
1821         .ndo_set_mac_address    = eth_mac_addr,
1822         .ndo_validate_addr      = eth_validate_addr,
1823         .ndo_do_ioctl           = netsec_netdev_ioctl,
1824         .ndo_xdp_xmit           = netsec_xdp_xmit,
1825         .ndo_bpf                = netsec_xdp,
1826 };
1827 
1828 static int netsec_of_probe(struct platform_device *pdev,
1829                            struct netsec_priv *priv, u32 *phy_addr)
1830 {
1831         priv->phy_np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0);
1832         if (!priv->phy_np) {
1833                 dev_err(&pdev->dev, "missing required property 'phy-handle'\n");
1834                 return -EINVAL;
1835         }
1836 
1837         *phy_addr = of_mdio_parse_addr(&pdev->dev, priv->phy_np);
1838 
1839         priv->clk = devm_clk_get(&pdev->dev, NULL); /* get by 'phy_ref_clk' */
1840         if (IS_ERR(priv->clk)) {
1841                 dev_err(&pdev->dev, "phy_ref_clk not found\n");
1842                 return PTR_ERR(priv->clk);
1843         }
1844         priv->freq = clk_get_rate(priv->clk);
1845 
1846         return 0;
1847 }
1848 
1849 static int netsec_acpi_probe(struct platform_device *pdev,
1850                              struct netsec_priv *priv, u32 *phy_addr)
1851 {
1852         int ret;
1853 
1854         if (!IS_ENABLED(CONFIG_ACPI))
1855                 return -ENODEV;
1856 
1857         ret = device_property_read_u32(&pdev->dev, "phy-channel", phy_addr);
1858         if (ret) {
1859                 dev_err(&pdev->dev,
1860                         "missing required property 'phy-channel'\n");
1861                 return ret;
1862         }
1863 
1864         ret = device_property_read_u32(&pdev->dev,
1865                                        "socionext,phy-clock-frequency",
1866                                        &priv->freq);
1867         if (ret)
1868                 dev_err(&pdev->dev,
1869                         "missing required property 'socionext,phy-clock-frequency'\n");
1870         return ret;
1871 }
1872 
1873 static void netsec_unregister_mdio(struct netsec_priv *priv)
1874 {
1875         struct phy_device *phydev = priv->phydev;
1876 
1877         if (!dev_of_node(priv->dev) && phydev) {
1878                 phy_device_remove(phydev);
1879                 phy_device_free(phydev);
1880         }
1881 
1882         mdiobus_unregister(priv->mii_bus);
1883 }
1884 
1885 static int netsec_register_mdio(struct netsec_priv *priv, u32 phy_addr)
1886 {
1887         struct mii_bus *bus;
1888         int ret;
1889 
1890         bus = devm_mdiobus_alloc(priv->dev);
1891         if (!bus)
1892                 return -ENOMEM;
1893 
1894         snprintf(bus->id, MII_BUS_ID_SIZE, "%s", dev_name(priv->dev));
1895         bus->priv = priv;
1896         bus->name = "SNI NETSEC MDIO";
1897         bus->read = netsec_phy_read;
1898         bus->write = netsec_phy_write;
1899         bus->parent = priv->dev;
1900         priv->mii_bus = bus;
1901 
1902         if (dev_of_node(priv->dev)) {
1903                 struct device_node *mdio_node, *parent = dev_of_node(priv->dev);
1904 
1905                 mdio_node = of_get_child_by_name(parent, "mdio");
1906                 if (mdio_node) {
1907                         parent = mdio_node;
1908                 } else {
1909                         /* older f/w doesn't populate the mdio subnode,
1910                          * allow relaxed upgrade of f/w in due time.
1911                          */
1912                         dev_info(priv->dev, "Upgrade f/w for mdio subnode!\n");
1913                 }
1914 
1915                 ret = of_mdiobus_register(bus, parent);
1916                 of_node_put(mdio_node);
1917 
1918                 if (ret) {
1919                         dev_err(priv->dev, "mdiobus register err(%d)\n", ret);
1920                         return ret;
1921                 }
1922         } else {
1923                 /* Mask out all PHYs from auto probing. */
1924                 bus->phy_mask = ~0;
1925                 ret = mdiobus_register(bus);
1926                 if (ret) {
1927                         dev_err(priv->dev, "mdiobus register err(%d)\n", ret);
1928                         return ret;
1929                 }
1930 
1931                 priv->phydev = get_phy_device(bus, phy_addr, false);
1932                 if (IS_ERR(priv->phydev)) {
1933                         ret = PTR_ERR(priv->phydev);
1934                         dev_err(priv->dev, "get_phy_device err(%d)\n", ret);
1935                         priv->phydev = NULL;
1936                         return -ENODEV;
1937                 }
1938 
1939                 ret = phy_device_register(priv->phydev);
1940                 if (ret) {
1941                         mdiobus_unregister(bus);
1942                         dev_err(priv->dev,
1943                                 "phy_device_register err(%d)\n", ret);
1944                 }
1945         }
1946 
1947         return ret;
1948 }
1949 
1950 static int netsec_probe(struct platform_device *pdev)
1951 {
1952         struct resource *mmio_res, *eeprom_res, *irq_res;
1953         u8 *mac, macbuf[ETH_ALEN];
1954         struct netsec_priv *priv;
1955         u32 hw_ver, phy_addr = 0;
1956         struct net_device *ndev;
1957         int ret;
1958 
1959         mmio_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1960         if (!mmio_res) {
1961                 dev_err(&pdev->dev, "No MMIO resource found.\n");
1962                 return -ENODEV;
1963         }
1964 
1965         eeprom_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1966         if (!eeprom_res) {
1967                 dev_info(&pdev->dev, "No EEPROM resource found.\n");
1968                 return -ENODEV;
1969         }
1970 
1971         irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1972         if (!irq_res) {
1973                 dev_err(&pdev->dev, "No IRQ resource found.\n");
1974                 return -ENODEV;
1975         }
1976 
1977         ndev = alloc_etherdev(sizeof(*priv));
1978         if (!ndev)
1979                 return -ENOMEM;
1980 
1981         priv = netdev_priv(ndev);
1982 
1983         spin_lock_init(&priv->reglock);
1984         SET_NETDEV_DEV(ndev, &pdev->dev);
1985         platform_set_drvdata(pdev, priv);
1986         ndev->irq = irq_res->start;
1987         priv->dev = &pdev->dev;
1988         priv->ndev = ndev;
1989 
1990         priv->msg_enable = NETIF_MSG_TX_ERR | NETIF_MSG_HW | NETIF_MSG_DRV |
1991                            NETIF_MSG_LINK | NETIF_MSG_PROBE;
1992 
1993         priv->phy_interface = device_get_phy_mode(&pdev->dev);
1994         if ((int)priv->phy_interface < 0) {
1995                 dev_err(&pdev->dev, "missing required property 'phy-mode'\n");
1996                 ret = -ENODEV;
1997                 goto free_ndev;
1998         }
1999 
2000         priv->ioaddr = devm_ioremap(&pdev->dev, mmio_res->start,
2001                                     resource_size(mmio_res));
2002         if (!priv->ioaddr) {
2003                 dev_err(&pdev->dev, "devm_ioremap() failed\n");
2004                 ret = -ENXIO;
2005                 goto free_ndev;
2006         }
2007 
2008         priv->eeprom_base = devm_ioremap(&pdev->dev, eeprom_res->start,
2009                                          resource_size(eeprom_res));
2010         if (!priv->eeprom_base) {
2011                 dev_err(&pdev->dev, "devm_ioremap() failed for EEPROM\n");
2012                 ret = -ENXIO;
2013                 goto free_ndev;
2014         }
2015 
2016         mac = device_get_mac_address(&pdev->dev, macbuf, sizeof(macbuf));
2017         if (mac)
2018                 ether_addr_copy(ndev->dev_addr, mac);
2019 
2020         if (priv->eeprom_base &&
2021             (!mac || !is_valid_ether_addr(ndev->dev_addr))) {
2022                 void __iomem *macp = priv->eeprom_base +
2023                                         NETSEC_EEPROM_MAC_ADDRESS;
2024 
2025                 ndev->dev_addr[0] = readb(macp + 3);
2026                 ndev->dev_addr[1] = readb(macp + 2);
2027                 ndev->dev_addr[2] = readb(macp + 1);
2028                 ndev->dev_addr[3] = readb(macp + 0);
2029                 ndev->dev_addr[4] = readb(macp + 7);
2030                 ndev->dev_addr[5] = readb(macp + 6);
2031         }
2032 
2033         if (!is_valid_ether_addr(ndev->dev_addr)) {
2034                 dev_warn(&pdev->dev, "No MAC address found, using random\n");
2035                 eth_hw_addr_random(ndev);
2036         }
2037 
2038         if (dev_of_node(&pdev->dev))
2039                 ret = netsec_of_probe(pdev, priv, &phy_addr);
2040         else
2041                 ret = netsec_acpi_probe(pdev, priv, &phy_addr);
2042         if (ret)
2043                 goto free_ndev;
2044 
2045         priv->phy_addr = phy_addr;
2046 
2047         if (!priv->freq) {
2048                 dev_err(&pdev->dev, "missing PHY reference clock frequency\n");
2049                 ret = -ENODEV;
2050                 goto free_ndev;
2051         }
2052 
2053         /* default for throughput */
2054         priv->et_coalesce.rx_coalesce_usecs = 500;
2055         priv->et_coalesce.rx_max_coalesced_frames = 8;
2056         priv->et_coalesce.tx_coalesce_usecs = 500;
2057         priv->et_coalesce.tx_max_coalesced_frames = 8;
2058 
2059         ret = device_property_read_u32(&pdev->dev, "max-frame-size",
2060                                        &ndev->max_mtu);
2061         if (ret < 0)
2062                 ndev->max_mtu = ETH_DATA_LEN;
2063 
2064         /* runtime_pm coverage just for probe, open/close also cover it */
2065         pm_runtime_enable(&pdev->dev);
2066         pm_runtime_get_sync(&pdev->dev);
2067 
2068         hw_ver = netsec_read(priv, NETSEC_REG_F_TAIKI_VER);
2069         /* this driver only supports F_TAIKI style NETSEC */
2070         if (NETSEC_F_NETSEC_VER_MAJOR_NUM(hw_ver) !=
2071             NETSEC_F_NETSEC_VER_MAJOR_NUM(NETSEC_REG_NETSEC_VER_F_TAIKI)) {
2072                 ret = -ENODEV;
2073                 goto pm_disable;
2074         }
2075 
2076         dev_info(&pdev->dev, "hardware revision %d.%d\n",
2077                  hw_ver >> 16, hw_ver & 0xffff);
2078 
2079         netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_POLL_WEIGHT);
2080 
2081         ndev->netdev_ops = &netsec_netdev_ops;
2082         ndev->ethtool_ops = &netsec_ethtool_ops;
2083 
2084         ndev->features |= NETIF_F_HIGHDMA | NETIF_F_RXCSUM | NETIF_F_GSO |
2085                                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2086         ndev->hw_features = ndev->features;
2087 
2088         priv->rx_cksum_offload_flag = true;
2089 
2090         ret = netsec_register_mdio(priv, phy_addr);
2091         if (ret)
2092                 goto unreg_napi;
2093 
2094         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)))
2095                 dev_warn(&pdev->dev, "Failed to set DMA mask\n");
2096 
2097         ret = register_netdev(ndev);
2098         if (ret) {
2099                 netif_err(priv, probe, ndev, "register_netdev() failed\n");
2100                 goto unreg_mii;
2101         }
2102 
2103         pm_runtime_put_sync(&pdev->dev);
2104         return 0;
2105 
2106 unreg_mii:
2107         netsec_unregister_mdio(priv);
2108 unreg_napi:
2109         netif_napi_del(&priv->napi);
2110 pm_disable:
2111         pm_runtime_put_sync(&pdev->dev);
2112         pm_runtime_disable(&pdev->dev);
2113 free_ndev:
2114         free_netdev(ndev);
2115         dev_err(&pdev->dev, "init failed\n");
2116 
2117         return ret;
2118 }
2119 
2120 static int netsec_remove(struct platform_device *pdev)
2121 {
2122         struct netsec_priv *priv = platform_get_drvdata(pdev);
2123 
2124         unregister_netdev(priv->ndev);
2125 
2126         netsec_unregister_mdio(priv);
2127 
2128         netif_napi_del(&priv->napi);
2129 
2130         pm_runtime_disable(&pdev->dev);
2131         free_netdev(priv->ndev);
2132 
2133         return 0;
2134 }
2135 
2136 #ifdef CONFIG_PM
2137 static int netsec_runtime_suspend(struct device *dev)
2138 {
2139         struct netsec_priv *priv = dev_get_drvdata(dev);
2140 
2141         netsec_write(priv, NETSEC_REG_CLK_EN, 0);
2142 
2143         clk_disable_unprepare(priv->clk);
2144 
2145         return 0;
2146 }
2147 
2148 static int netsec_runtime_resume(struct device *dev)
2149 {
2150         struct netsec_priv *priv = dev_get_drvdata(dev);
2151 
2152         clk_prepare_enable(priv->clk);
2153 
2154         netsec_write(priv, NETSEC_REG_CLK_EN, NETSEC_CLK_EN_REG_DOM_D |
2155                                                NETSEC_CLK_EN_REG_DOM_C |
2156                                                NETSEC_CLK_EN_REG_DOM_G);
2157         return 0;
2158 }
2159 #endif
2160 
2161 static const struct dev_pm_ops netsec_pm_ops = {
2162         SET_RUNTIME_PM_OPS(netsec_runtime_suspend, netsec_runtime_resume, NULL)
2163 };
2164 
2165 static const struct of_device_id netsec_dt_ids[] = {
2166         { .compatible = "socionext,synquacer-netsec" },
2167         { }
2168 };
2169 MODULE_DEVICE_TABLE(of, netsec_dt_ids);
2170 
2171 #ifdef CONFIG_ACPI
2172 static const struct acpi_device_id netsec_acpi_ids[] = {
2173         { "SCX0001" },
2174         { }
2175 };
2176 MODULE_DEVICE_TABLE(acpi, netsec_acpi_ids);
2177 #endif
2178 
2179 static struct platform_driver netsec_driver = {
2180         .probe  = netsec_probe,
2181         .remove = netsec_remove,
2182         .driver = {
2183                 .name = "netsec",
2184                 .pm = &netsec_pm_ops,
2185                 .of_match_table = netsec_dt_ids,
2186                 .acpi_match_table = ACPI_PTR(netsec_acpi_ids),
2187         },
2188 };
2189 module_platform_driver(netsec_driver);
2190 
2191 MODULE_AUTHOR("Jassi Brar <jaswinder.singh@linaro.org>");
2192 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
2193 MODULE_DESCRIPTION("NETSEC Ethernet driver");
2194 MODULE_LICENSE("GPL");

/* [<][>][^][v][top][bottom][index][help] */