71 #define nr64(reg)		readq(np->regs + (reg))
72 #define nw64(reg, val)		writeq((val), np->regs + (reg))
74 #define nr64_mac(reg)		readq(np->mac_regs + (reg))
75 #define nw64_mac(reg, val)	writeq((val), np->mac_regs + (reg))
77 #define nr64_ipp(reg)		readq(np->regs + np->ipp_off + (reg))
78 #define nw64_ipp(reg, val)	writeq((val), np->regs + np->ipp_off + (reg))
80 #define nr64_pcs(reg)		readq(np->regs + np->pcs_off + (reg))
81 #define nw64_pcs(reg, val)	writeq((val), np->regs + np->pcs_off + (reg))
83 #define nr64_xpcs(reg)		readq(np->regs + np->xpcs_off + (reg))
84 #define nw64_xpcs(reg, val)	writeq((val), np->regs + np->xpcs_off + (reg))
93 #define niu_lock_parent(np, flags) \  argument
94 	spin_lock_irqsave(&np->parent->lock, flags)
95 #define niu_unlock_parent(np, flags) \  argument
96 	spin_unlock_irqrestore(&np->parent->lock, flags)
98 static int serdes_init_10g_serdes(struct niu *np);
100 static int __niu_wait_bits_clear_mac(struct niu *np, unsigned long reg,  in __niu_wait_bits_clear_mac()  argument
115 static int __niu_set_and_wait_clear_mac(struct niu *np, unsigned long reg,  in __niu_set_and_wait_clear_mac()  argument
122 	err = __niu_wait_bits_clear_mac(np, reg, bits, limit, delay);  in __niu_set_and_wait_clear_mac()
124 		netdev_err(np->dev, "bits (%llx) of register %s would not clear, val[%llx]\n",  in __niu_set_and_wait_clear_mac()
135 static int __niu_wait_bits_clear_ipp(struct niu *np, unsigned long reg,  in __niu_wait_bits_clear_ipp()  argument
150 static int __niu_set_and_wait_clear_ipp(struct niu *np, unsigned long reg,  in __niu_set_and_wait_clear_ipp()  argument
161 	err = __niu_wait_bits_clear_ipp(np, reg, bits, limit, delay);  in __niu_set_and_wait_clear_ipp()
163 		netdev_err(np->dev, "bits (%llx) of register %s would not clear, val[%llx]\n",  in __niu_set_and_wait_clear_ipp()
174 static int __niu_wait_bits_clear(struct niu *np, unsigned long reg,  in __niu_wait_bits_clear()  argument
194 static int __niu_set_and_wait_clear(struct niu *np, unsigned long reg,  in __niu_set_and_wait_clear()  argument
201 	err = __niu_wait_bits_clear(np, reg, bits, limit, delay);  in __niu_set_and_wait_clear()
203 		netdev_err(np->dev, "bits (%llx) of register %s would not clear, val[%llx]\n",  in __niu_set_and_wait_clear()
214 static void niu_ldg_rearm(struct niu *np, struct niu_ldg *lp, int on)  in niu_ldg_rearm()  argument
224 static int niu_ldn_irq_enable(struct niu *np, int ldn, int on)  in niu_ldn_irq_enable()  argument
250 static int niu_enable_ldn_in_ldg(struct niu *np, struct niu_ldg *lp, int on)  in niu_enable_ldn_in_ldg()  argument
252 	struct niu_parent *parent = np->parent;  in niu_enable_ldn_in_ldg()
261 		err = niu_ldn_irq_enable(np, i, on);  in niu_enable_ldn_in_ldg()
268 static int niu_enable_interrupts(struct niu *np, int on)  in niu_enable_interrupts()  argument
272 	for (i = 0; i < np->num_ldg; i++) {  in niu_enable_interrupts()
273 		struct niu_ldg *lp = &np->ldg[i];  in niu_enable_interrupts()
276 		err = niu_enable_ldn_in_ldg(np, lp, on);  in niu_enable_interrupts()
280 	for (i = 0; i < np->num_ldg; i++)  in niu_enable_interrupts()
281 		niu_ldg_rearm(np, &np->ldg[i], on);  in niu_enable_interrupts()
296 static int mdio_wait(struct niu *np)  in mdio_wait()  argument
312 static int mdio_read(struct niu *np, int port, int dev, int reg)  in mdio_read()  argument
317 	err = mdio_wait(np);  in mdio_read()
322 	return mdio_wait(np);  in mdio_read()
325 static int mdio_write(struct niu *np, int port, int dev, int reg, int data)  in mdio_write()  argument
330 	err = mdio_wait(np);  in mdio_write()
335 	err = mdio_wait(np);  in mdio_write()
342 static int mii_read(struct niu *np, int port, int reg)  in mii_read()  argument
345 	return mdio_wait(np);  in mii_read()
348 static int mii_write(struct niu *np, int port, int reg, int data)  in mii_write()  argument
353 	err = mdio_wait(np);  in mii_write()
360 static int esr2_set_tx_cfg(struct niu *np, unsigned long channel, u32 val)  in esr2_set_tx_cfg()  argument
364 	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in esr2_set_tx_cfg()
368 		err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in esr2_set_tx_cfg()
374 static int esr2_set_rx_cfg(struct niu *np, unsigned long channel, u32 val)  in esr2_set_rx_cfg()  argument
378 	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in esr2_set_rx_cfg()
382 		err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in esr2_set_rx_cfg()
389 static int serdes_init_niu_10g_fiber(struct niu *np)  in serdes_init_niu_10g_fiber()  argument
391 	struct niu_link_config *lp = &np->link_config;  in serdes_init_niu_10g_fiber()
403 		mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in serdes_init_niu_10g_fiber()
412 		int err = esr2_set_tx_cfg(np, i, tx_cfg);  in serdes_init_niu_10g_fiber()
418 		int err = esr2_set_rx_cfg(np, i, rx_cfg);  in serdes_init_niu_10g_fiber()
426 static int serdes_init_niu_1g_serdes(struct niu *np)  in serdes_init_niu_1g_serdes()  argument
428 	struct niu_link_config *lp = &np->link_config;  in serdes_init_niu_1g_serdes()
442 	if (np->port == 0)  in serdes_init_niu_1g_serdes()
448 		mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in serdes_init_niu_1g_serdes()
458 	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in serdes_init_niu_1g_serdes()
461 		netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_CFG_L failed\n",  in serdes_init_niu_1g_serdes()
462 			   np->port, __func__);  in serdes_init_niu_1g_serdes()
468 	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in serdes_init_niu_1g_serdes()
471 		netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_STS_L failed\n",  in serdes_init_niu_1g_serdes()
472 			   np->port, __func__);  in serdes_init_niu_1g_serdes()
480 		err = esr2_set_tx_cfg(np, i, tx_cfg);  in serdes_init_niu_1g_serdes()
486 		err = esr2_set_rx_cfg(np, i, rx_cfg);  in serdes_init_niu_1g_serdes()
491 	switch (np->port) {  in serdes_init_niu_1g_serdes()
515 		netdev_err(np->dev, "Port %u signal bits [%08x] are not [%08x]\n",  in serdes_init_niu_1g_serdes()
516 			   np->port, (int)(sig & mask), (int)val);  in serdes_init_niu_1g_serdes()
523 static int serdes_init_niu_10g_serdes(struct niu *np)  in serdes_init_niu_10g_serdes()  argument
525 	struct niu_link_config *lp = &np->link_config;  in serdes_init_niu_10g_serdes()
540 		mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in serdes_init_niu_10g_serdes()
550 	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in serdes_init_niu_10g_serdes()
553 		netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_CFG_L failed\n",  in serdes_init_niu_10g_serdes()
554 			   np->port, __func__);  in serdes_init_niu_10g_serdes()
560 	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,  in serdes_init_niu_10g_serdes()
563 		netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_STS_L failed\n",  in serdes_init_niu_10g_serdes()
564 			   np->port, __func__);  in serdes_init_niu_10g_serdes()
572 		err = esr2_set_tx_cfg(np, i, tx_cfg);  in serdes_init_niu_10g_serdes()
578 		err = esr2_set_rx_cfg(np, i, rx_cfg);  in serdes_init_niu_10g_serdes()
585 	switch (np->port) {  in serdes_init_niu_10g_serdes()
622 			np->port, (int)(sig & mask), (int)val);  in serdes_init_niu_10g_serdes()
625 		err = serdes_init_niu_1g_serdes(np);  in serdes_init_niu_10g_serdes()
627 			np->flags &= ~NIU_FLAGS_10G;  in serdes_init_niu_10g_serdes()
628 			np->mac_xcvr = MAC_XCVR_PCS;  in serdes_init_niu_10g_serdes()
630 			netdev_err(np->dev, "Port %u 10G/1G SERDES Link Failed\n",  in serdes_init_niu_10g_serdes()
631 				   np->port);  in serdes_init_niu_10g_serdes()
638 static int esr_read_rxtx_ctrl(struct niu *np, unsigned long chan, u32 *val)  in esr_read_rxtx_ctrl()  argument
642 	err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR, ESR_RXTX_CTRL_L(chan));  in esr_read_rxtx_ctrl()
645 		err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,  in esr_read_rxtx_ctrl()
654 static int esr_read_glue0(struct niu *np, unsigned long chan, u32 *val)  in esr_read_glue0()  argument
658 	err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,  in esr_read_glue0()
662 		err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,  in esr_read_glue0()
672 static int esr_read_reset(struct niu *np, u32 *val)  in esr_read_reset()  argument
676 	err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,  in esr_read_reset()
680 		err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,  in esr_read_reset()
690 static int esr_write_rxtx_ctrl(struct niu *np, unsigned long chan, u32 val)  in esr_write_rxtx_ctrl()  argument
694 	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_write_rxtx_ctrl()
697 		err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_write_rxtx_ctrl()
702 static int esr_write_glue0(struct niu *np, unsigned long chan, u32 val)  in esr_write_glue0()  argument
706 	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_write_glue0()
709 		err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_write_glue0()
714 static int esr_reset(struct niu *np)  in esr_reset()  argument
719 	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_reset()
723 	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_reset()
729 	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_reset()
735 	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,  in esr_reset()
741 	err = esr_read_reset(np, &reset);  in esr_reset()
745 		netdev_err(np->dev, "Port %u ESR_RESET did not clear [%08x]\n",  in esr_reset()
746 			   np->port, reset);  in esr_reset()
753 static int serdes_init_10g(struct niu *np)  in serdes_init_10g()  argument
755 	struct niu_link_config *lp = &np->link_config;  in serdes_init_10g()
760 	switch (np->port) {  in serdes_init_10g()
805 		err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);  in serdes_init_10g()
808 		err = esr_read_glue0(np, i, &glue0);  in serdes_init_10g()
825 		err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);  in serdes_init_10g()
828 		err = esr_write_glue0(np, i, glue0);  in serdes_init_10g()
833 	err = esr_reset(np);  in serdes_init_10g()
838 	switch (np->port) {  in serdes_init_10g()
866 		if (np->flags & NIU_FLAGS_HOTPLUG_PHY) {  in serdes_init_10g()
867 			np->flags &= ~NIU_FLAGS_HOTPLUG_PHY_PRESENT;  in serdes_init_10g()
870 		netdev_err(np->dev, "Port %u signal bits [%08x] are not [%08x]\n",  in serdes_init_10g()
871 			   np->port, (int)(sig & mask), (int)val);  in serdes_init_10g()
874 	if (np->flags & NIU_FLAGS_HOTPLUG_PHY)  in serdes_init_10g()
875 		np->flags |= NIU_FLAGS_HOTPLUG_PHY_PRESENT;  in serdes_init_10g()
879 static int serdes_init_1g(struct niu *np)  in serdes_init_1g()  argument
885 	switch (np->port) {  in serdes_init_1g()
906 static int serdes_init_1g_serdes(struct niu *np)  in serdes_init_1g_serdes()  argument
908 	struct niu_link_config *lp = &np->link_config;  in serdes_init_1g_serdes()
917 	switch (np->port) {  in serdes_init_1g_serdes()
973 		err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);  in serdes_init_1g_serdes()
976 		err = esr_read_glue0(np, i, &glue0);  in serdes_init_1g_serdes()
993 		err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);  in serdes_init_1g_serdes()
996 		err = esr_write_glue0(np, i, glue0);  in serdes_init_1g_serdes()
1003 	switch (np->port) {  in serdes_init_1g_serdes()
1019 		netdev_err(np->dev, "Port %u signal bits [%08x] are not [%08x]\n",  in serdes_init_1g_serdes()
1020 			   np->port, (int)(sig & mask), (int)val);  in serdes_init_1g_serdes()
1027 static int link_status_1g_serdes(struct niu *np, int *link_up_p)  in link_status_1g_serdes()  argument
1029 	struct niu_link_config *lp = &np->link_config;  in link_status_1g_serdes()
1040 	spin_lock_irqsave(&np->lock, flags);  in link_status_1g_serdes()
1052 	spin_unlock_irqrestore(&np->lock, flags);  in link_status_1g_serdes()
1058 static int link_status_10g_serdes(struct niu *np, int *link_up_p)  in link_status_10g_serdes()  argument
1061 	struct niu_link_config *lp = &np->link_config;  in link_status_10g_serdes()
1068 	if (!(np->flags & NIU_FLAGS_10G))  in link_status_10g_serdes()
1069 		return link_status_1g_serdes(np, link_up_p);  in link_status_10g_serdes()
1073 	spin_lock_irqsave(&np->lock, flags);  in link_status_10g_serdes()
1087 	spin_unlock_irqrestore(&np->lock, flags);  in link_status_10g_serdes()
1092 static int link_status_mii(struct niu *np, int *link_up_p)  in link_status_mii()  argument
1094 	struct niu_link_config *lp = &np->link_config;  in link_status_mii()
1099 	err = mii_read(np, np->phy_addr, MII_BMCR);  in link_status_mii()
1104 	err = mii_read(np, np->phy_addr, MII_BMSR);  in link_status_mii()
1109 	err = mii_read(np, np->phy_addr, MII_ADVERTISE);  in link_status_mii()
1114 	err = mii_read(np, np->phy_addr, MII_LPA);  in link_status_mii()
1120 		err = mii_read(np, np->phy_addr, MII_ESTATUS);  in link_status_mii()
1125 		err = mii_read(np, np->phy_addr, MII_CTRL1000);  in link_status_mii()
1130 		err = mii_read(np, np->phy_addr, MII_STAT1000);  in link_status_mii()
1205 static int link_status_1g_rgmii(struct niu *np, int *link_up_p)  in link_status_1g_rgmii()  argument
1207 	struct niu_link_config *lp = &np->link_config;  in link_status_1g_rgmii()
1217 	spin_lock_irqsave(&np->lock, flags);  in link_status_1g_rgmii()
1221 	err = mii_read(np, np->phy_addr, MII_BMSR);  in link_status_1g_rgmii()
1229 		err = mii_read(np, np->phy_addr, MII_ADVERTISE);  in link_status_1g_rgmii()
1234 		err = mii_read(np, np->phy_addr, MII_LPA);  in link_status_1g_rgmii()
1239 		err = mii_read(np, np->phy_addr, MII_ESTATUS);  in link_status_1g_rgmii()
1252 	spin_unlock_irqrestore(&np->lock, flags);  in link_status_1g_rgmii()
1258 static int link_status_1g(struct niu *np, int *link_up_p)  in link_status_1g()  argument
1260 	struct niu_link_config *lp = &np->link_config;  in link_status_1g()
1264 	spin_lock_irqsave(&np->lock, flags);  in link_status_1g()
1266 	err = link_status_mii(np, link_up_p);  in link_status_1g()
1270 	spin_unlock_irqrestore(&np->lock, flags);  in link_status_1g()
1274 static int bcm8704_reset(struct niu *np)  in bcm8704_reset()  argument
1278 	err = mdio_read(np, np->phy_addr,  in bcm8704_reset()
1283 	err = mdio_write(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,  in bcm8704_reset()
1290 		err = mdio_read(np, np->phy_addr,  in bcm8704_reset()
1298 		netdev_err(np->dev, "Port %u PHY will not reset (bmcr=%04x)\n",  in bcm8704_reset()
1299 			   np->port, (err & 0xffff));  in bcm8704_reset()
1308 static int bcm8704_user_dev3_readback(struct niu *np, int reg)  in bcm8704_user_dev3_readback()  argument
1310 	int err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);  in bcm8704_user_dev3_readback()
1313 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);  in bcm8704_user_dev3_readback()
1319 static int bcm8706_init_user_dev3(struct niu *np)  in bcm8706_init_user_dev3()  argument
1324 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in bcm8706_init_user_dev3()
1331 	err = mdio_write(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in bcm8706_init_user_dev3()
1341 static int bcm8704_init_user_dev3(struct niu *np)  in bcm8704_init_user_dev3()  argument
1345 	err = mdio_write(np, np->phy_addr,  in bcm8704_init_user_dev3()
1359 	err = mdio_write(np, np->phy_addr,  in bcm8704_init_user_dev3()
1368 	err = bcm8704_user_dev3_readback(np, BCM8704_USER_CONTROL);  in bcm8704_init_user_dev3()
1371 	err = bcm8704_user_dev3_readback(np, BCM8704_USER_PMD_TX_CONTROL);  in bcm8704_init_user_dev3()
1375 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in bcm8704_init_user_dev3()
1381 	err = mdio_write(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in bcm8704_init_user_dev3()
1391 static int mrvl88x2011_act_led(struct niu *np, int val)  in mrvl88x2011_act_led()  argument
1395 	err  = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,  in mrvl88x2011_act_led()
1403 	return mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,  in mrvl88x2011_act_led()
1407 static int mrvl88x2011_led_blink_rate(struct niu *np, int rate)  in mrvl88x2011_led_blink_rate()  argument
1411 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,  in mrvl88x2011_led_blink_rate()
1417 		err = mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,  in mrvl88x2011_led_blink_rate()
1424 static int xcvr_init_10g_mrvl88x2011(struct niu *np)  in xcvr_init_10g_mrvl88x2011()  argument
1429 	err = mrvl88x2011_led_blink_rate(np, MRVL88X2011_LED_BLKRATE_134MS);  in xcvr_init_10g_mrvl88x2011()
1434 	err = mrvl88x2011_act_led(np, MRVL88X2011_LED_CTL_OFF);  in xcvr_init_10g_mrvl88x2011()
1438 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,  in xcvr_init_10g_mrvl88x2011()
1445 	err = mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,  in xcvr_init_10g_mrvl88x2011()
1450 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,  in xcvr_init_10g_mrvl88x2011()
1455 	if (np->link_config.loopback_mode == LOOPBACK_MAC)  in xcvr_init_10g_mrvl88x2011()
1460 	err = mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,  in xcvr_init_10g_mrvl88x2011()
1466 	return mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,  in xcvr_init_10g_mrvl88x2011()
1471 static int xcvr_diag_bcm870x(struct niu *np)  in xcvr_diag_bcm870x()  argument
1477 	err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,  in xcvr_diag_bcm870x()
1481 	pr_info("Port %u PMA_PMD(MII_STAT1000) [%04x]\n", np->port, err);  in xcvr_diag_bcm870x()
1483 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, 0x20);  in xcvr_diag_bcm870x()
1486 	pr_info("Port %u USER_DEV3(0x20) [%04x]\n", np->port, err);  in xcvr_diag_bcm870x()
1488 	err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,  in xcvr_diag_bcm870x()
1492 	pr_info("Port %u PHYXS(MII_NWAYTEST) [%04x]\n", np->port, err);  in xcvr_diag_bcm870x()
1496 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in xcvr_diag_bcm870x()
1500 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in xcvr_diag_bcm870x()
1506 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in xcvr_diag_bcm870x()
1510 	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,  in xcvr_diag_bcm870x()
1519 				np->port);  in xcvr_diag_bcm870x()
1522 				np->port);  in xcvr_diag_bcm870x()
1529 static int xcvr_10g_set_lb_bcm870x(struct niu *np)  in xcvr_10g_set_lb_bcm870x()  argument
1531 	struct niu_link_config *lp = &np->link_config;  in xcvr_10g_set_lb_bcm870x()
1534 	err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,  in xcvr_10g_set_lb_bcm870x()
1544 	err = mdio_write(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,  in xcvr_10g_set_lb_bcm870x()
1552 static int xcvr_init_10g_bcm8706(struct niu *np)  in xcvr_init_10g_bcm8706()  argument
1557 	if ((np->flags & NIU_FLAGS_HOTPLUG_PHY) &&  in xcvr_init_10g_bcm8706()
1558 	    (np->flags & NIU_FLAGS_HOTPLUG_PHY_PRESENT) == 0)  in xcvr_init_10g_bcm8706()
1570 	err = bcm8704_reset(np);  in xcvr_init_10g_bcm8706()
1574 	err = xcvr_10g_set_lb_bcm870x(np);  in xcvr_init_10g_bcm8706()
1578 	err = bcm8706_init_user_dev3(np);  in xcvr_init_10g_bcm8706()
1582 	err = xcvr_diag_bcm870x(np);  in xcvr_init_10g_bcm8706()
1589 static int xcvr_init_10g_bcm8704(struct niu *np)  in xcvr_init_10g_bcm8704()  argument
1593 	err = bcm8704_reset(np);  in xcvr_init_10g_bcm8704()
1597 	err = bcm8704_init_user_dev3(np);  in xcvr_init_10g_bcm8704()
1601 	err = xcvr_10g_set_lb_bcm870x(np);  in xcvr_init_10g_bcm8704()
1605 	err =  xcvr_diag_bcm870x(np);  in xcvr_init_10g_bcm8704()
1612 static int xcvr_init_10g(struct niu *np)  in xcvr_init_10g()  argument
1627 	phy_id = phy_decode(np->parent->port_phy, np->port);  in xcvr_init_10g()
1628 	phy_id = np->parent->phy_probe_info.phy_id[phy_id][np->port];  in xcvr_init_10g()
1633 		err = xcvr_init_10g_mrvl88x2011(np);  in xcvr_init_10g()
1637 		err = xcvr_init_10g_bcm8704(np);  in xcvr_init_10g()
1644 static int mii_reset(struct niu *np)  in mii_reset()  argument
1648 	err = mii_write(np, np->phy_addr, MII_BMCR, BMCR_RESET);  in mii_reset()
1655 		err = mii_read(np, np->phy_addr, MII_BMCR);  in mii_reset()
1662 		netdev_err(np->dev, "Port %u MII would not reset, bmcr[%04x]\n",  in mii_reset()
1663 			   np->port, err);  in mii_reset()
1670 static int xcvr_init_1g_rgmii(struct niu *np)  in xcvr_init_1g_rgmii()  argument
1680 	err = mii_reset(np);  in xcvr_init_1g_rgmii()
1684 	err = mii_read(np, np->phy_addr, MII_BMSR);  in xcvr_init_1g_rgmii()
1691 		err = mii_read(np, np->phy_addr, MII_ESTATUS);  in xcvr_init_1g_rgmii()
1698 	err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);  in xcvr_init_1g_rgmii()
1707 		err = mii_write(np, np->phy_addr, MII_CTRL1000, ctrl1000);  in xcvr_init_1g_rgmii()
1714 	err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);  in xcvr_init_1g_rgmii()
1718 	err = mii_read(np, np->phy_addr, MII_BMCR);  in xcvr_init_1g_rgmii()
1721 	bmcr = mii_read(np, np->phy_addr, MII_BMCR);  in xcvr_init_1g_rgmii()
1723 	err = mii_read(np, np->phy_addr, MII_BMSR);  in xcvr_init_1g_rgmii()
1730 static int mii_init_common(struct niu *np)  in mii_init_common()  argument
1732 	struct niu_link_config *lp = &np->link_config;  in mii_init_common()
1736 	err = mii_reset(np);  in mii_init_common()
1740 	err = mii_read(np, np->phy_addr, MII_BMSR);  in mii_init_common()
1747 		err = mii_read(np, np->phy_addr, MII_ESTATUS);  in mii_init_common()
1754 	err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);  in mii_init_common()
1771 		err = mii_write(np, np->phy_addr, BCM5464R_AUX_CTL, aux);  in mii_init_common()
1792 		err = mii_write(np, np->phy_addr, MII_ADVERTISE, adv);  in mii_init_common()
1804 			err = mii_write(np, np->phy_addr,  in mii_init_common()
1843 	err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);  in mii_init_common()
1848 	err = mii_read(np, np->phy_addr, MII_BMCR);  in mii_init_common()
1853 	err = mii_read(np, np->phy_addr, MII_BMSR);  in mii_init_common()
1859 		np->port, bmcr, bmsr);  in mii_init_common()
1865 static int xcvr_init_1g(struct niu *np)  in xcvr_init_1g()  argument
1874 	return mii_init_common(np);  in xcvr_init_1g()
1877 static int niu_xcvr_init(struct niu *np)  in niu_xcvr_init()  argument
1879 	const struct niu_phy_ops *ops = np->phy_ops;  in niu_xcvr_init()
1884 		err = ops->xcvr_init(np);  in niu_xcvr_init()
1889 static int niu_serdes_init(struct niu *np)  in niu_serdes_init()  argument
1891 	const struct niu_phy_ops *ops = np->phy_ops;  in niu_serdes_init()
1896 		err = ops->serdes_init(np);  in niu_serdes_init()
1904 static int niu_link_status_common(struct niu *np, int link_up)  in niu_link_status_common()  argument
1906 	struct niu_link_config *lp = &np->link_config;  in niu_link_status_common()
1907 	struct net_device *dev = np->dev;  in niu_link_status_common()
1911 		netif_info(np, link, dev, "Link is up at %s, %s duplex\n",  in niu_link_status_common()
1918 		spin_lock_irqsave(&np->lock, flags);  in niu_link_status_common()
1919 		niu_init_xif(np);  in niu_link_status_common()
1920 		niu_handle_led(np, 1);  in niu_link_status_common()
1921 		spin_unlock_irqrestore(&np->lock, flags);  in niu_link_status_common()
1925 		netif_warn(np, link, dev, "Link is down\n");  in niu_link_status_common()
1926 		spin_lock_irqsave(&np->lock, flags);  in niu_link_status_common()
1927 		niu_handle_led(np, 0);  in niu_link_status_common()
1928 		spin_unlock_irqrestore(&np->lock, flags);  in niu_link_status_common()
1935 static int link_status_10g_mrvl(struct niu *np, int *link_up_p)  in link_status_10g_mrvl()  argument
1941 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,  in link_status_10g_mrvl()
1947 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,  in link_status_10g_mrvl()
1955 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,  in link_status_10g_mrvl()
1960 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,  in link_status_10g_mrvl()
1968 	err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV4_ADDR,  in link_status_10g_mrvl()
1979 	np->link_config.active_speed = SPEED_10000;  in link_status_10g_mrvl()
1980 	np->link_config.active_duplex = DUPLEX_FULL;  in link_status_10g_mrvl()
1983 	mrvl88x2011_act_led(np, (link_up ?  in link_status_10g_mrvl()
1991 static int link_status_10g_bcm8706(struct niu *np, int *link_up_p)  in link_status_10g_bcm8706()  argument
1996 	err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,  in link_status_10g_bcm8706()
2005 	err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,  in link_status_10g_bcm8706()
2015 	err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,  in link_status_10g_bcm8706()
2027 		np->link_config.active_speed = SPEED_INVALID;  in link_status_10g_bcm8706()
2028 		np->link_config.active_duplex = DUPLEX_INVALID;  in link_status_10g_bcm8706()
2033 	np->link_config.active_speed = SPEED_10000;  in link_status_10g_bcm8706()
2034 	np->link_config.active_duplex = DUPLEX_FULL;  in link_status_10g_bcm8706()
2042 static int link_status_10g_bcom(struct niu *np, int *link_up_p)  in link_status_10g_bcom()  argument
2048 	err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,  in link_status_10g_bcom()
2057 	err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,  in link_status_10g_bcom()
2066 	err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,  in link_status_10g_bcom()
2082 	np->link_config.active_speed = SPEED_10000;  in link_status_10g_bcom()
2083 	np->link_config.active_duplex = DUPLEX_FULL;  in link_status_10g_bcom()
2091 static int link_status_10g(struct niu *np, int *link_up_p)  in link_status_10g()  argument
2096 	spin_lock_irqsave(&np->lock, flags);  in link_status_10g()
2098 	if (np->link_config.loopback_mode == LOOPBACK_DISABLED) {  in link_status_10g()
2101 		phy_id = phy_decode(np->parent->port_phy, np->port);  in link_status_10g()
2102 		phy_id = np->parent->phy_probe_info.phy_id[phy_id][np->port];  in link_status_10g()
2107 			err = link_status_10g_mrvl(np, link_up_p);  in link_status_10g()
2111 			err = link_status_10g_bcom(np, link_up_p);  in link_status_10g()
2116 	spin_unlock_irqrestore(&np->lock, flags);  in link_status_10g()
2121 static int niu_10g_phy_present(struct niu *np)  in niu_10g_phy_present()  argument
2126 	switch (np->port) {  in niu_10g_phy_present()
2158 static int link_status_10g_hotplug(struct niu *np, int *link_up_p)  in link_status_10g_hotplug()  argument
2165 	spin_lock_irqsave(&np->lock, flags);  in link_status_10g_hotplug()
2167 	if (np->link_config.loopback_mode == LOOPBACK_DISABLED) {  in link_status_10g_hotplug()
2168 		phy_present_prev = (np->flags & NIU_FLAGS_HOTPLUG_PHY_PRESENT) ?  in link_status_10g_hotplug()
2170 		phy_present = niu_10g_phy_present(np);  in link_status_10g_hotplug()
2175 				np->flags |= NIU_FLAGS_HOTPLUG_PHY_PRESENT;  in link_status_10g_hotplug()
2176 				if (np->phy_ops->xcvr_init)  in link_status_10g_hotplug()
2177 					err = np->phy_ops->xcvr_init(np);  in link_status_10g_hotplug()
2179 					err = mdio_read(np, np->phy_addr,  in link_status_10g_hotplug()
2186 					np->flags &= ~NIU_FLAGS_HOTPLUG_PHY_PRESENT;  in link_status_10g_hotplug()
2189 				np->flags &= ~NIU_FLAGS_HOTPLUG_PHY_PRESENT;  in link_status_10g_hotplug()
2191 				netif_warn(np, link, np->dev,  in link_status_10g_hotplug()
2196 		if (np->flags & NIU_FLAGS_HOTPLUG_PHY_PRESENT) {  in link_status_10g_hotplug()
2197 			err = link_status_10g_bcm8706(np, link_up_p);  in link_status_10g_hotplug()
2201 				np->link_config.active_speed = SPEED_10000;  in link_status_10g_hotplug()
2202 				np->link_config.active_duplex = DUPLEX_FULL;  in link_status_10g_hotplug()
2207 	spin_unlock_irqrestore(&np->lock, flags);  in link_status_10g_hotplug()
2212 static int niu_link_status(struct niu *np, int *link_up_p)  in niu_link_status()  argument
2214 	const struct niu_phy_ops *ops = np->phy_ops;  in niu_link_status()
2219 		err = ops->link_status(np, link_up_p);  in niu_link_status()
2226 	struct niu *np = (struct niu *) __opaque;  in niu_timer()  local
2230 	err = niu_link_status(np, &link_up);  in niu_timer()
2232 		niu_link_status_common(np, link_up);  in niu_timer()
2234 	if (netif_carrier_ok(np->dev))  in niu_timer()
2238 	np->timer.expires = jiffies + off;  in niu_timer()
2240 	add_timer(&np->timer);  in niu_timer()
2367 static int serdes_init_10g_serdes(struct niu *np)  in serdes_init_10g_serdes()  argument
2369 	struct niu_link_config *lp = &np->link_config;  in serdes_init_10g_serdes()
2373 	switch (np->port) {  in serdes_init_10g_serdes()
2413 	esr_reset(np);  in serdes_init_10g_serdes()
2423 		err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);  in serdes_init_10g_serdes()
2426 		err = esr_read_glue0(np, i, &glue0);  in serdes_init_10g_serdes()
2443 		err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);  in serdes_init_10g_serdes()
2446 		err = esr_write_glue0(np, i, glue0);  in serdes_init_10g_serdes()
2453 	switch (np->port) {  in serdes_init_10g_serdes()
2482 		err = serdes_init_1g_serdes(np);  in serdes_init_10g_serdes()
2484 			np->flags &= ~NIU_FLAGS_10G;  in serdes_init_10g_serdes()
2485 			np->mac_xcvr = MAC_XCVR_PCS;  in serdes_init_10g_serdes()
2487 			netdev_err(np->dev, "Port %u 10G/1G SERDES Link Failed\n",  in serdes_init_10g_serdes()
2488 				   np->port);  in serdes_init_10g_serdes()
2496 static int niu_determine_phy_disposition(struct niu *np)  in niu_determine_phy_disposition()  argument
2498 	struct niu_parent *parent = np->parent;  in niu_determine_phy_disposition()
2504 		switch (np->flags &  in niu_determine_phy_disposition()
2519 			if (np->flags & NIU_FLAGS_HOTPLUG_PHY) {  in niu_determine_phy_disposition()
2521 				if (np->port == 0)  in niu_determine_phy_disposition()
2523 				if (np->port == 1)  in niu_determine_phy_disposition()
2527 				phy_addr_off += np->port;  in niu_determine_phy_disposition()
2532 		switch (np->flags &  in niu_determine_phy_disposition()
2544 			phy_addr_off += (np->port ^ 0x3);  in niu_determine_phy_disposition()
2563 			phy_addr_off += np->port;  in niu_determine_phy_disposition()
2564 			if (np->flags & NIU_FLAGS_HOTPLUG_PHY) {  in niu_determine_phy_disposition()
2566 				if (np->port == 0)  in niu_determine_phy_disposition()
2568 				if (np->port == 1)  in niu_determine_phy_disposition()
2576 			switch(np->port) {  in niu_determine_phy_disposition()
2588 			phy_addr_off = niu_atca_port_num[np->port];  in niu_determine_phy_disposition()
2596 	np->phy_ops = tp->ops;  in niu_determine_phy_disposition()
2597 	np->phy_addr = tp->phy_addr_base + phy_addr_off;  in niu_determine_phy_disposition()
2602 static int niu_init_link(struct niu *np)  in niu_init_link()  argument
2604 	struct niu_parent *parent = np->parent;  in niu_init_link()
2608 		err = niu_xcvr_init(np);  in niu_init_link()
2613 	err = niu_serdes_init(np);  in niu_init_link()
2614 	if (err && !(np->flags & NIU_FLAGS_HOTPLUG_PHY))  in niu_init_link()
2617 	err = niu_xcvr_init(np);  in niu_init_link()
2618 	if (!err || (np->flags & NIU_FLAGS_HOTPLUG_PHY))  in niu_init_link()
2619 		niu_link_status(np, &ignore);  in niu_init_link()
2623 static void niu_set_primary_mac(struct niu *np, unsigned char *addr)  in niu_set_primary_mac()  argument
2629 	if (np->flags & NIU_FLAGS_XMAC) {  in niu_set_primary_mac()
2640 static int niu_num_alt_addr(struct niu *np)  in niu_num_alt_addr()  argument
2642 	if (np->flags & NIU_FLAGS_XMAC)  in niu_num_alt_addr()
2648 static int niu_set_alt_mac(struct niu *np, int index, unsigned char *addr)  in niu_set_alt_mac()  argument
2654 	if (index >= niu_num_alt_addr(np))  in niu_set_alt_mac()
2657 	if (np->flags & NIU_FLAGS_XMAC) {  in niu_set_alt_mac()
2670 static int niu_enable_alt_mac(struct niu *np, int index, int on)  in niu_enable_alt_mac()  argument
2675 	if (index >= niu_num_alt_addr(np))  in niu_enable_alt_mac()
2678 	if (np->flags & NIU_FLAGS_XMAC) {  in niu_enable_alt_mac()
2696 static void __set_rdc_table_num_hw(struct niu *np, unsigned long reg,  in __set_rdc_table_num_hw()  argument
2707 static int __set_rdc_table_num(struct niu *np,  in __set_rdc_table_num()  argument
2715 	if (np->flags & NIU_FLAGS_XMAC)  in __set_rdc_table_num()
2719 	__set_rdc_table_num_hw(np, reg, rdc_table_num, mac_pref);  in __set_rdc_table_num()
2723 static int niu_set_primary_mac_rdc_table(struct niu *np, int table_num,  in niu_set_primary_mac_rdc_table()  argument
2726 	return __set_rdc_table_num(np, 17, 0, table_num, mac_pref);  in niu_set_primary_mac_rdc_table()
2729 static int niu_set_multicast_mac_rdc_table(struct niu *np, int table_num,  in niu_set_multicast_mac_rdc_table()  argument
2732 	return __set_rdc_table_num(np, 16, 8, table_num, mac_pref);  in niu_set_multicast_mac_rdc_table()
2735 static int niu_set_alt_mac_rdc_table(struct niu *np, int idx,  in niu_set_alt_mac_rdc_table()  argument
2738 	if (idx >= niu_num_alt_addr(np))  in niu_set_alt_mac_rdc_table()
2740 	return __set_rdc_table_num(np, idx, idx + 1, table_num, mac_pref);  in niu_set_alt_mac_rdc_table()
2764 static void vlan_tbl_write(struct niu *np, unsigned long index,  in vlan_tbl_write()  argument
2782 static void vlan_tbl_clear(struct niu *np)  in vlan_tbl_clear()  argument
2790 static int tcam_wait_bit(struct niu *np, u64 bit)  in tcam_wait_bit()  argument
2805 static int tcam_flush(struct niu *np, int index)  in tcam_flush()  argument
2811 	return tcam_wait_bit(np, TCAM_CTL_STAT);  in tcam_flush()
2815 static int tcam_read(struct niu *np, int index,
2821 	err = tcam_wait_bit(np, TCAM_CTL_STAT);
2836 static int tcam_write(struct niu *np, int index,  in tcam_write()  argument
2849 	return tcam_wait_bit(np, TCAM_CTL_STAT);  in tcam_write()
2853 static int tcam_assoc_read(struct niu *np, int index, u64 *data)
2858 	err = tcam_wait_bit(np, TCAM_CTL_STAT);
2866 static int tcam_assoc_write(struct niu *np, int index, u64 assoc_data)  in tcam_assoc_write()  argument
2871 	return tcam_wait_bit(np, TCAM_CTL_STAT);  in tcam_assoc_write()
2874 static void tcam_enable(struct niu *np, int on)  in tcam_enable()  argument
2885 static void tcam_set_lat_and_ratio(struct niu *np, u64 latency, u64 ratio)  in tcam_set_lat_and_ratio()  argument
2901 static int tcam_user_eth_class_enable(struct niu *np, unsigned long class,  in tcam_user_eth_class_enable()  argument
2923 static int tcam_user_eth_class_set(struct niu *np, unsigned long class,
2944 static int tcam_user_ip_class_enable(struct niu *np, unsigned long class,  in tcam_user_ip_class_enable()  argument
2965 static int tcam_user_ip_class_set(struct niu *np, unsigned long class,  in tcam_user_ip_class_set()  argument
2993 static int tcam_early_init(struct niu *np)  in tcam_early_init()  argument
2998 	tcam_enable(np, 0);  in tcam_early_init()
2999 	tcam_set_lat_and_ratio(np,  in tcam_early_init()
3003 		err = tcam_user_eth_class_enable(np, i, 0);  in tcam_early_init()
3008 		err = tcam_user_ip_class_enable(np, i, 0);  in tcam_early_init()
3016 static int tcam_flush_all(struct niu *np)  in tcam_flush_all()  argument
3020 	for (i = 0; i < np->parent->tcam_num_entries; i++) {  in tcam_flush_all()
3021 		int err = tcam_flush(np, i);  in tcam_flush_all()
3034 static int hash_read(struct niu *np, unsigned long partition,
3053 static int hash_write(struct niu *np, unsigned long partition,  in hash_write()  argument
3071 static void fflp_reset(struct niu *np)  in fflp_reset()  argument
3083 static void fflp_set_timings(struct niu *np)  in fflp_set_timings()  argument
3102 static int fflp_set_partition(struct niu *np, u64 partition,  in fflp_set_partition()  argument
3126 static int fflp_disable_all_partitions(struct niu *np)  in fflp_disable_all_partitions()  argument
3131 		int err = fflp_set_partition(np, 0, 0, 0, 0);  in fflp_disable_all_partitions()
3138 static void fflp_llcsnap_enable(struct niu *np, int on)  in fflp_llcsnap_enable()  argument
3149 static void fflp_errors_enable(struct niu *np, int on)  in fflp_errors_enable()  argument
3160 static int fflp_hash_clear(struct niu *np)  in fflp_hash_clear()  argument
3170 		int err = hash_write(np, 0, i, 1, (u64 *) &ent);  in fflp_hash_clear()
3177 static int fflp_early_init(struct niu *np)  in fflp_early_init()  argument
3183 	niu_lock_parent(np, flags);  in fflp_early_init()
3185 	parent = np->parent;  in fflp_early_init()
3188 		if (np->parent->plat_type != PLAT_TYPE_NIU) {  in fflp_early_init()
3189 			fflp_reset(np);  in fflp_early_init()
3190 			fflp_set_timings(np);  in fflp_early_init()
3191 			err = fflp_disable_all_partitions(np);  in fflp_early_init()
3193 				netif_printk(np, probe, KERN_DEBUG, np->dev,  in fflp_early_init()
3200 		err = tcam_early_init(np);  in fflp_early_init()
3202 			netif_printk(np, probe, KERN_DEBUG, np->dev,  in fflp_early_init()
3206 		fflp_llcsnap_enable(np, 1);  in fflp_early_init()
3207 		fflp_errors_enable(np, 0);  in fflp_early_init()
3211 		err = tcam_flush_all(np);  in fflp_early_init()
3213 			netif_printk(np, probe, KERN_DEBUG, np->dev,  in fflp_early_init()
3217 		if (np->parent->plat_type != PLAT_TYPE_NIU) {  in fflp_early_init()
3218 			err = fflp_hash_clear(np);  in fflp_early_init()
3220 				netif_printk(np, probe, KERN_DEBUG, np->dev,  in fflp_early_init()
3227 		vlan_tbl_clear(np);  in fflp_early_init()
3232 	niu_unlock_parent(np, flags);  in fflp_early_init()
3236 static int niu_set_flow_key(struct niu *np, unsigned long class_code, u64 key)  in niu_set_flow_key()  argument
3246 static int niu_set_tcam_key(struct niu *np, unsigned long class_code, u64 key)  in niu_set_tcam_key()  argument
3257 static u16 tcam_get_index(struct niu *np, u16 idx)  in tcam_get_index()  argument
3260 	if (idx >= (np->clas.tcam_sz - 1))  in tcam_get_index()
3262 	return np->clas.tcam_top + ((idx+1) * np->parent->num_ports);  in tcam_get_index()
3265 static u16 tcam_get_size(struct niu *np)  in tcam_get_size()  argument
3268 	return np->clas.tcam_sz - 1;  in tcam_get_size()
3271 static u16 tcam_get_valid_entry_cnt(struct niu *np)  in tcam_get_valid_entry_cnt()  argument
3274 	return np->clas.tcam_valid_entries - 1;  in tcam_get_valid_entry_cnt()
3324 static int niu_rbr_add_page(struct niu *np, struct rx_ring_info *rp,  in niu_rbr_add_page()  argument
3335 	addr = np->ops->map_page(np->device, page, 0,  in niu_rbr_add_page()
3356 static void niu_rbr_refill(struct niu *np, struct rx_ring_info *rp, gfp_t mask)  in niu_rbr_refill()  argument
3362 		int err = niu_rbr_add_page(np, rp, mask, index);  in niu_rbr_refill()
3381 static int niu_rx_pkt_ignore(struct niu *np, struct rx_ring_info *rp)  in niu_rx_pkt_ignore()  argument
3403 			np->ops->unmap_page(np->device, page->index,  in niu_rx_pkt_ignore()
3421 static int niu_process_rx_pkt(struct napi_struct *napi, struct niu *np,  in niu_process_rx_pkt()  argument
3429 	skb = netdev_alloc_skb(np->dev, RX_SKB_ALLOC_SIZE);  in niu_process_rx_pkt()
3431 		return niu_rx_pkt_ignore(np, rp);  in niu_process_rx_pkt()
3473 			np->ops->unmap_page(np->device, page->index,  in niu_process_rx_pkt()
3493 	if (np->dev->features & NETIF_F_RXHASH)  in niu_process_rx_pkt()
3505 	skb->protocol = eth_type_trans(skb, np->dev);  in niu_process_rx_pkt()
3512 static int niu_rbr_fill(struct niu *np, struct rx_ring_info *rp, gfp_t mask)  in niu_rbr_fill()  argument
3519 		err = niu_rbr_add_page(np, rp, mask, index);  in niu_rbr_fill()
3530 static void niu_rbr_free(struct niu *np, struct rx_ring_info *rp)  in niu_rbr_free()  argument
3542 			np->ops->unmap_page(np->device, base, PAGE_SIZE,  in niu_rbr_free()
3558 static int release_tx_packet(struct niu *np, struct tx_ring_info *rp, int idx)  in release_tx_packet()  argument
3574 	np->ops->unmap_single(np->device, tb->mapping,  in release_tx_packet()
3589 		np->ops->unmap_page(np->device, tb->mapping,  in release_tx_packet()
3602 static void niu_tx_work(struct niu *np, struct tx_ring_info *rp)  in niu_tx_work()  argument
3609 	index = (rp - np->tx_rings);  in niu_tx_work()
3610 	txq = netdev_get_tx_queue(np->dev, index);  in niu_tx_work()
3624 	netif_printk(np, tx_done, KERN_DEBUG, np->dev,  in niu_tx_work()
3628 		cons = release_tx_packet(np, rp, cons);  in niu_tx_work()
3644 static inline void niu_sync_rx_discard_stats(struct niu *np,  in niu_sync_rx_discard_stats()  argument
3673 			dev_err(np->device, "rx-%d: Counter overflow RXMISC discard\n",  in niu_sync_rx_discard_stats()
3676 		netif_printk(np, rx_err, KERN_DEBUG, np->dev,  in niu_sync_rx_discard_stats()
3688 			dev_err(np->device, "rx-%d: Counter overflow WRED discard\n", rx_channel);  in niu_sync_rx_discard_stats()
3690 		netif_printk(np, rx_err, KERN_DEBUG, np->dev,  in niu_sync_rx_discard_stats()
3696 static int niu_rx_work(struct napi_struct *napi, struct niu *np,  in niu_rx_work()  argument
3713 	netif_printk(np, rx_status, KERN_DEBUG, np->dev,  in niu_rx_work()
3720 		rcr_done += niu_process_rx_pkt(napi, np, rp);  in niu_rx_work()
3728 			niu_rbr_refill(np, rp, GFP_ATOMIC);  in niu_rx_work()
3740 		niu_sync_rx_discard_stats(np, rp, 0x7FFF);  in niu_rx_work()
3745 static int niu_poll_core(struct niu *np, struct niu_ldg *lp, int budget)  in niu_poll_core()  argument
3752 	netif_printk(np, intr, KERN_DEBUG, np->dev,  in niu_poll_core()
3755 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_poll_core()
3756 		struct tx_ring_info *rp = &np->tx_rings[i];  in niu_poll_core()
3758 			niu_tx_work(np, rp);  in niu_poll_core()
3762 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_poll_core()
3763 		struct rx_ring_info *rp = &np->rx_rings[i];  in niu_poll_core()
3768 			this_work_done = niu_rx_work(&lp->napi, np, rp,  in niu_poll_core()
3783 	struct niu *np = lp->np;  in niu_poll()  local
3786 	work_done = niu_poll_core(np, lp, budget);  in niu_poll()
3790 		niu_ldg_rearm(np, lp, 1);  in niu_poll()
3795 static void niu_log_rxchan_errors(struct niu *np, struct rx_ring_info *rp,  in niu_log_rxchan_errors()  argument
3798 	netdev_err(np->dev, "RX channel %u errors ( ", rp->rx_channel);  in niu_log_rxchan_errors()
3832 static int niu_rx_error(struct niu *np, struct rx_ring_info *rp)  in niu_rx_error()  argument
3843 		netdev_err(np->dev, "RX channel %u error, stat[%llx]\n",  in niu_rx_error()
3847 		niu_log_rxchan_errors(np, rp, stat);  in niu_rx_error()
3856 static void niu_log_txchan_errors(struct niu *np, struct tx_ring_info *rp,  in niu_log_txchan_errors()  argument
3859 	netdev_err(np->dev, "TX channel %u errors ( ", rp->tx_channel);  in niu_log_txchan_errors()
3881 static int niu_tx_error(struct niu *np, struct tx_ring_info *rp)  in niu_tx_error()  argument
3889 	netdev_err(np->dev, "TX channel %u error, cs[%llx] logh[%llx] logl[%llx]\n",  in niu_tx_error()
3895 	niu_log_txchan_errors(np, rp, cs);  in niu_tx_error()
3900 static int niu_mif_interrupt(struct niu *np)  in niu_mif_interrupt()  argument
3905 	if (np->flags & NIU_FLAGS_XMAC) {  in niu_mif_interrupt()
3912 	netdev_err(np->dev, "MIF interrupt, stat[%llx] phy_mdint(%d)\n",  in niu_mif_interrupt()
3918 static void niu_xmac_interrupt(struct niu *np)  in niu_xmac_interrupt()  argument
3920 	struct niu_xmac_stats *mp = &np->mac_stats.xmac;  in niu_xmac_interrupt()
3990 static void niu_bmac_interrupt(struct niu *np)  in niu_bmac_interrupt()  argument
3992 	struct niu_bmac_stats *mp = &np->mac_stats.bmac;  in niu_bmac_interrupt()
4026 static int niu_mac_interrupt(struct niu *np)  in niu_mac_interrupt()  argument
4028 	if (np->flags & NIU_FLAGS_XMAC)  in niu_mac_interrupt()
4029 		niu_xmac_interrupt(np);  in niu_mac_interrupt()
4031 		niu_bmac_interrupt(np);  in niu_mac_interrupt()
4036 static void niu_log_device_error(struct niu *np, u64 stat)  in niu_log_device_error()  argument
4038 	netdev_err(np->dev, "Core device errors ( ");  in niu_log_device_error()
4066 static int niu_device_error(struct niu *np)  in niu_device_error()  argument
4070 	netdev_err(np->dev, "Core device error, stat[%llx]\n",  in niu_device_error()
4073 	niu_log_device_error(np, stat);  in niu_device_error()
4078 static int niu_slowpath_interrupt(struct niu *np, struct niu_ldg *lp,  in niu_slowpath_interrupt()  argument
4091 		for (i = 0; i < np->num_rx_rings; i++) {  in niu_slowpath_interrupt()
4092 			struct rx_ring_info *rp = &np->rx_rings[i];  in niu_slowpath_interrupt()
4095 				int r = niu_rx_error(np, rp);  in niu_slowpath_interrupt()
4109 		for (i = 0; i < np->num_tx_rings; i++) {  in niu_slowpath_interrupt()
4110 			struct tx_ring_info *rp = &np->tx_rings[i];  in niu_slowpath_interrupt()
4113 				int r = niu_tx_error(np, rp);  in niu_slowpath_interrupt()
4120 		int r = niu_mif_interrupt(np);  in niu_slowpath_interrupt()
4126 			int r = niu_mac_interrupt(np);  in niu_slowpath_interrupt()
4131 			int r = niu_device_error(np);  in niu_slowpath_interrupt()
4138 		niu_enable_interrupts(np, 0);  in niu_slowpath_interrupt()
4143 static void niu_rxchan_intr(struct niu *np, struct rx_ring_info *rp,  in niu_rxchan_intr()  argument
4153 	netif_printk(np, intr, KERN_DEBUG, np->dev,  in niu_rxchan_intr()
4157 static void niu_txchan_intr(struct niu *np, struct tx_ring_info *rp,  in niu_txchan_intr()  argument
4162 	netif_printk(np, intr, KERN_DEBUG, np->dev,  in niu_txchan_intr()
4166 static void __niu_fastpath_interrupt(struct niu *np, int ldg, u64 v0)  in __niu_fastpath_interrupt()  argument
4168 	struct niu_parent *parent = np->parent;  in __niu_fastpath_interrupt()
4175 	for (i = 0; i < np->num_rx_rings; i++) {  in __niu_fastpath_interrupt()
4176 		struct rx_ring_info *rp = &np->rx_rings[i];  in __niu_fastpath_interrupt()
4184 			niu_rxchan_intr(np, rp, ldn);  in __niu_fastpath_interrupt()
4187 	for (i = 0; i < np->num_tx_rings; i++) {  in __niu_fastpath_interrupt()
4188 		struct tx_ring_info *rp = &np->tx_rings[i];  in __niu_fastpath_interrupt()
4196 			niu_txchan_intr(np, rp, ldn);  in __niu_fastpath_interrupt()
4200 static void niu_schedule_napi(struct niu *np, struct niu_ldg *lp,  in niu_schedule_napi()  argument
4207 		__niu_fastpath_interrupt(np, lp->ldg_num, v0);  in niu_schedule_napi()
4215 	struct niu *np = lp->np;  in niu_interrupt()  local
4220 	if (netif_msg_intr(np))  in niu_interrupt()
4224 	spin_lock_irqsave(&np->lock, flags);  in niu_interrupt()
4230 	if (netif_msg_intr(np))  in niu_interrupt()
4237 		spin_unlock_irqrestore(&np->lock, flags);  in niu_interrupt()
4242 		int err = niu_slowpath_interrupt(np, lp, v0, v1, v2);  in niu_interrupt()
4247 		niu_schedule_napi(np, lp, v0, v1, v2);  in niu_interrupt()
4249 		niu_ldg_rearm(np, lp, 1);  in niu_interrupt()
4251 	spin_unlock_irqrestore(&np->lock, flags);  in niu_interrupt()
4256 static void niu_free_rx_ring_info(struct niu *np, struct rx_ring_info *rp)  in niu_free_rx_ring_info()  argument
4259 		np->ops->free_coherent(np->device,  in niu_free_rx_ring_info()
4265 		np->ops->free_coherent(np->device,  in niu_free_rx_ring_info()
4273 		niu_rbr_free(np, rp);  in niu_free_rx_ring_info()
4275 		np->ops->free_coherent(np->device,  in niu_free_rx_ring_info()
4286 static void niu_free_tx_ring_info(struct niu *np, struct tx_ring_info *rp)  in niu_free_tx_ring_info()  argument
4289 		np->ops->free_coherent(np->device,  in niu_free_tx_ring_info()
4299 				(void) release_tx_packet(np, rp, i);  in niu_free_tx_ring_info()
4302 		np->ops->free_coherent(np->device,  in niu_free_tx_ring_info()
4313 static void niu_free_channels(struct niu *np)  in niu_free_channels()  argument
4317 	if (np->rx_rings) {  in niu_free_channels()
4318 		for (i = 0; i < np->num_rx_rings; i++) {  in niu_free_channels()
4319 			struct rx_ring_info *rp = &np->rx_rings[i];  in niu_free_channels()
4321 			niu_free_rx_ring_info(np, rp);  in niu_free_channels()
4323 		kfree(np->rx_rings);  in niu_free_channels()
4324 		np->rx_rings = NULL;  in niu_free_channels()
4325 		np->num_rx_rings = 0;  in niu_free_channels()
4328 	if (np->tx_rings) {  in niu_free_channels()
4329 		for (i = 0; i < np->num_tx_rings; i++) {  in niu_free_channels()
4330 			struct tx_ring_info *rp = &np->tx_rings[i];  in niu_free_channels()
4332 			niu_free_tx_ring_info(np, rp);  in niu_free_channels()
4334 		kfree(np->tx_rings);  in niu_free_channels()
4335 		np->tx_rings = NULL;  in niu_free_channels()
4336 		np->num_tx_rings = 0;  in niu_free_channels()
4340 static int niu_alloc_rx_ring_info(struct niu *np,  in niu_alloc_rx_ring_info()  argument
4350 	rp->mbox = np->ops->alloc_coherent(np->device,  in niu_alloc_rx_ring_info()
4356 		netdev_err(np->dev, "Coherent alloc gives misaligned RXDMA mailbox %p\n",  in niu_alloc_rx_ring_info()
4361 	rp->rcr = np->ops->alloc_coherent(np->device,  in niu_alloc_rx_ring_info()
4367 		netdev_err(np->dev, "Coherent alloc gives misaligned RXDMA RCR table %p\n",  in niu_alloc_rx_ring_info()
4374 	rp->rbr = np->ops->alloc_coherent(np->device,  in niu_alloc_rx_ring_info()
4380 		netdev_err(np->dev, "Coherent alloc gives misaligned RXDMA RBR table %p\n",  in niu_alloc_rx_ring_info()
4391 static void niu_set_max_burst(struct niu *np, struct tx_ring_info *rp)  in niu_set_max_burst()  argument
4393 	int mtu = np->dev->mtu;  in niu_set_max_burst()
4403 static int niu_alloc_tx_ring_info(struct niu *np,  in niu_alloc_tx_ring_info()  argument
4408 	rp->mbox = np->ops->alloc_coherent(np->device,  in niu_alloc_tx_ring_info()
4414 		netdev_err(np->dev, "Coherent alloc gives misaligned TXDMA mailbox %p\n",  in niu_alloc_tx_ring_info()
4419 	rp->descr = np->ops->alloc_coherent(np->device,  in niu_alloc_tx_ring_info()
4425 		netdev_err(np->dev, "Coherent alloc gives misaligned TXDMA descr table %p\n",  in niu_alloc_tx_ring_info()
4438 	niu_set_max_burst(np, rp);  in niu_alloc_tx_ring_info()
4443 static void niu_size_rbr(struct niu *np, struct rx_ring_info *rp)  in niu_size_rbr()  argument
4454 	if (np->dev->mtu > ETH_DATA_LEN) {  in niu_size_rbr()
4470 static int niu_alloc_channels(struct niu *np)  in niu_alloc_channels()  argument
4472 	struct niu_parent *parent = np->parent;  in niu_alloc_channels()
4479 	port = np->port;  in niu_alloc_channels()
4495 	np->num_rx_rings = num_rx_rings;  in niu_alloc_channels()
4497 	np->rx_rings = rx_rings;  in niu_alloc_channels()
4499 	netif_set_real_num_rx_queues(np->dev, num_rx_rings);  in niu_alloc_channels()
4501 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_alloc_channels()
4502 		struct rx_ring_info *rp = &np->rx_rings[i];  in niu_alloc_channels()
4504 		rp->np = np;  in niu_alloc_channels()
4507 		err = niu_alloc_rx_ring_info(np, rp);  in niu_alloc_channels()
4511 		niu_size_rbr(np, rp);  in niu_alloc_channels()
4524 		err = niu_rbr_fill(np, rp, GFP_KERNEL);  in niu_alloc_channels()
4535 	np->num_tx_rings = num_tx_rings;  in niu_alloc_channels()
4537 	np->tx_rings = tx_rings;  in niu_alloc_channels()
4539 	netif_set_real_num_tx_queues(np->dev, num_tx_rings);  in niu_alloc_channels()
4541 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_alloc_channels()
4542 		struct tx_ring_info *rp = &np->tx_rings[i];  in niu_alloc_channels()
4544 		rp->np = np;  in niu_alloc_channels()
4547 		err = niu_alloc_tx_ring_info(np, rp);  in niu_alloc_channels()
4555 	niu_free_channels(np);  in niu_alloc_channels()
4559 static int niu_tx_cs_sng_poll(struct niu *np, int channel)  in niu_tx_cs_sng_poll()  argument
4571 static int niu_tx_channel_stop(struct niu *np, int channel)  in niu_tx_channel_stop()  argument
4578 	return niu_tx_cs_sng_poll(np, channel);  in niu_tx_channel_stop()
4581 static int niu_tx_cs_reset_poll(struct niu *np, int channel)  in niu_tx_cs_reset_poll()  argument
4593 static int niu_tx_channel_reset(struct niu *np, int channel)  in niu_tx_channel_reset()  argument
4601 	err = niu_tx_cs_reset_poll(np, channel);  in niu_tx_channel_reset()
4608 static int niu_tx_channel_lpage_init(struct niu *np, int channel)  in niu_tx_channel_lpage_init()  argument
4620 	val  = (u64)np->port << TX_LOG_PAGE_VLD_FUNC_SHIFT;  in niu_tx_channel_lpage_init()
4629 static void niu_txc_enable_port(struct niu *np, int on)  in niu_txc_enable_port()  argument
4634 	niu_lock_parent(np, flags);  in niu_txc_enable_port()
4636 	mask = (u64)1 << np->port;  in niu_txc_enable_port()
4645 	niu_unlock_parent(np, flags);  in niu_txc_enable_port()
4648 static void niu_txc_set_imask(struct niu *np, u64 imask)  in niu_txc_set_imask()  argument
4653 	niu_lock_parent(np, flags);  in niu_txc_set_imask()
4655 	val &= ~TXC_INT_MASK_VAL(np->port);  in niu_txc_set_imask()
4656 	val |= (imask << TXC_INT_MASK_VAL_SHIFT(np->port));  in niu_txc_set_imask()
4657 	niu_unlock_parent(np, flags);  in niu_txc_set_imask()
4660 static void niu_txc_port_dma_enable(struct niu *np, int on)  in niu_txc_port_dma_enable()  argument
4667 		for (i = 0; i < np->num_tx_rings; i++)  in niu_txc_port_dma_enable()
4668 			val |= (1 << np->tx_rings[i].tx_channel);  in niu_txc_port_dma_enable()
4670 	nw64(TXC_PORT_DMA(np->port), val);  in niu_txc_port_dma_enable()
4673 static int niu_init_one_tx_channel(struct niu *np, struct tx_ring_info *rp)  in niu_init_one_tx_channel()  argument
4678 	err = niu_tx_channel_stop(np, channel);  in niu_init_one_tx_channel()
4682 	err = niu_tx_channel_reset(np, channel);  in niu_init_one_tx_channel()
4686 	err = niu_tx_channel_lpage_init(np, channel);  in niu_init_one_tx_channel()
4695 		netdev_err(np->dev, "TX ring channel %d DMA addr (%llx) is not aligned\n",  in niu_init_one_tx_channel()
4713 		netdev_err(np->dev, "TX ring channel %d MBOX addr (%llx) has invalid bits\n",  in niu_init_one_tx_channel()
4727 static void niu_init_rdc_groups(struct niu *np)  in niu_init_rdc_groups()  argument
4729 	struct niu_rdc_tables *tp = &np->parent->rdc_group_cfg[np->port];  in niu_init_rdc_groups()
4742 	nw64(DEF_RDC(np->port), np->parent->rdc_default[np->port]);  in niu_init_rdc_groups()
4745 static void niu_init_drr_weight(struct niu *np)  in niu_init_drr_weight()  argument
4747 	int type = phy_decode(np->parent->port_phy, np->port);  in niu_init_drr_weight()
4760 	nw64(PT_DRR_WT(np->port), val);  in niu_init_drr_weight()
4763 static int niu_init_hostinfo(struct niu *np)  in niu_init_hostinfo()  argument
4765 	struct niu_parent *parent = np->parent;  in niu_init_hostinfo()
4766 	struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];  in niu_init_hostinfo()
4767 	int i, err, num_alt = niu_num_alt_addr(np);  in niu_init_hostinfo()
4770 	err = niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);  in niu_init_hostinfo()
4774 	err = niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);  in niu_init_hostinfo()
4779 		err = niu_set_alt_mac_rdc_table(np, i, first_rdc_table, 1);  in niu_init_hostinfo()
4787 static int niu_rx_channel_reset(struct niu *np, int channel)  in niu_rx_channel_reset()  argument
4789 	return niu_set_and_wait_clear(np, RXDMA_CFIG1(channel),  in niu_rx_channel_reset()
4794 static int niu_rx_channel_lpage_init(struct niu *np, int channel)  in niu_rx_channel_lpage_init()  argument
4806 	val  = (u64)np->port << RX_LOG_PAGE_VLD_FUNC_SHIFT;  in niu_rx_channel_lpage_init()
4813 static void niu_rx_channel_wred_init(struct niu *np, struct rx_ring_info *rp)  in niu_rx_channel_wred_init()  argument
4904 static int niu_enable_rx_channel(struct niu *np, int channel, int on)  in niu_enable_rx_channel()  argument
4926 static int niu_init_one_rx_channel(struct niu *np, struct rx_ring_info *rp)  in niu_init_one_rx_channel()  argument
4931 	err = niu_rx_channel_reset(np, channel);  in niu_init_one_rx_channel()
4935 	err = niu_rx_channel_lpage_init(np, channel);  in niu_init_one_rx_channel()
4939 	niu_rx_channel_wred_init(np, rp);  in niu_init_one_rx_channel()
4966 	err = niu_enable_rx_channel(np, channel, 1);  in niu_init_one_rx_channel()
4979 static int niu_init_rx_channels(struct niu *np)  in niu_init_rx_channels()  argument
4985 	niu_lock_parent(np, flags);  in niu_init_rx_channels()
4986 	nw64(RX_DMA_CK_DIV, np->parent->rxdma_clock_divider);  in niu_init_rx_channels()
4988 	niu_unlock_parent(np, flags);  in niu_init_rx_channels()
4992 	niu_init_rdc_groups(np);  in niu_init_rx_channels()
4993 	niu_init_drr_weight(np);  in niu_init_rx_channels()
4995 	err = niu_init_hostinfo(np);  in niu_init_rx_channels()
4999 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_init_rx_channels()
5000 		struct rx_ring_info *rp = &np->rx_rings[i];  in niu_init_rx_channels()
5002 		err = niu_init_one_rx_channel(np, rp);  in niu_init_rx_channels()
5010 static int niu_set_ip_frag_rule(struct niu *np)  in niu_set_ip_frag_rule()  argument
5012 	struct niu_parent *parent = np->parent;  in niu_set_ip_frag_rule()
5013 	struct niu_classifier *cp = &np->clas;  in niu_set_ip_frag_rule()
5028 	err = tcam_write(np, index, tp->key, tp->key_mask);  in niu_set_ip_frag_rule()
5031 	err = tcam_assoc_write(np, index, tp->assoc_data);  in niu_set_ip_frag_rule()
5040 static int niu_init_classifier_hw(struct niu *np)  in niu_init_classifier_hw()  argument
5042 	struct niu_parent *parent = np->parent;  in niu_init_classifier_hw()
5043 	struct niu_classifier *cp = &np->clas;  in niu_init_classifier_hw()
5049 	err = niu_init_hostinfo(np);  in niu_init_classifier_hw()
5056 		vlan_tbl_write(np, i, np->port,  in niu_init_classifier_hw()
5063 		err = niu_set_alt_mac_rdc_table(np, ap->alt_mac_num,  in niu_init_classifier_hw()
5072 		err = niu_set_tcam_key(np, i, parent->tcam_key[index]);  in niu_init_classifier_hw()
5075 		err = niu_set_flow_key(np, i, parent->flow_key[index]);  in niu_init_classifier_hw()
5080 	err = niu_set_ip_frag_rule(np);  in niu_init_classifier_hw()
5084 	tcam_enable(np, 1);  in niu_init_classifier_hw()
5089 static int niu_zcp_write(struct niu *np, int index, u64 *data)  in niu_zcp_write()  argument
5100 	      (ZCP_RAM_SEL_CFIFO(np->port) << ZCP_RAM_ACC_RAM_SEL_SHIFT)));  in niu_zcp_write()
5102 	return niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,  in niu_zcp_write()
5106 static int niu_zcp_read(struct niu *np, int index, u64 *data)  in niu_zcp_read()  argument
5110 	err = niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,  in niu_zcp_read()
5113 		netdev_err(np->dev, "ZCP read busy won't clear, ZCP_RAM_ACC[%llx]\n",  in niu_zcp_read()
5121 	      (ZCP_RAM_SEL_CFIFO(np->port) << ZCP_RAM_ACC_RAM_SEL_SHIFT)));  in niu_zcp_read()
5123 	err = niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,  in niu_zcp_read()
5126 		netdev_err(np->dev, "ZCP read busy2 won't clear, ZCP_RAM_ACC[%llx]\n",  in niu_zcp_read()
5140 static void niu_zcp_cfifo_reset(struct niu *np)  in niu_zcp_cfifo_reset()  argument
5144 	val |= RESET_CFIFO_RST(np->port);  in niu_zcp_cfifo_reset()
5148 	val &= ~RESET_CFIFO_RST(np->port);  in niu_zcp_cfifo_reset()
5152 static int niu_init_zcp(struct niu *np)  in niu_init_zcp()  argument
5157 	if (np->parent->plat_type != PLAT_TYPE_NIU) {  in niu_init_zcp()
5158 		if (np->port == 0 || np->port == 1)  in niu_init_zcp()
5172 		err = niu_zcp_write(np, i, data);  in niu_init_zcp()
5175 		err = niu_zcp_read(np, i, rbuf);  in niu_init_zcp()
5180 	niu_zcp_cfifo_reset(np);  in niu_init_zcp()
5181 	nw64(CFIFO_ECC(np->port), 0);  in niu_init_zcp()
5189 static void niu_ipp_write(struct niu *np, int index, u64 *data)  in niu_ipp_write()  argument
5203 static void niu_ipp_read(struct niu *np, int index, u64 *data)  in niu_ipp_read()  argument
5213 static int niu_ipp_reset(struct niu *np)  in niu_ipp_reset()  argument
5215 	return niu_set_and_wait_clear_ipp(np, IPP_CFIG, IPP_CFIG_SOFT_RST,  in niu_ipp_reset()
5219 static int niu_init_ipp(struct niu *np)  in niu_init_ipp()  argument
5224 	if (np->parent->plat_type != PLAT_TYPE_NIU) {  in niu_init_ipp()
5225 		if (np->port == 0 || np->port == 1)  in niu_init_ipp()
5239 		niu_ipp_write(np, i, data);  in niu_init_ipp()
5240 		niu_ipp_read(np, i, rbuf);  in niu_init_ipp()
5246 	err = niu_ipp_reset(np);  in niu_init_ipp()
5270 static void niu_handle_led(struct niu *np, int status)  in niu_handle_led()  argument
5275 	if ((np->flags & NIU_FLAGS_10G) != 0 &&  in niu_handle_led()
5276 	    (np->flags & NIU_FLAGS_FIBER) != 0) {  in niu_handle_led()
5289 static void niu_init_xif_xmac(struct niu *np)  in niu_init_xif_xmac()  argument
5291 	struct niu_link_config *lp = &np->link_config;  in niu_init_xif_xmac()
5294 	if (np->flags & NIU_FLAGS_XCVR_SERDES) {  in niu_init_xif_xmac()
5312 	if (np->flags & NIU_FLAGS_10G) {  in niu_init_xif_xmac()
5316 		if (!(np->flags & NIU_FLAGS_FIBER) &&  in niu_init_xif_xmac()
5317 		    !(np->flags & NIU_FLAGS_XCVR_SERDES))  in niu_init_xif_xmac()
5334 	if (np->flags & NIU_FLAGS_10G) {  in niu_init_xif_xmac()
5346 static void niu_init_xif_bmac(struct niu *np)  in niu_init_xif_bmac()  argument
5348 	struct niu_link_config *lp = &np->link_config;  in niu_init_xif_bmac()
5366 	if (!(np->flags & NIU_FLAGS_10G) &&  in niu_init_xif_bmac()
5367 	    !(np->flags & NIU_FLAGS_FIBER) &&  in niu_init_xif_bmac()
5376 static void niu_init_xif(struct niu *np)  in niu_init_xif()  argument
5378 	if (np->flags & NIU_FLAGS_XMAC)  in niu_init_xif()
5379 		niu_init_xif_xmac(np);  in niu_init_xif()
5381 		niu_init_xif_bmac(np);  in niu_init_xif()
5384 static void niu_pcs_mii_reset(struct niu *np)  in niu_pcs_mii_reset()  argument
5396 static void niu_xpcs_reset(struct niu *np)  in niu_xpcs_reset()  argument
5408 static int niu_init_pcs(struct niu *np)  in niu_init_pcs()  argument
5410 	struct niu_link_config *lp = &np->link_config;  in niu_init_pcs()
5413 	switch (np->flags & (NIU_FLAGS_10G |  in niu_init_pcs()
5420 		niu_pcs_mii_reset(np);  in niu_init_pcs()
5427 		if (!(np->flags & NIU_FLAGS_XMAC))  in niu_init_pcs()
5435 		niu_xpcs_reset(np);  in niu_init_pcs()
5452 		niu_pcs_mii_reset(np);  in niu_init_pcs()
5462 		niu_pcs_mii_reset(np);  in niu_init_pcs()
5472 static int niu_reset_tx_xmac(struct niu *np)  in niu_reset_tx_xmac()  argument
5474 	return niu_set_and_wait_clear_mac(np, XTXMAC_SW_RST,  in niu_reset_tx_xmac()
5480 static int niu_reset_tx_bmac(struct niu *np)  in niu_reset_tx_bmac()  argument
5492 		dev_err(np->device, "Port %u TX BMAC would not reset, BTXMAC_SW_RST[%llx]\n",  in niu_reset_tx_bmac()
5493 			np->port,  in niu_reset_tx_bmac()
5501 static int niu_reset_tx_mac(struct niu *np)  in niu_reset_tx_mac()  argument
5503 	if (np->flags & NIU_FLAGS_XMAC)  in niu_reset_tx_mac()
5504 		return niu_reset_tx_xmac(np);  in niu_reset_tx_mac()
5506 		return niu_reset_tx_bmac(np);  in niu_reset_tx_mac()
5509 static void niu_init_tx_xmac(struct niu *np, u64 min, u64 max)  in niu_init_tx_xmac()  argument
5525 	if (np->flags & NIU_FLAGS_10G) {  in niu_init_tx_xmac()
5545 static void niu_init_tx_bmac(struct niu *np, u64 min, u64 max)  in niu_init_tx_bmac()  argument
5562 static void niu_init_tx_mac(struct niu *np)  in niu_init_tx_mac()  argument
5567 	if (np->dev->mtu > ETH_DATA_LEN)  in niu_init_tx_mac()
5577 	if (np->flags & NIU_FLAGS_XMAC)  in niu_init_tx_mac()
5578 		niu_init_tx_xmac(np, min, max);  in niu_init_tx_mac()
5580 		niu_init_tx_bmac(np, min, max);  in niu_init_tx_mac()
5583 static int niu_reset_rx_xmac(struct niu *np)  in niu_reset_rx_xmac()  argument
5597 		dev_err(np->device, "Port %u RX XMAC would not reset, XRXMAC_SW_RST[%llx]\n",  in niu_reset_rx_xmac()
5598 			np->port,  in niu_reset_rx_xmac()
5606 static int niu_reset_rx_bmac(struct niu *np)  in niu_reset_rx_bmac()  argument
5618 		dev_err(np->device, "Port %u RX BMAC would not reset, BRXMAC_SW_RST[%llx]\n",  in niu_reset_rx_bmac()
5619 			np->port,  in niu_reset_rx_bmac()
5627 static int niu_reset_rx_mac(struct niu *np)  in niu_reset_rx_mac()  argument
5629 	if (np->flags & NIU_FLAGS_XMAC)  in niu_reset_rx_mac()
5630 		return niu_reset_rx_xmac(np);  in niu_reset_rx_mac()
5632 		return niu_reset_rx_bmac(np);  in niu_reset_rx_mac()
5635 static void niu_init_rx_xmac(struct niu *np)  in niu_init_rx_xmac()  argument
5637 	struct niu_parent *parent = np->parent;  in niu_init_rx_xmac()
5638 	struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];  in niu_init_rx_xmac()
5651 	niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);  in niu_init_rx_xmac()
5652 	niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);  in niu_init_rx_xmac()
5687 static void niu_init_rx_bmac(struct niu *np)  in niu_init_rx_bmac()  argument
5689 	struct niu_parent *parent = np->parent;  in niu_init_rx_bmac()
5690 	struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];  in niu_init_rx_bmac()
5702 	niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);  in niu_init_rx_bmac()
5703 	niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);  in niu_init_rx_bmac()
5722 static void niu_init_rx_mac(struct niu *np)  in niu_init_rx_mac()  argument
5724 	niu_set_primary_mac(np, np->dev->dev_addr);  in niu_init_rx_mac()
5726 	if (np->flags & NIU_FLAGS_XMAC)  in niu_init_rx_mac()
5727 		niu_init_rx_xmac(np);  in niu_init_rx_mac()
5729 		niu_init_rx_bmac(np);  in niu_init_rx_mac()
5732 static void niu_enable_tx_xmac(struct niu *np, int on)  in niu_enable_tx_xmac()  argument
5743 static void niu_enable_tx_bmac(struct niu *np, int on)  in niu_enable_tx_bmac()  argument
5754 static void niu_enable_tx_mac(struct niu *np, int on)  in niu_enable_tx_mac()  argument
5756 	if (np->flags & NIU_FLAGS_XMAC)  in niu_enable_tx_mac()
5757 		niu_enable_tx_xmac(np, on);  in niu_enable_tx_mac()
5759 		niu_enable_tx_bmac(np, on);  in niu_enable_tx_mac()
5762 static void niu_enable_rx_xmac(struct niu *np, int on)  in niu_enable_rx_xmac()  argument
5769 	if (np->flags & NIU_FLAGS_MCAST)  in niu_enable_rx_xmac()
5771 	if (np->flags & NIU_FLAGS_PROMISC)  in niu_enable_rx_xmac()
5781 static void niu_enable_rx_bmac(struct niu *np, int on)  in niu_enable_rx_bmac()  argument
5788 	if (np->flags & NIU_FLAGS_MCAST)  in niu_enable_rx_bmac()
5790 	if (np->flags & NIU_FLAGS_PROMISC)  in niu_enable_rx_bmac()
5800 static void niu_enable_rx_mac(struct niu *np, int on)  in niu_enable_rx_mac()  argument
5802 	if (np->flags & NIU_FLAGS_XMAC)  in niu_enable_rx_mac()
5803 		niu_enable_rx_xmac(np, on);  in niu_enable_rx_mac()
5805 		niu_enable_rx_bmac(np, on);  in niu_enable_rx_mac()
5808 static int niu_init_mac(struct niu *np)  in niu_init_mac()  argument
5812 	niu_init_xif(np);  in niu_init_mac()
5813 	err = niu_init_pcs(np);  in niu_init_mac()
5817 	err = niu_reset_tx_mac(np);  in niu_init_mac()
5820 	niu_init_tx_mac(np);  in niu_init_mac()
5821 	err = niu_reset_rx_mac(np);  in niu_init_mac()
5824 	niu_init_rx_mac(np);  in niu_init_mac()
5831 	niu_init_tx_mac(np);  in niu_init_mac()
5832 	niu_enable_tx_mac(np, 1);  in niu_init_mac()
5834 	niu_enable_rx_mac(np, 1);  in niu_init_mac()
5839 static void niu_stop_one_tx_channel(struct niu *np, struct tx_ring_info *rp)  in niu_stop_one_tx_channel()  argument
5841 	(void) niu_tx_channel_stop(np, rp->tx_channel);  in niu_stop_one_tx_channel()
5844 static void niu_stop_tx_channels(struct niu *np)  in niu_stop_tx_channels()  argument
5848 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_stop_tx_channels()
5849 		struct tx_ring_info *rp = &np->tx_rings[i];  in niu_stop_tx_channels()
5851 		niu_stop_one_tx_channel(np, rp);  in niu_stop_tx_channels()
5855 static void niu_reset_one_tx_channel(struct niu *np, struct tx_ring_info *rp)  in niu_reset_one_tx_channel()  argument
5857 	(void) niu_tx_channel_reset(np, rp->tx_channel);  in niu_reset_one_tx_channel()
5860 static void niu_reset_tx_channels(struct niu *np)  in niu_reset_tx_channels()  argument
5864 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_reset_tx_channels()
5865 		struct tx_ring_info *rp = &np->tx_rings[i];  in niu_reset_tx_channels()
5867 		niu_reset_one_tx_channel(np, rp);  in niu_reset_tx_channels()
5871 static void niu_stop_one_rx_channel(struct niu *np, struct rx_ring_info *rp)  in niu_stop_one_rx_channel()  argument
5873 	(void) niu_enable_rx_channel(np, rp->rx_channel, 0);  in niu_stop_one_rx_channel()
5876 static void niu_stop_rx_channels(struct niu *np)  in niu_stop_rx_channels()  argument
5880 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_stop_rx_channels()
5881 		struct rx_ring_info *rp = &np->rx_rings[i];  in niu_stop_rx_channels()
5883 		niu_stop_one_rx_channel(np, rp);  in niu_stop_rx_channels()
5887 static void niu_reset_one_rx_channel(struct niu *np, struct rx_ring_info *rp)  in niu_reset_one_rx_channel()  argument
5891 	(void) niu_rx_channel_reset(np, channel);  in niu_reset_one_rx_channel()
5894 	(void) niu_enable_rx_channel(np, channel, 0);  in niu_reset_one_rx_channel()
5897 static void niu_reset_rx_channels(struct niu *np)  in niu_reset_rx_channels()  argument
5901 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_reset_rx_channels()
5902 		struct rx_ring_info *rp = &np->rx_rings[i];  in niu_reset_rx_channels()
5904 		niu_reset_one_rx_channel(np, rp);  in niu_reset_rx_channels()
5908 static void niu_disable_ipp(struct niu *np)  in niu_disable_ipp()  argument
5922 		netdev_err(np->dev, "IPP would not quiesce, rd_ptr[%llx] wr_ptr[%llx]\n",  in niu_disable_ipp()
5934 	(void) niu_ipp_reset(np);  in niu_disable_ipp()
5937 static int niu_init_hw(struct niu *np)  in niu_init_hw()  argument
5941 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize TXC\n");  in niu_init_hw()
5942 	niu_txc_enable_port(np, 1);  in niu_init_hw()
5943 	niu_txc_port_dma_enable(np, 1);  in niu_init_hw()
5944 	niu_txc_set_imask(np, 0);  in niu_init_hw()
5946 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize TX channels\n");  in niu_init_hw()
5947 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_init_hw()
5948 		struct tx_ring_info *rp = &np->tx_rings[i];  in niu_init_hw()
5950 		err = niu_init_one_tx_channel(np, rp);  in niu_init_hw()
5955 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize RX channels\n");  in niu_init_hw()
5956 	err = niu_init_rx_channels(np);  in niu_init_hw()
5960 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize classifier\n");  in niu_init_hw()
5961 	err = niu_init_classifier_hw(np);  in niu_init_hw()
5965 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize ZCP\n");  in niu_init_hw()
5966 	err = niu_init_zcp(np);  in niu_init_hw()
5970 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize IPP\n");  in niu_init_hw()
5971 	err = niu_init_ipp(np);  in niu_init_hw()
5975 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize MAC\n");  in niu_init_hw()
5976 	err = niu_init_mac(np);  in niu_init_hw()
5983 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Uninit IPP\n");  in niu_init_hw()
5984 	niu_disable_ipp(np);  in niu_init_hw()
5987 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Uninit RX channels\n");  in niu_init_hw()
5988 	niu_stop_rx_channels(np);  in niu_init_hw()
5989 	niu_reset_rx_channels(np);  in niu_init_hw()
5992 	netif_printk(np, ifup, KERN_DEBUG, np->dev, "Uninit TX channels\n");  in niu_init_hw()
5993 	niu_stop_tx_channels(np);  in niu_init_hw()
5994 	niu_reset_tx_channels(np);  in niu_init_hw()
5999 static void niu_stop_hw(struct niu *np)  in niu_stop_hw()  argument
6001 	netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Disable interrupts\n");  in niu_stop_hw()
6002 	niu_enable_interrupts(np, 0);  in niu_stop_hw()
6004 	netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Disable RX MAC\n");  in niu_stop_hw()
6005 	niu_enable_rx_mac(np, 0);  in niu_stop_hw()
6007 	netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Disable IPP\n");  in niu_stop_hw()
6008 	niu_disable_ipp(np);  in niu_stop_hw()
6010 	netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Stop TX channels\n");  in niu_stop_hw()
6011 	niu_stop_tx_channels(np);  in niu_stop_hw()
6013 	netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Stop RX channels\n");  in niu_stop_hw()
6014 	niu_stop_rx_channels(np);  in niu_stop_hw()
6016 	netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Reset TX channels\n");  in niu_stop_hw()
6017 	niu_reset_tx_channels(np);  in niu_stop_hw()
6019 	netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Reset RX channels\n");  in niu_stop_hw()
6020 	niu_reset_rx_channels(np);  in niu_stop_hw()
6023 static void niu_set_irq_name(struct niu *np)  in niu_set_irq_name()  argument
6025 	int port = np->port;  in niu_set_irq_name()
6028 	sprintf(np->irq_name[0], "%s:MAC", np->dev->name);  in niu_set_irq_name()
6031 		sprintf(np->irq_name[1], "%s:MIF", np->dev->name);  in niu_set_irq_name()
6032 		sprintf(np->irq_name[2], "%s:SYSERR", np->dev->name);  in niu_set_irq_name()
6036 	for (i = 0; i < np->num_ldg - j; i++) {  in niu_set_irq_name()
6037 		if (i < np->num_rx_rings)  in niu_set_irq_name()
6038 			sprintf(np->irq_name[i+j], "%s-rx-%d",  in niu_set_irq_name()
6039 				np->dev->name, i);  in niu_set_irq_name()
6040 		else if (i < np->num_tx_rings + np->num_rx_rings)  in niu_set_irq_name()
6041 			sprintf(np->irq_name[i+j], "%s-tx-%d", np->dev->name,  in niu_set_irq_name()
6042 				i - np->num_rx_rings);  in niu_set_irq_name()
6046 static int niu_request_irq(struct niu *np)  in niu_request_irq()  argument
6050 	niu_set_irq_name(np);  in niu_request_irq()
6053 	for (i = 0; i < np->num_ldg; i++) {  in niu_request_irq()
6054 		struct niu_ldg *lp = &np->ldg[i];  in niu_request_irq()
6057 				  np->irq_name[i], lp);  in niu_request_irq()
6067 		struct niu_ldg *lp = &np->ldg[j];  in niu_request_irq()
6074 static void niu_free_irq(struct niu *np)  in niu_free_irq()  argument
6078 	for (i = 0; i < np->num_ldg; i++) {  in niu_free_irq()
6079 		struct niu_ldg *lp = &np->ldg[i];  in niu_free_irq()
6085 static void niu_enable_napi(struct niu *np)  in niu_enable_napi()  argument
6089 	for (i = 0; i < np->num_ldg; i++)  in niu_enable_napi()
6090 		napi_enable(&np->ldg[i].napi);  in niu_enable_napi()
6093 static void niu_disable_napi(struct niu *np)  in niu_disable_napi()  argument
6097 	for (i = 0; i < np->num_ldg; i++)  in niu_disable_napi()
6098 		napi_disable(&np->ldg[i].napi);  in niu_disable_napi()
6103 	struct niu *np = netdev_priv(dev);  in niu_open()  local
6108 	err = niu_alloc_channels(np);  in niu_open()
6112 	err = niu_enable_interrupts(np, 0);  in niu_open()
6116 	err = niu_request_irq(np);  in niu_open()
6120 	niu_enable_napi(np);  in niu_open()
6122 	spin_lock_irq(&np->lock);  in niu_open()
6124 	err = niu_init_hw(np);  in niu_open()
6126 		init_timer(&np->timer);  in niu_open()
6127 		np->timer.expires = jiffies + HZ;  in niu_open()
6128 		np->timer.data = (unsigned long) np;  in niu_open()
6129 		np->timer.function = niu_timer;  in niu_open()
6131 		err = niu_enable_interrupts(np, 1);  in niu_open()
6133 			niu_stop_hw(np);  in niu_open()
6136 	spin_unlock_irq(&np->lock);  in niu_open()
6139 		niu_disable_napi(np);  in niu_open()
6145 	if (np->link_config.loopback_mode != LOOPBACK_DISABLED)  in niu_open()
6148 	add_timer(&np->timer);  in niu_open()
6153 	niu_free_irq(np);  in niu_open()
6156 	niu_free_channels(np);  in niu_open()
6162 static void niu_full_shutdown(struct niu *np, struct net_device *dev)  in niu_full_shutdown()  argument
6164 	cancel_work_sync(&np->reset_task);  in niu_full_shutdown()
6166 	niu_disable_napi(np);  in niu_full_shutdown()
6169 	del_timer_sync(&np->timer);  in niu_full_shutdown()
6171 	spin_lock_irq(&np->lock);  in niu_full_shutdown()
6173 	niu_stop_hw(np);  in niu_full_shutdown()
6175 	spin_unlock_irq(&np->lock);  in niu_full_shutdown()
6180 	struct niu *np = netdev_priv(dev);  in niu_close()  local
6182 	niu_full_shutdown(np, dev);  in niu_close()
6184 	niu_free_irq(np);  in niu_close()
6186 	niu_free_channels(np);  in niu_close()
6188 	niu_handle_led(np, 0);  in niu_close()
6193 static void niu_sync_xmac_stats(struct niu *np)  in niu_sync_xmac_stats()  argument
6195 	struct niu_xmac_stats *mp = &np->mac_stats.xmac;  in niu_sync_xmac_stats()
6218 static void niu_sync_bmac_stats(struct niu *np)  in niu_sync_bmac_stats()  argument
6220 	struct niu_bmac_stats *mp = &np->mac_stats.bmac;  in niu_sync_bmac_stats()
6231 static void niu_sync_mac_stats(struct niu *np)  in niu_sync_mac_stats()  argument
6233 	if (np->flags & NIU_FLAGS_XMAC)  in niu_sync_mac_stats()
6234 		niu_sync_xmac_stats(np);  in niu_sync_mac_stats()
6236 		niu_sync_bmac_stats(np);  in niu_sync_mac_stats()
6239 static void niu_get_rx_stats(struct niu *np,  in niu_get_rx_stats()  argument
6248 	rx_rings = ACCESS_ONCE(np->rx_rings);  in niu_get_rx_stats()
6252 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_get_rx_stats()
6255 		niu_sync_rx_discard_stats(np, rp, 0);  in niu_get_rx_stats()
6270 static void niu_get_tx_stats(struct niu *np,  in niu_get_tx_stats()  argument
6279 	tx_rings = ACCESS_ONCE(np->tx_rings);  in niu_get_tx_stats()
6283 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_get_tx_stats()
6300 	struct niu *np = netdev_priv(dev);  in niu_get_stats()  local
6303 		niu_get_rx_stats(np, stats);  in niu_get_stats()
6304 		niu_get_tx_stats(np, stats);  in niu_get_stats()
6310 static void niu_load_hash_xmac(struct niu *np, u16 *hash)  in niu_load_hash_xmac()  argument
6318 static void niu_load_hash_bmac(struct niu *np, u16 *hash)  in niu_load_hash_bmac()  argument
6326 static void niu_load_hash(struct niu *np, u16 *hash)  in niu_load_hash()  argument
6328 	if (np->flags & NIU_FLAGS_XMAC)  in niu_load_hash()
6329 		niu_load_hash_xmac(np, hash);  in niu_load_hash()
6331 		niu_load_hash_bmac(np, hash);  in niu_load_hash()
6336 	struct niu *np = netdev_priv(dev);  in niu_set_rx_mode()  local
6342 	spin_lock_irqsave(&np->lock, flags);  in niu_set_rx_mode()
6343 	niu_enable_rx_mac(np, 0);  in niu_set_rx_mode()
6345 	np->flags &= ~(NIU_FLAGS_MCAST | NIU_FLAGS_PROMISC);  in niu_set_rx_mode()
6347 		np->flags |= NIU_FLAGS_PROMISC;  in niu_set_rx_mode()
6349 		np->flags |= NIU_FLAGS_MCAST;  in niu_set_rx_mode()
6352 	if (alt_cnt > niu_num_alt_addr(np)) {  in niu_set_rx_mode()
6354 		np->flags |= NIU_FLAGS_PROMISC;  in niu_set_rx_mode()
6361 			err = niu_set_alt_mac(np, index, ha->addr);  in niu_set_rx_mode()
6365 			err = niu_enable_alt_mac(np, index, 1);  in niu_set_rx_mode()
6374 		if (np->flags & NIU_FLAGS_XMAC)  in niu_set_rx_mode()
6378 		for (i = alt_start; i < niu_num_alt_addr(np); i++) {  in niu_set_rx_mode()
6379 			err = niu_enable_alt_mac(np, i, 0);  in niu_set_rx_mode()
6397 	if (np->flags & NIU_FLAGS_MCAST)  in niu_set_rx_mode()
6398 		niu_load_hash(np, hash);  in niu_set_rx_mode()
6400 	niu_enable_rx_mac(np, 1);  in niu_set_rx_mode()
6401 	spin_unlock_irqrestore(&np->lock, flags);  in niu_set_rx_mode()
6406 	struct niu *np = netdev_priv(dev);  in niu_set_mac_addr()  local
6418 	spin_lock_irqsave(&np->lock, flags);  in niu_set_mac_addr()
6419 	niu_enable_rx_mac(np, 0);  in niu_set_mac_addr()
6420 	niu_set_primary_mac(np, dev->dev_addr);  in niu_set_mac_addr()
6421 	niu_enable_rx_mac(np, 1);  in niu_set_mac_addr()
6422 	spin_unlock_irqrestore(&np->lock, flags);  in niu_set_mac_addr()
6432 static void niu_netif_stop(struct niu *np)  in niu_netif_stop()  argument
6434 	np->dev->trans_start = jiffies;	/* prevent tx timeout */  in niu_netif_stop()
6436 	niu_disable_napi(np);  in niu_netif_stop()
6438 	netif_tx_disable(np->dev);  in niu_netif_stop()
6441 static void niu_netif_start(struct niu *np)  in niu_netif_start()  argument
6447 	netif_tx_wake_all_queues(np->dev);  in niu_netif_start()
6449 	niu_enable_napi(np);  in niu_netif_start()
6451 	niu_enable_interrupts(np, 1);  in niu_netif_start()
6454 static void niu_reset_buffers(struct niu *np)  in niu_reset_buffers()  argument
6458 	if (np->rx_rings) {  in niu_reset_buffers()
6459 		for (i = 0; i < np->num_rx_rings; i++) {  in niu_reset_buffers()
6460 			struct rx_ring_info *rp = &np->rx_rings[i];  in niu_reset_buffers()
6476 				err = niu_rbr_add_page(np, rp, GFP_ATOMIC, k);  in niu_reset_buffers()
6487 	if (np->tx_rings) {  in niu_reset_buffers()
6488 		for (i = 0; i < np->num_tx_rings; i++) {  in niu_reset_buffers()
6489 			struct tx_ring_info *rp = &np->tx_rings[i];  in niu_reset_buffers()
6493 					(void) release_tx_packet(np, rp, j);  in niu_reset_buffers()
6506 	struct niu *np = container_of(work, struct niu, reset_task);  in niu_reset_task()  local
6510 	spin_lock_irqsave(&np->lock, flags);  in niu_reset_task()
6511 	if (!netif_running(np->dev)) {  in niu_reset_task()
6512 		spin_unlock_irqrestore(&np->lock, flags);  in niu_reset_task()
6516 	spin_unlock_irqrestore(&np->lock, flags);  in niu_reset_task()
6518 	del_timer_sync(&np->timer);  in niu_reset_task()
6520 	niu_netif_stop(np);  in niu_reset_task()
6522 	spin_lock_irqsave(&np->lock, flags);  in niu_reset_task()
6524 	niu_stop_hw(np);  in niu_reset_task()
6526 	spin_unlock_irqrestore(&np->lock, flags);  in niu_reset_task()
6528 	niu_reset_buffers(np);  in niu_reset_task()
6530 	spin_lock_irqsave(&np->lock, flags);  in niu_reset_task()
6532 	err = niu_init_hw(np);  in niu_reset_task()
6534 		np->timer.expires = jiffies + HZ;  in niu_reset_task()
6535 		add_timer(&np->timer);  in niu_reset_task()
6536 		niu_netif_start(np);  in niu_reset_task()
6539 	spin_unlock_irqrestore(&np->lock, flags);  in niu_reset_task()
6544 	struct niu *np = netdev_priv(dev);  in niu_tx_timeout()  local
6546 	dev_err(np->device, "%s: Transmit timed out, resetting\n",  in niu_tx_timeout()
6549 	schedule_work(&np->reset_task);  in niu_tx_timeout()
6632 	struct niu *np = netdev_priv(dev);  in niu_start_xmit()  local
6643 	rp = &np->tx_rings[i];  in niu_start_xmit()
6648 		dev_err(np->device, "%s: BUG! Tx ring full when queue awake!\n", dev->name);  in niu_start_xmit()
6681 	mapping = np->ops->map_single(np->device, skb->data,  in niu_start_xmit()
6721 		mapping = np->ops->map_page(np->device, skb_frag_page(frag),  in niu_start_xmit()
6756 	struct niu *np = netdev_priv(dev);  in niu_change_mtu()  local
6771 	niu_full_shutdown(np, dev);  in niu_change_mtu()
6773 	niu_free_channels(np);  in niu_change_mtu()
6775 	niu_enable_napi(np);  in niu_change_mtu()
6777 	err = niu_alloc_channels(np);  in niu_change_mtu()
6781 	spin_lock_irq(&np->lock);  in niu_change_mtu()
6783 	err = niu_init_hw(np);  in niu_change_mtu()
6785 		init_timer(&np->timer);  in niu_change_mtu()
6786 		np->timer.expires = jiffies + HZ;  in niu_change_mtu()
6787 		np->timer.data = (unsigned long) np;  in niu_change_mtu()
6788 		np->timer.function = niu_timer;  in niu_change_mtu()
6790 		err = niu_enable_interrupts(np, 1);  in niu_change_mtu()
6792 			niu_stop_hw(np);  in niu_change_mtu()
6795 	spin_unlock_irq(&np->lock);  in niu_change_mtu()
6799 		if (np->link_config.loopback_mode != LOOPBACK_DISABLED)  in niu_change_mtu()
6802 		add_timer(&np->timer);  in niu_change_mtu()
6811 	struct niu *np = netdev_priv(dev);  in niu_get_drvinfo()  local
6812 	struct niu_vpd *vpd = &np->vpd;  in niu_get_drvinfo()
6818 	if (np->parent->plat_type != PLAT_TYPE_NIU)  in niu_get_drvinfo()
6819 		strlcpy(info->bus_info, pci_name(np->pdev),  in niu_get_drvinfo()
6825 	struct niu *np = netdev_priv(dev);  in niu_get_settings()  local
6828 	lp = &np->link_config;  in niu_get_settings()
6831 	cmd->phy_address = np->phy_addr;  in niu_get_settings()
6837 	cmd->port = (np->flags & NIU_FLAGS_FIBER) ? PORT_FIBRE : PORT_TP;  in niu_get_settings()
6838 	cmd->transceiver = (np->flags & NIU_FLAGS_XCVR_SERDES) ?  in niu_get_settings()
6846 	struct niu *np = netdev_priv(dev);  in niu_set_settings()  local
6847 	struct niu_link_config *lp = &np->link_config;  in niu_set_settings()
6853 	return niu_init_link(np);  in niu_set_settings()
6858 	struct niu *np = netdev_priv(dev);  in niu_get_msglevel()  local
6859 	return np->msg_enable;  in niu_get_msglevel()
6864 	struct niu *np = netdev_priv(dev);  in niu_set_msglevel()  local
6865 	np->msg_enable = value;  in niu_set_msglevel()
6870 	struct niu *np = netdev_priv(dev);  in niu_nway_reset()  local
6872 	if (np->link_config.autoneg)  in niu_nway_reset()
6873 		return niu_init_link(np);  in niu_nway_reset()
6880 	struct niu *np = netdev_priv(dev);  in niu_get_eeprom_len()  local
6882 	return np->eeprom_len;  in niu_get_eeprom_len()
6888 	struct niu *np = netdev_priv(dev);  in niu_get_eeprom()  local
6896 	if (offset >= np->eeprom_len)  in niu_get_eeprom()
6898 	if (offset + len > np->eeprom_len)  in niu_get_eeprom()
6899 		len = eeprom->len = np->eeprom_len - offset;  in niu_get_eeprom()
7084 static int niu_get_hash_opts(struct niu *np, struct ethtool_rxnfc *nfc)  in niu_get_hash_opts()  argument
7093 	if (np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] &  in niu_get_hash_opts()
7097 		nfc->data = niu_flowkey_to_ethflow(np->parent->flow_key[class -  in niu_get_hash_opts()
7178 static int niu_get_ethtool_tcam_entry(struct niu *np,  in niu_get_ethtool_tcam_entry()  argument
7181 	struct niu_parent *parent = np->parent;  in niu_get_ethtool_tcam_entry()
7188 	idx = tcam_get_index(np, (u16)nfc->fs.location);  in niu_get_ethtool_tcam_entry()
7192 		netdev_info(np->dev, "niu%d: entry [%d] invalid for idx[%d]\n",  in niu_get_ethtool_tcam_entry()
7202 		netdev_info(np->dev, "niu%d: niu_class_to_ethflow failed\n",  in niu_get_ethtool_tcam_entry()
7252 	nfc->data = tcam_get_size(np);  in niu_get_ethtool_tcam_entry()
7257 static int niu_get_ethtool_tcam_all(struct niu *np,  in niu_get_ethtool_tcam_all()  argument
7261 	struct niu_parent *parent = np->parent;  in niu_get_ethtool_tcam_all()
7268 	nfc->data = tcam_get_size(np);  in niu_get_ethtool_tcam_all()
7270 	niu_lock_parent(np, flags);  in niu_get_ethtool_tcam_all()
7272 		idx = tcam_get_index(np, i);  in niu_get_ethtool_tcam_all()
7283 	niu_unlock_parent(np, flags);  in niu_get_ethtool_tcam_all()
7293 	struct niu *np = netdev_priv(dev);  in niu_get_nfc()  local
7298 		ret = niu_get_hash_opts(np, cmd);  in niu_get_nfc()
7301 		cmd->data = np->num_rx_rings;  in niu_get_nfc()
7304 		cmd->rule_cnt = tcam_get_valid_entry_cnt(np);  in niu_get_nfc()
7307 		ret = niu_get_ethtool_tcam_entry(np, cmd);  in niu_get_nfc()
7310 		ret = niu_get_ethtool_tcam_all(np, cmd, rule_locs);  in niu_get_nfc()
7320 static int niu_set_hash_opts(struct niu *np, struct ethtool_rxnfc *nfc)  in niu_set_hash_opts()  argument
7334 		niu_lock_parent(np, flags);  in niu_set_hash_opts()
7335 		flow_key = np->parent->tcam_key[class -  in niu_set_hash_opts()
7339 		np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] = flow_key;  in niu_set_hash_opts()
7340 		niu_unlock_parent(np, flags);  in niu_set_hash_opts()
7344 		if (np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] &  in niu_set_hash_opts()
7346 			niu_lock_parent(np, flags);  in niu_set_hash_opts()
7347 			flow_key = np->parent->tcam_key[class -  in niu_set_hash_opts()
7352 			np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] =  in niu_set_hash_opts()
7354 			niu_unlock_parent(np, flags);  in niu_set_hash_opts()
7361 	niu_lock_parent(np, flags);  in niu_set_hash_opts()
7363 	np->parent->flow_key[class - CLASS_CODE_USER_PROG1] = flow_key;  in niu_set_hash_opts()
7364 	niu_unlock_parent(np, flags);  in niu_set_hash_opts()
7437 static int niu_add_ethtool_tcam_entry(struct niu *np,  in niu_add_ethtool_tcam_entry()  argument
7440 	struct niu_parent *parent = np->parent;  in niu_add_ethtool_tcam_entry()
7443 	struct niu_rdc_tables *rdc_table = &parent->rdc_group_cfg[np->port];  in niu_add_ethtool_tcam_entry()
7453 	if (idx >= tcam_get_size(np))  in niu_add_ethtool_tcam_entry()
7465 		niu_lock_parent(np, flags);  in niu_add_ethtool_tcam_entry()
7493 				ret = tcam_user_ip_class_set(np, class, 0,  in niu_add_ethtool_tcam_entry()
7500 				ret = tcam_user_ip_class_enable(np, class, 1);  in niu_add_ethtool_tcam_entry()
7511 			netdev_info(np->dev, "niu%d: %s(): Could not find/insert class for pid %d\n",  in niu_add_ethtool_tcam_entry()
7516 		niu_unlock_parent(np, flags);  in niu_add_ethtool_tcam_entry()
7523 	niu_lock_parent(np, flags);  in niu_add_ethtool_tcam_entry()
7525 	idx = tcam_get_index(np, idx);  in niu_add_ethtool_tcam_entry()
7545 		netdev_info(np->dev, "niu%d: In %s(): flow %d for IPv6 not implemented\n",  in niu_add_ethtool_tcam_entry()
7553 		netdev_info(np->dev, "niu%d: In %s(): Unknown flow type %d\n",  in niu_add_ethtool_tcam_entry()
7563 		if (fsp->ring_cookie >= np->num_rx_rings) {  in niu_add_ethtool_tcam_entry()
7564 			netdev_info(np->dev, "niu%d: In %s(): Invalid RX ring %lld\n",  in niu_add_ethtool_tcam_entry()
7575 	err = tcam_write(np, idx, tp->key, tp->key_mask);  in niu_add_ethtool_tcam_entry()
7580 	err = tcam_assoc_write(np, idx, tp->assoc_data);  in niu_add_ethtool_tcam_entry()
7588 	np->clas.tcam_valid_entries++;  in niu_add_ethtool_tcam_entry()
7590 	niu_unlock_parent(np, flags);  in niu_add_ethtool_tcam_entry()
7595 static int niu_del_ethtool_tcam_entry(struct niu *np, u32 loc)  in niu_del_ethtool_tcam_entry()  argument
7597 	struct niu_parent *parent = np->parent;  in niu_del_ethtool_tcam_entry()
7604 	if (loc >= tcam_get_size(np))  in niu_del_ethtool_tcam_entry()
7607 	niu_lock_parent(np, flags);  in niu_del_ethtool_tcam_entry()
7609 	idx = tcam_get_index(np, loc);  in niu_del_ethtool_tcam_entry()
7623 					ret = tcam_user_ip_class_enable(np,  in niu_del_ethtool_tcam_entry()
7635 			netdev_info(np->dev, "niu%d: In %s(): Usr class 0x%llx not found\n",  in niu_del_ethtool_tcam_entry()
7643 	ret = tcam_flush(np, idx);  in niu_del_ethtool_tcam_entry()
7649 	np->clas.tcam_valid_entries--;  in niu_del_ethtool_tcam_entry()
7651 	niu_unlock_parent(np, flags);  in niu_del_ethtool_tcam_entry()
7658 	struct niu *np = netdev_priv(dev);  in niu_set_nfc()  local
7663 		ret = niu_set_hash_opts(np, cmd);  in niu_set_nfc()
7666 		ret = niu_add_ethtool_tcam_entry(np, cmd);  in niu_set_nfc()
7669 		ret = niu_del_ethtool_tcam_entry(np, cmd->fs.location);  in niu_set_nfc()
7759 	struct niu *np = netdev_priv(dev);  in niu_get_strings()  local
7765 	if (np->flags & NIU_FLAGS_XMAC) {  in niu_get_strings()
7774 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_get_strings()
7779 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_get_strings()
7788 	struct niu *np = netdev_priv(dev);  in niu_get_sset_count()  local
7793 	return (np->flags & NIU_FLAGS_XMAC ?  in niu_get_sset_count()
7796 		(np->num_rx_rings * NUM_RXCHAN_STAT_KEYS) +  in niu_get_sset_count()
7797 		(np->num_tx_rings * NUM_TXCHAN_STAT_KEYS);  in niu_get_sset_count()
7803 	struct niu *np = netdev_priv(dev);  in niu_get_ethtool_stats()  local
7806 	niu_sync_mac_stats(np);  in niu_get_ethtool_stats()
7807 	if (np->flags & NIU_FLAGS_XMAC) {  in niu_get_ethtool_stats()
7808 		memcpy(data, &np->mac_stats.xmac,  in niu_get_ethtool_stats()
7812 		memcpy(data, &np->mac_stats.bmac,  in niu_get_ethtool_stats()
7816 	for (i = 0; i < np->num_rx_rings; i++) {  in niu_get_ethtool_stats()
7817 		struct rx_ring_info *rp = &np->rx_rings[i];  in niu_get_ethtool_stats()
7819 		niu_sync_rx_discard_stats(np, rp, 0);  in niu_get_ethtool_stats()
7828 	for (i = 0; i < np->num_tx_rings; i++) {  in niu_get_ethtool_stats()
7829 		struct tx_ring_info *rp = &np->tx_rings[i];  in niu_get_ethtool_stats()
7839 static u64 niu_led_state_save(struct niu *np)  in niu_led_state_save()  argument
7841 	if (np->flags & NIU_FLAGS_XMAC)  in niu_led_state_save()
7847 static void niu_led_state_restore(struct niu *np, u64 val)  in niu_led_state_restore()  argument
7849 	if (np->flags & NIU_FLAGS_XMAC)  in niu_led_state_restore()
7855 static void niu_force_led(struct niu *np, int on)  in niu_force_led()  argument
7859 	if (np->flags & NIU_FLAGS_XMAC) {  in niu_force_led()
7879 	struct niu *np = netdev_priv(dev);  in niu_set_phys_id()  local
7886 		np->orig_led_state = niu_led_state_save(np);  in niu_set_phys_id()
7890 		niu_force_led(np, 1);  in niu_set_phys_id()
7894 		niu_force_led(np, 0);  in niu_set_phys_id()
7898 		niu_led_state_restore(np, np->orig_led_state);  in niu_set_phys_id()
7922 static int niu_ldg_assign_ldn(struct niu *np, struct niu_parent *parent,  in niu_ldg_assign_ldn()  argument
7932 	if (np->parent->plat_type == PLAT_TYPE_NIU) {  in niu_ldg_assign_ldn()
7939 			dev_err(np->device, "Port %u, mis-matched LDG assignment for ldn %d, should be %d is %llu\n",  in niu_ldg_assign_ldn()
7940 				np->port, ldn, ldg,  in niu_ldg_assign_ldn()
7950 static int niu_set_ldg_timer_res(struct niu *np, int res)  in niu_set_ldg_timer_res()  argument
7961 static int niu_set_ldg_sid(struct niu *np, int ldg, int func, int vector)  in niu_set_ldg_sid()  argument
7973 static int niu_pci_eeprom_read(struct niu *np, u32 addr)  in niu_pci_eeprom_read()  argument
7992 		dev_err(np->device, "EEPROM read timeout frame[%llx]\n",  in niu_pci_eeprom_read()
8007 		dev_err(np->device, "EEPROM read timeout frame[%llx]\n",  in niu_pci_eeprom_read()
8016 static int niu_pci_eeprom_read16(struct niu *np, u32 off)  in niu_pci_eeprom_read16()  argument
8018 	int err = niu_pci_eeprom_read(np, off);  in niu_pci_eeprom_read16()
8024 	err = niu_pci_eeprom_read(np, off + 1);  in niu_pci_eeprom_read16()
8032 static int niu_pci_eeprom_read16_swp(struct niu *np, u32 off)  in niu_pci_eeprom_read16_swp()  argument
8034 	int err = niu_pci_eeprom_read(np, off);  in niu_pci_eeprom_read16_swp()
8041 	err = niu_pci_eeprom_read(np, off + 1);  in niu_pci_eeprom_read16_swp()
8050 static int niu_pci_vpd_get_propname(struct niu *np, u32 off, char *namebuf,  in niu_pci_vpd_get_propname()  argument
8056 		int err = niu_pci_eeprom_read(np, off + i);  in niu_pci_vpd_get_propname()
8069 static void niu_vpd_parse_version(struct niu *np)  in niu_vpd_parse_version()  argument
8071 	struct niu_vpd *vpd = &np->vpd;  in niu_vpd_parse_version()
8086 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_vpd_parse_version()
8092 		np->flags |= NIU_FLAGS_VPD_VALID;  in niu_vpd_parse_version()
8096 static int niu_pci_vpd_scan_props(struct niu *np, u32 start, u32 end)  in niu_pci_vpd_scan_props()  argument
8107 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_vpd_scan_props()
8116 			niu_vpd_parse_version(np);  in niu_pci_vpd_scan_props()
8120 		err = niu_pci_eeprom_read(np, start + 2);  in niu_pci_vpd_scan_props()
8126 		prop_len = niu_pci_eeprom_read(np, start + 4);  in niu_pci_vpd_scan_props()
8127 		err = niu_pci_vpd_get_propname(np, start + 5, namebuf, 64);  in niu_pci_vpd_scan_props()
8134 			prop_buf = np->vpd.model;  in niu_pci_vpd_scan_props()
8138 			prop_buf = np->vpd.board_model;  in niu_pci_vpd_scan_props()
8142 			prop_buf = np->vpd.version;  in niu_pci_vpd_scan_props()
8146 			prop_buf = np->vpd.local_mac;  in niu_pci_vpd_scan_props()
8150 			prop_buf = &np->vpd.mac_num;  in niu_pci_vpd_scan_props()
8154 			prop_buf = np->vpd.phy_type;  in niu_pci_vpd_scan_props()
8160 			dev_err(np->device, "Property '%s' length (%d) is too long\n", namebuf, prop_len);  in niu_pci_vpd_scan_props()
8168 			netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_vpd_scan_props()
8172 				*prop_buf++ = niu_pci_eeprom_read(np, off + i);  in niu_pci_vpd_scan_props()
8182 static void niu_pci_vpd_fetch(struct niu *np, u32 start)  in niu_pci_vpd_fetch()  argument
8187 	err = niu_pci_eeprom_read16_swp(np, start + 1);  in niu_pci_vpd_fetch()
8197 		err = niu_pci_eeprom_read(np, here);  in niu_pci_vpd_fetch()
8201 		err = niu_pci_eeprom_read16_swp(np, here + 1);  in niu_pci_vpd_fetch()
8210 		err = niu_pci_vpd_scan_props(np, here, end);  in niu_pci_vpd_fetch()
8217 static u32 niu_pci_vpd_offset(struct niu *np)  in niu_pci_vpd_offset()  argument
8226 		err = niu_pci_eeprom_read16(np, start +  0);  in niu_pci_vpd_offset()
8231 		err = niu_pci_eeprom_read16(np, start + 23);  in niu_pci_vpd_offset()
8237 		err = niu_pci_eeprom_read16(np, start +  0);  in niu_pci_vpd_offset()
8240 		err = niu_pci_eeprom_read16(np, start +  2);  in niu_pci_vpd_offset()
8245 		err = niu_pci_eeprom_read(np, start + 20);  in niu_pci_vpd_offset()
8249 			err = niu_pci_eeprom_read(np, ret + 2);  in niu_pci_vpd_offset()
8257 		err = niu_pci_eeprom_read16_swp(np, start + 8);  in niu_pci_vpd_offset()
8262 		err = niu_pci_eeprom_read(np, ret + 0);  in niu_pci_vpd_offset()
8272 static int niu_phy_type_prop_decode(struct niu *np, const char *phy_prop)  in niu_phy_type_prop_decode()  argument
8276 		np->flags &= ~(NIU_FLAGS_FIBER |  in niu_phy_type_prop_decode()
8278 		np->mac_xcvr = MAC_XCVR_MII;  in niu_phy_type_prop_decode()
8281 		np->flags |= (NIU_FLAGS_10G |  in niu_phy_type_prop_decode()
8283 		np->mac_xcvr = MAC_XCVR_XPCS;  in niu_phy_type_prop_decode()
8286 		np->flags &= ~NIU_FLAGS_10G;  in niu_phy_type_prop_decode()
8287 		np->flags |= NIU_FLAGS_FIBER;  in niu_phy_type_prop_decode()
8288 		np->mac_xcvr = MAC_XCVR_PCS;  in niu_phy_type_prop_decode()
8291 		np->flags |= NIU_FLAGS_10G;  in niu_phy_type_prop_decode()
8292 		np->flags &= ~NIU_FLAGS_FIBER;  in niu_phy_type_prop_decode()
8293 		np->mac_xcvr = MAC_XCVR_XPCS;  in niu_phy_type_prop_decode()
8296 		np->flags |= NIU_FLAGS_10G;  in niu_phy_type_prop_decode()
8297 		np->flags &= ~NIU_FLAGS_FIBER;  in niu_phy_type_prop_decode()
8298 		np->flags |= NIU_FLAGS_XCVR_SERDES;  in niu_phy_type_prop_decode()
8299 		np->mac_xcvr = MAC_XCVR_XPCS;  in niu_phy_type_prop_decode()
8306 static int niu_pci_vpd_get_nports(struct niu *np)  in niu_pci_vpd_get_nports()  argument
8310 	if ((!strcmp(np->vpd.model, NIU_QGC_LP_MDL_STR)) ||  in niu_pci_vpd_get_nports()
8311 	    (!strcmp(np->vpd.model, NIU_QGC_PEM_MDL_STR)) ||  in niu_pci_vpd_get_nports()
8312 	    (!strcmp(np->vpd.model, NIU_MARAMBA_MDL_STR)) ||  in niu_pci_vpd_get_nports()
8313 	    (!strcmp(np->vpd.model, NIU_KIMI_MDL_STR)) ||  in niu_pci_vpd_get_nports()
8314 	    (!strcmp(np->vpd.model, NIU_ALONSO_MDL_STR))) {  in niu_pci_vpd_get_nports()
8316 	} else if ((!strcmp(np->vpd.model, NIU_2XGF_LP_MDL_STR)) ||  in niu_pci_vpd_get_nports()
8317 		   (!strcmp(np->vpd.model, NIU_2XGF_PEM_MDL_STR)) ||  in niu_pci_vpd_get_nports()
8318 		   (!strcmp(np->vpd.model, NIU_FOXXY_MDL_STR)) ||  in niu_pci_vpd_get_nports()
8319 		   (!strcmp(np->vpd.model, NIU_2XGF_MRVL_MDL_STR))) {  in niu_pci_vpd_get_nports()
8326 static void niu_pci_vpd_validate(struct niu *np)  in niu_pci_vpd_validate()  argument
8328 	struct net_device *dev = np->dev;  in niu_pci_vpd_validate()
8329 	struct niu_vpd *vpd = &np->vpd;  in niu_pci_vpd_validate()
8333 		dev_err(np->device, "VPD MAC invalid, falling back to SPROM\n");  in niu_pci_vpd_validate()
8335 		np->flags &= ~NIU_FLAGS_VPD_VALID;  in niu_pci_vpd_validate()
8339 	if (!strcmp(np->vpd.model, NIU_ALONSO_MDL_STR) ||  in niu_pci_vpd_validate()
8340 	    !strcmp(np->vpd.model, NIU_KIMI_MDL_STR)) {  in niu_pci_vpd_validate()
8341 		np->flags |= NIU_FLAGS_10G;  in niu_pci_vpd_validate()
8342 		np->flags &= ~NIU_FLAGS_FIBER;  in niu_pci_vpd_validate()
8343 		np->flags |= NIU_FLAGS_XCVR_SERDES;  in niu_pci_vpd_validate()
8344 		np->mac_xcvr = MAC_XCVR_PCS;  in niu_pci_vpd_validate()
8345 		if (np->port > 1) {  in niu_pci_vpd_validate()
8346 			np->flags |= NIU_FLAGS_FIBER;  in niu_pci_vpd_validate()
8347 			np->flags &= ~NIU_FLAGS_10G;  in niu_pci_vpd_validate()
8349 		if (np->flags & NIU_FLAGS_10G)  in niu_pci_vpd_validate()
8350 			np->mac_xcvr = MAC_XCVR_XPCS;  in niu_pci_vpd_validate()
8351 	} else if (!strcmp(np->vpd.model, NIU_FOXXY_MDL_STR)) {  in niu_pci_vpd_validate()
8352 		np->flags |= (NIU_FLAGS_10G | NIU_FLAGS_FIBER |  in niu_pci_vpd_validate()
8354 	} else if (niu_phy_type_prop_decode(np, np->vpd.phy_type)) {  in niu_pci_vpd_validate()
8355 		dev_err(np->device, "Illegal phy string [%s]\n",  in niu_pci_vpd_validate()
8356 			np->vpd.phy_type);  in niu_pci_vpd_validate()
8357 		dev_err(np->device, "Falling back to SPROM\n");  in niu_pci_vpd_validate()
8358 		np->flags &= ~NIU_FLAGS_VPD_VALID;  in niu_pci_vpd_validate()
8365 	dev->dev_addr[5] += np->port;  in niu_pci_vpd_validate()
8370 static int niu_pci_probe_sprom(struct niu *np)  in niu_pci_probe_sprom()  argument
8372 	struct net_device *dev = np->dev;  in niu_pci_probe_sprom()
8381 	np->eeprom_len = len;  in niu_pci_probe_sprom()
8383 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8394 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8397 		dev_err(np->device, "Bad SPROM checksum (%x, should be 0xab)\n", (int)(sum & 0xff));  in niu_pci_probe_sprom()
8402 	switch (np->port) {  in niu_pci_probe_sprom()
8420 		dev_err(np->device, "Bogus port number %u\n",  in niu_pci_probe_sprom()
8421 			np->port);  in niu_pci_probe_sprom()
8424 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8430 		np->flags &= ~(NIU_FLAGS_FIBER |  in niu_pci_probe_sprom()
8432 		np->mac_xcvr = MAC_XCVR_MII;  in niu_pci_probe_sprom()
8437 		np->flags &= ~NIU_FLAGS_10G;  in niu_pci_probe_sprom()
8438 		np->flags |= NIU_FLAGS_FIBER;  in niu_pci_probe_sprom()
8439 		np->mac_xcvr = MAC_XCVR_PCS;  in niu_pci_probe_sprom()
8444 		np->flags |= NIU_FLAGS_10G;  in niu_pci_probe_sprom()
8445 		np->flags &= ~NIU_FLAGS_FIBER;  in niu_pci_probe_sprom()
8446 		np->mac_xcvr = MAC_XCVR_XPCS;  in niu_pci_probe_sprom()
8451 		np->flags |= (NIU_FLAGS_10G |  in niu_pci_probe_sprom()
8453 		np->mac_xcvr = MAC_XCVR_XPCS;  in niu_pci_probe_sprom()
8457 		dev_err(np->device, "Bogus SPROM phy type %u\n", val8);  in niu_pci_probe_sprom()
8462 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8470 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8476 		dev_err(np->device, "SPROM MAC address invalid [ %pM ]\n",  in niu_pci_probe_sprom()
8482 	dev->dev_addr[5] += np->port;  in niu_pci_probe_sprom()
8487 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8495 		np->vpd.model[i + 3] = (tmp >>  0) & 0xff;  in niu_pci_probe_sprom()
8496 		np->vpd.model[i + 2] = (tmp >>  8) & 0xff;  in niu_pci_probe_sprom()
8497 		np->vpd.model[i + 1] = (tmp >> 16) & 0xff;  in niu_pci_probe_sprom()
8498 		np->vpd.model[i + 0] = (tmp >> 24) & 0xff;  in niu_pci_probe_sprom()
8500 	np->vpd.model[val] = '\0';  in niu_pci_probe_sprom()
8503 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8511 		np->vpd.board_model[i + 3] = (tmp >>  0) & 0xff;  in niu_pci_probe_sprom()
8512 		np->vpd.board_model[i + 2] = (tmp >>  8) & 0xff;  in niu_pci_probe_sprom()
8513 		np->vpd.board_model[i + 1] = (tmp >> 16) & 0xff;  in niu_pci_probe_sprom()
8514 		np->vpd.board_model[i + 0] = (tmp >> 24) & 0xff;  in niu_pci_probe_sprom()
8516 	np->vpd.board_model[val] = '\0';  in niu_pci_probe_sprom()
8518 	np->vpd.mac_num =  in niu_pci_probe_sprom()
8520 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_pci_probe_sprom()
8521 		     "SPROM: NUM_PORTS_MACS[%d]\n", np->vpd.mac_num);  in niu_pci_probe_sprom()
8526 static int niu_get_and_validate_port(struct niu *np)  in niu_get_and_validate_port()  argument
8528 	struct niu_parent *parent = np->parent;  in niu_get_and_validate_port()
8530 	if (np->port <= 1)  in niu_get_and_validate_port()
8531 		np->flags |= NIU_FLAGS_XMAC;  in niu_get_and_validate_port()
8537 			parent->num_ports = niu_pci_vpd_get_nports(np);  in niu_get_and_validate_port()
8554 	if (np->port >= parent->num_ports)  in niu_get_and_validate_port()
8762 static int fill_phy_probe_info(struct niu *np, struct niu_parent *parent,  in fill_phy_probe_info()  argument
8771 	niu_lock_parent(np, flags);  in fill_phy_probe_info()
8776 		dev_id_1 = mdio_read(np, port,  in fill_phy_probe_info()
8778 		dev_id_2 = mdio_read(np, port,  in fill_phy_probe_info()
8784 		dev_id_1 = mdio_read(np, port,  in fill_phy_probe_info()
8786 		dev_id_2 = mdio_read(np, port,  in fill_phy_probe_info()
8792 		dev_id_1 = mii_read(np, port, MII_PHYSID1);  in fill_phy_probe_info()
8793 		dev_id_2 = mii_read(np, port, MII_PHYSID2);  in fill_phy_probe_info()
8799 	niu_unlock_parent(np, flags);  in fill_phy_probe_info()
8804 static int walk_phys(struct niu *np, struct niu_parent *parent)  in walk_phys()  argument
8814 	if (!strcmp(np->vpd.model, NIU_ALONSO_MDL_STR) ||  in walk_phys()
8815 	    !strcmp(np->vpd.model, NIU_KIMI_MDL_STR)) {  in walk_phys()
8824 	} else if (!strcmp(np->vpd.model, NIU_FOXXY_MDL_STR)) {  in walk_phys()
8830 	} else if ((np->flags & NIU_FLAGS_XCVR_SERDES) &&  in walk_phys()
8833 		if (np->flags & NIU_FLAGS_10G) {  in walk_phys()
8841 		err = fill_phy_probe_info(np, parent, info);  in walk_phys()
8871 			val = phy_encode(PORT_TYPE_10G, np->port);  in walk_phys()
8933 static int niu_probe_ports(struct niu *np)  in niu_probe_ports()  argument
8935 	struct niu_parent *parent = np->parent;  in niu_probe_ports()
8939 		err = walk_phys(np, parent);  in niu_probe_ports()
8943 		niu_set_ldg_timer_res(np, 2);  in niu_probe_ports()
8945 			niu_ldn_irq_enable(np, i, 0);  in niu_probe_ports()
8954 static int niu_classifier_swstate_init(struct niu *np)  in niu_classifier_swstate_init()  argument
8956 	struct niu_classifier *cp = &np->clas;  in niu_classifier_swstate_init()
8958 	cp->tcam_top = (u16) np->port;  in niu_classifier_swstate_init()
8959 	cp->tcam_sz = np->parent->tcam_num_entries / np->parent->num_ports;  in niu_classifier_swstate_init()
8963 	return fflp_early_init(np);  in niu_classifier_swstate_init()
8966 static void niu_link_config_init(struct niu *np)  in niu_link_config_init()  argument
8968 	struct niu_link_config *lp = &np->link_config;  in niu_link_config_init()
8991 static int niu_init_mac_ipp_pcs_base(struct niu *np)  in niu_init_mac_ipp_pcs_base()  argument
8993 	switch (np->port) {  in niu_init_mac_ipp_pcs_base()
8995 		np->mac_regs = np->regs + XMAC_PORT0_OFF;  in niu_init_mac_ipp_pcs_base()
8996 		np->ipp_off  = 0x00000;  in niu_init_mac_ipp_pcs_base()
8997 		np->pcs_off  = 0x04000;  in niu_init_mac_ipp_pcs_base()
8998 		np->xpcs_off = 0x02000;  in niu_init_mac_ipp_pcs_base()
9002 		np->mac_regs = np->regs + XMAC_PORT1_OFF;  in niu_init_mac_ipp_pcs_base()
9003 		np->ipp_off  = 0x08000;  in niu_init_mac_ipp_pcs_base()
9004 		np->pcs_off  = 0x0a000;  in niu_init_mac_ipp_pcs_base()
9005 		np->xpcs_off = 0x08000;  in niu_init_mac_ipp_pcs_base()
9009 		np->mac_regs = np->regs + BMAC_PORT2_OFF;  in niu_init_mac_ipp_pcs_base()
9010 		np->ipp_off  = 0x04000;  in niu_init_mac_ipp_pcs_base()
9011 		np->pcs_off  = 0x0e000;  in niu_init_mac_ipp_pcs_base()
9012 		np->xpcs_off = ~0UL;  in niu_init_mac_ipp_pcs_base()
9016 		np->mac_regs = np->regs + BMAC_PORT3_OFF;  in niu_init_mac_ipp_pcs_base()
9017 		np->ipp_off  = 0x0c000;  in niu_init_mac_ipp_pcs_base()
9018 		np->pcs_off  = 0x12000;  in niu_init_mac_ipp_pcs_base()
9019 		np->xpcs_off = ~0UL;  in niu_init_mac_ipp_pcs_base()
9023 		dev_err(np->device, "Port %u is invalid, cannot compute MAC block offset\n", np->port);  in niu_init_mac_ipp_pcs_base()
9030 static void niu_try_msix(struct niu *np, u8 *ldg_num_map)  in niu_try_msix()  argument
9033 	struct niu_parent *parent = np->parent;  in niu_try_msix()
9034 	struct pci_dev *pdev = np->pdev;  in niu_try_msix()
9038 	first_ldg = (NIU_NUM_LDG / parent->num_ports) * np->port;  in niu_try_msix()
9042 	num_irqs = (parent->rxchan_per_port[np->port] +  in niu_try_msix()
9043 		    parent->txchan_per_port[np->port] +  in niu_try_msix()
9044 		    (np->port == 0 ? 3 : 1));  in niu_try_msix()
9054 		np->flags &= ~NIU_FLAGS_MSIX;  in niu_try_msix()
9058 	np->flags |= NIU_FLAGS_MSIX;  in niu_try_msix()
9060 		np->ldg[i].irq = msi_vec[i].vector;  in niu_try_msix()
9061 	np->num_ldg = num_irqs;  in niu_try_msix()
9064 static int niu_n2_irq_init(struct niu *np, u8 *ldg_num_map)  in niu_n2_irq_init()  argument
9067 	struct platform_device *op = np->op;  in niu_n2_irq_init()
9077 		np->ldg[i].irq = op->archdata.irqs[i];  in niu_n2_irq_init()
9080 	np->num_ldg = op->archdata.num_irqs;  in niu_n2_irq_init()
9088 static int niu_ldg_init(struct niu *np)  in niu_ldg_init()  argument
9090 	struct niu_parent *parent = np->parent;  in niu_ldg_init()
9096 	np->num_ldg = 1;  in niu_ldg_init()
9097 	np->ldg[0].irq = np->dev->irq;  in niu_ldg_init()
9099 		err = niu_n2_irq_init(np, ldg_num_map);  in niu_ldg_init()
9103 		niu_try_msix(np, ldg_num_map);  in niu_ldg_init()
9105 	port = np->port;  in niu_ldg_init()
9106 	for (i = 0; i < np->num_ldg; i++) {  in niu_ldg_init()
9107 		struct niu_ldg *lp = &np->ldg[i];  in niu_ldg_init()
9109 		netif_napi_add(np->dev, &lp->napi, niu_poll, 64);  in niu_ldg_init()
9111 		lp->np = np;  in niu_ldg_init()
9119 		if (np->parent->plat_type != PLAT_TYPE_NIU) {  in niu_ldg_init()
9120 			err = niu_set_ldg_sid(np, lp->ldg_num, port, i);  in niu_ldg_init()
9139 	err = niu_ldg_assign_ldn(np, parent, ldg_num_map[ldg_rotor],  in niu_ldg_init()
9145 	if (ldg_rotor == np->num_ldg)  in niu_ldg_init()
9149 		err = niu_ldg_assign_ldn(np, parent,  in niu_ldg_init()
9156 		if (ldg_rotor == np->num_ldg)  in niu_ldg_init()
9159 		err = niu_ldg_assign_ldn(np, parent,  in niu_ldg_init()
9166 		if (ldg_rotor == np->num_ldg)  in niu_ldg_init()
9177 		err = niu_ldg_assign_ldn(np, parent,  in niu_ldg_init()
9183 		if (ldg_rotor == np->num_ldg)  in niu_ldg_init()
9192 		err = niu_ldg_assign_ldn(np, parent,  in niu_ldg_init()
9198 		if (ldg_rotor == np->num_ldg)  in niu_ldg_init()
9205 static void niu_ldg_free(struct niu *np)  in niu_ldg_free()  argument
9207 	if (np->flags & NIU_FLAGS_MSIX)  in niu_ldg_free()
9208 		pci_disable_msix(np->pdev);  in niu_ldg_free()
9211 static int niu_get_of_props(struct niu *np)  in niu_get_of_props()  argument
9214 	struct net_device *dev = np->dev;  in niu_get_of_props()
9221 	if (np->parent->plat_type == PLAT_TYPE_NIU)  in niu_get_of_props()
9222 		dp = np->op->dev.of_node;  in niu_get_of_props()
9224 		dp = pci_device_to_OF_node(np->pdev);  in niu_get_of_props()
9236 	strcpy(np->vpd.phy_type, phy_type);  in niu_get_of_props()
9238 	if (niu_phy_type_prop_decode(np, np->vpd.phy_type)) {  in niu_get_of_props()
9240 			   dp->full_name, np->vpd.phy_type);  in niu_get_of_props()
9265 		strcpy(np->vpd.model, model);  in niu_get_of_props()
9268 		np->flags |= (NIU_FLAGS_10G | NIU_FLAGS_FIBER |  in niu_get_of_props()
9278 static int niu_get_invariants(struct niu *np)  in niu_get_invariants()  argument
9283 	err = niu_get_of_props(np);  in niu_get_invariants()
9289 	err = niu_init_mac_ipp_pcs_base(np);  in niu_get_invariants()
9294 		err = niu_get_and_validate_port(np);  in niu_get_invariants()
9299 		if (np->parent->plat_type == PLAT_TYPE_NIU)  in niu_get_invariants()
9303 		offset = niu_pci_vpd_offset(np);  in niu_get_invariants()
9304 		netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_get_invariants()
9307 			niu_pci_vpd_fetch(np, offset);  in niu_get_invariants()
9310 		if (np->flags & NIU_FLAGS_VPD_VALID) {  in niu_get_invariants()
9311 			niu_pci_vpd_validate(np);  in niu_get_invariants()
9312 			err = niu_get_and_validate_port(np);  in niu_get_invariants()
9317 		if (!(np->flags & NIU_FLAGS_VPD_VALID)) {  in niu_get_invariants()
9318 			err = niu_get_and_validate_port(np);  in niu_get_invariants()
9321 			err = niu_pci_probe_sprom(np);  in niu_get_invariants()
9327 	err = niu_probe_ports(np);  in niu_get_invariants()
9331 	niu_ldg_init(np);  in niu_get_invariants()
9333 	niu_classifier_swstate_init(np);  in niu_get_invariants()
9334 	niu_link_config_init(np);  in niu_get_invariants()
9336 	err = niu_determine_phy_disposition(np);  in niu_get_invariants()
9338 		err = niu_init_link(np);  in niu_get_invariants()
9457 static struct niu_parent *niu_new_parent(struct niu *np,  in niu_new_parent()  argument
9521 static struct niu_parent *niu_get_parent(struct niu *np,  in niu_get_parent()  argument
9525 	int port = np->port;  in niu_get_parent()
9536 		p = niu_new_parent(np, id, ptype);  in niu_get_parent()
9544 					&np->device->kobj,  in niu_get_parent()
9547 			p->ports[port] = np;  in niu_get_parent()
9556 static void niu_put_parent(struct niu *np)  in niu_put_parent()  argument
9558 	struct niu_parent *p = np->parent;  in niu_put_parent()
9559 	u8 port = np->port;  in niu_put_parent()
9562 	BUG_ON(!p || p->ports[port] != np);  in niu_put_parent()
9564 	netif_printk(np, probe, KERN_DEBUG, np->dev,  in niu_put_parent()
9574 	np->parent = NULL;  in niu_put_parent()
9652 	struct niu *np;  in niu_alloc_and_init()  local
9660 	np = netdev_priv(dev);  in niu_alloc_and_init()
9661 	np->dev = dev;  in niu_alloc_and_init()
9662 	np->pdev = pdev;  in niu_alloc_and_init()
9663 	np->op = op;  in niu_alloc_and_init()
9664 	np->device = gen_dev;  in niu_alloc_and_init()
9665 	np->ops = ops;  in niu_alloc_and_init()
9667 	np->msg_enable = niu_debug;  in niu_alloc_and_init()
9669 	spin_lock_init(&np->lock);  in niu_alloc_and_init()
9670 	INIT_WORK(&np->reset_task, niu_reset_task);  in niu_alloc_and_init()
9672 	np->port = port;  in niu_alloc_and_init()
9697 static void niu_device_announce(struct niu *np)  in niu_device_announce()  argument
9699 	struct net_device *dev = np->dev;  in niu_device_announce()
9703 	if (np->parent->plat_type == PLAT_TYPE_ATCA_CP3220) {  in niu_device_announce()
9706 				(np->flags & NIU_FLAGS_XMAC ? "XMAC" : "BMAC"),  in niu_device_announce()
9707 				(np->flags & NIU_FLAGS_10G ? "10G" : "1G"),  in niu_device_announce()
9708 				(np->flags & NIU_FLAGS_FIBER ? "RGMII FIBER" : "SERDES"),  in niu_device_announce()
9709 				(np->mac_xcvr == MAC_XCVR_MII ? "MII" :  in niu_device_announce()
9710 				 (np->mac_xcvr == MAC_XCVR_PCS ? "PCS" : "XPCS")),  in niu_device_announce()
9711 				np->vpd.phy_type);  in niu_device_announce()
9715 				(np->flags & NIU_FLAGS_XMAC ? "XMAC" : "BMAC"),  in niu_device_announce()
9716 				(np->flags & NIU_FLAGS_10G ? "10G" : "1G"),  in niu_device_announce()
9717 				(np->flags & NIU_FLAGS_FIBER ? "FIBER" :  in niu_device_announce()
9718 				 (np->flags & NIU_FLAGS_XCVR_SERDES ? "SERDES" :  in niu_device_announce()
9720 				(np->mac_xcvr == MAC_XCVR_MII ? "MII" :  in niu_device_announce()
9721 				 (np->mac_xcvr == MAC_XCVR_PCS ? "PCS" : "XPCS")),  in niu_device_announce()
9722 				np->vpd.phy_type);  in niu_device_announce()
9737 	struct niu *np;  in niu_pci_init_one()  local
9774 	np = netdev_priv(dev);  in niu_pci_init_one()
9781 	np->parent = niu_get_parent(np, &parent_id,  in niu_pci_init_one()
9783 	if (!np->parent) {  in niu_pci_init_one()
9816 	np->regs = pci_ioremap_bar(pdev, 0);  in niu_pci_init_one()
9817 	if (!np->regs) {  in niu_pci_init_one()
9830 	err = niu_get_invariants(np);  in niu_pci_init_one()
9845 	niu_device_announce(np);  in niu_pci_init_one()
9850 	if (np->regs) {  in niu_pci_init_one()
9851 		iounmap(np->regs);  in niu_pci_init_one()
9852 		np->regs = NULL;  in niu_pci_init_one()
9856 	niu_put_parent(np);  in niu_pci_init_one()
9875 		struct niu *np = netdev_priv(dev);  in niu_pci_remove_one()  local
9878 		if (np->regs) {  in niu_pci_remove_one()
9879 			iounmap(np->regs);  in niu_pci_remove_one()
9880 			np->regs = NULL;  in niu_pci_remove_one()
9883 		niu_ldg_free(np);  in niu_pci_remove_one()
9885 		niu_put_parent(np);  in niu_pci_remove_one()
9896 	struct niu *np = netdev_priv(dev);  in niu_suspend()  local
9902 	flush_work(&np->reset_task);  in niu_suspend()
9903 	niu_netif_stop(np);  in niu_suspend()
9905 	del_timer_sync(&np->timer);  in niu_suspend()
9907 	spin_lock_irqsave(&np->lock, flags);  in niu_suspend()
9908 	niu_enable_interrupts(np, 0);  in niu_suspend()
9909 	spin_unlock_irqrestore(&np->lock, flags);  in niu_suspend()
9913 	spin_lock_irqsave(&np->lock, flags);  in niu_suspend()
9914 	niu_stop_hw(np);  in niu_suspend()
9915 	spin_unlock_irqrestore(&np->lock, flags);  in niu_suspend()
9925 	struct niu *np = netdev_priv(dev);  in niu_resume()  local
9936 	spin_lock_irqsave(&np->lock, flags);  in niu_resume()
9938 	err = niu_init_hw(np);  in niu_resume()
9940 		np->timer.expires = jiffies + HZ;  in niu_resume()
9941 		add_timer(&np->timer);  in niu_resume()
9942 		niu_netif_start(np);  in niu_resume()
9945 	spin_unlock_irqrestore(&np->lock, flags);  in niu_resume()
10022 	struct niu *np;  in niu_of_probe()  local
10041 	np = netdev_priv(dev);  in niu_of_probe()
10046 	np->parent = niu_get_parent(np, &parent_id,  in niu_of_probe()
10048 	if (!np->parent) {  in niu_of_probe()
10055 	np->regs = of_ioremap(&op->resource[1], 0,  in niu_of_probe()
10058 	if (!np->regs) {  in niu_of_probe()
10064 	np->vir_regs_1 = of_ioremap(&op->resource[2], 0,  in niu_of_probe()
10067 	if (!np->vir_regs_1) {  in niu_of_probe()
10073 	np->vir_regs_2 = of_ioremap(&op->resource[3], 0,  in niu_of_probe()
10076 	if (!np->vir_regs_2) {  in niu_of_probe()
10084 	err = niu_get_invariants(np);  in niu_of_probe()
10099 	niu_device_announce(np);  in niu_of_probe()
10104 	if (np->vir_regs_1) {  in niu_of_probe()
10105 		of_iounmap(&op->resource[2], np->vir_regs_1,  in niu_of_probe()
10107 		np->vir_regs_1 = NULL;  in niu_of_probe()
10110 	if (np->vir_regs_2) {  in niu_of_probe()
10111 		of_iounmap(&op->resource[3], np->vir_regs_2,  in niu_of_probe()
10113 		np->vir_regs_2 = NULL;  in niu_of_probe()
10116 	if (np->regs) {  in niu_of_probe()
10117 		of_iounmap(&op->resource[1], np->regs,  in niu_of_probe()
10119 		np->regs = NULL;  in niu_of_probe()
10123 	niu_put_parent(np);  in niu_of_probe()
10137 		struct niu *np = netdev_priv(dev);  in niu_of_remove()  local
10141 		if (np->vir_regs_1) {  in niu_of_remove()
10142 			of_iounmap(&op->resource[2], np->vir_regs_1,  in niu_of_remove()
10144 			np->vir_regs_1 = NULL;  in niu_of_remove()
10147 		if (np->vir_regs_2) {  in niu_of_remove()
10148 			of_iounmap(&op->resource[3], np->vir_regs_2,  in niu_of_remove()
10150 			np->vir_regs_2 = NULL;  in niu_of_remove()
10153 		if (np->regs) {  in niu_of_remove()
10154 			of_iounmap(&op->resource[1], np->regs,  in niu_of_remove()
10156 			np->regs = NULL;  in niu_of_remove()
10159 		niu_ldg_free(np);  in niu_of_remove()
10161 		niu_put_parent(np);  in niu_of_remove()