350 					   unsigned int ring,  in bcmgenet_tdma_ring_readl()  argument
354 			(DMA_RING_SIZE * ring) +  in bcmgenet_tdma_ring_readl()
359 					     unsigned int ring, u32 val,  in bcmgenet_tdma_ring_writel()  argument
363 			(DMA_RING_SIZE * ring) +  in bcmgenet_tdma_ring_writel()
368 					   unsigned int ring,  in bcmgenet_rdma_ring_readl()  argument
372 			(DMA_RING_SIZE * ring) +  in bcmgenet_rdma_ring_readl()
377 					     unsigned int ring, u32 val,  in bcmgenet_rdma_ring_writel()  argument
381 			(DMA_RING_SIZE * ring) +  in bcmgenet_rdma_ring_writel()
943 					 struct bcmgenet_tx_ring *ring)  in bcmgenet_get_txcb()  argument
947 	tx_cb_ptr = ring->cbs;  in bcmgenet_get_txcb()
948 	tx_cb_ptr += ring->write_ptr - ring->cb_ptr;  in bcmgenet_get_txcb()
951 	if (ring->write_ptr == ring->end_ptr)  in bcmgenet_get_txcb()
952 		ring->write_ptr = ring->cb_ptr;  in bcmgenet_get_txcb()
954 		ring->write_ptr++;  in bcmgenet_get_txcb()
967 static inline void bcmgenet_rx_ring16_int_disable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring16_int_disable()  argument
969 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,  in bcmgenet_rx_ring16_int_disable()
973 static inline void bcmgenet_rx_ring16_int_enable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring16_int_enable()  argument
975 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,  in bcmgenet_rx_ring16_int_enable()
979 static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring_int_disable()  argument
981 	bcmgenet_intrl2_1_writel(ring->priv,  in bcmgenet_rx_ring_int_disable()
982 				 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),  in bcmgenet_rx_ring_int_disable()
986 static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring_int_enable()  argument
988 	bcmgenet_intrl2_1_writel(ring->priv,  in bcmgenet_rx_ring_int_enable()
989 				 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),  in bcmgenet_rx_ring_int_enable()
993 static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring16_int_disable()  argument
995 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,  in bcmgenet_tx_ring16_int_disable()
999 static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring16_int_enable()  argument
1001 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,  in bcmgenet_tx_ring16_int_enable()
1005 static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring_int_enable()  argument
1007 	bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,  in bcmgenet_tx_ring_int_enable()
1011 static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring_int_disable()  argument
1013 	bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,  in bcmgenet_tx_ring_int_disable()
1019 					  struct bcmgenet_tx_ring *ring)  in __bcmgenet_tx_reclaim()  argument
1030 	c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);  in __bcmgenet_tx_reclaim()
1033 	if (likely(c_index >= ring->c_index))  in __bcmgenet_tx_reclaim()
1034 		txbds_ready = c_index - ring->c_index;  in __bcmgenet_tx_reclaim()
1036 		txbds_ready = (DMA_C_INDEX_MASK + 1) - ring->c_index + c_index;  in __bcmgenet_tx_reclaim()
1040 		  __func__, ring->index, ring->c_index, c_index, txbds_ready);  in __bcmgenet_tx_reclaim()
1044 		tx_cb_ptr = &priv->tx_cbs[ring->clean_ptr];  in __bcmgenet_tx_reclaim()
1065 		if (likely(ring->clean_ptr < ring->end_ptr))  in __bcmgenet_tx_reclaim()
1066 			ring->clean_ptr++;  in __bcmgenet_tx_reclaim()
1068 			ring->clean_ptr = ring->cb_ptr;  in __bcmgenet_tx_reclaim()
1071 	ring->free_bds += txbds_processed;  in __bcmgenet_tx_reclaim()
1072 	ring->c_index = (ring->c_index + txbds_processed) & DMA_C_INDEX_MASK;  in __bcmgenet_tx_reclaim()
1074 	if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {  in __bcmgenet_tx_reclaim()
1075 		txq = netdev_get_tx_queue(dev, ring->queue);  in __bcmgenet_tx_reclaim()
1084 				struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_reclaim()  argument
1089 	spin_lock_irqsave(&ring->lock, flags);  in bcmgenet_tx_reclaim()
1090 	released = __bcmgenet_tx_reclaim(dev, ring);  in bcmgenet_tx_reclaim()
1091 	spin_unlock_irqrestore(&ring->lock, flags);  in bcmgenet_tx_reclaim()
1098 	struct bcmgenet_tx_ring *ring =  in bcmgenet_tx_poll()  local
1102 	work_done = bcmgenet_tx_reclaim(ring->priv->dev, ring);  in bcmgenet_tx_poll()
1106 		ring->int_enable(ring);  in bcmgenet_tx_poll()
1133 				struct bcmgenet_tx_ring *ring)  in bcmgenet_xmit_single()  argument
1143 	tx_cb_ptr = bcmgenet_get_txcb(priv, ring);  in bcmgenet_xmit_single()
1179 			      struct bcmgenet_tx_ring *ring)  in bcmgenet_xmit_frag()  argument
1187 	tx_cb_ptr = bcmgenet_get_txcb(priv, ring);  in bcmgenet_xmit_frag()
1280 	struct bcmgenet_tx_ring *ring = NULL;  in bcmgenet_xmit()  local
1302 	ring = &priv->tx_rings[index];  in bcmgenet_xmit()
1303 	txq = netdev_get_tx_queue(dev, ring->queue);  in bcmgenet_xmit()
1305 	spin_lock_irqsave(&ring->lock, flags);  in bcmgenet_xmit()
1306 	if (ring->free_bds <= nr_frags + 1) {  in bcmgenet_xmit()
1309 			   __func__, index, ring->queue);  in bcmgenet_xmit()
1333 	ret = bcmgenet_xmit_single(dev, skb, dma_desc_flags, ring);  in bcmgenet_xmit()
1344 					 ring);  in bcmgenet_xmit()
1354 	ring->free_bds -= nr_frags + 1;  in bcmgenet_xmit()
1355 	ring->prod_index += nr_frags + 1;  in bcmgenet_xmit()
1356 	ring->prod_index &= DMA_P_INDEX_MASK;  in bcmgenet_xmit()
1358 	if (ring->free_bds <= (MAX_SKB_FRAGS + 1))  in bcmgenet_xmit()
1363 		bcmgenet_tdma_ring_writel(priv, ring->index,  in bcmgenet_xmit()
1364 					  ring->prod_index, TDMA_PROD_INDEX);  in bcmgenet_xmit()
1366 	spin_unlock_irqrestore(&ring->lock, flags);  in bcmgenet_xmit()
1417 static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,  in bcmgenet_desc_rx()  argument
1420 	struct bcmgenet_priv *priv = ring->priv;  in bcmgenet_desc_rx()
1432 	p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);  in bcmgenet_desc_rx()
1436 	if (discards > ring->old_discards) {  in bcmgenet_desc_rx()
1437 		discards = discards - ring->old_discards;  in bcmgenet_desc_rx()
1440 		ring->old_discards += discards;  in bcmgenet_desc_rx()
1443 		if (ring->old_discards >= 0xC000) {  in bcmgenet_desc_rx()
1444 			ring->old_discards = 0;  in bcmgenet_desc_rx()
1445 			bcmgenet_rdma_ring_writel(priv, ring->index, 0,  in bcmgenet_desc_rx()
1452 	if (likely(p_index >= ring->c_index))  in bcmgenet_desc_rx()
1453 		rxpkttoprocess = p_index - ring->c_index;  in bcmgenet_desc_rx()
1455 		rxpkttoprocess = (DMA_C_INDEX_MASK + 1) - ring->c_index +  in bcmgenet_desc_rx()
1463 		cb = &priv->rx_cbs[ring->read_ptr];  in bcmgenet_desc_rx()
1490 			  __func__, p_index, ring->c_index,  in bcmgenet_desc_rx()
1491 			  ring->read_ptr, dma_length_status);  in bcmgenet_desc_rx()
1553 		napi_gro_receive(&ring->napi, skb);  in bcmgenet_desc_rx()
1558 		if (likely(ring->read_ptr < ring->end_ptr))  in bcmgenet_desc_rx()
1559 			ring->read_ptr++;  in bcmgenet_desc_rx()
1561 			ring->read_ptr = ring->cb_ptr;  in bcmgenet_desc_rx()
1563 		ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;  in bcmgenet_desc_rx()
1564 		bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);  in bcmgenet_desc_rx()
1573 	struct bcmgenet_rx_ring *ring = container_of(napi,  in bcmgenet_rx_poll()  local
1577 	work_done = bcmgenet_desc_rx(ring, budget);  in bcmgenet_rx_poll()
1581 		ring->int_enable(ring);  in bcmgenet_rx_poll()
1589 				     struct bcmgenet_rx_ring *ring)  in bcmgenet_alloc_rx_buffers()  argument
1598 	for (i = 0; i < ring->size; i++) {  in bcmgenet_alloc_rx_buffers()
1599 		cb = ring->cbs + i;  in bcmgenet_alloc_rx_buffers()
1777 	struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];  in bcmgenet_init_tx_ring()  local
1781 	spin_lock_init(&ring->lock);  in bcmgenet_init_tx_ring()
1782 	ring->priv = priv;  in bcmgenet_init_tx_ring()
1783 	ring->index = index;  in bcmgenet_init_tx_ring()
1785 		ring->queue = 0;  in bcmgenet_init_tx_ring()
1786 		ring->int_enable = bcmgenet_tx_ring16_int_enable;  in bcmgenet_init_tx_ring()
1787 		ring->int_disable = bcmgenet_tx_ring16_int_disable;  in bcmgenet_init_tx_ring()
1789 		ring->queue = index + 1;  in bcmgenet_init_tx_ring()
1790 		ring->int_enable = bcmgenet_tx_ring_int_enable;  in bcmgenet_init_tx_ring()
1791 		ring->int_disable = bcmgenet_tx_ring_int_disable;  in bcmgenet_init_tx_ring()
1793 	ring->cbs = priv->tx_cbs + start_ptr;  in bcmgenet_init_tx_ring()
1794 	ring->size = size;  in bcmgenet_init_tx_ring()
1795 	ring->clean_ptr = start_ptr;  in bcmgenet_init_tx_ring()
1796 	ring->c_index = 0;  in bcmgenet_init_tx_ring()
1797 	ring->free_bds = size;  in bcmgenet_init_tx_ring()
1798 	ring->write_ptr = start_ptr;  in bcmgenet_init_tx_ring()
1799 	ring->cb_ptr = start_ptr;  in bcmgenet_init_tx_ring()
1800 	ring->end_ptr = end_ptr - 1;  in bcmgenet_init_tx_ring()
1801 	ring->prod_index = 0;  in bcmgenet_init_tx_ring()
1833 	struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];  in bcmgenet_init_rx_ring()  local
1837 	ring->priv = priv;  in bcmgenet_init_rx_ring()
1838 	ring->index = index;  in bcmgenet_init_rx_ring()
1840 		ring->int_enable = bcmgenet_rx_ring16_int_enable;  in bcmgenet_init_rx_ring()
1841 		ring->int_disable = bcmgenet_rx_ring16_int_disable;  in bcmgenet_init_rx_ring()
1843 		ring->int_enable = bcmgenet_rx_ring_int_enable;  in bcmgenet_init_rx_ring()
1844 		ring->int_disable = bcmgenet_rx_ring_int_disable;  in bcmgenet_init_rx_ring()
1846 	ring->cbs = priv->rx_cbs + start_ptr;  in bcmgenet_init_rx_ring()
1847 	ring->size = size;  in bcmgenet_init_rx_ring()
1848 	ring->c_index = 0;  in bcmgenet_init_rx_ring()
1849 	ring->read_ptr = start_ptr;  in bcmgenet_init_rx_ring()
1850 	ring->cb_ptr = start_ptr;  in bcmgenet_init_rx_ring()
1851 	ring->end_ptr = end_ptr - 1;  in bcmgenet_init_rx_ring()
1853 	ret = bcmgenet_alloc_rx_buffers(priv, ring);  in bcmgenet_init_rx_ring()
1884 	struct bcmgenet_tx_ring *ring;  in bcmgenet_init_tx_napi()  local
1887 		ring = &priv->tx_rings[i];  in bcmgenet_init_tx_napi()
1888 		netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64);  in bcmgenet_init_tx_napi()
1891 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_init_tx_napi()
1892 	netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64);  in bcmgenet_init_tx_napi()
1898 	struct bcmgenet_tx_ring *ring;  in bcmgenet_enable_tx_napi()  local
1901 		ring = &priv->tx_rings[i];  in bcmgenet_enable_tx_napi()
1902 		napi_enable(&ring->napi);  in bcmgenet_enable_tx_napi()
1905 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_enable_tx_napi()
1906 	napi_enable(&ring->napi);  in bcmgenet_enable_tx_napi()
1912 	struct bcmgenet_tx_ring *ring;  in bcmgenet_disable_tx_napi()  local
1915 		ring = &priv->tx_rings[i];  in bcmgenet_disable_tx_napi()
1916 		napi_disable(&ring->napi);  in bcmgenet_disable_tx_napi()
1919 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_disable_tx_napi()
1920 	napi_disable(&ring->napi);  in bcmgenet_disable_tx_napi()
1926 	struct bcmgenet_tx_ring *ring;  in bcmgenet_fini_tx_napi()  local
1929 		ring = &priv->tx_rings[i];  in bcmgenet_fini_tx_napi()
1930 		netif_napi_del(&ring->napi);  in bcmgenet_fini_tx_napi()
1933 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_fini_tx_napi()
1934 	netif_napi_del(&ring->napi);  in bcmgenet_fini_tx_napi()
2012 	struct bcmgenet_rx_ring *ring;  in bcmgenet_init_rx_napi()  local
2015 		ring = &priv->rx_rings[i];  in bcmgenet_init_rx_napi()
2016 		netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);  in bcmgenet_init_rx_napi()
2019 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_init_rx_napi()
2020 	netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);  in bcmgenet_init_rx_napi()
2026 	struct bcmgenet_rx_ring *ring;  in bcmgenet_enable_rx_napi()  local
2029 		ring = &priv->rx_rings[i];  in bcmgenet_enable_rx_napi()
2030 		napi_enable(&ring->napi);  in bcmgenet_enable_rx_napi()
2033 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_enable_rx_napi()
2034 	napi_enable(&ring->napi);  in bcmgenet_enable_rx_napi()
2040 	struct bcmgenet_rx_ring *ring;  in bcmgenet_disable_rx_napi()  local
2043 		ring = &priv->rx_rings[i];  in bcmgenet_disable_rx_napi()
2044 		napi_disable(&ring->napi);  in bcmgenet_disable_rx_napi()
2047 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_disable_rx_napi()
2048 	napi_disable(&ring->napi);  in bcmgenet_disable_rx_napi()
2054 	struct bcmgenet_rx_ring *ring;  in bcmgenet_fini_rx_napi()  local
2057 		ring = &priv->rx_rings[i];  in bcmgenet_fini_rx_napi()
2058 		netif_napi_del(&ring->napi);  in bcmgenet_fini_rx_napi()
2061 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_fini_rx_napi()
2062 	netif_napi_del(&ring->napi);  in bcmgenet_fini_rx_napi()