418 					   unsigned int ring,  in bcmgenet_tdma_ring_readl()  argument
422 			(DMA_RING_SIZE * ring) +  in bcmgenet_tdma_ring_readl()
427 					     unsigned int ring, u32 val,  in bcmgenet_tdma_ring_writel()  argument
431 			(DMA_RING_SIZE * ring) +  in bcmgenet_tdma_ring_writel()
436 					   unsigned int ring,  in bcmgenet_rdma_ring_readl()  argument
440 			(DMA_RING_SIZE * ring) +  in bcmgenet_rdma_ring_readl()
445 					     unsigned int ring, u32 val,  in bcmgenet_rdma_ring_writel()  argument
449 			(DMA_RING_SIZE * ring) +  in bcmgenet_rdma_ring_writel()
1092 					 struct bcmgenet_tx_ring *ring)  in bcmgenet_get_txcb()  argument
1096 	tx_cb_ptr = ring->cbs;  in bcmgenet_get_txcb()
1097 	tx_cb_ptr += ring->write_ptr - ring->cb_ptr;  in bcmgenet_get_txcb()
1100 	if (ring->write_ptr == ring->end_ptr)  in bcmgenet_get_txcb()
1101 		ring->write_ptr = ring->cb_ptr;  in bcmgenet_get_txcb()
1103 		ring->write_ptr++;  in bcmgenet_get_txcb()
1116 static inline void bcmgenet_rx_ring16_int_disable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring16_int_disable()  argument
1118 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,  in bcmgenet_rx_ring16_int_disable()
1122 static inline void bcmgenet_rx_ring16_int_enable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring16_int_enable()  argument
1124 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,  in bcmgenet_rx_ring16_int_enable()
1128 static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring_int_disable()  argument
1130 	bcmgenet_intrl2_1_writel(ring->priv,  in bcmgenet_rx_ring_int_disable()
1131 				 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),  in bcmgenet_rx_ring_int_disable()
1135 static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)  in bcmgenet_rx_ring_int_enable()  argument
1137 	bcmgenet_intrl2_1_writel(ring->priv,  in bcmgenet_rx_ring_int_enable()
1138 				 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),  in bcmgenet_rx_ring_int_enable()
1142 static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring16_int_disable()  argument
1144 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,  in bcmgenet_tx_ring16_int_disable()
1148 static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring16_int_enable()  argument
1150 	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,  in bcmgenet_tx_ring16_int_enable()
1154 static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring_int_enable()  argument
1156 	bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,  in bcmgenet_tx_ring_int_enable()
1160 static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_ring_int_disable()  argument
1162 	bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,  in bcmgenet_tx_ring_int_disable()
1168 					  struct bcmgenet_tx_ring *ring)  in __bcmgenet_tx_reclaim()  argument
1179 	c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);  in __bcmgenet_tx_reclaim()
1182 	if (likely(c_index >= ring->c_index))  in __bcmgenet_tx_reclaim()
1183 		txbds_ready = c_index - ring->c_index;  in __bcmgenet_tx_reclaim()
1185 		txbds_ready = (DMA_C_INDEX_MASK + 1) - ring->c_index + c_index;  in __bcmgenet_tx_reclaim()
1189 		  __func__, ring->index, ring->c_index, c_index, txbds_ready);  in __bcmgenet_tx_reclaim()
1193 		tx_cb_ptr = &priv->tx_cbs[ring->clean_ptr];  in __bcmgenet_tx_reclaim()
1214 		if (likely(ring->clean_ptr < ring->end_ptr))  in __bcmgenet_tx_reclaim()
1215 			ring->clean_ptr++;  in __bcmgenet_tx_reclaim()
1217 			ring->clean_ptr = ring->cb_ptr;  in __bcmgenet_tx_reclaim()
1220 	ring->free_bds += txbds_processed;  in __bcmgenet_tx_reclaim()
1221 	ring->c_index = (ring->c_index + txbds_processed) & DMA_C_INDEX_MASK;  in __bcmgenet_tx_reclaim()
1223 	if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {  in __bcmgenet_tx_reclaim()
1224 		txq = netdev_get_tx_queue(dev, ring->queue);  in __bcmgenet_tx_reclaim()
1233 				struct bcmgenet_tx_ring *ring)  in bcmgenet_tx_reclaim()  argument
1238 	spin_lock_irqsave(&ring->lock, flags);  in bcmgenet_tx_reclaim()
1239 	released = __bcmgenet_tx_reclaim(dev, ring);  in bcmgenet_tx_reclaim()
1240 	spin_unlock_irqrestore(&ring->lock, flags);  in bcmgenet_tx_reclaim()
1247 	struct bcmgenet_tx_ring *ring =  in bcmgenet_tx_poll()  local
1251 	work_done = bcmgenet_tx_reclaim(ring->priv->dev, ring);  in bcmgenet_tx_poll()
1255 		ring->int_enable(ring);  in bcmgenet_tx_poll()
1282 				struct bcmgenet_tx_ring *ring)  in bcmgenet_xmit_single()  argument
1292 	tx_cb_ptr = bcmgenet_get_txcb(priv, ring);  in bcmgenet_xmit_single()
1328 			      struct bcmgenet_tx_ring *ring)  in bcmgenet_xmit_frag()  argument
1336 	tx_cb_ptr = bcmgenet_get_txcb(priv, ring);  in bcmgenet_xmit_frag()
1428 	struct bcmgenet_tx_ring *ring = NULL;  in bcmgenet_xmit()  local
1450 	ring = &priv->tx_rings[index];  in bcmgenet_xmit()
1451 	txq = netdev_get_tx_queue(dev, ring->queue);  in bcmgenet_xmit()
1453 	spin_lock_irqsave(&ring->lock, flags);  in bcmgenet_xmit()
1454 	if (ring->free_bds <= nr_frags + 1) {  in bcmgenet_xmit()
1457 			   __func__, index, ring->queue);  in bcmgenet_xmit()
1481 	ret = bcmgenet_xmit_single(dev, skb, dma_desc_flags, ring);  in bcmgenet_xmit()
1492 					 ring);  in bcmgenet_xmit()
1502 	ring->free_bds -= nr_frags + 1;  in bcmgenet_xmit()
1503 	ring->prod_index += nr_frags + 1;  in bcmgenet_xmit()
1504 	ring->prod_index &= DMA_P_INDEX_MASK;  in bcmgenet_xmit()
1506 	if (ring->free_bds <= (MAX_SKB_FRAGS + 1))  in bcmgenet_xmit()
1511 		bcmgenet_tdma_ring_writel(priv, ring->index,  in bcmgenet_xmit()
1512 					  ring->prod_index, TDMA_PROD_INDEX);  in bcmgenet_xmit()
1514 	spin_unlock_irqrestore(&ring->lock, flags);  in bcmgenet_xmit()
1565 static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,  in bcmgenet_desc_rx()  argument
1568 	struct bcmgenet_priv *priv = ring->priv;  in bcmgenet_desc_rx()
1580 	p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);  in bcmgenet_desc_rx()
1584 	if (discards > ring->old_discards) {  in bcmgenet_desc_rx()
1585 		discards = discards - ring->old_discards;  in bcmgenet_desc_rx()
1588 		ring->old_discards += discards;  in bcmgenet_desc_rx()
1591 		if (ring->old_discards >= 0xC000) {  in bcmgenet_desc_rx()
1592 			ring->old_discards = 0;  in bcmgenet_desc_rx()
1593 			bcmgenet_rdma_ring_writel(priv, ring->index, 0,  in bcmgenet_desc_rx()
1600 	if (likely(p_index >= ring->c_index))  in bcmgenet_desc_rx()
1601 		rxpkttoprocess = p_index - ring->c_index;  in bcmgenet_desc_rx()
1603 		rxpkttoprocess = (DMA_C_INDEX_MASK + 1) - ring->c_index +  in bcmgenet_desc_rx()
1611 		cb = &priv->rx_cbs[ring->read_ptr];  in bcmgenet_desc_rx()
1637 			  __func__, p_index, ring->c_index,  in bcmgenet_desc_rx()
1638 			  ring->read_ptr, dma_length_status);  in bcmgenet_desc_rx()
1698 		napi_gro_receive(&ring->napi, skb);  in bcmgenet_desc_rx()
1703 		if (likely(ring->read_ptr < ring->end_ptr))  in bcmgenet_desc_rx()
1704 			ring->read_ptr++;  in bcmgenet_desc_rx()
1706 			ring->read_ptr = ring->cb_ptr;  in bcmgenet_desc_rx()
1708 		ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;  in bcmgenet_desc_rx()
1709 		bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);  in bcmgenet_desc_rx()
1718 	struct bcmgenet_rx_ring *ring = container_of(napi,  in bcmgenet_rx_poll()  local
1722 	work_done = bcmgenet_desc_rx(ring, budget);  in bcmgenet_rx_poll()
1726 		ring->int_enable(ring);  in bcmgenet_rx_poll()
1734 				     struct bcmgenet_rx_ring *ring)  in bcmgenet_alloc_rx_buffers()  argument
1743 	for (i = 0; i < ring->size; i++) {  in bcmgenet_alloc_rx_buffers()
1744 		cb = ring->cbs + i;  in bcmgenet_alloc_rx_buffers()
1933 	struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];  in bcmgenet_init_tx_ring()  local
1937 	spin_lock_init(&ring->lock);  in bcmgenet_init_tx_ring()
1938 	ring->priv = priv;  in bcmgenet_init_tx_ring()
1939 	ring->index = index;  in bcmgenet_init_tx_ring()
1941 		ring->queue = 0;  in bcmgenet_init_tx_ring()
1942 		ring->int_enable = bcmgenet_tx_ring16_int_enable;  in bcmgenet_init_tx_ring()
1943 		ring->int_disable = bcmgenet_tx_ring16_int_disable;  in bcmgenet_init_tx_ring()
1945 		ring->queue = index + 1;  in bcmgenet_init_tx_ring()
1946 		ring->int_enable = bcmgenet_tx_ring_int_enable;  in bcmgenet_init_tx_ring()
1947 		ring->int_disable = bcmgenet_tx_ring_int_disable;  in bcmgenet_init_tx_ring()
1949 	ring->cbs = priv->tx_cbs + start_ptr;  in bcmgenet_init_tx_ring()
1950 	ring->size = size;  in bcmgenet_init_tx_ring()
1951 	ring->clean_ptr = start_ptr;  in bcmgenet_init_tx_ring()
1952 	ring->c_index = 0;  in bcmgenet_init_tx_ring()
1953 	ring->free_bds = size;  in bcmgenet_init_tx_ring()
1954 	ring->write_ptr = start_ptr;  in bcmgenet_init_tx_ring()
1955 	ring->cb_ptr = start_ptr;  in bcmgenet_init_tx_ring()
1956 	ring->end_ptr = end_ptr - 1;  in bcmgenet_init_tx_ring()
1957 	ring->prod_index = 0;  in bcmgenet_init_tx_ring()
1989 	struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];  in bcmgenet_init_rx_ring()  local
1993 	ring->priv = priv;  in bcmgenet_init_rx_ring()
1994 	ring->index = index;  in bcmgenet_init_rx_ring()
1996 		ring->int_enable = bcmgenet_rx_ring16_int_enable;  in bcmgenet_init_rx_ring()
1997 		ring->int_disable = bcmgenet_rx_ring16_int_disable;  in bcmgenet_init_rx_ring()
1999 		ring->int_enable = bcmgenet_rx_ring_int_enable;  in bcmgenet_init_rx_ring()
2000 		ring->int_disable = bcmgenet_rx_ring_int_disable;  in bcmgenet_init_rx_ring()
2002 	ring->cbs = priv->rx_cbs + start_ptr;  in bcmgenet_init_rx_ring()
2003 	ring->size = size;  in bcmgenet_init_rx_ring()
2004 	ring->c_index = 0;  in bcmgenet_init_rx_ring()
2005 	ring->read_ptr = start_ptr;  in bcmgenet_init_rx_ring()
2006 	ring->cb_ptr = start_ptr;  in bcmgenet_init_rx_ring()
2007 	ring->end_ptr = end_ptr - 1;  in bcmgenet_init_rx_ring()
2009 	ret = bcmgenet_alloc_rx_buffers(priv, ring);  in bcmgenet_init_rx_ring()
2040 	struct bcmgenet_tx_ring *ring;  in bcmgenet_init_tx_napi()  local
2043 		ring = &priv->tx_rings[i];  in bcmgenet_init_tx_napi()
2044 		netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64);  in bcmgenet_init_tx_napi()
2047 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_init_tx_napi()
2048 	netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64);  in bcmgenet_init_tx_napi()
2054 	struct bcmgenet_tx_ring *ring;  in bcmgenet_enable_tx_napi()  local
2057 		ring = &priv->tx_rings[i];  in bcmgenet_enable_tx_napi()
2058 		napi_enable(&ring->napi);  in bcmgenet_enable_tx_napi()
2061 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_enable_tx_napi()
2062 	napi_enable(&ring->napi);  in bcmgenet_enable_tx_napi()
2068 	struct bcmgenet_tx_ring *ring;  in bcmgenet_disable_tx_napi()  local
2071 		ring = &priv->tx_rings[i];  in bcmgenet_disable_tx_napi()
2072 		napi_disable(&ring->napi);  in bcmgenet_disable_tx_napi()
2075 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_disable_tx_napi()
2076 	napi_disable(&ring->napi);  in bcmgenet_disable_tx_napi()
2082 	struct bcmgenet_tx_ring *ring;  in bcmgenet_fini_tx_napi()  local
2085 		ring = &priv->tx_rings[i];  in bcmgenet_fini_tx_napi()
2086 		netif_napi_del(&ring->napi);  in bcmgenet_fini_tx_napi()
2089 	ring = &priv->tx_rings[DESC_INDEX];  in bcmgenet_fini_tx_napi()
2090 	netif_napi_del(&ring->napi);  in bcmgenet_fini_tx_napi()
2168 	struct bcmgenet_rx_ring *ring;  in bcmgenet_init_rx_napi()  local
2171 		ring = &priv->rx_rings[i];  in bcmgenet_init_rx_napi()
2172 		netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);  in bcmgenet_init_rx_napi()
2175 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_init_rx_napi()
2176 	netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);  in bcmgenet_init_rx_napi()
2182 	struct bcmgenet_rx_ring *ring;  in bcmgenet_enable_rx_napi()  local
2185 		ring = &priv->rx_rings[i];  in bcmgenet_enable_rx_napi()
2186 		napi_enable(&ring->napi);  in bcmgenet_enable_rx_napi()
2189 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_enable_rx_napi()
2190 	napi_enable(&ring->napi);  in bcmgenet_enable_rx_napi()
2196 	struct bcmgenet_rx_ring *ring;  in bcmgenet_disable_rx_napi()  local
2199 		ring = &priv->rx_rings[i];  in bcmgenet_disable_rx_napi()
2200 		napi_disable(&ring->napi);  in bcmgenet_disable_rx_napi()
2203 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_disable_rx_napi()
2204 	napi_disable(&ring->napi);  in bcmgenet_disable_rx_napi()
2210 	struct bcmgenet_rx_ring *ring;  in bcmgenet_fini_rx_napi()  local
2213 		ring = &priv->rx_rings[i];  in bcmgenet_fini_rx_napi()
2214 		netif_napi_del(&ring->napi);  in bcmgenet_fini_rx_napi()
2217 	ring = &priv->rx_rings[DESC_INDEX];  in bcmgenet_fini_rx_napi()
2218 	netif_napi_del(&ring->napi);  in bcmgenet_fini_rx_napi()
2964 static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring)  in bcmgenet_dump_tx_queue()  argument
2966 	struct bcmgenet_priv *priv = ring->priv;  in bcmgenet_dump_tx_queue()
2976 	txq = netdev_get_tx_queue(priv->dev, ring->queue);  in bcmgenet_dump_tx_queue()
2978 	spin_lock_irqsave(&ring->lock, flags);  in bcmgenet_dump_tx_queue()
2979 	if (ring->index == DESC_INDEX) {  in bcmgenet_dump_tx_queue()
2984 		intmsk = 1 << ring->index;  in bcmgenet_dump_tx_queue()
2986 	c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);  in bcmgenet_dump_tx_queue()
2987 	p_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_PROD_INDEX);  in bcmgenet_dump_tx_queue()
2989 	free_bds = ring->free_bds;  in bcmgenet_dump_tx_queue()
2990 	spin_unlock_irqrestore(&ring->lock, flags);  in bcmgenet_dump_tx_queue()
2999 		  ring->index, ring->queue,  in bcmgenet_dump_tx_queue()
3002 		  free_bds, ring->size,  in bcmgenet_dump_tx_queue()
3003 		  ring->prod_index, p_index & DMA_P_INDEX_MASK,  in bcmgenet_dump_tx_queue()
3004 		  ring->c_index, c_index & DMA_C_INDEX_MASK,  in bcmgenet_dump_tx_queue()
3005 		  ring->clean_ptr, ring->write_ptr,  in bcmgenet_dump_tx_queue()
3006 		  ring->cb_ptr, ring->end_ptr);  in bcmgenet_dump_tx_queue()