192 #define IS_TSO_HEADER(txq, addr) \  argument
193 	((addr >= txq->tso_hdrs_dma) && \
194 	 (addr < txq->tso_hdrs_dma + txq->tx_ring_size * TSO_HEADER_SIZE))
423 	struct tx_queue txq[8];  member
461 static struct mv643xx_eth_private *txq_to_mp(struct tx_queue *txq)  in txq_to_mp()  argument
463 	return container_of(txq, struct mv643xx_eth_private, txq[txq->index]);  in txq_to_mp()
482 static void txq_reset_hw_ptr(struct tx_queue *txq)  in txq_reset_hw_ptr()  argument
484 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_reset_hw_ptr()
487 	addr = (u32)txq->tx_desc_dma;  in txq_reset_hw_ptr()
488 	addr += txq->tx_curr_desc * sizeof(struct tx_desc);  in txq_reset_hw_ptr()
489 	wrlp(mp, TXQ_CURRENT_DESC_PTR(txq->index), addr);  in txq_reset_hw_ptr()
492 static void txq_enable(struct tx_queue *txq)  in txq_enable()  argument
494 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_enable()
495 	wrlp(mp, TXQ_COMMAND, 1 << txq->index);  in txq_enable()
498 static void txq_disable(struct tx_queue *txq)  in txq_disable()  argument
500 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_disable()
501 	u8 mask = 1 << txq->index;  in txq_disable()
508 static void txq_maybe_wake(struct tx_queue *txq)  in txq_maybe_wake()  argument
510 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_maybe_wake()
511 	struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index);  in txq_maybe_wake()
515 		if (txq->tx_desc_count <= txq->tx_wake_threshold)  in txq_maybe_wake()
746 txq_put_data_tso(struct net_device *dev, struct tx_queue *txq,  in txq_put_data_tso()  argument
754 	tx_index = txq->tx_curr_desc++;  in txq_put_data_tso()
755 	if (txq->tx_curr_desc == txq->tx_ring_size)  in txq_put_data_tso()
756 		txq->tx_curr_desc = 0;  in txq_put_data_tso()
757 	desc = &txq->tx_desc_area[tx_index];  in txq_put_data_tso()
758 	txq->tx_desc_mapping[tx_index] = DESC_DMA_MAP_SINGLE;  in txq_put_data_tso()
782 txq_put_hdr_tso(struct sk_buff *skb, struct tx_queue *txq, int length)  in txq_put_hdr_tso()  argument
784 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_put_hdr_tso()
792 	tx_index = txq->tx_curr_desc;  in txq_put_hdr_tso()
793 	desc = &txq->tx_desc_area[tx_index];  in txq_put_hdr_tso()
804 	desc->buf_ptr = txq->tso_hdrs_dma +  in txq_put_hdr_tso()
805 			txq->tx_curr_desc * TSO_HEADER_SIZE;  in txq_put_hdr_tso()
809 	txq->tx_curr_desc++;  in txq_put_hdr_tso()
810 	if (txq->tx_curr_desc == txq->tx_ring_size)  in txq_put_hdr_tso()
811 		txq->tx_curr_desc = 0;  in txq_put_hdr_tso()
814 static int txq_submit_tso(struct tx_queue *txq, struct sk_buff *skb,  in txq_submit_tso()  argument
817 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_submit_tso()
824 	if ((txq->tx_desc_count + tso_count_descs(skb)) >= txq->tx_ring_size) {  in txq_submit_tso()
841 		hdr = txq->tso_hdrs + txq->tx_curr_desc * TSO_HEADER_SIZE;  in txq_submit_tso()
843 		txq_put_hdr_tso(skb, txq, data_left);  in txq_submit_tso()
850 			ret = txq_put_data_tso(dev, txq, skb, tso.data, size,  in txq_submit_tso()
860 	__skb_queue_tail(&txq->tx_skb, skb);  in txq_submit_tso()
864 	mp->work_tx_end &= ~(1 << txq->index);  in txq_submit_tso()
868 	txq_enable(txq);  in txq_submit_tso()
869 	txq->tx_desc_count += desc_count;  in txq_submit_tso()
878 static void txq_submit_frag_skb(struct tx_queue *txq, struct sk_buff *skb)  in txq_submit_frag_skb()  argument
880 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_submit_frag_skb()
890 		tx_index = txq->tx_curr_desc++;  in txq_submit_frag_skb()
891 		if (txq->tx_curr_desc == txq->tx_ring_size)  in txq_submit_frag_skb()
892 			txq->tx_curr_desc = 0;  in txq_submit_frag_skb()
893 		desc = &txq->tx_desc_area[tx_index];  in txq_submit_frag_skb()
894 		txq->tx_desc_mapping[tx_index] = DESC_DMA_MAP_PAGE;  in txq_submit_frag_skb()
916 static int txq_submit_skb(struct tx_queue *txq, struct sk_buff *skb,  in txq_submit_skb()  argument
919 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_submit_skb()
930 	if (txq->tx_ring_size - txq->tx_desc_count < MAX_SKB_FRAGS + 1) {  in txq_submit_skb()
941 	tx_index = txq->tx_curr_desc++;  in txq_submit_skb()
942 	if (txq->tx_curr_desc == txq->tx_ring_size)  in txq_submit_skb()
943 		txq->tx_curr_desc = 0;  in txq_submit_skb()
944 	desc = &txq->tx_desc_area[tx_index];  in txq_submit_skb()
945 	txq->tx_desc_mapping[tx_index] = DESC_DMA_MAP_SINGLE;  in txq_submit_skb()
948 		txq_submit_frag_skb(txq, skb);  in txq_submit_skb()
960 	__skb_queue_tail(&txq->tx_skb, skb);  in txq_submit_skb()
969 	mp->work_tx_end &= ~(1 << txq->index);  in txq_submit_skb()
973 	txq_enable(txq);  in txq_submit_skb()
975 	txq->tx_desc_count += nr_frags + 1;  in txq_submit_skb()
984 	struct tx_queue *txq;  in mv643xx_eth_xmit()  local
988 	txq = mp->txq + queue;  in mv643xx_eth_xmit()
1000 		ret = txq_submit_tso(txq, skb, dev);  in mv643xx_eth_xmit()
1002 		ret = txq_submit_skb(txq, skb, dev);  in mv643xx_eth_xmit()
1004 		txq->tx_bytes += length;  in mv643xx_eth_xmit()
1005 		txq->tx_packets++;  in mv643xx_eth_xmit()
1007 		if (txq->tx_desc_count >= txq->tx_stop_threshold)  in mv643xx_eth_xmit()
1010 		txq->tx_dropped++;  in mv643xx_eth_xmit()
1019 static void txq_kick(struct tx_queue *txq)  in txq_kick()  argument
1021 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_kick()
1022 	struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index);  in txq_kick()
1028 	if (rdlp(mp, TXQ_COMMAND) & (1 << txq->index))  in txq_kick()
1031 	hw_desc_ptr = rdlp(mp, TXQ_CURRENT_DESC_PTR(txq->index));  in txq_kick()
1032 	expected_ptr = (u32)txq->tx_desc_dma +  in txq_kick()
1033 				txq->tx_curr_desc * sizeof(struct tx_desc);  in txq_kick()
1036 		txq_enable(txq);  in txq_kick()
1041 	mp->work_tx_end &= ~(1 << txq->index);  in txq_kick()
1044 static int txq_reclaim(struct tx_queue *txq, int budget, int force)  in txq_reclaim()  argument
1046 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_reclaim()
1047 	struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index);  in txq_reclaim()
1053 	while (reclaimed < budget && txq->tx_desc_count > 0) {  in txq_reclaim()
1059 		tx_index = txq->tx_used_desc;  in txq_reclaim()
1060 		desc = &txq->tx_desc_area[tx_index];  in txq_reclaim()
1061 		desc_dma_map = txq->tx_desc_mapping[tx_index];  in txq_reclaim()
1071 		txq->tx_used_desc = tx_index + 1;  in txq_reclaim()
1072 		if (txq->tx_used_desc == txq->tx_ring_size)  in txq_reclaim()
1073 			txq->tx_used_desc = 0;  in txq_reclaim()
1076 		txq->tx_desc_count--;  in txq_reclaim()
1078 		if (!IS_TSO_HEADER(txq, desc->buf_ptr)) {  in txq_reclaim()
1093 			struct sk_buff *skb = __skb_dequeue(&txq->tx_skb);  in txq_reclaim()
1109 		mp->work_tx &= ~(1 << txq->index);  in txq_reclaim()
1152 static void txq_set_rate(struct tx_queue *txq, int rate, int burst)  in txq_set_rate()  argument
1154 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_set_rate()
1166 	wrlp(mp, TXQ_BW_TOKENS(txq->index), token_rate << 14);  in txq_set_rate()
1167 	wrlp(mp, TXQ_BW_CONF(txq->index), (bucket_size << 10) | token_rate);  in txq_set_rate()
1170 static void txq_set_fixed_prio_mode(struct tx_queue *txq)  in txq_set_fixed_prio_mode()  argument
1172 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_set_fixed_prio_mode()
1191 		val |= 1 << txq->index;  in txq_set_fixed_prio_mode()
1249 		struct tx_queue *txq = mp->txq + i;  in mv643xx_eth_get_stats()  local
1251 		tx_packets += txq->tx_packets;  in mv643xx_eth_get_stats()
1252 		tx_bytes += txq->tx_bytes;  in mv643xx_eth_get_stats()
1253 		tx_dropped += txq->tx_dropped;  in mv643xx_eth_get_stats()
2016 	struct tx_queue *txq = mp->txq + index;  in txq_init()  local
2022 	txq->index = index;  in txq_init()
2024 	txq->tx_ring_size = mp->tx_ring_size;  in txq_init()
2030 	txq->tx_stop_threshold = txq->tx_ring_size - MV643XX_MAX_SKB_DESCS;  in txq_init()
2031 	txq->tx_wake_threshold = txq->tx_stop_threshold / 2;  in txq_init()
2033 	txq->tx_desc_count = 0;  in txq_init()
2034 	txq->tx_curr_desc = 0;  in txq_init()
2035 	txq->tx_used_desc = 0;  in txq_init()
2037 	size = txq->tx_ring_size * sizeof(struct tx_desc);  in txq_init()
2040 		txq->tx_desc_area = ioremap(mp->tx_desc_sram_addr,  in txq_init()
2042 		txq->tx_desc_dma = mp->tx_desc_sram_addr;  in txq_init()
2044 		txq->tx_desc_area = dma_alloc_coherent(mp->dev->dev.parent,  in txq_init()
2045 						       size, &txq->tx_desc_dma,  in txq_init()
2049 	if (txq->tx_desc_area == NULL) {  in txq_init()
2054 	memset(txq->tx_desc_area, 0, size);  in txq_init()
2056 	txq->tx_desc_area_size = size;  in txq_init()
2058 	tx_desc = txq->tx_desc_area;  in txq_init()
2059 	for (i = 0; i < txq->tx_ring_size; i++) {  in txq_init()
2064 		if (nexti == txq->tx_ring_size)  in txq_init()
2068 		txd->next_desc_ptr = txq->tx_desc_dma +  in txq_init()
2072 	txq->tx_desc_mapping = kcalloc(txq->tx_ring_size, sizeof(char),  in txq_init()
2074 	if (!txq->tx_desc_mapping) {  in txq_init()
2080 	txq->tso_hdrs = dma_alloc_coherent(mp->dev->dev.parent,  in txq_init()
2081 					   txq->tx_ring_size * TSO_HEADER_SIZE,  in txq_init()
2082 					   &txq->tso_hdrs_dma, GFP_KERNEL);  in txq_init()
2083 	if (txq->tso_hdrs == NULL) {  in txq_init()
2087 	skb_queue_head_init(&txq->tx_skb);  in txq_init()
2092 	kfree(txq->tx_desc_mapping);  in txq_init()
2095 		iounmap(txq->tx_desc_area);  in txq_init()
2097 		dma_free_coherent(mp->dev->dev.parent, txq->tx_desc_area_size,  in txq_init()
2098 				  txq->tx_desc_area, txq->tx_desc_dma);  in txq_init()
2102 static void txq_deinit(struct tx_queue *txq)  in txq_deinit()  argument
2104 	struct mv643xx_eth_private *mp = txq_to_mp(txq);  in txq_deinit()
2106 	txq_disable(txq);  in txq_deinit()
2107 	txq_reclaim(txq, txq->tx_ring_size, 1);  in txq_deinit()
2109 	BUG_ON(txq->tx_used_desc != txq->tx_curr_desc);  in txq_deinit()
2111 	if (txq->index == 0 &&  in txq_deinit()
2112 	    txq->tx_desc_area_size <= mp->tx_desc_sram_size)  in txq_deinit()
2113 		iounmap(txq->tx_desc_area);  in txq_deinit()
2115 		dma_free_coherent(mp->dev->dev.parent, txq->tx_desc_area_size,  in txq_deinit()
2116 				  txq->tx_desc_area, txq->tx_desc_dma);  in txq_deinit()
2117 	kfree(txq->tx_desc_mapping);  in txq_deinit()
2119 	if (txq->tso_hdrs)  in txq_deinit()
2121 				  txq->tx_ring_size * TSO_HEADER_SIZE,  in txq_deinit()
2122 				  txq->tso_hdrs, txq->tso_hdrs_dma);  in txq_deinit()
2192 				struct tx_queue *txq = mp->txq + i;  in handle_link_event()  local
2194 				txq_reclaim(txq, txq->tx_ring_size, 1);  in handle_link_event()
2195 				txq_reset_hw_ptr(txq);  in handle_link_event()
2268 			txq_kick(mp->txq + queue);  in mv643xx_eth_poll()
2270 			work_done += txq_reclaim(mp->txq + queue, work_tbd, 0);  in mv643xx_eth_poll()
2271 			txq_maybe_wake(mp->txq + queue);  in mv643xx_eth_poll()
2333 		struct tx_queue *txq = mp->txq + i;  in port_start()  local
2335 		txq_reset_hw_ptr(txq);  in port_start()
2336 		txq_set_rate(txq, 1000000000, 16777216);  in port_start()
2337 		txq_set_fixed_prio_mode(txq);  in port_start()
2444 				txq_deinit(mp->txq + i);  in mv643xx_eth_open()
2476 		txq_disable(mp->txq + i);  in port_reset()
2520 		txq_deinit(mp->txq + i);  in mv643xx_eth_stop()