204 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_txq_notify()  argument
212 	val |= txo->q.id & DB_TXULP_RING_ID_MASK;  in be_txq_notify()
216 	iowrite32(val, adapter->db + txo->db_offset);  in be_txq_notify()
602 	struct be_tx_obj *txo;  in be_get_stats64()  local
622 	for_all_tx_queues(adapter, txo, i) {  in be_get_stats64()
623 		const struct be_tx_stats *tx_stats = tx_stats(txo);  in be_get_stats64()
627 			pkts = tx_stats(txo)->tx_pkts;  in be_get_stats64()
628 			bytes = tx_stats(txo)->tx_bytes;  in be_get_stats64()
681 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)  in be_tx_stats_update()  argument
683 	struct be_tx_stats *stats = tx_stats(txo);  in be_tx_stats_update()
750 static inline bool be_is_txq_full(struct be_tx_obj *txo)  in be_is_txq_full()  argument
752 	return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;  in be_is_txq_full()
755 static inline bool be_can_txq_wake(struct be_tx_obj *txo)  in be_can_txq_wake()  argument
757 	return atomic_read(&txo->q.used) < txo->q.len / 2;  in be_can_txq_wake()
760 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)  in be_is_tx_compl_pending()  argument
762 	return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;  in be_is_tx_compl_pending()
852 static u16 be_tx_get_wrb_hdr(struct be_tx_obj *txo)  in be_tx_get_wrb_hdr()  argument
854 	u16 head = txo->q.head;  in be_tx_get_wrb_hdr()
856 	queue_head_inc(&txo->q);  in be_tx_get_wrb_hdr()
862 				struct be_tx_obj *txo,  in be_tx_setup_wrb_hdr()  argument
867 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_hdr()
873 	BUG_ON(txo->sent_skb_list[head]);  in be_tx_setup_wrb_hdr()
874 	txo->sent_skb_list[head] = skb;  in be_tx_setup_wrb_hdr()
875 	txo->last_req_hdr = head;  in be_tx_setup_wrb_hdr()
877 	txo->last_req_wrb_cnt = num_frags;  in be_tx_setup_wrb_hdr()
878 	txo->pend_wrb_cnt += num_frags;  in be_tx_setup_wrb_hdr()
882 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,  in be_tx_setup_wrb_frag()  argument
886 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_frag()
898 			    struct be_tx_obj *txo, u16 head, bool map_single,  in be_xmit_restore()  argument
903 	struct be_queue_info *txq = &txo->q;  in be_xmit_restore()
925 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_xmit_enqueue()  argument
931 	struct be_queue_info *txq = &txo->q;  in be_xmit_enqueue()
937 	head = be_tx_get_wrb_hdr(txo);  in be_xmit_enqueue()
946 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
957 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
961 	be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);  in be_xmit_enqueue()
963 	be_tx_stats_update(txo, skb);  in be_xmit_enqueue()
968 	be_xmit_restore(adapter, txo, head, map_single, copied);  in be_xmit_enqueue()
1145 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)  in be_xmit_flush()  argument
1147 	struct be_queue_info *txq = &txo->q;  in be_xmit_flush()
1148 	struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);  in be_xmit_flush()
1155 	if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {  in be_xmit_flush()
1159 		txo->pend_wrb_cnt++;  in be_xmit_flush()
1162 		hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<  in be_xmit_flush()
1165 	be_txq_notify(adapter, txo, txo->pend_wrb_cnt);  in be_xmit_flush()
1166 	txo->pend_wrb_cnt = 0;  in be_xmit_flush()
1297 	struct be_tx_obj *txo = &adapter->tx_obj[q_idx];  in be_xmit()  local
1308 	wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);  in be_xmit()
1319 		wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);  in be_xmit()
1326 	if (be_is_txq_full(txo)) {  in be_xmit()
1328 		tx_stats(txo)->tx_stops++;  in be_xmit()
1332 		be_xmit_flush(adapter, txo);  in be_xmit()
1336 	tx_stats(txo)->tx_drv_drops++;  in be_xmit()
1338 	if (flush && txo->pend_wrb_cnt)  in be_xmit()
1339 		be_xmit_flush(adapter, txo);  in be_xmit()
1850 	struct be_tx_obj *txo;  in be_get_new_eqd()  local
1871 	for_all_tx_queues_on_eq(adapter, eqo, txo, i) {  in be_get_new_eqd()
1873 			start = u64_stats_fetch_begin_irq(&txo->stats.sync);  in be_get_new_eqd()
1874 			tx_pkts += txo->stats.tx_reqs;  in be_get_new_eqd()
1875 		} while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));  in be_get_new_eqd()
2377 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo)  in be_tx_compl_get()  argument
2379 	struct be_queue_info *tx_cq = &txo->cq;  in be_tx_compl_get()
2380 	struct be_tx_compl_info *txcp = &txo->txcp;  in be_tx_compl_get()
2399 			       struct be_tx_obj *txo, u16 last_index)  in be_tx_compl_process()  argument
2401 	struct sk_buff **sent_skbs = txo->sent_skb_list;  in be_tx_compl_process()
2402 	struct be_queue_info *txq = &txo->q;  in be_tx_compl_process()
2522 	struct be_tx_obj *txo;  in be_tx_compl_clean()  local
2529 		for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2532 			txq = &txo->q;  in be_tx_compl_clean()
2533 			while ((txcp = be_tx_compl_get(txo))) {  in be_tx_compl_clean()
2535 					be_tx_compl_process(adapter, txo,  in be_tx_compl_clean()
2540 				be_cq_notify(adapter, txo->cq.id, false, cmpl);  in be_tx_compl_clean()
2544 			if (!be_is_tx_compl_pending(txo))  in be_tx_compl_clean()
2556 	for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2557 		txq = &txo->q;  in be_tx_compl_clean()
2569 			num_wrbs = be_tx_compl_process(adapter, txo, end_idx);  in be_tx_compl_clean()
2572 			txo->pend_wrb_cnt = 0;  in be_tx_compl_clean()
2690 	struct be_tx_obj *txo;  in be_tx_queues_destroy()  local
2693 	for_all_tx_queues(adapter, txo, i) {  in be_tx_queues_destroy()
2694 		q = &txo->q;  in be_tx_queues_destroy()
2699 		q = &txo->cq;  in be_tx_queues_destroy()
2709 	struct be_tx_obj *txo;  in be_tx_qs_create()  local
2715 	for_all_tx_queues(adapter, txo, i) {  in be_tx_qs_create()
2716 		cq = &txo->cq;  in be_tx_qs_create()
2722 		u64_stats_init(&txo->stats.sync);  in be_tx_qs_create()
2723 		u64_stats_init(&txo->stats.sync_compl);  in be_tx_qs_create()
2733 		status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,  in be_tx_qs_create()
2738 		status = be_cmd_txq_create(adapter, txo);  in be_tx_qs_create()
2913 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)  in be_update_tx_err()  argument
2917 		tx_stats(txo)->tx_hdr_parse_err++;  in be_update_tx_err()
2920 		tx_stats(txo)->tx_dma_err++;  in be_update_tx_err()
2923 		tx_stats(txo)->tx_spoof_check_err++;  in be_update_tx_err()
2928 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)  in lancer_update_tx_err()  argument
2932 		tx_stats(txo)->tx_tso_err++;  in lancer_update_tx_err()
2936 		tx_stats(txo)->tx_spoof_check_err++;  in lancer_update_tx_err()
2939 		tx_stats(txo)->tx_qinq_err++;  in lancer_update_tx_err()
2942 		tx_stats(txo)->tx_internal_parity_err++;  in lancer_update_tx_err()
2945 		tx_stats(txo)->tx_dma_err++;  in lancer_update_tx_err()
2950 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_process_tx()  argument
2956 	while ((txcp = be_tx_compl_get(txo))) {  in be_process_tx()
2957 		num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);  in be_process_tx()
2962 				lancer_update_tx_err(txo, txcp->status);  in be_process_tx()
2964 				be_update_tx_err(txo, txcp->status);  in be_process_tx()
2969 		be_cq_notify(adapter, txo->cq.id, true, work_done);  in be_process_tx()
2970 		atomic_sub(num_wrbs, &txo->q.used);  in be_process_tx()
2975 		    be_can_txq_wake(txo)) {  in be_process_tx()
2979 		u64_stats_update_begin(&tx_stats(txo)->sync_compl);  in be_process_tx()
2980 		tx_stats(txo)->tx_compl += work_done;  in be_process_tx()
2981 		u64_stats_update_end(&tx_stats(txo)->sync_compl);  in be_process_tx()
3091 	struct be_tx_obj *txo;  in be_poll()  local
3096 	for_all_tx_queues_on_eq(adapter, eqo, txo, i)  in be_poll()
3097 		be_process_tx(adapter, txo, i);  in be_poll()
3568 	struct be_tx_obj *txo;  in be_open()  local
3587 	for_all_tx_queues(adapter, txo, i)  in be_open()
3588 		be_cq_notify(adapter, txo->cq.id, true, 0);  in be_open()