726 static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)  in init_qset_cntxt()  argument
728 	qs->rspq.cntxt_id = id;  in init_qset_cntxt()
729 	qs->fl[0].cntxt_id = 2 * id;  in init_qset_cntxt()
730 	qs->fl[1].cntxt_id = 2 * id + 1;  in init_qset_cntxt()
731 	qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;  in init_qset_cntxt()
732 	qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;  in init_qset_cntxt()
733 	qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;  in init_qset_cntxt()
734 	qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;  in init_qset_cntxt()
735 	qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;  in init_qset_cntxt()
1207 				    struct sge_qset *qs, struct sge_txq *q)  in t3_stop_tx_queue()  argument
1210 	set_bit(TXQ_ETH, &qs->txq_stopped);  in t3_stop_tx_queue()
1228 	struct sge_qset *qs;  in t3_eth_xmit()  local
1241 	qs = &pi->qs[qidx];  in t3_eth_xmit()
1242 	q = &qs->txq[TXQ_ETH];  in t3_eth_xmit()
1251 		t3_stop_tx_queue(txq, qs, q);  in t3_eth_xmit()
1260 		t3_stop_tx_queue(txq, qs, q);  in t3_eth_xmit()
1263 		    test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {  in t3_eth_xmit()
1282 		qs->port_stats[SGE_PSTAT_TX_CSUM]++;  in t3_eth_xmit()
1284 		qs->port_stats[SGE_PSTAT_TSO]++;  in t3_eth_xmit()
1286 		qs->port_stats[SGE_PSTAT_VLANINS]++;  in t3_eth_xmit()
1379 		struct sge_qset *qs = txq_to_qset(q, qid);  in check_desc_avail()  local
1381 		set_bit(qid, &qs->txq_stopped);  in check_desc_avail()
1385 		    test_and_clear_bit(qid, &qs->txq_stopped))  in check_desc_avail()
1475 	struct sge_qset *qs = (struct sge_qset *)data;  in restart_ctrlq()  local
1476 	struct sge_txq *q = &qs->txq[TXQ_CTRL];  in restart_ctrlq()
1494 		set_bit(TXQ_CTRL, &qs->txq_stopped);  in restart_ctrlq()
1498 		    test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))  in restart_ctrlq()
1505 	t3_write_reg(qs->adap, A_SG_KDOORBELL,  in restart_ctrlq()
1516 	ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);  in t3_mgmt_tx()
1686 	struct sge_qset *qs = (struct sge_qset *)data;  in restart_offloadq()  local
1687 	struct sge_txq *q = &qs->txq[TXQ_OFLD];  in restart_offloadq()
1688 	const struct port_info *pi = netdev_priv(qs->netdev);  in restart_offloadq()
1699 			set_bit(TXQ_OFLD, &qs->txq_stopped);  in restart_offloadq()
1703 			    test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))  in restart_offloadq()
1770 	struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];  in t3_offload_tx()  local
1773 		return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);  in t3_offload_tx()
1775 	return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);  in t3_offload_tx()
1794 		struct sge_qset *qs = rspq_to_qset(q);  in offload_enqueue()  local
1796 		napi_schedule(&qs->napi);  in offload_enqueue()
1832 	struct sge_qset *qs = container_of(napi, struct sge_qset, napi);  in ofld_poll()  local
1833 	struct sge_rspq *q = &qs->rspq;  in ofld_poll()
1834 	struct adapter *adapter = qs->adap;  in ofld_poll()
1919 static void restart_tx(struct sge_qset *qs)  in restart_tx()  argument
1921 	if (test_bit(TXQ_ETH, &qs->txq_stopped) &&  in restart_tx()
1922 	    should_restart_tx(&qs->txq[TXQ_ETH]) &&  in restart_tx()
1923 	    test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {  in restart_tx()
1924 		qs->txq[TXQ_ETH].restarts++;  in restart_tx()
1925 		if (netif_running(qs->netdev))  in restart_tx()
1926 			netif_tx_wake_queue(qs->tx_q);  in restart_tx()
1929 	if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&  in restart_tx()
1930 	    should_restart_tx(&qs->txq[TXQ_OFLD]) &&  in restart_tx()
1931 	    test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {  in restart_tx()
1932 		qs->txq[TXQ_OFLD].restarts++;  in restart_tx()
1933 		tasklet_schedule(&qs->txq[TXQ_OFLD].qresume_tsk);  in restart_tx()
1935 	if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&  in restart_tx()
1936 	    should_restart_tx(&qs->txq[TXQ_CTRL]) &&  in restart_tx()
1937 	    test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {  in restart_tx()
1938 		qs->txq[TXQ_CTRL].restarts++;  in restart_tx()
1939 		tasklet_schedule(&qs->txq[TXQ_CTRL].qresume_tsk);  in restart_tx()
2017 	struct sge_qset *qs = rspq_to_qset(rq);  in rx_eth()  local
2025 		qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;  in rx_eth()
2029 	skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);  in rx_eth()
2032 		qs->port_stats[SGE_PSTAT_VLANEX]++;  in rx_eth()
2037 			napi_gro_receive(&qs->napi, skb);  in rx_eth()
2063 static void lro_add_page(struct adapter *adap, struct sge_qset *qs,  in lro_add_page()  argument
2067 	struct port_info *pi = netdev_priv(qs->netdev);  in lro_add_page()
2074 	if (!qs->nomem) {  in lro_add_page()
2075 		skb = napi_get_frags(&qs->napi);  in lro_add_page()
2076 		qs->nomem = !skb;  in lro_add_page()
2096 			qs->nomem = 0;  in lro_add_page()
2105 		cpl = qs->lro_va = sd->pg_chunk.va + 2;  in lro_add_page()
2107 		if ((qs->netdev->features & NETIF_F_RXCSUM) &&  in lro_add_page()
2110 			qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;  in lro_add_page()
2114 		cpl = qs->lro_va;  in lro_add_page()
2131 	skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);  in lro_add_page()
2134 		qs->port_stats[SGE_PSTAT_VLANEX]++;  in lro_add_page()
2137 	napi_gro_frags(&qs->napi);  in lro_add_page()
2149 static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)  in handle_rsp_cntrl_info()  argument
2155 		clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);  in handle_rsp_cntrl_info()
2160 		qs->txq[TXQ_ETH].processed += credits;  in handle_rsp_cntrl_info()
2164 		qs->txq[TXQ_CTRL].processed += credits;  in handle_rsp_cntrl_info()
2168 		clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);  in handle_rsp_cntrl_info()
2172 		qs->txq[TXQ_OFLD].processed += credits;  in handle_rsp_cntrl_info()
2185 static void check_ring_db(struct adapter *adap, struct sge_qset *qs,  in check_ring_db()  argument
2189 		struct sge_txq *txq = &qs->txq[TXQ_ETH];  in check_ring_db()
2200 		struct sge_txq *txq = &qs->txq[TXQ_OFLD];  in check_ring_db()
2255 static int process_responses(struct adapter *adap, struct sge_qset *qs,  in process_responses()  argument
2258 	struct sge_rspq *q = &qs->rspq;  in process_responses()
2269 		int lro = !!(qs->netdev->features & NETIF_F_GRO);  in process_responses()
2306 			fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];  in process_responses()
2316 					lro_add_page(adap, qs, fl,  in process_responses()
2344 			handle_rsp_cntrl_info(qs, flags);  in process_responses()
2386 		check_ring_db(adap, qs, sleeping);  in process_responses()
2389 	if (unlikely(qs->txq_stopped != 0))  in process_responses()
2390 		restart_tx(qs);  in process_responses()
2412 	struct sge_qset *qs = container_of(napi, struct sge_qset, napi);  in napi_rx_handler()  local
2413 	struct adapter *adap = qs->adap;  in napi_rx_handler()
2414 	int work_done = process_responses(adap, qs, budget);  in napi_rx_handler()
2433 		t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |  in napi_rx_handler()
2434 			     V_NEWTIMER(qs->rspq.next_holdoff) |  in napi_rx_handler()
2435 			     V_NEWINDEX(qs->rspq.cidx));  in napi_rx_handler()
2462 static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,  in process_pure_responses()  argument
2465 	struct sge_rspq *q = &qs->rspq;  in process_pure_responses()
2481 			handle_rsp_cntrl_info(qs, flags);  in process_pure_responses()
2495 		check_ring_db(adap, qs, sleeping);  in process_pure_responses()
2498 	if (unlikely(qs->txq_stopped != 0))  in process_pure_responses()
2499 		restart_tx(qs);  in process_pure_responses()
2521 	struct sge_qset *qs = rspq_to_qset(q);  in handle_responses()  local
2527 	if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {  in handle_responses()
2532 	napi_schedule(&qs->napi);  in handle_responses()
2542 	struct sge_qset *qs = cookie;  in t3_sge_intr_msix()  local
2543 	struct adapter *adap = qs->adap;  in t3_sge_intr_msix()
2544 	struct sge_rspq *q = &qs->rspq;  in t3_sge_intr_msix()
2547 	if (process_responses(adap, qs, -1) == 0)  in t3_sge_intr_msix()
2561 	struct sge_qset *qs = cookie;  in t3_sge_intr_msix_napi()  local
2562 	struct sge_rspq *q = &qs->rspq;  in t3_sge_intr_msix_napi()
2566 	if (handle_responses(qs->adap, q) < 0)  in t3_sge_intr_msix_napi()
2582 	struct sge_rspq *q = &adap->sge.qs[0].rspq;  in t3_intr_msi()
2586 	if (process_responses(adap, &adap->sge.qs[0], -1)) {  in t3_intr_msi()
2593 	    process_responses(adap, &adap->sge.qs[1], -1)) {  in t3_intr_msi()
2594 		struct sge_rspq *q1 = &adap->sge.qs[1].rspq;  in t3_intr_msi()
2609 static int rspq_check_napi(struct sge_qset *qs)  in rspq_check_napi()  argument
2611 	struct sge_rspq *q = &qs->rspq;  in rspq_check_napi()
2613 	if (!napi_is_scheduled(&qs->napi) &&  in rspq_check_napi()
2615 		napi_schedule(&qs->napi);  in rspq_check_napi()
2632 	struct sge_rspq *q = &adap->sge.qs[0].rspq;  in t3_intr_msi_napi()
2636 	new_packets = rspq_check_napi(&adap->sge.qs[0]);  in t3_intr_msi_napi()
2638 		new_packets += rspq_check_napi(&adap->sge.qs[1]);  in t3_intr_msi_napi()
2670 	struct sge_rspq *q0 = &adap->sge.qs[0].rspq;  in t3_intr()
2671 	struct sge_rspq *q1 = &adap->sge.qs[1].rspq;  in t3_intr()
2708 	struct sge_rspq *q0 = &adap->sge.qs[0].rspq;  in t3b_intr()
2725 		process_responses_gts(adap, &adap->sge.qs[1].rspq);  in t3b_intr()
2742 	struct sge_qset *qs0 = &adap->sge.qs[0];  in t3b_intr_napi()
2760 		napi_schedule(&adap->sge.qs[1].napi);  in t3b_intr_napi()
2858 	struct sge_qset *qs = (struct sge_qset *)data;  in sge_timer_tx()  local
2859 	struct port_info *pi = netdev_priv(qs->netdev);  in sge_timer_tx()
2864 	if (__netif_tx_trylock(qs->tx_q)) {  in sge_timer_tx()
2865                 tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],  in sge_timer_tx()
2867 		__netif_tx_unlock(qs->tx_q);  in sge_timer_tx()
2870 	if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {  in sge_timer_tx()
2871 		tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],  in sge_timer_tx()
2873 		spin_unlock(&qs->txq[TXQ_OFLD].lock);  in sge_timer_tx()
2879 	mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);  in sge_timer_tx()
2899 	struct sge_qset *qs = (struct sge_qset *)data;  in sge_timer_rx()  local
2900 	struct port_info *pi = netdev_priv(qs->netdev);  in sge_timer_rx()
2905 	       &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;  in sge_timer_rx()
2910 	if (napi_is_scheduled(&qs->napi))  in sge_timer_rx()
2916 		if (status & (1 << qs->rspq.cntxt_id)) {  in sge_timer_rx()
2917 			qs->rspq.starved++;  in sge_timer_rx()
2918 			if (qs->rspq.credits) {  in sge_timer_rx()
2919 				qs->rspq.credits--;  in sge_timer_rx()
2920 				refill_rspq(adap, &qs->rspq, 1);  in sge_timer_rx()
2921 				qs->rspq.restarted++;  in sge_timer_rx()
2923 					     1 << qs->rspq.cntxt_id);  in sge_timer_rx()
2928 	if (qs->fl[0].credits < qs->fl[0].size)  in sge_timer_rx()
2929 		__refill_fl(adap, &qs->fl[0]);  in sge_timer_rx()
2930 	if (qs->fl[1].credits < qs->fl[1].size)  in sge_timer_rx()
2931 		__refill_fl(adap, &qs->fl[1]);  in sge_timer_rx()
2936 	mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);  in sge_timer_rx()
2947 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)  in t3_update_qset_coalesce()  argument
2949 	qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */  in t3_update_qset_coalesce()
2950 	qs->rspq.polling = p->polling;  in t3_update_qset_coalesce()
2951 	qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;  in t3_update_qset_coalesce()
2976 	struct sge_qset *q = &adapter->sge.qs[id];  in t3_sge_alloc_qset()
3150 		struct sge_qset *q = &adap->sge.qs[i];  in t3_start_sge_timers()
3171 		struct sge_qset *q = &adap->sge.qs[i];  in t3_stop_sge_timers()
3191 		t3_free_qset(adap, &adap->sge.qs[i]);  in t3_free_sge_resources()
3226 			struct sge_qset *qs = &adap->sge.qs[i];  in t3_sge_stop()  local
3228 			tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk);  in t3_sge_stop()
3229 			tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk);  in t3_sge_stop()