261 static inline bool fl_starving(const struct adapter *adapter,  in fl_starving()  argument
264 	const struct sge *s = &adapter->sge;  in fl_starving()
377 static void free_tx_desc(struct adapter *adapter, struct sge_txq *tq,  in free_tx_desc()  argument
382 	struct device *dev = adapter->pdev_dev;  in free_tx_desc()
430 static inline void reclaim_completed_tx(struct adapter *adapter,  in reclaim_completed_tx()  argument
444 		free_tx_desc(adapter, tq, avail, unmap);  in reclaim_completed_tx()
454 static inline int get_buf_size(const struct adapter *adapter,  in get_buf_size()  argument
457 	const struct sge *s = &adapter->sge;  in get_buf_size()
473 static void free_rx_bufs(struct adapter *adapter, struct sge_fl *fl, int n)  in free_rx_bufs()  argument
479 			dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),  in free_rx_bufs()
480 				       get_buf_size(adapter, sdesc),  in free_rx_bufs()
503 static void unmap_rx_buf(struct adapter *adapter, struct sge_fl *fl)  in unmap_rx_buf()  argument
508 		dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),  in unmap_rx_buf()
509 			       get_buf_size(adapter, sdesc),  in unmap_rx_buf()
525 static inline void ring_fl_db(struct adapter *adapter, struct sge_fl *fl)  in ring_fl_db()  argument
527 	u32 val = adapter->params.arch.sge_fl_db;  in ring_fl_db()
534 		if (is_t4(adapter->params.chip))  in ring_fl_db()
549 			t4_write_reg(adapter,  in ring_fl_db()
604 static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl,  in refill_fl()  argument
607 	struct sge *s = &adapter->sge;  in refill_fl()
645 		dma_addr = dma_map_page(adapter->pdev_dev, page, 0,  in refill_fl()
648 		if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {  in refill_fl()
684 		dma_addr = dma_map_page(adapter->pdev_dev, page, 0, PAGE_SIZE,  in refill_fl()
686 		if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {  in refill_fl()
711 	ring_fl_db(adapter, fl);  in refill_fl()
713 	if (unlikely(fl_starving(adapter, fl))) {  in refill_fl()
715 		set_bit(fl->cntxt_id, adapter->sge.starving_fl);  in refill_fl()
725 static inline void __refill_fl(struct adapter *adapter, struct sge_fl *fl)  in __refill_fl()  argument
727 	refill_fl(adapter, fl,  in __refill_fl()
969 static inline void ring_tx_db(struct adapter *adapter, struct sge_txq *tq,  in ring_tx_db()  argument
983 		t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,  in ring_tx_db()
1168 	struct adapter *adapter;  in t4vf_eth_xmit()  local
1200 	adapter = pi->adapter;  in t4vf_eth_xmit()
1203 	txq = &adapter->sge.ethtxq[pi->first_qset + qidx];  in t4vf_eth_xmit()
1209 	reclaim_completed_tx(adapter, &txq->q, true);  in t4vf_eth_xmit()
1228 		dev_err(adapter->pdev_dev,  in t4vf_eth_xmit()
1235 	    unlikely(map_skb(adapter->pdev_dev, skb, addr) < 0)) {  in t4vf_eth_xmit()
1304 		if (is_t4(adapter->params.chip))  in t4vf_eth_xmit()
1315 		if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)  in t4vf_eth_xmit()
1339 			cntrl = hwcsum(adapter->params.chip, skb) |  in t4vf_eth_xmit()
1366 	T4_TRACE5(adapter->tb[txq->q.cntxt_id & 7],  in t4vf_eth_xmit()
1452 	ring_tx_db(adapter, &txq->q, ndesc);  in t4vf_eth_xmit()
1571 	struct adapter *adapter = rxq->rspq.adapter;  in do_gro()  local
1572 	struct sge *s = &adapter->sge;  in do_gro()
1621 	struct adapter *adapter = rspq->adapter;  in t4vf_ethrx_handler()  local
1622 	struct sge *s = &adapter->sge;  in t4vf_ethrx_handler()
1755 	struct adapter *adapter = rspq->adapter;  in process_responses()  local
1756 	struct sge *s = &adapter->sge;  in process_responses()
1791 					free_rx_bufs(rspq->adapter, &rxq->fl,  in process_responses()
1806 				bufsz = get_buf_size(adapter, sdesc);  in process_responses()
1813 				unmap_rx_buf(rspq->adapter, &rxq->fl);  in process_responses()
1822 			dma_sync_single_for_cpu(rspq->adapter->pdev_dev,  in process_responses()
1868 		__refill_fl(rspq->adapter, &rxq->fl);  in process_responses()
1905 		t4_write_reg(rspq->adapter,  in napi_rx_handler()
1932 static unsigned int process_intrq(struct adapter *adapter)  in process_intrq()  argument
1934 	struct sge *s = &adapter->sge;  in process_intrq()
1939 	spin_lock(&adapter->sge.intrq_lock);  in process_intrq()
1960 			dev_err(adapter->pdev_dev,  in process_intrq()
1977 			dev_err(adapter->pdev_dev,  in process_intrq()
1983 			dev_err(adapter->pdev_dev,  in process_intrq()
1988 			dev_err(adapter->pdev_dev,  in process_intrq()
2008 		t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,  in process_intrq()
2016 	spin_unlock(&adapter->sge.intrq_lock);  in process_intrq()
2027 	struct adapter *adapter = cookie;  in t4vf_intr_msi()  local
2029 	process_intrq(adapter);  in t4vf_intr_msi()
2040 irq_handler_t t4vf_intr_handler(struct adapter *adapter)  in t4vf_intr_handler()  argument
2042 	BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);  in t4vf_intr_handler()
2043 	if (adapter->flags & USING_MSIX)  in t4vf_intr_handler()
2062 	struct adapter *adapter = (struct adapter *)data;  in sge_rx_timer_cb()  local
2063 	struct sge *s = &adapter->sge;  in sge_rx_timer_cb()
2090 			if (fl_starving(adapter, fl)) {  in sge_rx_timer_cb()
2121 	struct adapter *adapter = (struct adapter *)data;  in sge_tx_timer_cb()  local
2122 	struct sge *s = &adapter->sge;  in sge_tx_timer_cb()
2136 			free_tx_desc(adapter, &txq->q, avail, true);  in sge_tx_timer_cb()
2172 static void __iomem *bar2_address(struct adapter *adapter,  in bar2_address()  argument
2180 	ret = t4vf_bar2_sge_qregs(adapter, qid, qtype,  in bar2_address()
2185 	return adapter->bar2 + bar2_qoffset;  in bar2_address()
2198 int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,  in t4vf_sge_alloc_rxq()  argument
2203 	struct sge *s = &adapter->sge;  in t4vf_sge_alloc_rxq()
2215 	if ((adapter->flags & USING_MSI) && rspq != &adapter->sge.intrq) {  in t4vf_sge_alloc_rxq()
2217 		intr_dest = adapter->sge.intrq.abs_id;  in t4vf_sge_alloc_rxq()
2228 	rspq->desc = alloc_ring(adapter->pdev_dev, rspq->size, rspq->iqe_len,  in t4vf_sge_alloc_rxq()
2266 			CHELSIO_CHIP_VERSION(adapter->params.chip);  in t4vf_sge_alloc_rxq()
2278 		fl->desc = alloc_ring(adapter->pdev_dev, fl->size,  in t4vf_sge_alloc_rxq()
2317 	ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);  in t4vf_sge_alloc_rxq()
2327 	rspq->bar2_addr = bar2_address(adapter,  in t4vf_sge_alloc_rxq()
2333 	rspq->adapter = adapter;  in t4vf_sge_alloc_rxq()
2353 		fl->bar2_addr = bar2_address(adapter,  in t4vf_sge_alloc_rxq()
2358 		refill_fl(adapter, fl, fl_cap(fl), GFP_KERNEL);  in t4vf_sge_alloc_rxq()
2369 		dma_free_coherent(adapter->pdev_dev, rspq->size * rspq->iqe_len,  in t4vf_sge_alloc_rxq()
2376 		dma_free_coherent(adapter->pdev_dev, flsz * EQ_UNIT,  in t4vf_sge_alloc_rxq()
2391 int t4vf_sge_alloc_eth_txq(struct adapter *adapter, struct sge_eth_txq *txq,  in t4vf_sge_alloc_eth_txq()  argument
2395 	struct sge *s = &adapter->sge;  in t4vf_sge_alloc_eth_txq()
2410 	txq->q.desc = alloc_ring(adapter->pdev_dev, txq->q.size,  in t4vf_sge_alloc_eth_txq()
2450 	ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);  in t4vf_sge_alloc_eth_txq()
2458 		dma_free_coherent(adapter->pdev_dev,  in t4vf_sge_alloc_eth_txq()
2470 	txq->q.bar2_addr = bar2_address(adapter,  in t4vf_sge_alloc_eth_txq()
2489 static void free_txq(struct adapter *adapter, struct sge_txq *tq)  in free_txq()  argument
2491 	struct sge *s = &adapter->sge;  in free_txq()
2493 	dma_free_coherent(adapter->pdev_dev,  in free_txq()
2505 static void free_rspq_fl(struct adapter *adapter, struct sge_rspq *rspq,  in free_rspq_fl()  argument
2508 	struct sge *s = &adapter->sge;  in free_rspq_fl()
2511 	t4vf_iq_free(adapter, FW_IQ_TYPE_FL_INT_CAP,  in free_rspq_fl()
2513 	dma_free_coherent(adapter->pdev_dev, (rspq->size + 1) * rspq->iqe_len,  in free_rspq_fl()
2522 		free_rx_bufs(adapter, fl, fl->avail);  in free_rspq_fl()
2523 		dma_free_coherent(adapter->pdev_dev,  in free_rspq_fl()
2539 void t4vf_free_sge_resources(struct adapter *adapter)  in t4vf_free_sge_resources()  argument
2541 	struct sge *s = &adapter->sge;  in t4vf_free_sge_resources()
2548 	for (qs = 0; qs < adapter->sge.ethqsets; qs++, rxq++, txq++) {  in t4vf_free_sge_resources()
2550 			free_rspq_fl(adapter, &rxq->rspq, &rxq->fl);  in t4vf_free_sge_resources()
2552 			t4vf_eth_eq_free(adapter, txq->q.cntxt_id);  in t4vf_free_sge_resources()
2553 			free_tx_desc(adapter, &txq->q, txq->q.in_use, true);  in t4vf_free_sge_resources()
2555 			free_txq(adapter, &txq->q);  in t4vf_free_sge_resources()
2559 		free_rspq_fl(adapter, evtq, NULL);  in t4vf_free_sge_resources()
2561 		free_rspq_fl(adapter, intrq, NULL);  in t4vf_free_sge_resources()
2570 void t4vf_sge_start(struct adapter *adapter)  in t4vf_sge_start()  argument
2572 	adapter->sge.ethtxq_rover = 0;  in t4vf_sge_start()
2573 	mod_timer(&adapter->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);  in t4vf_sge_start()
2574 	mod_timer(&adapter->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);  in t4vf_sge_start()
2585 void t4vf_sge_stop(struct adapter *adapter)  in t4vf_sge_stop()  argument
2587 	struct sge *s = &adapter->sge;  in t4vf_sge_stop()
2604 int t4vf_sge_init(struct adapter *adapter)  in t4vf_sge_init()  argument
2606 	struct sge_params *sge_params = &adapter->params.sge;  in t4vf_sge_init()
2609 	struct sge *s = &adapter->sge;  in t4vf_sge_init()
2618 		dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n",  in t4vf_sge_init()
2623 		dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n");  in t4vf_sge_init()
2648 	if (is_t4(adapter->params.chip)) {  in t4vf_sge_init()
2671 	switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {  in t4vf_sge_init()
2691 	setup_timer(&s->rx_timer, sge_rx_timer_cb, (unsigned long)adapter);  in t4vf_sge_init()
2692 	setup_timer(&s->tx_timer, sge_tx_timer_cb, (unsigned long)adapter);  in t4vf_sge_init()