261 static inline bool fl_starving(const struct adapter *adapter,  in fl_starving()  argument
264 	const struct sge *s = &adapter->sge;  in fl_starving()
377 static void free_tx_desc(struct adapter *adapter, struct sge_txq *tq,  in free_tx_desc()  argument
382 	struct device *dev = adapter->pdev_dev;  in free_tx_desc()
430 static inline void reclaim_completed_tx(struct adapter *adapter,  in reclaim_completed_tx()  argument
444 		free_tx_desc(adapter, tq, avail, unmap);  in reclaim_completed_tx()
454 static inline int get_buf_size(const struct adapter *adapter,  in get_buf_size()  argument
457 	const struct sge *s = &adapter->sge;  in get_buf_size()
473 static void free_rx_bufs(struct adapter *adapter, struct sge_fl *fl, int n)  in free_rx_bufs()  argument
479 			dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),  in free_rx_bufs()
480 				       get_buf_size(adapter, sdesc),  in free_rx_bufs()
503 static void unmap_rx_buf(struct adapter *adapter, struct sge_fl *fl)  in unmap_rx_buf()  argument
508 		dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),  in unmap_rx_buf()
509 			       get_buf_size(adapter, sdesc),  in unmap_rx_buf()
525 static inline void ring_fl_db(struct adapter *adapter, struct sge_fl *fl)  in ring_fl_db()  argument
534 		if (is_t4(adapter->params.chip))  in ring_fl_db()
551 			t4_write_reg(adapter,  in ring_fl_db()
606 static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl,  in refill_fl()  argument
609 	struct sge *s = &adapter->sge;  in refill_fl()
647 		dma_addr = dma_map_page(adapter->pdev_dev, page, 0,  in refill_fl()
650 		if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {  in refill_fl()
686 		dma_addr = dma_map_page(adapter->pdev_dev, page, 0, PAGE_SIZE,  in refill_fl()
688 		if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {  in refill_fl()
713 	ring_fl_db(adapter, fl);  in refill_fl()
715 	if (unlikely(fl_starving(adapter, fl))) {  in refill_fl()
717 		set_bit(fl->cntxt_id, adapter->sge.starving_fl);  in refill_fl()
727 static inline void __refill_fl(struct adapter *adapter, struct sge_fl *fl)  in __refill_fl()  argument
729 	refill_fl(adapter, fl,  in __refill_fl()
971 static inline void ring_tx_db(struct adapter *adapter, struct sge_txq *tq,  in ring_tx_db()  argument
985 		t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,  in ring_tx_db()
1165 	struct adapter *adapter;  in t4vf_eth_xmit()  local
1190 	adapter = pi->adapter;  in t4vf_eth_xmit()
1193 	txq = &adapter->sge.ethtxq[pi->first_qset + qidx];  in t4vf_eth_xmit()
1199 	reclaim_completed_tx(adapter, &txq->q, true);  in t4vf_eth_xmit()
1218 		dev_err(adapter->pdev_dev,  in t4vf_eth_xmit()
1225 	    unlikely(map_skb(adapter->pdev_dev, skb, addr) < 0)) {  in t4vf_eth_xmit()
1294 		if (is_t4(adapter->params.chip))  in t4vf_eth_xmit()
1349 	T4_TRACE5(adapter->tb[txq->q.cntxt_id & 7],  in t4vf_eth_xmit()
1435 	ring_tx_db(adapter, &txq->q, ndesc);  in t4vf_eth_xmit()
1554 	struct adapter *adapter = rxq->rspq.adapter;  in do_gro()  local
1555 	struct sge *s = &adapter->sge;  in do_gro()
1604 	struct adapter *adapter = rspq->adapter;  in t4vf_ethrx_handler()  local
1605 	struct sge *s = &adapter->sge;  in t4vf_ethrx_handler()
1738 	struct adapter *adapter = rspq->adapter;  in process_responses()  local
1739 	struct sge *s = &adapter->sge;  in process_responses()
1774 					free_rx_bufs(rspq->adapter, &rxq->fl,  in process_responses()
1789 				bufsz = get_buf_size(adapter, sdesc);  in process_responses()
1796 				unmap_rx_buf(rspq->adapter, &rxq->fl);  in process_responses()
1805 			dma_sync_single_for_cpu(rspq->adapter->pdev_dev,  in process_responses()
1851 		__refill_fl(rspq->adapter, &rxq->fl);  in process_responses()
1884 	if (is_t4(rspq->adapter->params.chip)) {  in napi_rx_handler()
1885 		t4_write_reg(rspq->adapter,  in napi_rx_handler()
1912 static unsigned int process_intrq(struct adapter *adapter)  in process_intrq()  argument
1914 	struct sge *s = &adapter->sge;  in process_intrq()
1919 	spin_lock(&adapter->sge.intrq_lock);  in process_intrq()
1940 			dev_err(adapter->pdev_dev,  in process_intrq()
1957 			dev_err(adapter->pdev_dev,  in process_intrq()
1963 			dev_err(adapter->pdev_dev,  in process_intrq()
1968 			dev_err(adapter->pdev_dev,  in process_intrq()
1984 	if (is_t4(adapter->params.chip))  in process_intrq()
1985 		t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,  in process_intrq()
1993 	spin_unlock(&adapter->sge.intrq_lock);  in process_intrq()
2004 	struct adapter *adapter = cookie;  in t4vf_intr_msi()  local
2006 	process_intrq(adapter);  in t4vf_intr_msi()
2017 irq_handler_t t4vf_intr_handler(struct adapter *adapter)  in t4vf_intr_handler()  argument
2019 	BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);  in t4vf_intr_handler()
2020 	if (adapter->flags & USING_MSIX)  in t4vf_intr_handler()
2039 	struct adapter *adapter = (struct adapter *)data;  in sge_rx_timer_cb()  local
2040 	struct sge *s = &adapter->sge;  in sge_rx_timer_cb()
2067 			if (fl_starving(adapter, fl)) {  in sge_rx_timer_cb()
2098 	struct adapter *adapter = (struct adapter *)data;  in sge_tx_timer_cb()  local
2099 	struct sge *s = &adapter->sge;  in sge_tx_timer_cb()
2113 			free_tx_desc(adapter, &txq->q, avail, true);  in sge_tx_timer_cb()
2149 static void __iomem *bar2_address(struct adapter *adapter,  in bar2_address()  argument
2157 	ret = t4_bar2_sge_qregs(adapter, qid, qtype,  in bar2_address()
2162 	return adapter->bar2 + bar2_qoffset;  in bar2_address()
2175 int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,  in t4vf_sge_alloc_rxq()  argument
2180 	struct sge *s = &adapter->sge;  in t4vf_sge_alloc_rxq()
2192 	if ((adapter->flags & USING_MSI) && rspq != &adapter->sge.intrq) {  in t4vf_sge_alloc_rxq()
2194 		intr_dest = adapter->sge.intrq.abs_id;  in t4vf_sge_alloc_rxq()
2205 	rspq->desc = alloc_ring(adapter->pdev_dev, rspq->size, rspq->iqe_len,  in t4vf_sge_alloc_rxq()
2249 		fl->desc = alloc_ring(adapter->pdev_dev, fl->size,  in t4vf_sge_alloc_rxq()
2286 	ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);  in t4vf_sge_alloc_rxq()
2296 	rspq->bar2_addr = bar2_address(adapter,  in t4vf_sge_alloc_rxq()
2302 	rspq->adapter = adapter;  in t4vf_sge_alloc_rxq()
2322 		fl->bar2_addr = bar2_address(adapter,  in t4vf_sge_alloc_rxq()
2327 		refill_fl(adapter, fl, fl_cap(fl), GFP_KERNEL);  in t4vf_sge_alloc_rxq()
2338 		dma_free_coherent(adapter->pdev_dev, rspq->size * rspq->iqe_len,  in t4vf_sge_alloc_rxq()
2345 		dma_free_coherent(adapter->pdev_dev, flsz * EQ_UNIT,  in t4vf_sge_alloc_rxq()
2360 int t4vf_sge_alloc_eth_txq(struct adapter *adapter, struct sge_eth_txq *txq,  in t4vf_sge_alloc_eth_txq()  argument
2364 	struct sge *s = &adapter->sge;  in t4vf_sge_alloc_eth_txq()
2379 	txq->q.desc = alloc_ring(adapter->pdev_dev, txq->q.size,  in t4vf_sge_alloc_eth_txq()
2419 	ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);  in t4vf_sge_alloc_eth_txq()
2427 		dma_free_coherent(adapter->pdev_dev,  in t4vf_sge_alloc_eth_txq()
2439 	txq->q.bar2_addr = bar2_address(adapter,  in t4vf_sge_alloc_eth_txq()
2458 static void free_txq(struct adapter *adapter, struct sge_txq *tq)  in free_txq()  argument
2460 	struct sge *s = &adapter->sge;  in free_txq()
2462 	dma_free_coherent(adapter->pdev_dev,  in free_txq()
2474 static void free_rspq_fl(struct adapter *adapter, struct sge_rspq *rspq,  in free_rspq_fl()  argument
2477 	struct sge *s = &adapter->sge;  in free_rspq_fl()
2480 	t4vf_iq_free(adapter, FW_IQ_TYPE_FL_INT_CAP,  in free_rspq_fl()
2482 	dma_free_coherent(adapter->pdev_dev, (rspq->size + 1) * rspq->iqe_len,  in free_rspq_fl()
2491 		free_rx_bufs(adapter, fl, fl->avail);  in free_rspq_fl()
2492 		dma_free_coherent(adapter->pdev_dev,  in free_rspq_fl()
2508 void t4vf_free_sge_resources(struct adapter *adapter)  in t4vf_free_sge_resources()  argument
2510 	struct sge *s = &adapter->sge;  in t4vf_free_sge_resources()
2517 	for (qs = 0; qs < adapter->sge.ethqsets; qs++, rxq++, txq++) {  in t4vf_free_sge_resources()
2519 			free_rspq_fl(adapter, &rxq->rspq, &rxq->fl);  in t4vf_free_sge_resources()
2521 			t4vf_eth_eq_free(adapter, txq->q.cntxt_id);  in t4vf_free_sge_resources()
2522 			free_tx_desc(adapter, &txq->q, txq->q.in_use, true);  in t4vf_free_sge_resources()
2524 			free_txq(adapter, &txq->q);  in t4vf_free_sge_resources()
2528 		free_rspq_fl(adapter, evtq, NULL);  in t4vf_free_sge_resources()
2530 		free_rspq_fl(adapter, intrq, NULL);  in t4vf_free_sge_resources()
2539 void t4vf_sge_start(struct adapter *adapter)  in t4vf_sge_start()  argument
2541 	adapter->sge.ethtxq_rover = 0;  in t4vf_sge_start()
2542 	mod_timer(&adapter->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);  in t4vf_sge_start()
2543 	mod_timer(&adapter->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);  in t4vf_sge_start()
2554 void t4vf_sge_stop(struct adapter *adapter)  in t4vf_sge_stop()  argument
2556 	struct sge *s = &adapter->sge;  in t4vf_sge_stop()
2573 int t4vf_sge_init(struct adapter *adapter)  in t4vf_sge_init()  argument
2575 	struct sge_params *sge_params = &adapter->params.sge;  in t4vf_sge_init()
2578 	struct sge *s = &adapter->sge;  in t4vf_sge_init()
2587 		dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n",  in t4vf_sge_init()
2592 		dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n");  in t4vf_sge_init()
2617 	if (is_t4(adapter->params.chip)) {  in t4vf_sge_init()
2646 	setup_timer(&s->rx_timer, sge_rx_timer_cb, (unsigned long)adapter);  in t4vf_sge_init()
2647 	setup_timer(&s->tx_timer, sge_tx_timer_cb, (unsigned long)adapter);  in t4vf_sge_init()