480 		struct cmdQ *q = &sge->cmdQ[0];  in sched_skb()  local
481 		clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);  in sched_skb()
482 		if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {  in sched_skb()
483 			set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);  in sched_skb()
505 static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q)  in free_freelQ_buffers()  argument
507 	unsigned int cidx = q->cidx;  in free_freelQ_buffers()
509 	while (q->credits--) {  in free_freelQ_buffers()
510 		struct freelQ_ce *ce = &q->centries[cidx];  in free_freelQ_buffers()
517 		if (++cidx == q->size)  in free_freelQ_buffers()
537 		struct freelQ *q = &sge->freelQ[i];  in free_rx_resources()  local
539 		if (q->centries) {  in free_rx_resources()
540 			free_freelQ_buffers(pdev, q);  in free_rx_resources()
541 			kfree(q->centries);  in free_rx_resources()
543 		if (q->entries) {  in free_rx_resources()
544 			size = sizeof(struct freelQ_e) * q->size;  in free_rx_resources()
545 			pci_free_consistent(pdev, size, q->entries,  in free_rx_resources()
546 					    q->dma_addr);  in free_rx_resources()
561 		struct freelQ *q = &sge->freelQ[i];  in alloc_rx_resources()  local
563 		q->genbit = 1;  in alloc_rx_resources()
564 		q->size = p->freelQ_size[i];  in alloc_rx_resources()
565 		q->dma_offset = sge->rx_pkt_pad ? 0 : NET_IP_ALIGN;  in alloc_rx_resources()
566 		size = sizeof(struct freelQ_e) * q->size;  in alloc_rx_resources()
567 		q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);  in alloc_rx_resources()
568 		if (!q->entries)  in alloc_rx_resources()
571 		size = sizeof(struct freelQ_ce) * q->size;  in alloc_rx_resources()
572 		q->centries = kzalloc(size, GFP_KERNEL);  in alloc_rx_resources()
573 		if (!q->centries)  in alloc_rx_resources()
618 static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n)  in free_cmdQ_buffers()  argument
622 	unsigned int cidx = q->cidx;  in free_cmdQ_buffers()
624 	q->in_use -= n;  in free_cmdQ_buffers()
625 	ce = &q->centries[cidx];  in free_cmdQ_buffers()
631 			if (q->sop)  in free_cmdQ_buffers()
632 				q->sop = 0;  in free_cmdQ_buffers()
636 			q->sop = 1;  in free_cmdQ_buffers()
639 		if (++cidx == q->size) {  in free_cmdQ_buffers()
641 			ce = q->centries;  in free_cmdQ_buffers()
644 	q->cidx = cidx;  in free_cmdQ_buffers()
658 		struct cmdQ *q = &sge->cmdQ[i];  in free_tx_resources()  local
660 		if (q->centries) {  in free_tx_resources()
661 			if (q->in_use)  in free_tx_resources()
662 				free_cmdQ_buffers(sge, q, q->in_use);  in free_tx_resources()
663 			kfree(q->centries);  in free_tx_resources()
665 		if (q->entries) {  in free_tx_resources()
666 			size = sizeof(struct cmdQ_e) * q->size;  in free_tx_resources()
667 			pci_free_consistent(pdev, size, q->entries,  in free_tx_resources()
668 					    q->dma_addr);  in free_tx_resources()
682 		struct cmdQ *q = &sge->cmdQ[i];  in alloc_tx_resources()  local
684 		q->genbit = 1;  in alloc_tx_resources()
685 		q->sop = 1;  in alloc_tx_resources()
686 		q->size = p->cmdQ_size[i];  in alloc_tx_resources()
687 		q->in_use = 0;  in alloc_tx_resources()
688 		q->status = 0;  in alloc_tx_resources()
689 		q->processed = q->cleaned = 0;  in alloc_tx_resources()
690 		q->stop_thres = 0;  in alloc_tx_resources()
691 		spin_lock_init(&q->lock);  in alloc_tx_resources()
692 		size = sizeof(struct cmdQ_e) * q->size;  in alloc_tx_resources()
693 		q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);  in alloc_tx_resources()
694 		if (!q->entries)  in alloc_tx_resources()
697 		size = sizeof(struct cmdQ_ce) * q->size;  in alloc_tx_resources()
698 		q->centries = kzalloc(size, GFP_KERNEL);  in alloc_tx_resources()
699 		if (!q->centries)  in alloc_tx_resources()
825 static void refill_free_list(struct sge *sge, struct freelQ *q)  in refill_free_list()  argument
828 	struct freelQ_ce *ce = &q->centries[q->pidx];  in refill_free_list()
829 	struct freelQ_e *e = &q->entries[q->pidx];  in refill_free_list()
830 	unsigned int dma_len = q->rx_buffer_size - q->dma_offset;  in refill_free_list()
832 	while (q->credits < q->size) {  in refill_free_list()
836 		skb = dev_alloc_skb(q->rx_buffer_size);  in refill_free_list()
840 		skb_reserve(skb, q->dma_offset);  in refill_free_list()
850 		e->len_gen = V_CMD_LEN(dma_len) | V_CMD_GEN1(q->genbit);  in refill_free_list()
852 		e->gen2 = V_CMD_GEN2(q->genbit);  in refill_free_list()
856 		if (++q->pidx == q->size) {  in refill_free_list()
857 			q->pidx = 0;  in refill_free_list()
858 			q->genbit ^= 1;  in refill_free_list()
859 			ce = q->centries;  in refill_free_list()
860 			e = q->entries;  in refill_free_list()
862 		q->credits++;  in refill_free_list()
1166 						     struct cmdQ *q)  in write_large_page_tx_descs()  argument
1182 				if (++pidx == q->size) {  in write_large_page_tx_descs()
1185 					ce1 = q->centries;  in write_large_page_tx_descs()
1186 					e1 = q->entries;  in write_large_page_tx_descs()
1202 				  struct cmdQ *q)  in write_tx_descs()  argument
1210 	e = e1 = &q->entries[pidx];  in write_tx_descs()
1211 	ce = &q->centries[pidx];  in write_tx_descs()
1236 		if (++pidx == q->size) {  in write_tx_descs()
1239 			e1 = q->entries;  in write_tx_descs()
1240 			ce = q->centries;  in write_tx_descs()
1244 						 nfrags, q);  in write_tx_descs()
1259 		if (++pidx == q->size) {  in write_tx_descs()
1262 			e1 = q->entries;  in write_tx_descs()
1263 			ce = q->centries;  in write_tx_descs()
1273 						 nfrags, q);  in write_tx_descs()
1289 static inline void reclaim_completed_tx(struct sge *sge, struct cmdQ *q)  in reclaim_completed_tx()  argument
1291 	unsigned int reclaim = q->processed - q->cleaned;  in reclaim_completed_tx()
1295 			 q->processed, q->cleaned);  in reclaim_completed_tx()
1296 		free_cmdQ_buffers(sge, q, reclaim);  in reclaim_completed_tx()
1297 		q->cleaned += reclaim;  in reclaim_completed_tx()
1309 	struct cmdQ *q = &sge->cmdQ[0];  in restart_sched()  local
1313 	spin_lock(&q->lock);  in restart_sched()
1314 	reclaim_completed_tx(sge, q);  in restart_sched()
1316 	credits = q->size - q->in_use;  in restart_sched()
1322 		q->in_use += count;  in restart_sched()
1323 		genbit = q->genbit;  in restart_sched()
1324 		pidx = q->pidx;  in restart_sched()
1325 		q->pidx += count;  in restart_sched()
1326 		if (q->pidx >= q->size) {  in restart_sched()
1327 			q->pidx -= q->size;  in restart_sched()
1328 			q->genbit ^= 1;  in restart_sched()
1330 		write_tx_descs(adapter, skb, pidx, genbit, q);  in restart_sched()
1331 	        credits = q->size - q->in_use;  in restart_sched()
1336 		clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);  in restart_sched()
1337 		if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {  in restart_sched()
1338 			set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);  in restart_sched()
1342 	spin_unlock(&q->lock);  in restart_sched()
1397 static inline int enough_free_Tx_descs(const struct cmdQ *q)  in enough_free_Tx_descs()  argument
1399 	unsigned int r = q->processed - q->cleaned;  in enough_free_Tx_descs()
1401 	return q->in_use - r < (q->size >> 1);  in enough_free_Tx_descs()
1470 	struct respQ *q = &sge->respQ;  in process_responses()  local
1471 	struct respQ_e *e = &q->entries[q->cidx];  in process_responses()
1476 	while (done < budget && e->GenerationBit == q->genbit) {  in process_responses()
1522 		if (unlikely(++q->cidx == q->size)) {  in process_responses()
1523 			q->cidx = 0;  in process_responses()
1524 			q->genbit ^= 1;  in process_responses()
1525 			e = q->entries;  in process_responses()
1529 		if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {  in process_responses()
1530 			writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);  in process_responses()
1531 			q->credits = 0;  in process_responses()
1560 	struct respQ *q = &sge->respQ;  in process_pure_responses()  local
1561 	struct respQ_e *e = &q->entries[q->cidx];  in process_pure_responses()
1577 		if (unlikely(++q->cidx == q->size)) {  in process_pure_responses()
1578 			q->cidx = 0;  in process_pure_responses()
1579 			q->genbit ^= 1;  in process_pure_responses()
1580 			e = q->entries;  in process_pure_responses()
1584 		if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {  in process_pure_responses()
1585 			writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);  in process_pure_responses()
1586 			q->credits = 0;  in process_pure_responses()
1589 	} while (e->GenerationBit == q->genbit && !e->DataValid);  in process_pure_responses()
1594 	return e->GenerationBit == q->genbit;  in process_pure_responses()
1664 	struct cmdQ *q = &sge->cmdQ[qid];  in t1_sge_tx()  local
1667 	if (!spin_trylock(&q->lock))  in t1_sge_tx()
1670 	reclaim_completed_tx(sge, q);  in t1_sge_tx()
1672 	pidx = q->pidx;  in t1_sge_tx()
1673 	credits = q->size - q->in_use;  in t1_sge_tx()
1686 		spin_unlock(&q->lock);  in t1_sge_tx()
1690 	if (unlikely(credits - count < q->stop_thres)) {  in t1_sge_tx()
1707 			spin_unlock(&q->lock);  in t1_sge_tx()
1710 		pidx = q->pidx;  in t1_sge_tx()
1715 	q->in_use += count;  in t1_sge_tx()
1716 	genbit = q->genbit;  in t1_sge_tx()
1717 	pidx = q->pidx;  in t1_sge_tx()
1718 	q->pidx += count;  in t1_sge_tx()
1719 	if (q->pidx >= q->size) {  in t1_sge_tx()
1720 		q->pidx -= q->size;  in t1_sge_tx()
1721 		q->genbit ^= 1;  in t1_sge_tx()
1723 	spin_unlock(&q->lock);  in t1_sge_tx()
1725 	write_tx_descs(adapter, skb, pidx, genbit, q);  in t1_sge_tx()
1737 		clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);  in t1_sge_tx()
1738 		if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {  in t1_sge_tx()
1739 			set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);  in t1_sge_tx()
1745 		if (spin_trylock(&q->lock)) {  in t1_sge_tx()
1746 			credits = q->size - q->in_use;  in t1_sge_tx()
1892 		struct cmdQ *q = &sge->cmdQ[i];  in sge_tx_reclaim_cb()  local
1894 		if (!spin_trylock(&q->lock))  in sge_tx_reclaim_cb()
1897 		reclaim_completed_tx(sge, q);  in sge_tx_reclaim_cb()
1898 		if (i == 0 && q->in_use) {    /* flush pending credits */  in sge_tx_reclaim_cb()
1901 		spin_unlock(&q->lock);  in sge_tx_reclaim_cb()