506 				     struct bnxt_rx_ring_info *rxr,  in bnxt_alloc_rx_data()  argument
509 	struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];  in bnxt_alloc_rx_data()
510 	struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];  in bnxt_alloc_rx_data()
526 static void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons,  in bnxt_reuse_rx_data()  argument
529 	u16 prod = rxr->rx_prod;  in bnxt_reuse_rx_data()
533 	prod_rx_buf = &rxr->rx_buf_ring[prod];  in bnxt_reuse_rx_data()
534 	cons_rx_buf = &rxr->rx_buf_ring[cons];  in bnxt_reuse_rx_data()
541 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];  in bnxt_reuse_rx_data()
542 	cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];  in bnxt_reuse_rx_data()
547 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)  in bnxt_find_next_agg_idx()  argument
549 	u16 next, max = rxr->rx_agg_bmap_size;  in bnxt_find_next_agg_idx()
551 	next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);  in bnxt_find_next_agg_idx()
553 		next = find_first_zero_bit(rxr->rx_agg_bmap, max);  in bnxt_find_next_agg_idx()
558 				     struct bnxt_rx_ring_info *rxr,  in bnxt_alloc_rx_page()  argument
562 		&rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];  in bnxt_alloc_rx_page()
567 	u16 sw_prod = rxr->rx_sw_agg_prod;  in bnxt_alloc_rx_page()
580 	if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))  in bnxt_alloc_rx_page()
581 		sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);  in bnxt_alloc_rx_page()
583 	__set_bit(sw_prod, rxr->rx_agg_bmap);  in bnxt_alloc_rx_page()
584 	rx_agg_buf = &rxr->rx_agg_ring[sw_prod];  in bnxt_alloc_rx_page()
585 	rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);  in bnxt_alloc_rx_page()
599 	struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_reuse_rx_agg_bufs()  local
600 	u16 prod = rxr->rx_agg_prod;  in bnxt_reuse_rx_agg_bufs()
601 	u16 sw_prod = rxr->rx_sw_agg_prod;  in bnxt_reuse_rx_agg_bufs()
614 		__clear_bit(cons, rxr->rx_agg_bmap);  in bnxt_reuse_rx_agg_bufs()
616 		if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))  in bnxt_reuse_rx_agg_bufs()
617 			sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);  in bnxt_reuse_rx_agg_bufs()
619 		__set_bit(sw_prod, rxr->rx_agg_bmap);  in bnxt_reuse_rx_agg_bufs()
620 		prod_rx_buf = &rxr->rx_agg_ring[sw_prod];  in bnxt_reuse_rx_agg_bufs()
621 		cons_rx_buf = &rxr->rx_agg_ring[cons];  in bnxt_reuse_rx_agg_bufs()
632 		prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];  in bnxt_reuse_rx_agg_bufs()
641 	rxr->rx_agg_prod = prod;  in bnxt_reuse_rx_agg_bufs()
642 	rxr->rx_sw_agg_prod = sw_prod;  in bnxt_reuse_rx_agg_bufs()
646 				   struct bnxt_rx_ring_info *rxr, u16 cons,  in bnxt_rx_skb()  argument
653 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);  in bnxt_rx_skb()
655 		bnxt_reuse_rx_data(rxr, cons, data);  in bnxt_rx_skb()
678 	struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_rx_pages()  local
679 	u16 prod = rxr->rx_agg_prod;  in bnxt_rx_pages()
695 		cons_rx_buf = &rxr->rx_agg_ring[cons];  in bnxt_rx_pages()
697 		__clear_bit(cons, rxr->rx_agg_bmap);  in bnxt_rx_pages()
707 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {  in bnxt_rx_pages()
722 			rxr->rx_agg_prod = prod;  in bnxt_rx_pages()
737 	rxr->rx_agg_prod = prod;  in bnxt_rx_pages()
779 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,  in bnxt_tpa_start()  argument
791 	prod = rxr->rx_prod;  in bnxt_tpa_start()
792 	cons_rx_buf = &rxr->rx_buf_ring[cons];  in bnxt_tpa_start()
793 	prod_rx_buf = &rxr->rx_buf_ring[prod];  in bnxt_tpa_start()
794 	tpa_info = &rxr->rx_tpa[agg_id];  in bnxt_tpa_start()
801 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];  in bnxt_tpa_start()
831 	rxr->rx_prod = NEXT_RX(prod);  in bnxt_tpa_start()
833 	cons_rx_buf = &rxr->rx_buf_ring[cons];  in bnxt_tpa_start()
835 	bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);  in bnxt_tpa_start()
836 	rxr->rx_prod = NEXT_RX(rxr->rx_prod);  in bnxt_tpa_start()
932 	struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_tpa_end()  local
941 	tpa_info = &rxr->rx_tpa[agg_id];  in bnxt_tpa_end()
1048 	struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_rx_pkt()  local
1074 	prod = rxr->rx_prod;  in bnxt_rx_pkt()
1077 		bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,  in bnxt_rx_pkt()
1105 	rx_buf = &rxr->rx_buf_ring[cons];  in bnxt_rx_pkt()
1122 		bnxt_reuse_rx_data(rxr, cons, data);  in bnxt_rx_pkt()
1135 		bnxt_reuse_rx_data(rxr, cons, data);  in bnxt_rx_pkt()
1141 		skb = bnxt_rx_skb(bp, rxr, cons, prod, data, dma_addr, len);  in bnxt_rx_pkt()
1203 	rxr->rx_prod = NEXT_RX(prod);  in bnxt_rx_pkt()
1380 		struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_poll_work()  local
1382 		writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);  in bnxt_poll_work()
1383 		writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);  in bnxt_poll_work()
1385 			writel(DB_KEY_RX | rxr->rx_agg_prod,  in bnxt_poll_work()
1386 			       rxr->rx_agg_doorbell);  in bnxt_poll_work()
1387 			writel(DB_KEY_RX | rxr->rx_agg_prod,  in bnxt_poll_work()
1388 			       rxr->rx_agg_doorbell);  in bnxt_poll_work()
1514 		struct bnxt_rx_ring_info *rxr;  in bnxt_free_rx_skbs()  local
1520 		rxr = &bnapi->rx_ring;  in bnxt_free_rx_skbs()
1522 		if (rxr->rx_tpa) {  in bnxt_free_rx_skbs()
1525 							&rxr->rx_tpa[j];  in bnxt_free_rx_skbs()
1544 			struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];  in bnxt_free_rx_skbs()
1562 				&rxr->rx_agg_ring[j];  in bnxt_free_rx_skbs()
1573 			__clear_bit(j, rxr->rx_agg_bmap);  in bnxt_free_rx_skbs()
1654 		struct bnxt_rx_ring_info *rxr;  in bnxt_free_rx_rings()  local
1660 		rxr = &bnapi->rx_ring;  in bnxt_free_rx_rings()
1662 		kfree(rxr->rx_tpa);  in bnxt_free_rx_rings()
1663 		rxr->rx_tpa = NULL;  in bnxt_free_rx_rings()
1665 		kfree(rxr->rx_agg_bmap);  in bnxt_free_rx_rings()
1666 		rxr->rx_agg_bmap = NULL;  in bnxt_free_rx_rings()
1668 		ring = &rxr->rx_ring_struct;  in bnxt_free_rx_rings()
1671 		ring = &rxr->rx_agg_ring_struct;  in bnxt_free_rx_rings()
1688 		struct bnxt_rx_ring_info *rxr;  in bnxt_alloc_rx_rings()  local
1694 		rxr = &bnapi->rx_ring;  in bnxt_alloc_rx_rings()
1695 		ring = &rxr->rx_ring_struct;  in bnxt_alloc_rx_rings()
1704 			ring = &rxr->rx_agg_ring_struct;  in bnxt_alloc_rx_rings()
1709 			rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;  in bnxt_alloc_rx_rings()
1710 			mem_size = rxr->rx_agg_bmap_size / 8;  in bnxt_alloc_rx_rings()
1711 			rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);  in bnxt_alloc_rx_rings()
1712 			if (!rxr->rx_agg_bmap)  in bnxt_alloc_rx_rings()
1716 				rxr->rx_tpa = kcalloc(MAX_TPA,  in bnxt_alloc_rx_rings()
1719 				if (!rxr->rx_tpa)  in bnxt_alloc_rx_rings()
1873 		struct bnxt_rx_ring_info *rxr;  in bnxt_init_ring_struct()  local
1888 		rxr = &bnapi->rx_ring;  in bnxt_init_ring_struct()
1889 		ring = &rxr->rx_ring_struct;  in bnxt_init_ring_struct()
1892 		ring->pg_arr = (void **)rxr->rx_desc_ring;  in bnxt_init_ring_struct()
1893 		ring->dma_arr = rxr->rx_desc_mapping;  in bnxt_init_ring_struct()
1895 		ring->vmem = (void **)&rxr->rx_buf_ring;  in bnxt_init_ring_struct()
1897 		ring = &rxr->rx_agg_ring_struct;  in bnxt_init_ring_struct()
1900 		ring->pg_arr = (void **)rxr->rx_agg_desc_ring;  in bnxt_init_ring_struct()
1901 		ring->dma_arr = rxr->rx_agg_desc_mapping;  in bnxt_init_ring_struct()
1903 		ring->vmem = (void **)&rxr->rx_agg_ring;  in bnxt_init_ring_struct()
1942 	struct bnxt_rx_ring_info *rxr;  in bnxt_init_one_rx_ring()  local
1956 	rxr = &bnapi->rx_ring;  in bnxt_init_one_rx_ring()
1957 	ring = &rxr->rx_ring_struct;  in bnxt_init_one_rx_ring()
1960 	prod = rxr->rx_prod;  in bnxt_init_one_rx_ring()
1962 		if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {  in bnxt_init_one_rx_ring()
1969 	rxr->rx_prod = prod;  in bnxt_init_one_rx_ring()
1975 	ring = &rxr->rx_agg_ring_struct;  in bnxt_init_one_rx_ring()
1982 	prod = rxr->rx_agg_prod;  in bnxt_init_one_rx_ring()
1984 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {  in bnxt_init_one_rx_ring()
1991 	rxr->rx_agg_prod = prod;  in bnxt_init_one_rx_ring()
1995 		if (rxr->rx_tpa) {  in bnxt_init_one_rx_ring()
2005 				rxr->rx_tpa[i].data = data;  in bnxt_init_one_rx_ring()
2006 				rxr->rx_tpa[i].mapping = mapping;  in bnxt_init_one_rx_ring()
2417 		struct bnxt_rx_ring_info *rxr;  in bnxt_clear_ring_indices()  local
2430 		rxr = &bnapi->rx_ring;  in bnxt_clear_ring_indices()
2431 		rxr->rx_prod = 0;  in bnxt_clear_ring_indices()
2432 		rxr->rx_agg_prod = 0;  in bnxt_clear_ring_indices()
2433 		rxr->rx_sw_agg_prod = 0;  in bnxt_clear_ring_indices()
3332 			struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_hwrm_ring_alloc()  local
3333 			struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;  in bnxt_hwrm_ring_alloc()
3340 			rxr->rx_doorbell = bp->bar1 + i * 0x80;  in bnxt_hwrm_ring_alloc()
3341 			writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);  in bnxt_hwrm_ring_alloc()
3349 			struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_hwrm_ring_alloc()  local
3351 						&rxr->rx_agg_ring_struct;  in bnxt_hwrm_ring_alloc()
3360 			rxr->rx_agg_doorbell =  in bnxt_hwrm_ring_alloc()
3362 			writel(DB_KEY_RX | rxr->rx_agg_prod,  in bnxt_hwrm_ring_alloc()
3363 			       rxr->rx_agg_doorbell);  in bnxt_hwrm_ring_alloc()
3439 			struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_hwrm_ring_free()  local
3440 			struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;  in bnxt_hwrm_ring_free()
3459 			struct bnxt_rx_ring_info *rxr = &bnapi->rx_ring;  in bnxt_hwrm_ring_free()  local
3461 						&rxr->rx_agg_ring_struct;  in bnxt_hwrm_ring_free()
5007 	struct bnxt_rx_ring_info *rxr;  in bnxt_dbg_dump_states()  local
5013 		rxr = &bnapi->rx_ring;  in bnxt_dbg_dump_states()
5020 				    i, rxr->rx_ring_struct.fw_ring_id,  in bnxt_dbg_dump_states()
5021 				    rxr->rx_prod,  in bnxt_dbg_dump_states()
5022 				    rxr->rx_agg_ring_struct.fw_ring_id,  in bnxt_dbg_dump_states()
5023 				    rxr->rx_agg_prod, rxr->rx_sw_agg_prod);  in bnxt_dbg_dump_states()