558 			       struct be_rx_obj *rxo, u32 erx_stat)  in populate_erx_stats()  argument
561 		rx_stats(rxo)->rx_drops_no_frags = erx_stat;  in populate_erx_stats()
566 		accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,  in populate_erx_stats()
573 	struct be_rx_obj *rxo;  in be_parse_stats()  local
589 		for_all_rx_queues(adapter, rxo, i) {  in be_parse_stats()
590 			erx_stat = erx->rx_drops_no_fragments[rxo->q.id];  in be_parse_stats()
591 			populate_erx_stats(adapter, rxo, erx_stat);  in be_parse_stats()
601 	struct be_rx_obj *rxo;  in be_get_stats64()  local
607 	for_all_rx_queues(adapter, rxo, i) {  in be_get_stats64()
608 		const struct be_rx_stats *rx_stats = rx_stats(rxo);  in be_get_stats64()
612 			pkts = rx_stats(rxo)->rx_pkts;  in be_get_stats64()
613 			bytes = rx_stats(rxo)->rx_bytes;  in be_get_stats64()
617 		stats->multicast += rx_stats(rxo)->rx_mcast_pkts;  in be_get_stats64()
618 		stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +  in be_get_stats64()
619 					rx_stats(rxo)->rx_drops_no_frags;  in be_get_stats64()
1849 	struct be_rx_obj *rxo;  in be_get_new_eqd()  local
1864 	for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {  in be_get_new_eqd()
1866 			start = u64_stats_fetch_begin_irq(&rxo->stats.sync);  in be_get_new_eqd()
1867 			rx_pkts += rxo->stats.rx_pkts;  in be_get_new_eqd()
1868 		} while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));  in be_get_new_eqd()
1959 static void be_rx_stats_update(struct be_rx_obj *rxo,  in be_rx_stats_update()  argument
1962 	struct be_rx_stats *stats = rx_stats(rxo);  in be_rx_stats_update()
1986 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)  in get_rx_page_info()  argument
1988 	struct be_adapter *adapter = rxo->adapter;  in get_rx_page_info()
1990 	struct be_queue_info *rxq = &rxo->q;  in get_rx_page_info()
1993 	rx_page_info = &rxo->page_info_tbl[frag_idx];  in get_rx_page_info()
2013 static void be_rx_compl_discard(struct be_rx_obj *rxo,  in be_rx_compl_discard()  argument
2020 		page_info = get_rx_page_info(rxo);  in be_rx_compl_discard()
2030 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,  in skb_fill_rx_data()  argument
2038 	page_info = get_rx_page_info(rxo);  in skb_fill_rx_data()
2075 		page_info = get_rx_page_info(rxo);  in skb_fill_rx_data()
2102 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,  in be_rx_compl_process()  argument
2105 	struct be_adapter *adapter = rxo->adapter;  in be_rx_compl_process()
2111 		rx_stats(rxo)->rx_drops_no_skbs++;  in be_rx_compl_process()
2112 		be_rx_compl_discard(rxo, rxcp);  in be_rx_compl_process()
2116 	skb_fill_rx_data(rxo, skb, rxcp);  in be_rx_compl_process()
2124 	skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);  in be_rx_compl_process()
2138 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,  in be_rx_compl_process_gro()  argument
2142 	struct be_adapter *adapter = rxo->adapter;  in be_rx_compl_process_gro()
2150 		be_rx_compl_discard(rxo, rxcp);  in be_rx_compl_process_gro()
2156 		page_info = get_rx_page_info(rxo);  in be_rx_compl_process_gro()
2182 	skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);  in be_rx_compl_process_gro()
2240 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)  in be_rx_compl_get()  argument
2242 	struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);  in be_rx_compl_get()
2243 	struct be_rx_compl_info *rxcp = &rxo->rxcp;  in be_rx_compl_get()
2244 	struct be_adapter *adapter = rxo->adapter;  in be_rx_compl_get()
2281 	queue_tail_inc(&rxo->cq);  in be_rx_compl_get()
2298 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)  in be_post_rx_frags()  argument
2300 	struct be_adapter *adapter = rxo->adapter;  in be_post_rx_frags()
2302 	struct be_queue_info *rxq = &rxo->q;  in be_post_rx_frags()
2309 	page_info = &rxo->page_info_tbl[rxq->head];  in be_post_rx_frags()
2314 				rx_stats(rxo)->rx_post_fail++;  in be_post_rx_frags()
2351 		page_info = &rxo->page_info_tbl[rxq->head];  in be_post_rx_frags()
2364 		if (rxo->rx_post_starved)  in be_post_rx_frags()
2365 			rxo->rx_post_starved = false;  in be_post_rx_frags()
2373 		rxo->rx_post_starved = true;  in be_post_rx_frags()
2461 static void be_rxq_clean(struct be_rx_obj *rxo)  in be_rxq_clean()  argument
2463 	struct be_queue_info *rxq = &rxo->q;  in be_rxq_clean()
2467 		page_info = get_rx_page_info(rxo);  in be_rxq_clean()
2476 static void be_rx_cq_clean(struct be_rx_obj *rxo)  in be_rx_cq_clean()  argument
2478 	struct be_queue_info *rx_cq = &rxo->cq;  in be_rx_cq_clean()
2480 	struct be_adapter *adapter = rxo->adapter;  in be_rx_cq_clean()
2490 		rxcp = be_rx_compl_get(rxo);  in be_rx_cq_clean()
2505 			be_rx_compl_discard(rxo, rxcp);  in be_rx_cq_clean()
2754 	struct be_rx_obj *rxo;  in be_rx_cqs_destroy()  local
2757 	for_all_rx_queues(adapter, rxo, i) {  in be_rx_cqs_destroy()
2758 		q = &rxo->cq;  in be_rx_cqs_destroy()
2768 	struct be_rx_obj *rxo;  in be_rx_cqs_create()  local
2787 	for_all_rx_queues(adapter, rxo, i) {  in be_rx_cqs_create()
2788 		rxo->adapter = adapter;  in be_rx_cqs_create()
2789 		cq = &rxo->cq;  in be_rx_cqs_create()
2795 		u64_stats_init(&rxo->stats.sync);  in be_rx_cqs_create()
2853 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,  in be_process_rx()  argument
2856 	struct be_adapter *adapter = rxo->adapter;  in be_process_rx()
2857 	struct be_queue_info *rx_cq = &rxo->cq;  in be_process_rx()
2863 		rxcp = be_rx_compl_get(rxo);  in be_process_rx()
2873 			be_rx_compl_discard(rxo, rxcp);  in be_process_rx()
2882 			be_rx_compl_discard(rxo, rxcp);  in be_process_rx()
2888 			be_rx_compl_process_gro(rxo, napi, rxcp);  in be_process_rx()
2890 			be_rx_compl_process(rxo, napi, rxcp);  in be_process_rx()
2894 		be_rx_stats_update(rxo, rxcp);  in be_process_rx()
2903 		if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&  in be_process_rx()
2904 		    !rxo->rx_post_starved)  in be_process_rx()
2905 			be_post_rx_frags(rxo, GFP_ATOMIC,  in be_process_rx()
3090 	struct be_rx_obj *rxo;  in be_poll()  local
3104 		for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {  in be_poll()
3105 			work = be_process_rx(rxo, napi, budget, NAPI_POLLING);  in be_poll()
3139 	struct be_rx_obj *rxo;  in be_busy_poll()  local
3145 	for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {  in be_busy_poll()
3146 		work = be_process_rx(rxo, napi, 4, BUSY_POLLING);  in be_busy_poll()
3369 	struct be_rx_obj *rxo;  in be_rx_qs_destroy()  local
3372 	for_all_rx_queues(adapter, rxo, i) {  in be_rx_qs_destroy()
3373 		q = &rxo->q;  in be_rx_qs_destroy()
3382 				be_rx_cq_clean(rxo);  in be_rx_qs_destroy()
3384 					be_post_rx_frags(rxo, GFP_KERNEL,  in be_rx_qs_destroy()
3389 			be_rx_cq_clean(rxo);  in be_rx_qs_destroy()
3390 			be_rxq_clean(rxo);  in be_rx_qs_destroy()
3472 	struct be_rx_obj *rxo;  in be_rx_qs_create()  local
3475 	for_all_rx_queues(adapter, rxo, i) {  in be_rx_qs_create()
3476 		rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,  in be_rx_qs_create()
3483 		rxo = default_rxo(adapter);  in be_rx_qs_create()
3484 		rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,  in be_rx_qs_create()
3486 				       false, &rxo->rss_id);  in be_rx_qs_create()
3491 	for_all_rss_queues(adapter, rxo, i) {  in be_rx_qs_create()
3492 		rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,  in be_rx_qs_create()
3494 				       true, &rxo->rss_id);  in be_rx_qs_create()
3501 			for_all_rss_queues(adapter, rxo, i) {  in be_rx_qs_create()
3504 				rss->rsstable[j + i] = rxo->rss_id;  in be_rx_qs_create()
3532 	for_all_rx_queues(adapter, rxo, i)  in be_rx_qs_create()
3533 		be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1);  in be_rx_qs_create()
3567 	struct be_rx_obj *rxo;  in be_open()  local
3584 	for_all_rx_queues(adapter, rxo, i)  in be_open()
3585 		be_cq_notify(adapter, rxo->cq.id, true, 0);  in be_open()
5472 	struct be_rx_obj *rxo;  in be_worker()  local
5497 	for_all_rx_queues(adapter, rxo, i) {  in be_worker()
5501 		if (rxo->rx_post_starved)  in be_worker()
5502 			be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);  in be_worker()