515 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)  in prb_del_retire_blk_timer()  argument
517 	del_timer_sync(&pkc->retire_blk_timer);  in prb_del_retire_blk_timer()
524 	struct tpacket_kbdq_core *pkc;  in prb_shutdown_retire_blk_timer()  local
526 	pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :  in prb_shutdown_retire_blk_timer()
530 	pkc->delete_blk_timer = 1;  in prb_shutdown_retire_blk_timer()
533 	prb_del_retire_blk_timer(pkc);  in prb_shutdown_retire_blk_timer()
537 		struct tpacket_kbdq_core *pkc,  in prb_init_blk_timer()  argument
540 	init_timer(&pkc->retire_blk_timer);  in prb_init_blk_timer()
541 	pkc->retire_blk_timer.data = (long)po;  in prb_init_blk_timer()
542 	pkc->retire_blk_timer.function = func;  in prb_init_blk_timer()
543 	pkc->retire_blk_timer.expires = jiffies;  in prb_init_blk_timer()
548 	struct tpacket_kbdq_core *pkc;  in prb_setup_retire_blk_timer()  local
553 	pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :  in prb_setup_retire_blk_timer()
555 	prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);  in prb_setup_retire_blk_timer()
644 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)  in _prb_refresh_rx_retire_blk_timer()  argument
646 	mod_timer(&pkc->retire_blk_timer,  in _prb_refresh_rx_retire_blk_timer()
647 			jiffies + pkc->tov_in_jiffies);  in _prb_refresh_rx_retire_blk_timer()
648 	pkc->last_kactive_blk_num = pkc->kactive_blk_num;  in _prb_refresh_rx_retire_blk_timer()
677 	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);  in prb_retire_rx_blk_timer_expired()  local
683 	frozen = prb_queue_frozen(pkc);  in prb_retire_rx_blk_timer_expired()
684 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);  in prb_retire_rx_blk_timer_expired()
686 	if (unlikely(pkc->delete_blk_timer))  in prb_retire_rx_blk_timer_expired()
699 		while (atomic_read(&pkc->blk_fill_in_prog)) {  in prb_retire_rx_blk_timer_expired()
705 	if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {  in prb_retire_rx_blk_timer_expired()
711 			prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);  in prb_retire_rx_blk_timer_expired()
712 			if (!prb_dispatch_next_block(pkc, po))  in prb_retire_rx_blk_timer_expired()
720 			if (prb_curr_blk_in_use(pkc, pbd)) {  in prb_retire_rx_blk_timer_expired()
734 				prb_open_block(pkc, pbd);  in prb_retire_rx_blk_timer_expired()
741 	_prb_refresh_rx_retire_blk_timer(pkc);  in prb_retire_rx_blk_timer_expired()
832 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)  in prb_thaw_queue()  argument
834 	pkc->reset_pending_on_curr_blk = 0;  in prb_thaw_queue()
904 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,  in prb_freeze_queue()  argument
907 	pkc->reset_pending_on_curr_blk = 1;  in prb_freeze_queue()
919 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,  in prb_dispatch_next_block()  argument
927 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);  in prb_dispatch_next_block()
931 		prb_freeze_queue(pkc, po);  in prb_dispatch_next_block()
940 	prb_open_block(pkc, pbd);  in prb_dispatch_next_block()
941 	return (void *)pkc->nxt_offset;  in prb_dispatch_next_block()
944 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,  in prb_retire_current_block()  argument
947 	struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);  in prb_retire_current_block()
961 			while (atomic_read(&pkc->blk_fill_in_prog)) {  in prb_retire_current_block()
966 		prb_close_block(pkc, pbd, po, status);  in prb_retire_current_block()
971 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,  in prb_curr_blk_in_use()  argument
977 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)  in prb_queue_frozen()  argument
979 	return pkc->reset_pending_on_curr_blk;  in prb_queue_frozen()
984 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);  in prb_clear_blk_fill_status()  local
985 	atomic_dec(&pkc->blk_fill_in_prog);  in prb_clear_blk_fill_status()
988 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,  in prb_fill_rxhash()  argument
991 	ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);  in prb_fill_rxhash()
994 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,  in prb_clear_rxhash()  argument
1000 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,  in prb_fill_vlan_info()  argument
1003 	if (skb_vlan_tag_present(pkc->skb)) {  in prb_fill_vlan_info()
1004 		ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);  in prb_fill_vlan_info()
1005 		ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);  in prb_fill_vlan_info()
1014 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,  in prb_run_all_ft_ops()  argument
1018 	prb_fill_vlan_info(pkc, ppd);  in prb_run_all_ft_ops()
1020 	if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)  in prb_run_all_ft_ops()
1021 		prb_fill_rxhash(pkc, ppd);  in prb_run_all_ft_ops()
1023 		prb_clear_rxhash(pkc, ppd);  in prb_run_all_ft_ops()
1027 				struct tpacket_kbdq_core *pkc,  in prb_fill_curr_block()  argument
1035 	pkc->prev = curr;  in prb_fill_curr_block()
1036 	pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);  in prb_fill_curr_block()
1039 	atomic_inc(&pkc->blk_fill_in_prog);  in prb_fill_curr_block()
1040 	prb_run_all_ft_ops(pkc, ppd);  in prb_fill_curr_block()
1050 	struct tpacket_kbdq_core *pkc;  in __packet_lookup_frame_in_block()  local
1054 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);  in __packet_lookup_frame_in_block()
1055 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);  in __packet_lookup_frame_in_block()
1058 	if (prb_queue_frozen(pkc)) {  in __packet_lookup_frame_in_block()
1063 		if (prb_curr_blk_in_use(pkc, pbd)) {  in __packet_lookup_frame_in_block()
1073 			prb_open_block(pkc, pbd);  in __packet_lookup_frame_in_block()
1078 	curr = pkc->nxt_offset;  in __packet_lookup_frame_in_block()
1079 	pkc->skb = skb;  in __packet_lookup_frame_in_block()
1080 	end = (char *)pbd + pkc->kblk_size;  in __packet_lookup_frame_in_block()
1084 		prb_fill_curr_block(curr, pkc, pbd, len);  in __packet_lookup_frame_in_block()
1089 	prb_retire_current_block(pkc, po, 0);  in __packet_lookup_frame_in_block()
1092 	curr = (char *)prb_dispatch_next_block(pkc, po);  in __packet_lookup_frame_in_block()
1094 		pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);  in __packet_lookup_frame_in_block()
1095 		prb_fill_curr_block(curr, pkc, pbd, len);  in __packet_lookup_frame_in_block()
1131 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);  in prb_lookup_block()  local
1132 	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);  in prb_lookup_block()