101 	.gpnum = 0UL - 300UL, \
238 	return READ_ONCE(rsp->completed) != READ_ONCE(rsp->gpnum);  in rcu_gp_in_progress()
253 				       __this_cpu_read(rcu_sched_data.gpnum),  in rcu_sched_qs()
273 				       __this_cpu_read(rcu_bh_data.gpnum),  in rcu_bh_qs()
422 	return rcu_state_p->gpnum;  in rcu_batches_started()
431 	return rcu_sched_state.gpnum;  in rcu_batches_started_sched()
440 	return rcu_bh_state.gpnum;  in rcu_batches_started_bh()
531 			    unsigned long *gpnum, unsigned long *completed)  in rcutorture_get_gp_data()  argument
550 		*gpnum = READ_ONCE(rsp->gpnum);  in rcutorture_get_gp_data()
555 	*gpnum = 0;  in rcutorture_get_gp_data()
1073 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));  in dyntick_save_progress_counter()
1076 		if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4,  in dyntick_save_progress_counter()
1077 				 rdp->mynode->gpnum))  in dyntick_save_progress_counter()
1108 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));  in rcu_implicit_dynticks_qs()
1128 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl"));  in rcu_implicit_dynticks_qs()
1202 		       rsp->gpnum, rsp->completed,  in rcu_check_gp_kthread_starvation()
1227 static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)  in print_other_cpu_stall()  argument
1277 	       (long)rsp->gpnum, (long)rsp->completed, totqlen);  in print_other_cpu_stall()
1281 		if (READ_ONCE(rsp->gpnum) != gpnum ||  in print_other_cpu_stall()
1282 		    READ_ONCE(rsp->completed) == gpnum) {  in print_other_cpu_stall()
1324 		(long)rsp->gpnum, (long)rsp->completed, totqlen);  in print_cpu_stall()
1349 	unsigned long gpnum;  in check_cpu_stall()  local
1376 	gpnum = READ_ONCE(rsp->gpnum);  in check_cpu_stall()
1383 	if (ULONG_CMP_GE(completed, gpnum) ||  in check_cpu_stall()
1398 		print_other_cpu_stall(rsp, gpnum);  in check_cpu_stall()
1462 	if (rcu_get_root(rsp) == rnp && rnp->gpnum == rnp->completed)  in rcu_cbs_completed()
1479 	trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum,  in trace_rcu_future_gp()
1525 	if (rnp->gpnum != rnp->completed ||  in rcu_start_future_gp()
1526 	    READ_ONCE(rnp_root->gpnum) != READ_ONCE(rnp_root->completed)) {  in rcu_start_future_gp()
1566 	if (rnp_root->gpnum != rnp_root->completed) {  in rcu_start_future_gp()
1683 		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB"));  in rcu_accelerate_cbs()
1685 		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB"));  in rcu_accelerate_cbs()
1758 		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend"));  in __note_gp_changes()
1761 	if (rdp->gpnum != rnp->gpnum || unlikely(READ_ONCE(rdp->gpwrap))) {  in __note_gp_changes()
1767 		rdp->gpnum = rnp->gpnum;  in __note_gp_changes()
1768 		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));  in __note_gp_changes()
1786 	if ((rdp->gpnum == READ_ONCE(rnp->gpnum) &&  in note_gp_changes()
1803 	    !(rsp->gpnum % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))  in rcu_gp_slow()
1838 	smp_store_release(&rsp->gpnum, rsp->gpnum + 1);  in rcu_gp_init()
1839 	trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));  in rcu_gp_init()
1912 		WRITE_ONCE(rnp->gpnum, rsp->gpnum);  in rcu_gp_init()
1918 		trace_rcu_grace_period_init(rsp->name, rnp->gpnum,  in rcu_gp_init()
2026 		WRITE_ONCE(rnp->completed, rsp->gpnum);  in rcu_gp_cleanup()
2043 	WRITE_ONCE(rsp->completed, rsp->gpnum);  in rcu_gp_cleanup()
2052 				       READ_ONCE(rsp->gpnum),  in rcu_gp_cleanup()
2076 					       READ_ONCE(rsp->gpnum),  in rcu_gp_kthread()
2090 					       READ_ONCE(rsp->gpnum),  in rcu_gp_kthread()
2106 					       READ_ONCE(rsp->gpnum),  in rcu_gp_kthread()
2121 						       READ_ONCE(rsp->gpnum),  in rcu_gp_kthread()
2126 						       READ_ONCE(rsp->gpnum),  in rcu_gp_kthread()
2136 						       READ_ONCE(rsp->gpnum),  in rcu_gp_kthread()
2181 	trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum),  in rcu_start_gp_advanced()
2256 		if (!(rnp->qsmask & mask) || rnp->gpnum != gps) {  in rcu_report_qs_rnp()
2267 		trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,  in rcu_report_qs_rnp()
2332 	gps = rnp->gpnum;  in rcu_report_unblock_qs_rnp()
2362 	    rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||  in rcu_report_qs_rdp()
2388 		rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);  in rcu_report_qs_rdp()
2551 			       rnp->gpnum + 1 - !!(rnp->qsmask & mask),  in rcu_cleanup_dying_cpu()
2849 			rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);  in force_qs_rnp()
3263 	return smp_load_acquire(&rcu_state_p->gpnum);  in get_state_synchronize_rcu()
3315 	return smp_load_acquire(&rcu_sched_state.gpnum);  in get_state_synchronize_sched()
3917 	if (READ_ONCE(rnp->gpnum) != rdp->gpnum ||  in __rcu_pending()
4208 	rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */  in rcu_init_percpu_data()
4213 	trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));  in rcu_init_percpu_data()
4431 			rnp->gpnum = rsp->gpnum;  in rcu_init_one()