104 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
250 		rcu_report_exp_rdp(rdp->rsp, rdp, true);  in rcu_preempt_ctxt_queue()
316 		trace_rcu_preempt_task(rdp->rsp->name,  in rcu_preempt_note_context_switch()
546 static void rcu_print_detail_task_stall(struct rcu_state *rsp)  in rcu_print_detail_task_stall()  argument
548 	struct rcu_node *rnp = rcu_get_root(rsp);  in rcu_print_detail_task_stall()
551 	rcu_for_each_leaf_node(rsp, rnp)  in rcu_print_detail_task_stall()
705 	struct rcu_state *rsp = info;  in sync_rcu_exp_handler()  local
728 	rdp = this_cpu_ptr(rsp->rda);  in sync_rcu_exp_handler()
729 	rcu_report_exp_rdp(rsp, rdp, true);  in sync_rcu_exp_handler()
748 	struct rcu_state *rsp = rcu_state_p;  in synchronize_rcu_expedited()  local
751 	s = rcu_exp_gp_seq_snap(rsp);  in synchronize_rcu_expedited()
753 	rnp_unlock = exp_funnel_lock(rsp, s);  in synchronize_rcu_expedited()
757 	rcu_exp_gp_seq_start(rsp);  in synchronize_rcu_expedited()
760 	sync_rcu_exp_select_cpus(rsp, sync_rcu_exp_handler);  in synchronize_rcu_expedited()
763 	rnp = rcu_get_root(rsp);  in synchronize_rcu_expedited()
764 	synchronize_sched_expedited_wait(rsp);  in synchronize_rcu_expedited()
767 	rcu_exp_gp_seq_end(rsp);  in synchronize_rcu_expedited()
855 static void rcu_print_detail_task_stall(struct rcu_state *rsp)  in rcu_print_detail_task_stall()  argument
1158 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,  in rcu_spawn_one_boost_kthread()  argument
1161 	int rnp_index = rnp - &rsp->node[0];  in rcu_spawn_one_boost_kthread()
1166 	if (rcu_state_p != rsp)  in rcu_spawn_one_boost_kthread()
1172 	rsp->boost = 1;  in rcu_spawn_one_boost_kthread()
1435 	struct rcu_state *rsp;  in rcu_try_advance_all_cbs()  local
1442 	for_each_rcu_flavor(rsp) {  in rcu_try_advance_all_cbs()
1443 		rdp = this_cpu_ptr(rsp->rda);  in rcu_try_advance_all_cbs()
1454 			note_gp_changes(rsp, rdp);  in rcu_try_advance_all_cbs()
1524 	struct rcu_state *rsp;  in rcu_prepare_for_idle()  local
1565 	for_each_rcu_flavor(rsp) {  in rcu_prepare_for_idle()
1566 		rdp = this_cpu_ptr(rsp->rda);  in rcu_prepare_for_idle()
1572 		needwake = rcu_accelerate_cbs(rsp, rnp, rdp);  in rcu_prepare_for_idle()
1575 			rcu_gp_kthread_wake(rsp);  in rcu_prepare_for_idle()
1631 	struct rcu_state *rsp;  in rcu_oom_notify_cpu()  local
1634 	for_each_rcu_flavor(rsp) {  in rcu_oom_notify_cpu()
1635 		rdp = raw_cpu_ptr(rsp->rda);  in rcu_oom_notify_cpu()
1638 			rsp->call(&rdp->oom_head, rcu_oom_callback);  in rcu_oom_notify_cpu()
1730 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)  in print_cpu_stall_info()  argument
1733 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in print_cpu_stall_info()
1738 	if (rsp->gpnum == rdp->gpnum) {  in print_cpu_stall_info()
1743 		ticks_value = rsp->gpnum - rdp->gpnum;  in print_cpu_stall_info()
1755 	       READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,  in print_cpu_stall_info()
1775 	struct rcu_state *rsp;  in increment_cpu_stall_ticks()  local
1777 	for_each_rcu_flavor(rsp)  in increment_cpu_stall_ticks()
1778 		raw_cpu_inc(rsp->rda->ticks_this_gp);  in increment_cpu_stall_ticks()
1825 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)  in rcu_nocb_gp_cleanup()  argument
1879 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)  in rcu_nocb_cpu_needs_barrier()  argument
1881 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_nocb_cpu_needs_barrier()
1950 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1959 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1963 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1971 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1975 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1980 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_enqueue()
2002 		trace_rcu_kfree_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2007 		trace_rcu_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2028 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,  in rcu_nocb_adopt_orphan_cbs()  argument
2032 	long ql = rsp->qlen;  in rcu_nocb_adopt_orphan_cbs()
2033 	long qll = rsp->qlen_lazy;  in rcu_nocb_adopt_orphan_cbs()
2038 	rsp->qlen = 0;  in rcu_nocb_adopt_orphan_cbs()
2039 	rsp->qlen_lazy = 0;  in rcu_nocb_adopt_orphan_cbs()
2042 	if (rsp->orphan_donelist != NULL) {  in rcu_nocb_adopt_orphan_cbs()
2043 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,  in rcu_nocb_adopt_orphan_cbs()
2044 					rsp->orphan_donetail, ql, qll, flags);  in rcu_nocb_adopt_orphan_cbs()
2046 		rsp->orphan_donelist = NULL;  in rcu_nocb_adopt_orphan_cbs()
2047 		rsp->orphan_donetail = &rsp->orphan_donelist;  in rcu_nocb_adopt_orphan_cbs()
2049 	if (rsp->orphan_nxtlist != NULL) {  in rcu_nocb_adopt_orphan_cbs()
2050 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,  in rcu_nocb_adopt_orphan_cbs()
2051 					rsp->orphan_nxttail, ql, qll, flags);  in rcu_nocb_adopt_orphan_cbs()
2053 		rsp->orphan_nxtlist = NULL;  in rcu_nocb_adopt_orphan_cbs()
2054 		rsp->orphan_nxttail = &rsp->orphan_nxtlist;  in rcu_nocb_adopt_orphan_cbs()
2076 		rcu_gp_kthread_wake(rdp->rsp);  in rcu_nocb_wait_gp()
2111 		trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");  in nocb_leader_wait()
2117 		trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");  in nocb_leader_wait()
2143 			trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,  in nocb_leader_wait()
2206 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2213 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");  in nocb_follower_wait()
2220 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2252 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");  in rcu_nocb_kthread()
2257 		trace_rcu_batch_start(rdp->rsp->name,  in rcu_nocb_kthread()
2265 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2268 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2274 			if (__rcu_reclaim(rdp->rsp->name, list))  in rcu_nocb_kthread()
2280 		trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);  in rcu_nocb_kthread()
2305 	trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));  in do_nocb_deferred_wakeup()
2312 	struct rcu_state *rsp;  in rcu_init_nohz()  local
2356 	for_each_rcu_flavor(rsp) {  in rcu_init_nohz()
2358 			init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));  in rcu_init_nohz()
2359 		rcu_organize_nocb_kthreads(rsp);  in rcu_init_nohz()
2377 static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)  in rcu_spawn_one_nocb_kthread()  argument
2382 	struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);  in rcu_spawn_one_nocb_kthread()
2414 			"rcuo%c/%d", rsp->abbr, cpu);  in rcu_spawn_one_nocb_kthread()
2425 	struct rcu_state *rsp;  in rcu_spawn_all_nocb_kthreads()  local
2428 		for_each_rcu_flavor(rsp)  in rcu_spawn_all_nocb_kthreads()
2429 			rcu_spawn_one_nocb_kthread(rsp, cpu);  in rcu_spawn_all_nocb_kthreads()
2453 static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)  in rcu_organize_nocb_kthreads()  argument
2474 		rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_organize_nocb_kthreads()
2511 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)  in rcu_nocb_cpu_needs_barrier()  argument
2517 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)  in rcu_nocb_gp_cleanup()  argument
2535 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,  in rcu_nocb_adopt_orphan_cbs()  argument
2747 	if (!*isidle || rdp->rsp != rcu_state_p ||  in rcu_sysidle_check_cpu()
2771 static bool is_sysidle_rcu_state(struct rcu_state *rsp)  in is_sysidle_rcu_state()  argument
2773 	return rsp == rcu_state_p;  in is_sysidle_rcu_state()
2848 static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,  in rcu_sysidle_report()  argument
2851 	if (rsp != rcu_state_p)  in rcu_sysidle_report()
2865 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,  in rcu_sysidle_report_gp()  argument
2872 	rcu_sysidle_report(rsp, isidle, maxj, true);  in rcu_sysidle_report_gp()
2986 static bool is_sysidle_rcu_state(struct rcu_state *rsp)  in is_sysidle_rcu_state()  argument
2991 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,  in rcu_sysidle_report_gp()  argument
3011 static bool rcu_nohz_full_cpu(struct rcu_state *rsp)  in rcu_nohz_full_cpu()  argument
3015 	    (!rcu_gp_in_progress(rsp) ||  in rcu_nohz_full_cpu()
3016 	     ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))  in rcu_nohz_full_cpu()