146 	struct rcu_data *rdp;  in rcu_preempt_note_context_switch()  local
153 		rdp = this_cpu_ptr(rcu_preempt_state.rda);  in rcu_preempt_note_context_switch()
154 		rnp = rdp->mynode;  in rcu_preempt_note_context_switch()
178 		WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);  in rcu_preempt_note_context_switch()
180 		if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {  in rcu_preempt_note_context_switch()
189 			if (rnp->qsmask & rdp->grpmask)  in rcu_preempt_note_context_switch()
192 		trace_rcu_preempt_task(rdp->rsp->name,  in rcu_preempt_note_context_switch()
194 				       (rnp->qsmask & rdp->grpmask)  in rcu_preempt_note_context_switch()
1315 	struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);  in rcu_prepare_kthreads()  local
1316 	struct rcu_node *rnp = rdp->mynode;  in rcu_prepare_kthreads()
1445 	struct rcu_data *rdp;  in rcu_try_advance_all_cbs()  local
1456 		rdp = this_cpu_ptr(rsp->rda);  in rcu_try_advance_all_cbs()
1457 		rnp = rdp->mynode;  in rcu_try_advance_all_cbs()
1464 		if ((rdp->completed != rnp->completed ||  in rcu_try_advance_all_cbs()
1465 		     unlikely(ACCESS_ONCE(rdp->gpwrap))) &&  in rcu_try_advance_all_cbs()
1466 		    rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])  in rcu_try_advance_all_cbs()
1467 			note_gp_changes(rsp, rdp);  in rcu_try_advance_all_cbs()
1469 		if (cpu_has_callbacks_ready_to_invoke(rdp))  in rcu_try_advance_all_cbs()
1530 	struct rcu_data *rdp;  in rcu_prepare_for_idle()  local
1572 		rdp = this_cpu_ptr(rsp->rda);  in rcu_prepare_for_idle()
1573 		if (!*rdp->nxttail[RCU_DONE_TAIL])  in rcu_prepare_for_idle()
1575 		rnp = rdp->mynode;  in rcu_prepare_for_idle()
1578 		needwake = rcu_accelerate_cbs(rsp, rnp, rdp);  in rcu_prepare_for_idle()
1640 	struct rcu_data *rdp;  in rcu_oom_notify_cpu()  local
1643 		rdp = raw_cpu_ptr(rsp->rda);  in rcu_oom_notify_cpu()
1644 		if (rdp->qlen_lazy != 0) {  in rcu_oom_notify_cpu()
1646 			rsp->call(&rdp->oom_head, rcu_oom_callback);  in rcu_oom_notify_cpu()
1745 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in print_cpu_stall_info()  local
1746 	struct rcu_dynticks *rdtp = rdp->dynticks;  in print_cpu_stall_info()
1750 	if (rsp->gpnum == rdp->gpnum) {  in print_cpu_stall_info()
1752 		ticks_value = rdp->ticks_this_gp;  in print_cpu_stall_info()
1755 		ticks_value = rsp->gpnum - rdp->gpnum;  in print_cpu_stall_info()
1762 	       rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),  in print_cpu_stall_info()
1774 static void zero_cpu_stall_ticks(struct rcu_data *rdp)  in zero_cpu_stall_ticks()  argument
1776 	rdp->ticks_this_gp = 0;  in zero_cpu_stall_ticks()
1777 	rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());  in zero_cpu_stall_ticks()
1806 static void zero_cpu_stall_ticks(struct rcu_data *rdp)  in zero_cpu_stall_ticks()  argument
1897 static void wake_nocb_leader(struct rcu_data *rdp, bool force)  in wake_nocb_leader()  argument
1899 	struct rcu_data *rdp_leader = rdp->nocb_leader;  in wake_nocb_leader()
1916 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_nocb_cpu_needs_barrier()  local
1934 	ret = atomic_long_read(&rdp->nocb_q_count);  in rcu_nocb_cpu_needs_barrier()
1937 	rhp = ACCESS_ONCE(rdp->nocb_head);  in rcu_nocb_cpu_needs_barrier()
1939 		rhp = ACCESS_ONCE(rdp->nocb_gp_head);  in rcu_nocb_cpu_needs_barrier()
1941 		rhp = ACCESS_ONCE(rdp->nocb_follower_head);  in rcu_nocb_cpu_needs_barrier()
1944 	if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp &&  in rcu_nocb_cpu_needs_barrier()
1964 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,  in __call_rcu_nocb_enqueue()  argument
1975 	atomic_long_add(rhcount, &rdp->nocb_q_count);  in __call_rcu_nocb_enqueue()
1977 	old_rhpp = xchg(&rdp->nocb_tail, rhtp);  in __call_rcu_nocb_enqueue()
1979 	atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);  in __call_rcu_nocb_enqueue()
1983 	t = ACCESS_ONCE(rdp->nocb_kthread);  in __call_rcu_nocb_enqueue()
1985 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1989 	len = atomic_long_read(&rdp->nocb_q_count);  in __call_rcu_nocb_enqueue()
1990 	if (old_rhpp == &rdp->nocb_head) {  in __call_rcu_nocb_enqueue()
1993 			wake_nocb_leader(rdp, false);  in __call_rcu_nocb_enqueue()
1994 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1997 			rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;  in __call_rcu_nocb_enqueue()
1998 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2001 		rdp->qlen_last_fqs_check = 0;  in __call_rcu_nocb_enqueue()
2002 	} else if (len > rdp->qlen_last_fqs_check + qhimark) {  in __call_rcu_nocb_enqueue()
2005 			wake_nocb_leader(rdp, true);  in __call_rcu_nocb_enqueue()
2006 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2009 			rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;  in __call_rcu_nocb_enqueue()
2010 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2013 		rdp->qlen_last_fqs_check = LONG_MAX / 2;  in __call_rcu_nocb_enqueue()
2015 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_enqueue()
2029 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,  in __call_rcu_nocb()  argument
2033 	if (!rcu_is_nocb_cpu(rdp->cpu))  in __call_rcu_nocb()
2035 	__call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);  in __call_rcu_nocb()
2037 		trace_rcu_kfree_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2039 					 -atomic_long_read(&rdp->nocb_q_count_lazy),  in __call_rcu_nocb()
2040 					 -atomic_long_read(&rdp->nocb_q_count));  in __call_rcu_nocb()
2042 		trace_rcu_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2043 				   -atomic_long_read(&rdp->nocb_q_count_lazy),  in __call_rcu_nocb()
2044 				   -atomic_long_read(&rdp->nocb_q_count));  in __call_rcu_nocb()
2064 						     struct rcu_data *rdp,  in rcu_nocb_adopt_orphan_cbs()  argument
2078 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,  in rcu_nocb_adopt_orphan_cbs()
2085 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,  in rcu_nocb_adopt_orphan_cbs()
2098 static void rcu_nocb_wait_gp(struct rcu_data *rdp)  in rcu_nocb_wait_gp()  argument
2104 	struct rcu_node *rnp = rdp->mynode;  in rcu_nocb_wait_gp()
2108 	needwake = rcu_start_future_gp(rnp, rdp, &c);  in rcu_nocb_wait_gp()
2111 		rcu_gp_kthread_wake(rdp->rsp);  in rcu_nocb_wait_gp()
2117 	trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));  in rcu_nocb_wait_gp()
2125 		trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));  in rcu_nocb_wait_gp()
2127 	trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));  in rcu_nocb_wait_gp()
2139 	struct rcu_data *rdp;  in nocb_leader_wait()  local
2161 	for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {  in nocb_leader_wait()
2162 		rdp->nocb_gp_head = ACCESS_ONCE(rdp->nocb_head);  in nocb_leader_wait()
2163 		if (!rdp->nocb_gp_head)  in nocb_leader_wait()
2167 		ACCESS_ONCE(rdp->nocb_head) = NULL;  in nocb_leader_wait()
2168 		rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);  in nocb_leader_wait()
2186 		for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)  in nocb_leader_wait()
2187 			if (ACCESS_ONCE(rdp->nocb_head)) {  in nocb_leader_wait()
2207 	for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {  in nocb_leader_wait()
2208 		if (ACCESS_ONCE(rdp->nocb_head))  in nocb_leader_wait()
2210 		if (!rdp->nocb_gp_head)  in nocb_leader_wait()
2214 		tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);  in nocb_leader_wait()
2215 		*tail = rdp->nocb_gp_head;  in nocb_leader_wait()
2217 		if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {  in nocb_leader_wait()
2222 			wake_up(&rdp->nocb_wq);  in nocb_leader_wait()
2235 static void nocb_follower_wait(struct rcu_data *rdp)  in nocb_follower_wait()  argument
2241 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2243 			wait_event_interruptible(rdp->nocb_wq,  in nocb_follower_wait()
2244 						 ACCESS_ONCE(rdp->nocb_follower_head));  in nocb_follower_wait()
2248 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");  in nocb_follower_wait()
2250 		if (smp_load_acquire(&rdp->nocb_follower_head)) {  in nocb_follower_wait()
2255 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2274 	struct rcu_data *rdp = arg;  in rcu_nocb_kthread()  local
2279 		if (rdp->nocb_leader == rdp)  in rcu_nocb_kthread()
2280 			nocb_leader_wait(rdp);  in rcu_nocb_kthread()
2282 			nocb_follower_wait(rdp);  in rcu_nocb_kthread()
2285 		list = ACCESS_ONCE(rdp->nocb_follower_head);  in rcu_nocb_kthread()
2287 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");  in rcu_nocb_kthread()
2288 		ACCESS_ONCE(rdp->nocb_follower_head) = NULL;  in rcu_nocb_kthread()
2289 		tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);  in rcu_nocb_kthread()
2292 		trace_rcu_batch_start(rdp->rsp->name,  in rcu_nocb_kthread()
2293 				      atomic_long_read(&rdp->nocb_q_count_lazy),  in rcu_nocb_kthread()
2294 				      atomic_long_read(&rdp->nocb_q_count), -1);  in rcu_nocb_kthread()
2300 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2303 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2309 			if (__rcu_reclaim(rdp->rsp->name, list))  in rcu_nocb_kthread()
2315 		trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);  in rcu_nocb_kthread()
2317 		atomic_long_add(-c, &rdp->nocb_q_count);  in rcu_nocb_kthread()
2318 		atomic_long_add(-cl, &rdp->nocb_q_count_lazy);  in rcu_nocb_kthread()
2319 		rdp->n_nocbs_invoked += c;  in rcu_nocb_kthread()
2325 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)  in rcu_nocb_need_deferred_wakeup()  argument
2327 	return ACCESS_ONCE(rdp->nocb_defer_wakeup);  in rcu_nocb_need_deferred_wakeup()
2331 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
2335 	if (!rcu_nocb_need_deferred_wakeup(rdp))  in do_nocb_deferred_wakeup()
2337 	ndw = ACCESS_ONCE(rdp->nocb_defer_wakeup);  in do_nocb_deferred_wakeup()
2338 	ACCESS_ONCE(rdp->nocb_defer_wakeup) = RCU_NOGP_WAKE_NOT;  in do_nocb_deferred_wakeup()
2339 	wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);  in do_nocb_deferred_wakeup()
2340 	trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));  in do_nocb_deferred_wakeup()
2399 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
2401 	rdp->nocb_tail = &rdp->nocb_head;  in rcu_boot_init_nocb_percpu_data()
2402 	init_waitqueue_head(&rdp->nocb_wq);  in rcu_boot_init_nocb_percpu_data()
2403 	rdp->nocb_follower_tail = &rdp->nocb_follower_head;  in rcu_boot_init_nocb_percpu_data()
2414 	struct rcu_data *rdp;  in rcu_spawn_one_nocb_kthread()  local
2431 		rdp = rdp_old_leader;  in rcu_spawn_one_nocb_kthread()
2433 			rdp->nocb_leader = rdp_spawn;  in rcu_spawn_one_nocb_kthread()
2434 			if (rdp_last && rdp != rdp_spawn)  in rcu_spawn_one_nocb_kthread()
2435 				rdp_last->nocb_next_follower = rdp;  in rcu_spawn_one_nocb_kthread()
2436 			if (rdp == rdp_spawn) {  in rcu_spawn_one_nocb_kthread()
2437 				rdp = rdp->nocb_next_follower;  in rcu_spawn_one_nocb_kthread()
2439 				rdp_last = rdp;  in rcu_spawn_one_nocb_kthread()
2440 				rdp = rdp->nocb_next_follower;  in rcu_spawn_one_nocb_kthread()
2443 		} while (rdp);  in rcu_spawn_one_nocb_kthread()
2493 	struct rcu_data *rdp;  in rcu_organize_nocb_kthreads()  local
2509 		rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_organize_nocb_kthreads()
2510 		if (rdp->cpu >= nl) {  in rcu_organize_nocb_kthreads()
2512 			nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;  in rcu_organize_nocb_kthreads()
2513 			rdp->nocb_leader = rdp;  in rcu_organize_nocb_kthreads()
2514 			rdp_leader = rdp;  in rcu_organize_nocb_kthreads()
2517 			rdp->nocb_leader = rdp_leader;  in rcu_organize_nocb_kthreads()
2518 			rdp_prev->nocb_next_follower = rdp;  in rcu_organize_nocb_kthreads()
2520 		rdp_prev = rdp;  in rcu_organize_nocb_kthreads()
2525 static bool init_nocb_callback_list(struct rcu_data *rdp)  in init_nocb_callback_list()  argument
2527 	if (!rcu_is_nocb_cpu(rdp->cpu))  in init_nocb_callback_list()
2531 	if (rdp->nxtlist) {  in init_nocb_callback_list()
2532 		rdp->nocb_head = rdp->nxtlist;  in init_nocb_callback_list()
2533 		rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];  in init_nocb_callback_list()
2534 		atomic_long_set(&rdp->nocb_q_count, rdp->qlen);  in init_nocb_callback_list()
2535 		atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);  in init_nocb_callback_list()
2536 		rdp->nxtlist = NULL;  in init_nocb_callback_list()
2537 		rdp->qlen = 0;  in init_nocb_callback_list()
2538 		rdp->qlen_lazy = 0;  in init_nocb_callback_list()
2540 	rdp->nxttail[RCU_NEXT_TAIL] = NULL;  in init_nocb_callback_list()
2564 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,  in __call_rcu_nocb()  argument
2571 						     struct rcu_data *rdp,  in rcu_nocb_adopt_orphan_cbs()  argument
2577 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
2581 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)  in rcu_nocb_need_deferred_wakeup()  argument
2586 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
2598 static bool init_nocb_callback_list(struct rcu_data *rdp)  in init_nocb_callback_list()  argument
2766 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,  in rcu_sysidle_check_cpu()  argument
2771 	struct rcu_dynticks *rdtp = rdp->dynticks;  in rcu_sysidle_check_cpu()
2782 	if (!*isidle || rdp->rsp != rcu_state_p ||  in rcu_sysidle_check_cpu()
2783 	    cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)  in rcu_sysidle_check_cpu()
2956 			struct rcu_data *rdp;  in rcu_sys_is_idle()  local
2960 				rdp = per_cpu_ptr(rcu_state_p->rda, cpu);  in rcu_sys_is_idle()
2961 				rcu_sysidle_check_cpu(rdp, &isidle, &maxj);  in rcu_sys_is_idle()
3016 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,  in rcu_sysidle_check_cpu()  argument