150 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp,  in rcu_preempt_ctxt_queue()  argument
155 			 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +  in rcu_preempt_ctxt_queue()
156 			 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);  in rcu_preempt_ctxt_queue()
250 		rcu_report_exp_rdp(rdp->rsp, rdp, true);  in rcu_preempt_ctxt_queue()
295 	struct rcu_data *rdp;  in rcu_preempt_note_context_switch()  local
302 		rdp = this_cpu_ptr(rcu_state_p->rda);  in rcu_preempt_note_context_switch()
303 		rnp = rdp->mynode;  in rcu_preempt_note_context_switch()
314 		WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);  in rcu_preempt_note_context_switch()
316 		trace_rcu_preempt_task(rdp->rsp->name,  in rcu_preempt_note_context_switch()
318 				       (rnp->qsmask & rdp->grpmask)  in rcu_preempt_note_context_switch()
321 		rcu_preempt_ctxt_queue(rnp, rdp, flags);  in rcu_preempt_note_context_switch()
391 	struct rcu_data *rdp;  in rcu_read_unlock_special()  local
426 		rdp = this_cpu_ptr(rcu_state_p->rda);  in rcu_read_unlock_special()
427 		rcu_report_exp_rdp(rcu_state_p, rdp, true);  in rcu_read_unlock_special()
704 	struct rcu_data *rdp;  in sync_rcu_exp_handler()  local
728 	rdp = this_cpu_ptr(rsp->rda);  in sync_rcu_exp_handler()
729 	rcu_report_exp_rdp(rsp, rdp, true);  in sync_rcu_exp_handler()
1305 	struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);  in rcu_prepare_kthreads()  local
1306 	struct rcu_node *rnp = rdp->mynode;  in rcu_prepare_kthreads()
1432 	struct rcu_data *rdp;  in rcu_try_advance_all_cbs()  local
1443 		rdp = this_cpu_ptr(rsp->rda);  in rcu_try_advance_all_cbs()
1444 		rnp = rdp->mynode;  in rcu_try_advance_all_cbs()
1451 		if ((rdp->completed != rnp->completed ||  in rcu_try_advance_all_cbs()
1452 		     unlikely(READ_ONCE(rdp->gpwrap))) &&  in rcu_try_advance_all_cbs()
1453 		    rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])  in rcu_try_advance_all_cbs()
1454 			note_gp_changes(rsp, rdp);  in rcu_try_advance_all_cbs()
1456 		if (cpu_has_callbacks_ready_to_invoke(rdp))  in rcu_try_advance_all_cbs()
1521 	struct rcu_data *rdp;  in rcu_prepare_for_idle()  local
1566 		rdp = this_cpu_ptr(rsp->rda);  in rcu_prepare_for_idle()
1567 		if (!*rdp->nxttail[RCU_DONE_TAIL])  in rcu_prepare_for_idle()
1569 		rnp = rdp->mynode;  in rcu_prepare_for_idle()
1572 		needwake = rcu_accelerate_cbs(rsp, rnp, rdp);  in rcu_prepare_for_idle()
1632 	struct rcu_data *rdp;  in rcu_oom_notify_cpu()  local
1635 		rdp = raw_cpu_ptr(rsp->rda);  in rcu_oom_notify_cpu()
1636 		if (rdp->qlen_lazy != 0) {  in rcu_oom_notify_cpu()
1638 			rsp->call(&rdp->oom_head, rcu_oom_callback);  in rcu_oom_notify_cpu()
1733 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in print_cpu_stall_info()  local
1734 	struct rcu_dynticks *rdtp = rdp->dynticks;  in print_cpu_stall_info()
1738 	if (rsp->gpnum == rdp->gpnum) {  in print_cpu_stall_info()
1740 		ticks_value = rdp->ticks_this_gp;  in print_cpu_stall_info()
1743 		ticks_value = rsp->gpnum - rdp->gpnum;  in print_cpu_stall_info()
1749 	       "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],  in print_cpu_stall_info()
1750 	       "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],  in print_cpu_stall_info()
1754 	       rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),  in print_cpu_stall_info()
1766 static void zero_cpu_stall_ticks(struct rcu_data *rdp)  in zero_cpu_stall_ticks()  argument
1768 	rdp->ticks_this_gp = 0;  in zero_cpu_stall_ticks()
1769 	rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());  in zero_cpu_stall_ticks()
1862 static void wake_nocb_leader(struct rcu_data *rdp, bool force)  in wake_nocb_leader()  argument
1864 	struct rcu_data *rdp_leader = rdp->nocb_leader;  in wake_nocb_leader()
1881 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_nocb_cpu_needs_barrier()  local
1899 	ret = atomic_long_read(&rdp->nocb_q_count);  in rcu_nocb_cpu_needs_barrier()
1902 	rhp = READ_ONCE(rdp->nocb_head);  in rcu_nocb_cpu_needs_barrier()
1904 		rhp = READ_ONCE(rdp->nocb_gp_head);  in rcu_nocb_cpu_needs_barrier()
1906 		rhp = READ_ONCE(rdp->nocb_follower_head);  in rcu_nocb_cpu_needs_barrier()
1909 	if (!READ_ONCE(rdp->nocb_kthread) && rhp &&  in rcu_nocb_cpu_needs_barrier()
1929 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,  in __call_rcu_nocb_enqueue()  argument
1940 	atomic_long_add(rhcount, &rdp->nocb_q_count);  in __call_rcu_nocb_enqueue()
1942 	old_rhpp = xchg(&rdp->nocb_tail, rhtp);  in __call_rcu_nocb_enqueue()
1944 	atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);  in __call_rcu_nocb_enqueue()
1948 	t = READ_ONCE(rdp->nocb_kthread);  in __call_rcu_nocb_enqueue()
1950 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1954 	len = atomic_long_read(&rdp->nocb_q_count);  in __call_rcu_nocb_enqueue()
1955 	if (old_rhpp == &rdp->nocb_head) {  in __call_rcu_nocb_enqueue()
1958 			wake_nocb_leader(rdp, false);  in __call_rcu_nocb_enqueue()
1959 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1962 			rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;  in __call_rcu_nocb_enqueue()
1963 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1966 		rdp->qlen_last_fqs_check = 0;  in __call_rcu_nocb_enqueue()
1967 	} else if (len > rdp->qlen_last_fqs_check + qhimark) {  in __call_rcu_nocb_enqueue()
1970 			wake_nocb_leader(rdp, true);  in __call_rcu_nocb_enqueue()
1971 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1974 			rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;  in __call_rcu_nocb_enqueue()
1975 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
1978 		rdp->qlen_last_fqs_check = LONG_MAX / 2;  in __call_rcu_nocb_enqueue()
1980 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_enqueue()
1994 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,  in __call_rcu_nocb()  argument
1998 	if (!rcu_is_nocb_cpu(rdp->cpu))  in __call_rcu_nocb()
2000 	__call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);  in __call_rcu_nocb()
2002 		trace_rcu_kfree_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2004 					 -atomic_long_read(&rdp->nocb_q_count_lazy),  in __call_rcu_nocb()
2005 					 -atomic_long_read(&rdp->nocb_q_count));  in __call_rcu_nocb()
2007 		trace_rcu_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2008 				   -atomic_long_read(&rdp->nocb_q_count_lazy),  in __call_rcu_nocb()
2009 				   -atomic_long_read(&rdp->nocb_q_count));  in __call_rcu_nocb()
2029 						     struct rcu_data *rdp,  in rcu_nocb_adopt_orphan_cbs()  argument
2043 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,  in rcu_nocb_adopt_orphan_cbs()
2050 		__call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,  in rcu_nocb_adopt_orphan_cbs()
2063 static void rcu_nocb_wait_gp(struct rcu_data *rdp)  in rcu_nocb_wait_gp()  argument
2069 	struct rcu_node *rnp = rdp->mynode;  in rcu_nocb_wait_gp()
2073 	needwake = rcu_start_future_gp(rnp, rdp, &c);  in rcu_nocb_wait_gp()
2076 		rcu_gp_kthread_wake(rdp->rsp);  in rcu_nocb_wait_gp()
2082 	trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));  in rcu_nocb_wait_gp()
2090 		trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));  in rcu_nocb_wait_gp()
2092 	trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));  in rcu_nocb_wait_gp()
2104 	struct rcu_data *rdp;  in nocb_leader_wait()  local
2126 	for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {  in nocb_leader_wait()
2127 		rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);  in nocb_leader_wait()
2128 		if (!rdp->nocb_gp_head)  in nocb_leader_wait()
2132 		WRITE_ONCE(rdp->nocb_head, NULL);  in nocb_leader_wait()
2133 		rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);  in nocb_leader_wait()
2151 		for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)  in nocb_leader_wait()
2152 			if (READ_ONCE(rdp->nocb_head)) {  in nocb_leader_wait()
2172 	for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {  in nocb_leader_wait()
2173 		if (READ_ONCE(rdp->nocb_head))  in nocb_leader_wait()
2175 		if (!rdp->nocb_gp_head)  in nocb_leader_wait()
2179 		tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);  in nocb_leader_wait()
2180 		*tail = rdp->nocb_gp_head;  in nocb_leader_wait()
2182 		if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {  in nocb_leader_wait()
2187 			wake_up(&rdp->nocb_wq);  in nocb_leader_wait()
2200 static void nocb_follower_wait(struct rcu_data *rdp)  in nocb_follower_wait()  argument
2206 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2208 			wait_event_interruptible(rdp->nocb_wq,  in nocb_follower_wait()
2209 						 READ_ONCE(rdp->nocb_follower_head));  in nocb_follower_wait()
2213 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");  in nocb_follower_wait()
2215 		if (smp_load_acquire(&rdp->nocb_follower_head)) {  in nocb_follower_wait()
2220 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in nocb_follower_wait()
2239 	struct rcu_data *rdp = arg;  in rcu_nocb_kthread()  local
2244 		if (rdp->nocb_leader == rdp)  in rcu_nocb_kthread()
2245 			nocb_leader_wait(rdp);  in rcu_nocb_kthread()
2247 			nocb_follower_wait(rdp);  in rcu_nocb_kthread()
2250 		list = READ_ONCE(rdp->nocb_follower_head);  in rcu_nocb_kthread()
2252 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");  in rcu_nocb_kthread()
2253 		WRITE_ONCE(rdp->nocb_follower_head, NULL);  in rcu_nocb_kthread()
2254 		tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);  in rcu_nocb_kthread()
2257 		trace_rcu_batch_start(rdp->rsp->name,  in rcu_nocb_kthread()
2258 				      atomic_long_read(&rdp->nocb_q_count_lazy),  in rcu_nocb_kthread()
2259 				      atomic_long_read(&rdp->nocb_q_count), -1);  in rcu_nocb_kthread()
2265 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2268 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2274 			if (__rcu_reclaim(rdp->rsp->name, list))  in rcu_nocb_kthread()
2280 		trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);  in rcu_nocb_kthread()
2282 		atomic_long_add(-c, &rdp->nocb_q_count);  in rcu_nocb_kthread()
2283 		atomic_long_add(-cl, &rdp->nocb_q_count_lazy);  in rcu_nocb_kthread()
2284 		rdp->n_nocbs_invoked += c;  in rcu_nocb_kthread()
2290 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)  in rcu_nocb_need_deferred_wakeup()  argument
2292 	return READ_ONCE(rdp->nocb_defer_wakeup);  in rcu_nocb_need_deferred_wakeup()
2296 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
2300 	if (!rcu_nocb_need_deferred_wakeup(rdp))  in do_nocb_deferred_wakeup()
2302 	ndw = READ_ONCE(rdp->nocb_defer_wakeup);  in do_nocb_deferred_wakeup()
2303 	WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_NOT);  in do_nocb_deferred_wakeup()
2304 	wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);  in do_nocb_deferred_wakeup()
2305 	trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));  in do_nocb_deferred_wakeup()
2364 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
2366 	rdp->nocb_tail = &rdp->nocb_head;  in rcu_boot_init_nocb_percpu_data()
2367 	init_waitqueue_head(&rdp->nocb_wq);  in rcu_boot_init_nocb_percpu_data()
2368 	rdp->nocb_follower_tail = &rdp->nocb_follower_head;  in rcu_boot_init_nocb_percpu_data()
2379 	struct rcu_data *rdp;  in rcu_spawn_one_nocb_kthread()  local
2396 		rdp = rdp_old_leader;  in rcu_spawn_one_nocb_kthread()
2398 			rdp->nocb_leader = rdp_spawn;  in rcu_spawn_one_nocb_kthread()
2399 			if (rdp_last && rdp != rdp_spawn)  in rcu_spawn_one_nocb_kthread()
2400 				rdp_last->nocb_next_follower = rdp;  in rcu_spawn_one_nocb_kthread()
2401 			if (rdp == rdp_spawn) {  in rcu_spawn_one_nocb_kthread()
2402 				rdp = rdp->nocb_next_follower;  in rcu_spawn_one_nocb_kthread()
2404 				rdp_last = rdp;  in rcu_spawn_one_nocb_kthread()
2405 				rdp = rdp->nocb_next_follower;  in rcu_spawn_one_nocb_kthread()
2408 		} while (rdp);  in rcu_spawn_one_nocb_kthread()
2458 	struct rcu_data *rdp;  in rcu_organize_nocb_kthreads()  local
2474 		rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_organize_nocb_kthreads()
2475 		if (rdp->cpu >= nl) {  in rcu_organize_nocb_kthreads()
2477 			nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;  in rcu_organize_nocb_kthreads()
2478 			rdp->nocb_leader = rdp;  in rcu_organize_nocb_kthreads()
2479 			rdp_leader = rdp;  in rcu_organize_nocb_kthreads()
2482 			rdp->nocb_leader = rdp_leader;  in rcu_organize_nocb_kthreads()
2483 			rdp_prev->nocb_next_follower = rdp;  in rcu_organize_nocb_kthreads()
2485 		rdp_prev = rdp;  in rcu_organize_nocb_kthreads()
2490 static bool init_nocb_callback_list(struct rcu_data *rdp)  in init_nocb_callback_list()  argument
2492 	if (!rcu_is_nocb_cpu(rdp->cpu))  in init_nocb_callback_list()
2496 	if (rdp->nxtlist) {  in init_nocb_callback_list()
2497 		rdp->nocb_head = rdp->nxtlist;  in init_nocb_callback_list()
2498 		rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];  in init_nocb_callback_list()
2499 		atomic_long_set(&rdp->nocb_q_count, rdp->qlen);  in init_nocb_callback_list()
2500 		atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);  in init_nocb_callback_list()
2501 		rdp->nxtlist = NULL;  in init_nocb_callback_list()
2502 		rdp->qlen = 0;  in init_nocb_callback_list()
2503 		rdp->qlen_lazy = 0;  in init_nocb_callback_list()
2505 	rdp->nxttail[RCU_NEXT_TAIL] = NULL;  in init_nocb_callback_list()
2529 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,  in __call_rcu_nocb()  argument
2536 						     struct rcu_data *rdp,  in rcu_nocb_adopt_orphan_cbs()  argument
2542 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
2546 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)  in rcu_nocb_need_deferred_wakeup()  argument
2551 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
2563 static bool init_nocb_callback_list(struct rcu_data *rdp)  in init_nocb_callback_list()  argument
2731 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,  in rcu_sysidle_check_cpu()  argument
2736 	struct rcu_dynticks *rdtp = rdp->dynticks;  in rcu_sysidle_check_cpu()
2747 	if (!*isidle || rdp->rsp != rcu_state_p ||  in rcu_sysidle_check_cpu()
2748 	    cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)  in rcu_sysidle_check_cpu()
2921 			struct rcu_data *rdp;  in rcu_sys_is_idle()  local
2925 				rdp = per_cpu_ptr(rcu_state_p->rda, cpu);  in rcu_sys_is_idle()
2926 				rcu_sysidle_check_cpu(rdp, &isidle, &maxj);  in rcu_sys_is_idle()
2981 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,  in rcu_sysidle_check_cpu()  argument