Lines Matching refs:rdp
159 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
261 struct rcu_data *rdp; in rcu_momentary_dyntick_idle() local
277 rdp = raw_cpu_ptr(rsp->rda); in rcu_momentary_dyntick_idle()
281 if (ACCESS_ONCE(rdp->mynode->completed) != in rcu_momentary_dyntick_idle()
282 ACCESS_ONCE(rdp->cond_resched_completed)) in rcu_momentary_dyntick_idle()
354 struct rcu_data *rdp);
520 cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) in cpu_has_callbacks_ready_to_invoke() argument
522 return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] && in cpu_has_callbacks_ready_to_invoke()
523 rdp->nxttail[RCU_DONE_TAIL] != NULL; in cpu_has_callbacks_ready_to_invoke()
554 cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) in cpu_needs_another_gp() argument
562 if (!rdp->nxttail[RCU_NEXT_TAIL]) in cpu_needs_another_gp()
564 if (*rdp->nxttail[RCU_NEXT_READY_TAIL]) in cpu_needs_another_gp()
567 if (rdp->nxttail[i - 1] != rdp->nxttail[i] && in cpu_needs_another_gp()
569 rdp->nxtcompleted[i])) in cpu_needs_another_gp()
584 struct rcu_data *rdp; in rcu_eqs_enter_common() local
599 rdp = this_cpu_ptr(rsp->rda); in rcu_eqs_enter_common()
600 do_nocb_deferred_wakeup(rdp); in rcu_eqs_enter_common()
970 struct rcu_data *rdp; in rcu_lockdep_current_cpu_online() local
977 rdp = this_cpu_ptr(&rcu_sched_data); in rcu_lockdep_current_cpu_online()
978 rnp = rdp->mynode; in rcu_lockdep_current_cpu_online()
979 ret = (rdp->grpmask & rcu_rnp_online_cpus(rnp)) || in rcu_lockdep_current_cpu_online()
1005 static int dyntick_save_progress_counter(struct rcu_data *rdp, in dyntick_save_progress_counter() argument
1008 rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks); in dyntick_save_progress_counter()
1009 rcu_sysidle_check_cpu(rdp, isidle, maxj); in dyntick_save_progress_counter()
1010 if ((rdp->dynticks_snap & 0x1) == 0) { in dyntick_save_progress_counter()
1011 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
1014 if (ULONG_CMP_LT(ACCESS_ONCE(rdp->gpnum) + ULONG_MAX / 4, in dyntick_save_progress_counter()
1015 rdp->mynode->gpnum)) in dyntick_save_progress_counter()
1016 ACCESS_ONCE(rdp->gpwrap) = true; in dyntick_save_progress_counter()
1027 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp, in rcu_implicit_dynticks_qs() argument
1034 curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks); in rcu_implicit_dynticks_qs()
1035 snap = (unsigned int)rdp->dynticks_snap; in rcu_implicit_dynticks_qs()
1046 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
1047 rdp->dynticks_fqs++; in rcu_implicit_dynticks_qs()
1062 if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies)) in rcu_implicit_dynticks_qs()
1065 if (cpu_is_offline(rdp->cpu)) { in rcu_implicit_dynticks_qs()
1066 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl")); in rcu_implicit_dynticks_qs()
1067 rdp->offline_fqs++; in rcu_implicit_dynticks_qs()
1092 rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu); in rcu_implicit_dynticks_qs()
1094 rdp->rsp->gp_start + jiffies_till_sched_qs) || in rcu_implicit_dynticks_qs()
1095 ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { in rcu_implicit_dynticks_qs()
1096 if (!(ACCESS_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) { in rcu_implicit_dynticks_qs()
1097 ACCESS_ONCE(rdp->cond_resched_completed) = in rcu_implicit_dynticks_qs()
1098 ACCESS_ONCE(rdp->mynode->completed); in rcu_implicit_dynticks_qs()
1101 ACCESS_ONCE(*rcrmp) + rdp->rsp->flavor_mask; in rcu_implicit_dynticks_qs()
1102 resched_cpu(rdp->cpu); /* Force CPU into scheduler. */ in rcu_implicit_dynticks_qs()
1103 rdp->rsp->jiffies_resched += 5; /* Enable beating. */ in rcu_implicit_dynticks_qs()
1104 } else if (ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { in rcu_implicit_dynticks_qs()
1106 resched_cpu(rdp->cpu); /* Force CPU into scheduler. */ in rcu_implicit_dynticks_qs()
1107 rdp->rsp->jiffies_resched += 5; /* Re-enable beating. */ in rcu_implicit_dynticks_qs()
1280 static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) in check_cpu_stall() argument
1321 rnp = rdp->mynode; in check_cpu_stall()
1323 (ACCESS_ONCE(rnp->qsmask) & rdp->grpmask)) { in check_cpu_stall()
1358 static void init_default_callback_list(struct rcu_data *rdp) in init_default_callback_list() argument
1362 rdp->nxtlist = NULL; in init_default_callback_list()
1364 rdp->nxttail[i] = &rdp->nxtlist; in init_default_callback_list()
1370 static void init_callback_list(struct rcu_data *rdp) in init_callback_list() argument
1372 if (init_nocb_callback_list(rdp)) in init_callback_list()
1374 init_default_callback_list(rdp); in init_callback_list()
1410 static void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_future_gp() argument
1413 trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum, in trace_rcu_future_gp()
1427 rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, in rcu_start_future_gp() argument
1433 struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); in rcu_start_future_gp()
1439 c = rcu_cbs_completed(rdp->rsp, rnp); in rcu_start_future_gp()
1440 trace_rcu_future_gp(rnp, rdp, c, TPS("Startleaf")); in rcu_start_future_gp()
1442 trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartleaf")); in rcu_start_future_gp()
1462 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf")); in rcu_start_future_gp()
1482 c = rcu_cbs_completed(rdp->rsp, rnp_root); in rcu_start_future_gp()
1484 if (ULONG_CMP_LT(c, rdp->nxtcompleted[i])) in rcu_start_future_gp()
1485 rdp->nxtcompleted[i] = c; in rcu_start_future_gp()
1492 trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartedroot")); in rcu_start_future_gp()
1501 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleafroot")); in rcu_start_future_gp()
1503 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedroot")); in rcu_start_future_gp()
1504 ret = rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp); in rcu_start_future_gp()
1525 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in rcu_future_gp_cleanup() local
1530 trace_rcu_future_gp(rnp, rdp, c, in rcu_future_gp_cleanup()
1564 struct rcu_data *rdp) in rcu_accelerate_cbs() argument
1571 if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL]) in rcu_accelerate_cbs()
1590 if (rdp->nxttail[i] != rdp->nxttail[i - 1] && in rcu_accelerate_cbs()
1591 !ULONG_CMP_GE(rdp->nxtcompleted[i], c)) in rcu_accelerate_cbs()
1609 rdp->nxttail[i] = rdp->nxttail[RCU_NEXT_TAIL]; in rcu_accelerate_cbs()
1610 rdp->nxtcompleted[i] = c; in rcu_accelerate_cbs()
1613 ret = rcu_start_future_gp(rnp, rdp, NULL); in rcu_accelerate_cbs()
1616 if (!*rdp->nxttail[RCU_WAIT_TAIL]) in rcu_accelerate_cbs()
1617 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1619 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1634 struct rcu_data *rdp) in rcu_advance_cbs() argument
1639 if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL]) in rcu_advance_cbs()
1647 if (ULONG_CMP_LT(rnp->completed, rdp->nxtcompleted[i])) in rcu_advance_cbs()
1649 rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[i]; in rcu_advance_cbs()
1653 rdp->nxttail[j] = rdp->nxttail[RCU_DONE_TAIL]; in rcu_advance_cbs()
1657 if (rdp->nxttail[j] == rdp->nxttail[RCU_NEXT_TAIL]) in rcu_advance_cbs()
1659 rdp->nxttail[j] = rdp->nxttail[i]; in rcu_advance_cbs()
1660 rdp->nxtcompleted[j] = rdp->nxtcompleted[i]; in rcu_advance_cbs()
1664 return rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_advance_cbs()
1674 struct rcu_data *rdp) in __note_gp_changes() argument
1679 if (rdp->completed == rnp->completed && in __note_gp_changes()
1680 !unlikely(ACCESS_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1683 ret = rcu_accelerate_cbs(rsp, rnp, rdp); in __note_gp_changes()
1688 ret = rcu_advance_cbs(rsp, rnp, rdp); in __note_gp_changes()
1691 rdp->completed = rnp->completed; in __note_gp_changes()
1692 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend")); in __note_gp_changes()
1695 if (rdp->gpnum != rnp->gpnum || unlikely(ACCESS_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1701 rdp->gpnum = rnp->gpnum; in __note_gp_changes()
1702 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); in __note_gp_changes()
1703 rdp->passed_quiesce = 0; in __note_gp_changes()
1704 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); in __note_gp_changes()
1705 rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1706 zero_cpu_stall_ticks(rdp); in __note_gp_changes()
1707 ACCESS_ONCE(rdp->gpwrap) = false; in __note_gp_changes()
1712 static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp) in note_gp_changes() argument
1719 rnp = rdp->mynode; in note_gp_changes()
1720 if ((rdp->gpnum == ACCESS_ONCE(rnp->gpnum) && in note_gp_changes()
1721 rdp->completed == ACCESS_ONCE(rnp->completed) && in note_gp_changes()
1722 !unlikely(ACCESS_ONCE(rdp->gpwrap))) || /* w/out lock. */ in note_gp_changes()
1728 needwake = __note_gp_changes(rsp, rnp, rdp); in note_gp_changes()
1740 struct rcu_data *rdp; in rcu_gp_init() local
1834 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_init()
1840 if (rnp == rdp->mynode) in rcu_gp_init()
1841 (void)__note_gp_changes(rsp, rnp, rdp); in rcu_gp_init()
1903 struct rcu_data *rdp; in rcu_gp_cleanup() local
1938 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
1939 if (rnp == rdp->mynode) in rcu_gp_cleanup()
1940 needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; in rcu_gp_cleanup()
1956 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
1958 needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp; in rcu_gp_cleanup()
1959 if (needgp || cpu_needs_another_gp(rsp, rdp)) { in rcu_gp_cleanup()
2078 struct rcu_data *rdp) in rcu_start_gp_advanced() argument
2080 if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) { in rcu_start_gp_advanced()
2112 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in rcu_start_gp() local
2124 ret = rcu_advance_cbs(rsp, rnp, rdp) || ret; in rcu_start_gp()
2125 ret = rcu_start_gp_advanced(rsp, rnp, rdp) || ret; in rcu_start_gp()
2258 rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) in rcu_report_qs_rdp() argument
2265 rnp = rdp->mynode; in rcu_report_qs_rdp()
2268 if ((rdp->passed_quiesce == 0 && in rcu_report_qs_rdp()
2269 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) || in rcu_report_qs_rdp()
2270 rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum || in rcu_report_qs_rdp()
2271 rdp->gpwrap) { in rcu_report_qs_rdp()
2279 rdp->passed_quiesce = 0; /* need qs for new gp. */ in rcu_report_qs_rdp()
2280 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); in rcu_report_qs_rdp()
2284 mask = rdp->grpmask; in rcu_report_qs_rdp()
2288 rdp->qs_pending = 0; in rcu_report_qs_rdp()
2294 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_report_qs_rdp()
2310 rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_check_quiescent_state() argument
2313 note_gp_changes(rsp, rdp); in rcu_check_quiescent_state()
2319 if (!rdp->qs_pending) in rcu_check_quiescent_state()
2326 if (!rdp->passed_quiesce && in rcu_check_quiescent_state()
2327 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) in rcu_check_quiescent_state()
2334 rcu_report_qs_rdp(rdp->cpu, rsp, rdp); in rcu_check_quiescent_state()
2346 struct rcu_node *rnp, struct rcu_data *rdp) in rcu_send_cbs_to_orphanage() argument
2349 if (rcu_is_nocb_cpu(rdp->cpu)) in rcu_send_cbs_to_orphanage()
2357 if (rdp->nxtlist != NULL) { in rcu_send_cbs_to_orphanage()
2358 rsp->qlen_lazy += rdp->qlen_lazy; in rcu_send_cbs_to_orphanage()
2359 rsp->qlen += rdp->qlen; in rcu_send_cbs_to_orphanage()
2360 rdp->n_cbs_orphaned += rdp->qlen; in rcu_send_cbs_to_orphanage()
2361 rdp->qlen_lazy = 0; in rcu_send_cbs_to_orphanage()
2362 ACCESS_ONCE(rdp->qlen) = 0; in rcu_send_cbs_to_orphanage()
2374 if (*rdp->nxttail[RCU_DONE_TAIL] != NULL) { in rcu_send_cbs_to_orphanage()
2375 *rsp->orphan_nxttail = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_send_cbs_to_orphanage()
2376 rsp->orphan_nxttail = rdp->nxttail[RCU_NEXT_TAIL]; in rcu_send_cbs_to_orphanage()
2377 *rdp->nxttail[RCU_DONE_TAIL] = NULL; in rcu_send_cbs_to_orphanage()
2385 if (rdp->nxtlist != NULL) { in rcu_send_cbs_to_orphanage()
2386 *rsp->orphan_donetail = rdp->nxtlist; in rcu_send_cbs_to_orphanage()
2387 rsp->orphan_donetail = rdp->nxttail[RCU_DONE_TAIL]; in rcu_send_cbs_to_orphanage()
2394 init_callback_list(rdp); in rcu_send_cbs_to_orphanage()
2395 rdp->nxttail[RCU_NEXT_TAIL] = NULL; in rcu_send_cbs_to_orphanage()
2405 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in rcu_adopt_orphan_cbs() local
2408 if (rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags)) in rcu_adopt_orphan_cbs()
2412 rdp->qlen_lazy += rsp->qlen_lazy; in rcu_adopt_orphan_cbs()
2413 rdp->qlen += rsp->qlen; in rcu_adopt_orphan_cbs()
2414 rdp->n_cbs_adopted += rsp->qlen; in rcu_adopt_orphan_cbs()
2428 *rsp->orphan_donetail = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_adopt_orphan_cbs()
2429 *rdp->nxttail[RCU_DONE_TAIL] = rsp->orphan_donelist; in rcu_adopt_orphan_cbs()
2431 if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL]) in rcu_adopt_orphan_cbs()
2432 rdp->nxttail[i] = rsp->orphan_donetail; in rcu_adopt_orphan_cbs()
2439 *rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxtlist; in rcu_adopt_orphan_cbs()
2440 rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxttail; in rcu_adopt_orphan_cbs()
2452 RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda)); in rcu_cleanup_dying_cpu()
2453 RCU_TRACE(struct rcu_node *rnp = rdp->mynode); in rcu_cleanup_dying_cpu()
2455 RCU_TRACE(mask = rdp->grpmask); in rcu_cleanup_dying_cpu()
2511 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dying_idle_cpu() local
2512 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcu_cleanup_dying_idle_cpu()
2515 mask = rdp->grpmask; in rcu_cleanup_dying_idle_cpu()
2532 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dead_cpu() local
2533 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcu_cleanup_dead_cpu()
2540 rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp); in rcu_cleanup_dead_cpu()
2544 WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL, in rcu_cleanup_dead_cpu()
2546 cpu, rdp->qlen, rdp->nxtlist); in rcu_cleanup_dead_cpu()
2573 static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_do_batch() argument
2581 if (!cpu_has_callbacks_ready_to_invoke(rdp)) { in rcu_do_batch()
2582 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0); in rcu_do_batch()
2583 trace_rcu_batch_end(rsp->name, 0, !!ACCESS_ONCE(rdp->nxtlist), in rcu_do_batch()
2595 bl = rdp->blimit; in rcu_do_batch()
2596 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, bl); in rcu_do_batch()
2597 list = rdp->nxtlist; in rcu_do_batch()
2598 rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_do_batch()
2599 *rdp->nxttail[RCU_DONE_TAIL] = NULL; in rcu_do_batch()
2600 tail = rdp->nxttail[RCU_DONE_TAIL]; in rcu_do_batch()
2602 if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL]) in rcu_do_batch()
2603 rdp->nxttail[i] = &rdp->nxtlist; in rcu_do_batch()
2629 *tail = rdp->nxtlist; in rcu_do_batch()
2630 rdp->nxtlist = list; in rcu_do_batch()
2632 if (&rdp->nxtlist == rdp->nxttail[i]) in rcu_do_batch()
2633 rdp->nxttail[i] = tail; in rcu_do_batch()
2638 rdp->qlen_lazy -= count_lazy; in rcu_do_batch()
2639 ACCESS_ONCE(rdp->qlen) = rdp->qlen - count; in rcu_do_batch()
2640 rdp->n_cbs_invoked += count; in rcu_do_batch()
2643 if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark) in rcu_do_batch()
2644 rdp->blimit = blimit; in rcu_do_batch()
2647 if (rdp->qlen == 0 && rdp->qlen_last_fqs_check != 0) { in rcu_do_batch()
2648 rdp->qlen_last_fqs_check = 0; in rcu_do_batch()
2649 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_do_batch()
2650 } else if (rdp->qlen < rdp->qlen_last_fqs_check - qhimark) in rcu_do_batch()
2651 rdp->qlen_last_fqs_check = rdp->qlen; in rcu_do_batch()
2652 WARN_ON_ONCE((rdp->nxtlist == NULL) != (rdp->qlen == 0)); in rcu_do_batch()
2657 if (cpu_has_callbacks_ready_to_invoke(rdp)) in rcu_do_batch()
2833 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in __rcu_process_callbacks() local
2835 WARN_ON_ONCE(rdp->beenonline == 0); in __rcu_process_callbacks()
2838 rcu_check_quiescent_state(rsp, rdp); in __rcu_process_callbacks()
2842 if (cpu_needs_another_gp(rsp, rdp)) { in __rcu_process_callbacks()
2853 if (cpu_has_callbacks_ready_to_invoke(rdp)) in __rcu_process_callbacks()
2854 invoke_rcu_callbacks(rsp, rdp); in __rcu_process_callbacks()
2857 do_nocb_deferred_wakeup(rdp); in __rcu_process_callbacks()
2882 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) in invoke_rcu_callbacks() argument
2887 rcu_do_batch(rsp, rdp); in invoke_rcu_callbacks()
2902 static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, in __call_rcu_core() argument
2925 if (unlikely(rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) { in __call_rcu_core()
2928 note_gp_changes(rsp, rdp); in __call_rcu_core()
2942 rdp->blimit = LONG_MAX; in __call_rcu_core()
2943 if (rsp->n_force_qs == rdp->n_force_qs_snap && in __call_rcu_core()
2944 *rdp->nxttail[RCU_DONE_TAIL] != head) in __call_rcu_core()
2946 rdp->n_force_qs_snap = rsp->n_force_qs; in __call_rcu_core()
2947 rdp->qlen_last_fqs_check = rdp->qlen; in __call_rcu_core()
2970 struct rcu_data *rdp; in __call_rcu() local
2989 rdp = this_cpu_ptr(rsp->rda); in __call_rcu()
2992 if (unlikely(rdp->nxttail[RCU_NEXT_TAIL] == NULL) || cpu != -1) { in __call_rcu()
2996 rdp = per_cpu_ptr(rsp->rda, cpu); in __call_rcu()
2997 if (likely(rdp->mynode)) { in __call_rcu()
2999 offline = !__call_rcu_nocb(rdp, head, lazy, flags); in __call_rcu()
3011 if (!likely(rdp->nxtlist)) in __call_rcu()
3012 init_default_callback_list(rdp); in __call_rcu()
3014 ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1; in __call_rcu()
3016 rdp->qlen_lazy++; in __call_rcu()
3020 *rdp->nxttail[RCU_NEXT_TAIL] = head; in __call_rcu()
3021 rdp->nxttail[RCU_NEXT_TAIL] = &head->next; in __call_rcu()
3025 rdp->qlen_lazy, rdp->qlen); in __call_rcu()
3027 trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen); in __call_rcu()
3030 __call_rcu_core(rsp, rdp, head, flags); in __call_rcu()
3415 static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) in __rcu_pending() argument
3417 struct rcu_node *rnp = rdp->mynode; in __rcu_pending()
3419 rdp->n_rcu_pending++; in __rcu_pending()
3422 check_cpu_stall(rsp, rdp); in __rcu_pending()
3430 rdp->qs_pending && !rdp->passed_quiesce && in __rcu_pending()
3431 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) { in __rcu_pending()
3432 rdp->n_rp_qs_pending++; in __rcu_pending()
3433 } else if (rdp->qs_pending && in __rcu_pending()
3434 (rdp->passed_quiesce || in __rcu_pending()
3435 rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) { in __rcu_pending()
3436 rdp->n_rp_report_qs++; in __rcu_pending()
3441 if (cpu_has_callbacks_ready_to_invoke(rdp)) { in __rcu_pending()
3442 rdp->n_rp_cb_ready++; in __rcu_pending()
3447 if (cpu_needs_another_gp(rsp, rdp)) { in __rcu_pending()
3448 rdp->n_rp_cpu_needs_gp++; in __rcu_pending()
3453 if (ACCESS_ONCE(rnp->completed) != rdp->completed) { /* outside lock */ in __rcu_pending()
3454 rdp->n_rp_gp_completed++; in __rcu_pending()
3459 if (ACCESS_ONCE(rnp->gpnum) != rdp->gpnum || in __rcu_pending()
3460 unlikely(ACCESS_ONCE(rdp->gpwrap))) { /* outside lock */ in __rcu_pending()
3461 rdp->n_rp_gp_started++; in __rcu_pending()
3466 if (rcu_nocb_need_deferred_wakeup(rdp)) { in __rcu_pending()
3467 rdp->n_rp_nocb_defer_wakeup++; in __rcu_pending()
3472 rdp->n_rp_need_nothing++; in __rcu_pending()
3500 struct rcu_data *rdp; in rcu_cpu_has_callbacks() local
3504 rdp = this_cpu_ptr(rsp->rda); in rcu_cpu_has_callbacks()
3505 if (!rdp->nxtlist) in rcu_cpu_has_callbacks()
3508 if (rdp->qlen != rdp->qlen_lazy || !all_lazy) { in rcu_cpu_has_callbacks()
3535 struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head); in rcu_barrier_callback() local
3536 struct rcu_state *rsp = rdp->rsp; in rcu_barrier_callback()
3552 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in rcu_barrier_func() local
3556 rsp->call(&rdp->barrier_head, rcu_barrier_callback); in rcu_barrier_func()
3566 struct rcu_data *rdp; in _rcu_barrier() local
3635 rdp = per_cpu_ptr(rsp->rda, cpu); in _rcu_barrier()
3645 __call_rcu(&rdp->barrier_head, in _rcu_barrier()
3648 } else if (ACCESS_ONCE(rdp->qlen)) { in _rcu_barrier()
3727 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_boot_init_percpu_data() local
3732 rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo); in rcu_boot_init_percpu_data()
3733 rdp->dynticks = &per_cpu(rcu_dynticks, cpu); in rcu_boot_init_percpu_data()
3734 WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE); in rcu_boot_init_percpu_data()
3735 WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1); in rcu_boot_init_percpu_data()
3736 rdp->cpu = cpu; in rcu_boot_init_percpu_data()
3737 rdp->rsp = rsp; in rcu_boot_init_percpu_data()
3738 rcu_boot_init_nocb_percpu_data(rdp); in rcu_boot_init_percpu_data()
3753 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_init_percpu_data() local
3758 rdp->beenonline = 1; /* We have now been online. */ in rcu_init_percpu_data()
3759 rdp->qlen_last_fqs_check = 0; in rcu_init_percpu_data()
3760 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_init_percpu_data()
3761 rdp->blimit = blimit; in rcu_init_percpu_data()
3762 if (!rdp->nxtlist) in rcu_init_percpu_data()
3763 init_callback_list(rdp); /* Re-enable callbacks on this CPU. */ in rcu_init_percpu_data()
3764 rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE; in rcu_init_percpu_data()
3765 rcu_sysidle_init_percpu_data(rdp->dynticks); in rcu_init_percpu_data()
3766 atomic_set(&rdp->dynticks->dynticks, in rcu_init_percpu_data()
3767 (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1); in rcu_init_percpu_data()
3775 rnp = rdp->mynode; in rcu_init_percpu_data()
3776 mask = rdp->grpmask; in rcu_init_percpu_data()
3780 rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */ in rcu_init_percpu_data()
3781 rdp->completed = rnp->completed; in rcu_init_percpu_data()
3782 rdp->passed_quiesce = false; in rcu_init_percpu_data()
3783 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); in rcu_init_percpu_data()
3784 rdp->qs_pending = false; in rcu_init_percpu_data()
3785 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); in rcu_init_percpu_data()
3804 struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu); in rcu_cpu_notify() local
3805 struct rcu_node *rnp = rdp->mynode; in rcu_cpu_notify()