Lines Matching refs:rdp
162 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
164 struct rcu_data *rdp, bool wake);
307 struct rcu_data *rdp; in rcu_momentary_dyntick_idle() local
323 rdp = raw_cpu_ptr(rsp->rda); in rcu_momentary_dyntick_idle()
327 if (READ_ONCE(rdp->mynode->completed) != in rcu_momentary_dyntick_idle()
328 READ_ONCE(rdp->cond_resched_completed)) in rcu_momentary_dyntick_idle()
409 struct rcu_data *rdp);
575 cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) in cpu_has_callbacks_ready_to_invoke() argument
577 return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] && in cpu_has_callbacks_ready_to_invoke()
578 rdp->nxttail[RCU_DONE_TAIL] != NULL; in cpu_has_callbacks_ready_to_invoke()
609 cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) in cpu_needs_another_gp() argument
617 if (!rdp->nxttail[RCU_NEXT_TAIL]) in cpu_needs_another_gp()
619 if (*rdp->nxttail[RCU_NEXT_READY_TAIL]) in cpu_needs_another_gp()
622 if (rdp->nxttail[i - 1] != rdp->nxttail[i] && in cpu_needs_another_gp()
624 rdp->nxtcompleted[i])) in cpu_needs_another_gp()
639 struct rcu_data *rdp; in rcu_eqs_enter_common() local
655 rdp = this_cpu_ptr(rsp->rda); in rcu_eqs_enter_common()
656 do_nocb_deferred_wakeup(rdp); in rcu_eqs_enter_common()
1032 struct rcu_data *rdp; in rcu_lockdep_current_cpu_online() local
1039 rdp = this_cpu_ptr(&rcu_sched_data); in rcu_lockdep_current_cpu_online()
1040 rnp = rdp->mynode; in rcu_lockdep_current_cpu_online()
1041 ret = (rdp->grpmask & rcu_rnp_online_cpus(rnp)) || in rcu_lockdep_current_cpu_online()
1067 static int dyntick_save_progress_counter(struct rcu_data *rdp, in dyntick_save_progress_counter() argument
1070 rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks); in dyntick_save_progress_counter()
1071 rcu_sysidle_check_cpu(rdp, isidle, maxj); in dyntick_save_progress_counter()
1072 if ((rdp->dynticks_snap & 0x1) == 0) { in dyntick_save_progress_counter()
1073 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
1076 if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4, in dyntick_save_progress_counter()
1077 rdp->mynode->gpnum)) in dyntick_save_progress_counter()
1078 WRITE_ONCE(rdp->gpwrap, true); in dyntick_save_progress_counter()
1089 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp, in rcu_implicit_dynticks_qs() argument
1096 curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks); in rcu_implicit_dynticks_qs()
1097 snap = (unsigned int)rdp->dynticks_snap; in rcu_implicit_dynticks_qs()
1108 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
1109 rdp->dynticks_fqs++; in rcu_implicit_dynticks_qs()
1124 if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies)) in rcu_implicit_dynticks_qs()
1127 if (cpu_is_offline(rdp->cpu)) { in rcu_implicit_dynticks_qs()
1128 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl")); in rcu_implicit_dynticks_qs()
1129 rdp->offline_fqs++; in rcu_implicit_dynticks_qs()
1154 rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu); in rcu_implicit_dynticks_qs()
1156 rdp->rsp->gp_start + jiffies_till_sched_qs) || in rcu_implicit_dynticks_qs()
1157 ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { in rcu_implicit_dynticks_qs()
1158 if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) { in rcu_implicit_dynticks_qs()
1159 WRITE_ONCE(rdp->cond_resched_completed, in rcu_implicit_dynticks_qs()
1160 READ_ONCE(rdp->mynode->completed)); in rcu_implicit_dynticks_qs()
1163 READ_ONCE(*rcrmp) + rdp->rsp->flavor_mask); in rcu_implicit_dynticks_qs()
1164 resched_cpu(rdp->cpu); /* Force CPU into scheduler. */ in rcu_implicit_dynticks_qs()
1165 rdp->rsp->jiffies_resched += 5; /* Enable beating. */ in rcu_implicit_dynticks_qs()
1166 } else if (ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { in rcu_implicit_dynticks_qs()
1168 resched_cpu(rdp->cpu); /* Force CPU into scheduler. */ in rcu_implicit_dynticks_qs()
1169 rdp->rsp->jiffies_resched += 5; /* Re-enable beating. */ in rcu_implicit_dynticks_qs()
1346 static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) in check_cpu_stall() argument
1387 rnp = rdp->mynode; in check_cpu_stall()
1389 (READ_ONCE(rnp->qsmask) & rdp->grpmask)) { in check_cpu_stall()
1424 static void init_default_callback_list(struct rcu_data *rdp) in init_default_callback_list() argument
1428 rdp->nxtlist = NULL; in init_default_callback_list()
1430 rdp->nxttail[i] = &rdp->nxtlist; in init_default_callback_list()
1436 static void init_callback_list(struct rcu_data *rdp) in init_callback_list() argument
1438 if (init_nocb_callback_list(rdp)) in init_callback_list()
1440 init_default_callback_list(rdp); in init_callback_list()
1476 static void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_future_gp() argument
1479 trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum, in trace_rcu_future_gp()
1493 rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, in rcu_start_future_gp() argument
1499 struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); in rcu_start_future_gp()
1505 c = rcu_cbs_completed(rdp->rsp, rnp); in rcu_start_future_gp()
1506 trace_rcu_future_gp(rnp, rdp, c, TPS("Startleaf")); in rcu_start_future_gp()
1508 trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartleaf")); in rcu_start_future_gp()
1528 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf")); in rcu_start_future_gp()
1548 c = rcu_cbs_completed(rdp->rsp, rnp_root); in rcu_start_future_gp()
1550 if (ULONG_CMP_LT(c, rdp->nxtcompleted[i])) in rcu_start_future_gp()
1551 rdp->nxtcompleted[i] = c; in rcu_start_future_gp()
1558 trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartedroot")); in rcu_start_future_gp()
1567 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleafroot")); in rcu_start_future_gp()
1569 trace_rcu_future_gp(rnp, rdp, c, TPS("Startedroot")); in rcu_start_future_gp()
1570 ret = rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp); in rcu_start_future_gp()
1591 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in rcu_future_gp_cleanup() local
1596 trace_rcu_future_gp(rnp, rdp, c, in rcu_future_gp_cleanup()
1630 struct rcu_data *rdp) in rcu_accelerate_cbs() argument
1637 if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL]) in rcu_accelerate_cbs()
1656 if (rdp->nxttail[i] != rdp->nxttail[i - 1] && in rcu_accelerate_cbs()
1657 !ULONG_CMP_GE(rdp->nxtcompleted[i], c)) in rcu_accelerate_cbs()
1675 rdp->nxttail[i] = rdp->nxttail[RCU_NEXT_TAIL]; in rcu_accelerate_cbs()
1676 rdp->nxtcompleted[i] = c; in rcu_accelerate_cbs()
1679 ret = rcu_start_future_gp(rnp, rdp, NULL); in rcu_accelerate_cbs()
1682 if (!*rdp->nxttail[RCU_WAIT_TAIL]) in rcu_accelerate_cbs()
1683 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1685 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1700 struct rcu_data *rdp) in rcu_advance_cbs() argument
1705 if (!rdp->nxttail[RCU_NEXT_TAIL] || !*rdp->nxttail[RCU_DONE_TAIL]) in rcu_advance_cbs()
1713 if (ULONG_CMP_LT(rnp->completed, rdp->nxtcompleted[i])) in rcu_advance_cbs()
1715 rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[i]; in rcu_advance_cbs()
1719 rdp->nxttail[j] = rdp->nxttail[RCU_DONE_TAIL]; in rcu_advance_cbs()
1723 if (rdp->nxttail[j] == rdp->nxttail[RCU_NEXT_TAIL]) in rcu_advance_cbs()
1725 rdp->nxttail[j] = rdp->nxttail[i]; in rcu_advance_cbs()
1726 rdp->nxtcompleted[j] = rdp->nxtcompleted[i]; in rcu_advance_cbs()
1730 return rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_advance_cbs()
1740 struct rcu_data *rdp) in __note_gp_changes() argument
1745 if (rdp->completed == rnp->completed && in __note_gp_changes()
1746 !unlikely(READ_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1749 ret = rcu_accelerate_cbs(rsp, rnp, rdp); in __note_gp_changes()
1754 ret = rcu_advance_cbs(rsp, rnp, rdp); in __note_gp_changes()
1757 rdp->completed = rnp->completed; in __note_gp_changes()
1758 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend")); in __note_gp_changes()
1761 if (rdp->gpnum != rnp->gpnum || unlikely(READ_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1767 rdp->gpnum = rnp->gpnum; in __note_gp_changes()
1768 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); in __note_gp_changes()
1769 rdp->cpu_no_qs.b.norm = true; in __note_gp_changes()
1770 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); in __note_gp_changes()
1771 rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1772 zero_cpu_stall_ticks(rdp); in __note_gp_changes()
1773 WRITE_ONCE(rdp->gpwrap, false); in __note_gp_changes()
1778 static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp) in note_gp_changes() argument
1785 rnp = rdp->mynode; in note_gp_changes()
1786 if ((rdp->gpnum == READ_ONCE(rnp->gpnum) && in note_gp_changes()
1787 rdp->completed == READ_ONCE(rnp->completed) && in note_gp_changes()
1788 !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */ in note_gp_changes()
1794 needwake = __note_gp_changes(rsp, rnp, rdp); in note_gp_changes()
1813 struct rcu_data *rdp; in rcu_gp_init() local
1909 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_init()
1915 if (rnp == rdp->mynode) in rcu_gp_init()
1916 (void)__note_gp_changes(rsp, rnp, rdp); in rcu_gp_init()
1992 struct rcu_data *rdp; in rcu_gp_cleanup() local
2027 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
2028 if (rnp == rdp->mynode) in rcu_gp_cleanup()
2029 needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; in rcu_gp_cleanup()
2046 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
2048 needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp; in rcu_gp_cleanup()
2049 if (needgp || cpu_needs_another_gp(rsp, rdp)) { in rcu_gp_cleanup()
2169 struct rcu_data *rdp) in rcu_start_gp_advanced() argument
2171 if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) { in rcu_start_gp_advanced()
2203 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in rcu_start_gp() local
2215 ret = rcu_advance_cbs(rsp, rnp, rdp) || ret; in rcu_start_gp()
2216 ret = rcu_start_gp_advanced(rsp, rnp, rdp) || ret; in rcu_start_gp()
2350 rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) in rcu_report_qs_rdp() argument
2357 rnp = rdp->mynode; in rcu_report_qs_rdp()
2360 if ((rdp->cpu_no_qs.b.norm && in rcu_report_qs_rdp()
2361 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) || in rcu_report_qs_rdp()
2362 rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum || in rcu_report_qs_rdp()
2363 rdp->gpwrap) { in rcu_report_qs_rdp()
2371 rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */ in rcu_report_qs_rdp()
2372 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); in rcu_report_qs_rdp()
2376 mask = rdp->grpmask; in rcu_report_qs_rdp()
2380 rdp->core_needs_qs = 0; in rcu_report_qs_rdp()
2386 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_report_qs_rdp()
2402 rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_check_quiescent_state() argument
2405 note_gp_changes(rsp, rdp); in rcu_check_quiescent_state()
2411 if (!rdp->core_needs_qs) in rcu_check_quiescent_state()
2418 if (rdp->cpu_no_qs.b.norm && in rcu_check_quiescent_state()
2419 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) in rcu_check_quiescent_state()
2426 rcu_report_qs_rdp(rdp->cpu, rsp, rdp); in rcu_check_quiescent_state()
2436 struct rcu_node *rnp, struct rcu_data *rdp) in rcu_send_cbs_to_orphanage() argument
2439 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) || rcu_is_nocb_cpu(rdp->cpu)) in rcu_send_cbs_to_orphanage()
2447 if (rdp->nxtlist != NULL) { in rcu_send_cbs_to_orphanage()
2448 rsp->qlen_lazy += rdp->qlen_lazy; in rcu_send_cbs_to_orphanage()
2449 rsp->qlen += rdp->qlen; in rcu_send_cbs_to_orphanage()
2450 rdp->n_cbs_orphaned += rdp->qlen; in rcu_send_cbs_to_orphanage()
2451 rdp->qlen_lazy = 0; in rcu_send_cbs_to_orphanage()
2452 WRITE_ONCE(rdp->qlen, 0); in rcu_send_cbs_to_orphanage()
2464 if (*rdp->nxttail[RCU_DONE_TAIL] != NULL) { in rcu_send_cbs_to_orphanage()
2465 *rsp->orphan_nxttail = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_send_cbs_to_orphanage()
2466 rsp->orphan_nxttail = rdp->nxttail[RCU_NEXT_TAIL]; in rcu_send_cbs_to_orphanage()
2467 *rdp->nxttail[RCU_DONE_TAIL] = NULL; in rcu_send_cbs_to_orphanage()
2475 if (rdp->nxtlist != NULL) { in rcu_send_cbs_to_orphanage()
2476 *rsp->orphan_donetail = rdp->nxtlist; in rcu_send_cbs_to_orphanage()
2477 rsp->orphan_donetail = rdp->nxttail[RCU_DONE_TAIL]; in rcu_send_cbs_to_orphanage()
2484 init_callback_list(rdp); in rcu_send_cbs_to_orphanage()
2485 rdp->nxttail[RCU_NEXT_TAIL] = NULL; in rcu_send_cbs_to_orphanage()
2495 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in rcu_adopt_orphan_cbs() local
2499 rcu_nocb_adopt_orphan_cbs(rsp, rdp, flags)) in rcu_adopt_orphan_cbs()
2503 rdp->qlen_lazy += rsp->qlen_lazy; in rcu_adopt_orphan_cbs()
2504 rdp->qlen += rsp->qlen; in rcu_adopt_orphan_cbs()
2505 rdp->n_cbs_adopted += rsp->qlen; in rcu_adopt_orphan_cbs()
2519 *rsp->orphan_donetail = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_adopt_orphan_cbs()
2520 *rdp->nxttail[RCU_DONE_TAIL] = rsp->orphan_donelist; in rcu_adopt_orphan_cbs()
2522 if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL]) in rcu_adopt_orphan_cbs()
2523 rdp->nxttail[i] = rsp->orphan_donetail; in rcu_adopt_orphan_cbs()
2530 *rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxtlist; in rcu_adopt_orphan_cbs()
2531 rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_nxttail; in rcu_adopt_orphan_cbs()
2543 RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda)); in rcu_cleanup_dying_cpu()
2544 RCU_TRACE(struct rcu_node *rnp = rdp->mynode); in rcu_cleanup_dying_cpu()
2549 RCU_TRACE(mask = rdp->grpmask); in rcu_cleanup_dying_cpu()
2606 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dying_idle_cpu() local
2607 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcu_cleanup_dying_idle_cpu()
2613 mask = rdp->grpmask; in rcu_cleanup_dying_idle_cpu()
2630 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dead_cpu() local
2631 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcu_cleanup_dead_cpu()
2641 rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp); in rcu_cleanup_dead_cpu()
2645 WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL, in rcu_cleanup_dead_cpu()
2647 cpu, rdp->qlen, rdp->nxtlist); in rcu_cleanup_dead_cpu()
2654 static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_do_batch() argument
2662 if (!cpu_has_callbacks_ready_to_invoke(rdp)) { in rcu_do_batch()
2663 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0); in rcu_do_batch()
2664 trace_rcu_batch_end(rsp->name, 0, !!READ_ONCE(rdp->nxtlist), in rcu_do_batch()
2676 bl = rdp->blimit; in rcu_do_batch()
2677 trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, bl); in rcu_do_batch()
2678 list = rdp->nxtlist; in rcu_do_batch()
2679 rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL]; in rcu_do_batch()
2680 *rdp->nxttail[RCU_DONE_TAIL] = NULL; in rcu_do_batch()
2681 tail = rdp->nxttail[RCU_DONE_TAIL]; in rcu_do_batch()
2683 if (rdp->nxttail[i] == rdp->nxttail[RCU_DONE_TAIL]) in rcu_do_batch()
2684 rdp->nxttail[i] = &rdp->nxtlist; in rcu_do_batch()
2710 *tail = rdp->nxtlist; in rcu_do_batch()
2711 rdp->nxtlist = list; in rcu_do_batch()
2713 if (&rdp->nxtlist == rdp->nxttail[i]) in rcu_do_batch()
2714 rdp->nxttail[i] = tail; in rcu_do_batch()
2719 rdp->qlen_lazy -= count_lazy; in rcu_do_batch()
2720 WRITE_ONCE(rdp->qlen, rdp->qlen - count); in rcu_do_batch()
2721 rdp->n_cbs_invoked += count; in rcu_do_batch()
2724 if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark) in rcu_do_batch()
2725 rdp->blimit = blimit; in rcu_do_batch()
2728 if (rdp->qlen == 0 && rdp->qlen_last_fqs_check != 0) { in rcu_do_batch()
2729 rdp->qlen_last_fqs_check = 0; in rcu_do_batch()
2730 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_do_batch()
2731 } else if (rdp->qlen < rdp->qlen_last_fqs_check - qhimark) in rcu_do_batch()
2732 rdp->qlen_last_fqs_check = rdp->qlen; in rcu_do_batch()
2733 WARN_ON_ONCE((rdp->nxtlist == NULL) != (rdp->qlen == 0)); in rcu_do_batch()
2738 if (cpu_has_callbacks_ready_to_invoke(rdp)) in rcu_do_batch()
2907 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in __rcu_process_callbacks() local
2909 WARN_ON_ONCE(rdp->beenonline == 0); in __rcu_process_callbacks()
2912 rcu_check_quiescent_state(rsp, rdp); in __rcu_process_callbacks()
2916 if (cpu_needs_another_gp(rsp, rdp)) { in __rcu_process_callbacks()
2927 if (cpu_has_callbacks_ready_to_invoke(rdp)) in __rcu_process_callbacks()
2928 invoke_rcu_callbacks(rsp, rdp); in __rcu_process_callbacks()
2931 do_nocb_deferred_wakeup(rdp); in __rcu_process_callbacks()
2956 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) in invoke_rcu_callbacks() argument
2961 rcu_do_batch(rsp, rdp); in invoke_rcu_callbacks()
2976 static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, in __call_rcu_core() argument
2999 if (unlikely(rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) { in __call_rcu_core()
3002 note_gp_changes(rsp, rdp); in __call_rcu_core()
3016 rdp->blimit = LONG_MAX; in __call_rcu_core()
3017 if (rsp->n_force_qs == rdp->n_force_qs_snap && in __call_rcu_core()
3018 *rdp->nxttail[RCU_DONE_TAIL] != head) in __call_rcu_core()
3020 rdp->n_force_qs_snap = rsp->n_force_qs; in __call_rcu_core()
3021 rdp->qlen_last_fqs_check = rdp->qlen; in __call_rcu_core()
3044 struct rcu_data *rdp; in __call_rcu() local
3063 rdp = this_cpu_ptr(rsp->rda); in __call_rcu()
3066 if (unlikely(rdp->nxttail[RCU_NEXT_TAIL] == NULL) || cpu != -1) { in __call_rcu()
3070 rdp = per_cpu_ptr(rsp->rda, cpu); in __call_rcu()
3071 if (likely(rdp->mynode)) { in __call_rcu()
3073 offline = !__call_rcu_nocb(rdp, head, lazy, flags); in __call_rcu()
3085 if (!likely(rdp->nxtlist)) in __call_rcu()
3086 init_default_callback_list(rdp); in __call_rcu()
3088 WRITE_ONCE(rdp->qlen, rdp->qlen + 1); in __call_rcu()
3090 rdp->qlen_lazy++; in __call_rcu()
3094 *rdp->nxttail[RCU_NEXT_TAIL] = head; in __call_rcu()
3095 rdp->nxttail[RCU_NEXT_TAIL] = &head->next; in __call_rcu()
3099 rdp->qlen_lazy, rdp->qlen); in __call_rcu()
3101 trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen); in __call_rcu()
3104 __call_rcu_core(rsp, rdp, head, flags); in __call_rcu()
3581 static void rcu_report_exp_rdp(struct rcu_state *rsp, struct rcu_data *rdp, in rcu_report_exp_rdp() argument
3584 rcu_report_exp_cpu_mult(rsp, rdp->mynode, rdp->grpmask, wake); in rcu_report_exp_rdp()
3589 struct rcu_data *rdp, in sync_exp_work_done() argument
3595 else if (rdp) in sync_exp_work_done()
3596 mutex_unlock(&rdp->exp_funnel_mutex); in sync_exp_work_done()
3612 struct rcu_data *rdp; in exp_funnel_lock() local
3640 rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id()); in exp_funnel_lock()
3643 mutex_lock(&rdp->exp_funnel_mutex); in exp_funnel_lock()
3644 rnp0 = rdp->mynode; in exp_funnel_lock()
3646 if (sync_exp_work_done(rsp, rnp1, rdp, in exp_funnel_lock()
3653 mutex_unlock(&rdp->exp_funnel_mutex); in exp_funnel_lock()
3656 if (sync_exp_work_done(rsp, rnp1, rdp, in exp_funnel_lock()
3665 struct rcu_data *rdp; in sync_sched_exp_handler() local
3669 rdp = this_cpu_ptr(rsp->rda); in sync_sched_exp_handler()
3670 rnp = rdp->mynode; in sync_sched_exp_handler()
3671 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in sync_sched_exp_handler()
3681 struct rcu_data *rdp; in sync_sched_exp_online_cleanup() local
3686 rdp = per_cpu_ptr(rsp->rda, cpu); in sync_sched_exp_online_cleanup()
3687 rnp = rdp->mynode; in sync_sched_exp_online_cleanup()
3688 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask)) in sync_sched_exp_online_cleanup()
3717 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in sync_rcu_exp_select_cpus() local
3722 mask_ofl_test |= rdp->grpmask; in sync_rcu_exp_select_cpus()
3802 struct rcu_data *rdp; in synchronize_sched_expedited_wait() local
3806 rdp = per_cpu_ptr(rsp->rda, cpu); in synchronize_sched_expedited_wait()
3809 "o."[!!(rdp->grpmask & rnp->expmaskinit)], in synchronize_sched_expedited_wait()
3810 "N."[!!(rdp->grpmask & rnp->expmaskinitnext)]); in synchronize_sched_expedited_wait()
3873 static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) in __rcu_pending() argument
3875 struct rcu_node *rnp = rdp->mynode; in __rcu_pending()
3877 rdp->n_rcu_pending++; in __rcu_pending()
3880 check_cpu_stall(rsp, rdp); in __rcu_pending()
3888 rdp->core_needs_qs && rdp->cpu_no_qs.b.norm && in __rcu_pending()
3889 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) { in __rcu_pending()
3890 rdp->n_rp_core_needs_qs++; in __rcu_pending()
3891 } else if (rdp->core_needs_qs && in __rcu_pending()
3892 (!rdp->cpu_no_qs.b.norm || in __rcu_pending()
3893 rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) { in __rcu_pending()
3894 rdp->n_rp_report_qs++; in __rcu_pending()
3899 if (cpu_has_callbacks_ready_to_invoke(rdp)) { in __rcu_pending()
3900 rdp->n_rp_cb_ready++; in __rcu_pending()
3905 if (cpu_needs_another_gp(rsp, rdp)) { in __rcu_pending()
3906 rdp->n_rp_cpu_needs_gp++; in __rcu_pending()
3911 if (READ_ONCE(rnp->completed) != rdp->completed) { /* outside lock */ in __rcu_pending()
3912 rdp->n_rp_gp_completed++; in __rcu_pending()
3917 if (READ_ONCE(rnp->gpnum) != rdp->gpnum || in __rcu_pending()
3918 unlikely(READ_ONCE(rdp->gpwrap))) { /* outside lock */ in __rcu_pending()
3919 rdp->n_rp_gp_started++; in __rcu_pending()
3924 if (rcu_nocb_need_deferred_wakeup(rdp)) { in __rcu_pending()
3925 rdp->n_rp_nocb_defer_wakeup++; in __rcu_pending()
3930 rdp->n_rp_need_nothing++; in __rcu_pending()
3958 struct rcu_data *rdp; in rcu_cpu_has_callbacks() local
3962 rdp = this_cpu_ptr(rsp->rda); in rcu_cpu_has_callbacks()
3963 if (!rdp->nxtlist) in rcu_cpu_has_callbacks()
3966 if (rdp->qlen != rdp->qlen_lazy || !all_lazy) { in rcu_cpu_has_callbacks()
3993 struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head); in rcu_barrier_callback() local
3994 struct rcu_state *rsp = rdp->rsp; in rcu_barrier_callback()
4010 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in rcu_barrier_func() local
4014 rsp->call(&rdp->barrier_head, rcu_barrier_callback); in rcu_barrier_func()
4024 struct rcu_data *rdp; in _rcu_barrier() local
4062 rdp = per_cpu_ptr(rsp->rda, cpu); in _rcu_barrier()
4072 __call_rcu(&rdp->barrier_head, in _rcu_barrier()
4075 } else if (READ_ONCE(rdp->qlen)) { in _rcu_barrier()
4151 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_boot_init_percpu_data() local
4156 rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo); in rcu_boot_init_percpu_data()
4157 rdp->dynticks = &per_cpu(rcu_dynticks, cpu); in rcu_boot_init_percpu_data()
4158 WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE); in rcu_boot_init_percpu_data()
4159 WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1); in rcu_boot_init_percpu_data()
4160 rdp->cpu = cpu; in rcu_boot_init_percpu_data()
4161 rdp->rsp = rsp; in rcu_boot_init_percpu_data()
4162 mutex_init(&rdp->exp_funnel_mutex); in rcu_boot_init_percpu_data()
4163 rcu_boot_init_nocb_percpu_data(rdp); in rcu_boot_init_percpu_data()
4178 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_init_percpu_data() local
4183 rdp->qlen_last_fqs_check = 0; in rcu_init_percpu_data()
4184 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_init_percpu_data()
4185 rdp->blimit = blimit; in rcu_init_percpu_data()
4186 if (!rdp->nxtlist) in rcu_init_percpu_data()
4187 init_callback_list(rdp); /* Re-enable callbacks on this CPU. */ in rcu_init_percpu_data()
4188 rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE; in rcu_init_percpu_data()
4189 rcu_sysidle_init_percpu_data(rdp->dynticks); in rcu_init_percpu_data()
4190 atomic_set(&rdp->dynticks->dynticks, in rcu_init_percpu_data()
4191 (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1); in rcu_init_percpu_data()
4199 rnp = rdp->mynode; in rcu_init_percpu_data()
4200 mask = rdp->grpmask; in rcu_init_percpu_data()
4205 if (!rdp->beenonline) in rcu_init_percpu_data()
4207 rdp->beenonline = true; /* We have now been online. */ in rcu_init_percpu_data()
4208 rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */ in rcu_init_percpu_data()
4209 rdp->completed = rnp->completed; in rcu_init_percpu_data()
4210 rdp->cpu_no_qs.b.norm = true; in rcu_init_percpu_data()
4211 rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu); in rcu_init_percpu_data()
4212 rdp->core_needs_qs = false; in rcu_init_percpu_data()
4213 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); in rcu_init_percpu_data()
4232 struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu); in rcu_cpu_notify() local
4233 struct rcu_node *rnp = rdp->mynode; in rcu_cpu_notify()