Lines Matching refs:READ_ONCE
228 return READ_ONCE(rnp->qsmaskinitnext); in rcu_rnp_online_cpus()
238 return READ_ONCE(rsp->completed) != READ_ONCE(rsp->gpnum); in rcu_gp_in_progress()
327 if (READ_ONCE(rdp->mynode->completed) != in rcu_momentary_dyntick_idle()
328 READ_ONCE(rdp->cond_resched_completed)) in rcu_momentary_dyntick_idle()
549 *flags = READ_ONCE(rsp->gp_flags); in rcutorture_get_gp_data()
550 *gpnum = READ_ONCE(rsp->gpnum); in rcutorture_get_gp_data()
551 *completed = READ_ONCE(rsp->completed); in rcutorture_get_gp_data()
597 int idx = (READ_ONCE(rnp->completed) + 1) & 0x1; in rcu_future_needs_gp()
600 return READ_ONCE(*fp); in rcu_future_needs_gp()
623 ULONG_CMP_LT(READ_ONCE(rsp->completed), in cpu_needs_another_gp()
1076 if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4, in dyntick_save_progress_counter()
1158 if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) { in rcu_implicit_dynticks_qs()
1160 READ_ONCE(rdp->mynode->completed)); in rcu_implicit_dynticks_qs()
1163 READ_ONCE(*rcrmp) + rdp->rsp->flavor_mask); in rcu_implicit_dynticks_qs()
1186 rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs); in record_gp_stall_check_time()
1198 gpa = READ_ONCE(rsp->gp_activity); in rcu_check_gp_kthread_starvation()
1241 delta = jiffies - READ_ONCE(rsp->jiffies_stall); in print_other_cpu_stall()
1281 if (READ_ONCE(rsp->gpnum) != gpnum || in print_other_cpu_stall()
1282 READ_ONCE(rsp->completed) == gpnum) { in print_other_cpu_stall()
1286 gpa = READ_ONCE(rsp->gp_activity); in print_other_cpu_stall()
1331 if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) in print_cpu_stall()
1376 gpnum = READ_ONCE(rsp->gpnum); in check_cpu_stall()
1378 js = READ_ONCE(rsp->jiffies_stall); in check_cpu_stall()
1380 gps = READ_ONCE(rsp->gp_start); in check_cpu_stall()
1382 completed = READ_ONCE(rsp->completed); in check_cpu_stall()
1389 (READ_ONCE(rnp->qsmask) & rdp->grpmask)) { in check_cpu_stall()
1526 READ_ONCE(rnp_root->gpnum) != READ_ONCE(rnp_root->completed)) { in rcu_start_future_gp()
1611 !READ_ONCE(rsp->gp_flags) || in rcu_gp_kthread_wake()
1746 !unlikely(READ_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1761 if (rdp->gpnum != rnp->gpnum || unlikely(READ_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1786 if ((rdp->gpnum == READ_ONCE(rnp->gpnum) && in note_gp_changes()
1787 rdp->completed == READ_ONCE(rnp->completed) && in note_gp_changes()
1788 !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */ in note_gp_changes()
1819 if (!READ_ONCE(rsp->gp_flags)) { in rcu_gp_init()
1938 *gfp = READ_ONCE(rsp->gp_flags); in rcu_gp_fqs_check_wake()
1943 if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp)) in rcu_gp_fqs_check_wake()
1975 if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { in rcu_gp_fqs()
1979 READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS); in rcu_gp_fqs()
2052 READ_ONCE(rsp->gpnum), in rcu_gp_cleanup()
2076 READ_ONCE(rsp->gpnum), in rcu_gp_kthread()
2080 READ_ONCE(rsp->gp_flags) & in rcu_gp_kthread()
2090 READ_ONCE(rsp->gpnum), in rcu_gp_kthread()
2106 READ_ONCE(rsp->gpnum), in rcu_gp_kthread()
2114 if (!READ_ONCE(rnp->qsmask) && in rcu_gp_kthread()
2121 READ_ONCE(rsp->gpnum), in rcu_gp_kthread()
2126 READ_ONCE(rsp->gpnum), in rcu_gp_kthread()
2136 READ_ONCE(rsp->gpnum), in rcu_gp_kthread()
2181 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), in rcu_start_gp_advanced()
2231 WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS); in rcu_report_qs_rsp()
2664 trace_rcu_batch_end(rsp->name, 0, !!READ_ONCE(rdp->nxtlist), in rcu_do_batch()
2871 ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) || in force_quiescent_state()
2887 if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { in force_quiescent_state()
2892 WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS); in force_quiescent_state()
2958 if (unlikely(!READ_ONCE(rcu_scheduler_fully_active))) in invoke_rcu_callbacks()
3369 s = (READ_ONCE(*sp) + 3) & ~0x1; in rcu_seq_snap()
3380 return ULONG_CMP_GE(READ_ONCE(*sp), s); in rcu_seq_done()
3415 int ncpus = READ_ONCE(rsp->ncpus); in sync_exp_reset_tree_hotplug()
3495 READ_ONCE(rnp->expmask) == 0; in sync_rcu_preempt_exp_done()
3671 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in sync_sched_exp_handler()
3688 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask)) in sync_sched_exp_online_cleanup()
3911 if (READ_ONCE(rnp->completed) != rdp->completed) { /* outside lock */ in __rcu_pending()
3917 if (READ_ONCE(rnp->gpnum) != rdp->gpnum || in __rcu_pending()
3918 unlikely(READ_ONCE(rdp->gpwrap))) { /* outside lock */ in __rcu_pending()
4075 } else if (READ_ONCE(rdp->qlen)) { in _rcu_barrier()
4206 WRITE_ONCE(rsp->ncpus, READ_ONCE(rsp->ncpus) + 1); in rcu_init_percpu_data()