root/kernel/rcu/tiny.c

DEFINITIONS

1. rcu_barrier
2. rcu_qs
3. rcu_sched_clock_irq
4. rcu_process_callbacks
5. synchronize_rcu
6. call_rcu
7. rcu_init

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
   4  *
   5  * Copyright IBM Corporation, 2008
   6  *
   7  * Author: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *
   9  * For detailed explanation of Read-Copy Update mechanism see -
  10  *              Documentation/RCU
  11  */
  12 #include <linux/completion.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/notifier.h>
  15 #include <linux/rcupdate_wait.h>
  16 #include <linux/kernel.h>
  17 #include <linux/export.h>
  18 #include <linux/mutex.h>
  19 #include <linux/sched.h>
  20 #include <linux/types.h>
  21 #include <linux/init.h>
  22 #include <linux/time.h>
  23 #include <linux/cpu.h>
  24 #include <linux/prefetch.h>
  25 
  26 #include "rcu.h"
  27 
  28 /* Global control variables for rcupdate callback mechanism. */
  29 struct rcu_ctrlblk {
  30         struct rcu_head *rcucblist;     /* List of pending callbacks (CBs). */
  31         struct rcu_head **donetail;     /* ->next pointer of last "done" CB. */
  32         struct rcu_head **curtail;      /* ->next pointer of last CB. */
  33 };
  34 
  35 /* Definition for rcupdate control block. */
  36 static struct rcu_ctrlblk rcu_ctrlblk = {
  37         .donetail       = &rcu_ctrlblk.rcucblist,
  38         .curtail        = &rcu_ctrlblk.rcucblist,
  39 };
  40 
  41 void rcu_barrier(void)
  42 {
  43         wait_rcu_gp(call_rcu);
  44 }
  45 EXPORT_SYMBOL(rcu_barrier);
  46 
  47 /* Record an rcu quiescent state.  */
  48 void rcu_qs(void)
  49 {
  50         unsigned long flags;
  51 
  52         local_irq_save(flags);
  53         if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  54                 rcu_ctrlblk.donetail = rcu_ctrlblk.curtail;
  55                 raise_softirq_irqoff(RCU_SOFTIRQ);
  56         }
  57         local_irq_restore(flags);
  58 }
  59 
  60 /*
  61  * Check to see if the scheduling-clock interrupt came from an extended
  62  * quiescent state, and, if so, tell RCU about it.  This function must
  63  * be called from hardirq context.  It is normally called from the
  64  * scheduling-clock interrupt.
  65  */
  66 void rcu_sched_clock_irq(int user)
  67 {
  68         if (user) {
  69                 rcu_qs();
  70         } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  71                 set_tsk_need_resched(current);
  72                 set_preempt_need_resched();
  73         }
  74 }
  75 
  76 /* Invoke the RCU callbacks whose grace period has elapsed.  */
  77 static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
  78 {
  79         struct rcu_head *next, *list;
  80         unsigned long flags;
  81 
  82         /* Move the ready-to-invoke callbacks to a local list. */
  83         local_irq_save(flags);
  84         if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) {
  85                 /* No callbacks ready, so just leave. */
  86                 local_irq_restore(flags);
  87                 return;
  88         }
  89         list = rcu_ctrlblk.rcucblist;
  90         rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail;
  91         *rcu_ctrlblk.donetail = NULL;
  92         if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail)
  93                 rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist;
  94         rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist;
  95         local_irq_restore(flags);
  96 
  97         /* Invoke the callbacks on the local list. */
  98         while (list) {
  99                 next = list->next;
 100                 prefetch(next);
 101                 debug_rcu_head_unqueue(list);
 102                 local_bh_disable();
 103                 __rcu_reclaim("", list);
 104                 local_bh_enable();
 105                 list = next;
 106         }
 107 }
 108 
 109 /*
 110  * Wait for a grace period to elapse.  But it is illegal to invoke
 111  * synchronize_rcu() from within an RCU read-side critical section.
 112  * Therefore, any legal call to synchronize_rcu() is a quiescent
 113  * state, and so on a UP system, synchronize_rcu() need do nothing.
 114  * (But Lai Jiangshan points out the benefits of doing might_sleep()
 115  * to reduce latency.)
 116  *
 117  * Cool, huh?  (Due to Josh Triplett.)
 118  */
 119 void synchronize_rcu(void)
 120 {
 121         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
 122                          lock_is_held(&rcu_lock_map) ||
 123                          lock_is_held(&rcu_sched_lock_map),
 124                          "Illegal synchronize_rcu() in RCU read-side critical section");
 125 }
 126 EXPORT_SYMBOL_GPL(synchronize_rcu);
 127 
 128 /*
 129  * Post an RCU callback to be invoked after the end of an RCU grace
 130  * period.  But since we have but one CPU, that would be after any
 131  * quiescent state.
 132  */
 133 void call_rcu(struct rcu_head *head, rcu_callback_t func)
 134 {
 135         unsigned long flags;
 136 
 137         debug_rcu_head_queue(head);
 138         head->func = func;
 139         head->next = NULL;
 140 
 141         local_irq_save(flags);
 142         *rcu_ctrlblk.curtail = head;
 143         rcu_ctrlblk.curtail = &head->next;
 144         local_irq_restore(flags);
 145 
 146         if (unlikely(is_idle_task(current))) {
 147                 /* force scheduling for rcu_qs() */
 148                 resched_cpu(0);
 149         }
 150 }
 151 EXPORT_SYMBOL_GPL(call_rcu);
 152 
 153 void __init rcu_init(void)
 154 {
 155         open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 156         rcu_early_boot_tests();
 157         srcu_init();
 158 }