rsclp 91 kernel/rcu/rcu_segcblist.c void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v) rsclp 94 kernel/rcu/rcu_segcblist.c atomic_long_set(&rsclp->len, v); rsclp 96 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->len, v); rsclp 107 kernel/rcu/rcu_segcblist.c void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v) rsclp 111 kernel/rcu/rcu_segcblist.c atomic_long_add(v, &rsclp->len); rsclp 115 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->len, rsclp->len + v); rsclp 126 kernel/rcu/rcu_segcblist.c void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp) rsclp 128 kernel/rcu/rcu_segcblist.c rcu_segcblist_add_len(rsclp, 1); rsclp 137 kernel/rcu/rcu_segcblist.c long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v) rsclp 140 kernel/rcu/rcu_segcblist.c return atomic_long_xchg(&rsclp->len, v); rsclp 142 kernel/rcu/rcu_segcblist.c long ret = rsclp->len; rsclp 145 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->len, v); rsclp 154 kernel/rcu/rcu_segcblist.c void rcu_segcblist_init(struct rcu_segcblist *rsclp) rsclp 158 kernel/rcu/rcu_segcblist.c BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq)); rsclp 159 kernel/rcu/rcu_segcblist.c BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq)); rsclp 160 kernel/rcu/rcu_segcblist.c rsclp->head = NULL; rsclp 162 kernel/rcu/rcu_segcblist.c rsclp->tails[i] = &rsclp->head; rsclp 163 kernel/rcu/rcu_segcblist.c rcu_segcblist_set_len(rsclp, 0); rsclp 164 kernel/rcu/rcu_segcblist.c rsclp->len_lazy = 0; rsclp 165 kernel/rcu/rcu_segcblist.c rsclp->enabled = 1; rsclp 172 kernel/rcu/rcu_segcblist.c void rcu_segcblist_disable(struct rcu_segcblist *rsclp) rsclp 174 kernel/rcu/rcu_segcblist.c WARN_ON_ONCE(!rcu_segcblist_empty(rsclp)); rsclp 175 kernel/rcu/rcu_segcblist.c WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp)); rsclp 176 kernel/rcu/rcu_segcblist.c WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp)); rsclp 177 kernel/rcu/rcu_segcblist.c rsclp->enabled = 0; rsclp 184 kernel/rcu/rcu_segcblist.c void rcu_segcblist_offload(struct rcu_segcblist *rsclp) rsclp 186 kernel/rcu/rcu_segcblist.c rsclp->offloaded = 1; rsclp 193 kernel/rcu/rcu_segcblist.c bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp) rsclp 195 kernel/rcu/rcu_segcblist.c return rcu_segcblist_is_enabled(rsclp) && rsclp 196 kernel/rcu/rcu_segcblist.c &rsclp->head != rsclp->tails[RCU_DONE_TAIL]; rsclp 203 kernel/rcu/rcu_segcblist.c bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp) rsclp 205 kernel/rcu/rcu_segcblist.c return rcu_segcblist_is_enabled(rsclp) && rsclp 206 kernel/rcu/rcu_segcblist.c !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL); rsclp 213 kernel/rcu/rcu_segcblist.c struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp) rsclp 215 kernel/rcu/rcu_segcblist.c if (rcu_segcblist_is_enabled(rsclp)) rsclp 216 kernel/rcu/rcu_segcblist.c return rsclp->head; rsclp 227 kernel/rcu/rcu_segcblist.c struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) rsclp 229 kernel/rcu/rcu_segcblist.c if (rcu_segcblist_is_enabled(rsclp)) rsclp 230 kernel/rcu/rcu_segcblist.c return *rsclp->tails[RCU_DONE_TAIL]; rsclp 238 kernel/rcu/rcu_segcblist.c bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) rsclp 240 kernel/rcu/rcu_segcblist.c if (!rcu_segcblist_pend_cbs(rsclp)) rsclp 242 kernel/rcu/rcu_segcblist.c *lp = rsclp->gp_seq[RCU_WAIT_TAIL]; rsclp 255 kernel/rcu/rcu_segcblist.c void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, rsclp 258 kernel/rcu/rcu_segcblist.c rcu_segcblist_inc_len(rsclp); rsclp 260 kernel/rcu/rcu_segcblist.c rsclp->len_lazy++; rsclp 263 kernel/rcu/rcu_segcblist.c WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp); rsclp 264 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next); rsclp 277 kernel/rcu/rcu_segcblist.c bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, rsclp 282 kernel/rcu/rcu_segcblist.c if (rcu_segcblist_n_cbs(rsclp) == 0) rsclp 284 kernel/rcu/rcu_segcblist.c rcu_segcblist_inc_len(rsclp); rsclp 286 kernel/rcu/rcu_segcblist.c rsclp->len_lazy++; rsclp 290 kernel/rcu/rcu_segcblist.c if (rsclp->tails[i] != rsclp->tails[i - 1]) rsclp 292 kernel/rcu/rcu_segcblist.c WRITE_ONCE(*rsclp->tails[i], rhp); rsclp 294 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[i], &rhp->next); rsclp 307 kernel/rcu/rcu_segcblist.c void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, rsclp 310 kernel/rcu/rcu_segcblist.c rclp->len_lazy += rsclp->len_lazy; rsclp 311 kernel/rcu/rcu_segcblist.c rsclp->len_lazy = 0; rsclp 312 kernel/rcu/rcu_segcblist.c rclp->len = rcu_segcblist_xchg_len(rsclp, 0); rsclp 320 kernel/rcu/rcu_segcblist.c void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, rsclp 325 kernel/rcu/rcu_segcblist.c if (!rcu_segcblist_ready_cbs(rsclp)) rsclp 327 kernel/rcu/rcu_segcblist.c *rclp->tail = rsclp->head; rsclp 328 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]); rsclp 329 kernel/rcu/rcu_segcblist.c WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); rsclp 330 kernel/rcu/rcu_segcblist.c rclp->tail = rsclp->tails[RCU_DONE_TAIL]; rsclp 332 kernel/rcu/rcu_segcblist.c if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL]) rsclp 333 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[i], &rsclp->head); rsclp 343 kernel/rcu/rcu_segcblist.c void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, rsclp 348 kernel/rcu/rcu_segcblist.c if (!rcu_segcblist_pend_cbs(rsclp)) rsclp 350 kernel/rcu/rcu_segcblist.c *rclp->tail = *rsclp->tails[RCU_DONE_TAIL]; rsclp 351 kernel/rcu/rcu_segcblist.c rclp->tail = rsclp->tails[RCU_NEXT_TAIL]; rsclp 352 kernel/rcu/rcu_segcblist.c WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); rsclp 354 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]); rsclp 361 kernel/rcu/rcu_segcblist.c void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, rsclp 364 kernel/rcu/rcu_segcblist.c rsclp->len_lazy += rclp->len_lazy; rsclp 365 kernel/rcu/rcu_segcblist.c rcu_segcblist_add_len(rsclp, rclp->len); rsclp 374 kernel/rcu/rcu_segcblist.c void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, rsclp 381 kernel/rcu/rcu_segcblist.c *rclp->tail = rsclp->head; rsclp 382 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->head, rclp->head); rsclp 384 kernel/rcu/rcu_segcblist.c if (&rsclp->head == rsclp->tails[i]) rsclp 385 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[i], rclp->tail); rsclp 396 kernel/rcu/rcu_segcblist.c void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, rsclp 401 kernel/rcu/rcu_segcblist.c WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head); rsclp 402 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail); rsclp 411 kernel/rcu/rcu_segcblist.c void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) rsclp 415 kernel/rcu/rcu_segcblist.c WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); rsclp 416 kernel/rcu/rcu_segcblist.c if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) rsclp 424 kernel/rcu/rcu_segcblist.c if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) rsclp 426 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); rsclp 435 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); rsclp 444 kernel/rcu/rcu_segcblist.c if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) rsclp 446 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); rsclp 447 kernel/rcu/rcu_segcblist.c rsclp->gp_seq[j] = rsclp->gp_seq[i]; rsclp 466 kernel/rcu/rcu_segcblist.c bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) rsclp 470 kernel/rcu/rcu_segcblist.c WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); rsclp 471 kernel/rcu/rcu_segcblist.c if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) rsclp 483 kernel/rcu/rcu_segcblist.c if (rsclp->tails[i] != rsclp->tails[i - 1] && rsclp 484 kernel/rcu/rcu_segcblist.c ULONG_CMP_LT(rsclp->gp_seq[i], seq)) rsclp 511 kernel/rcu/rcu_segcblist.c WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]); rsclp 512 kernel/rcu/rcu_segcblist.c rsclp->gp_seq[i] = seq; rsclp 47 kernel/rcu/rcu_segcblist.h static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) rsclp 49 kernel/rcu/rcu_segcblist.h return !READ_ONCE(rsclp->head); rsclp 53 kernel/rcu/rcu_segcblist.h static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) rsclp 56 kernel/rcu/rcu_segcblist.h return atomic_long_read(&rsclp->len); rsclp 58 kernel/rcu/rcu_segcblist.h return READ_ONCE(rsclp->len); rsclp 63 kernel/rcu/rcu_segcblist.h static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp) rsclp 65 kernel/rcu/rcu_segcblist.h return rsclp->len_lazy; rsclp 69 kernel/rcu/rcu_segcblist.h static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp) rsclp 71 kernel/rcu/rcu_segcblist.h return rcu_segcblist_n_cbs(rsclp) - rsclp->len_lazy; rsclp 78 kernel/rcu/rcu_segcblist.h static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) rsclp 80 kernel/rcu/rcu_segcblist.h return rsclp->enabled; rsclp 84 kernel/rcu/rcu_segcblist.h static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) rsclp 86 kernel/rcu/rcu_segcblist.h return rsclp->offloaded; rsclp 94 kernel/rcu/rcu_segcblist.h static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) rsclp 96 kernel/rcu/rcu_segcblist.h return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); rsclp 99 kernel/rcu/rcu_segcblist.h void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); rsclp 100 kernel/rcu/rcu_segcblist.h void rcu_segcblist_init(struct rcu_segcblist *rsclp); rsclp 101 kernel/rcu/rcu_segcblist.h void rcu_segcblist_disable(struct rcu_segcblist *rsclp); rsclp 102 kernel/rcu/rcu_segcblist.h void rcu_segcblist_offload(struct rcu_segcblist *rsclp); rsclp 103 kernel/rcu/rcu_segcblist.h bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); rsclp 104 kernel/rcu/rcu_segcblist.h bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); rsclp 105 kernel/rcu/rcu_segcblist.h struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); rsclp 106 kernel/rcu/rcu_segcblist.h struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); rsclp 107 kernel/rcu/rcu_segcblist.h bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); rsclp 108 kernel/rcu/rcu_segcblist.h void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, rsclp 110 kernel/rcu/rcu_segcblist.h bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, rsclp 112 kernel/rcu/rcu_segcblist.h void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, rsclp 114 kernel/rcu/rcu_segcblist.h void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, rsclp 116 kernel/rcu/rcu_segcblist.h void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, rsclp 118 kernel/rcu/rcu_segcblist.h void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, rsclp 120 kernel/rcu/rcu_segcblist.h void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, rsclp 122 kernel/rcu/rcu_segcblist.h void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, rsclp 124 kernel/rcu/rcu_segcblist.h void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); rsclp 125 kernel/rcu/rcu_segcblist.h bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); rsclp 2415 kernel/rcu/tree_plugin.h struct rcu_segcblist *rsclp = &rdp->cblist; rsclp 2434 kernel/rcu/tree_plugin.h ".D"[rcu_segcblist_ready_cbs(rsclp)], rsclp 2435 kernel/rcu/tree_plugin.h ".W"[!rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)], rsclp 2436 kernel/rcu/tree_plugin.h ".R"[!rcu_segcblist_restempty(rsclp, RCU_WAIT_TAIL)], rsclp 2437 kernel/rcu/tree_plugin.h ".N"[!rcu_segcblist_restempty(rsclp, RCU_NEXT_READY_TAIL)],