cblist 214 kernel/rcu/tree.c if (rcu_segcblist_is_enabled(&rdp->cblist)) cblist 215 kernel/rcu/tree.c return rcu_segcblist_n_cbs(&rdp->cblist); cblist 1266 kernel/rcu/tree.c if (!rcu_segcblist_pend_cbs(&rdp->cblist)) cblist 1280 kernel/rcu/tree.c if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) cblist 1284 kernel/rcu/tree.c if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) cblist 1308 kernel/rcu/tree.c (void)rcu_segcblist_accelerate(&rdp->cblist, c); cblist 1334 kernel/rcu/tree.c if (!rcu_segcblist_pend_cbs(&rdp->cblist)) cblist 1341 kernel/rcu/tree.c rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); cblist 1373 kernel/rcu/tree.c rcu_segcblist_is_offloaded(&rdp->cblist); cblist 1756 kernel/rcu/tree.c rcu_segcblist_is_offloaded(&rdp->cblist); cblist 1953 kernel/rcu/tree.c rcu_segcblist_is_offloaded(&rdp->cblist); cblist 2116 kernel/rcu/tree.c rcu_segcblist_is_offloaded(&rdp->cblist); cblist 2123 kernel/rcu/tree.c if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { cblist 2125 kernel/rcu/tree.c rcu_segcblist_n_lazy_cbs(&rdp->cblist), cblist 2126 kernel/rcu/tree.c rcu_segcblist_n_cbs(&rdp->cblist), 0); cblist 2128 kernel/rcu/tree.c !rcu_segcblist_empty(&rdp->cblist), cblist 2142 kernel/rcu/tree.c pending = rcu_segcblist_n_cbs(&rdp->cblist); cblist 2147 kernel/rcu/tree.c rcu_segcblist_n_lazy_cbs(&rdp->cblist), cblist 2148 kernel/rcu/tree.c rcu_segcblist_n_cbs(&rdp->cblist), bl); cblist 2149 kernel/rcu/tree.c rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); cblist 2151 kernel/rcu/tree.c rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); cblist 2192 kernel/rcu/tree.c rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); cblist 2194 kernel/rcu/tree.c rcu_segcblist_insert_count(&rdp->cblist, &rcl); cblist 2197 kernel/rcu/tree.c count = rcu_segcblist_n_cbs(&rdp->cblist); cblist 2212 kernel/rcu/tree.c WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); cblist 2214 kernel/rcu/tree.c count != 0 && rcu_segcblist_empty(&rdp->cblist)); cblist 2219 kernel/rcu/tree.c if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist)) cblist 2346 kernel/rcu/tree.c rcu_segcblist_is_offloaded(&rdp->cblist); cblist 2366 kernel/rcu/tree.c rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) { cblist 2368 kernel/rcu/tree.c if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) cblist 2376 kernel/rcu/tree.c if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) && cblist 2521 kernel/rcu/tree.c if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > cblist 2534 kernel/rcu/tree.c rcu_segcblist_first_pend_cb(&rdp->cblist) != head) cblist 2537 kernel/rcu/tree.c rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); cblist 2582 kernel/rcu/tree.c if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { cblist 2588 kernel/rcu/tree.c if (rcu_segcblist_empty(&rdp->cblist)) cblist 2589 kernel/rcu/tree.c rcu_segcblist_init(&rdp->cblist); cblist 2594 kernel/rcu/tree.c rcu_segcblist_enqueue(&rdp->cblist, head, lazy); cblist 2598 kernel/rcu/tree.c rcu_segcblist_n_lazy_cbs(&rdp->cblist), cblist 2599 kernel/rcu/tree.c rcu_segcblist_n_cbs(&rdp->cblist)); cblist 2602 kernel/rcu/tree.c rcu_segcblist_n_lazy_cbs(&rdp->cblist), cblist 2603 kernel/rcu/tree.c rcu_segcblist_n_cbs(&rdp->cblist)); cblist 2607 kernel/rcu/tree.c unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) { cblist 2811 kernel/rcu/tree.c if (rcu_segcblist_ready_cbs(&rdp->cblist)) cblist 2816 kernel/rcu/tree.c rcu_segcblist_is_enabled(&rdp->cblist) && cblist 2818 kernel/rcu/tree.c !rcu_segcblist_is_offloaded(&rdp->cblist)) && cblist 2819 kernel/rcu/tree.c !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) cblist 2868 kernel/rcu/tree.c if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) { cblist 2928 kernel/rcu/tree.c !rcu_segcblist_is_offloaded(&rdp->cblist)) cblist 2930 kernel/rcu/tree.c if (rcu_segcblist_n_cbs(&rdp->cblist)) { cblist 3029 kernel/rcu/tree.c if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */ cblist 3030 kernel/rcu/tree.c !rcu_segcblist_is_offloaded(&rdp->cblist)) cblist 3031 kernel/rcu/tree.c rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ cblist 3213 kernel/rcu/tree.c if (rcu_segcblist_is_offloaded(&rdp->cblist) || cblist 3214 kernel/rcu/tree.c rcu_segcblist_empty(&rdp->cblist)) cblist 3226 kernel/rcu/tree.c rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); cblist 3228 kernel/rcu/tree.c rcu_segcblist_disable(&rdp->cblist); cblist 3229 kernel/rcu/tree.c WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != cblist 3230 kernel/rcu/tree.c !rcu_segcblist_n_cbs(&my_rdp->cblist)); cblist 3231 kernel/rcu/tree.c if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) { cblist 3241 kernel/rcu/tree.c WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || cblist 3242 kernel/rcu/tree.c !rcu_segcblist_empty(&rdp->cblist), cblist 3244 kernel/rcu/tree.c cpu, rcu_segcblist_n_cbs(&rdp->cblist), cblist 3245 kernel/rcu/tree.c rcu_segcblist_first_cb(&rdp->cblist)); cblist 168 kernel/rcu/tree.h struct rcu_segcblist cblist; /* Segmented callback list, with */ cblist 455 kernel/rcu/tree.h if (!rcu_segcblist_is_offloaded(&(rdp)->cblist)) \ cblist 1241 kernel/rcu/tree_plugin.h return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && cblist 1242 kernel/rcu/tree_plugin.h !rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist); cblist 1320 kernel/rcu/tree_plugin.h rcu_segcblist_pend_cbs(&rdp->cblist)) cblist 1323 kernel/rcu/tree_plugin.h if (rcu_segcblist_ready_cbs(&rdp->cblist)) cblist 1344 kernel/rcu/tree_plugin.h if (rcu_segcblist_empty(&rdp->cblist) || cblist 1345 kernel/rcu/tree_plugin.h rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist)) { cblist 1359 kernel/rcu/tree_plugin.h rdp->all_lazy = !rcu_segcblist_n_nonlazy_cbs(&rdp->cblist); cblist 1388 kernel/rcu/tree_plugin.h if (rcu_segcblist_is_offloaded(&rdp->cblist)) cblist 1394 kernel/rcu/tree_plugin.h if (!rcu_segcblist_empty(&rdp->cblist)) cblist 1407 kernel/rcu/tree_plugin.h if (rdp->all_lazy && rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)) { cblist 1420 kernel/rcu/tree_plugin.h if (rcu_segcblist_pend_cbs(&rdp->cblist)) { cblist 1440 kernel/rcu/tree_plugin.h if (rcu_segcblist_is_offloaded(&rdp->cblist)) cblist 1569 kernel/rcu/tree_plugin.h if (!rcu_segcblist_is_offloaded(&rdp->cblist)) cblist 1580 kernel/rcu/tree_plugin.h if (rcu_segcblist_is_offloaded(&rdp->cblist)) { cblist 1593 kernel/rcu/tree_plugin.h if (rcu_segcblist_is_offloaded(&rdp->cblist)) { cblist 1605 kernel/rcu/tree_plugin.h if (rcu_segcblist_is_offloaded(&rdp->cblist) && cblist 1696 kernel/rcu/tree_plugin.h WARN_ON_ONCE(!rcu_segcblist_is_offloaded(&rdp->cblist)); cblist 1705 kernel/rcu/tree_plugin.h rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ cblist 1707 kernel/rcu/tree_plugin.h rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); cblist 1724 kernel/rcu/tree_plugin.h if (!rcu_segcblist_is_offloaded(&rdp->cblist)) cblist 1738 kernel/rcu/tree_plugin.h if (!rcu_segcblist_is_offloaded(&rdp->cblist) || cblist 1770 kernel/rcu/tree_plugin.h if (!rcu_segcblist_is_offloaded(&rdp->cblist)) { cblist 1771 kernel/rcu/tree_plugin.h *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); cblist 1780 kernel/rcu/tree_plugin.h *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); cblist 1804 kernel/rcu/tree_plugin.h *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); cblist 1819 kernel/rcu/tree_plugin.h *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); cblist 1827 kernel/rcu/tree_plugin.h rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && cblist 1840 kernel/rcu/tree_plugin.h rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ cblist 1853 kernel/rcu/tree_plugin.h if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { cblist 1890 kernel/rcu/tree_plugin.h len = rcu_segcblist_n_cbs(&rdp->cblist); cblist 1908 kernel/rcu/tree_plugin.h rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && cblist 1915 kernel/rcu/tree_plugin.h !rcu_segcblist_ready_cbs(&rdp->cblist)) && cblist 1974 kernel/rcu/tree_plugin.h } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { cblist 1991 kernel/rcu/tree_plugin.h if (!rcu_segcblist_restempty(&rdp->cblist, cblist 1993 kernel/rcu/tree_plugin.h (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && cblist 2000 kernel/rcu/tree_plugin.h WARN_ON_ONCE(!rcu_segcblist_restempty(&rdp->cblist, cblist 2002 kernel/rcu/tree_plugin.h if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { cblist 2010 kernel/rcu/tree_plugin.h if (rcu_segcblist_ready_cbs(&rdp->cblist)) { cblist 2106 kernel/rcu/tree_plugin.h if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && cblist 2112 kernel/rcu/tree_plugin.h if (rcu_segcblist_ready_cbs(&rdp->cblist)) { cblist 2233 kernel/rcu/tree_plugin.h if (rcu_segcblist_empty(&rdp->cblist)) cblist 2234 kernel/rcu/tree_plugin.h rcu_segcblist_init(&rdp->cblist); cblist 2235 kernel/rcu/tree_plugin.h rcu_segcblist_offload(&rdp->cblist); cblist 2415 kernel/rcu/tree_plugin.h struct rcu_segcblist *rsclp = &rdp->cblist; cblist 2439 kernel/rcu/tree_plugin.h rcu_segcblist_n_cbs(&rdp->cblist)); cblist 269 kernel/rcu/tree_stall.h ".L"[!rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)], cblist 594 kernel/rcu/tree_stall.h if (rcu_segcblist_is_offloaded(&rdp->cblist))