This source file includes following definitions.
- rcu_cblist_init
- rcu_cblist_enqueue
- rcu_cblist_flush_enqueue
- rcu_cblist_dequeue
- rcu_segcblist_set_len
- rcu_segcblist_add_len
- rcu_segcblist_inc_len
- rcu_segcblist_xchg_len
- rcu_segcblist_init
- rcu_segcblist_disable
- rcu_segcblist_offload
- rcu_segcblist_ready_cbs
- rcu_segcblist_pend_cbs
- rcu_segcblist_first_cb
- rcu_segcblist_first_pend_cb
- rcu_segcblist_nextgp
- rcu_segcblist_enqueue
- rcu_segcblist_entrain
- rcu_segcblist_extract_count
- rcu_segcblist_extract_done_cbs
- rcu_segcblist_extract_pend_cbs
- rcu_segcblist_insert_count
- rcu_segcblist_insert_done_cbs
- rcu_segcblist_insert_pend_cbs
- rcu_segcblist_advance
- rcu_segcblist_accelerate
- rcu_segcblist_merge
1
2
3
4
5
6
7
8
9
10 #include <linux/types.h>
11 #include <linux/kernel.h>
12 #include <linux/interrupt.h>
13 #include <linux/rcupdate.h>
14
15 #include "rcu_segcblist.h"
16
17
18 void rcu_cblist_init(struct rcu_cblist *rclp)
19 {
20 rclp->head = NULL;
21 rclp->tail = &rclp->head;
22 rclp->len = 0;
23 rclp->len_lazy = 0;
24 }
25
26
27
28
29
30
31
32 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp)
33 {
34 *rclp->tail = rhp;
35 rclp->tail = &rhp->next;
36 WRITE_ONCE(rclp->len, rclp->len + 1);
37 }
38
39
40
41
42
43
44
45
46
47 void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
48 struct rcu_cblist *srclp,
49 struct rcu_head *rhp)
50 {
51 drclp->head = srclp->head;
52 if (drclp->head)
53 drclp->tail = srclp->tail;
54 else
55 drclp->tail = &drclp->head;
56 drclp->len = srclp->len;
57 drclp->len_lazy = srclp->len_lazy;
58 if (!rhp) {
59 rcu_cblist_init(srclp);
60 } else {
61 rhp->next = NULL;
62 srclp->head = rhp;
63 srclp->tail = &rhp->next;
64 WRITE_ONCE(srclp->len, 1);
65 srclp->len_lazy = 0;
66 }
67 }
68
69
70
71
72
73
74
75
76 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp)
77 {
78 struct rcu_head *rhp;
79
80 rhp = rclp->head;
81 if (!rhp)
82 return NULL;
83 rclp->len--;
84 rclp->head = rhp->next;
85 if (!rclp->head)
86 rclp->tail = &rclp->head;
87 return rhp;
88 }
89
90
91 void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v)
92 {
93 #ifdef CONFIG_RCU_NOCB_CPU
94 atomic_long_set(&rsclp->len, v);
95 #else
96 WRITE_ONCE(rsclp->len, v);
97 #endif
98 }
99
100
101
102
103
104
105
106
107 void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)
108 {
109 #ifdef CONFIG_RCU_NOCB_CPU
110 smp_mb__before_atomic();
111 atomic_long_add(v, &rsclp->len);
112 smp_mb__after_atomic();
113 #else
114 smp_mb();
115 WRITE_ONCE(rsclp->len, rsclp->len + v);
116 smp_mb();
117 #endif
118 }
119
120
121
122
123
124
125
126 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp)
127 {
128 rcu_segcblist_add_len(rsclp, 1);
129 }
130
131
132
133
134
135
136
137 long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v)
138 {
139 #ifdef CONFIG_RCU_NOCB_CPU
140 return atomic_long_xchg(&rsclp->len, v);
141 #else
142 long ret = rsclp->len;
143
144 smp_mb();
145 WRITE_ONCE(rsclp->len, v);
146 smp_mb();
147 return ret;
148 #endif
149 }
150
151
152
153
154 void rcu_segcblist_init(struct rcu_segcblist *rsclp)
155 {
156 int i;
157
158 BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq));
159 BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq));
160 rsclp->head = NULL;
161 for (i = 0; i < RCU_CBLIST_NSEGS; i++)
162 rsclp->tails[i] = &rsclp->head;
163 rcu_segcblist_set_len(rsclp, 0);
164 rsclp->len_lazy = 0;
165 rsclp->enabled = 1;
166 }
167
168
169
170
171
172 void rcu_segcblist_disable(struct rcu_segcblist *rsclp)
173 {
174 WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));
175 WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));
176 WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp));
177 rsclp->enabled = 0;
178 }
179
180
181
182
183
184 void rcu_segcblist_offload(struct rcu_segcblist *rsclp)
185 {
186 rsclp->offloaded = 1;
187 }
188
189
190
191
192
193 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
194 {
195 return rcu_segcblist_is_enabled(rsclp) &&
196 &rsclp->head != rsclp->tails[RCU_DONE_TAIL];
197 }
198
199
200
201
202
203 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp)
204 {
205 return rcu_segcblist_is_enabled(rsclp) &&
206 !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL);
207 }
208
209
210
211
212
213 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp)
214 {
215 if (rcu_segcblist_is_enabled(rsclp))
216 return rsclp->head;
217 return NULL;
218 }
219
220
221
222
223
224
225
226
227 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)
228 {
229 if (rcu_segcblist_is_enabled(rsclp))
230 return *rsclp->tails[RCU_DONE_TAIL];
231 return NULL;
232 }
233
234
235
236
237
238 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)
239 {
240 if (!rcu_segcblist_pend_cbs(rsclp))
241 return false;
242 *lp = rsclp->gp_seq[RCU_WAIT_TAIL];
243 return true;
244 }
245
246
247
248
249
250
251
252
253
254
255 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
256 struct rcu_head *rhp, bool lazy)
257 {
258 rcu_segcblist_inc_len(rsclp);
259 if (lazy)
260 rsclp->len_lazy++;
261 smp_mb();
262 rhp->next = NULL;
263 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);
264 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);
265 }
266
267
268
269
270
271
272
273
274
275
276
277 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
278 struct rcu_head *rhp, bool lazy)
279 {
280 int i;
281
282 if (rcu_segcblist_n_cbs(rsclp) == 0)
283 return false;
284 rcu_segcblist_inc_len(rsclp);
285 if (lazy)
286 rsclp->len_lazy++;
287 smp_mb();
288 rhp->next = NULL;
289 for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)
290 if (rsclp->tails[i] != rsclp->tails[i - 1])
291 break;
292 WRITE_ONCE(*rsclp->tails[i], rhp);
293 for (; i <= RCU_NEXT_TAIL; i++)
294 WRITE_ONCE(rsclp->tails[i], &rhp->next);
295 return true;
296 }
297
298
299
300
301
302
303
304
305
306
307 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
308 struct rcu_cblist *rclp)
309 {
310 rclp->len_lazy += rsclp->len_lazy;
311 rsclp->len_lazy = 0;
312 rclp->len = rcu_segcblist_xchg_len(rsclp, 0);
313 }
314
315
316
317
318
319
320 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
321 struct rcu_cblist *rclp)
322 {
323 int i;
324
325 if (!rcu_segcblist_ready_cbs(rsclp))
326 return;
327 *rclp->tail = rsclp->head;
328 WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]);
329 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
330 rclp->tail = rsclp->tails[RCU_DONE_TAIL];
331 for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--)
332 if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])
333 WRITE_ONCE(rsclp->tails[i], &rsclp->head);
334 }
335
336
337
338
339
340
341
342
343 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
344 struct rcu_cblist *rclp)
345 {
346 int i;
347
348 if (!rcu_segcblist_pend_cbs(rsclp))
349 return;
350 *rclp->tail = *rsclp->tails[RCU_DONE_TAIL];
351 rclp->tail = rsclp->tails[RCU_NEXT_TAIL];
352 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
353 for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++)
354 WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);
355 }
356
357
358
359
360
361 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
362 struct rcu_cblist *rclp)
363 {
364 rsclp->len_lazy += rclp->len_lazy;
365 rcu_segcblist_add_len(rsclp, rclp->len);
366 rclp->len_lazy = 0;
367 rclp->len = 0;
368 }
369
370
371
372
373
374 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
375 struct rcu_cblist *rclp)
376 {
377 int i;
378
379 if (!rclp->head)
380 return;
381 *rclp->tail = rsclp->head;
382 WRITE_ONCE(rsclp->head, rclp->head);
383 for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++)
384 if (&rsclp->head == rsclp->tails[i])
385 WRITE_ONCE(rsclp->tails[i], rclp->tail);
386 else
387 break;
388 rclp->head = NULL;
389 rclp->tail = &rclp->head;
390 }
391
392
393
394
395
396 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
397 struct rcu_cblist *rclp)
398 {
399 if (!rclp->head)
400 return;
401 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
402 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
403 rclp->head = NULL;
404 rclp->tail = &rclp->head;
405 }
406
407
408
409
410
411 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
412 {
413 int i, j;
414
415 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
416 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
417 return;
418
419
420
421
422
423 for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {
424 if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
425 break;
426 WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);
427 }
428
429
430 if (i == RCU_WAIT_TAIL)
431 return;
432
433
434 for (j = RCU_WAIT_TAIL; j < i; j++)
435 WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);
436
437
438
439
440
441
442
443 for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {
444 if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])
445 break;
446 WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);
447 rsclp->gp_seq[j] = rsclp->gp_seq[i];
448 }
449 }
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)
467 {
468 int i;
469
470 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
471 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
472 return false;
473
474
475
476
477
478
479
480
481
482 for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--)
483 if (rsclp->tails[i] != rsclp->tails[i - 1] &&
484 ULONG_CMP_LT(rsclp->gp_seq[i], seq))
485 break;
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500 if (++i >= RCU_NEXT_TAIL)
501 return false;
502
503
504
505
506
507
508
509
510 for (; i < RCU_NEXT_TAIL; i++) {
511 WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);
512 rsclp->gp_seq[i] = seq;
513 }
514 return true;
515 }
516
517
518
519
520
521
522
523
524 void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
525 struct rcu_segcblist *src_rsclp)
526 {
527 struct rcu_cblist donecbs;
528 struct rcu_cblist pendcbs;
529
530 rcu_cblist_init(&donecbs);
531 rcu_cblist_init(&pendcbs);
532 rcu_segcblist_extract_count(src_rsclp, &donecbs);
533 rcu_segcblist_extract_done_cbs(src_rsclp, &donecbs);
534 rcu_segcblist_extract_pend_cbs(src_rsclp, &pendcbs);
535 rcu_segcblist_insert_count(dst_rsclp, &donecbs);
536 rcu_segcblist_insert_done_cbs(dst_rsclp, &donecbs);
537 rcu_segcblist_insert_pend_cbs(dst_rsclp, &pendcbs);
538 rcu_segcblist_init(src_rsclp);
539 }