This source file includes following definitions.
- __find_holder
- __add_holder
- __del_holder
- __check_holder
- __wait
- __wake_waiter
- __wake_many
- bl_init
- __available_for_read
- bl_down_read
- bl_down_read_nonblock
- bl_up_read
- bl_down_write
- bl_up_write
- report_recursive_bug
- to_buffer
- dm_block_location
- dm_block_data
- dm_block_manager_alloc_callback
- dm_block_manager_write_callback
- dm_block_manager_create
- dm_block_manager_destroy
- dm_bm_block_size
- dm_bm_nr_blocks
- dm_bm_validate_buffer
- dm_bm_read_lock
- dm_bm_write_lock
- dm_bm_read_try_lock
- dm_bm_write_lock_zero
- dm_bm_unlock
- dm_bm_flush
- dm_bm_prefetch
- dm_bm_is_read_only
- dm_bm_set_read_only
- dm_bm_set_read_write
- dm_bm_checksum
1
2
3
4
5
6 #include "dm-block-manager.h"
7 #include "dm-persistent-data-internal.h"
8
9 #include <linux/dm-bufio.h>
10 #include <linux/crc32c.h>
11 #include <linux/module.h>
12 #include <linux/slab.h>
13 #include <linux/rwsem.h>
14 #include <linux/device-mapper.h>
15 #include <linux/stacktrace.h>
16 #include <linux/sched/task.h>
17
18 #define DM_MSG_PREFIX "block manager"
19
20
21
22 #ifdef CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING
23
24
25
26
27
28
29
30
31
32
33
34
35 #define MAX_HOLDERS 4
36 #define MAX_STACK 10
37
38 struct stack_store {
39 unsigned int nr_entries;
40 unsigned long entries[MAX_STACK];
41 };
42
43 struct block_lock {
44 spinlock_t lock;
45 __s32 count;
46 struct list_head waiters;
47 struct task_struct *holders[MAX_HOLDERS];
48
49 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
50 struct stack_store traces[MAX_HOLDERS];
51 #endif
52 };
53
54 struct waiter {
55 struct list_head list;
56 struct task_struct *task;
57 int wants_write;
58 };
59
60 static unsigned __find_holder(struct block_lock *lock,
61 struct task_struct *task)
62 {
63 unsigned i;
64
65 for (i = 0; i < MAX_HOLDERS; i++)
66 if (lock->holders[i] == task)
67 break;
68
69 BUG_ON(i == MAX_HOLDERS);
70 return i;
71 }
72
73
74 static void __add_holder(struct block_lock *lock, struct task_struct *task)
75 {
76 unsigned h = __find_holder(lock, NULL);
77 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
78 struct stack_store *t;
79 #endif
80
81 get_task_struct(task);
82 lock->holders[h] = task;
83
84 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
85 t = lock->traces + h;
86 t->nr_entries = stack_trace_save(t->entries, MAX_STACK, 2);
87 #endif
88 }
89
90
91 static void __del_holder(struct block_lock *lock, struct task_struct *task)
92 {
93 unsigned h = __find_holder(lock, task);
94 lock->holders[h] = NULL;
95 put_task_struct(task);
96 }
97
98 static int __check_holder(struct block_lock *lock)
99 {
100 unsigned i;
101
102 for (i = 0; i < MAX_HOLDERS; i++) {
103 if (lock->holders[i] == current) {
104 DMERR("recursive lock detected in metadata");
105 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
106 DMERR("previously held here:");
107 stack_trace_print(lock->traces[i].entries,
108 lock->traces[i].nr_entries, 4);
109
110 DMERR("subsequent acquisition attempted here:");
111 dump_stack();
112 #endif
113 return -EINVAL;
114 }
115 }
116
117 return 0;
118 }
119
120 static void __wait(struct waiter *w)
121 {
122 for (;;) {
123 set_current_state(TASK_UNINTERRUPTIBLE);
124
125 if (!w->task)
126 break;
127
128 schedule();
129 }
130
131 set_current_state(TASK_RUNNING);
132 }
133
134 static void __wake_waiter(struct waiter *w)
135 {
136 struct task_struct *task;
137
138 list_del(&w->list);
139 task = w->task;
140 smp_mb();
141 w->task = NULL;
142 wake_up_process(task);
143 }
144
145
146
147
148 static void __wake_many(struct block_lock *lock)
149 {
150 struct waiter *w, *tmp;
151
152 BUG_ON(lock->count < 0);
153 list_for_each_entry_safe(w, tmp, &lock->waiters, list) {
154 if (lock->count >= MAX_HOLDERS)
155 return;
156
157 if (w->wants_write) {
158 if (lock->count > 0)
159 return;
160
161 lock->count = -1;
162 __add_holder(lock, w->task);
163 __wake_waiter(w);
164 return;
165 }
166
167 lock->count++;
168 __add_holder(lock, w->task);
169 __wake_waiter(w);
170 }
171 }
172
173 static void bl_init(struct block_lock *lock)
174 {
175 int i;
176
177 spin_lock_init(&lock->lock);
178 lock->count = 0;
179 INIT_LIST_HEAD(&lock->waiters);
180 for (i = 0; i < MAX_HOLDERS; i++)
181 lock->holders[i] = NULL;
182 }
183
184 static int __available_for_read(struct block_lock *lock)
185 {
186 return lock->count >= 0 &&
187 lock->count < MAX_HOLDERS &&
188 list_empty(&lock->waiters);
189 }
190
191 static int bl_down_read(struct block_lock *lock)
192 {
193 int r;
194 struct waiter w;
195
196 spin_lock(&lock->lock);
197 r = __check_holder(lock);
198 if (r) {
199 spin_unlock(&lock->lock);
200 return r;
201 }
202
203 if (__available_for_read(lock)) {
204 lock->count++;
205 __add_holder(lock, current);
206 spin_unlock(&lock->lock);
207 return 0;
208 }
209
210 get_task_struct(current);
211
212 w.task = current;
213 w.wants_write = 0;
214 list_add_tail(&w.list, &lock->waiters);
215 spin_unlock(&lock->lock);
216
217 __wait(&w);
218 put_task_struct(current);
219 return 0;
220 }
221
222 static int bl_down_read_nonblock(struct block_lock *lock)
223 {
224 int r;
225
226 spin_lock(&lock->lock);
227 r = __check_holder(lock);
228 if (r)
229 goto out;
230
231 if (__available_for_read(lock)) {
232 lock->count++;
233 __add_holder(lock, current);
234 r = 0;
235 } else
236 r = -EWOULDBLOCK;
237
238 out:
239 spin_unlock(&lock->lock);
240 return r;
241 }
242
243 static void bl_up_read(struct block_lock *lock)
244 {
245 spin_lock(&lock->lock);
246 BUG_ON(lock->count <= 0);
247 __del_holder(lock, current);
248 --lock->count;
249 if (!list_empty(&lock->waiters))
250 __wake_many(lock);
251 spin_unlock(&lock->lock);
252 }
253
254 static int bl_down_write(struct block_lock *lock)
255 {
256 int r;
257 struct waiter w;
258
259 spin_lock(&lock->lock);
260 r = __check_holder(lock);
261 if (r) {
262 spin_unlock(&lock->lock);
263 return r;
264 }
265
266 if (lock->count == 0 && list_empty(&lock->waiters)) {
267 lock->count = -1;
268 __add_holder(lock, current);
269 spin_unlock(&lock->lock);
270 return 0;
271 }
272
273 get_task_struct(current);
274 w.task = current;
275 w.wants_write = 1;
276
277
278
279
280
281 list_add(&w.list, &lock->waiters);
282 spin_unlock(&lock->lock);
283
284 __wait(&w);
285 put_task_struct(current);
286
287 return 0;
288 }
289
290 static void bl_up_write(struct block_lock *lock)
291 {
292 spin_lock(&lock->lock);
293 __del_holder(lock, current);
294 lock->count = 0;
295 if (!list_empty(&lock->waiters))
296 __wake_many(lock);
297 spin_unlock(&lock->lock);
298 }
299
300 static void report_recursive_bug(dm_block_t b, int r)
301 {
302 if (r == -EINVAL)
303 DMERR("recursive acquisition of block %llu requested.",
304 (unsigned long long) b);
305 }
306
307 #else
308
309 #define bl_init(x) do { } while (0)
310 #define bl_down_read(x) 0
311 #define bl_down_read_nonblock(x) 0
312 #define bl_up_read(x) do { } while (0)
313 #define bl_down_write(x) 0
314 #define bl_up_write(x) do { } while (0)
315 #define report_recursive_bug(x, y) do { } while (0)
316
317 #endif
318
319
320
321
322
323
324
325
326
327
328 static struct dm_buffer *to_buffer(struct dm_block *b)
329 {
330 return (struct dm_buffer *) b;
331 }
332
333 dm_block_t dm_block_location(struct dm_block *b)
334 {
335 return dm_bufio_get_block_number(to_buffer(b));
336 }
337 EXPORT_SYMBOL_GPL(dm_block_location);
338
339 void *dm_block_data(struct dm_block *b)
340 {
341 return dm_bufio_get_block_data(to_buffer(b));
342 }
343 EXPORT_SYMBOL_GPL(dm_block_data);
344
345 struct buffer_aux {
346 struct dm_block_validator *validator;
347 int write_locked;
348
349 #ifdef CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING
350 struct block_lock lock;
351 #endif
352 };
353
354 static void dm_block_manager_alloc_callback(struct dm_buffer *buf)
355 {
356 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
357 aux->validator = NULL;
358 bl_init(&aux->lock);
359 }
360
361 static void dm_block_manager_write_callback(struct dm_buffer *buf)
362 {
363 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
364 if (aux->validator) {
365 aux->validator->prepare_for_write(aux->validator, (struct dm_block *) buf,
366 dm_bufio_get_block_size(dm_bufio_get_client(buf)));
367 }
368 }
369
370
371
372
373 struct dm_block_manager {
374 struct dm_bufio_client *bufio;
375 bool read_only:1;
376 };
377
378 struct dm_block_manager *dm_block_manager_create(struct block_device *bdev,
379 unsigned block_size,
380 unsigned max_held_per_thread)
381 {
382 int r;
383 struct dm_block_manager *bm;
384
385 bm = kmalloc(sizeof(*bm), GFP_KERNEL);
386 if (!bm) {
387 r = -ENOMEM;
388 goto bad;
389 }
390
391 bm->bufio = dm_bufio_client_create(bdev, block_size, max_held_per_thread,
392 sizeof(struct buffer_aux),
393 dm_block_manager_alloc_callback,
394 dm_block_manager_write_callback);
395 if (IS_ERR(bm->bufio)) {
396 r = PTR_ERR(bm->bufio);
397 kfree(bm);
398 goto bad;
399 }
400
401 bm->read_only = false;
402
403 return bm;
404
405 bad:
406 return ERR_PTR(r);
407 }
408 EXPORT_SYMBOL_GPL(dm_block_manager_create);
409
410 void dm_block_manager_destroy(struct dm_block_manager *bm)
411 {
412 dm_bufio_client_destroy(bm->bufio);
413 kfree(bm);
414 }
415 EXPORT_SYMBOL_GPL(dm_block_manager_destroy);
416
417 unsigned dm_bm_block_size(struct dm_block_manager *bm)
418 {
419 return dm_bufio_get_block_size(bm->bufio);
420 }
421 EXPORT_SYMBOL_GPL(dm_bm_block_size);
422
423 dm_block_t dm_bm_nr_blocks(struct dm_block_manager *bm)
424 {
425 return dm_bufio_get_device_size(bm->bufio);
426 }
427
428 static int dm_bm_validate_buffer(struct dm_block_manager *bm,
429 struct dm_buffer *buf,
430 struct buffer_aux *aux,
431 struct dm_block_validator *v)
432 {
433 if (unlikely(!aux->validator)) {
434 int r;
435 if (!v)
436 return 0;
437 r = v->check(v, (struct dm_block *) buf, dm_bufio_get_block_size(bm->bufio));
438 if (unlikely(r)) {
439 DMERR_LIMIT("%s validator check failed for block %llu", v->name,
440 (unsigned long long) dm_bufio_get_block_number(buf));
441 return r;
442 }
443 aux->validator = v;
444 } else {
445 if (unlikely(aux->validator != v)) {
446 DMERR_LIMIT("validator mismatch (old=%s vs new=%s) for block %llu",
447 aux->validator->name, v ? v->name : "NULL",
448 (unsigned long long) dm_bufio_get_block_number(buf));
449 return -EINVAL;
450 }
451 }
452
453 return 0;
454 }
455 int dm_bm_read_lock(struct dm_block_manager *bm, dm_block_t b,
456 struct dm_block_validator *v,
457 struct dm_block **result)
458 {
459 struct buffer_aux *aux;
460 void *p;
461 int r;
462
463 p = dm_bufio_read(bm->bufio, b, (struct dm_buffer **) result);
464 if (IS_ERR(p))
465 return PTR_ERR(p);
466
467 aux = dm_bufio_get_aux_data(to_buffer(*result));
468 r = bl_down_read(&aux->lock);
469 if (unlikely(r)) {
470 dm_bufio_release(to_buffer(*result));
471 report_recursive_bug(b, r);
472 return r;
473 }
474
475 aux->write_locked = 0;
476
477 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);
478 if (unlikely(r)) {
479 bl_up_read(&aux->lock);
480 dm_bufio_release(to_buffer(*result));
481 return r;
482 }
483
484 return 0;
485 }
486 EXPORT_SYMBOL_GPL(dm_bm_read_lock);
487
488 int dm_bm_write_lock(struct dm_block_manager *bm,
489 dm_block_t b, struct dm_block_validator *v,
490 struct dm_block **result)
491 {
492 struct buffer_aux *aux;
493 void *p;
494 int r;
495
496 if (bm->read_only)
497 return -EPERM;
498
499 p = dm_bufio_read(bm->bufio, b, (struct dm_buffer **) result);
500 if (IS_ERR(p))
501 return PTR_ERR(p);
502
503 aux = dm_bufio_get_aux_data(to_buffer(*result));
504 r = bl_down_write(&aux->lock);
505 if (r) {
506 dm_bufio_release(to_buffer(*result));
507 report_recursive_bug(b, r);
508 return r;
509 }
510
511 aux->write_locked = 1;
512
513 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);
514 if (unlikely(r)) {
515 bl_up_write(&aux->lock);
516 dm_bufio_release(to_buffer(*result));
517 return r;
518 }
519
520 return 0;
521 }
522 EXPORT_SYMBOL_GPL(dm_bm_write_lock);
523
524 int dm_bm_read_try_lock(struct dm_block_manager *bm,
525 dm_block_t b, struct dm_block_validator *v,
526 struct dm_block **result)
527 {
528 struct buffer_aux *aux;
529 void *p;
530 int r;
531
532 p = dm_bufio_get(bm->bufio, b, (struct dm_buffer **) result);
533 if (IS_ERR(p))
534 return PTR_ERR(p);
535 if (unlikely(!p))
536 return -EWOULDBLOCK;
537
538 aux = dm_bufio_get_aux_data(to_buffer(*result));
539 r = bl_down_read_nonblock(&aux->lock);
540 if (r < 0) {
541 dm_bufio_release(to_buffer(*result));
542 report_recursive_bug(b, r);
543 return r;
544 }
545 aux->write_locked = 0;
546
547 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);
548 if (unlikely(r)) {
549 bl_up_read(&aux->lock);
550 dm_bufio_release(to_buffer(*result));
551 return r;
552 }
553
554 return 0;
555 }
556
557 int dm_bm_write_lock_zero(struct dm_block_manager *bm,
558 dm_block_t b, struct dm_block_validator *v,
559 struct dm_block **result)
560 {
561 int r;
562 struct buffer_aux *aux;
563 void *p;
564
565 if (bm->read_only)
566 return -EPERM;
567
568 p = dm_bufio_new(bm->bufio, b, (struct dm_buffer **) result);
569 if (IS_ERR(p))
570 return PTR_ERR(p);
571
572 memset(p, 0, dm_bm_block_size(bm));
573
574 aux = dm_bufio_get_aux_data(to_buffer(*result));
575 r = bl_down_write(&aux->lock);
576 if (r) {
577 dm_bufio_release(to_buffer(*result));
578 return r;
579 }
580
581 aux->write_locked = 1;
582 aux->validator = v;
583
584 return 0;
585 }
586 EXPORT_SYMBOL_GPL(dm_bm_write_lock_zero);
587
588 void dm_bm_unlock(struct dm_block *b)
589 {
590 struct buffer_aux *aux;
591 aux = dm_bufio_get_aux_data(to_buffer(b));
592
593 if (aux->write_locked) {
594 dm_bufio_mark_buffer_dirty(to_buffer(b));
595 bl_up_write(&aux->lock);
596 } else
597 bl_up_read(&aux->lock);
598
599 dm_bufio_release(to_buffer(b));
600 }
601 EXPORT_SYMBOL_GPL(dm_bm_unlock);
602
603 int dm_bm_flush(struct dm_block_manager *bm)
604 {
605 if (bm->read_only)
606 return -EPERM;
607
608 return dm_bufio_write_dirty_buffers(bm->bufio);
609 }
610 EXPORT_SYMBOL_GPL(dm_bm_flush);
611
612 void dm_bm_prefetch(struct dm_block_manager *bm, dm_block_t b)
613 {
614 dm_bufio_prefetch(bm->bufio, b, 1);
615 }
616
617 bool dm_bm_is_read_only(struct dm_block_manager *bm)
618 {
619 return bm->read_only;
620 }
621 EXPORT_SYMBOL_GPL(dm_bm_is_read_only);
622
623 void dm_bm_set_read_only(struct dm_block_manager *bm)
624 {
625 bm->read_only = true;
626 }
627 EXPORT_SYMBOL_GPL(dm_bm_set_read_only);
628
629 void dm_bm_set_read_write(struct dm_block_manager *bm)
630 {
631 bm->read_only = false;
632 }
633 EXPORT_SYMBOL_GPL(dm_bm_set_read_write);
634
635 u32 dm_bm_checksum(const void *data, size_t len, u32 init_xor)
636 {
637 return crc32c(~(u32) 0, data, len) ^ init_xor;
638 }
639 EXPORT_SYMBOL_GPL(dm_bm_checksum);
640
641
642
643 MODULE_LICENSE("GPL");
644 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
645 MODULE_DESCRIPTION("Immutable metadata library for dm");
646
647