drivers/block/drbd/drbd

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
__bm_print_lock_info
drbd_bm_lock
drbd_bm_unlock
bm_store_page_idx
bm_page_to_idx
bm_page_lock_io
bm_page_unlock_io
bm_set_page_unchanged
bm_set_page_need_writeout
drbd_bm_reset_al_hints
drbd_bm_mark_for_writeout
bm_test_page_unchanged
bm_set_page_io_err
bm_clear_page_io_err
bm_set_page_lazy_writeout
bm_test_page_lazy_writeout
bm_word_to_page_idx
bm_bit_to_page_idx
__bm_map_pidx
bm_map_pidx
__bm_unmap
bm_unmap
bm_free_pages
bm_vk_free
bm_realloc_pages
drbd_bm_init
drbd_bm_capacity
drbd_bm_cleanup
bm_clear_surplus
bm_set_surplus
bm_count_bits
bm_memset
drbd_md_on_disk_bits
drbd_bm_resize
_drbd_bm_total_weight
drbd_bm_total_weight
drbd_bm_words
drbd_bm_bits
drbd_bm_merge_lel
drbd_bm_get_lel
drbd_bm_set_all
drbd_bm_clear_all
drbd_bm_aio_ctx_destroy
drbd_bm_endio
bm_page_io_async
bm_rw
drbd_bm_read
drbd_bm_write
drbd_bm_write_all
drbd_bm_write_lazy
drbd_bm_write_copy_pages
drbd_bm_write_hinted
__bm_find_next
bm_find_next
drbd_bm_find_next
drbd_bm_find_next_zero
_drbd_bm_find_next
_drbd_bm_find_next_zero
__bm_change_bits_to
bm_change_bits_to
drbd_bm_set_bits
drbd_bm_clear_bits
bm_set_full_words_within_one_page
_drbd_bm_set_bits
drbd_bm_test_bit
drbd_bm_count_bits
drbd_bm_e_weight
   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3    drbd_bitmap.c
   4 
   5    This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
   6 
   7    Copyright (C) 2004-2008, LINBIT Information Technologies GmbH.
   8    Copyright (C) 2004-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   9    Copyright (C) 2004-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
  10 
  11  */
  12 
  13 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  14 
  15 #include <linux/bitmap.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/string.h>
  18 #include <linux/drbd.h>
  19 #include <linux/slab.h>
  20 #include <linux/highmem.h>
  21 
  22 #include "drbd_int.h"
  23 
  24 
  25 /* OPAQUE outside this file!
  26  * interface defined in drbd_int.h
  27 
  28  * convention:
  29  * function name drbd_bm_... => used elsewhere, "public".
  30  * function name      bm_... => internal to implementation, "private".
  31  */
  32 
  33 
  34 /*
  35  * LIMITATIONS:
  36  * We want to support >= peta byte of backend storage, while for now still using
  37  * a granularity of one bit per 4KiB of storage.
  38  * 1 << 50              bytes backend storage (1 PiB)
  39  * 1 << (50 - 12)       bits needed
  40  *      38 --> we need u64 to index and count bits
  41  * 1 << (38 - 3)        bitmap bytes needed
  42  *      35 --> we still need u64 to index and count bytes
  43  *                      (that's 32 GiB of bitmap for 1 PiB storage)
  44  * 1 << (35 - 2)        32bit longs needed
  45  *      33 --> we'd even need u64 to index and count 32bit long words.
  46  * 1 << (35 - 3)        64bit longs needed
  47  *      32 --> we could get away with a 32bit unsigned int to index and count
  48  *      64bit long words, but I rather stay with unsigned long for now.
  49  *      We probably should neither count nor point to bytes or long words
  50  *      directly, but either by bitnumber, or by page index and offset.
  51  * 1 << (35 - 12)
  52  *      22 --> we need that much 4KiB pages of bitmap.
  53  *      1 << (22 + 3) --> on a 64bit arch,
  54  *      we need 32 MiB to store the array of page pointers.
  55  *
  56  * Because I'm lazy, and because the resulting patch was too large, too ugly
  57  * and still incomplete, on 32bit we still "only" support 16 TiB (minus some),
  58  * (1 << 32) bits * 4k storage.
  59  *
  60 
  61  * bitmap storage and IO:
  62  *      Bitmap is stored little endian on disk, and is kept little endian in
  63  *      core memory. Currently we still hold the full bitmap in core as long
  64  *      as we are "attached" to a local disk, which at 32 GiB for 1PiB storage
  65  *      seems excessive.
  66  *
  67  *      We plan to reduce the amount of in-core bitmap pages by paging them in
  68  *      and out against their on-disk location as necessary, but need to make
  69  *      sure we don't cause too much meta data IO, and must not deadlock in
  70  *      tight memory situations. This needs some more work.
  71  */
  72 
  73 /*
  74  * NOTE
  75  *  Access to the *bm_pages is protected by bm_lock.
  76  *  It is safe to read the other members within the lock.
  77  *
  78  *  drbd_bm_set_bits is called from bio_endio callbacks,
  79  *  We may be called with irq already disabled,
  80  *  so we need spin_lock_irqsave().
  81  *  And we need the kmap_atomic.
  82  */
  83 struct drbd_bitmap {
  84         struct page **bm_pages;
  85         spinlock_t bm_lock;
  86 
  87         /* exclusively to be used by __al_write_transaction(),
  88          * drbd_bm_mark_for_writeout() and
  89          * and drbd_bm_write_hinted() -> bm_rw() called from there.
  90          */
  91         unsigned int n_bitmap_hints;
  92         unsigned int al_bitmap_hints[AL_UPDATES_PER_TRANSACTION];
  93 
  94         /* see LIMITATIONS: above */
  95 
  96         unsigned long bm_set;       /* nr of set bits; THINK maybe atomic_t? */
  97         unsigned long bm_bits;
  98         size_t   bm_words;
  99         size_t   bm_number_of_pages;
 100         sector_t bm_dev_capacity;
 101         struct mutex bm_change; /* serializes resize operations */
 102 
 103         wait_queue_head_t bm_io_wait; /* used to serialize IO of single pages */
 104 
 105         enum bm_flag bm_flags;
 106 
 107         /* debugging aid, in case we are still racy somewhere */
 108         char          *bm_why;
 109         struct task_struct *bm_task;
 110 };
 111 
 112 #define bm_print_lock_info(m) __bm_print_lock_info(m, __func__)
 113 static void __bm_print_lock_info(struct drbd_device *device, const char *func)
 114 {
 115         struct drbd_bitmap *b = device->bitmap;
 116         if (!__ratelimit(&drbd_ratelimit_state))
 117                 return;
 118         drbd_err(device, "FIXME %s[%d] in %s, bitmap locked for '%s' by %s[%d]\n",
 119                  current->comm, task_pid_nr(current),
 120                  func, b->bm_why ?: "?",
 121                  b->bm_task->comm, task_pid_nr(b->bm_task));
 122 }
 123 
 124 void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags)
 125 {
 126         struct drbd_bitmap *b = device->bitmap;
 127         int trylock_failed;
 128 
 129         if (!b) {
 130                 drbd_err(device, "FIXME no bitmap in drbd_bm_lock!?\n");
 131                 return;
 132         }
 133 
 134         trylock_failed = !mutex_trylock(&b->bm_change);
 135 
 136         if (trylock_failed) {
 137                 drbd_warn(device, "%s[%d] going to '%s' but bitmap already locked for '%s' by %s[%d]\n",
 138                           current->comm, task_pid_nr(current),
 139                           why, b->bm_why ?: "?",
 140                           b->bm_task->comm, task_pid_nr(b->bm_task));
 141                 mutex_lock(&b->bm_change);
 142         }
 143         if (BM_LOCKED_MASK & b->bm_flags)
 144                 drbd_err(device, "FIXME bitmap already locked in bm_lock\n");
 145         b->bm_flags |= flags & BM_LOCKED_MASK;
 146 
 147         b->bm_why  = why;
 148         b->bm_task = current;
 149 }
 150 
 151 void drbd_bm_unlock(struct drbd_device *device)
 152 {
 153         struct drbd_bitmap *b = device->bitmap;
 154         if (!b) {
 155                 drbd_err(device, "FIXME no bitmap in drbd_bm_unlock!?\n");
 156                 return;
 157         }
 158 
 159         if (!(BM_LOCKED_MASK & device->bitmap->bm_flags))
 160                 drbd_err(device, "FIXME bitmap not locked in bm_unlock\n");
 161 
 162         b->bm_flags &= ~BM_LOCKED_MASK;
 163         b->bm_why  = NULL;
 164         b->bm_task = NULL;
 165         mutex_unlock(&b->bm_change);
 166 }
 167 
 168 /* we store some "meta" info about our pages in page->private */
 169 /* at a granularity of 4k storage per bitmap bit:
 170  * one peta byte storage: 1<<50 byte, 1<<38 * 4k storage blocks
 171  *  1<<38 bits,
 172  *  1<<23 4k bitmap pages.
 173  * Use 24 bits as page index, covers 2 peta byte storage
 174  * at a granularity of 4k per bit.
 175  * Used to report the failed page idx on io error from the endio handlers.
 176  */
 177 #define BM_PAGE_IDX_MASK        ((1UL<<24)-1)
 178 /* this page is currently read in, or written back */
 179 #define BM_PAGE_IO_LOCK         31
 180 /* if there has been an IO error for this page */
 181 #define BM_PAGE_IO_ERROR        30
 182 /* this is to be able to intelligently skip disk IO,
 183  * set if bits have been set since last IO. */
 184 #define BM_PAGE_NEED_WRITEOUT   29
 185 /* to mark for lazy writeout once syncer cleared all clearable bits,
 186  * we if bits have been cleared since last IO. */
 187 #define BM_PAGE_LAZY_WRITEOUT   28
 188 /* pages marked with this "HINT" will be considered for writeout
 189  * on activity log transactions */
 190 #define BM_PAGE_HINT_WRITEOUT   27
 191 
 192 /* store_page_idx uses non-atomic assignment. It is only used directly after
 193  * allocating the page.  All other bm_set_page_* and bm_clear_page_* need to
 194  * use atomic bit manipulation, as set_out_of_sync (and therefore bitmap
 195  * changes) may happen from various contexts, and wait_on_bit/wake_up_bit
 196  * requires it all to be atomic as well. */
 197 static void bm_store_page_idx(struct page *page, unsigned long idx)
 198 {
 199         BUG_ON(0 != (idx & ~BM_PAGE_IDX_MASK));
 200         set_page_private(page, idx);
 201 }
 202 
 203 static unsigned long bm_page_to_idx(struct page *page)
 204 {
 205         return page_private(page) & BM_PAGE_IDX_MASK;
 206 }
 207 
 208 /* As is very unlikely that the same page is under IO from more than one
 209  * context, we can get away with a bit per page and one wait queue per bitmap.
 210  */
 211 static void bm_page_lock_io(struct drbd_device *device, int page_nr)
 212 {
 213         struct drbd_bitmap *b = device->bitmap;
 214         void *addr = &page_private(b->bm_pages[page_nr]);
 215         wait_event(b->bm_io_wait, !test_and_set_bit(BM_PAGE_IO_LOCK, addr));
 216 }
 217 
 218 static void bm_page_unlock_io(struct drbd_device *device, int page_nr)
 219 {
 220         struct drbd_bitmap *b = device->bitmap;
 221         void *addr = &page_private(b->bm_pages[page_nr]);
 222         clear_bit_unlock(BM_PAGE_IO_LOCK, addr);
 223         wake_up(&device->bitmap->bm_io_wait);
 224 }
 225 
 226 /* set _before_ submit_io, so it may be reset due to being changed
 227  * while this page is in flight... will get submitted later again */
 228 static void bm_set_page_unchanged(struct page *page)
 229 {
 230         /* use cmpxchg? */
 231         clear_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page));
 232         clear_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));
 233 }
 234 
 235 static void bm_set_page_need_writeout(struct page *page)
 236 {
 237         set_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page));
 238 }
 239 
 240 void drbd_bm_reset_al_hints(struct drbd_device *device)
 241 {
 242         device->bitmap->n_bitmap_hints = 0;
 243 }
 244 
 245 /**
 246  * drbd_bm_mark_for_writeout() - mark a page with a "hint" to be considered for writeout
 247  * @device:     DRBD device.
 248  * @page_nr:    the bitmap page to mark with the "hint" flag
 249  *
 250  * From within an activity log transaction, we mark a few pages with these
 251  * hints, then call drbd_bm_write_hinted(), which will only write out changed
 252  * pages which are flagged with this mark.
 253  */
 254 void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr)
 255 {
 256         struct drbd_bitmap *b = device->bitmap;
 257         struct page *page;
 258         if (page_nr >= device->bitmap->bm_number_of_pages) {
 259                 drbd_warn(device, "BAD: page_nr: %u, number_of_pages: %u\n",
 260                          page_nr, (int)device->bitmap->bm_number_of_pages);
 261                 return;
 262         }
 263         page = device->bitmap->bm_pages[page_nr];
 264         BUG_ON(b->n_bitmap_hints >= ARRAY_SIZE(b->al_bitmap_hints));
 265         if (!test_and_set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page)))
 266                 b->al_bitmap_hints[b->n_bitmap_hints++] = page_nr;
 267 }
 268 
 269 static int bm_test_page_unchanged(struct page *page)
 270 {
 271         volatile const unsigned long *addr = &page_private(page);
 272         return (*addr & ((1UL<<BM_PAGE_NEED_WRITEOUT)|(1UL<<BM_PAGE_LAZY_WRITEOUT))) == 0;
 273 }
 274 
 275 static void bm_set_page_io_err(struct page *page)
 276 {
 277         set_bit(BM_PAGE_IO_ERROR, &page_private(page));
 278 }
 279 
 280 static void bm_clear_page_io_err(struct page *page)
 281 {
 282         clear_bit(BM_PAGE_IO_ERROR, &page_private(page));
 283 }
 284 
 285 static void bm_set_page_lazy_writeout(struct page *page)
 286 {
 287         set_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));
 288 }
 289 
 290 static int bm_test_page_lazy_writeout(struct page *page)
 291 {
 292         return test_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));
 293 }
 294 
 295 /* on a 32bit box, this would allow for exactly (2<<38) bits. */
 296 static unsigned int bm_word_to_page_idx(struct drbd_bitmap *b, unsigned long long_nr)
 297 {
 298         /* page_nr = (word*sizeof(long)) >> PAGE_SHIFT; */
 299         unsigned int page_nr = long_nr >> (PAGE_SHIFT - LN2_BPL + 3);
 300         BUG_ON(page_nr >= b->bm_number_of_pages);
 301         return page_nr;
 302 }
 303 
 304 static unsigned int bm_bit_to_page_idx(struct drbd_bitmap *b, u64 bitnr)
 305 {
 306         /* page_nr = (bitnr/8) >> PAGE_SHIFT; */
 307         unsigned int page_nr = bitnr >> (PAGE_SHIFT + 3);
 308         BUG_ON(page_nr >= b->bm_number_of_pages);
 309         return page_nr;
 310 }
 311 
 312 static unsigned long *__bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)
 313 {
 314         struct page *page = b->bm_pages[idx];
 315         return (unsigned long *) kmap_atomic(page);
 316 }
 317 
 318 static unsigned long *bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)
 319 {
 320         return __bm_map_pidx(b, idx);
 321 }
 322 
 323 static void __bm_unmap(unsigned long *p_addr)
 324 {
 325         kunmap_atomic(p_addr);
 326 };
 327 
 328 static void bm_unmap(unsigned long *p_addr)
 329 {
 330         return __bm_unmap(p_addr);
 331 }
 332 
 333 /* long word offset of _bitmap_ sector */
 334 #define S2W(s)  ((s)<<(BM_EXT_SHIFT-BM_BLOCK_SHIFT-LN2_BPL))
 335 /* word offset from start of bitmap to word number _in_page_
 336  * modulo longs per page
 337 #define MLPP(X) ((X) % (PAGE_SIZE/sizeof(long))
 338  hm, well, Philipp thinks gcc might not optimize the % into & (... - 1)
 339  so do it explicitly:
 340  */
 341 #define MLPP(X) ((X) & ((PAGE_SIZE/sizeof(long))-1))
 342 
 343 /* Long words per page */
 344 #define LWPP (PAGE_SIZE/sizeof(long))
 345 
 346 /*
 347  * actually most functions herein should take a struct drbd_bitmap*, not a
 348  * struct drbd_device*, but for the debug macros I like to have the device around
 349  * to be able to report device specific.
 350  */
 351 
 352 
 353 static void bm_free_pages(struct page **pages, unsigned long number)
 354 {
 355         unsigned long i;
 356         if (!pages)
 357                 return;
 358 
 359         for (i = 0; i < number; i++) {
 360                 if (!pages[i]) {
 361                         pr_alert("bm_free_pages tried to free a NULL pointer; i=%lu n=%lu\n",
 362                                  i, number);
 363                         continue;
 364                 }
 365                 __free_page(pages[i]);
 366                 pages[i] = NULL;
 367         }
 368 }
 369 
 370 static inline void bm_vk_free(void *ptr)
 371 {
 372         kvfree(ptr);
 373 }
 374 
 375 /*
 376  * "have" and "want" are NUMBER OF PAGES.
 377  */
 378 static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 379 {
 380         struct page **old_pages = b->bm_pages;
 381         struct page **new_pages, *page;
 382         unsigned int i, bytes;
 383         unsigned long have = b->bm_number_of_pages;
 384 
 385         BUG_ON(have == 0 && old_pages != NULL);
 386         BUG_ON(have != 0 && old_pages == NULL);
 387 
 388         if (have == want)
 389                 return old_pages;
 390 
 391         /* Trying kmalloc first, falling back to vmalloc.
 392          * GFP_NOIO, as this is called while drbd IO is "suspended",
 393          * and during resize or attach on diskless Primary,
 394          * we must not block on IO to ourselves.
 395          * Context is receiver thread or dmsetup. */
 396         bytes = sizeof(struct page *)*want;
 397         new_pages = kzalloc(bytes, GFP_NOIO | __GFP_NOWARN);
 398         if (!new_pages) {
 399                 new_pages = __vmalloc(bytes,
 400                                 GFP_NOIO | __GFP_ZERO,
 401                                 PAGE_KERNEL);
 402                 if (!new_pages)
 403                         return NULL;
 404         }
 405 
 406         if (want >= have) {
 407                 for (i = 0; i < have; i++)
 408                         new_pages[i] = old_pages[i];
 409                 for (; i < want; i++) {
 410                         page = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
 411                         if (!page) {
 412                                 bm_free_pages(new_pages + have, i - have);
 413                                 bm_vk_free(new_pages);
 414                                 return NULL;
 415                         }
 416                         /* we want to know which page it is
 417                          * from the endio handlers */
 418                         bm_store_page_idx(page, i);
 419                         new_pages[i] = page;
 420                 }
 421         } else {
 422                 for (i = 0; i < want; i++)
 423                         new_pages[i] = old_pages[i];
 424                 /* NOT HERE, we are outside the spinlock!
 425                 bm_free_pages(old_pages + want, have - want);
 426                 */
 427         }
 428 
 429         return new_pages;
 430 }
 431 
 432 /*
 433  * allocates the drbd_bitmap and stores it in device->bitmap.
 434  */
 435 int drbd_bm_init(struct drbd_device *device)
 436 {
 437         struct drbd_bitmap *b = device->bitmap;
 438         WARN_ON(b != NULL);
 439         b = kzalloc(sizeof(struct drbd_bitmap), GFP_KERNEL);
 440         if (!b)
 441                 return -ENOMEM;
 442         spin_lock_init(&b->bm_lock);
 443         mutex_init(&b->bm_change);
 444         init_waitqueue_head(&b->bm_io_wait);
 445 
 446         device->bitmap = b;
 447 
 448         return 0;
 449 }
 450 
 451 sector_t drbd_bm_capacity(struct drbd_device *device)
 452 {
 453         if (!expect(device->bitmap))
 454                 return 0;
 455         return device->bitmap->bm_dev_capacity;
 456 }
 457 
 458 /* called on driver unload. TODO: call when a device is destroyed.
 459  */
 460 void drbd_bm_cleanup(struct drbd_device *device)
 461 {
 462         if (!expect(device->bitmap))
 463                 return;
 464         bm_free_pages(device->bitmap->bm_pages, device->bitmap->bm_number_of_pages);
 465         bm_vk_free(device->bitmap->bm_pages);
 466         kfree(device->bitmap);
 467         device->bitmap = NULL;
 468 }
 469 
 470 /*
 471  * since (b->bm_bits % BITS_PER_LONG) != 0,
 472  * this masks out the remaining bits.
 473  * Returns the number of bits cleared.
 474  */
 475 #ifndef BITS_PER_PAGE
 476 #define BITS_PER_PAGE           (1UL << (PAGE_SHIFT + 3))
 477 #define BITS_PER_PAGE_MASK      (BITS_PER_PAGE - 1)
 478 #else
 479 # if BITS_PER_PAGE != (1UL << (PAGE_SHIFT + 3))
 480 #  error "ambiguous BITS_PER_PAGE"
 481 # endif
 482 #endif
 483 #define BITS_PER_LONG_MASK      (BITS_PER_LONG - 1)
 484 static int bm_clear_surplus(struct drbd_bitmap *b)
 485 {
 486         unsigned long mask;
 487         unsigned long *p_addr, *bm;
 488         int tmp;
 489         int cleared = 0;
 490 
 491         /* number of bits modulo bits per page */
 492         tmp = (b->bm_bits & BITS_PER_PAGE_MASK);
 493         /* mask the used bits of the word containing the last bit */
 494         mask = (1UL << (tmp & BITS_PER_LONG_MASK)) -1;
 495         /* bitmap is always stored little endian,
 496          * on disk and in core memory alike */
 497         mask = cpu_to_lel(mask);
 498 
 499         p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);
 500         bm = p_addr + (tmp/BITS_PER_LONG);
 501         if (mask) {
 502                 /* If mask != 0, we are not exactly aligned, so bm now points
 503                  * to the long containing the last bit.
 504                  * If mask == 0, bm already points to the word immediately
 505                  * after the last (long word aligned) bit. */
 506                 cleared = hweight_long(*bm & ~mask);
 507                 *bm &= mask;
 508                 bm++;
 509         }
 510 
 511         if (BITS_PER_LONG == 32 && ((bm - p_addr) & 1) == 1) {
 512                 /* on a 32bit arch, we may need to zero out
 513                  * a padding long to align with a 64bit remote */
 514                 cleared += hweight_long(*bm);
 515                 *bm = 0;
 516         }
 517         bm_unmap(p_addr);
 518         return cleared;
 519 }
 520 
 521 static void bm_set_surplus(struct drbd_bitmap *b)
 522 {
 523         unsigned long mask;
 524         unsigned long *p_addr, *bm;
 525         int tmp;
 526 
 527         /* number of bits modulo bits per page */
 528         tmp = (b->bm_bits & BITS_PER_PAGE_MASK);
 529         /* mask the used bits of the word containing the last bit */
 530         mask = (1UL << (tmp & BITS_PER_LONG_MASK)) -1;
 531         /* bitmap is always stored little endian,
 532          * on disk and in core memory alike */
 533         mask = cpu_to_lel(mask);
 534 
 535         p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);
 536         bm = p_addr + (tmp/BITS_PER_LONG);
 537         if (mask) {
 538                 /* If mask != 0, we are not exactly aligned, so bm now points
 539                  * to the long containing the last bit.
 540                  * If mask == 0, bm already points to the word immediately
 541                  * after the last (long word aligned) bit. */
 542                 *bm |= ~mask;
 543                 bm++;
 544         }
 545 
 546         if (BITS_PER_LONG == 32 && ((bm - p_addr) & 1) == 1) {
 547                 /* on a 32bit arch, we may need to zero out
 548                  * a padding long to align with a 64bit remote */
 549                 *bm = ~0UL;
 550         }
 551         bm_unmap(p_addr);
 552 }
 553 
 554 /* you better not modify the bitmap while this is running,
 555  * or its results will be stale */
 556 static unsigned long bm_count_bits(struct drbd_bitmap *b)
 557 {
 558         unsigned long *p_addr;
 559         unsigned long bits = 0;
 560         unsigned long mask = (1UL << (b->bm_bits & BITS_PER_LONG_MASK)) -1;
 561         int idx, last_word;
 562 
 563         /* all but last page */
 564         for (idx = 0; idx < b->bm_number_of_pages - 1; idx++) {
 565                 p_addr = __bm_map_pidx(b, idx);
 566                 bits += bitmap_weight(p_addr, BITS_PER_PAGE);
 567                 __bm_unmap(p_addr);
 568                 cond_resched();
 569         }
 570         /* last (or only) page */
 571         last_word = ((b->bm_bits - 1) & BITS_PER_PAGE_MASK) >> LN2_BPL;
 572         p_addr = __bm_map_pidx(b, idx);
 573         bits += bitmap_weight(p_addr, last_word * BITS_PER_LONG);
 574         p_addr[last_word] &= cpu_to_lel(mask);
 575         bits += hweight_long(p_addr[last_word]);
 576         /* 32bit arch, may have an unused padding long */
 577         if (BITS_PER_LONG == 32 && (last_word & 1) == 0)
 578                 p_addr[last_word+1] = 0;
 579         __bm_unmap(p_addr);
 580         return bits;
 581 }
 582 
 583 /* offset and len in long words.*/
 584 static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len)
 585 {
 586         unsigned long *p_addr, *bm;
 587         unsigned int idx;
 588         size_t do_now, end;
 589 
 590         end = offset + len;
 591 
 592         if (end > b->bm_words) {
 593                 pr_alert("bm_memset end > bm_words\n");
 594                 return;
 595         }
 596 
 597         while (offset < end) {
 598                 do_now = min_t(size_t, ALIGN(offset + 1, LWPP), end) - offset;
 599                 idx = bm_word_to_page_idx(b, offset);
 600                 p_addr = bm_map_pidx(b, idx);
 601                 bm = p_addr + MLPP(offset);
 602                 if (bm+do_now > p_addr + LWPP) {
 603                         pr_alert("BUG BUG BUG! p_addr:%p bm:%p do_now:%d\n",
 604                                p_addr, bm, (int)do_now);
 605                 } else
 606                         memset(bm, c, do_now * sizeof(long));
 607                 bm_unmap(p_addr);
 608                 bm_set_page_need_writeout(b->bm_pages[idx]);
 609                 offset += do_now;
 610         }
 611 }
 612 
 613 /* For the layout, see comment above drbd_md_set_sector_offsets(). */
 614 static u64 drbd_md_on_disk_bits(struct drbd_backing_dev *ldev)
 615 {
 616         u64 bitmap_sectors;
 617         if (ldev->md.al_offset == 8)
 618                 bitmap_sectors = ldev->md.md_size_sect - ldev->md.bm_offset;
 619         else
 620                 bitmap_sectors = ldev->md.al_offset - ldev->md.bm_offset;
 621         return bitmap_sectors << (9 + 3);
 622 }
 623 
 624 /*
 625  * make sure the bitmap has enough room for the attached storage,
 626  * if necessary, resize.
 627  * called whenever we may have changed the device size.
 628  * returns -ENOMEM if we could not allocate enough memory, 0 on success.
 629  * In case this is actually a resize, we copy the old bitmap into the new one.
 630  * Otherwise, the bitmap is initialized to all bits set.
 631  */
 632 int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bits)
 633 {
 634         struct drbd_bitmap *b = device->bitmap;
 635         unsigned long bits, words, owords, obits;
 636         unsigned long want, have, onpages; /* number of pages */
 637         struct page **npages, **opages = NULL;
 638         int err = 0;
 639         bool growing;
 640 
 641         if (!expect(b))
 642                 return -ENOMEM;
 643 
 644         drbd_bm_lock(device, "resize", BM_LOCKED_MASK);
 645 
 646         drbd_info(device, "drbd_bm_resize called with capacity == %llu\n",
 647                         (unsigned long long)capacity);
 648 
 649         if (capacity == b->bm_dev_capacity)
 650                 goto out;
 651 
 652         if (capacity == 0) {
 653                 spin_lock_irq(&b->bm_lock);
 654                 opages = b->bm_pages;
 655                 onpages = b->bm_number_of_pages;
 656                 owords = b->bm_words;
 657                 b->bm_pages = NULL;
 658                 b->bm_number_of_pages =
 659                 b->bm_set   =
 660                 b->bm_bits  =
 661                 b->bm_words =
 662                 b->bm_dev_capacity = 0;
 663                 spin_unlock_irq(&b->bm_lock);
 664                 bm_free_pages(opages, onpages);
 665                 bm_vk_free(opages);
 666                 goto out;
 667         }
 668         bits  = BM_SECT_TO_BIT(ALIGN(capacity, BM_SECT_PER_BIT));
 669 
 670         /* if we would use
 671            words = ALIGN(bits,BITS_PER_LONG) >> LN2_BPL;
 672            a 32bit host could present the wrong number of words
 673            to a 64bit host.
 674         */
 675         words = ALIGN(bits, 64) >> LN2_BPL;
 676 
 677         if (get_ldev(device)) {
 678                 u64 bits_on_disk = drbd_md_on_disk_bits(device->ldev);
 679                 put_ldev(device);
 680                 if (bits > bits_on_disk) {
 681                         drbd_info(device, "bits = %lu\n", bits);
 682                         drbd_info(device, "bits_on_disk = %llu\n", bits_on_disk);
 683                         err = -ENOSPC;
 684                         goto out;
 685                 }
 686         }
 687 
 688         want = ALIGN(words*sizeof(long), PAGE_SIZE) >> PAGE_SHIFT;
 689         have = b->bm_number_of_pages;
 690         if (want == have) {
 691                 D_ASSERT(device, b->bm_pages != NULL);
 692                 npages = b->bm_pages;
 693         } else {
 694                 if (drbd_insert_fault(device, DRBD_FAULT_BM_ALLOC))
 695                         npages = NULL;
 696                 else
 697                         npages = bm_realloc_pages(b, want);
 698         }
 699 
 700         if (!npages) {
 701                 err = -ENOMEM;
 702                 goto out;
 703         }
 704 
 705         spin_lock_irq(&b->bm_lock);
 706         opages = b->bm_pages;
 707         owords = b->bm_words;
 708         obits  = b->bm_bits;
 709 
 710         growing = bits > obits;
 711         if (opages && growing && set_new_bits)
 712                 bm_set_surplus(b);
 713 
 714         b->bm_pages = npages;
 715         b->bm_number_of_pages = want;
 716         b->bm_bits  = bits;
 717         b->bm_words = words;
 718         b->bm_dev_capacity = capacity;
 719 
 720         if (growing) {
 721                 if (set_new_bits) {
 722                         bm_memset(b, owords, 0xff, words-owords);
 723                         b->bm_set += bits - obits;
 724                 } else
 725                         bm_memset(b, owords, 0x00, words-owords);
 726 
 727         }
 728 
 729         if (want < have) {
 730                 /* implicit: (opages != NULL) && (opages != npages) */
 731                 bm_free_pages(opages + want, have - want);
 732         }
 733 
 734         (void)bm_clear_surplus(b);
 735 
 736         spin_unlock_irq(&b->bm_lock);
 737         if (opages != npages)
 738                 bm_vk_free(opages);
 739         if (!growing)
 740                 b->bm_set = bm_count_bits(b);
 741         drbd_info(device, "resync bitmap: bits=%lu words=%lu pages=%lu\n", bits, words, want);
 742 
 743  out:
 744         drbd_bm_unlock(device);
 745         return err;
 746 }
 747 
 748 /* inherently racy:
 749  * if not protected by other means, return value may be out of date when
 750  * leaving this function...
 751  * we still need to lock it, since it is important that this returns
 752  * bm_set == 0 precisely.
 753  *
 754  * maybe bm_set should be atomic_t ?
 755  */
 756 unsigned long _drbd_bm_total_weight(struct drbd_device *device)
 757 {
 758         struct drbd_bitmap *b = device->bitmap;
 759         unsigned long s;
 760         unsigned long flags;
 761 
 762         if (!expect(b))
 763                 return 0;
 764         if (!expect(b->bm_pages))
 765                 return 0;
 766 
 767         spin_lock_irqsave(&b->bm_lock, flags);
 768         s = b->bm_set;
 769         spin_unlock_irqrestore(&b->bm_lock, flags);
 770 
 771         return s;
 772 }
 773 
 774 unsigned long drbd_bm_total_weight(struct drbd_device *device)
 775 {
 776         unsigned long s;
 777         /* if I don't have a disk, I don't know about out-of-sync status */
 778         if (!get_ldev_if_state(device, D_NEGOTIATING))
 779                 return 0;
 780         s = _drbd_bm_total_weight(device);
 781         put_ldev(device);
 782         return s;
 783 }
 784 
 785 size_t drbd_bm_words(struct drbd_device *device)
 786 {
 787         struct drbd_bitmap *b = device->bitmap;
 788         if (!expect(b))
 789                 return 0;
 790         if (!expect(b->bm_pages))
 791                 return 0;
 792 
 793         return b->bm_words;
 794 }
 795 
 796 unsigned long drbd_bm_bits(struct drbd_device *device)
 797 {
 798         struct drbd_bitmap *b = device->bitmap;
 799         if (!expect(b))
 800                 return 0;
 801 
 802         return b->bm_bits;
 803 }
 804 
 805 /* merge number words from buffer into the bitmap starting at offset.
 806  * buffer[i] is expected to be little endian unsigned long.
 807  * bitmap must be locked by drbd_bm_lock.
 808  * currently only used from receive_bitmap.
 809  */
 810 void drbd_bm_merge_lel(struct drbd_device *device, size_t offset, size_t number,
 811                         unsigned long *buffer)
 812 {
 813         struct drbd_bitmap *b = device->bitmap;
 814         unsigned long *p_addr, *bm;
 815         unsigned long word, bits;
 816         unsigned int idx;
 817         size_t end, do_now;
 818 
 819         end = offset + number;
 820 
 821         if (!expect(b))
 822                 return;
 823         if (!expect(b->bm_pages))
 824                 return;
 825         if (number == 0)
 826                 return;
 827         WARN_ON(offset >= b->bm_words);
 828         WARN_ON(end    >  b->bm_words);
 829 
 830         spin_lock_irq(&b->bm_lock);
 831         while (offset < end) {
 832                 do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset;
 833                 idx = bm_word_to_page_idx(b, offset);
 834                 p_addr = bm_map_pidx(b, idx);
 835                 bm = p_addr + MLPP(offset);
 836                 offset += do_now;
 837                 while (do_now--) {
 838                         bits = hweight_long(*bm);
 839                         word = *bm | *buffer++;
 840                         *bm++ = word;
 841                         b->bm_set += hweight_long(word) - bits;
 842                 }
 843                 bm_unmap(p_addr);
 844                 bm_set_page_need_writeout(b->bm_pages[idx]);
 845         }
 846         /* with 32bit <-> 64bit cross-platform connect
 847          * this is only correct for current usage,
 848          * where we _know_ that we are 64 bit aligned,
 849          * and know that this function is used in this way, too...
 850          */
 851         if (end == b->bm_words)
 852                 b->bm_set -= bm_clear_surplus(b);
 853         spin_unlock_irq(&b->bm_lock);
 854 }
 855 
 856 /* copy number words from the bitmap starting at offset into the buffer.
 857  * buffer[i] will be little endian unsigned long.
 858  */
 859 void drbd_bm_get_lel(struct drbd_device *device, size_t offset, size_t number,
 860                      unsigned long *buffer)
 861 {
 862         struct drbd_bitmap *b = device->bitmap;
 863         unsigned long *p_addr, *bm;
 864         size_t end, do_now;
 865 
 866         end = offset + number;
 867 
 868         if (!expect(b))
 869                 return;
 870         if (!expect(b->bm_pages))
 871                 return;
 872 
 873         spin_lock_irq(&b->bm_lock);
 874         if ((offset >= b->bm_words) ||
 875             (end    >  b->bm_words) ||
 876             (number <= 0))
 877                 drbd_err(device, "offset=%lu number=%lu bm_words=%lu\n",
 878                         (unsigned long) offset,
 879                         (unsigned long) number,
 880                         (unsigned long) b->bm_words);
 881         else {
 882                 while (offset < end) {
 883                         do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset;
 884                         p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, offset));
 885                         bm = p_addr + MLPP(offset);
 886                         offset += do_now;
 887                         while (do_now--)
 888                                 *buffer++ = *bm++;
 889                         bm_unmap(p_addr);
 890                 }
 891         }
 892         spin_unlock_irq(&b->bm_lock);
 893 }
 894 
 895 /* set all bits in the bitmap */
 896 void drbd_bm_set_all(struct drbd_device *device)
 897 {
 898         struct drbd_bitmap *b = device->bitmap;
 899         if (!expect(b))
 900                 return;
 901         if (!expect(b->bm_pages))
 902                 return;
 903 
 904         spin_lock_irq(&b->bm_lock);
 905         bm_memset(b, 0, 0xff, b->bm_words);
 906         (void)bm_clear_surplus(b);
 907         b->bm_set = b->bm_bits;
 908         spin_unlock_irq(&b->bm_lock);
 909 }
 910 
 911 /* clear all bits in the bitmap */
 912 void drbd_bm_clear_all(struct drbd_device *device)
 913 {
 914         struct drbd_bitmap *b = device->bitmap;
 915         if (!expect(b))
 916                 return;
 917         if (!expect(b->bm_pages))
 918                 return;
 919 
 920         spin_lock_irq(&b->bm_lock);
 921         bm_memset(b, 0, 0, b->bm_words);
 922         b->bm_set = 0;
 923         spin_unlock_irq(&b->bm_lock);
 924 }
 925 
 926 static void drbd_bm_aio_ctx_destroy(struct kref *kref)
 927 {
 928         struct drbd_bm_aio_ctx *ctx = container_of(kref, struct drbd_bm_aio_ctx, kref);
 929         unsigned long flags;
 930 
 931         spin_lock_irqsave(&ctx->device->resource->req_lock, flags);
 932         list_del(&ctx->list);
 933         spin_unlock_irqrestore(&ctx->device->resource->req_lock, flags);
 934         put_ldev(ctx->device);
 935         kfree(ctx);
 936 }
 937 
 938 /* bv_page may be a copy, or may be the original */
 939 static void drbd_bm_endio(struct bio *bio)
 940 {
 941         struct drbd_bm_aio_ctx *ctx = bio->bi_private;
 942         struct drbd_device *device = ctx->device;
 943         struct drbd_bitmap *b = device->bitmap;
 944         unsigned int idx = bm_page_to_idx(bio_first_page_all(bio));
 945 
 946         if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 &&
 947             !bm_test_page_unchanged(b->bm_pages[idx]))
 948                 drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx);
 949 
 950         if (bio->bi_status) {
 951                 /* ctx error will hold the completed-last non-zero error code,
 952                  * in case error codes differ. */
 953                 ctx->error = blk_status_to_errno(bio->bi_status);
 954                 bm_set_page_io_err(b->bm_pages[idx]);
 955                 /* Not identical to on disk version of it.
 956                  * Is BM_PAGE_IO_ERROR enough? */
 957                 if (__ratelimit(&drbd_ratelimit_state))
 958                         drbd_err(device, "IO ERROR %d on bitmap page idx %u\n",
 959                                         bio->bi_status, idx);
 960         } else {
 961                 bm_clear_page_io_err(b->bm_pages[idx]);
 962                 dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx);
 963         }
 964 
 965         bm_page_unlock_io(device, idx);
 966 
 967         if (ctx->flags & BM_AIO_COPY_PAGES)
 968                 mempool_free(bio->bi_io_vec[0].bv_page, &drbd_md_io_page_pool);
 969 
 970         bio_put(bio);
 971 
 972         if (atomic_dec_and_test(&ctx->in_flight)) {
 973                 ctx->done = 1;
 974                 wake_up(&device->misc_wait);
 975                 kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy);
 976         }
 977 }
 978 
 979 static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_hold(local)
 980 {
 981         struct bio *bio = bio_alloc_drbd(GFP_NOIO);
 982         struct drbd_device *device = ctx->device;
 983         struct drbd_bitmap *b = device->bitmap;
 984         struct page *page;
 985         unsigned int len;
 986         unsigned int op = (ctx->flags & BM_AIO_READ) ? REQ_OP_READ : REQ_OP_WRITE;
 987 
 988         sector_t on_disk_sector =
 989                 device->ldev->md.md_offset + device->ldev->md.bm_offset;
 990         on_disk_sector += ((sector_t)page_nr) << (PAGE_SHIFT-9);
 991 
 992         /* this might happen with very small
 993          * flexible external meta data device,
 994          * or with PAGE_SIZE > 4k */
 995         len = min_t(unsigned int, PAGE_SIZE,
 996                 (drbd_md_last_sector(device->ldev) - on_disk_sector + 1)<<9);
 997 
 998         /* serialize IO on this page */
 999         bm_page_lock_io(device, page_nr);
1000         /* before memcpy and submit,
1001          * so it can be redirtied any time */
1002         bm_set_page_unchanged(b->bm_pages[page_nr]);
1003 
1004         if (ctx->flags & BM_AIO_COPY_PAGES) {
1005                 page = mempool_alloc(&drbd_md_io_page_pool,
1006                                 GFP_NOIO | __GFP_HIGHMEM);
1007                 copy_highpage(page, b->bm_pages[page_nr]);
1008                 bm_store_page_idx(page, page_nr);
1009         } else
1010                 page = b->bm_pages[page_nr];
1011         bio_set_dev(bio, device->ldev->md_bdev);
1012         bio->bi_iter.bi_sector = on_disk_sector;
1013         /* bio_add_page of a single page to an empty bio will always succeed,
1014          * according to api.  Do we want to assert that? */
1015         bio_add_page(bio, page, len, 0);
1016         bio->bi_private = ctx;
1017         bio->bi_end_io = drbd_bm_endio;
1018         bio_set_op_attrs(bio, op, 0);
1019 
1020         if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) {
1021                 bio_io_error(bio);
1022         } else {
1023                 submit_bio(bio);
1024                 /* this should not count as user activity and cause the
1025                  * resync to throttle -- see drbd_rs_should_slow_down(). */
1026                 atomic_add(len >> 9, &device->rs_sect_ev);
1027         }
1028 }
1029 
1030 /*
1031  * bm_rw: read/write the whole bitmap from/to its on disk location.
1032  */
1033 static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned lazy_writeout_upper_idx) __must_hold(local)
1034 {
1035         struct drbd_bm_aio_ctx *ctx;
1036         struct drbd_bitmap *b = device->bitmap;
1037         unsigned int num_pages, i, count = 0;
1038         unsigned long now;
1039         char ppb[10];
1040         int err = 0;
1041 
1042         /*
1043          * We are protected against bitmap disappearing/resizing by holding an
1044          * ldev reference (caller must have called get_ldev()).
1045          * For read/write, we are protected against changes to the bitmap by
1046          * the bitmap lock (see drbd_bitmap_io).
1047          * For lazy writeout, we don't care for ongoing changes to the bitmap,
1048          * as we submit copies of pages anyways.
1049          */
1050 
1051         ctx = kmalloc(sizeof(struct drbd_bm_aio_ctx), GFP_NOIO);
1052         if (!ctx)
1053                 return -ENOMEM;
1054 
1055         *ctx = (struct drbd_bm_aio_ctx) {
1056                 .device = device,
1057                 .start_jif = jiffies,
1058                 .in_flight = ATOMIC_INIT(1),
1059                 .done = 0,
1060                 .flags = flags,
1061                 .error = 0,
1062                 .kref = KREF_INIT(2),
1063         };
1064 
1065         if (!get_ldev_if_state(device, D_ATTACHING)) {  /* put is in drbd_bm_aio_ctx_destroy() */
1066                 drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in bm_rw()\n");
1067                 kfree(ctx);
1068                 return -ENODEV;
1069         }
1070         /* Here D_ATTACHING is sufficient since drbd_bm_read() is called only from
1071            drbd_adm_attach(), after device->ldev was assigned. */
1072 
1073         if (0 == (ctx->flags & ~BM_AIO_READ))
1074                 WARN_ON(!(BM_LOCKED_MASK & b->bm_flags));
1075 
1076         spin_lock_irq(&device->resource->req_lock);
1077         list_add_tail(&ctx->list, &device->pending_bitmap_io);
1078         spin_unlock_irq(&device->resource->req_lock);
1079 
1080         num_pages = b->bm_number_of_pages;
1081 
1082         now = jiffies;
1083 
1084         /* let the layers below us try to merge these bios... */
1085 
1086         if (flags & BM_AIO_READ) {
1087                 for (i = 0; i < num_pages; i++) {
1088                         atomic_inc(&ctx->in_flight);
1089                         bm_page_io_async(ctx, i);
1090                         ++count;
1091                         cond_resched();
1092                 }
1093         } else if (flags & BM_AIO_WRITE_HINTED) {
1094                 /* ASSERT: BM_AIO_WRITE_ALL_PAGES is not set. */
1095                 unsigned int hint;
1096                 for (hint = 0; hint < b->n_bitmap_hints; hint++) {
1097                         i = b->al_bitmap_hints[hint];
1098                         if (i >= num_pages) /* == -1U: no hint here. */
1099                                 continue;
1100                         /* Several AL-extents may point to the same page. */
1101                         if (!test_and_clear_bit(BM_PAGE_HINT_WRITEOUT,
1102                             &page_private(b->bm_pages[i])))
1103                                 continue;
1104                         /* Has it even changed? */
1105                         if (bm_test_page_unchanged(b->bm_pages[i]))
1106                                 continue;
1107                         atomic_inc(&ctx->in_flight);
1108                         bm_page_io_async(ctx, i);
1109                         ++count;
1110                 }
1111         } else {
1112                 for (i = 0; i < num_pages; i++) {
1113                         /* ignore completely unchanged pages */
1114                         if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx)
1115                                 break;
1116                         if (!(flags & BM_AIO_WRITE_ALL_PAGES) &&
1117                             bm_test_page_unchanged(b->bm_pages[i])) {
1118                                 dynamic_drbd_dbg(device, "skipped bm write for idx %u\n", i);
1119                                 continue;
1120                         }
1121                         /* during lazy writeout,
1122                          * ignore those pages not marked for lazy writeout. */
1123                         if (lazy_writeout_upper_idx &&
1124                             !bm_test_page_lazy_writeout(b->bm_pages[i])) {
1125                                 dynamic_drbd_dbg(device, "skipped bm lazy write for idx %u\n", i);
1126                                 continue;
1127                         }
1128                         atomic_inc(&ctx->in_flight);
1129                         bm_page_io_async(ctx, i);
1130                         ++count;
1131                         cond_resched();
1132                 }
1133         }
1134 
1135         /*
1136          * We initialize ctx->in_flight to one to make sure drbd_bm_endio
1137          * will not set ctx->done early, and decrement / test it here.  If there
1138          * are still some bios in flight, we need to wait for them here.
1139          * If all IO is done already (or nothing had been submitted), there is
1140          * no need to wait.  Still, we need to put the kref associated with the
1141          * "in_flight reached zero, all done" event.
1142          */
1143         if (!atomic_dec_and_test(&ctx->in_flight))
1144                 wait_until_done_or_force_detached(device, device->ldev, &ctx->done);
1145         else
1146                 kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy);
1147 
1148         /* summary for global bitmap IO */
1149         if (flags == 0) {
1150                 unsigned int ms = jiffies_to_msecs(jiffies - now);
1151                 if (ms > 5) {
1152                         drbd_info(device, "bitmap %s of %u pages took %u ms\n",
1153                                  (flags & BM_AIO_READ) ? "READ" : "WRITE",
1154                                  count, ms);
1155                 }
1156         }
1157 
1158         if (ctx->error) {
1159                 drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n");
1160                 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
1161                 err = -EIO; /* ctx->error ? */
1162         }
1163 
1164         if (atomic_read(&ctx->in_flight))
1165                 err = -EIO; /* Disk timeout/force-detach during IO... */
1166 
1167         now = jiffies;
1168         if (flags & BM_AIO_READ) {
1169                 b->bm_set = bm_count_bits(b);
1170                 drbd_info(device, "recounting of set bits took additional %lu jiffies\n",
1171                      jiffies - now);
1172         }
1173         now = b->bm_set;
1174 
1175         if ((flags & ~BM_AIO_READ) == 0)
1176                 drbd_info(device, "%s (%lu bits) marked out-of-sync by on disk bit-map.\n",
1177                      ppsize(ppb, now << (BM_BLOCK_SHIFT-10)), now);
1178 
1179         kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy);
1180         return err;
1181 }
1182 
1183 /**
1184  * drbd_bm_read() - Read the whole bitmap from its on disk location.
1185  * @device:     DRBD device.
1186  */
1187 int drbd_bm_read(struct drbd_device *device) __must_hold(local)
1188 {
1189         return bm_rw(device, BM_AIO_READ, 0);
1190 }
1191 
1192 /**
1193  * drbd_bm_write() - Write the whole bitmap to its on disk location.
1194  * @device:     DRBD device.
1195  *
1196  * Will only write pages that have changed since last IO.
1197  */
1198 int drbd_bm_write(struct drbd_device *device) __must_hold(local)
1199 {
1200         return bm_rw(device, 0, 0);
1201 }
1202 
1203 /**
1204  * drbd_bm_write_all() - Write the whole bitmap to its on disk location.
1205  * @device:     DRBD device.
1206  *
1207  * Will write all pages.
1208  */
1209 int drbd_bm_write_all(struct drbd_device *device) __must_hold(local)
1210 {
1211         return bm_rw(device, BM_AIO_WRITE_ALL_PAGES, 0);
1212 }
1213 
1214 /**
1215  * drbd_bm_write_lazy() - Write bitmap pages 0 to @upper_idx-1, if they have changed.
1216  * @device:     DRBD device.
1217  * @upper_idx:  0: write all changed pages; +ve: page index to stop scanning for changed pages
1218  */
1219 int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local)
1220 {
1221         return bm_rw(device, BM_AIO_COPY_PAGES, upper_idx);
1222 }
1223 
1224 /**
1225  * drbd_bm_write_copy_pages() - Write the whole bitmap to its on disk location.
1226  * @device:     DRBD device.
1227  *
1228  * Will only write pages that have changed since last IO.
1229  * In contrast to drbd_bm_write(), this will copy the bitmap pages
1230  * to temporary writeout pages. It is intended to trigger a full write-out
1231  * while still allowing the bitmap to change, for example if a resync or online
1232  * verify is aborted due to a failed peer disk, while local IO continues, or
1233  * pending resync acks are still being processed.
1234  */
1235 int drbd_bm_write_copy_pages(struct drbd_device *device) __must_hold(local)
1236 {
1237         return bm_rw(device, BM_AIO_COPY_PAGES, 0);
1238 }
1239 
1240 /**
1241  * drbd_bm_write_hinted() - Write bitmap pages with "hint" marks, if they have changed.
1242  * @device:     DRBD device.
1243  */
1244 int drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local)
1245 {
1246         return bm_rw(device, BM_AIO_WRITE_HINTED | BM_AIO_COPY_PAGES, 0);
1247 }
1248 
1249 /* NOTE
1250  * find_first_bit returns int, we return unsigned long.
1251  * For this to work on 32bit arch with bitnumbers > (1<<32),
1252  * we'd need to return u64, and get a whole lot of other places
1253  * fixed where we still use unsigned long.
1254  *
1255  * this returns a bit number, NOT a sector!
1256  */
1257 static unsigned long __bm_find_next(struct drbd_device *device, unsigned long bm_fo,
1258         const int find_zero_bit)
1259 {
1260         struct drbd_bitmap *b = device->bitmap;
1261         unsigned long *p_addr;
1262         unsigned long bit_offset;
1263         unsigned i;
1264 
1265 
1266         if (bm_fo > b->bm_bits) {
1267                 drbd_err(device, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits);
1268                 bm_fo = DRBD_END_OF_BITMAP;
1269         } else {
1270                 while (bm_fo < b->bm_bits) {
1271                         /* bit offset of the first bit in the page */
1272                         bit_offset = bm_fo & ~BITS_PER_PAGE_MASK;
1273                         p_addr = __bm_map_pidx(b, bm_bit_to_page_idx(b, bm_fo));
1274 
1275                         if (find_zero_bit)
1276                                 i = find_next_zero_bit_le(p_addr,
1277                                                 PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK);
1278                         else
1279                                 i = find_next_bit_le(p_addr,
1280                                                 PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK);
1281 
1282                         __bm_unmap(p_addr);
1283                         if (i < PAGE_SIZE*8) {
1284                                 bm_fo = bit_offset + i;
1285                                 if (bm_fo >= b->bm_bits)
1286                                         break;
1287                                 goto found;
1288                         }
1289                         bm_fo = bit_offset + PAGE_SIZE*8;
1290                 }
1291                 bm_fo = DRBD_END_OF_BITMAP;
1292         }
1293  found:
1294         return bm_fo;
1295 }
1296 
1297 static unsigned long bm_find_next(struct drbd_device *device,
1298         unsigned long bm_fo, const int find_zero_bit)
1299 {
1300         struct drbd_bitmap *b = device->bitmap;
1301         unsigned long i = DRBD_END_OF_BITMAP;
1302 
1303         if (!expect(b))
1304                 return i;
1305         if (!expect(b->bm_pages))
1306                 return i;
1307 
1308         spin_lock_irq(&b->bm_lock);
1309         if (BM_DONT_TEST & b->bm_flags)
1310                 bm_print_lock_info(device);
1311 
1312         i = __bm_find_next(device, bm_fo, find_zero_bit);
1313 
1314         spin_unlock_irq(&b->bm_lock);
1315         return i;
1316 }
1317 
1318 unsigned long drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo)
1319 {
1320         return bm_find_next(device, bm_fo, 0);
1321 }
1322 
1323 #if 0
1324 /* not yet needed for anything. */
1325 unsigned long drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo)
1326 {
1327         return bm_find_next(device, bm_fo, 1);
1328 }
1329 #endif
1330 
1331 /* does not spin_lock_irqsave.
1332  * you must take drbd_bm_lock() first */
1333 unsigned long _drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo)
1334 {
1335         /* WARN_ON(!(BM_DONT_SET & device->b->bm_flags)); */
1336         return __bm_find_next(device, bm_fo, 0);
1337 }
1338 
1339 unsigned long _drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo)
1340 {
1341         /* WARN_ON(!(BM_DONT_SET & device->b->bm_flags)); */
1342         return __bm_find_next(device, bm_fo, 1);
1343 }
1344 
1345 /* returns number of bits actually changed.
1346  * for val != 0, we change 0 -> 1, return code positive
1347  * for val == 0, we change 1 -> 0, return code negative
1348  * wants bitnr, not sector.
1349  * expected to be called for only a few bits (e - s about BITS_PER_LONG).
1350  * Must hold bitmap lock already. */
1351 static int __bm_change_bits_to(struct drbd_device *device, const unsigned long s,
1352         unsigned long e, int val)
1353 {
1354         struct drbd_bitmap *b = device->bitmap;
1355         unsigned long *p_addr = NULL;
1356         unsigned long bitnr;
1357         unsigned int last_page_nr = -1U;
1358         int c = 0;
1359         int changed_total = 0;
1360 
1361         if (e >= b->bm_bits) {
1362                 drbd_err(device, "ASSERT FAILED: bit_s=%lu bit_e=%lu bm_bits=%lu\n",
1363                                 s, e, b->bm_bits);
1364                 e = b->bm_bits ? b->bm_bits -1 : 0;
1365         }
1366         for (bitnr = s; bitnr <= e; bitnr++) {
1367                 unsigned int page_nr = bm_bit_to_page_idx(b, bitnr);
1368                 if (page_nr != last_page_nr) {
1369                         if (p_addr)
1370                                 __bm_unmap(p_addr);
1371                         if (c < 0)
1372                                 bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);
1373                         else if (c > 0)
1374                                 bm_set_page_need_writeout(b->bm_pages[last_page_nr]);
1375                         changed_total += c;
1376                         c = 0;
1377                         p_addr = __bm_map_pidx(b, page_nr);
1378                         last_page_nr = page_nr;
1379                 }
1380                 if (val)
1381                         c += (0 == __test_and_set_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr));
1382                 else
1383                         c -= (0 != __test_and_clear_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr));
1384         }
1385         if (p_addr)
1386                 __bm_unmap(p_addr);
1387         if (c < 0)
1388                 bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);
1389         else if (c > 0)
1390                 bm_set_page_need_writeout(b->bm_pages[last_page_nr]);
1391         changed_total += c;
1392         b->bm_set += changed_total;
1393         return changed_total;
1394 }
1395 
1396 /* returns number of bits actually changed.
1397  * for val != 0, we change 0 -> 1, return code positive
1398  * for val == 0, we change 1 -> 0, return code negative
1399  * wants bitnr, not sector */
1400 static int bm_change_bits_to(struct drbd_device *device, const unsigned long s,
1401         const unsigned long e, int val)
1402 {
1403         unsigned long flags;
1404         struct drbd_bitmap *b = device->bitmap;
1405         int c = 0;
1406 
1407         if (!expect(b))
1408                 return 1;
1409         if (!expect(b->bm_pages))
1410                 return 0;
1411 
1412         spin_lock_irqsave(&b->bm_lock, flags);
1413         if ((val ? BM_DONT_SET : BM_DONT_CLEAR) & b->bm_flags)
1414                 bm_print_lock_info(device);
1415 
1416         c = __bm_change_bits_to(device, s, e, val);
1417 
1418         spin_unlock_irqrestore(&b->bm_lock, flags);
1419         return c;
1420 }
1421 
1422 /* returns number of bits changed 0 -> 1 */
1423 int drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
1424 {
1425         return bm_change_bits_to(device, s, e, 1);
1426 }
1427 
1428 /* returns number of bits changed 1 -> 0 */
1429 int drbd_bm_clear_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
1430 {
1431         return -bm_change_bits_to(device, s, e, 0);
1432 }
1433 
1434 /* sets all bits in full words,
1435  * from first_word up to, but not including, last_word */
1436 static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b,
1437                 int page_nr, int first_word, int last_word)
1438 {
1439         int i;
1440         int bits;
1441         int changed = 0;
1442         unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr]);
1443 
1444         /* I think it is more cache line friendly to hweight_long then set to ~0UL,
1445          * than to first bitmap_weight() all words, then bitmap_fill() all words */
1446         for (i = first_word; i < last_word; i++) {
1447                 bits = hweight_long(paddr[i]);
1448                 paddr[i] = ~0UL;
1449                 changed += BITS_PER_LONG - bits;
1450         }
1451         kunmap_atomic(paddr);
1452         if (changed) {
1453                 /* We only need lazy writeout, the information is still in the
1454                  * remote bitmap as well, and is reconstructed during the next
1455                  * bitmap exchange, if lost locally due to a crash. */
1456                 bm_set_page_lazy_writeout(b->bm_pages[page_nr]);
1457                 b->bm_set += changed;
1458         }
1459 }
1460 
1461 /* Same thing as drbd_bm_set_bits,
1462  * but more efficient for a large bit range.
1463  * You must first drbd_bm_lock().
1464  * Can be called to set the whole bitmap in one go.
1465  * Sets bits from s to e _inclusive_. */
1466 void _drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
1467 {
1468         /* First set_bit from the first bit (s)
1469          * up to the next long boundary (sl),
1470          * then assign full words up to the last long boundary (el),
1471          * then set_bit up to and including the last bit (e).
1472          *
1473          * Do not use memset, because we must account for changes,
1474          * so we need to loop over the words with hweight() anyways.
1475          */
1476         struct drbd_bitmap *b = device->bitmap;
1477         unsigned long sl = ALIGN(s,BITS_PER_LONG);
1478         unsigned long el = (e+1) & ~((unsigned long)BITS_PER_LONG-1);
1479         int first_page;
1480         int last_page;
1481         int page_nr;
1482         int first_word;
1483         int last_word;
1484 
1485         if (e - s <= 3*BITS_PER_LONG) {
1486                 /* don't bother; el and sl may even be wrong. */
1487                 spin_lock_irq(&b->bm_lock);
1488                 __bm_change_bits_to(device, s, e, 1);
1489                 spin_unlock_irq(&b->bm_lock);
1490                 return;
1491         }
1492 
1493         /* difference is large enough that we can trust sl and el */
1494 
1495         spin_lock_irq(&b->bm_lock);
1496 
1497         /* bits filling the current long */
1498         if (sl)
1499                 __bm_change_bits_to(device, s, sl-1, 1);
1500 
1501         first_page = sl >> (3 + PAGE_SHIFT);
1502         last_page = el >> (3 + PAGE_SHIFT);
1503 
1504         /* MLPP: modulo longs per page */
1505         /* LWPP: long words per page */
1506         first_word = MLPP(sl >> LN2_BPL);
1507         last_word = LWPP;
1508 
1509         /* first and full pages, unless first page == last page */
1510         for (page_nr = first_page; page_nr < last_page; page_nr++) {
1511                 bm_set_full_words_within_one_page(device->bitmap, page_nr, first_word, last_word);
1512                 spin_unlock_irq(&b->bm_lock);
1513                 cond_resched();
1514                 first_word = 0;
1515                 spin_lock_irq(&b->bm_lock);
1516         }
1517         /* last page (respectively only page, for first page == last page) */
1518         last_word = MLPP(el >> LN2_BPL);
1519 
1520         /* consider bitmap->bm_bits = 32768, bitmap->bm_number_of_pages = 1. (or multiples).
1521          * ==> e = 32767, el = 32768, last_page = 2,
1522          * and now last_word = 0.
1523          * We do not want to touch last_page in this case,
1524          * as we did not allocate it, it is not present in bitmap->bm_pages.
1525          */
1526         if (last_word)
1527                 bm_set_full_words_within_one_page(device->bitmap, last_page, first_word, last_word);
1528 
1529         /* possibly trailing bits.
1530          * example: (e & 63) == 63, el will be e+1.
1531          * if that even was the very last bit,
1532          * it would trigger an assert in __bm_change_bits_to()
1533          */
1534         if (el <= e)
1535                 __bm_change_bits_to(device, el, e, 1);
1536         spin_unlock_irq(&b->bm_lock);
1537 }
1538 
1539 /* returns bit state
1540  * wants bitnr, NOT sector.
1541  * inherently racy... area needs to be locked by means of {al,rs}_lru
1542  *  1 ... bit set
1543  *  0 ... bit not set
1544  * -1 ... first out of bounds access, stop testing for bits!
1545  */
1546 int drbd_bm_test_bit(struct drbd_device *device, const unsigned long bitnr)
1547 {
1548         unsigned long flags;
1549         struct drbd_bitmap *b = device->bitmap;
1550         unsigned long *p_addr;
1551         int i;
1552 
1553         if (!expect(b))
1554                 return 0;
1555         if (!expect(b->bm_pages))
1556                 return 0;
1557 
1558         spin_lock_irqsave(&b->bm_lock, flags);
1559         if (BM_DONT_TEST & b->bm_flags)
1560                 bm_print_lock_info(device);
1561         if (bitnr < b->bm_bits) {
1562                 p_addr = bm_map_pidx(b, bm_bit_to_page_idx(b, bitnr));
1563                 i = test_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr) ? 1 : 0;
1564                 bm_unmap(p_addr);
1565         } else if (bitnr == b->bm_bits) {
1566                 i = -1;
1567         } else { /* (bitnr > b->bm_bits) */
1568                 drbd_err(device, "bitnr=%lu > bm_bits=%lu\n", bitnr, b->bm_bits);
1569                 i = 0;
1570         }
1571 
1572         spin_unlock_irqrestore(&b->bm_lock, flags);
1573         return i;
1574 }
1575 
1576 /* returns number of bits set in the range [s, e] */
1577 int drbd_bm_count_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
1578 {
1579         unsigned long flags;
1580         struct drbd_bitmap *b = device->bitmap;
1581         unsigned long *p_addr = NULL;
1582         unsigned long bitnr;
1583         unsigned int page_nr = -1U;
1584         int c = 0;
1585 
1586         /* If this is called without a bitmap, that is a bug.  But just to be
1587          * robust in case we screwed up elsewhere, in that case pretend there
1588          * was one dirty bit in the requested area, so we won't try to do a
1589          * local read there (no bitmap probably implies no disk) */
1590         if (!expect(b))
1591                 return 1;
1592         if (!expect(b->bm_pages))
1593                 return 1;
1594 
1595         spin_lock_irqsave(&b->bm_lock, flags);
1596         if (BM_DONT_TEST & b->bm_flags)
1597                 bm_print_lock_info(device);
1598         for (bitnr = s; bitnr <= e; bitnr++) {
1599                 unsigned int idx = bm_bit_to_page_idx(b, bitnr);
1600                 if (page_nr != idx) {
1601                         page_nr = idx;
1602                         if (p_addr)
1603                                 bm_unmap(p_addr);
1604                         p_addr = bm_map_pidx(b, idx);
1605                 }
1606                 if (expect(bitnr < b->bm_bits))
1607                         c += (0 != test_bit_le(bitnr - (page_nr << (PAGE_SHIFT+3)), p_addr));
1608                 else
1609                         drbd_err(device, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits);
1610         }
1611         if (p_addr)
1612                 bm_unmap(p_addr);
1613         spin_unlock_irqrestore(&b->bm_lock, flags);
1614         return c;
1615 }
1616 
1617 
1618 /* inherently racy...
1619  * return value may be already out-of-date when this function returns.
1620  * but the general usage is that this is only use during a cstate when bits are
1621  * only cleared, not set, and typically only care for the case when the return
1622  * value is zero, or we already "locked" this "bitmap extent" by other means.
1623  *
1624  * enr is bm-extent number, since we chose to name one sector (512 bytes)
1625  * worth of the bitmap a "bitmap extent".
1626  *
1627  * TODO
1628  * I think since we use it like a reference count, we should use the real
1629  * reference count of some bitmap extent element from some lru instead...
1630  *
1631  */
1632 int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr)
1633 {
1634         struct drbd_bitmap *b = device->bitmap;
1635         int count, s, e;
1636         unsigned long flags;
1637         unsigned long *p_addr, *bm;
1638 
1639         if (!expect(b))
1640                 return 0;
1641         if (!expect(b->bm_pages))
1642                 return 0;
1643 
1644         spin_lock_irqsave(&b->bm_lock, flags);
1645         if (BM_DONT_TEST & b->bm_flags)
1646                 bm_print_lock_info(device);
1647 
1648         s = S2W(enr);
1649         e = min((size_t)S2W(enr+1), b->bm_words);
1650         count = 0;
1651         if (s < b->bm_words) {
1652                 int n = e-s;
1653                 p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, s));
1654                 bm = p_addr + MLPP(s);
1655                 count += bitmap_weight(bm, n * BITS_PER_LONG);
1656                 bm_unmap(p_addr);
1657         } else {
1658                 drbd_err(device, "start offset (%d) too large in drbd_bm_e_weight\n", s);
1659         }
1660         spin_unlock_irqrestore(&b->bm_lock, flags);
1661         return count;
1662 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/block/drbd/drbd_bitmap.c

DEFINITIONS
