118 static void __bm_print_lock_info(struct drbd_device *device, const char *func)  in __bm_print_lock_info()  argument
120 	struct drbd_bitmap *b = device->bitmap;  in __bm_print_lock_info()
123 	drbd_err(device, "FIXME %s[%d] in %s, bitmap locked for '%s' by %s[%d]\n",  in __bm_print_lock_info()
129 void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags)  in drbd_bm_lock()  argument
131 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_lock()
135 		drbd_err(device, "FIXME no bitmap in drbd_bm_lock!?\n");  in drbd_bm_lock()
142 		drbd_warn(device, "%s[%d] going to '%s' but bitmap already locked for '%s' by %s[%d]\n",  in drbd_bm_lock()
149 		drbd_err(device, "FIXME bitmap already locked in bm_lock\n");  in drbd_bm_lock()
156 void drbd_bm_unlock(struct drbd_device *device)  in drbd_bm_unlock()  argument
158 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_unlock()
160 		drbd_err(device, "FIXME no bitmap in drbd_bm_unlock!?\n");  in drbd_bm_unlock()
164 	if (!(BM_LOCKED_MASK & device->bitmap->bm_flags))  in drbd_bm_unlock()
165 		drbd_err(device, "FIXME bitmap not locked in bm_unlock\n");  in drbd_bm_unlock()
216 static void bm_page_lock_io(struct drbd_device *device, int page_nr)  in bm_page_lock_io()  argument
218 	struct drbd_bitmap *b = device->bitmap;  in bm_page_lock_io()
223 static void bm_page_unlock_io(struct drbd_device *device, int page_nr)  in bm_page_unlock_io()  argument
225 	struct drbd_bitmap *b = device->bitmap;  in bm_page_unlock_io()
228 	wake_up(&device->bitmap->bm_io_wait);  in bm_page_unlock_io()
254 void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr)  in drbd_bm_mark_for_writeout()  argument
257 	if (page_nr >= device->bitmap->bm_number_of_pages) {  in drbd_bm_mark_for_writeout()
258 		drbd_warn(device, "BAD: page_nr: %u, number_of_pages: %u\n",  in drbd_bm_mark_for_writeout()
259 			 page_nr, (int)device->bitmap->bm_number_of_pages);  in drbd_bm_mark_for_writeout()
262 	page = device->bitmap->bm_pages[page_nr];  in drbd_bm_mark_for_writeout()
442 int drbd_bm_init(struct drbd_device *device)  in drbd_bm_init()  argument
444 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_init()
453 	device->bitmap = b;  in drbd_bm_init()
458 sector_t drbd_bm_capacity(struct drbd_device *device)  in drbd_bm_capacity()  argument
460 	if (!expect(device->bitmap))  in drbd_bm_capacity()
462 	return device->bitmap->bm_dev_capacity;  in drbd_bm_capacity()
467 void drbd_bm_cleanup(struct drbd_device *device)  in drbd_bm_cleanup()  argument
469 	if (!expect(device->bitmap))  in drbd_bm_cleanup()
471 	bm_free_pages(device->bitmap->bm_pages, device->bitmap->bm_number_of_pages);  in drbd_bm_cleanup()
472 	bm_vk_free(device->bitmap->bm_pages, (BM_P_VMALLOCED & device->bitmap->bm_flags));  in drbd_bm_cleanup()
473 	kfree(device->bitmap);  in drbd_bm_cleanup()
474 	device->bitmap = NULL;  in drbd_bm_cleanup()
635 int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bits)  in drbd_bm_resize()  argument
637 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_resize()
647 	drbd_bm_lock(device, "resize", BM_LOCKED_MASK);  in drbd_bm_resize()
649 	drbd_info(device, "drbd_bm_resize called with capacity == %llu\n",  in drbd_bm_resize()
682 	if (get_ldev(device)) {  in drbd_bm_resize()
683 		u64 bits_on_disk = drbd_md_on_disk_bits(device->ldev);  in drbd_bm_resize()
684 		put_ldev(device);  in drbd_bm_resize()
686 			drbd_info(device, "bits = %lu\n", bits);  in drbd_bm_resize()
687 			drbd_info(device, "bits_on_disk = %llu\n", bits_on_disk);  in drbd_bm_resize()
696 		D_ASSERT(device, b->bm_pages != NULL);  in drbd_bm_resize()
699 		if (drbd_insert_fault(device, DRBD_FAULT_BM_ALLOC))  in drbd_bm_resize()
746 	drbd_info(device, "resync bitmap: bits=%lu words=%lu pages=%lu\n", bits, words, want);  in drbd_bm_resize()
749 	drbd_bm_unlock(device);  in drbd_bm_resize()
761 unsigned long _drbd_bm_total_weight(struct drbd_device *device)  in _drbd_bm_total_weight()  argument
763 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_total_weight()
779 unsigned long drbd_bm_total_weight(struct drbd_device *device)  in drbd_bm_total_weight()  argument
783 	if (!get_ldev_if_state(device, D_NEGOTIATING))  in drbd_bm_total_weight()
785 	s = _drbd_bm_total_weight(device);  in drbd_bm_total_weight()
786 	put_ldev(device);  in drbd_bm_total_weight()
790 size_t drbd_bm_words(struct drbd_device *device)  in drbd_bm_words()  argument
792 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_words()
801 unsigned long drbd_bm_bits(struct drbd_device *device)  in drbd_bm_bits()  argument
803 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_bits()
815 void drbd_bm_merge_lel(struct drbd_device *device, size_t offset, size_t number,  in drbd_bm_merge_lel()  argument
818 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_merge_lel()
864 void drbd_bm_get_lel(struct drbd_device *device, size_t offset, size_t number,  in drbd_bm_get_lel()  argument
867 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_get_lel()
882 		drbd_err(device, "offset=%lu number=%lu bm_words=%lu\n",  in drbd_bm_get_lel()
901 void drbd_bm_set_all(struct drbd_device *device)  in drbd_bm_set_all()  argument
903 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_set_all()
917 void drbd_bm_clear_all(struct drbd_device *device)  in drbd_bm_clear_all()  argument
919 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_clear_all()
936 	spin_lock_irqsave(&ctx->device->resource->req_lock, flags);  in drbd_bm_aio_ctx_destroy()
938 	spin_unlock_irqrestore(&ctx->device->resource->req_lock, flags);  in drbd_bm_aio_ctx_destroy()
939 	put_ldev(ctx->device);  in drbd_bm_aio_ctx_destroy()
947 	struct drbd_device *device = ctx->device;  in drbd_bm_endio()  local
948 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_endio()
962 		drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx);  in drbd_bm_endio()
972 			drbd_err(device, "IO ERROR %d on bitmap page idx %u\n",  in drbd_bm_endio()
976 		dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx);  in drbd_bm_endio()
979 	bm_page_unlock_io(device, idx);  in drbd_bm_endio()
988 		wake_up(&device->misc_wait);  in drbd_bm_endio()
996 	struct drbd_device *device = ctx->device;  in bm_page_io_async()  local
997 	struct drbd_bitmap *b = device->bitmap;  in bm_page_io_async()
1003 		device->ldev->md.md_offset + device->ldev->md.bm_offset;  in bm_page_io_async()
1010 		(drbd_md_last_sector(device->ldev) - on_disk_sector + 1)<<9);  in bm_page_io_async()
1013 	bm_page_lock_io(device, page_nr);  in bm_page_io_async()
1024 	bio->bi_bdev = device->ldev->md_bdev;  in bm_page_io_async()
1032 	if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) {  in bm_page_io_async()
1039 		atomic_add(len >> 9, &device->rs_sect_ev);  in bm_page_io_async()
1046 static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned lazy_writeout_upper…  in bm_rw()  argument
1049 	struct drbd_bitmap *b = device->bitmap;  in bm_rw()
1069 		.device = device,  in bm_rw()
1078 	if (!get_ldev_if_state(device, D_ATTACHING)) {  /* put is in drbd_bm_aio_ctx_destroy() */  in bm_rw()
1079 		drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in bm_rw()\n");  in bm_rw()
1089 	spin_lock_irq(&device->resource->req_lock);  in bm_rw()
1090 	list_add_tail(&ctx->list, &device->pending_bitmap_io);  in bm_rw()
1091 	spin_unlock_irq(&device->resource->req_lock);  in bm_rw()
1110 				dynamic_drbd_dbg(device, "skipped bm write for idx %u\n", i);  in bm_rw()
1117 				dynamic_drbd_dbg(device, "skipped bm lazy write for idx %u\n", i);  in bm_rw()
1136 		wait_until_done_or_force_detached(device, device->ldev, &ctx->done);  in bm_rw()
1142 		drbd_info(device, "bitmap %s of %u pages took %lu jiffies\n",  in bm_rw()
1147 		drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n");  in bm_rw()
1148 		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);  in bm_rw()
1158 		drbd_info(device, "recounting of set bits took additional %lu jiffies\n",  in bm_rw()
1164 		drbd_info(device, "%s (%lu bits) marked out-of-sync by on disk bit-map.\n",  in bm_rw()
1175 int drbd_bm_read(struct drbd_device *device) __must_hold(local)  in drbd_bm_read()  argument
1177 	return bm_rw(device, BM_AIO_READ, 0);  in drbd_bm_read()
1186 int drbd_bm_write(struct drbd_device *device) __must_hold(local)  in drbd_bm_write()  argument
1188 	return bm_rw(device, 0, 0);  in drbd_bm_write()
1197 int drbd_bm_write_all(struct drbd_device *device) __must_hold(local)  in drbd_bm_write_all()  argument
1199 	return bm_rw(device, BM_AIO_WRITE_ALL_PAGES, 0);  in drbd_bm_write_all()
1207 int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local)  in drbd_bm_write_lazy()  argument
1209 	return bm_rw(device, BM_AIO_COPY_PAGES, upper_idx);  in drbd_bm_write_lazy()
1223 int drbd_bm_write_copy_pages(struct drbd_device *device) __must_hold(local)  in drbd_bm_write_copy_pages()  argument
1225 	return bm_rw(device, BM_AIO_COPY_PAGES, 0);  in drbd_bm_write_copy_pages()
1232 int drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local)  in drbd_bm_write_hinted()  argument
1234 	return bm_rw(device, BM_AIO_WRITE_HINTED | BM_AIO_COPY_PAGES, 0);  in drbd_bm_write_hinted()
1245 static unsigned long __bm_find_next(struct drbd_device *device, unsigned long bm_fo,  in __bm_find_next()  argument
1248 	struct drbd_bitmap *b = device->bitmap;  in __bm_find_next()
1255 		drbd_err(device, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits);  in __bm_find_next()
1285 static unsigned long bm_find_next(struct drbd_device *device,  in bm_find_next()  argument
1288 	struct drbd_bitmap *b = device->bitmap;  in bm_find_next()
1298 		bm_print_lock_info(device);  in bm_find_next()
1300 	i = __bm_find_next(device, bm_fo, find_zero_bit);  in bm_find_next()
1306 unsigned long drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo)  in drbd_bm_find_next()  argument
1308 	return bm_find_next(device, bm_fo, 0);  in drbd_bm_find_next()
1313 unsigned long drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo)
1315 	return bm_find_next(device, bm_fo, 1);
1321 unsigned long _drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo)  in _drbd_bm_find_next()  argument
1324 	return __bm_find_next(device, bm_fo, 0);  in _drbd_bm_find_next()
1327 unsigned long _drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo)  in _drbd_bm_find_next_zero()  argument
1330 	return __bm_find_next(device, bm_fo, 1);  in _drbd_bm_find_next_zero()
1339 static int __bm_change_bits_to(struct drbd_device *device, const unsigned long s,  in __bm_change_bits_to()  argument
1342 	struct drbd_bitmap *b = device->bitmap;  in __bm_change_bits_to()
1350 		drbd_err(device, "ASSERT FAILED: bit_s=%lu bit_e=%lu bm_bits=%lu\n",  in __bm_change_bits_to()
1388 static int bm_change_bits_to(struct drbd_device *device, const unsigned long s,  in bm_change_bits_to()  argument
1392 	struct drbd_bitmap *b = device->bitmap;  in bm_change_bits_to()
1402 		bm_print_lock_info(device);  in bm_change_bits_to()
1404 	c = __bm_change_bits_to(device, s, e, val);  in bm_change_bits_to()
1411 int drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)  in drbd_bm_set_bits()  argument
1413 	return bm_change_bits_to(device, s, e, 1);  in drbd_bm_set_bits()
1417 int drbd_bm_clear_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)  in drbd_bm_clear_bits()  argument
1419 	return -bm_change_bits_to(device, s, e, 0);  in drbd_bm_clear_bits()
1451 void _drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)  in _drbd_bm_set_bits()  argument
1461 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_set_bits()
1473 		__bm_change_bits_to(device, s, e, 1);  in _drbd_bm_set_bits()
1484 		__bm_change_bits_to(device, s, sl-1, 1);  in _drbd_bm_set_bits()
1496 		bm_set_full_words_within_one_page(device->bitmap, page_nr, first_word, last_word);  in _drbd_bm_set_bits()
1512 		bm_set_full_words_within_one_page(device->bitmap, last_page, first_word, last_word);  in _drbd_bm_set_bits()
1520 		__bm_change_bits_to(device, el, e, 1);  in _drbd_bm_set_bits()
1531 int drbd_bm_test_bit(struct drbd_device *device, const unsigned long bitnr)  in drbd_bm_test_bit()  argument
1534 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_test_bit()
1545 		bm_print_lock_info(device);  in drbd_bm_test_bit()
1553 		drbd_err(device, "bitnr=%lu > bm_bits=%lu\n", bitnr, b->bm_bits);  in drbd_bm_test_bit()
1562 int drbd_bm_count_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)  in drbd_bm_count_bits()  argument
1565 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_count_bits()
1582 		bm_print_lock_info(device);  in drbd_bm_count_bits()
1594 			drbd_err(device, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits);  in drbd_bm_count_bits()
1617 int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr)  in drbd_bm_e_weight()  argument
1619 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_e_weight()
1631 		bm_print_lock_info(device);  in drbd_bm_e_weight()
1644 		drbd_err(device, "start offset (%d) too large in drbd_bm_e_weight\n", s);  in drbd_bm_e_weight()