Lines Matching refs:bitmap
33 static inline char *bmname(struct bitmap *bitmap) in bmname() argument
35 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX"; in bmname()
48 static int bitmap_checkpage(struct bitmap_counts *bitmap, in bitmap_checkpage() argument
50 __releases(bitmap->lock) in bitmap_checkpage()
51 __acquires(bitmap->lock) in bitmap_checkpage()
55 if (page >= bitmap->pages) { in bitmap_checkpage()
63 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */ in bitmap_checkpage()
66 if (bitmap->bp[page].map) /* page is already allocated, just return */ in bitmap_checkpage()
74 spin_unlock_irq(&bitmap->lock); in bitmap_checkpage()
89 spin_lock_irq(&bitmap->lock); in bitmap_checkpage()
95 if (!bitmap->bp[page].map) in bitmap_checkpage()
96 bitmap->bp[page].hijacked = 1; in bitmap_checkpage()
97 } else if (bitmap->bp[page].map || in bitmap_checkpage()
98 bitmap->bp[page].hijacked) { in bitmap_checkpage()
106 bitmap->bp[page].map = mappage; in bitmap_checkpage()
107 bitmap->missing_pages--; in bitmap_checkpage()
115 static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page) in bitmap_checkfree() argument
119 if (bitmap->bp[page].count) /* page is still busy */ in bitmap_checkfree()
124 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */ in bitmap_checkfree()
125 bitmap->bp[page].hijacked = 0; in bitmap_checkfree()
126 bitmap->bp[page].map = NULL; in bitmap_checkfree()
129 ptr = bitmap->bp[page].map; in bitmap_checkfree()
130 bitmap->bp[page].map = NULL; in bitmap_checkfree()
131 bitmap->missing_pages++; in bitmap_checkfree()
207 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait) in write_sb_page() argument
211 struct mddev *mddev = bitmap->mddev; in write_sb_page()
212 struct bitmap_storage *store = &bitmap->storage; in write_sb_page()
215 if (mddev_is_clustered(bitmap->mddev)) in write_sb_page()
216 node_offset = bitmap->cluster_slot * store->file_pages; in write_sb_page()
281 static void bitmap_file_kick(struct bitmap *bitmap);
285 static void write_page(struct bitmap *bitmap, struct page *page, int wait) in write_page() argument
289 if (bitmap->storage.file == NULL) { in write_page()
290 switch (write_sb_page(bitmap, page, wait)) { in write_page()
292 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in write_page()
299 atomic_inc(&bitmap->pending_writes); in write_page()
307 wait_event(bitmap->write_wait, in write_page()
308 atomic_read(&bitmap->pending_writes)==0); in write_page()
310 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in write_page()
311 bitmap_file_kick(bitmap); in write_page()
316 struct bitmap *bitmap = bh->b_private; in end_bitmap_write() local
319 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in end_bitmap_write()
320 if (atomic_dec_and_test(&bitmap->pending_writes)) in end_bitmap_write()
321 wake_up(&bitmap->write_wait); in end_bitmap_write()
357 struct bitmap *bitmap, in read_page() argument
393 bh->b_private = bitmap; in read_page()
394 atomic_inc(&bitmap->pending_writes); in read_page()
404 wait_event(bitmap->write_wait, in read_page()
405 atomic_read(&bitmap->pending_writes)==0); in read_page()
406 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in read_page()
422 void bitmap_update_sb(struct bitmap *bitmap) in bitmap_update_sb() argument
426 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */ in bitmap_update_sb()
428 if (bitmap->mddev->bitmap_info.external) in bitmap_update_sb()
430 if (!bitmap->storage.sb_page) /* no superblock */ in bitmap_update_sb()
432 sb = kmap_atomic(bitmap->storage.sb_page); in bitmap_update_sb()
433 sb->events = cpu_to_le64(bitmap->mddev->events); in bitmap_update_sb()
434 if (bitmap->mddev->events < bitmap->events_cleared) in bitmap_update_sb()
436 bitmap->events_cleared = bitmap->mddev->events; in bitmap_update_sb()
437 sb->events_cleared = cpu_to_le64(bitmap->events_cleared); in bitmap_update_sb()
438 sb->state = cpu_to_le32(bitmap->flags); in bitmap_update_sb()
440 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ); in bitmap_update_sb()
441 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind); in bitmap_update_sb()
443 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in bitmap_update_sb()
444 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize); in bitmap_update_sb()
445 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes); in bitmap_update_sb()
446 sb->sectors_reserved = cpu_to_le32(bitmap->mddev-> in bitmap_update_sb()
449 write_page(bitmap, bitmap->storage.sb_page, 1); in bitmap_update_sb()
453 void bitmap_print_sb(struct bitmap *bitmap) in bitmap_print_sb() argument
457 if (!bitmap || !bitmap->storage.sb_page) in bitmap_print_sb()
459 sb = kmap_atomic(bitmap->storage.sb_page); in bitmap_print_sb()
460 printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap)); in bitmap_print_sb()
492 static int bitmap_new_disk_sb(struct bitmap *bitmap) in bitmap_new_disk_sb() argument
497 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO); in bitmap_new_disk_sb()
498 if (bitmap->storage.sb_page == NULL) in bitmap_new_disk_sb()
500 bitmap->storage.sb_page->index = 0; in bitmap_new_disk_sb()
502 sb = kmap_atomic(bitmap->storage.sb_page); in bitmap_new_disk_sb()
507 chunksize = bitmap->mddev->bitmap_info.chunksize; in bitmap_new_disk_sb()
516 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep; in bitmap_new_disk_sb()
523 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in bitmap_new_disk_sb()
529 write_behind = bitmap->mddev->bitmap_info.max_write_behind; in bitmap_new_disk_sb()
533 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in bitmap_new_disk_sb()
536 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in bitmap_new_disk_sb()
538 memcpy(sb->uuid, bitmap->mddev->uuid, 16); in bitmap_new_disk_sb()
540 set_bit(BITMAP_STALE, &bitmap->flags); in bitmap_new_disk_sb()
541 sb->state = cpu_to_le32(bitmap->flags); in bitmap_new_disk_sb()
542 bitmap->events_cleared = bitmap->mddev->events; in bitmap_new_disk_sb()
543 sb->events_cleared = cpu_to_le64(bitmap->mddev->events); in bitmap_new_disk_sb()
544 bitmap->mddev->bitmap_info.nodes = 0; in bitmap_new_disk_sb()
552 static int bitmap_read_sb(struct bitmap *bitmap) in bitmap_read_sb() argument
563 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) { in bitmap_read_sb()
567 set_bit(BITMAP_STALE, &bitmap->flags); in bitmap_read_sb()
575 bitmap->storage.sb_page = sb_page; in bitmap_read_sb()
579 if (bitmap->cluster_slot >= 0) { in bitmap_read_sb()
580 sector_t bm_blocks = bitmap->mddev->resync_max_sectors; in bitmap_read_sb()
583 bitmap->mddev->bitmap_info.chunksize >> 9); in bitmap_read_sb()
588 bitmap->mddev->bitmap_info.offset += bitmap->cluster_slot * (bm_blocks << 3); in bitmap_read_sb()
590 bitmap->cluster_slot, (unsigned long long)bitmap->mddev->bitmap_info.offset); in bitmap_read_sb()
593 if (bitmap->storage.file) { in bitmap_read_sb()
594 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host); in bitmap_read_sb()
597 err = read_page(bitmap->storage.file, 0, in bitmap_read_sb()
598 bitmap, bytes, sb_page); in bitmap_read_sb()
600 err = read_sb_page(bitmap->mddev, in bitmap_read_sb()
601 bitmap->mddev->bitmap_info.offset, in bitmap_read_sb()
620 if (!bitmap->mddev->sync_super) { in bitmap_read_sb()
622 strlcpy(bitmap->mddev->bitmap_info.cluster_name, in bitmap_read_sb()
642 bmname(bitmap), reason); in bitmap_read_sb()
647 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in bitmap_read_sb()
649 if (bitmap->mddev->persistent) { in bitmap_read_sb()
654 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) { in bitmap_read_sb()
657 bmname(bitmap)); in bitmap_read_sb()
661 if (!nodes && (events < bitmap->mddev->events)) { in bitmap_read_sb()
665 bmname(bitmap), events, in bitmap_read_sb()
666 (unsigned long long) bitmap->mddev->events); in bitmap_read_sb()
667 set_bit(BITMAP_STALE, &bitmap->flags); in bitmap_read_sb()
672 bitmap->flags |= le32_to_cpu(sb->state); in bitmap_read_sb()
674 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags); in bitmap_read_sb()
675 bitmap->events_cleared = le64_to_cpu(sb->events_cleared); in bitmap_read_sb()
676 strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64); in bitmap_read_sb()
682 bitmap->mddev->bitmap_info.chunksize = chunksize; in bitmap_read_sb()
683 if (nodes && (bitmap->cluster_slot < 0)) { in bitmap_read_sb()
684 err = md_setup_cluster(bitmap->mddev, nodes); in bitmap_read_sb()
687 bmname(bitmap), err); in bitmap_read_sb()
690 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev); in bitmap_read_sb()
696 if (test_bit(BITMAP_STALE, &bitmap->flags)) in bitmap_read_sb()
697 bitmap->events_cleared = bitmap->mddev->events; in bitmap_read_sb()
698 bitmap->mddev->bitmap_info.chunksize = chunksize; in bitmap_read_sb()
699 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in bitmap_read_sb()
700 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in bitmap_read_sb()
701 bitmap->mddev->bitmap_info.nodes = nodes; in bitmap_read_sb()
702 if (bitmap->mddev->bitmap_info.space == 0 || in bitmap_read_sb()
703 bitmap->mddev->bitmap_info.space > sectors_reserved) in bitmap_read_sb()
704 bitmap->mddev->bitmap_info.space = sectors_reserved; in bitmap_read_sb()
706 bitmap_print_sb(bitmap); in bitmap_read_sb()
707 if (bitmap->cluster_slot < 0) in bitmap_read_sb()
708 md_cluster_stop(bitmap->mddev); in bitmap_read_sb()
841 static void bitmap_file_kick(struct bitmap *bitmap) in bitmap_file_kick() argument
845 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) { in bitmap_file_kick()
846 bitmap_update_sb(bitmap); in bitmap_file_kick()
848 if (bitmap->storage.file) { in bitmap_file_kick()
851 ptr = d_path(&bitmap->storage.file->f_path, in bitmap_file_kick()
856 bmname(bitmap), IS_ERR(ptr) ? "" : ptr); in bitmap_file_kick()
862 bmname(bitmap)); in bitmap_file_kick()
873 static inline void set_page_attr(struct bitmap *bitmap, int pnum, in set_page_attr() argument
876 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in set_page_attr()
879 static inline void clear_page_attr(struct bitmap *bitmap, int pnum, in clear_page_attr() argument
882 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in clear_page_attr()
885 static inline int test_page_attr(struct bitmap *bitmap, int pnum, in test_page_attr() argument
888 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in test_page_attr()
891 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum, in test_and_clear_page_attr() argument
895 bitmap->storage.filemap_attr); in test_and_clear_page_attr()
904 static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) in bitmap_file_set_bit() argument
909 unsigned long chunk = block >> bitmap->counts.chunkshift; in bitmap_file_set_bit()
911 page = filemap_get_page(&bitmap->storage, chunk); in bitmap_file_set_bit()
914 bit = file_page_offset(&bitmap->storage, chunk); in bitmap_file_set_bit()
918 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in bitmap_file_set_bit()
925 set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY); in bitmap_file_set_bit()
928 static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block) in bitmap_file_clear_bit() argument
933 unsigned long chunk = block >> bitmap->counts.chunkshift; in bitmap_file_clear_bit()
935 page = filemap_get_page(&bitmap->storage, chunk); in bitmap_file_clear_bit()
938 bit = file_page_offset(&bitmap->storage, chunk); in bitmap_file_clear_bit()
940 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in bitmap_file_clear_bit()
945 if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) { in bitmap_file_clear_bit()
946 set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING); in bitmap_file_clear_bit()
947 bitmap->allclean = 0; in bitmap_file_clear_bit()
951 static int bitmap_file_test_bit(struct bitmap *bitmap, sector_t block) in bitmap_file_test_bit() argument
956 unsigned long chunk = block >> bitmap->counts.chunkshift; in bitmap_file_test_bit()
959 page = filemap_get_page(&bitmap->storage, chunk); in bitmap_file_test_bit()
962 bit = file_page_offset(&bitmap->storage, chunk); in bitmap_file_test_bit()
964 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in bitmap_file_test_bit()
976 void bitmap_unplug(struct bitmap *bitmap) in bitmap_unplug() argument
981 if (!bitmap || !bitmap->storage.filemap || in bitmap_unplug()
982 test_bit(BITMAP_STALE, &bitmap->flags)) in bitmap_unplug()
987 for (i = 0; i < bitmap->storage.file_pages; i++) { in bitmap_unplug()
988 if (!bitmap->storage.filemap) in bitmap_unplug()
990 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in bitmap_unplug()
991 need_write = test_and_clear_page_attr(bitmap, i, in bitmap_unplug()
994 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING); in bitmap_unplug()
995 write_page(bitmap, bitmap->storage.filemap[i], 0); in bitmap_unplug()
998 if (bitmap->storage.file) in bitmap_unplug()
999 wait_event(bitmap->write_wait, in bitmap_unplug()
1000 atomic_read(&bitmap->pending_writes)==0); in bitmap_unplug()
1002 md_super_wait(bitmap->mddev); in bitmap_unplug()
1004 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in bitmap_unplug()
1005 bitmap_file_kick(bitmap); in bitmap_unplug()
1009 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1021 static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) in bitmap_init_from_disk() argument
1031 struct bitmap_storage *store = &bitmap->storage; in bitmap_init_from_disk()
1033 chunks = bitmap->counts.chunks; in bitmap_init_from_disk()
1036 if (!file && !bitmap->mddev->bitmap_info.offset) { in bitmap_init_from_disk()
1042 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift) in bitmap_init_from_disk()
1044 bitmap_set_memory_bits(bitmap, in bitmap_init_from_disk()
1045 (sector_t)i << bitmap->counts.chunkshift, in bitmap_init_from_disk()
1051 outofdate = test_bit(BITMAP_STALE, &bitmap->flags); in bitmap_init_from_disk()
1054 "recovery\n", bmname(bitmap)); in bitmap_init_from_disk()
1058 bmname(bitmap), in bitmap_init_from_disk()
1066 if (!bitmap->mddev->bitmap_info.external) in bitmap_init_from_disk()
1069 if (mddev_is_clustered(bitmap->mddev)) in bitmap_init_from_disk()
1070 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE)); in bitmap_init_from_disk()
1074 index = file_page_index(&bitmap->storage, i); in bitmap_init_from_disk()
1075 bit = file_page_offset(&bitmap->storage, i); in bitmap_init_from_disk()
1085 ret = read_page(file, index, bitmap, in bitmap_init_from_disk()
1089 bitmap->mddev, in bitmap_init_from_disk()
1090 bitmap->mddev->bitmap_info.offset, in bitmap_init_from_disk()
1108 write_page(bitmap, page, 1); in bitmap_init_from_disk()
1112 &bitmap->flags)) in bitmap_init_from_disk()
1117 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in bitmap_init_from_disk()
1124 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift in bitmap_init_from_disk()
1126 bitmap_set_memory_bits(bitmap, in bitmap_init_from_disk()
1127 (sector_t)i << bitmap->counts.chunkshift, in bitmap_init_from_disk()
1136 bmname(bitmap), store->file_pages, in bitmap_init_from_disk()
1143 bmname(bitmap), ret); in bitmap_init_from_disk()
1147 void bitmap_write_all(struct bitmap *bitmap) in bitmap_write_all() argument
1154 if (!bitmap || !bitmap->storage.filemap) in bitmap_write_all()
1156 if (bitmap->storage.file) in bitmap_write_all()
1160 for (i = 0; i < bitmap->storage.file_pages; i++) in bitmap_write_all()
1161 set_page_attr(bitmap, i, in bitmap_write_all()
1163 bitmap->allclean = 0; in bitmap_write_all()
1166 static void bitmap_count_page(struct bitmap_counts *bitmap, in bitmap_count_page() argument
1169 sector_t chunk = offset >> bitmap->chunkshift; in bitmap_count_page()
1171 bitmap->bp[page].count += inc; in bitmap_count_page()
1172 bitmap_checkfree(bitmap, page); in bitmap_count_page()
1175 static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset) in bitmap_set_pending() argument
1177 sector_t chunk = offset >> bitmap->chunkshift; in bitmap_set_pending()
1179 struct bitmap_page *bp = &bitmap->bp[page]; in bitmap_set_pending()
1185 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1196 struct bitmap *bitmap; in bitmap_daemon_work() local
1206 bitmap = mddev->bitmap; in bitmap_daemon_work()
1207 if (bitmap == NULL) { in bitmap_daemon_work()
1211 if (time_before(jiffies, bitmap->daemon_lastrun in bitmap_daemon_work()
1215 bitmap->daemon_lastrun = jiffies; in bitmap_daemon_work()
1216 if (bitmap->allclean) { in bitmap_daemon_work()
1220 bitmap->allclean = 1; in bitmap_daemon_work()
1226 for (j = 0; j < bitmap->storage.file_pages; j++) in bitmap_daemon_work()
1227 if (test_and_clear_page_attr(bitmap, j, in bitmap_daemon_work()
1229 set_page_attr(bitmap, j, in bitmap_daemon_work()
1232 if (bitmap->need_sync && in bitmap_daemon_work()
1237 bitmap->need_sync = 0; in bitmap_daemon_work()
1238 if (bitmap->storage.filemap) { in bitmap_daemon_work()
1239 sb = kmap_atomic(bitmap->storage.sb_page); in bitmap_daemon_work()
1241 cpu_to_le64(bitmap->events_cleared); in bitmap_daemon_work()
1243 set_page_attr(bitmap, 0, in bitmap_daemon_work()
1250 counts = &bitmap->counts; in bitmap_daemon_work()
1273 if (*bmc == 1 && !bitmap->need_sync) { in bitmap_daemon_work()
1277 bitmap_file_clear_bit(bitmap, block); in bitmap_daemon_work()
1281 bitmap->allclean = 0; in bitmap_daemon_work()
1295 j < bitmap->storage.file_pages in bitmap_daemon_work()
1296 && !test_bit(BITMAP_STALE, &bitmap->flags); in bitmap_daemon_work()
1298 if (test_page_attr(bitmap, j, in bitmap_daemon_work()
1302 if (test_and_clear_page_attr(bitmap, j, in bitmap_daemon_work()
1304 write_page(bitmap, bitmap->storage.filemap[j], 0); in bitmap_daemon_work()
1309 if (bitmap->allclean == 0) in bitmap_daemon_work()
1315 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap, in bitmap_get_counter() argument
1318 __releases(bitmap->lock) in bitmap_get_counter()
1319 __acquires(bitmap->lock) in bitmap_get_counter()
1325 sector_t chunk = offset >> bitmap->chunkshift; in bitmap_get_counter()
1331 err = bitmap_checkpage(bitmap, page, create); in bitmap_get_counter()
1333 if (bitmap->bp[page].hijacked || in bitmap_get_counter()
1334 bitmap->bp[page].map == NULL) in bitmap_get_counter()
1335 csize = ((sector_t)1) << (bitmap->chunkshift + in bitmap_get_counter()
1338 csize = ((sector_t)1) << bitmap->chunkshift; in bitmap_get_counter()
1346 if (bitmap->bp[page].hijacked) { /* hijacked pointer */ in bitmap_get_counter()
1351 &bitmap->bp[page].map)[hi]; in bitmap_get_counter()
1354 &(bitmap->bp[page].map[pageoff]); in bitmap_get_counter()
1357 int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind) in bitmap_startwrite() argument
1359 if (!bitmap) in bitmap_startwrite()
1364 atomic_inc(&bitmap->behind_writes); in bitmap_startwrite()
1365 bw = atomic_read(&bitmap->behind_writes); in bitmap_startwrite()
1366 if (bw > bitmap->behind_writes_used) in bitmap_startwrite()
1367 bitmap->behind_writes_used = bw; in bitmap_startwrite()
1370 bw, bitmap->mddev->bitmap_info.max_write_behind); in bitmap_startwrite()
1377 spin_lock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1378 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1); in bitmap_startwrite()
1380 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1390 prepare_to_wait(&bitmap->overflow_wait, &__wait, in bitmap_startwrite()
1392 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1394 finish_wait(&bitmap->overflow_wait, &__wait); in bitmap_startwrite()
1400 bitmap_file_set_bit(bitmap, offset); in bitmap_startwrite()
1401 bitmap_count_page(&bitmap->counts, offset, 1); in bitmap_startwrite()
1409 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1421 void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, in bitmap_endwrite() argument
1424 if (!bitmap) in bitmap_endwrite()
1427 if (atomic_dec_and_test(&bitmap->behind_writes)) in bitmap_endwrite()
1428 wake_up(&bitmap->behind_wait); in bitmap_endwrite()
1430 atomic_read(&bitmap->behind_writes), in bitmap_endwrite()
1431 bitmap->mddev->bitmap_info.max_write_behind); in bitmap_endwrite()
1439 spin_lock_irqsave(&bitmap->counts.lock, flags); in bitmap_endwrite()
1440 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0); in bitmap_endwrite()
1442 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in bitmap_endwrite()
1446 if (success && !bitmap->mddev->degraded && in bitmap_endwrite()
1447 bitmap->events_cleared < bitmap->mddev->events) { in bitmap_endwrite()
1448 bitmap->events_cleared = bitmap->mddev->events; in bitmap_endwrite()
1449 bitmap->need_sync = 1; in bitmap_endwrite()
1450 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear); in bitmap_endwrite()
1457 wake_up(&bitmap->overflow_wait); in bitmap_endwrite()
1461 bitmap_set_pending(&bitmap->counts, offset); in bitmap_endwrite()
1462 bitmap->allclean = 0; in bitmap_endwrite()
1464 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in bitmap_endwrite()
1474 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, in __bitmap_start_sync() argument
1479 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */ in __bitmap_start_sync()
1483 spin_lock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1484 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in __bitmap_start_sync()
1498 spin_unlock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1502 int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, in bitmap_start_sync() argument
1517 rv |= __bitmap_start_sync(bitmap, offset, in bitmap_start_sync()
1526 void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted) in bitmap_end_sync() argument
1531 if (bitmap == NULL) { in bitmap_end_sync()
1535 spin_lock_irqsave(&bitmap->counts.lock, flags); in bitmap_end_sync()
1536 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in bitmap_end_sync()
1547 bitmap_set_pending(&bitmap->counts, offset); in bitmap_end_sync()
1548 bitmap->allclean = 0; in bitmap_end_sync()
1553 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in bitmap_end_sync()
1557 void bitmap_close_sync(struct bitmap *bitmap) in bitmap_close_sync() argument
1565 if (!bitmap) in bitmap_close_sync()
1567 while (sector < bitmap->mddev->resync_max_sectors) { in bitmap_close_sync()
1568 bitmap_end_sync(bitmap, sector, &blocks, 0); in bitmap_close_sync()
1574 void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector) in bitmap_cond_end_sync() argument
1579 if (!bitmap) in bitmap_cond_end_sync()
1582 bitmap->last_end_sync = jiffies; in bitmap_cond_end_sync()
1585 if (time_before(jiffies, (bitmap->last_end_sync in bitmap_cond_end_sync()
1586 + bitmap->mddev->bitmap_info.daemon_sleep))) in bitmap_cond_end_sync()
1588 wait_event(bitmap->mddev->recovery_wait, in bitmap_cond_end_sync()
1589 atomic_read(&bitmap->mddev->recovery_active) == 0); in bitmap_cond_end_sync()
1591 bitmap->mddev->curr_resync_completed = sector; in bitmap_cond_end_sync()
1592 set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags); in bitmap_cond_end_sync()
1593 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1); in bitmap_cond_end_sync()
1595 while (s < sector && s < bitmap->mddev->resync_max_sectors) { in bitmap_cond_end_sync()
1596 bitmap_end_sync(bitmap, s, &blocks, 0); in bitmap_cond_end_sync()
1599 bitmap->last_end_sync = jiffies; in bitmap_cond_end_sync()
1600 sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed"); in bitmap_cond_end_sync()
1604 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed) in bitmap_set_memory_bits() argument
1613 spin_lock_irq(&bitmap->counts.lock); in bitmap_set_memory_bits()
1614 bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1); in bitmap_set_memory_bits()
1616 spin_unlock_irq(&bitmap->counts.lock); in bitmap_set_memory_bits()
1621 bitmap_count_page(&bitmap->counts, offset, 1); in bitmap_set_memory_bits()
1622 bitmap_set_pending(&bitmap->counts, offset); in bitmap_set_memory_bits()
1623 bitmap->allclean = 0; in bitmap_set_memory_bits()
1627 spin_unlock_irq(&bitmap->counts.lock); in bitmap_set_memory_bits()
1631 void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e) in bitmap_dirty_bits() argument
1636 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift; in bitmap_dirty_bits()
1637 bitmap_set_memory_bits(bitmap, sec, 1); in bitmap_dirty_bits()
1638 bitmap_file_set_bit(bitmap, sec); in bitmap_dirty_bits()
1639 if (sec < bitmap->mddev->recovery_cp) in bitmap_dirty_bits()
1644 bitmap->mddev->recovery_cp = sec; in bitmap_dirty_bits()
1653 struct bitmap *bitmap = mddev->bitmap; in bitmap_flush() local
1656 if (!bitmap) /* there was no bitmap */ in bitmap_flush()
1663 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
1665 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
1667 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
1669 bitmap_update_sb(bitmap); in bitmap_flush()
1675 static void bitmap_free(struct bitmap *bitmap) in bitmap_free() argument
1680 if (!bitmap) /* there was no bitmap */ in bitmap_free()
1683 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info && in bitmap_free()
1684 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev)) in bitmap_free()
1685 md_cluster_stop(bitmap->mddev); in bitmap_free()
1688 wait_event(bitmap->write_wait, in bitmap_free()
1689 atomic_read(&bitmap->pending_writes) == 0); in bitmap_free()
1692 bitmap_file_unmap(&bitmap->storage); in bitmap_free()
1694 bp = bitmap->counts.bp; in bitmap_free()
1695 pages = bitmap->counts.pages; in bitmap_free()
1704 kfree(bitmap); in bitmap_free()
1709 struct bitmap *bitmap = mddev->bitmap; in bitmap_destroy() local
1711 if (!bitmap) /* there was no bitmap */ in bitmap_destroy()
1716 mddev->bitmap = NULL; /* disconnect from the md device */ in bitmap_destroy()
1722 if (bitmap->sysfs_can_clear) in bitmap_destroy()
1723 sysfs_put(bitmap->sysfs_can_clear); in bitmap_destroy()
1725 bitmap_free(bitmap); in bitmap_destroy()
1732 struct bitmap *bitmap_create(struct mddev *mddev, int slot) in bitmap_create()
1734 struct bitmap *bitmap; in bitmap_create() local
1744 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); in bitmap_create()
1745 if (!bitmap) in bitmap_create()
1748 spin_lock_init(&bitmap->counts.lock); in bitmap_create()
1749 atomic_set(&bitmap->pending_writes, 0); in bitmap_create()
1750 init_waitqueue_head(&bitmap->write_wait); in bitmap_create()
1751 init_waitqueue_head(&bitmap->overflow_wait); in bitmap_create()
1752 init_waitqueue_head(&bitmap->behind_wait); in bitmap_create()
1754 bitmap->mddev = mddev; in bitmap_create()
1755 bitmap->cluster_slot = slot; in bitmap_create()
1760 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear"); in bitmap_create()
1763 bitmap->sysfs_can_clear = NULL; in bitmap_create()
1765 bitmap->storage.file = file; in bitmap_create()
1781 err = bitmap_new_disk_sb(bitmap); in bitmap_create()
1783 err = bitmap_read_sb(bitmap); in bitmap_create()
1795 bitmap->daemon_lastrun = jiffies; in bitmap_create()
1796 err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1); in bitmap_create()
1801 bitmap->counts.pages, bmname(bitmap)); in bitmap_create()
1803 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0; in bitmap_create()
1807 return bitmap; in bitmap_create()
1809 bitmap_free(bitmap); in bitmap_create()
1818 struct bitmap *bitmap = mddev->bitmap; in bitmap_load() local
1820 if (!bitmap) in bitmap_load()
1830 bitmap_start_sync(bitmap, sector, &blocks, 0); in bitmap_load()
1833 bitmap_close_sync(bitmap); in bitmap_load()
1836 || bitmap->events_cleared == mddev->events) in bitmap_load()
1842 err = bitmap_init_from_disk(bitmap, start); in bitmap_load()
1847 clear_bit(BITMAP_STALE, &bitmap->flags); in bitmap_load()
1850 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery); in bitmap_load()
1855 bitmap_update_sb(bitmap); in bitmap_load()
1857 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in bitmap_load()
1873 struct bitmap *bitmap = bitmap_create(mddev, slot); in bitmap_copy_from_slot() local
1875 if (IS_ERR(bitmap)) in bitmap_copy_from_slot()
1876 return PTR_ERR(bitmap); in bitmap_copy_from_slot()
1878 rv = bitmap_read_sb(bitmap); in bitmap_copy_from_slot()
1882 rv = bitmap_init_from_disk(bitmap, 0); in bitmap_copy_from_slot()
1886 counts = &bitmap->counts; in bitmap_copy_from_slot()
1889 if (bitmap_file_test_bit(bitmap, block)) { in bitmap_copy_from_slot()
1893 bitmap_file_clear_bit(bitmap, block); in bitmap_copy_from_slot()
1894 bitmap_set_memory_bits(mddev->bitmap, block, 1); in bitmap_copy_from_slot()
1895 bitmap_file_set_bit(mddev->bitmap, block); in bitmap_copy_from_slot()
1900 bitmap_update_sb(bitmap); in bitmap_copy_from_slot()
1904 for (i = 0; i < bitmap->storage.file_pages; i++) in bitmap_copy_from_slot()
1905 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in bitmap_copy_from_slot()
1906 bitmap_write_all(bitmap); in bitmap_copy_from_slot()
1907 bitmap_unplug(bitmap); in bitmap_copy_from_slot()
1912 bitmap_free(bitmap); in bitmap_copy_from_slot()
1918 void bitmap_status(struct seq_file *seq, struct bitmap *bitmap) in bitmap_status() argument
1923 if (!bitmap) in bitmap_status()
1926 counts = &bitmap->counts; in bitmap_status()
1928 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10; in bitmap_status()
1935 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize, in bitmap_status()
1937 if (bitmap->storage.file) { in bitmap_status()
1939 seq_path(seq, &bitmap->storage.file->f_path, " \t\n"); in bitmap_status()
1945 int bitmap_resize(struct bitmap *bitmap, sector_t blocks, in bitmap_resize() argument
1973 long space = bitmap->mddev->bitmap_info.space; in bitmap_resize()
1979 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8); in bitmap_resize()
1980 if (!bitmap->mddev->bitmap_info.external) in bitmap_resize()
1983 bitmap->mddev->bitmap_info.space = space; in bitmap_resize()
1985 chunkshift = bitmap->counts.chunkshift; in bitmap_resize()
1992 if (!bitmap->mddev->bitmap_info.external) in bitmap_resize()
2000 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file) in bitmap_resize()
2002 !bitmap->mddev->bitmap_info.external, in bitmap_resize()
2003 mddev_is_clustered(bitmap->mddev) in bitmap_resize()
2004 ? bitmap->cluster_slot : 0); in bitmap_resize()
2018 bitmap->mddev->pers->quiesce(bitmap->mddev, 1); in bitmap_resize()
2020 store.file = bitmap->storage.file; in bitmap_resize()
2021 bitmap->storage.file = NULL; in bitmap_resize()
2023 if (store.sb_page && bitmap->storage.sb_page) in bitmap_resize()
2025 page_address(bitmap->storage.sb_page), in bitmap_resize()
2027 bitmap_file_unmap(&bitmap->storage); in bitmap_resize()
2028 bitmap->storage = store; in bitmap_resize()
2030 old_counts = bitmap->counts; in bitmap_resize()
2031 bitmap->counts.bp = new_bp; in bitmap_resize()
2032 bitmap->counts.pages = pages; in bitmap_resize()
2033 bitmap->counts.missing_pages = pages; in bitmap_resize()
2034 bitmap->counts.chunkshift = chunkshift; in bitmap_resize()
2035 bitmap->counts.chunks = chunks; in bitmap_resize()
2036 bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift + in bitmap_resize()
2042 spin_lock_irq(&bitmap->counts.lock); in bitmap_resize()
2052 bmc_new = bitmap_get_counter(&bitmap->counts, block, in bitmap_resize()
2060 bitmap_file_set_bit(bitmap, block); in bitmap_resize()
2064 bitmap_count_page(&bitmap->counts, in bitmap_resize()
2066 bitmap_set_pending(&bitmap->counts, in bitmap_resize()
2080 bmc = bitmap_get_counter(&bitmap->counts, block, in bitmap_resize()
2088 bitmap_count_page(&bitmap->counts, in bitmap_resize()
2090 bitmap_set_pending(&bitmap->counts, in bitmap_resize()
2096 for (i = 0; i < bitmap->storage.file_pages; i++) in bitmap_resize()
2097 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in bitmap_resize()
2099 spin_unlock_irq(&bitmap->counts.lock); in bitmap_resize()
2102 bitmap_unplug(bitmap); in bitmap_resize()
2103 bitmap->mddev->pers->quiesce(bitmap->mddev, 0); in bitmap_resize()
2136 if (mddev->bitmap || mddev->bitmap_info.file || in location_store()
2176 struct bitmap *bitmap; in location_store() local
2178 bitmap = bitmap_create(mddev, -1); in location_store()
2179 if (IS_ERR(bitmap)) in location_store()
2180 rv = PTR_ERR(bitmap); in location_store()
2182 mddev->bitmap = bitmap; in location_store()
2231 if (mddev->bitmap && in space_store()
2232 sectors < (mddev->bitmap->storage.bytes + 511) >> 9) in space_store()
2329 if (mddev->bitmap) in chunksize_store()
2354 if (mddev->bitmap || in metadata_store()
2375 if (mddev->bitmap) in can_clear_show()
2376 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ? in can_clear_show()
2386 if (mddev->bitmap == NULL) in can_clear_store()
2389 mddev->bitmap->need_sync = 1; in can_clear_store()
2393 mddev->bitmap->need_sync = 0; in can_clear_store()
2407 if (mddev->bitmap == NULL) in behind_writes_used_show()
2411 mddev->bitmap->behind_writes_used); in behind_writes_used_show()
2419 if (mddev->bitmap) in behind_writes_used_reset()
2420 mddev->bitmap->behind_writes_used = 0; in behind_writes_used_reset()