120 	struct drbd_bitmap *b = device->bitmap;  in __bm_print_lock_info()  local
125 		 func, b->bm_why ?: "?",  in __bm_print_lock_info()
126 		 b->bm_task->comm, task_pid_nr(b->bm_task));  in __bm_print_lock_info()
131 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_lock()  local
134 	if (!b) {  in drbd_bm_lock()
139 	trylock_failed = !mutex_trylock(&b->bm_change);  in drbd_bm_lock()
144 			  why, b->bm_why ?: "?",  in drbd_bm_lock()
145 			  b->bm_task->comm, task_pid_nr(b->bm_task));  in drbd_bm_lock()
146 		mutex_lock(&b->bm_change);  in drbd_bm_lock()
148 	if (BM_LOCKED_MASK & b->bm_flags)  in drbd_bm_lock()
150 	b->bm_flags |= flags & BM_LOCKED_MASK;  in drbd_bm_lock()
152 	b->bm_why  = why;  in drbd_bm_lock()
153 	b->bm_task = current;  in drbd_bm_lock()
158 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_unlock()  local
159 	if (!b) {  in drbd_bm_unlock()
167 	b->bm_flags &= ~BM_LOCKED_MASK;  in drbd_bm_unlock()
168 	b->bm_why  = NULL;  in drbd_bm_unlock()
169 	b->bm_task = NULL;  in drbd_bm_unlock()
170 	mutex_unlock(&b->bm_change);  in drbd_bm_unlock()
218 	struct drbd_bitmap *b = device->bitmap;  in bm_page_lock_io()  local
219 	void *addr = &page_private(b->bm_pages[page_nr]);  in bm_page_lock_io()
220 	wait_event(b->bm_io_wait, !test_and_set_bit(BM_PAGE_IO_LOCK, addr));  in bm_page_lock_io()
225 	struct drbd_bitmap *b = device->bitmap;  in bm_page_unlock_io()  local
226 	void *addr = &page_private(b->bm_pages[page_nr]);  in bm_page_unlock_io()
293 static unsigned int bm_word_to_page_idx(struct drbd_bitmap *b, unsigned long long_nr)  in bm_word_to_page_idx()  argument
297 	BUG_ON(page_nr >= b->bm_number_of_pages);  in bm_word_to_page_idx()
301 static unsigned int bm_bit_to_page_idx(struct drbd_bitmap *b, u64 bitnr)  in bm_bit_to_page_idx()  argument
305 	BUG_ON(page_nr >= b->bm_number_of_pages);  in bm_bit_to_page_idx()
309 static unsigned long *__bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)  in __bm_map_pidx()  argument
311 	struct page *page = b->bm_pages[idx];  in __bm_map_pidx()
315 static unsigned long *bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)  in bm_map_pidx()  argument
317 	return __bm_map_pidx(b, idx);  in bm_map_pidx()
378 static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)  in bm_realloc_pages()  argument
380 	struct page **old_pages = b->bm_pages;  in bm_realloc_pages()
383 	unsigned long have = b->bm_number_of_pages;  in bm_realloc_pages()
431 		b->bm_flags |= BM_P_VMALLOCED;  in bm_realloc_pages()
433 		b->bm_flags &= ~BM_P_VMALLOCED;  in bm_realloc_pages()
444 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_init()  local
445 	WARN_ON(b != NULL);  in drbd_bm_init()
446 	b = kzalloc(sizeof(struct drbd_bitmap), GFP_KERNEL);  in drbd_bm_init()
447 	if (!b)  in drbd_bm_init()
449 	spin_lock_init(&b->bm_lock);  in drbd_bm_init()
450 	mutex_init(&b->bm_change);  in drbd_bm_init()
451 	init_waitqueue_head(&b->bm_io_wait);  in drbd_bm_init()
453 	device->bitmap = b;  in drbd_bm_init()
485 static int bm_clear_surplus(struct drbd_bitmap *b)  in bm_clear_surplus()  argument
493 	tmp = (b->bm_bits & BITS_PER_PAGE_MASK);  in bm_clear_surplus()
500 	p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);  in bm_clear_surplus()
522 static void bm_set_surplus(struct drbd_bitmap *b)  in bm_set_surplus()  argument
529 	tmp = (b->bm_bits & BITS_PER_PAGE_MASK);  in bm_set_surplus()
536 	p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);  in bm_set_surplus()
557 static unsigned long bm_count_bits(struct drbd_bitmap *b)  in bm_count_bits()  argument
561 	unsigned long mask = (1UL << (b->bm_bits & BITS_PER_LONG_MASK)) -1;  in bm_count_bits()
565 	for (idx = 0; idx < b->bm_number_of_pages - 1; idx++) {  in bm_count_bits()
566 		p_addr = __bm_map_pidx(b, idx);  in bm_count_bits()
573 	last_word = ((b->bm_bits - 1) & BITS_PER_PAGE_MASK) >> LN2_BPL;  in bm_count_bits()
574 	p_addr = __bm_map_pidx(b, idx);  in bm_count_bits()
587 static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len)  in bm_memset()  argument
595 	if (end > b->bm_words) {  in bm_memset()
602 		idx = bm_word_to_page_idx(b, offset);  in bm_memset()
603 		p_addr = bm_map_pidx(b, idx);  in bm_memset()
611 		bm_set_page_need_writeout(b->bm_pages[idx]);  in bm_memset()
637 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_resize()  local
644 	if (!expect(b))  in drbd_bm_resize()
652 	if (capacity == b->bm_dev_capacity)  in drbd_bm_resize()
655 	opages_vmalloced = (BM_P_VMALLOCED & b->bm_flags);  in drbd_bm_resize()
658 		spin_lock_irq(&b->bm_lock);  in drbd_bm_resize()
659 		opages = b->bm_pages;  in drbd_bm_resize()
660 		onpages = b->bm_number_of_pages;  in drbd_bm_resize()
661 		owords = b->bm_words;  in drbd_bm_resize()
662 		b->bm_pages = NULL;  in drbd_bm_resize()
663 		b->bm_number_of_pages =  in drbd_bm_resize()
664 		b->bm_set   =  in drbd_bm_resize()
665 		b->bm_bits  =  in drbd_bm_resize()
666 		b->bm_words =  in drbd_bm_resize()
667 		b->bm_dev_capacity = 0;  in drbd_bm_resize()
668 		spin_unlock_irq(&b->bm_lock);  in drbd_bm_resize()
694 	have = b->bm_number_of_pages;  in drbd_bm_resize()
696 		D_ASSERT(device, b->bm_pages != NULL);  in drbd_bm_resize()
697 		npages = b->bm_pages;  in drbd_bm_resize()
702 			npages = bm_realloc_pages(b, want);  in drbd_bm_resize()
710 	spin_lock_irq(&b->bm_lock);  in drbd_bm_resize()
711 	opages = b->bm_pages;  in drbd_bm_resize()
712 	owords = b->bm_words;  in drbd_bm_resize()
713 	obits  = b->bm_bits;  in drbd_bm_resize()
717 		bm_set_surplus(b);  in drbd_bm_resize()
719 	b->bm_pages = npages;  in drbd_bm_resize()
720 	b->bm_number_of_pages = want;  in drbd_bm_resize()
721 	b->bm_bits  = bits;  in drbd_bm_resize()
722 	b->bm_words = words;  in drbd_bm_resize()
723 	b->bm_dev_capacity = capacity;  in drbd_bm_resize()
727 			bm_memset(b, owords, 0xff, words-owords);  in drbd_bm_resize()
728 			b->bm_set += bits - obits;  in drbd_bm_resize()
730 			bm_memset(b, owords, 0x00, words-owords);  in drbd_bm_resize()
739 	(void)bm_clear_surplus(b);  in drbd_bm_resize()
741 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_resize()
745 		b->bm_set = bm_count_bits(b);  in drbd_bm_resize()
763 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_total_weight()  local
767 	if (!expect(b))  in _drbd_bm_total_weight()
769 	if (!expect(b->bm_pages))  in _drbd_bm_total_weight()
772 	spin_lock_irqsave(&b->bm_lock, flags);  in _drbd_bm_total_weight()
773 	s = b->bm_set;  in _drbd_bm_total_weight()
774 	spin_unlock_irqrestore(&b->bm_lock, flags);  in _drbd_bm_total_weight()
792 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_words()  local
793 	if (!expect(b))  in drbd_bm_words()
795 	if (!expect(b->bm_pages))  in drbd_bm_words()
798 	return b->bm_words;  in drbd_bm_words()
803 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_bits()  local
804 	if (!expect(b))  in drbd_bm_bits()
807 	return b->bm_bits;  in drbd_bm_bits()
818 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_merge_lel()  local
826 	if (!expect(b))  in drbd_bm_merge_lel()
828 	if (!expect(b->bm_pages))  in drbd_bm_merge_lel()
832 	WARN_ON(offset >= b->bm_words);  in drbd_bm_merge_lel()
833 	WARN_ON(end    >  b->bm_words);  in drbd_bm_merge_lel()
835 	spin_lock_irq(&b->bm_lock);  in drbd_bm_merge_lel()
838 		idx = bm_word_to_page_idx(b, offset);  in drbd_bm_merge_lel()
839 		p_addr = bm_map_pidx(b, idx);  in drbd_bm_merge_lel()
846 			b->bm_set += hweight_long(word) - bits;  in drbd_bm_merge_lel()
849 		bm_set_page_need_writeout(b->bm_pages[idx]);  in drbd_bm_merge_lel()
856 	if (end == b->bm_words)  in drbd_bm_merge_lel()
857 		b->bm_set -= bm_clear_surplus(b);  in drbd_bm_merge_lel()
858 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_merge_lel()
867 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_get_lel()  local
873 	if (!expect(b))  in drbd_bm_get_lel()
875 	if (!expect(b->bm_pages))  in drbd_bm_get_lel()
878 	spin_lock_irq(&b->bm_lock);  in drbd_bm_get_lel()
879 	if ((offset >= b->bm_words) ||  in drbd_bm_get_lel()
880 	    (end    >  b->bm_words) ||  in drbd_bm_get_lel()
885 			(unsigned long) b->bm_words);  in drbd_bm_get_lel()
889 			p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, offset));  in drbd_bm_get_lel()
897 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_get_lel()
903 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_set_all()  local
904 	if (!expect(b))  in drbd_bm_set_all()
906 	if (!expect(b->bm_pages))  in drbd_bm_set_all()
909 	spin_lock_irq(&b->bm_lock);  in drbd_bm_set_all()
910 	bm_memset(b, 0, 0xff, b->bm_words);  in drbd_bm_set_all()
911 	(void)bm_clear_surplus(b);  in drbd_bm_set_all()
912 	b->bm_set = b->bm_bits;  in drbd_bm_set_all()
913 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_set_all()
919 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_clear_all()  local
920 	if (!expect(b))  in drbd_bm_clear_all()
922 	if (!expect(b->bm_pages))  in drbd_bm_clear_all()
925 	spin_lock_irq(&b->bm_lock);  in drbd_bm_clear_all()
926 	bm_memset(b, 0, 0, b->bm_words);  in drbd_bm_clear_all()
927 	b->bm_set = 0;  in drbd_bm_clear_all()
928 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_clear_all()
948 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_endio()  local
952 	    !bm_test_page_unchanged(b->bm_pages[idx]))  in drbd_bm_endio()
959 		bm_set_page_io_err(b->bm_pages[idx]);  in drbd_bm_endio()
966 		bm_clear_page_io_err(b->bm_pages[idx]);  in drbd_bm_endio()
988 	struct drbd_bitmap *b = device->bitmap;  in bm_page_io_async()  local
1007 	bm_set_page_unchanged(b->bm_pages[page_nr]);  in bm_page_io_async()
1011 		copy_highpage(page, b->bm_pages[page_nr]);  in bm_page_io_async()
1014 		page = b->bm_pages[page_nr];  in bm_page_io_async()
1040 	struct drbd_bitmap *b = device->bitmap;  in bm_rw()  local
1078 		WARN_ON(!(BM_LOCKED_MASK & b->bm_flags));  in bm_rw()
1084 	num_pages = b->bm_number_of_pages;  in bm_rw()
1096 				    &page_private(b->bm_pages[i])))  in bm_rw()
1100 			    bm_test_page_unchanged(b->bm_pages[i])) {  in bm_rw()
1107 			    !bm_test_page_lazy_writeout(b->bm_pages[i])) {  in bm_rw()
1148 		b->bm_set = bm_count_bits(b);  in bm_rw()
1152 	now = b->bm_set;  in bm_rw()
1239 	struct drbd_bitmap *b = device->bitmap;  in __bm_find_next()  local
1245 	if (bm_fo > b->bm_bits) {  in __bm_find_next()
1246 		drbd_err(device, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits);  in __bm_find_next()
1249 		while (bm_fo < b->bm_bits) {  in __bm_find_next()
1252 			p_addr = __bm_map_pidx(b, bm_bit_to_page_idx(b, bm_fo));  in __bm_find_next()
1264 				if (bm_fo >= b->bm_bits)  in __bm_find_next()
1279 	struct drbd_bitmap *b = device->bitmap;  in bm_find_next()  local
1282 	if (!expect(b))  in bm_find_next()
1284 	if (!expect(b->bm_pages))  in bm_find_next()
1287 	spin_lock_irq(&b->bm_lock);  in bm_find_next()
1288 	if (BM_DONT_TEST & b->bm_flags)  in bm_find_next()
1293 	spin_unlock_irq(&b->bm_lock);  in bm_find_next()
1333 	struct drbd_bitmap *b = device->bitmap;  in __bm_change_bits_to()  local
1340 	if (e >= b->bm_bits) {  in __bm_change_bits_to()
1342 				s, e, b->bm_bits);  in __bm_change_bits_to()
1343 		e = b->bm_bits ? b->bm_bits -1 : 0;  in __bm_change_bits_to()
1346 		unsigned int page_nr = bm_bit_to_page_idx(b, bitnr);  in __bm_change_bits_to()
1351 				bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1353 				bm_set_page_need_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1356 			p_addr = __bm_map_pidx(b, page_nr);  in __bm_change_bits_to()
1367 		bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1369 		bm_set_page_need_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1371 	b->bm_set += changed_total;  in __bm_change_bits_to()
1383 	struct drbd_bitmap *b = device->bitmap;  in bm_change_bits_to()  local
1386 	if (!expect(b))  in bm_change_bits_to()
1388 	if (!expect(b->bm_pages))  in bm_change_bits_to()
1391 	spin_lock_irqsave(&b->bm_lock, flags);  in bm_change_bits_to()
1392 	if ((val ? BM_DONT_SET : BM_DONT_CLEAR) & b->bm_flags)  in bm_change_bits_to()
1397 	spin_unlock_irqrestore(&b->bm_lock, flags);  in bm_change_bits_to()
1415 static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b,  in bm_set_full_words_within_one_page()  argument
1421 	unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr]);  in bm_set_full_words_within_one_page()
1432 		bm_set_page_lazy_writeout(b->bm_pages[page_nr]);  in bm_set_full_words_within_one_page()
1433 		b->bm_set += changed;  in bm_set_full_words_within_one_page()
1452 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_set_bits()  local
1463 		spin_lock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1465 		spin_unlock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1471 	spin_lock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1488 		spin_unlock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1491 		spin_lock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1512 	spin_unlock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1525 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_test_bit()  local
1529 	if (!expect(b))  in drbd_bm_test_bit()
1531 	if (!expect(b->bm_pages))  in drbd_bm_test_bit()
1534 	spin_lock_irqsave(&b->bm_lock, flags);  in drbd_bm_test_bit()
1535 	if (BM_DONT_TEST & b->bm_flags)  in drbd_bm_test_bit()
1537 	if (bitnr < b->bm_bits) {  in drbd_bm_test_bit()
1538 		p_addr = bm_map_pidx(b, bm_bit_to_page_idx(b, bitnr));  in drbd_bm_test_bit()
1541 	} else if (bitnr == b->bm_bits) {  in drbd_bm_test_bit()
1544 		drbd_err(device, "bitnr=%lu > bm_bits=%lu\n", bitnr, b->bm_bits);  in drbd_bm_test_bit()
1548 	spin_unlock_irqrestore(&b->bm_lock, flags);  in drbd_bm_test_bit()
1556 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_count_bits()  local
1566 	if (!expect(b))  in drbd_bm_count_bits()
1568 	if (!expect(b->bm_pages))  in drbd_bm_count_bits()
1571 	spin_lock_irqsave(&b->bm_lock, flags);  in drbd_bm_count_bits()
1572 	if (BM_DONT_TEST & b->bm_flags)  in drbd_bm_count_bits()
1575 		unsigned int idx = bm_bit_to_page_idx(b, bitnr);  in drbd_bm_count_bits()
1580 			p_addr = bm_map_pidx(b, idx);  in drbd_bm_count_bits()
1582 		if (expect(bitnr < b->bm_bits))  in drbd_bm_count_bits()
1585 			drbd_err(device, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits);  in drbd_bm_count_bits()
1589 	spin_unlock_irqrestore(&b->bm_lock, flags);  in drbd_bm_count_bits()
1610 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_e_weight()  local
1615 	if (!expect(b))  in drbd_bm_e_weight()
1617 	if (!expect(b->bm_pages))  in drbd_bm_e_weight()
1620 	spin_lock_irqsave(&b->bm_lock, flags);  in drbd_bm_e_weight()
1621 	if (BM_DONT_TEST & b->bm_flags)  in drbd_bm_e_weight()
1625 	e = min((size_t)S2W(enr+1), b->bm_words);  in drbd_bm_e_weight()
1627 	if (s < b->bm_words) {  in drbd_bm_e_weight()
1629 		p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, s));  in drbd_bm_e_weight()
1637 	spin_unlock_irqrestore(&b->bm_lock, flags);  in drbd_bm_e_weight()