70 	struct drbd_device *device;  in drbd_md_endio()  local
72 	device = bio->bi_private;  in drbd_md_endio()
73 	device->md_io.error = bio->bi_error;  in drbd_md_endio()
86 	drbd_md_put_buffer(device);  in drbd_md_endio()
87 	device->md_io.done = 1;  in drbd_md_endio()
88 	wake_up(&device->misc_wait);  in drbd_md_endio()
90 	if (device->ldev) /* special case: drbd_md_read() during drbd_adm_attach() */  in drbd_md_endio()
91 		put_ldev(device);  in drbd_md_endio()
101 	struct drbd_device *device = peer_device->device;  in drbd_endio_read_sec_final()  local
103 	spin_lock_irqsave(&device->resource->req_lock, flags);  in drbd_endio_read_sec_final()
104 	device->read_cnt += peer_req->i.size >> 9;  in drbd_endio_read_sec_final()
106 	if (list_empty(&device->read_ee))  in drbd_endio_read_sec_final()
107 		wake_up(&device->ee_wait);  in drbd_endio_read_sec_final()
109 		__drbd_chk_io_error(device, DRBD_READ_ERROR);  in drbd_endio_read_sec_final()
110 	spin_unlock_irqrestore(&device->resource->req_lock, flags);  in drbd_endio_read_sec_final()
113 	put_ldev(device);  in drbd_endio_read_sec_final()
122 	struct drbd_device *device = peer_device->device;  in drbd_endio_write_sec_final()  local
137 	spin_lock_irqsave(&device->resource->req_lock, flags);  in drbd_endio_write_sec_final()
138 	device->writ_cnt += peer_req->i.size >> 9;  in drbd_endio_write_sec_final()
139 	list_move_tail(&peer_req->w.list, &device->done_ee);  in drbd_endio_write_sec_final()
149 	do_wake = list_empty(block_id == ID_SYNCER ? &device->sync_ee : &device->active_ee);  in drbd_endio_write_sec_final()
154 		__drbd_chk_io_error(device, DRBD_WRITE_ERROR);  in drbd_endio_write_sec_final()
155 	spin_unlock_irqrestore(&device->resource->req_lock, flags);  in drbd_endio_write_sec_final()
158 		drbd_rs_complete_io(device, i.sector);  in drbd_endio_write_sec_final()
161 		wake_up(&device->ee_wait);  in drbd_endio_write_sec_final()
164 		drbd_al_complete_io(device, &i);  in drbd_endio_write_sec_final()
167 	put_ldev(device);  in drbd_endio_write_sec_final()
176 	struct drbd_device *device = peer_req->peer_device->device;  in drbd_peer_request_endio()  local
181 		drbd_warn(device, "%s: error=%d s=%llus\n",  in drbd_peer_request_endio()
204 	struct drbd_device *device = req->device;  in drbd_request_endio()  local
238 …drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressiv…  in drbd_request_endio()
263 	spin_lock_irqsave(&device->resource->req_lock, flags);  in drbd_request_endio()
265 	spin_unlock_irqrestore(&device->resource->req_lock, flags);  in drbd_request_endio()
266 	put_ldev(device);  in drbd_request_endio()
269 		complete_master_bio(device, &m);  in drbd_request_endio()
324 	struct drbd_device *device = peer_device->device;  in w_e_send_csum()  local
346 		drbd_free_peer_req(device, peer_req);  in w_e_send_csum()
348 		inc_rs_pending(device);  in w_e_send_csum()
354 		drbd_err(device, "kmalloc() of digest failed.\n");  in w_e_send_csum()
360 		drbd_free_peer_req(device, peer_req);  in w_e_send_csum()
363 		drbd_err(device, "drbd_send_drequest(..., csum) failed\n");  in w_e_send_csum()
371 	struct drbd_device *device = peer_device->device;  in read_for_csum()  local
374 	if (!get_ldev(device))  in read_for_csum()
385 	spin_lock_irq(&device->resource->req_lock);  in read_for_csum()
386 	list_add_tail(&peer_req->w.list, &device->read_ee);  in read_for_csum()
387 	spin_unlock_irq(&device->resource->req_lock);  in read_for_csum()
389 	atomic_add(size >> 9, &device->rs_sect_ev);  in read_for_csum()
390 	if (drbd_submit_peer_request(device, peer_req, READ, DRBD_FAULT_RS_RD) == 0)  in read_for_csum()
397 	spin_lock_irq(&device->resource->req_lock);  in read_for_csum()
399 	spin_unlock_irq(&device->resource->req_lock);  in read_for_csum()
401 	drbd_free_peer_req(device, peer_req);  in read_for_csum()
403 	put_ldev(device);  in read_for_csum()
409 	struct drbd_device *device =  in w_resync_timer()  local
412 	switch (device->state.conn) {  in w_resync_timer()
414 		make_ov_request(device, cancel);  in w_resync_timer()
417 		make_resync_request(device, cancel);  in w_resync_timer()
426 	struct drbd_device *device = (struct drbd_device *) data;  in resync_timer_fn()  local
429 		&first_peer_device(device)->connection->sender_work,  in resync_timer_fn()
430 		&device->resync_work);  in resync_timer_fn()
477 static int drbd_rs_controller(struct drbd_device *device, unsigned int sect_in)  in drbd_rs_controller()  argument
489 	dc = rcu_dereference(device->ldev->disk_conf);  in drbd_rs_controller()
490 	plan = rcu_dereference(device->rs_plan_s);  in drbd_rs_controller()
494 	if (device->rs_in_flight + sect_in == 0) { /* At start of resync */  in drbd_rs_controller()
501 	correction = want - device->rs_in_flight - plan->total;  in drbd_rs_controller()
529 static int drbd_rs_number_requests(struct drbd_device *device)  in drbd_rs_number_requests()  argument
534 	sect_in = atomic_xchg(&device->rs_sect_in, 0);  in drbd_rs_number_requests()
535 	device->rs_in_flight -= sect_in;  in drbd_rs_number_requests()
538 	mxb = drbd_get_max_buffers(device) / 2;  in drbd_rs_number_requests()
539 	if (rcu_dereference(device->rs_plan_s)->size) {  in drbd_rs_number_requests()
540 		number = drbd_rs_controller(device, sect_in) >> (BM_BLOCK_SHIFT - 9);  in drbd_rs_number_requests()
541 		device->c_sync_rate = number * HZ * (BM_BLOCK_SIZE / 1024) / SLEEP_TIME;  in drbd_rs_number_requests()
543 		device->c_sync_rate = rcu_dereference(device->ldev->disk_conf)->resync_rate;  in drbd_rs_number_requests()
544 		number = SLEEP_TIME * device->c_sync_rate  / ((BM_BLOCK_SIZE / 1024) * HZ);  in drbd_rs_number_requests()
558 	if (mxb - device->rs_in_flight/8 < number)  in drbd_rs_number_requests()
559 		number = mxb - device->rs_in_flight/8;  in drbd_rs_number_requests()
564 static int make_resync_request(struct drbd_device *const device, int cancel)  in make_resync_request()  argument
566 	struct drbd_peer_device *const peer_device = first_peer_device(device);  in make_resync_request()
570 	const sector_t capacity = drbd_get_capacity(device->this_bdev);  in make_resync_request()
579 	if (device->rs_total == 0) {  in make_resync_request()
581 		drbd_resync_finished(device);  in make_resync_request()
585 	if (!get_ldev(device)) {  in make_resync_request()
590 		drbd_err(device, "Disk broke down during resync!\n");  in make_resync_request()
594 	max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9;  in make_resync_request()
595 	number = drbd_rs_number_requests(device);  in make_resync_request()
620 		bit  = drbd_bm_find_next(device, device->bm_resync_fo);  in make_resync_request()
623 			device->bm_resync_fo = drbd_bm_bits(device);  in make_resync_request()
624 			put_ldev(device);  in make_resync_request()
630 		if (drbd_try_rs_begin_io(device, sector)) {  in make_resync_request()
631 			device->bm_resync_fo = bit;  in make_resync_request()
634 		device->bm_resync_fo = bit + 1;  in make_resync_request()
636 		if (unlikely(drbd_bm_test_bit(device, bit) == 0)) {  in make_resync_request()
637 			drbd_rs_complete_io(device, sector);  in make_resync_request()
666 			if (drbd_bm_test_bit(device, bit+1) != 1)  in make_resync_request()
677 			device->bm_resync_fo = bit + 1;  in make_resync_request()
684 		if (device->use_csums) {  in make_resync_request()
687 				put_ldev(device);  in make_resync_request()
690 				drbd_rs_complete_io(device, sector);  in make_resync_request()
691 				device->bm_resync_fo = BM_SECT_TO_BIT(sector);  in make_resync_request()
703 			inc_rs_pending(device);  in make_resync_request()
707 				drbd_err(device, "drbd_send_drequest() failed, aborting...\n");  in make_resync_request()
708 				dec_rs_pending(device);  in make_resync_request()
709 				put_ldev(device);  in make_resync_request()
715 	if (device->bm_resync_fo >= drbd_bm_bits(device)) {  in make_resync_request()
722 		put_ldev(device);  in make_resync_request()
727 	device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9));  in make_resync_request()
728 	mod_timer(&device->resync_timer, jiffies + SLEEP_TIME);  in make_resync_request()
729 	put_ldev(device);  in make_resync_request()
733 static int make_ov_request(struct drbd_device *device, int cancel)  in make_ov_request()  argument
737 	const sector_t capacity = drbd_get_capacity(device->this_bdev);  in make_ov_request()
743 	number = drbd_rs_number_requests(device);  in make_ov_request()
745 	sector = device->ov_position;  in make_ov_request()
754 			&& verify_can_do_stop_sector(device)  in make_ov_request()
755 			&& sector >= device->ov_stop_sector;  in make_ov_request()
761 		if (drbd_try_rs_begin_io(device, sector)) {  in make_ov_request()
762 			device->ov_position = sector;  in make_ov_request()
769 		inc_rs_pending(device);  in make_ov_request()
770 		if (drbd_send_ov_request(first_peer_device(device), sector, size)) {  in make_ov_request()
771 			dec_rs_pending(device);  in make_ov_request()
776 	device->ov_position = sector;  in make_ov_request()
779 	device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9));  in make_ov_request()
781 		mod_timer(&device->resync_timer, jiffies + SLEEP_TIME);  in make_ov_request()
789 	struct drbd_device *device = dw->device;  in w_ov_finished()  local
791 	ov_out_of_sync_print(device);  in w_ov_finished()
792 	drbd_resync_finished(device);  in w_ov_finished()
801 	struct drbd_device *device = dw->device;  in w_resync_finished()  local
804 	drbd_resync_finished(device);  in w_resync_finished()
809 static void ping_peer(struct drbd_device *device)  in ping_peer()  argument
811 	struct drbd_connection *connection = first_peer_device(device)->connection;  in ping_peer()
816 		   test_bit(GOT_PING_ACK, &connection->flags) || device->state.conn < C_CONNECTED);  in ping_peer()
819 int drbd_resync_finished(struct drbd_device *device)  in drbd_resync_finished()  argument
831 	if (drbd_rs_del_all(device)) {  in drbd_resync_finished()
841 			dw->device = device;  in drbd_resync_finished()
842 			drbd_queue_work(&first_peer_device(device)->connection->sender_work,  in drbd_resync_finished()
846 		drbd_err(device, "Warn failed to drbd_rs_del_all() and to kmalloc(dw).\n");  in drbd_resync_finished()
849 	dt = (jiffies - device->rs_start - device->rs_paused) / HZ;  in drbd_resync_finished()
853 	db = device->rs_total;  in drbd_resync_finished()
855 	if (device->state.conn == C_VERIFY_S || device->state.conn == C_VERIFY_T)  in drbd_resync_finished()
856 		db -= device->ov_left;  in drbd_resync_finished()
859 	device->rs_paused /= HZ;  in drbd_resync_finished()
861 	if (!get_ldev(device))  in drbd_resync_finished()
864 	ping_peer(device);  in drbd_resync_finished()
866 	spin_lock_irq(&device->resource->req_lock);  in drbd_resync_finished()
867 	os = drbd_read_state(device);  in drbd_resync_finished()
879 	drbd_info(device, "%s done (total %lu sec; paused %lu sec; %lu K/sec)\n",  in drbd_resync_finished()
881 	     dt + device->rs_paused, device->rs_paused, dbdt);  in drbd_resync_finished()
883 	n_oos = drbd_bm_total_weight(device);  in drbd_resync_finished()
887 			drbd_alert(device, "Online verify found %lu %dk block out of sync!\n",  in drbd_resync_finished()
892 		D_ASSERT(device, (n_oos - device->rs_failed) == 0);  in drbd_resync_finished()
897 		if (device->use_csums && device->rs_total) {  in drbd_resync_finished()
898 			const unsigned long s = device->rs_same_csum;  in drbd_resync_finished()
899 			const unsigned long t = device->rs_total;  in drbd_resync_finished()
903 			drbd_info(device, "%u %% had equal checksums, eliminated: %luK; "  in drbd_resync_finished()
906 			     Bit2KB(device->rs_same_csum),  in drbd_resync_finished()
907 			     Bit2KB(device->rs_total - device->rs_same_csum),  in drbd_resync_finished()
908 			     Bit2KB(device->rs_total));  in drbd_resync_finished()
912 	if (device->rs_failed) {  in drbd_resync_finished()
913 		drbd_info(device, "            %lu failed blocks\n", device->rs_failed);  in drbd_resync_finished()
927 			if (device->p_uuid) {  in drbd_resync_finished()
930 					_drbd_uuid_set(device, i, device->p_uuid[i]);  in drbd_resync_finished()
931 				drbd_uuid_set(device, UI_BITMAP, device->ldev->md.uuid[UI_CURRENT]);  in drbd_resync_finished()
932 				_drbd_uuid_set(device, UI_CURRENT, device->p_uuid[UI_CURRENT]);  in drbd_resync_finished()
934 				drbd_err(device, "device->p_uuid is NULL! BUG\n");  in drbd_resync_finished()
941 			drbd_uuid_set_bm(device, 0UL);  in drbd_resync_finished()
942 			drbd_print_uuids(device, "updated UUIDs");  in drbd_resync_finished()
943 			if (device->p_uuid) {  in drbd_resync_finished()
948 					device->p_uuid[i] = device->ldev->md.uuid[i];  in drbd_resync_finished()
953 	_drbd_set_state(device, ns, CS_VERBOSE, NULL);  in drbd_resync_finished()
955 	spin_unlock_irq(&device->resource->req_lock);  in drbd_resync_finished()
956 	put_ldev(device);  in drbd_resync_finished()
958 	device->rs_total  = 0;  in drbd_resync_finished()
959 	device->rs_failed = 0;  in drbd_resync_finished()
960 	device->rs_paused = 0;  in drbd_resync_finished()
963 	if (verify_done && device->ov_left == 0)  in drbd_resync_finished()
964 		device->ov_start_sector = 0;  in drbd_resync_finished()
966 	drbd_md_sync(device);  in drbd_resync_finished()
969 		drbd_khelper(device, khelper_cmd);  in drbd_resync_finished()
975 static void move_to_net_ee_or_free(struct drbd_device *device, struct drbd_peer_request *peer_req)  in move_to_net_ee_or_free()  argument
980 		atomic_add(i, &device->pp_in_use_by_net);  in move_to_net_ee_or_free()
981 		atomic_sub(i, &device->pp_in_use);  in move_to_net_ee_or_free()
982 		spin_lock_irq(&device->resource->req_lock);  in move_to_net_ee_or_free()
983 		list_add_tail(&peer_req->w.list, &device->net_ee);  in move_to_net_ee_or_free()
984 		spin_unlock_irq(&device->resource->req_lock);  in move_to_net_ee_or_free()
987 		drbd_free_peer_req(device, peer_req);  in move_to_net_ee_or_free()
1000 	struct drbd_device *device = peer_device->device;  in w_e_end_data_req()  local
1004 		drbd_free_peer_req(device, peer_req);  in w_e_end_data_req()
1005 		dec_unacked(device);  in w_e_end_data_req()
1013 			drbd_err(device, "Sending NegDReply. sector=%llus.\n",  in w_e_end_data_req()
1019 	dec_unacked(device);  in w_e_end_data_req()
1021 	move_to_net_ee_or_free(device, peer_req);  in w_e_end_data_req()
1024 		drbd_err(device, "drbd_send_block() failed\n");  in w_e_end_data_req()
1037 	struct drbd_device *device = peer_device->device;  in w_e_end_rsdata_req()  local
1041 		drbd_free_peer_req(device, peer_req);  in w_e_end_rsdata_req()
1042 		dec_unacked(device);  in w_e_end_rsdata_req()
1046 	if (get_ldev_if_state(device, D_FAILED)) {  in w_e_end_rsdata_req()
1047 		drbd_rs_complete_io(device, peer_req->i.sector);  in w_e_end_rsdata_req()
1048 		put_ldev(device);  in w_e_end_rsdata_req()
1051 	if (device->state.conn == C_AHEAD) {  in w_e_end_rsdata_req()
1054 		if (likely(device->state.pdsk >= D_INCONSISTENT)) {  in w_e_end_rsdata_req()
1055 			inc_rs_pending(device);  in w_e_end_rsdata_req()
1059 				drbd_err(device, "Not sending RSDataReply, "  in w_e_end_rsdata_req()
1065 			drbd_err(device, "Sending NegRSDReply. sector %llus.\n",  in w_e_end_rsdata_req()
1071 		drbd_rs_failed_io(device, peer_req->i.sector, peer_req->i.size);  in w_e_end_rsdata_req()
1074 	dec_unacked(device);  in w_e_end_rsdata_req()
1076 	move_to_net_ee_or_free(device, peer_req);  in w_e_end_rsdata_req()
1079 		drbd_err(device, "drbd_send_block() failed\n");  in w_e_end_rsdata_req()
1087 	struct drbd_device *device = peer_device->device;  in w_e_end_csum_rs_req()  local
1094 		drbd_free_peer_req(device, peer_req);  in w_e_end_csum_rs_req()
1095 		dec_unacked(device);  in w_e_end_csum_rs_req()
1099 	if (get_ldev(device)) {  in w_e_end_csum_rs_req()
1100 		drbd_rs_complete_io(device, peer_req->i.sector);  in w_e_end_csum_rs_req()
1101 		put_ldev(device);  in w_e_end_csum_rs_req()
1112 			D_ASSERT(device, digest_size == di->digest_size);  in w_e_end_csum_rs_req()
1122 			drbd_set_in_sync(device, peer_req->i.sector, peer_req->i.size);  in w_e_end_csum_rs_req()
1124 			device->rs_same_csum += peer_req->i.size >> BM_BLOCK_SHIFT;  in w_e_end_csum_rs_req()
1127 			inc_rs_pending(device);  in w_e_end_csum_rs_req()
1136 			drbd_err(device, "Sending NegDReply. I guess it gets messy.\n");  in w_e_end_csum_rs_req()
1139 	dec_unacked(device);  in w_e_end_csum_rs_req()
1140 	move_to_net_ee_or_free(device, peer_req);  in w_e_end_csum_rs_req()
1143 		drbd_err(device, "drbd_send_block/ack() failed\n");  in w_e_end_csum_rs_req()
1151 	struct drbd_device *device = peer_device->device;  in w_e_end_ov_req()  local
1178 	drbd_free_peer_req(device, peer_req);  in w_e_end_ov_req()
1180 	inc_rs_pending(device);  in w_e_end_ov_req()
1183 		dec_rs_pending(device);  in w_e_end_ov_req()
1188 		drbd_free_peer_req(device, peer_req);  in w_e_end_ov_req()
1189 	dec_unacked(device);  in w_e_end_ov_req()
1193 void drbd_ov_out_of_sync_found(struct drbd_device *device, sector_t sector, int size)  in drbd_ov_out_of_sync_found()  argument
1195 	if (device->ov_last_oos_start + device->ov_last_oos_size == sector) {  in drbd_ov_out_of_sync_found()
1196 		device->ov_last_oos_size += size>>9;  in drbd_ov_out_of_sync_found()
1198 		device->ov_last_oos_start = sector;  in drbd_ov_out_of_sync_found()
1199 		device->ov_last_oos_size = size>>9;  in drbd_ov_out_of_sync_found()
1201 	drbd_set_out_of_sync(device, sector, size);  in drbd_ov_out_of_sync_found()
1208 	struct drbd_device *device = peer_device->device;  in w_e_end_ov_reply()  local
1218 		drbd_free_peer_req(device, peer_req);  in w_e_end_ov_reply()
1219 		dec_unacked(device);  in w_e_end_ov_reply()
1225 	if (get_ldev(device)) {  in w_e_end_ov_reply()
1226 		drbd_rs_complete_io(device, peer_req->i.sector);  in w_e_end_ov_reply()
1227 		put_ldev(device);  in w_e_end_ov_reply()
1238 			D_ASSERT(device, digest_size == di->digest_size);  in w_e_end_ov_reply()
1249 	drbd_free_peer_req(device, peer_req);  in w_e_end_ov_reply()
1251 		drbd_ov_out_of_sync_found(device, sector, size);  in w_e_end_ov_reply()
1253 		ov_out_of_sync_print(device);  in w_e_end_ov_reply()
1258 	dec_unacked(device);  in w_e_end_ov_reply()
1260 	--device->ov_left;  in w_e_end_ov_reply()
1263 	if ((device->ov_left & 0x200) == 0x200)  in w_e_end_ov_reply()
1264 		drbd_advance_rs_marks(device, device->ov_left);  in w_e_end_ov_reply()
1266 	stop_sector_reached = verify_can_do_stop_sector(device) &&  in w_e_end_ov_reply()
1267 		(sector + (size>>9)) >= device->ov_stop_sector;  in w_e_end_ov_reply()
1269 	if (device->ov_left == 0 || stop_sector_reached) {  in w_e_end_ov_reply()
1270 		ov_out_of_sync_print(device);  in w_e_end_ov_reply()
1271 		drbd_resync_finished(device);  in w_e_end_ov_reply()
1300 	struct drbd_device *device =  in w_send_write_hint()  local
1306 	sock = &first_peer_device(device)->connection->data;  in w_send_write_hint()
1307 	if (!drbd_prepare_command(first_peer_device(device), sock))  in w_send_write_hint()
1309 	return drbd_send_command(first_peer_device(device), sock, P_UNPLUG_REMOTE, 0, NULL, 0);  in w_send_write_hint()
1336 	struct drbd_device *device = req->device;  in w_send_out_of_sync()  local
1337 	struct drbd_peer_device *const peer_device = first_peer_device(device);  in w_send_out_of_sync()
1367 	struct drbd_device *device = req->device;  in w_send_dblock()  local
1368 	struct drbd_peer_device *const peer_device = first_peer_device(device);  in w_send_dblock()
1396 	struct drbd_device *device = req->device;  in w_send_read_req()  local
1397 	struct drbd_peer_device *const peer_device = first_peer_device(device);  in w_send_read_req()
1422 	struct drbd_device *device = req->device;  in w_restart_disk_io()  local
1425 		drbd_al_begin_io(device, &req->i);  in w_restart_disk_io()
1428 	req->private_bio->bi_bdev = device->ldev->backing_bdev;  in w_restart_disk_io()
1434 static int _drbd_may_sync_now(struct drbd_device *device)  in _drbd_may_sync_now()  argument
1436 	struct drbd_device *odev = device;  in _drbd_may_sync_now()
1464 static int _drbd_pause_after(struct drbd_device *device)  in _drbd_pause_after()  argument
1488 static int _drbd_resume_next(struct drbd_device *device)  in _drbd_resume_next()  argument
1508 void resume_next_sg(struct drbd_device *device)  in resume_next_sg()  argument
1511 	_drbd_resume_next(device);  in resume_next_sg()
1515 void suspend_other_sg(struct drbd_device *device)  in suspend_other_sg()  argument
1518 	_drbd_pause_after(device);  in suspend_other_sg()
1523 enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor)  in drbd_resync_after_valid()  argument
1536 		if (odev == device)  in drbd_resync_after_valid()
1561 void drbd_resync_after_changed(struct drbd_device *device)  in drbd_resync_after_changed()  argument
1566 		changes  = _drbd_pause_after(device);  in drbd_resync_after_changed()
1567 		changes |= _drbd_resume_next(device);  in drbd_resync_after_changed()
1571 void drbd_rs_controller_reset(struct drbd_device *device)  in drbd_rs_controller_reset()  argument
1573 	struct gendisk *disk = device->ldev->backing_bdev->bd_contains->bd_disk;  in drbd_rs_controller_reset()
1576 	atomic_set(&device->rs_sect_in, 0);  in drbd_rs_controller_reset()
1577 	atomic_set(&device->rs_sect_ev, 0);  in drbd_rs_controller_reset()
1578 	device->rs_in_flight = 0;  in drbd_rs_controller_reset()
1579 	device->rs_last_events =  in drbd_rs_controller_reset()
1588 	plan = rcu_dereference(device->rs_plan_s);  in drbd_rs_controller_reset()
1596 	struct drbd_device *device = (struct drbd_device *) data;  in start_resync_timer_fn()  local
1597 	drbd_device_post_work(device, RS_START);  in start_resync_timer_fn()
1600 static void do_start_resync(struct drbd_device *device)  in do_start_resync()  argument
1602 	if (atomic_read(&device->unacked_cnt) || atomic_read(&device->rs_pending_cnt)) {  in do_start_resync()
1603 		drbd_warn(device, "postponing start_resync ...\n");  in do_start_resync()
1604 		device->start_resync_timer.expires = jiffies + HZ/10;  in do_start_resync()
1605 		add_timer(&device->start_resync_timer);  in do_start_resync()
1609 	drbd_start_resync(device, C_SYNC_SOURCE);  in do_start_resync()
1610 	clear_bit(AHEAD_TO_SYNC_SOURCE, &device->flags);  in do_start_resync()
1613 …atic bool use_checksum_based_resync(struct drbd_connection *connection, struct drbd_device *device)  in use_checksum_based_resync()  argument
1622 		 || test_bit(CRASHED_PRIMARY, &device->flags));	/* or only after Primary crash? */  in use_checksum_based_resync()
1633 void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)  in drbd_start_resync()  argument
1635 	struct drbd_peer_device *peer_device = first_peer_device(device);  in drbd_start_resync()
1640 	if (device->state.conn >= C_SYNC_SOURCE && device->state.conn < C_AHEAD) {  in drbd_start_resync()
1641 		drbd_err(device, "Resync already running!\n");  in drbd_start_resync()
1645 	if (!test_bit(B_RS_H_DONE, &device->flags)) {  in drbd_start_resync()
1650 			r = drbd_khelper(device, "before-resync-target");  in drbd_start_resync()
1653 				drbd_info(device, "before-resync-target handler returned %d, "  in drbd_start_resync()
1659 			r = drbd_khelper(device, "before-resync-source");  in drbd_start_resync()
1663 					drbd_info(device, "before-resync-source handler returned %d, "  in drbd_start_resync()
1666 					drbd_info(device, "before-resync-source handler returned %d, "  in drbd_start_resync()
1679 		if (!mutex_trylock(device->state_mutex)) {  in drbd_start_resync()
1680 			set_bit(B_RS_H_DONE, &device->flags);  in drbd_start_resync()
1681 			device->start_resync_timer.expires = jiffies + HZ/5;  in drbd_start_resync()
1682 			add_timer(&device->start_resync_timer);  in drbd_start_resync()
1686 		mutex_lock(device->state_mutex);  in drbd_start_resync()
1688 	clear_bit(B_RS_H_DONE, &device->flags);  in drbd_start_resync()
1692 	spin_lock_irq(&device->resource->req_lock);  in drbd_start_resync()
1695 	if (device->state.conn < C_CONNECTED  in drbd_start_resync()
1696 	|| !get_ldev_if_state(device, D_NEGOTIATING)) {  in drbd_start_resync()
1698 		spin_unlock_irq(&device->resource->req_lock);  in drbd_start_resync()
1699 		mutex_unlock(device->state_mutex);  in drbd_start_resync()
1703 	ns = drbd_read_state(device);  in drbd_start_resync()
1705 	ns.aftr_isp = !_drbd_may_sync_now(device);  in drbd_start_resync()
1714 	r = __drbd_set_state(device, ns, CS_VERBOSE, NULL);  in drbd_start_resync()
1715 	ns = drbd_read_state(device);  in drbd_start_resync()
1721 		unsigned long tw = drbd_bm_total_weight(device);  in drbd_start_resync()
1725 		device->rs_failed    = 0;  in drbd_start_resync()
1726 		device->rs_paused    = 0;  in drbd_start_resync()
1727 		device->rs_same_csum = 0;  in drbd_start_resync()
1728 		device->rs_last_sect_ev = 0;  in drbd_start_resync()
1729 		device->rs_total     = tw;  in drbd_start_resync()
1730 		device->rs_start     = now;  in drbd_start_resync()
1732 			device->rs_mark_left[i] = tw;  in drbd_start_resync()
1733 			device->rs_mark_time[i] = now;  in drbd_start_resync()
1735 		_drbd_pause_after(device);  in drbd_start_resync()
1739 		spin_lock(&device->al_lock);  in drbd_start_resync()
1740 		lc_reset(device->resync);  in drbd_start_resync()
1741 		device->resync_locked = 0;  in drbd_start_resync()
1742 		device->resync_wenr = LC_FREE;  in drbd_start_resync()
1743 		spin_unlock(&device->al_lock);  in drbd_start_resync()
1746 	spin_unlock_irq(&device->resource->req_lock);  in drbd_start_resync()
1749 		wake_up(&device->al_wait); /* for lc_reset() above */  in drbd_start_resync()
1752 		device->rs_last_bcast = jiffies - HZ;  in drbd_start_resync()
1754 		drbd_info(device, "Began resync as %s (will sync %lu KB [%lu bits set]).\n",  in drbd_start_resync()
1756 		     (unsigned long) device->rs_total << (BM_BLOCK_SHIFT-10),  in drbd_start_resync()
1757 		     (unsigned long) device->rs_total);  in drbd_start_resync()
1759 			device->bm_resync_fo = 0;  in drbd_start_resync()
1760 			device->use_csums = use_checksum_based_resync(connection, device);  in drbd_start_resync()
1762 			device->use_csums = 0;  in drbd_start_resync()
1775 		if (connection->agreed_pro_version < 95 && device->rs_total == 0) {  in drbd_start_resync()
1796 			drbd_resync_finished(device);  in drbd_start_resync()
1799 		drbd_rs_controller_reset(device);  in drbd_start_resync()
1805 			mod_timer(&device->resync_timer, jiffies);  in drbd_start_resync()
1807 		drbd_md_sync(device);  in drbd_start_resync()
1809 	put_ldev(device);  in drbd_start_resync()
1810 	mutex_unlock(device->state_mutex);  in drbd_start_resync()
1813 static void update_on_disk_bitmap(struct drbd_device *device, bool resync_done)  in update_on_disk_bitmap()  argument
1816 	device->rs_last_bcast = jiffies;  in update_on_disk_bitmap()
1818 	if (!get_ldev(device))  in update_on_disk_bitmap()
1821 	drbd_bm_write_lazy(device, 0);  in update_on_disk_bitmap()
1822 	if (resync_done && is_sync_state(device->state.conn))  in update_on_disk_bitmap()
1823 		drbd_resync_finished(device);  in update_on_disk_bitmap()
1825 	drbd_bcast_event(device, &sib);  in update_on_disk_bitmap()
1827 	device->rs_last_bcast = jiffies;  in update_on_disk_bitmap()
1828 	put_ldev(device);  in update_on_disk_bitmap()
1831 static void drbd_ldev_destroy(struct drbd_device *device)  in drbd_ldev_destroy()  argument
1833 	lc_destroy(device->resync);  in drbd_ldev_destroy()
1834 	device->resync = NULL;  in drbd_ldev_destroy()
1835 	lc_destroy(device->act_log);  in drbd_ldev_destroy()
1836 	device->act_log = NULL;  in drbd_ldev_destroy()
1839 	drbd_free_ldev(device->ldev);  in drbd_ldev_destroy()
1840 	device->ldev = NULL;  in drbd_ldev_destroy()
1843 	clear_bit(GOING_DISKLESS, &device->flags);  in drbd_ldev_destroy()
1844 	wake_up(&device->misc_wait);  in drbd_ldev_destroy()
1847 static void go_diskless(struct drbd_device *device)  in go_diskless()  argument
1849 	D_ASSERT(device, device->state.disk == D_FAILED);  in go_diskless()
1868 	if (device->bitmap && device->ldev) {  in go_diskless()
1873 		if (drbd_bitmap_io_from_worker(device, drbd_bm_write,  in go_diskless()
1875 			if (test_bit(WAS_READ_ERROR, &device->flags)) {  in go_diskless()
1876 				drbd_md_set_flag(device, MDF_FULL_SYNC);  in go_diskless()
1877 				drbd_md_sync(device);  in go_diskless()
1882 	drbd_force_state(device, NS(disk, D_DISKLESS));  in go_diskless()
1885 static int do_md_sync(struct drbd_device *device)  in do_md_sync()  argument
1887 	drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n");  in do_md_sync()
1888 	drbd_md_sync(device);  in do_md_sync()
1915 static void do_device_work(struct drbd_device *device, const unsigned long todo)  in do_device_work()  argument
1918 		do_md_sync(device);  in do_device_work()
1921 		update_on_disk_bitmap(device, test_bit(RS_DONE, &todo));  in do_device_work()
1923 		go_diskless(device);  in do_device_work()
1925 		drbd_ldev_destroy(device);  in do_device_work()
1927 		do_start_resync(device);  in do_device_work()
1956 		struct drbd_device *device = peer_device->device;  in do_unqueued_work()  local
1957 		unsigned long todo = get_work_bits(&device->flags);  in do_unqueued_work()
1961 		kref_get(&device->kref);  in do_unqueued_work()
1963 		do_device_work(device, todo);  in do_unqueued_work()
1964 		kref_put(&device->kref, drbd_destroy_device);  in do_unqueued_work()
2123 		struct drbd_device *device = peer_device->device;  in drbd_worker()  local
2124 		D_ASSERT(device, device->state.disk == D_DISKLESS && device->state.conn == C_STANDALONE);  in drbd_worker()
2125 		kref_get(&device->kref);  in drbd_worker()
2127 		drbd_device_cleanup(device);  in drbd_worker()
2128 		kref_put(&device->kref, drbd_destroy_device);  in drbd_worker()