This source file includes following definitions.
- bio_alloc_drbd
- _get_ldev_if_state
- tl_release
- _tl_restart
- tl_restart
- tl_clear
- tl_abort_disk_io
- drbd_thread_setup
- drbd_thread_init
- drbd_thread_start
- _drbd_thread_stop
- conn_lowest_minor
- drbd_calc_cpu_mask
- drbd_thread_current_set_cpu
- drbd_header_size
- prepare_header80
- prepare_header95
- prepare_header100
- prepare_header
- __conn_prepare_command
- conn_prepare_command
- drbd_prepare_command
- __send_command
- __conn_send_command
- conn_send_command
- drbd_send_command
- drbd_send_ping
- drbd_send_ping_ack
- drbd_send_sync_param
- __drbd_send_protocol
- drbd_send_protocol
- _drbd_send_uuids
- drbd_send_uuids
- drbd_send_uuids_skip_initial_sync
- drbd_print_uuids
- drbd_gen_and_send_sync_uuid
- assign_p_sizes_qlim
- drbd_send_sizes
- drbd_send_current_state
- drbd_send_state
- drbd_send_state_req
- conn_send_state_req
- drbd_send_sr_reply
- conn_send_sr_reply
- dcbp_set_code
- dcbp_set_start
- dcbp_set_pad_bits
- fill_bitmap_rle_bits
- send_bitmap_rle_or_plain
- _drbd_send_bitmap
- drbd_send_bitmap
- drbd_send_b_ack
- _drbd_send_ack
- drbd_send_ack_dp
- drbd_send_ack_rp
- drbd_send_ack
- drbd_send_ack_ex
- drbd_send_rs_deallocated
- drbd_send_drequest
- drbd_send_drequest_csum
- drbd_send_ov_request
- we_should_drop_the_connection
- drbd_update_congested
- _drbd_no_send_page
- _drbd_send_page
- _drbd_send_bio
- _drbd_send_zc_bio
- _drbd_send_zc_ee
- bio_flags_to_wire
- drbd_send_dblock
- drbd_send_block
- drbd_send_out_of_sync
- drbd_send
- drbd_send_all
- drbd_open
- drbd_release
- drbd_queue_unplug
- drbd_set_defaults
- drbd_init_set_defaults
- _drbd_set_my_capacity
- drbd_set_my_capacity
- drbd_device_cleanup
- drbd_destroy_mempools
- drbd_create_mempools
- drbd_release_all_peer_reqs
- drbd_destroy_device
- do_retry
- drbd_restart_request
- drbd_destroy_resource
- drbd_free_resource
- drbd_cleanup
- drbd_congested
- drbd_init_workqueue
- w_complete
- drbd_flush_workqueue
- drbd_find_resource
- conn_get_by_addrs
- drbd_alloc_socket
- drbd_free_socket
- conn_free_crypto
- set_resource_options
- drbd_create_resource
- conn_create
- drbd_destroy_connection
- init_submitter
- drbd_create_device
- drbd_delete_device
- drbd_init
- drbd_free_one_sock
- drbd_free_sock
- conn_md_sync
- drbd_md_write
- drbd_md_sync
- check_activity_log_stripe_size
- check_offsets_and_sizes
- drbd_md_read
- drbd_md_mark_dirty_
- drbd_md_mark_dirty
- drbd_uuid_move_history
- __drbd_uuid_set
- _drbd_uuid_set
- drbd_uuid_set
- drbd_uuid_new_current
- drbd_uuid_set_bm
- drbd_bmio_set_n_write
- drbd_bmio_clear_n_write
- w_bitmap_io
- drbd_queue_bitmap_io
- drbd_bitmap_io
- drbd_md_set_flag
- drbd_md_clear_flag
- drbd_md_test_flag
- md_sync_timer_fn
- cmdname
- drbd_wait_misc
- lock_all_resources
- unlock_all_resources
- _drbd_fault_random
- _drbd_fault_str
- _drbd_insert_fault
- drbd_buildtag
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
19 #include <linux/module.h>
20 #include <linux/jiffies.h>
21 #include <linux/drbd.h>
22 #include <linux/uaccess.h>
23 #include <asm/types.h>
24 #include <net/sock.h>
25 #include <linux/ctype.h>
26 #include <linux/mutex.h>
27 #include <linux/fs.h>
28 #include <linux/file.h>
29 #include <linux/proc_fs.h>
30 #include <linux/init.h>
31 #include <linux/mm.h>
32 #include <linux/memcontrol.h>
33 #include <linux/mm_inline.h>
34 #include <linux/slab.h>
35 #include <linux/random.h>
36 #include <linux/reboot.h>
37 #include <linux/notifier.h>
38 #include <linux/kthread.h>
39 #include <linux/workqueue.h>
40 #define __KERNEL_SYSCALLS__
41 #include <linux/unistd.h>
42 #include <linux/vmalloc.h>
43 #include <linux/sched/signal.h>
44
45 #include <linux/drbd_limits.h>
46 #include "drbd_int.h"
47 #include "drbd_protocol.h"
48 #include "drbd_req.h"
49 #include "drbd_vli.h"
50 #include "drbd_debugfs.h"
51
52 static DEFINE_MUTEX(drbd_main_mutex);
53 static int drbd_open(struct block_device *bdev, fmode_t mode);
54 static void drbd_release(struct gendisk *gd, fmode_t mode);
55 static void md_sync_timer_fn(struct timer_list *t);
56 static int w_bitmap_io(struct drbd_work *w, int unused);
57
58 MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
59 "Lars Ellenberg <lars@linbit.com>");
60 MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
61 MODULE_VERSION(REL_VERSION);
62 MODULE_LICENSE("GPL");
63 MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
64 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
65 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);
66
67 #include <linux/moduleparam.h>
68
69
70
71 #ifdef CONFIG_DRBD_FAULT_INJECTION
72 int drbd_enable_faults;
73 int drbd_fault_rate;
74 static int drbd_fault_count;
75 static int drbd_fault_devs;
76
77 module_param_named(enable_faults, drbd_enable_faults, int, 0664);
78
79 module_param_named(fault_rate, drbd_fault_rate, int, 0664);
80
81 module_param_named(fault_count, drbd_fault_count, int, 0664);
82
83 module_param_named(fault_devs, drbd_fault_devs, int, 0644);
84 #endif
85
86
87 static bool drbd_allow_oos;
88 static bool drbd_disable_sendpage;
89 MODULE_PARM_DESC(allow_oos, "DONT USE!");
90 module_param_named(allow_oos, drbd_allow_oos, bool, 0);
91 module_param_named(disable_sendpage, drbd_disable_sendpage, bool, 0644);
92
93
94 int drbd_proc_details;
95 module_param_named(proc_details, drbd_proc_details, int, 0644);
96
97 unsigned int drbd_minor_count = DRBD_MINOR_COUNT_DEF;
98
99
100 char drbd_usermode_helper[80] = "/sbin/drbdadm";
101 module_param_named(minor_count, drbd_minor_count, uint, 0444);
102 module_param_string(usermode_helper, drbd_usermode_helper, sizeof(drbd_usermode_helper), 0644);
103
104
105
106
107 struct idr drbd_devices;
108 struct list_head drbd_resources;
109 struct mutex resources_mutex;
110
111 struct kmem_cache *drbd_request_cache;
112 struct kmem_cache *drbd_ee_cache;
113 struct kmem_cache *drbd_bm_ext_cache;
114 struct kmem_cache *drbd_al_ext_cache;
115 mempool_t drbd_request_mempool;
116 mempool_t drbd_ee_mempool;
117 mempool_t drbd_md_io_page_pool;
118 struct bio_set drbd_md_io_bio_set;
119 struct bio_set drbd_io_bio_set;
120
121
122
123
124
125
126
127 struct page *drbd_pp_pool;
128 spinlock_t drbd_pp_lock;
129 int drbd_pp_vacant;
130 wait_queue_head_t drbd_pp_wait;
131
132 DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);
133
134 static const struct block_device_operations drbd_ops = {
135 .owner = THIS_MODULE,
136 .open = drbd_open,
137 .release = drbd_release,
138 };
139
140 struct bio *bio_alloc_drbd(gfp_t gfp_mask)
141 {
142 struct bio *bio;
143
144 if (!bioset_initialized(&drbd_md_io_bio_set))
145 return bio_alloc(gfp_mask, 1);
146
147 bio = bio_alloc_bioset(gfp_mask, 1, &drbd_md_io_bio_set);
148 if (!bio)
149 return NULL;
150 return bio;
151 }
152
153 #ifdef __CHECKER__
154
155
156
157 int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
158 {
159 int io_allowed;
160
161 atomic_inc(&device->local_cnt);
162 io_allowed = (device->state.disk >= mins);
163 if (!io_allowed) {
164 if (atomic_dec_and_test(&device->local_cnt))
165 wake_up(&device->misc_wait);
166 }
167 return io_allowed;
168 }
169
170 #endif
171
172
173
174
175
176
177
178
179
180
181
182 void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
183 unsigned int set_size)
184 {
185 struct drbd_request *r;
186 struct drbd_request *req = NULL;
187 int expect_epoch = 0;
188 int expect_size = 0;
189
190 spin_lock_irq(&connection->resource->req_lock);
191
192
193
194 list_for_each_entry(r, &connection->transfer_log, tl_requests) {
195 const unsigned s = r->rq_state;
196 if (!req) {
197 if (!(s & RQ_WRITE))
198 continue;
199 if (!(s & RQ_NET_MASK))
200 continue;
201 if (s & RQ_NET_DONE)
202 continue;
203 req = r;
204 expect_epoch = req->epoch;
205 expect_size ++;
206 } else {
207 if (r->epoch != expect_epoch)
208 break;
209 if (!(s & RQ_WRITE))
210 continue;
211
212
213 expect_size++;
214 }
215 }
216
217
218 if (req == NULL) {
219 drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
220 barrier_nr);
221 goto bail;
222 }
223 if (expect_epoch != barrier_nr) {
224 drbd_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
225 barrier_nr, expect_epoch);
226 goto bail;
227 }
228
229 if (expect_size != set_size) {
230 drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
231 barrier_nr, set_size, expect_size);
232 goto bail;
233 }
234
235
236
237
238
239 list_for_each_entry(req, &connection->transfer_log, tl_requests)
240 if (req->epoch == expect_epoch)
241 break;
242 list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
243 if (req->epoch != expect_epoch)
244 break;
245 _req_mod(req, BARRIER_ACKED);
246 }
247 spin_unlock_irq(&connection->resource->req_lock);
248
249 return;
250
251 bail:
252 spin_unlock_irq(&connection->resource->req_lock);
253 conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
254 }
255
256
257
258
259
260
261
262
263
264
265
266 void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
267 {
268 struct drbd_request *req, *r;
269
270 list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
271 _req_mod(req, what);
272 }
273
274 void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
275 {
276 spin_lock_irq(&connection->resource->req_lock);
277 _tl_restart(connection, what);
278 spin_unlock_irq(&connection->resource->req_lock);
279 }
280
281
282
283
284
285
286
287
288
289 void tl_clear(struct drbd_connection *connection)
290 {
291 tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
292 }
293
294
295
296
297
298 void tl_abort_disk_io(struct drbd_device *device)
299 {
300 struct drbd_connection *connection = first_peer_device(device)->connection;
301 struct drbd_request *req, *r;
302
303 spin_lock_irq(&connection->resource->req_lock);
304 list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
305 if (!(req->rq_state & RQ_LOCAL_PENDING))
306 continue;
307 if (req->device != device)
308 continue;
309 _req_mod(req, ABORT_DISK_IO);
310 }
311 spin_unlock_irq(&connection->resource->req_lock);
312 }
313
314 static int drbd_thread_setup(void *arg)
315 {
316 struct drbd_thread *thi = (struct drbd_thread *) arg;
317 struct drbd_resource *resource = thi->resource;
318 unsigned long flags;
319 int retval;
320
321 snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
322 thi->name[0],
323 resource->name);
324
325 allow_kernel_signal(DRBD_SIGKILL);
326 allow_kernel_signal(SIGXCPU);
327 restart:
328 retval = thi->function(thi);
329
330 spin_lock_irqsave(&thi->t_lock, flags);
331
332
333
334
335
336
337
338
339
340
341
342 if (thi->t_state == RESTARTING) {
343 drbd_info(resource, "Restarting %s thread\n", thi->name);
344 thi->t_state = RUNNING;
345 spin_unlock_irqrestore(&thi->t_lock, flags);
346 goto restart;
347 }
348
349 thi->task = NULL;
350 thi->t_state = NONE;
351 smp_mb();
352 complete_all(&thi->stop);
353 spin_unlock_irqrestore(&thi->t_lock, flags);
354
355 drbd_info(resource, "Terminating %s\n", current->comm);
356
357
358
359 if (thi->connection)
360 kref_put(&thi->connection->kref, drbd_destroy_connection);
361 kref_put(&resource->kref, drbd_destroy_resource);
362 module_put(THIS_MODULE);
363 return retval;
364 }
365
366 static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread *thi,
367 int (*func) (struct drbd_thread *), const char *name)
368 {
369 spin_lock_init(&thi->t_lock);
370 thi->task = NULL;
371 thi->t_state = NONE;
372 thi->function = func;
373 thi->resource = resource;
374 thi->connection = NULL;
375 thi->name = name;
376 }
377
378 int drbd_thread_start(struct drbd_thread *thi)
379 {
380 struct drbd_resource *resource = thi->resource;
381 struct task_struct *nt;
382 unsigned long flags;
383
384
385
386 spin_lock_irqsave(&thi->t_lock, flags);
387
388 switch (thi->t_state) {
389 case NONE:
390 drbd_info(resource, "Starting %s thread (from %s [%d])\n",
391 thi->name, current->comm, current->pid);
392
393
394 if (!try_module_get(THIS_MODULE)) {
395 drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
396 spin_unlock_irqrestore(&thi->t_lock, flags);
397 return false;
398 }
399
400 kref_get(&resource->kref);
401 if (thi->connection)
402 kref_get(&thi->connection->kref);
403
404 init_completion(&thi->stop);
405 thi->reset_cpu_mask = 1;
406 thi->t_state = RUNNING;
407 spin_unlock_irqrestore(&thi->t_lock, flags);
408 flush_signals(current);
409
410 nt = kthread_create(drbd_thread_setup, (void *) thi,
411 "drbd_%c_%s", thi->name[0], thi->resource->name);
412
413 if (IS_ERR(nt)) {
414 drbd_err(resource, "Couldn't start thread\n");
415
416 if (thi->connection)
417 kref_put(&thi->connection->kref, drbd_destroy_connection);
418 kref_put(&resource->kref, drbd_destroy_resource);
419 module_put(THIS_MODULE);
420 return false;
421 }
422 spin_lock_irqsave(&thi->t_lock, flags);
423 thi->task = nt;
424 thi->t_state = RUNNING;
425 spin_unlock_irqrestore(&thi->t_lock, flags);
426 wake_up_process(nt);
427 break;
428 case EXITING:
429 thi->t_state = RESTARTING;
430 drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
431 thi->name, current->comm, current->pid);
432
433 case RUNNING:
434 case RESTARTING:
435 default:
436 spin_unlock_irqrestore(&thi->t_lock, flags);
437 break;
438 }
439
440 return true;
441 }
442
443
444 void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
445 {
446 unsigned long flags;
447
448 enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
449
450
451 spin_lock_irqsave(&thi->t_lock, flags);
452
453 if (thi->t_state == NONE) {
454 spin_unlock_irqrestore(&thi->t_lock, flags);
455 if (restart)
456 drbd_thread_start(thi);
457 return;
458 }
459
460 if (thi->t_state != ns) {
461 if (thi->task == NULL) {
462 spin_unlock_irqrestore(&thi->t_lock, flags);
463 return;
464 }
465
466 thi->t_state = ns;
467 smp_mb();
468 init_completion(&thi->stop);
469 if (thi->task != current)
470 send_sig(DRBD_SIGKILL, thi->task, 1);
471 }
472
473 spin_unlock_irqrestore(&thi->t_lock, flags);
474
475 if (wait)
476 wait_for_completion(&thi->stop);
477 }
478
479 int conn_lowest_minor(struct drbd_connection *connection)
480 {
481 struct drbd_peer_device *peer_device;
482 int vnr = 0, minor = -1;
483
484 rcu_read_lock();
485 peer_device = idr_get_next(&connection->peer_devices, &vnr);
486 if (peer_device)
487 minor = device_to_minor(peer_device->device);
488 rcu_read_unlock();
489
490 return minor;
491 }
492
493 #ifdef CONFIG_SMP
494
495
496
497
498
499
500 static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
501 {
502 unsigned int *resources_per_cpu, min_index = ~0;
503
504 resources_per_cpu = kcalloc(nr_cpu_ids, sizeof(*resources_per_cpu),
505 GFP_KERNEL);
506 if (resources_per_cpu) {
507 struct drbd_resource *resource;
508 unsigned int cpu, min = ~0;
509
510 rcu_read_lock();
511 for_each_resource_rcu(resource, &drbd_resources) {
512 for_each_cpu(cpu, resource->cpu_mask)
513 resources_per_cpu[cpu]++;
514 }
515 rcu_read_unlock();
516 for_each_online_cpu(cpu) {
517 if (resources_per_cpu[cpu] < min) {
518 min = resources_per_cpu[cpu];
519 min_index = cpu;
520 }
521 }
522 kfree(resources_per_cpu);
523 }
524 if (min_index == ~0) {
525 cpumask_setall(*cpu_mask);
526 return;
527 }
528 cpumask_set_cpu(min_index, *cpu_mask);
529 }
530
531
532
533
534
535
536
537
538
539 void drbd_thread_current_set_cpu(struct drbd_thread *thi)
540 {
541 struct drbd_resource *resource = thi->resource;
542 struct task_struct *p = current;
543
544 if (!thi->reset_cpu_mask)
545 return;
546 thi->reset_cpu_mask = 0;
547 set_cpus_allowed_ptr(p, resource->cpu_mask);
548 }
549 #else
550 #define drbd_calc_cpu_mask(A) ({})
551 #endif
552
553
554
555
556
557
558
559
560 unsigned int drbd_header_size(struct drbd_connection *connection)
561 {
562 if (connection->agreed_pro_version >= 100) {
563 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
564 return sizeof(struct p_header100);
565 } else {
566 BUILD_BUG_ON(sizeof(struct p_header80) !=
567 sizeof(struct p_header95));
568 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
569 return sizeof(struct p_header80);
570 }
571 }
572
573 static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
574 {
575 h->magic = cpu_to_be32(DRBD_MAGIC);
576 h->command = cpu_to_be16(cmd);
577 h->length = cpu_to_be16(size);
578 return sizeof(struct p_header80);
579 }
580
581 static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
582 {
583 h->magic = cpu_to_be16(DRBD_MAGIC_BIG);
584 h->command = cpu_to_be16(cmd);
585 h->length = cpu_to_be32(size);
586 return sizeof(struct p_header95);
587 }
588
589 static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
590 int size, int vnr)
591 {
592 h->magic = cpu_to_be32(DRBD_MAGIC_100);
593 h->volume = cpu_to_be16(vnr);
594 h->command = cpu_to_be16(cmd);
595 h->length = cpu_to_be32(size);
596 h->pad = 0;
597 return sizeof(struct p_header100);
598 }
599
600 static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
601 void *buffer, enum drbd_packet cmd, int size)
602 {
603 if (connection->agreed_pro_version >= 100)
604 return prepare_header100(buffer, cmd, size, vnr);
605 else if (connection->agreed_pro_version >= 95 &&
606 size > DRBD_MAX_SIZE_H80_PACKET)
607 return prepare_header95(buffer, cmd, size);
608 else
609 return prepare_header80(buffer, cmd, size);
610 }
611
612 static void *__conn_prepare_command(struct drbd_connection *connection,
613 struct drbd_socket *sock)
614 {
615 if (!sock->socket)
616 return NULL;
617 return sock->sbuf + drbd_header_size(connection);
618 }
619
620 void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
621 {
622 void *p;
623
624 mutex_lock(&sock->mutex);
625 p = __conn_prepare_command(connection, sock);
626 if (!p)
627 mutex_unlock(&sock->mutex);
628
629 return p;
630 }
631
632 void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
633 {
634 return conn_prepare_command(peer_device->connection, sock);
635 }
636
637 static int __send_command(struct drbd_connection *connection, int vnr,
638 struct drbd_socket *sock, enum drbd_packet cmd,
639 unsigned int header_size, void *data,
640 unsigned int size)
641 {
642 int msg_flags;
643 int err;
644
645
646
647
648
649
650
651
652 msg_flags = data ? MSG_MORE : 0;
653
654 header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
655 header_size + size);
656 err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
657 msg_flags);
658 if (data && !err)
659 err = drbd_send_all(connection, sock->socket, data, size, 0);
660
661
662 if (!err && (cmd == P_PING || cmd == P_PING_ACK))
663 drbd_tcp_nodelay(sock->socket);
664
665 return err;
666 }
667
668 static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
669 enum drbd_packet cmd, unsigned int header_size,
670 void *data, unsigned int size)
671 {
672 return __send_command(connection, 0, sock, cmd, header_size, data, size);
673 }
674
675 int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
676 enum drbd_packet cmd, unsigned int header_size,
677 void *data, unsigned int size)
678 {
679 int err;
680
681 err = __conn_send_command(connection, sock, cmd, header_size, data, size);
682 mutex_unlock(&sock->mutex);
683 return err;
684 }
685
686 int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
687 enum drbd_packet cmd, unsigned int header_size,
688 void *data, unsigned int size)
689 {
690 int err;
691
692 err = __send_command(peer_device->connection, peer_device->device->vnr,
693 sock, cmd, header_size, data, size);
694 mutex_unlock(&sock->mutex);
695 return err;
696 }
697
698 int drbd_send_ping(struct drbd_connection *connection)
699 {
700 struct drbd_socket *sock;
701
702 sock = &connection->meta;
703 if (!conn_prepare_command(connection, sock))
704 return -EIO;
705 return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
706 }
707
708 int drbd_send_ping_ack(struct drbd_connection *connection)
709 {
710 struct drbd_socket *sock;
711
712 sock = &connection->meta;
713 if (!conn_prepare_command(connection, sock))
714 return -EIO;
715 return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
716 }
717
718 int drbd_send_sync_param(struct drbd_peer_device *peer_device)
719 {
720 struct drbd_socket *sock;
721 struct p_rs_param_95 *p;
722 int size;
723 const int apv = peer_device->connection->agreed_pro_version;
724 enum drbd_packet cmd;
725 struct net_conf *nc;
726 struct disk_conf *dc;
727
728 sock = &peer_device->connection->data;
729 p = drbd_prepare_command(peer_device, sock);
730 if (!p)
731 return -EIO;
732
733 rcu_read_lock();
734 nc = rcu_dereference(peer_device->connection->net_conf);
735
736 size = apv <= 87 ? sizeof(struct p_rs_param)
737 : apv == 88 ? sizeof(struct p_rs_param)
738 + strlen(nc->verify_alg) + 1
739 : apv <= 94 ? sizeof(struct p_rs_param_89)
740 : sizeof(struct p_rs_param_95);
741
742 cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
743
744
745 memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
746
747 if (get_ldev(peer_device->device)) {
748 dc = rcu_dereference(peer_device->device->ldev->disk_conf);
749 p->resync_rate = cpu_to_be32(dc->resync_rate);
750 p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
751 p->c_delay_target = cpu_to_be32(dc->c_delay_target);
752 p->c_fill_target = cpu_to_be32(dc->c_fill_target);
753 p->c_max_rate = cpu_to_be32(dc->c_max_rate);
754 put_ldev(peer_device->device);
755 } else {
756 p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
757 p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
758 p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
759 p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
760 p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
761 }
762
763 if (apv >= 88)
764 strcpy(p->verify_alg, nc->verify_alg);
765 if (apv >= 89)
766 strcpy(p->csums_alg, nc->csums_alg);
767 rcu_read_unlock();
768
769 return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
770 }
771
772 int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
773 {
774 struct drbd_socket *sock;
775 struct p_protocol *p;
776 struct net_conf *nc;
777 int size, cf;
778
779 sock = &connection->data;
780 p = __conn_prepare_command(connection, sock);
781 if (!p)
782 return -EIO;
783
784 rcu_read_lock();
785 nc = rcu_dereference(connection->net_conf);
786
787 if (nc->tentative && connection->agreed_pro_version < 92) {
788 rcu_read_unlock();
789 drbd_err(connection, "--dry-run is not supported by peer");
790 return -EOPNOTSUPP;
791 }
792
793 size = sizeof(*p);
794 if (connection->agreed_pro_version >= 87)
795 size += strlen(nc->integrity_alg) + 1;
796
797 p->protocol = cpu_to_be32(nc->wire_protocol);
798 p->after_sb_0p = cpu_to_be32(nc->after_sb_0p);
799 p->after_sb_1p = cpu_to_be32(nc->after_sb_1p);
800 p->after_sb_2p = cpu_to_be32(nc->after_sb_2p);
801 p->two_primaries = cpu_to_be32(nc->two_primaries);
802 cf = 0;
803 if (nc->discard_my_data)
804 cf |= CF_DISCARD_MY_DATA;
805 if (nc->tentative)
806 cf |= CF_DRY_RUN;
807 p->conn_flags = cpu_to_be32(cf);
808
809 if (connection->agreed_pro_version >= 87)
810 strcpy(p->integrity_alg, nc->integrity_alg);
811 rcu_read_unlock();
812
813 return __conn_send_command(connection, sock, cmd, size, NULL, 0);
814 }
815
816 int drbd_send_protocol(struct drbd_connection *connection)
817 {
818 int err;
819
820 mutex_lock(&connection->data.mutex);
821 err = __drbd_send_protocol(connection, P_PROTOCOL);
822 mutex_unlock(&connection->data.mutex);
823
824 return err;
825 }
826
827 static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
828 {
829 struct drbd_device *device = peer_device->device;
830 struct drbd_socket *sock;
831 struct p_uuids *p;
832 int i;
833
834 if (!get_ldev_if_state(device, D_NEGOTIATING))
835 return 0;
836
837 sock = &peer_device->connection->data;
838 p = drbd_prepare_command(peer_device, sock);
839 if (!p) {
840 put_ldev(device);
841 return -EIO;
842 }
843 spin_lock_irq(&device->ldev->md.uuid_lock);
844 for (i = UI_CURRENT; i < UI_SIZE; i++)
845 p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
846 spin_unlock_irq(&device->ldev->md.uuid_lock);
847
848 device->comm_bm_set = drbd_bm_total_weight(device);
849 p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
850 rcu_read_lock();
851 uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
852 rcu_read_unlock();
853 uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
854 uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
855 p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
856
857 put_ldev(device);
858 return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
859 }
860
861 int drbd_send_uuids(struct drbd_peer_device *peer_device)
862 {
863 return _drbd_send_uuids(peer_device, 0);
864 }
865
866 int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
867 {
868 return _drbd_send_uuids(peer_device, 8);
869 }
870
871 void drbd_print_uuids(struct drbd_device *device, const char *text)
872 {
873 if (get_ldev_if_state(device, D_NEGOTIATING)) {
874 u64 *uuid = device->ldev->md.uuid;
875 drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
876 text,
877 (unsigned long long)uuid[UI_CURRENT],
878 (unsigned long long)uuid[UI_BITMAP],
879 (unsigned long long)uuid[UI_HISTORY_START],
880 (unsigned long long)uuid[UI_HISTORY_END]);
881 put_ldev(device);
882 } else {
883 drbd_info(device, "%s effective data uuid: %016llX\n",
884 text,
885 (unsigned long long)device->ed_uuid);
886 }
887 }
888
889 void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
890 {
891 struct drbd_device *device = peer_device->device;
892 struct drbd_socket *sock;
893 struct p_rs_uuid *p;
894 u64 uuid;
895
896 D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
897
898 uuid = device->ldev->md.uuid[UI_BITMAP];
899 if (uuid && uuid != UUID_JUST_CREATED)
900 uuid = uuid + UUID_NEW_BM_OFFSET;
901 else
902 get_random_bytes(&uuid, sizeof(u64));
903 drbd_uuid_set(device, UI_BITMAP, uuid);
904 drbd_print_uuids(device, "updated sync UUID");
905 drbd_md_sync(device);
906
907 sock = &peer_device->connection->data;
908 p = drbd_prepare_command(peer_device, sock);
909 if (p) {
910 p->uuid = cpu_to_be64(uuid);
911 drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
912 }
913 }
914
915
916 static void
917 assign_p_sizes_qlim(struct drbd_device *device, struct p_sizes *p,
918 struct request_queue *q)
919 {
920 if (q) {
921 p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q));
922 p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q));
923 p->qlim->alignment_offset = cpu_to_be32(queue_alignment_offset(q));
924 p->qlim->io_min = cpu_to_be32(queue_io_min(q));
925 p->qlim->io_opt = cpu_to_be32(queue_io_opt(q));
926 p->qlim->discard_enabled = blk_queue_discard(q);
927 p->qlim->write_same_capable = !!q->limits.max_write_same_sectors;
928 } else {
929 q = device->rq_queue;
930 p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q));
931 p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q));
932 p->qlim->alignment_offset = 0;
933 p->qlim->io_min = cpu_to_be32(queue_io_min(q));
934 p->qlim->io_opt = cpu_to_be32(queue_io_opt(q));
935 p->qlim->discard_enabled = 0;
936 p->qlim->write_same_capable = 0;
937 }
938 }
939
940 int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
941 {
942 struct drbd_device *device = peer_device->device;
943 struct drbd_socket *sock;
944 struct p_sizes *p;
945 sector_t d_size, u_size;
946 int q_order_type;
947 unsigned int max_bio_size;
948 unsigned int packet_size;
949
950 sock = &peer_device->connection->data;
951 p = drbd_prepare_command(peer_device, sock);
952 if (!p)
953 return -EIO;
954
955 packet_size = sizeof(*p);
956 if (peer_device->connection->agreed_features & DRBD_FF_WSAME)
957 packet_size += sizeof(p->qlim[0]);
958
959 memset(p, 0, packet_size);
960 if (get_ldev_if_state(device, D_NEGOTIATING)) {
961 struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev);
962 d_size = drbd_get_max_capacity(device->ldev);
963 rcu_read_lock();
964 u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
965 rcu_read_unlock();
966 q_order_type = drbd_queue_order_type(device);
967 max_bio_size = queue_max_hw_sectors(q) << 9;
968 max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
969 assign_p_sizes_qlim(device, p, q);
970 put_ldev(device);
971 } else {
972 d_size = 0;
973 u_size = 0;
974 q_order_type = QUEUE_ORDERED_NONE;
975 max_bio_size = DRBD_MAX_BIO_SIZE;
976 assign_p_sizes_qlim(device, p, NULL);
977 }
978
979 if (peer_device->connection->agreed_pro_version <= 94)
980 max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
981 else if (peer_device->connection->agreed_pro_version < 100)
982 max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
983
984 p->d_size = cpu_to_be64(d_size);
985 p->u_size = cpu_to_be64(u_size);
986 p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
987 p->max_bio_size = cpu_to_be32(max_bio_size);
988 p->queue_order_type = cpu_to_be16(q_order_type);
989 p->dds_flags = cpu_to_be16(flags);
990
991 return drbd_send_command(peer_device, sock, P_SIZES, packet_size, NULL, 0);
992 }
993
994
995
996
997
998 int drbd_send_current_state(struct drbd_peer_device *peer_device)
999 {
1000 struct drbd_socket *sock;
1001 struct p_state *p;
1002
1003 sock = &peer_device->connection->data;
1004 p = drbd_prepare_command(peer_device, sock);
1005 if (!p)
1006 return -EIO;
1007 p->state = cpu_to_be32(peer_device->device->state.i);
1008 return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
1009 }
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021 int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
1022 {
1023 struct drbd_socket *sock;
1024 struct p_state *p;
1025
1026 sock = &peer_device->connection->data;
1027 p = drbd_prepare_command(peer_device, sock);
1028 if (!p)
1029 return -EIO;
1030 p->state = cpu_to_be32(state.i);
1031 return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
1032 }
1033
1034 int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
1035 {
1036 struct drbd_socket *sock;
1037 struct p_req_state *p;
1038
1039 sock = &peer_device->connection->data;
1040 p = drbd_prepare_command(peer_device, sock);
1041 if (!p)
1042 return -EIO;
1043 p->mask = cpu_to_be32(mask.i);
1044 p->val = cpu_to_be32(val.i);
1045 return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
1046 }
1047
1048 int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
1049 {
1050 enum drbd_packet cmd;
1051 struct drbd_socket *sock;
1052 struct p_req_state *p;
1053
1054 cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
1055 sock = &connection->data;
1056 p = conn_prepare_command(connection, sock);
1057 if (!p)
1058 return -EIO;
1059 p->mask = cpu_to_be32(mask.i);
1060 p->val = cpu_to_be32(val.i);
1061 return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1062 }
1063
1064 void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
1065 {
1066 struct drbd_socket *sock;
1067 struct p_req_state_reply *p;
1068
1069 sock = &peer_device->connection->meta;
1070 p = drbd_prepare_command(peer_device, sock);
1071 if (p) {
1072 p->retcode = cpu_to_be32(retcode);
1073 drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1074 }
1075 }
1076
1077 void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1078 {
1079 struct drbd_socket *sock;
1080 struct p_req_state_reply *p;
1081 enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
1082
1083 sock = &connection->meta;
1084 p = conn_prepare_command(connection, sock);
1085 if (p) {
1086 p->retcode = cpu_to_be32(retcode);
1087 conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1088 }
1089 }
1090
1091 static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
1092 {
1093 BUG_ON(code & ~0xf);
1094 p->encoding = (p->encoding & ~0xf) | code;
1095 }
1096
1097 static void dcbp_set_start(struct p_compressed_bm *p, int set)
1098 {
1099 p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
1100 }
1101
1102 static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
1103 {
1104 BUG_ON(n & ~0x7);
1105 p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
1106 }
1107
1108 static int fill_bitmap_rle_bits(struct drbd_device *device,
1109 struct p_compressed_bm *p,
1110 unsigned int size,
1111 struct bm_xfer_ctx *c)
1112 {
1113 struct bitstream bs;
1114 unsigned long plain_bits;
1115 unsigned long tmp;
1116 unsigned long rl;
1117 unsigned len;
1118 unsigned toggle;
1119 int bits, use_rle;
1120
1121
1122 rcu_read_lock();
1123 use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1124 rcu_read_unlock();
1125 if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1126 return 0;
1127
1128 if (c->bit_offset >= c->bm_bits)
1129 return 0;
1130
1131
1132 bitstream_init(&bs, p->code, size, 0);
1133 memset(p->code, 0, size);
1134
1135 plain_bits = 0;
1136
1137
1138
1139
1140 toggle = 2;
1141
1142
1143
1144 do {
1145 tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
1146 : _drbd_bm_find_next(device, c->bit_offset);
1147 if (tmp == -1UL)
1148 tmp = c->bm_bits;
1149 rl = tmp - c->bit_offset;
1150
1151 if (toggle == 2) {
1152 if (rl == 0) {
1153
1154
1155 dcbp_set_start(p, 1);
1156
1157 toggle = !toggle;
1158 continue;
1159 }
1160 dcbp_set_start(p, 0);
1161 }
1162
1163
1164
1165 if (rl == 0) {
1166 drbd_err(device, "unexpected zero runlength while encoding bitmap "
1167 "t:%u bo:%lu\n", toggle, c->bit_offset);
1168 return -1;
1169 }
1170
1171 bits = vli_encode_bits(&bs, rl);
1172 if (bits == -ENOBUFS)
1173 break;
1174 if (bits <= 0) {
1175 drbd_err(device, "error while encoding bitmap: %d\n", bits);
1176 return 0;
1177 }
1178
1179 toggle = !toggle;
1180 plain_bits += rl;
1181 c->bit_offset = tmp;
1182 } while (c->bit_offset < c->bm_bits);
1183
1184 len = bs.cur.b - p->code + !!bs.cur.bit;
1185
1186 if (plain_bits < (len << 3)) {
1187
1188
1189 c->bit_offset -= plain_bits;
1190 bm_xfer_ctx_bit_to_word_offset(c);
1191 c->bit_offset = c->word_offset * BITS_PER_LONG;
1192 return 0;
1193 }
1194
1195
1196
1197 bm_xfer_ctx_bit_to_word_offset(c);
1198
1199
1200 dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
1201
1202 return len;
1203 }
1204
1205
1206
1207
1208
1209
1210
1211 static int
1212 send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
1213 {
1214 struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1215 unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1216 struct p_compressed_bm *p = sock->sbuf + header_size;
1217 int len, err;
1218
1219 len = fill_bitmap_rle_bits(device, p,
1220 DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
1221 if (len < 0)
1222 return -EIO;
1223
1224 if (len) {
1225 dcbp_set_code(p, RLE_VLI_Bits);
1226 err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1227 P_COMPRESSED_BITMAP, sizeof(*p) + len,
1228 NULL, 0);
1229 c->packets[0]++;
1230 c->bytes[0] += header_size + sizeof(*p) + len;
1231
1232 if (c->bit_offset >= c->bm_bits)
1233 len = 0;
1234 } else {
1235
1236
1237 unsigned int data_size;
1238 unsigned long num_words;
1239 unsigned long *p = sock->sbuf + header_size;
1240
1241 data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1242 num_words = min_t(size_t, data_size / sizeof(*p),
1243 c->bm_words - c->word_offset);
1244 len = num_words * sizeof(*p);
1245 if (len)
1246 drbd_bm_get_lel(device, c->word_offset, num_words, p);
1247 err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
1248 c->word_offset += num_words;
1249 c->bit_offset = c->word_offset * BITS_PER_LONG;
1250
1251 c->packets[1]++;
1252 c->bytes[1] += header_size + len;
1253
1254 if (c->bit_offset > c->bm_bits)
1255 c->bit_offset = c->bm_bits;
1256 }
1257 if (!err) {
1258 if (len == 0) {
1259 INFO_bm_xfer_stats(device, "send", c);
1260 return 0;
1261 } else
1262 return 1;
1263 }
1264 return -EIO;
1265 }
1266
1267
1268 static int _drbd_send_bitmap(struct drbd_device *device)
1269 {
1270 struct bm_xfer_ctx c;
1271 int err;
1272
1273 if (!expect(device->bitmap))
1274 return false;
1275
1276 if (get_ldev(device)) {
1277 if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
1278 drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
1279 drbd_bm_set_all(device);
1280 if (drbd_bm_write(device)) {
1281
1282
1283
1284 drbd_err(device, "Failed to write bitmap to disk!\n");
1285 } else {
1286 drbd_md_clear_flag(device, MDF_FULL_SYNC);
1287 drbd_md_sync(device);
1288 }
1289 }
1290 put_ldev(device);
1291 }
1292
1293 c = (struct bm_xfer_ctx) {
1294 .bm_bits = drbd_bm_bits(device),
1295 .bm_words = drbd_bm_words(device),
1296 };
1297
1298 do {
1299 err = send_bitmap_rle_or_plain(device, &c);
1300 } while (err > 0);
1301
1302 return err == 0;
1303 }
1304
1305 int drbd_send_bitmap(struct drbd_device *device)
1306 {
1307 struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1308 int err = -1;
1309
1310 mutex_lock(&sock->mutex);
1311 if (sock->socket)
1312 err = !_drbd_send_bitmap(device);
1313 mutex_unlock(&sock->mutex);
1314 return err;
1315 }
1316
1317 void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
1318 {
1319 struct drbd_socket *sock;
1320 struct p_barrier_ack *p;
1321
1322 if (connection->cstate < C_WF_REPORT_PARAMS)
1323 return;
1324
1325 sock = &connection->meta;
1326 p = conn_prepare_command(connection, sock);
1327 if (!p)
1328 return;
1329 p->barrier = barrier_nr;
1330 p->set_size = cpu_to_be32(set_size);
1331 conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
1332 }
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342 static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1343 u64 sector, u32 blksize, u64 block_id)
1344 {
1345 struct drbd_socket *sock;
1346 struct p_block_ack *p;
1347
1348 if (peer_device->device->state.conn < C_CONNECTED)
1349 return -EIO;
1350
1351 sock = &peer_device->connection->meta;
1352 p = drbd_prepare_command(peer_device, sock);
1353 if (!p)
1354 return -EIO;
1355 p->sector = sector;
1356 p->block_id = block_id;
1357 p->blksize = blksize;
1358 p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
1359 return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
1360 }
1361
1362
1363
1364
1365 void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1366 struct p_data *dp, int data_size)
1367 {
1368 if (peer_device->connection->peer_integrity_tfm)
1369 data_size -= crypto_shash_digestsize(peer_device->connection->peer_integrity_tfm);
1370 _drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
1371 dp->block_id);
1372 }
1373
1374 void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1375 struct p_block_req *rp)
1376 {
1377 _drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
1378 }
1379
1380
1381
1382
1383
1384
1385
1386 int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1387 struct drbd_peer_request *peer_req)
1388 {
1389 return _drbd_send_ack(peer_device, cmd,
1390 cpu_to_be64(peer_req->i.sector),
1391 cpu_to_be32(peer_req->i.size),
1392 peer_req->block_id);
1393 }
1394
1395
1396
1397 int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1398 sector_t sector, int blksize, u64 block_id)
1399 {
1400 return _drbd_send_ack(peer_device, cmd,
1401 cpu_to_be64(sector),
1402 cpu_to_be32(blksize),
1403 cpu_to_be64(block_id));
1404 }
1405
1406 int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device,
1407 struct drbd_peer_request *peer_req)
1408 {
1409 struct drbd_socket *sock;
1410 struct p_block_desc *p;
1411
1412 sock = &peer_device->connection->data;
1413 p = drbd_prepare_command(peer_device, sock);
1414 if (!p)
1415 return -EIO;
1416 p->sector = cpu_to_be64(peer_req->i.sector);
1417 p->blksize = cpu_to_be32(peer_req->i.size);
1418 p->pad = 0;
1419 return drbd_send_command(peer_device, sock, P_RS_DEALLOCATED, sizeof(*p), NULL, 0);
1420 }
1421
1422 int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
1423 sector_t sector, int size, u64 block_id)
1424 {
1425 struct drbd_socket *sock;
1426 struct p_block_req *p;
1427
1428 sock = &peer_device->connection->data;
1429 p = drbd_prepare_command(peer_device, sock);
1430 if (!p)
1431 return -EIO;
1432 p->sector = cpu_to_be64(sector);
1433 p->block_id = block_id;
1434 p->blksize = cpu_to_be32(size);
1435 return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
1436 }
1437
1438 int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
1439 void *digest, int digest_size, enum drbd_packet cmd)
1440 {
1441 struct drbd_socket *sock;
1442 struct p_block_req *p;
1443
1444
1445
1446 sock = &peer_device->connection->data;
1447 p = drbd_prepare_command(peer_device, sock);
1448 if (!p)
1449 return -EIO;
1450 p->sector = cpu_to_be64(sector);
1451 p->block_id = ID_SYNCER ;
1452 p->blksize = cpu_to_be32(size);
1453 return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
1454 }
1455
1456 int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
1457 {
1458 struct drbd_socket *sock;
1459 struct p_block_req *p;
1460
1461 sock = &peer_device->connection->data;
1462 p = drbd_prepare_command(peer_device, sock);
1463 if (!p)
1464 return -EIO;
1465 p->sector = cpu_to_be64(sector);
1466 p->block_id = ID_SYNCER ;
1467 p->blksize = cpu_to_be32(size);
1468 return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
1469 }
1470
1471
1472
1473
1474
1475 static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
1476 {
1477 int drop_it;
1478
1479
1480 drop_it = connection->meta.socket == sock
1481 || !connection->ack_receiver.task
1482 || get_t_state(&connection->ack_receiver) != RUNNING
1483 || connection->cstate < C_WF_REPORT_PARAMS;
1484
1485 if (drop_it)
1486 return true;
1487
1488 drop_it = !--connection->ko_count;
1489 if (!drop_it) {
1490 drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1491 current->comm, current->pid, connection->ko_count);
1492 request_ping(connection);
1493 }
1494
1495 return drop_it; ;
1496 }
1497
1498 static void drbd_update_congested(struct drbd_connection *connection)
1499 {
1500 struct sock *sk = connection->data.socket->sk;
1501 if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1502 set_bit(NET_CONGESTED, &connection->flags);
1503 }
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526 static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
1527 int offset, size_t size, unsigned msg_flags)
1528 {
1529 struct socket *socket;
1530 void *addr;
1531 int err;
1532
1533 socket = peer_device->connection->data.socket;
1534 addr = kmap(page) + offset;
1535 err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
1536 kunmap(page);
1537 if (!err)
1538 peer_device->device->send_cnt += size >> 9;
1539 return err;
1540 }
1541
1542 static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
1543 int offset, size_t size, unsigned msg_flags)
1544 {
1545 struct socket *socket = peer_device->connection->data.socket;
1546 int len = size;
1547 int err = -EIO;
1548
1549
1550
1551
1552
1553
1554
1555 if (drbd_disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1556 return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
1557
1558 msg_flags |= MSG_NOSIGNAL;
1559 drbd_update_congested(peer_device->connection);
1560 do {
1561 int sent;
1562
1563 sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
1564 if (sent <= 0) {
1565 if (sent == -EAGAIN) {
1566 if (we_should_drop_the_connection(peer_device->connection, socket))
1567 break;
1568 continue;
1569 }
1570 drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
1571 __func__, (int)size, len, sent);
1572 if (sent < 0)
1573 err = sent;
1574 break;
1575 }
1576 len -= sent;
1577 offset += sent;
1578 } while (len > 0 );
1579 clear_bit(NET_CONGESTED, &peer_device->connection->flags);
1580
1581 if (len == 0) {
1582 err = 0;
1583 peer_device->device->send_cnt += size >> 9;
1584 }
1585 return err;
1586 }
1587
1588 static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
1589 {
1590 struct bio_vec bvec;
1591 struct bvec_iter iter;
1592
1593
1594 bio_for_each_segment(bvec, bio, iter) {
1595 int err;
1596
1597 err = _drbd_no_send_page(peer_device, bvec.bv_page,
1598 bvec.bv_offset, bvec.bv_len,
1599 bio_iter_last(bvec, iter)
1600 ? 0 : MSG_MORE);
1601 if (err)
1602 return err;
1603
1604 if (bio_op(bio) == REQ_OP_WRITE_SAME)
1605 break;
1606 }
1607 return 0;
1608 }
1609
1610 static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *bio)
1611 {
1612 struct bio_vec bvec;
1613 struct bvec_iter iter;
1614
1615
1616 bio_for_each_segment(bvec, bio, iter) {
1617 int err;
1618
1619 err = _drbd_send_page(peer_device, bvec.bv_page,
1620 bvec.bv_offset, bvec.bv_len,
1621 bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1622 if (err)
1623 return err;
1624
1625 if (bio_op(bio) == REQ_OP_WRITE_SAME)
1626 break;
1627 }
1628 return 0;
1629 }
1630
1631 static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
1632 struct drbd_peer_request *peer_req)
1633 {
1634 struct page *page = peer_req->pages;
1635 unsigned len = peer_req->i.size;
1636 int err;
1637
1638
1639 page_chain_for_each(page) {
1640 unsigned l = min_t(unsigned, len, PAGE_SIZE);
1641
1642 err = _drbd_send_page(peer_device, page, 0, l,
1643 page_chain_next(page) ? MSG_MORE : 0);
1644 if (err)
1645 return err;
1646 len -= l;
1647 }
1648 return 0;
1649 }
1650
1651 static u32 bio_flags_to_wire(struct drbd_connection *connection,
1652 struct bio *bio)
1653 {
1654 if (connection->agreed_pro_version >= 95)
1655 return (bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0) |
1656 (bio->bi_opf & REQ_FUA ? DP_FUA : 0) |
1657 (bio->bi_opf & REQ_PREFLUSH ? DP_FLUSH : 0) |
1658 (bio_op(bio) == REQ_OP_WRITE_SAME ? DP_WSAME : 0) |
1659 (bio_op(bio) == REQ_OP_DISCARD ? DP_DISCARD : 0) |
1660 (bio_op(bio) == REQ_OP_WRITE_ZEROES ?
1661 ((connection->agreed_features & DRBD_FF_WZEROES) ?
1662 (DP_ZEROES |(!(bio->bi_opf & REQ_NOUNMAP) ? DP_DISCARD : 0))
1663 : DP_DISCARD)
1664 : 0);
1665 else
1666 return bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0;
1667 }
1668
1669
1670
1671
1672 int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
1673 {
1674 struct drbd_device *device = peer_device->device;
1675 struct drbd_socket *sock;
1676 struct p_data *p;
1677 struct p_wsame *wsame = NULL;
1678 void *digest_out;
1679 unsigned int dp_flags = 0;
1680 int digest_size;
1681 int err;
1682
1683 sock = &peer_device->connection->data;
1684 p = drbd_prepare_command(peer_device, sock);
1685 digest_size = peer_device->connection->integrity_tfm ?
1686 crypto_shash_digestsize(peer_device->connection->integrity_tfm) : 0;
1687
1688 if (!p)
1689 return -EIO;
1690 p->sector = cpu_to_be64(req->i.sector);
1691 p->block_id = (unsigned long)req;
1692 p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
1693 dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio);
1694 if (device->state.conn >= C_SYNC_SOURCE &&
1695 device->state.conn <= C_PAUSED_SYNC_T)
1696 dp_flags |= DP_MAY_SET_IN_SYNC;
1697 if (peer_device->connection->agreed_pro_version >= 100) {
1698 if (req->rq_state & RQ_EXP_RECEIVE_ACK)
1699 dp_flags |= DP_SEND_RECEIVE_ACK;
1700
1701
1702 if (req->rq_state & RQ_EXP_WRITE_ACK
1703 || (dp_flags & DP_MAY_SET_IN_SYNC))
1704 dp_flags |= DP_SEND_WRITE_ACK;
1705 }
1706 p->dp_flags = cpu_to_be32(dp_flags);
1707
1708 if (dp_flags & (DP_DISCARD|DP_ZEROES)) {
1709 enum drbd_packet cmd = (dp_flags & DP_ZEROES) ? P_ZEROES : P_TRIM;
1710 struct p_trim *t = (struct p_trim*)p;
1711 t->size = cpu_to_be32(req->i.size);
1712 err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*t), NULL, 0);
1713 goto out;
1714 }
1715 if (dp_flags & DP_WSAME) {
1716
1717
1718
1719 wsame = (struct p_wsame*)p;
1720 digest_out = wsame + 1;
1721 wsame->size = cpu_to_be32(req->i.size);
1722 } else
1723 digest_out = p + 1;
1724
1725
1726
1727 if (digest_size)
1728 drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest_out);
1729 if (wsame) {
1730 err =
1731 __send_command(peer_device->connection, device->vnr, sock, P_WSAME,
1732 sizeof(*wsame) + digest_size, NULL,
1733 bio_iovec(req->master_bio).bv_len);
1734 } else
1735 err =
1736 __send_command(peer_device->connection, device->vnr, sock, P_DATA,
1737 sizeof(*p) + digest_size, NULL, req->i.size);
1738 if (!err) {
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750 if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || digest_size)
1751 err = _drbd_send_bio(peer_device, req->master_bio);
1752 else
1753 err = _drbd_send_zc_bio(peer_device, req->master_bio);
1754
1755
1756 if (digest_size > 0 && digest_size <= 64) {
1757
1758
1759 unsigned char digest[64];
1760 drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
1761 if (memcmp(p + 1, digest, digest_size)) {
1762 drbd_warn(device,
1763 "Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1764 (unsigned long long)req->i.sector, req->i.size);
1765 }
1766 }
1767
1768
1769 }
1770 out:
1771 mutex_unlock(&sock->mutex);
1772
1773 return err;
1774 }
1775
1776
1777
1778
1779
1780 int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1781 struct drbd_peer_request *peer_req)
1782 {
1783 struct drbd_device *device = peer_device->device;
1784 struct drbd_socket *sock;
1785 struct p_data *p;
1786 int err;
1787 int digest_size;
1788
1789 sock = &peer_device->connection->data;
1790 p = drbd_prepare_command(peer_device, sock);
1791
1792 digest_size = peer_device->connection->integrity_tfm ?
1793 crypto_shash_digestsize(peer_device->connection->integrity_tfm) : 0;
1794
1795 if (!p)
1796 return -EIO;
1797 p->sector = cpu_to_be64(peer_req->i.sector);
1798 p->block_id = peer_req->block_id;
1799 p->seq_num = 0;
1800 p->dp_flags = 0;
1801 if (digest_size)
1802 drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
1803 err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + digest_size, NULL, peer_req->i.size);
1804 if (!err)
1805 err = _drbd_send_zc_ee(peer_device, peer_req);
1806 mutex_unlock(&sock->mutex);
1807
1808 return err;
1809 }
1810
1811 int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
1812 {
1813 struct drbd_socket *sock;
1814 struct p_block_desc *p;
1815
1816 sock = &peer_device->connection->data;
1817 p = drbd_prepare_command(peer_device, sock);
1818 if (!p)
1819 return -EIO;
1820 p->sector = cpu_to_be64(req->i.sector);
1821 p->blksize = cpu_to_be32(req->i.size);
1822 return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1823 }
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841 int drbd_send(struct drbd_connection *connection, struct socket *sock,
1842 void *buf, size_t size, unsigned msg_flags)
1843 {
1844 struct kvec iov = {.iov_base = buf, .iov_len = size};
1845 struct msghdr msg = {.msg_flags = msg_flags | MSG_NOSIGNAL};
1846 int rv, sent = 0;
1847
1848 if (!sock)
1849 return -EBADR;
1850
1851
1852
1853 iov_iter_kvec(&msg.msg_iter, WRITE, &iov, 1, size);
1854
1855 if (sock == connection->data.socket) {
1856 rcu_read_lock();
1857 connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1858 rcu_read_unlock();
1859 drbd_update_congested(connection);
1860 }
1861 do {
1862 rv = sock_sendmsg(sock, &msg);
1863 if (rv == -EAGAIN) {
1864 if (we_should_drop_the_connection(connection, sock))
1865 break;
1866 else
1867 continue;
1868 }
1869 if (rv == -EINTR) {
1870 flush_signals(current);
1871 rv = 0;
1872 }
1873 if (rv < 0)
1874 break;
1875 sent += rv;
1876 } while (sent < size);
1877
1878 if (sock == connection->data.socket)
1879 clear_bit(NET_CONGESTED, &connection->flags);
1880
1881 if (rv <= 0) {
1882 if (rv != -EAGAIN) {
1883 drbd_err(connection, "%s_sendmsg returned %d\n",
1884 sock == connection->meta.socket ? "msock" : "sock",
1885 rv);
1886 conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
1887 } else
1888 conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
1889 }
1890
1891 return sent;
1892 }
1893
1894
1895
1896
1897
1898
1899 int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1900 size_t size, unsigned msg_flags)
1901 {
1902 int err;
1903
1904 err = drbd_send(connection, sock, buffer, size, msg_flags);
1905 if (err < 0)
1906 return err;
1907 if (err != size)
1908 return -EIO;
1909 return 0;
1910 }
1911
1912 static int drbd_open(struct block_device *bdev, fmode_t mode)
1913 {
1914 struct drbd_device *device = bdev->bd_disk->private_data;
1915 unsigned long flags;
1916 int rv = 0;
1917
1918 mutex_lock(&drbd_main_mutex);
1919 spin_lock_irqsave(&device->resource->req_lock, flags);
1920
1921
1922
1923 if (device->state.role != R_PRIMARY) {
1924 if (mode & FMODE_WRITE)
1925 rv = -EROFS;
1926 else if (!drbd_allow_oos)
1927 rv = -EMEDIUMTYPE;
1928 }
1929
1930 if (!rv)
1931 device->open_cnt++;
1932 spin_unlock_irqrestore(&device->resource->req_lock, flags);
1933 mutex_unlock(&drbd_main_mutex);
1934
1935 return rv;
1936 }
1937
1938 static void drbd_release(struct gendisk *gd, fmode_t mode)
1939 {
1940 struct drbd_device *device = gd->private_data;
1941 mutex_lock(&drbd_main_mutex);
1942 device->open_cnt--;
1943 mutex_unlock(&drbd_main_mutex);
1944 }
1945
1946
1947 void drbd_queue_unplug(struct drbd_device *device)
1948 {
1949 if (device->state.pdsk >= D_INCONSISTENT && device->state.conn >= C_CONNECTED) {
1950 D_ASSERT(device, device->state.role == R_PRIMARY);
1951 if (test_and_clear_bit(UNPLUG_REMOTE, &device->flags)) {
1952 drbd_queue_work_if_unqueued(
1953 &first_peer_device(device)->connection->sender_work,
1954 &device->unplug_work);
1955 }
1956 }
1957 }
1958
1959 static void drbd_set_defaults(struct drbd_device *device)
1960 {
1961
1962
1963 device->state = (union drbd_dev_state) {
1964 { .role = R_SECONDARY,
1965 .peer = R_UNKNOWN,
1966 .conn = C_STANDALONE,
1967 .disk = D_DISKLESS,
1968 .pdsk = D_UNKNOWN,
1969 } };
1970 }
1971
1972 void drbd_init_set_defaults(struct drbd_device *device)
1973 {
1974
1975
1976
1977 drbd_set_defaults(device);
1978
1979 atomic_set(&device->ap_bio_cnt, 0);
1980 atomic_set(&device->ap_actlog_cnt, 0);
1981 atomic_set(&device->ap_pending_cnt, 0);
1982 atomic_set(&device->rs_pending_cnt, 0);
1983 atomic_set(&device->unacked_cnt, 0);
1984 atomic_set(&device->local_cnt, 0);
1985 atomic_set(&device->pp_in_use_by_net, 0);
1986 atomic_set(&device->rs_sect_in, 0);
1987 atomic_set(&device->rs_sect_ev, 0);
1988 atomic_set(&device->ap_in_flight, 0);
1989 atomic_set(&device->md_io.in_use, 0);
1990
1991 mutex_init(&device->own_state_mutex);
1992 device->state_mutex = &device->own_state_mutex;
1993
1994 spin_lock_init(&device->al_lock);
1995 spin_lock_init(&device->peer_seq_lock);
1996
1997 INIT_LIST_HEAD(&device->active_ee);
1998 INIT_LIST_HEAD(&device->sync_ee);
1999 INIT_LIST_HEAD(&device->done_ee);
2000 INIT_LIST_HEAD(&device->read_ee);
2001 INIT_LIST_HEAD(&device->net_ee);
2002 INIT_LIST_HEAD(&device->resync_reads);
2003 INIT_LIST_HEAD(&device->resync_work.list);
2004 INIT_LIST_HEAD(&device->unplug_work.list);
2005 INIT_LIST_HEAD(&device->bm_io_work.w.list);
2006 INIT_LIST_HEAD(&device->pending_master_completion[0]);
2007 INIT_LIST_HEAD(&device->pending_master_completion[1]);
2008 INIT_LIST_HEAD(&device->pending_completion[0]);
2009 INIT_LIST_HEAD(&device->pending_completion[1]);
2010
2011 device->resync_work.cb = w_resync_timer;
2012 device->unplug_work.cb = w_send_write_hint;
2013 device->bm_io_work.w.cb = w_bitmap_io;
2014
2015 timer_setup(&device->resync_timer, resync_timer_fn, 0);
2016 timer_setup(&device->md_sync_timer, md_sync_timer_fn, 0);
2017 timer_setup(&device->start_resync_timer, start_resync_timer_fn, 0);
2018 timer_setup(&device->request_timer, request_timer_fn, 0);
2019
2020 init_waitqueue_head(&device->misc_wait);
2021 init_waitqueue_head(&device->state_wait);
2022 init_waitqueue_head(&device->ee_wait);
2023 init_waitqueue_head(&device->al_wait);
2024 init_waitqueue_head(&device->seq_wait);
2025
2026 device->resync_wenr = LC_FREE;
2027 device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
2028 device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
2029 }
2030
2031 static void _drbd_set_my_capacity(struct drbd_device *device, sector_t size)
2032 {
2033
2034 set_capacity(device->vdisk, size);
2035 device->this_bdev->bd_inode->i_size = (loff_t)size << 9;
2036 }
2037
2038 void drbd_set_my_capacity(struct drbd_device *device, sector_t size)
2039 {
2040 char ppb[10];
2041 _drbd_set_my_capacity(device, size);
2042 drbd_info(device, "size = %s (%llu KB)\n",
2043 ppsize(ppb, size>>1), (unsigned long long)size>>1);
2044 }
2045
2046 void drbd_device_cleanup(struct drbd_device *device)
2047 {
2048 int i;
2049 if (first_peer_device(device)->connection->receiver.t_state != NONE)
2050 drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
2051 first_peer_device(device)->connection->receiver.t_state);
2052
2053 device->al_writ_cnt =
2054 device->bm_writ_cnt =
2055 device->read_cnt =
2056 device->recv_cnt =
2057 device->send_cnt =
2058 device->writ_cnt =
2059 device->p_size =
2060 device->rs_start =
2061 device->rs_total =
2062 device->rs_failed = 0;
2063 device->rs_last_events = 0;
2064 device->rs_last_sect_ev = 0;
2065 for (i = 0; i < DRBD_SYNC_MARKS; i++) {
2066 device->rs_mark_left[i] = 0;
2067 device->rs_mark_time[i] = 0;
2068 }
2069 D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
2070
2071 _drbd_set_my_capacity(device, 0);
2072 if (device->bitmap) {
2073
2074 drbd_bm_resize(device, 0, 1);
2075 drbd_bm_cleanup(device);
2076 }
2077
2078 drbd_backing_dev_free(device, device->ldev);
2079 device->ldev = NULL;
2080
2081 clear_bit(AL_SUSPENDED, &device->flags);
2082
2083 D_ASSERT(device, list_empty(&device->active_ee));
2084 D_ASSERT(device, list_empty(&device->sync_ee));
2085 D_ASSERT(device, list_empty(&device->done_ee));
2086 D_ASSERT(device, list_empty(&device->read_ee));
2087 D_ASSERT(device, list_empty(&device->net_ee));
2088 D_ASSERT(device, list_empty(&device->resync_reads));
2089 D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
2090 D_ASSERT(device, list_empty(&device->resync_work.list));
2091 D_ASSERT(device, list_empty(&device->unplug_work.list));
2092
2093 drbd_set_defaults(device);
2094 }
2095
2096
2097 static void drbd_destroy_mempools(void)
2098 {
2099 struct page *page;
2100
2101 while (drbd_pp_pool) {
2102 page = drbd_pp_pool;
2103 drbd_pp_pool = (struct page *)page_private(page);
2104 __free_page(page);
2105 drbd_pp_vacant--;
2106 }
2107
2108
2109
2110 bioset_exit(&drbd_io_bio_set);
2111 bioset_exit(&drbd_md_io_bio_set);
2112 mempool_exit(&drbd_md_io_page_pool);
2113 mempool_exit(&drbd_ee_mempool);
2114 mempool_exit(&drbd_request_mempool);
2115 kmem_cache_destroy(drbd_ee_cache);
2116 kmem_cache_destroy(drbd_request_cache);
2117 kmem_cache_destroy(drbd_bm_ext_cache);
2118 kmem_cache_destroy(drbd_al_ext_cache);
2119
2120 drbd_ee_cache = NULL;
2121 drbd_request_cache = NULL;
2122 drbd_bm_ext_cache = NULL;
2123 drbd_al_ext_cache = NULL;
2124
2125 return;
2126 }
2127
2128 static int drbd_create_mempools(void)
2129 {
2130 struct page *page;
2131 const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * drbd_minor_count;
2132 int i, ret;
2133
2134
2135 drbd_request_cache = kmem_cache_create(
2136 "drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
2137 if (drbd_request_cache == NULL)
2138 goto Enomem;
2139
2140 drbd_ee_cache = kmem_cache_create(
2141 "drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
2142 if (drbd_ee_cache == NULL)
2143 goto Enomem;
2144
2145 drbd_bm_ext_cache = kmem_cache_create(
2146 "drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
2147 if (drbd_bm_ext_cache == NULL)
2148 goto Enomem;
2149
2150 drbd_al_ext_cache = kmem_cache_create(
2151 "drbd_al", sizeof(struct lc_element), 0, 0, NULL);
2152 if (drbd_al_ext_cache == NULL)
2153 goto Enomem;
2154
2155
2156 ret = bioset_init(&drbd_io_bio_set, BIO_POOL_SIZE, 0, 0);
2157 if (ret)
2158 goto Enomem;
2159
2160 ret = bioset_init(&drbd_md_io_bio_set, DRBD_MIN_POOL_PAGES, 0,
2161 BIOSET_NEED_BVECS);
2162 if (ret)
2163 goto Enomem;
2164
2165 ret = mempool_init_page_pool(&drbd_md_io_page_pool, DRBD_MIN_POOL_PAGES, 0);
2166 if (ret)
2167 goto Enomem;
2168
2169 ret = mempool_init_slab_pool(&drbd_request_mempool, number,
2170 drbd_request_cache);
2171 if (ret)
2172 goto Enomem;
2173
2174 ret = mempool_init_slab_pool(&drbd_ee_mempool, number, drbd_ee_cache);
2175 if (ret)
2176 goto Enomem;
2177
2178
2179 spin_lock_init(&drbd_pp_lock);
2180
2181 for (i = 0; i < number; i++) {
2182 page = alloc_page(GFP_HIGHUSER);
2183 if (!page)
2184 goto Enomem;
2185 set_page_private(page, (unsigned long)drbd_pp_pool);
2186 drbd_pp_pool = page;
2187 }
2188 drbd_pp_vacant = number;
2189
2190 return 0;
2191
2192 Enomem:
2193 drbd_destroy_mempools();
2194 return -ENOMEM;
2195 }
2196
2197 static void drbd_release_all_peer_reqs(struct drbd_device *device)
2198 {
2199 int rr;
2200
2201 rr = drbd_free_peer_reqs(device, &device->active_ee);
2202 if (rr)
2203 drbd_err(device, "%d EEs in active list found!\n", rr);
2204
2205 rr = drbd_free_peer_reqs(device, &device->sync_ee);
2206 if (rr)
2207 drbd_err(device, "%d EEs in sync list found!\n", rr);
2208
2209 rr = drbd_free_peer_reqs(device, &device->read_ee);
2210 if (rr)
2211 drbd_err(device, "%d EEs in read list found!\n", rr);
2212
2213 rr = drbd_free_peer_reqs(device, &device->done_ee);
2214 if (rr)
2215 drbd_err(device, "%d EEs in done list found!\n", rr);
2216
2217 rr = drbd_free_peer_reqs(device, &device->net_ee);
2218 if (rr)
2219 drbd_err(device, "%d EEs in net list found!\n", rr);
2220 }
2221
2222
2223 void drbd_destroy_device(struct kref *kref)
2224 {
2225 struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2226 struct drbd_resource *resource = device->resource;
2227 struct drbd_peer_device *peer_device, *tmp_peer_device;
2228
2229 del_timer_sync(&device->request_timer);
2230
2231
2232 D_ASSERT(device, device->open_cnt == 0);
2233
2234
2235
2236
2237
2238 if (device->this_bdev)
2239 bdput(device->this_bdev);
2240
2241 drbd_backing_dev_free(device, device->ldev);
2242 device->ldev = NULL;
2243
2244 drbd_release_all_peer_reqs(device);
2245
2246 lc_destroy(device->act_log);
2247 lc_destroy(device->resync);
2248
2249 kfree(device->p_uuid);
2250
2251
2252 if (device->bitmap)
2253 drbd_bm_cleanup(device);
2254 __free_page(device->md_io.page);
2255 put_disk(device->vdisk);
2256 blk_cleanup_queue(device->rq_queue);
2257 kfree(device->rs_plan_s);
2258
2259
2260
2261
2262 for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
2263 kref_put(&peer_device->connection->kref, drbd_destroy_connection);
2264 kfree(peer_device);
2265 }
2266 memset(device, 0xfd, sizeof(*device));
2267 kfree(device);
2268 kref_put(&resource->kref, drbd_destroy_resource);
2269 }
2270
2271
2272
2273
2274 static struct retry_worker {
2275 struct workqueue_struct *wq;
2276 struct work_struct worker;
2277
2278 spinlock_t lock;
2279 struct list_head writes;
2280 } retry;
2281
2282 static void do_retry(struct work_struct *ws)
2283 {
2284 struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
2285 LIST_HEAD(writes);
2286 struct drbd_request *req, *tmp;
2287
2288 spin_lock_irq(&retry->lock);
2289 list_splice_init(&retry->writes, &writes);
2290 spin_unlock_irq(&retry->lock);
2291
2292 list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2293 struct drbd_device *device = req->device;
2294 struct bio *bio = req->master_bio;
2295 unsigned long start_jif = req->start_jif;
2296 bool expected;
2297
2298 expected =
2299 expect(atomic_read(&req->completion_ref) == 0) &&
2300 expect(req->rq_state & RQ_POSTPONED) &&
2301 expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
2302 (req->rq_state & RQ_LOCAL_ABORTED) != 0);
2303
2304 if (!expected)
2305 drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
2306 req, atomic_read(&req->completion_ref),
2307 req->rq_state);
2308
2309
2310
2311
2312
2313
2314 kref_put(&req->kref, drbd_req_destroy);
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329 inc_ap_bio(device);
2330 __drbd_make_request(device, bio, start_jif);
2331 }
2332 }
2333
2334
2335
2336 void drbd_restart_request(struct drbd_request *req)
2337 {
2338 unsigned long flags;
2339 spin_lock_irqsave(&retry.lock, flags);
2340 list_move_tail(&req->tl_requests, &retry.writes);
2341 spin_unlock_irqrestore(&retry.lock, flags);
2342
2343
2344
2345
2346 dec_ap_bio(req->device);
2347
2348 queue_work(retry.wq, &retry.worker);
2349 }
2350
2351 void drbd_destroy_resource(struct kref *kref)
2352 {
2353 struct drbd_resource *resource =
2354 container_of(kref, struct drbd_resource, kref);
2355
2356 idr_destroy(&resource->devices);
2357 free_cpumask_var(resource->cpu_mask);
2358 kfree(resource->name);
2359 memset(resource, 0xf2, sizeof(*resource));
2360 kfree(resource);
2361 }
2362
2363 void drbd_free_resource(struct drbd_resource *resource)
2364 {
2365 struct drbd_connection *connection, *tmp;
2366
2367 for_each_connection_safe(connection, tmp, resource) {
2368 list_del(&connection->connections);
2369 drbd_debugfs_connection_cleanup(connection);
2370 kref_put(&connection->kref, drbd_destroy_connection);
2371 }
2372 drbd_debugfs_resource_cleanup(resource);
2373 kref_put(&resource->kref, drbd_destroy_resource);
2374 }
2375
2376 static void drbd_cleanup(void)
2377 {
2378 unsigned int i;
2379 struct drbd_device *device;
2380 struct drbd_resource *resource, *tmp;
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390 if (drbd_proc)
2391 remove_proc_entry("drbd", NULL);
2392
2393 if (retry.wq)
2394 destroy_workqueue(retry.wq);
2395
2396 drbd_genl_unregister();
2397
2398 idr_for_each_entry(&drbd_devices, device, i)
2399 drbd_delete_device(device);
2400
2401
2402 for_each_resource_safe(resource, tmp, &drbd_resources) {
2403 list_del(&resource->resources);
2404 drbd_free_resource(resource);
2405 }
2406
2407 drbd_debugfs_cleanup();
2408
2409 drbd_destroy_mempools();
2410 unregister_blkdev(DRBD_MAJOR, "drbd");
2411
2412 idr_destroy(&drbd_devices);
2413
2414 pr_info("module cleanup done.\n");
2415 }
2416
2417
2418
2419
2420
2421
2422
2423
2424 static int drbd_congested(void *congested_data, int bdi_bits)
2425 {
2426 struct drbd_device *device = congested_data;
2427 struct request_queue *q;
2428 char reason = '-';
2429 int r = 0;
2430
2431 if (!may_inc_ap_bio(device)) {
2432
2433 r = bdi_bits;
2434 reason = 'd';
2435 goto out;
2436 }
2437
2438 if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2439 r |= (1 << WB_async_congested);
2440
2441
2442
2443
2444
2445 if (!get_ldev_if_state(device, D_UP_TO_DATE))
2446 r |= (1 << WB_sync_congested);
2447 else
2448 put_ldev(device);
2449 r &= bdi_bits;
2450 reason = 'c';
2451 goto out;
2452 }
2453
2454 if (get_ldev(device)) {
2455 q = bdev_get_queue(device->ldev->backing_bdev);
2456 r = bdi_congested(q->backing_dev_info, bdi_bits);
2457 put_ldev(device);
2458 if (r)
2459 reason = 'b';
2460 }
2461
2462 if (bdi_bits & (1 << WB_async_congested) &&
2463 test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
2464 r |= (1 << WB_async_congested);
2465 reason = reason == 'b' ? 'a' : 'n';
2466 }
2467
2468 out:
2469 device->congestion_reason = reason;
2470 return r;
2471 }
2472
2473 static void drbd_init_workqueue(struct drbd_work_queue* wq)
2474 {
2475 spin_lock_init(&wq->q_lock);
2476 INIT_LIST_HEAD(&wq->q);
2477 init_waitqueue_head(&wq->q_wait);
2478 }
2479
2480 struct completion_work {
2481 struct drbd_work w;
2482 struct completion done;
2483 };
2484
2485 static int w_complete(struct drbd_work *w, int cancel)
2486 {
2487 struct completion_work *completion_work =
2488 container_of(w, struct completion_work, w);
2489
2490 complete(&completion_work->done);
2491 return 0;
2492 }
2493
2494 void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
2495 {
2496 struct completion_work completion_work;
2497
2498 completion_work.w.cb = w_complete;
2499 init_completion(&completion_work.done);
2500 drbd_queue_work(work_queue, &completion_work.w);
2501 wait_for_completion(&completion_work.done);
2502 }
2503
2504 struct drbd_resource *drbd_find_resource(const char *name)
2505 {
2506 struct drbd_resource *resource;
2507
2508 if (!name || !name[0])
2509 return NULL;
2510
2511 rcu_read_lock();
2512 for_each_resource_rcu(resource, &drbd_resources) {
2513 if (!strcmp(resource->name, name)) {
2514 kref_get(&resource->kref);
2515 goto found;
2516 }
2517 }
2518 resource = NULL;
2519 found:
2520 rcu_read_unlock();
2521 return resource;
2522 }
2523
2524 struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2525 void *peer_addr, int peer_addr_len)
2526 {
2527 struct drbd_resource *resource;
2528 struct drbd_connection *connection;
2529
2530 rcu_read_lock();
2531 for_each_resource_rcu(resource, &drbd_resources) {
2532 for_each_connection_rcu(connection, resource) {
2533 if (connection->my_addr_len == my_addr_len &&
2534 connection->peer_addr_len == peer_addr_len &&
2535 !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
2536 !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
2537 kref_get(&connection->kref);
2538 goto found;
2539 }
2540 }
2541 }
2542 connection = NULL;
2543 found:
2544 rcu_read_unlock();
2545 return connection;
2546 }
2547
2548 static int drbd_alloc_socket(struct drbd_socket *socket)
2549 {
2550 socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
2551 if (!socket->rbuf)
2552 return -ENOMEM;
2553 socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
2554 if (!socket->sbuf)
2555 return -ENOMEM;
2556 return 0;
2557 }
2558
2559 static void drbd_free_socket(struct drbd_socket *socket)
2560 {
2561 free_page((unsigned long) socket->sbuf);
2562 free_page((unsigned long) socket->rbuf);
2563 }
2564
2565 void conn_free_crypto(struct drbd_connection *connection)
2566 {
2567 drbd_free_sock(connection);
2568
2569 crypto_free_shash(connection->csums_tfm);
2570 crypto_free_shash(connection->verify_tfm);
2571 crypto_free_shash(connection->cram_hmac_tfm);
2572 crypto_free_shash(connection->integrity_tfm);
2573 crypto_free_shash(connection->peer_integrity_tfm);
2574 kfree(connection->int_dig_in);
2575 kfree(connection->int_dig_vv);
2576
2577 connection->csums_tfm = NULL;
2578 connection->verify_tfm = NULL;
2579 connection->cram_hmac_tfm = NULL;
2580 connection->integrity_tfm = NULL;
2581 connection->peer_integrity_tfm = NULL;
2582 connection->int_dig_in = NULL;
2583 connection->int_dig_vv = NULL;
2584 }
2585
2586 int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
2587 {
2588 struct drbd_connection *connection;
2589 cpumask_var_t new_cpu_mask;
2590 int err;
2591
2592 if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
2593 return -ENOMEM;
2594
2595
2596 if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
2597 err = bitmap_parse(res_opts->cpu_mask, DRBD_CPU_MASK_SIZE,
2598 cpumask_bits(new_cpu_mask), nr_cpu_ids);
2599 if (err == -EOVERFLOW) {
2600
2601 cpumask_var_t tmp_cpu_mask;
2602 if (zalloc_cpumask_var(&tmp_cpu_mask, GFP_KERNEL)) {
2603 cpumask_setall(tmp_cpu_mask);
2604 cpumask_and(new_cpu_mask, new_cpu_mask, tmp_cpu_mask);
2605 drbd_warn(resource, "Overflow in bitmap_parse(%.12s%s), truncating to %u bits\n",
2606 res_opts->cpu_mask,
2607 strlen(res_opts->cpu_mask) > 12 ? "..." : "",
2608 nr_cpu_ids);
2609 free_cpumask_var(tmp_cpu_mask);
2610 err = 0;
2611 }
2612 }
2613 if (err) {
2614 drbd_warn(resource, "bitmap_parse() failed with %d\n", err);
2615
2616 goto fail;
2617 }
2618 }
2619 resource->res_opts = *res_opts;
2620 if (cpumask_empty(new_cpu_mask))
2621 drbd_calc_cpu_mask(&new_cpu_mask);
2622 if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
2623 cpumask_copy(resource->cpu_mask, new_cpu_mask);
2624 for_each_connection_rcu(connection, resource) {
2625 connection->receiver.reset_cpu_mask = 1;
2626 connection->ack_receiver.reset_cpu_mask = 1;
2627 connection->worker.reset_cpu_mask = 1;
2628 }
2629 }
2630 err = 0;
2631
2632 fail:
2633 free_cpumask_var(new_cpu_mask);
2634 return err;
2635
2636 }
2637
2638 struct drbd_resource *drbd_create_resource(const char *name)
2639 {
2640 struct drbd_resource *resource;
2641
2642 resource = kzalloc(sizeof(struct drbd_resource), GFP_KERNEL);
2643 if (!resource)
2644 goto fail;
2645 resource->name = kstrdup(name, GFP_KERNEL);
2646 if (!resource->name)
2647 goto fail_free_resource;
2648 if (!zalloc_cpumask_var(&resource->cpu_mask, GFP_KERNEL))
2649 goto fail_free_name;
2650 kref_init(&resource->kref);
2651 idr_init(&resource->devices);
2652 INIT_LIST_HEAD(&resource->connections);
2653 resource->write_ordering = WO_BDEV_FLUSH;
2654 list_add_tail_rcu(&resource->resources, &drbd_resources);
2655 mutex_init(&resource->conf_update);
2656 mutex_init(&resource->adm_mutex);
2657 spin_lock_init(&resource->req_lock);
2658 drbd_debugfs_resource_add(resource);
2659 return resource;
2660
2661 fail_free_name:
2662 kfree(resource->name);
2663 fail_free_resource:
2664 kfree(resource);
2665 fail:
2666 return NULL;
2667 }
2668
2669
2670 struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2671 {
2672 struct drbd_resource *resource;
2673 struct drbd_connection *connection;
2674
2675 connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
2676 if (!connection)
2677 return NULL;
2678
2679 if (drbd_alloc_socket(&connection->data))
2680 goto fail;
2681 if (drbd_alloc_socket(&connection->meta))
2682 goto fail;
2683
2684 connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
2685 if (!connection->current_epoch)
2686 goto fail;
2687
2688 INIT_LIST_HEAD(&connection->transfer_log);
2689
2690 INIT_LIST_HEAD(&connection->current_epoch->list);
2691 connection->epochs = 1;
2692 spin_lock_init(&connection->epoch_lock);
2693
2694 connection->send.seen_any_write_yet = false;
2695 connection->send.current_epoch_nr = 0;
2696 connection->send.current_epoch_writes = 0;
2697
2698 resource = drbd_create_resource(name);
2699 if (!resource)
2700 goto fail;
2701
2702 connection->cstate = C_STANDALONE;
2703 mutex_init(&connection->cstate_mutex);
2704 init_waitqueue_head(&connection->ping_wait);
2705 idr_init(&connection->peer_devices);
2706
2707 drbd_init_workqueue(&connection->sender_work);
2708 mutex_init(&connection->data.mutex);
2709 mutex_init(&connection->meta.mutex);
2710
2711 drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
2712 connection->receiver.connection = connection;
2713 drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
2714 connection->worker.connection = connection;
2715 drbd_thread_init(resource, &connection->ack_receiver, drbd_ack_receiver, "ack_recv");
2716 connection->ack_receiver.connection = connection;
2717
2718 kref_init(&connection->kref);
2719
2720 connection->resource = resource;
2721
2722 if (set_resource_options(resource, res_opts))
2723 goto fail_resource;
2724
2725 kref_get(&resource->kref);
2726 list_add_tail_rcu(&connection->connections, &resource->connections);
2727 drbd_debugfs_connection_add(connection);
2728 return connection;
2729
2730 fail_resource:
2731 list_del(&resource->resources);
2732 drbd_free_resource(resource);
2733 fail:
2734 kfree(connection->current_epoch);
2735 drbd_free_socket(&connection->meta);
2736 drbd_free_socket(&connection->data);
2737 kfree(connection);
2738 return NULL;
2739 }
2740
2741 void drbd_destroy_connection(struct kref *kref)
2742 {
2743 struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2744 struct drbd_resource *resource = connection->resource;
2745
2746 if (atomic_read(&connection->current_epoch->epoch_size) != 0)
2747 drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
2748 kfree(connection->current_epoch);
2749
2750 idr_destroy(&connection->peer_devices);
2751
2752 drbd_free_socket(&connection->meta);
2753 drbd_free_socket(&connection->data);
2754 kfree(connection->int_dig_in);
2755 kfree(connection->int_dig_vv);
2756 memset(connection, 0xfc, sizeof(*connection));
2757 kfree(connection);
2758 kref_put(&resource->kref, drbd_destroy_resource);
2759 }
2760
2761 static int init_submitter(struct drbd_device *device)
2762 {
2763
2764
2765 device->submit.wq =
2766 alloc_ordered_workqueue("drbd%u_submit", WQ_MEM_RECLAIM, device->minor);
2767 if (!device->submit.wq)
2768 return -ENOMEM;
2769
2770 INIT_WORK(&device->submit.worker, do_submit);
2771 INIT_LIST_HEAD(&device->submit.writes);
2772 return 0;
2773 }
2774
2775 enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
2776 {
2777 struct drbd_resource *resource = adm_ctx->resource;
2778 struct drbd_connection *connection;
2779 struct drbd_device *device;
2780 struct drbd_peer_device *peer_device, *tmp_peer_device;
2781 struct gendisk *disk;
2782 struct request_queue *q;
2783 int id;
2784 int vnr = adm_ctx->volume;
2785 enum drbd_ret_code err = ERR_NOMEM;
2786
2787 device = minor_to_device(minor);
2788 if (device)
2789 return ERR_MINOR_OR_VOLUME_EXISTS;
2790
2791
2792 device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
2793 if (!device)
2794 return ERR_NOMEM;
2795 kref_init(&device->kref);
2796
2797 kref_get(&resource->kref);
2798 device->resource = resource;
2799 device->minor = minor;
2800 device->vnr = vnr;
2801
2802 drbd_init_set_defaults(device);
2803
2804 q = blk_alloc_queue_node(GFP_KERNEL, NUMA_NO_NODE);
2805 if (!q)
2806 goto out_no_q;
2807 device->rq_queue = q;
2808 q->queuedata = device;
2809
2810 disk = alloc_disk(1);
2811 if (!disk)
2812 goto out_no_disk;
2813 device->vdisk = disk;
2814
2815 set_disk_ro(disk, true);
2816
2817 disk->queue = q;
2818 disk->major = DRBD_MAJOR;
2819 disk->first_minor = minor;
2820 disk->fops = &drbd_ops;
2821 sprintf(disk->disk_name, "drbd%d", minor);
2822 disk->private_data = device;
2823
2824 device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
2825
2826 device->this_bdev->bd_contains = device->this_bdev;
2827
2828 q->backing_dev_info->congested_fn = drbd_congested;
2829 q->backing_dev_info->congested_data = device;
2830
2831 blk_queue_make_request(q, drbd_make_request);
2832 blk_queue_write_cache(q, true, true);
2833
2834
2835 blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
2836
2837 device->md_io.page = alloc_page(GFP_KERNEL);
2838 if (!device->md_io.page)
2839 goto out_no_io_page;
2840
2841 if (drbd_bm_init(device))
2842 goto out_no_bitmap;
2843 device->read_requests = RB_ROOT;
2844 device->write_requests = RB_ROOT;
2845
2846 id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
2847 if (id < 0) {
2848 if (id == -ENOSPC)
2849 err = ERR_MINOR_OR_VOLUME_EXISTS;
2850 goto out_no_minor_idr;
2851 }
2852 kref_get(&device->kref);
2853
2854 id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
2855 if (id < 0) {
2856 if (id == -ENOSPC)
2857 err = ERR_MINOR_OR_VOLUME_EXISTS;
2858 goto out_idr_remove_minor;
2859 }
2860 kref_get(&device->kref);
2861
2862 INIT_LIST_HEAD(&device->peer_devices);
2863 INIT_LIST_HEAD(&device->pending_bitmap_io);
2864 for_each_connection(connection, resource) {
2865 peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
2866 if (!peer_device)
2867 goto out_idr_remove_from_resource;
2868 peer_device->connection = connection;
2869 peer_device->device = device;
2870
2871 list_add(&peer_device->peer_devices, &device->peer_devices);
2872 kref_get(&device->kref);
2873
2874 id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
2875 if (id < 0) {
2876 if (id == -ENOSPC)
2877 err = ERR_INVALID_REQUEST;
2878 goto out_idr_remove_from_resource;
2879 }
2880 kref_get(&connection->kref);
2881 INIT_WORK(&peer_device->send_acks_work, drbd_send_acks_wf);
2882 }
2883
2884 if (init_submitter(device)) {
2885 err = ERR_NOMEM;
2886 goto out_idr_remove_vol;
2887 }
2888
2889 add_disk(disk);
2890
2891
2892 device->state.conn = first_connection(resource)->cstate;
2893 if (device->state.conn == C_WF_REPORT_PARAMS) {
2894 for_each_peer_device(peer_device, device)
2895 drbd_connected(peer_device);
2896 }
2897
2898 for_each_peer_device(peer_device, device)
2899 drbd_debugfs_peer_device_add(peer_device);
2900 drbd_debugfs_device_add(device);
2901 return NO_ERROR;
2902
2903 out_idr_remove_vol:
2904 idr_remove(&connection->peer_devices, vnr);
2905 out_idr_remove_from_resource:
2906 for_each_connection(connection, resource) {
2907 peer_device = idr_remove(&connection->peer_devices, vnr);
2908 if (peer_device)
2909 kref_put(&connection->kref, drbd_destroy_connection);
2910 }
2911 for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
2912 list_del(&peer_device->peer_devices);
2913 kfree(peer_device);
2914 }
2915 idr_remove(&resource->devices, vnr);
2916 out_idr_remove_minor:
2917 idr_remove(&drbd_devices, minor);
2918 synchronize_rcu();
2919 out_no_minor_idr:
2920 drbd_bm_cleanup(device);
2921 out_no_bitmap:
2922 __free_page(device->md_io.page);
2923 out_no_io_page:
2924 put_disk(disk);
2925 out_no_disk:
2926 blk_cleanup_queue(q);
2927 out_no_q:
2928 kref_put(&resource->kref, drbd_destroy_resource);
2929 kfree(device);
2930 return err;
2931 }
2932
2933 void drbd_delete_device(struct drbd_device *device)
2934 {
2935 struct drbd_resource *resource = device->resource;
2936 struct drbd_connection *connection;
2937 struct drbd_peer_device *peer_device;
2938
2939
2940 for_each_peer_device(peer_device, device)
2941 drbd_debugfs_peer_device_cleanup(peer_device);
2942 drbd_debugfs_device_cleanup(device);
2943 for_each_connection(connection, resource) {
2944 idr_remove(&connection->peer_devices, device->vnr);
2945 kref_put(&device->kref, drbd_destroy_device);
2946 }
2947 idr_remove(&resource->devices, device->vnr);
2948 kref_put(&device->kref, drbd_destroy_device);
2949 idr_remove(&drbd_devices, device_to_minor(device));
2950 kref_put(&device->kref, drbd_destroy_device);
2951 del_gendisk(device->vdisk);
2952 synchronize_rcu();
2953 kref_put(&device->kref, drbd_destroy_device);
2954 }
2955
2956 static int __init drbd_init(void)
2957 {
2958 int err;
2959
2960 if (drbd_minor_count < DRBD_MINOR_COUNT_MIN || drbd_minor_count > DRBD_MINOR_COUNT_MAX) {
2961 pr_err("invalid minor_count (%d)\n", drbd_minor_count);
2962 #ifdef MODULE
2963 return -EINVAL;
2964 #else
2965 drbd_minor_count = DRBD_MINOR_COUNT_DEF;
2966 #endif
2967 }
2968
2969 err = register_blkdev(DRBD_MAJOR, "drbd");
2970 if (err) {
2971 pr_err("unable to register block device major %d\n",
2972 DRBD_MAJOR);
2973 return err;
2974 }
2975
2976
2977
2978
2979 init_waitqueue_head(&drbd_pp_wait);
2980
2981 drbd_proc = NULL;
2982 idr_init(&drbd_devices);
2983
2984 mutex_init(&resources_mutex);
2985 INIT_LIST_HEAD(&drbd_resources);
2986
2987 err = drbd_genl_register();
2988 if (err) {
2989 pr_err("unable to register generic netlink family\n");
2990 goto fail;
2991 }
2992
2993 err = drbd_create_mempools();
2994 if (err)
2995 goto fail;
2996
2997 err = -ENOMEM;
2998 drbd_proc = proc_create_single("drbd", S_IFREG | 0444 , NULL, drbd_seq_show);
2999 if (!drbd_proc) {
3000 pr_err("unable to register proc file\n");
3001 goto fail;
3002 }
3003
3004 retry.wq = create_singlethread_workqueue("drbd-reissue");
3005 if (!retry.wq) {
3006 pr_err("unable to create retry workqueue\n");
3007 goto fail;
3008 }
3009 INIT_WORK(&retry.worker, do_retry);
3010 spin_lock_init(&retry.lock);
3011 INIT_LIST_HEAD(&retry.writes);
3012
3013 drbd_debugfs_init();
3014
3015 pr_info("initialized. "
3016 "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
3017 API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
3018 pr_info("%s\n", drbd_buildtag());
3019 pr_info("registered as block device major %d\n", DRBD_MAJOR);
3020 return 0;
3021
3022 fail:
3023 drbd_cleanup();
3024 if (err == -ENOMEM)
3025 pr_err("ran out of memory\n");
3026 else
3027 pr_err("initialization failure\n");
3028 return err;
3029 }
3030
3031 static void drbd_free_one_sock(struct drbd_socket *ds)
3032 {
3033 struct socket *s;
3034 mutex_lock(&ds->mutex);
3035 s = ds->socket;
3036 ds->socket = NULL;
3037 mutex_unlock(&ds->mutex);
3038 if (s) {
3039
3040 synchronize_rcu();
3041 kernel_sock_shutdown(s, SHUT_RDWR);
3042 sock_release(s);
3043 }
3044 }
3045
3046 void drbd_free_sock(struct drbd_connection *connection)
3047 {
3048 if (connection->data.socket)
3049 drbd_free_one_sock(&connection->data);
3050 if (connection->meta.socket)
3051 drbd_free_one_sock(&connection->meta);
3052 }
3053
3054
3055
3056 void conn_md_sync(struct drbd_connection *connection)
3057 {
3058 struct drbd_peer_device *peer_device;
3059 int vnr;
3060
3061 rcu_read_lock();
3062 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
3063 struct drbd_device *device = peer_device->device;
3064
3065 kref_get(&device->kref);
3066 rcu_read_unlock();
3067 drbd_md_sync(device);
3068 kref_put(&device->kref, drbd_destroy_device);
3069 rcu_read_lock();
3070 }
3071 rcu_read_unlock();
3072 }
3073
3074
3075 struct meta_data_on_disk {
3076 u64 la_size_sect;
3077 u64 uuid[UI_SIZE];
3078 u64 device_uuid;
3079 u64 reserved_u64_1;
3080 u32 flags;
3081 u32 magic;
3082 u32 md_size_sect;
3083 u32 al_offset;
3084 u32 al_nr_extents;
3085
3086 u32 bm_offset;
3087 u32 bm_bytes_per_bit;
3088 u32 la_peer_max_bio_size;
3089
3090
3091 u32 al_stripes;
3092 u32 al_stripe_size_4k;
3093
3094 u8 reserved_u8[4096 - (7*8 + 10*4)];
3095 } __packed;
3096
3097
3098
3099 void drbd_md_write(struct drbd_device *device, void *b)
3100 {
3101 struct meta_data_on_disk *buffer = b;
3102 sector_t sector;
3103 int i;
3104
3105 memset(buffer, 0, sizeof(*buffer));
3106
3107 buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
3108 for (i = UI_CURRENT; i < UI_SIZE; i++)
3109 buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
3110 buffer->flags = cpu_to_be32(device->ldev->md.flags);
3111 buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
3112
3113 buffer->md_size_sect = cpu_to_be32(device->ldev->md.md_size_sect);
3114 buffer->al_offset = cpu_to_be32(device->ldev->md.al_offset);
3115 buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
3116 buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
3117 buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
3118
3119 buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
3120 buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
3121
3122 buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
3123 buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
3124
3125 D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
3126 sector = device->ldev->md.md_offset;
3127
3128 if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) {
3129
3130 drbd_err(device, "meta data update failed!\n");
3131 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
3132 }
3133 }
3134
3135
3136
3137
3138
3139 void drbd_md_sync(struct drbd_device *device)
3140 {
3141 struct meta_data_on_disk *buffer;
3142
3143
3144 BUILD_BUG_ON(UI_SIZE != 4);
3145 BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);
3146
3147 del_timer(&device->md_sync_timer);
3148
3149 if (!test_and_clear_bit(MD_DIRTY, &device->flags))
3150 return;
3151
3152
3153
3154 if (!get_ldev_if_state(device, D_FAILED))
3155 return;
3156
3157 buffer = drbd_md_get_buffer(device, __func__);
3158 if (!buffer)
3159 goto out;
3160
3161 drbd_md_write(device, buffer);
3162
3163
3164
3165 device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
3166
3167 drbd_md_put_buffer(device);
3168 out:
3169 put_ldev(device);
3170 }
3171
3172 static int check_activity_log_stripe_size(struct drbd_device *device,
3173 struct meta_data_on_disk *on_disk,
3174 struct drbd_md *in_core)
3175 {
3176 u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
3177 u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
3178 u64 al_size_4k;
3179
3180
3181 if (al_stripes == 0 && al_stripe_size_4k == 0) {
3182 al_stripes = 1;
3183 al_stripe_size_4k = MD_32kB_SECT/8;
3184 }
3185
3186
3187
3188
3189 if (al_stripes == 0 || al_stripe_size_4k == 0)
3190 goto err;
3191
3192 al_size_4k = (u64)al_stripes * al_stripe_size_4k;
3193
3194
3195
3196
3197
3198 if (al_size_4k > (16 * 1024 * 1024/4))
3199 goto err;
3200
3201
3202
3203 if (al_size_4k < MD_32kB_SECT/8)
3204 goto err;
3205
3206 in_core->al_stripe_size_4k = al_stripe_size_4k;
3207 in_core->al_stripes = al_stripes;
3208 in_core->al_size_4k = al_size_4k;
3209
3210 return 0;
3211 err:
3212 drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
3213 al_stripes, al_stripe_size_4k);
3214 return -EINVAL;
3215 }
3216
3217 static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3218 {
3219 sector_t capacity = drbd_get_capacity(bdev->md_bdev);
3220 struct drbd_md *in_core = &bdev->md;
3221 s32 on_disk_al_sect;
3222 s32 on_disk_bm_sect;
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232 if (in_core->al_offset < 0) {
3233 if (in_core->bm_offset > in_core->al_offset)
3234 goto err;
3235 on_disk_al_sect = -in_core->al_offset;
3236 on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
3237 } else {
3238 if (in_core->al_offset != MD_4kB_SECT)
3239 goto err;
3240 if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
3241 goto err;
3242
3243 on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
3244 on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
3245 }
3246
3247
3248 if (in_core->meta_dev_idx >= 0) {
3249 if (in_core->md_size_sect != MD_128MB_SECT
3250 || in_core->al_offset != MD_4kB_SECT
3251 || in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
3252 || in_core->al_stripes != 1
3253 || in_core->al_stripe_size_4k != MD_32kB_SECT/8)
3254 goto err;
3255 }
3256
3257 if (capacity < in_core->md_size_sect)
3258 goto err;
3259 if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
3260 goto err;
3261
3262
3263 if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
3264 goto err;
3265
3266
3267
3268 if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
3269 goto err;
3270
3271
3272 if (in_core->bm_offset & 7)
3273 goto err;
3274
3275
3276
3277
3278 if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
3279 goto err;
3280
3281 return 0;
3282
3283 err:
3284 drbd_err(device, "meta data offsets don't make sense: idx=%d "
3285 "al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
3286 "md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
3287 in_core->meta_dev_idx,
3288 in_core->al_stripes, in_core->al_stripe_size_4k,
3289 in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
3290 (unsigned long long)in_core->la_size_sect,
3291 (unsigned long long)capacity);
3292
3293 return -EINVAL;
3294 }
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308 int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
3309 {
3310 struct meta_data_on_disk *buffer;
3311 u32 magic, flags;
3312 int i, rv = NO_ERROR;
3313
3314 if (device->state.disk != D_DISKLESS)
3315 return ERR_DISK_CONFIGURED;
3316
3317 buffer = drbd_md_get_buffer(device, __func__);
3318 if (!buffer)
3319 return ERR_NOMEM;
3320
3321
3322
3323 bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
3324 bdev->md.md_offset = drbd_md_ss(bdev);
3325
3326
3327
3328 bdev->md.md_size_sect = 8;
3329
3330 if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset,
3331 REQ_OP_READ)) {
3332
3333
3334 drbd_err(device, "Error while reading metadata.\n");
3335 rv = ERR_IO_MD_DISK;
3336 goto err;
3337 }
3338
3339 magic = be32_to_cpu(buffer->magic);
3340 flags = be32_to_cpu(buffer->flags);
3341 if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
3342 (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
3343
3344 drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
3345 rv = ERR_MD_UNCLEAN;
3346 goto err;
3347 }
3348
3349 rv = ERR_MD_INVALID;
3350 if (magic != DRBD_MD_MAGIC_08) {
3351 if (magic == DRBD_MD_MAGIC_07)
3352 drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
3353 else
3354 drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
3355 goto err;
3356 }
3357
3358 if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
3359 drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
3360 be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
3361 goto err;
3362 }
3363
3364
3365
3366 bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
3367 for (i = UI_CURRENT; i < UI_SIZE; i++)
3368 bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
3369 bdev->md.flags = be32_to_cpu(buffer->flags);
3370 bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);
3371
3372 bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
3373 bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
3374 bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);
3375
3376 if (check_activity_log_stripe_size(device, buffer, &bdev->md))
3377 goto err;
3378 if (check_offsets_and_sizes(device, bdev))
3379 goto err;
3380
3381 if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
3382 drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
3383 be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
3384 goto err;
3385 }
3386 if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
3387 drbd_err(device, "unexpected md_size: %u (expected %u)\n",
3388 be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
3389 goto err;
3390 }
3391
3392 rv = NO_ERROR;
3393
3394 spin_lock_irq(&device->resource->req_lock);
3395 if (device->state.conn < C_CONNECTED) {
3396 unsigned int peer;
3397 peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3398 peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3399 device->peer_max_bio_size = peer;
3400 }
3401 spin_unlock_irq(&device->resource->req_lock);
3402
3403 err:
3404 drbd_md_put_buffer(device);
3405
3406 return rv;
3407 }
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417 #ifdef DEBUG
3418 void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3419 {
3420 if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
3421 mod_timer(&device->md_sync_timer, jiffies + HZ);
3422 device->last_md_mark_dirty.line = line;
3423 device->last_md_mark_dirty.func = func;
3424 }
3425 }
3426 #else
3427 void drbd_md_mark_dirty(struct drbd_device *device)
3428 {
3429 if (!test_and_set_bit(MD_DIRTY, &device->flags))
3430 mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
3431 }
3432 #endif
3433
3434 void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
3435 {
3436 int i;
3437
3438 for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3439 device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
3440 }
3441
3442 void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3443 {
3444 if (idx == UI_CURRENT) {
3445 if (device->state.role == R_PRIMARY)
3446 val |= 1;
3447 else
3448 val &= ~((u64)1);
3449
3450 drbd_set_ed_uuid(device, val);
3451 }
3452
3453 device->ldev->md.uuid[idx] = val;
3454 drbd_md_mark_dirty(device);
3455 }
3456
3457 void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3458 {
3459 unsigned long flags;
3460 spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
3461 __drbd_uuid_set(device, idx, val);
3462 spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3463 }
3464
3465 void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3466 {
3467 unsigned long flags;
3468 spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
3469 if (device->ldev->md.uuid[idx]) {
3470 drbd_uuid_move_history(device);
3471 device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
3472 }
3473 __drbd_uuid_set(device, idx, val);
3474 spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3475 }
3476
3477
3478
3479
3480
3481
3482
3483
3484 void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
3485 {
3486 u64 val;
3487 unsigned long long bm_uuid;
3488
3489 get_random_bytes(&val, sizeof(u64));
3490
3491 spin_lock_irq(&device->ldev->md.uuid_lock);
3492 bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3493
3494 if (bm_uuid)
3495 drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
3496
3497 device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
3498 __drbd_uuid_set(device, UI_CURRENT, val);
3499 spin_unlock_irq(&device->ldev->md.uuid_lock);
3500
3501 drbd_print_uuids(device, "new current UUID");
3502
3503 drbd_md_sync(device);
3504 }
3505
3506 void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
3507 {
3508 unsigned long flags;
3509 if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
3510 return;
3511
3512 spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
3513 if (val == 0) {
3514 drbd_uuid_move_history(device);
3515 device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
3516 device->ldev->md.uuid[UI_BITMAP] = 0;
3517 } else {
3518 unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3519 if (bm_uuid)
3520 drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
3521
3522 device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
3523 }
3524 spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3525
3526 drbd_md_mark_dirty(device);
3527 }
3528
3529
3530
3531
3532
3533
3534
3535 int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
3536 {
3537 int rv = -EIO;
3538
3539 drbd_md_set_flag(device, MDF_FULL_SYNC);
3540 drbd_md_sync(device);
3541 drbd_bm_set_all(device);
3542
3543 rv = drbd_bm_write(device);
3544
3545 if (!rv) {
3546 drbd_md_clear_flag(device, MDF_FULL_SYNC);
3547 drbd_md_sync(device);
3548 }
3549
3550 return rv;
3551 }
3552
3553
3554
3555
3556
3557
3558
3559 int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local)
3560 {
3561 drbd_resume_al(device);
3562 drbd_bm_clear_all(device);
3563 return drbd_bm_write(device);
3564 }
3565
3566 static int w_bitmap_io(struct drbd_work *w, int unused)
3567 {
3568 struct drbd_device *device =
3569 container_of(w, struct drbd_device, bm_io_work.w);
3570 struct bm_io_work *work = &device->bm_io_work;
3571 int rv = -EIO;
3572
3573 if (work->flags != BM_LOCKED_CHANGE_ALLOWED) {
3574 int cnt = atomic_read(&device->ap_bio_cnt);
3575 if (cnt)
3576 drbd_err(device, "FIXME: ap_bio_cnt %d, expected 0; queued for '%s'\n",
3577 cnt, work->why);
3578 }
3579
3580 if (get_ldev(device)) {
3581 drbd_bm_lock(device, work->why, work->flags);
3582 rv = work->io_fn(device);
3583 drbd_bm_unlock(device);
3584 put_ldev(device);
3585 }
3586
3587 clear_bit_unlock(BITMAP_IO, &device->flags);
3588 wake_up(&device->misc_wait);
3589
3590 if (work->done)
3591 work->done(device, rv);
3592
3593 clear_bit(BITMAP_IO_QUEUED, &device->flags);
3594 work->why = NULL;
3595 work->flags = 0;
3596
3597 return 0;
3598 }
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615 void drbd_queue_bitmap_io(struct drbd_device *device,
3616 int (*io_fn)(struct drbd_device *),
3617 void (*done)(struct drbd_device *, int),
3618 char *why, enum bm_flag flags)
3619 {
3620 D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
3621
3622 D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
3623 D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
3624 D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
3625 if (device->bm_io_work.why)
3626 drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
3627 why, device->bm_io_work.why);
3628
3629 device->bm_io_work.io_fn = io_fn;
3630 device->bm_io_work.done = done;
3631 device->bm_io_work.why = why;
3632 device->bm_io_work.flags = flags;
3633
3634 spin_lock_irq(&device->resource->req_lock);
3635 set_bit(BITMAP_IO, &device->flags);
3636
3637
3638 if (flags == BM_LOCKED_CHANGE_ALLOWED || atomic_read(&device->ap_bio_cnt) == 0) {
3639 if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3640 drbd_queue_work(&first_peer_device(device)->connection->sender_work,
3641 &device->bm_io_work.w);
3642 }
3643 spin_unlock_irq(&device->resource->req_lock);
3644 }
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655 int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3656 char *why, enum bm_flag flags)
3657 {
3658
3659 const bool do_suspend_io = flags & (BM_DONT_CLEAR|BM_DONT_SET|BM_DONT_TEST);
3660 int rv;
3661
3662 D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
3663
3664 if (do_suspend_io)
3665 drbd_suspend_io(device);
3666
3667 drbd_bm_lock(device, why, flags);
3668 rv = io_fn(device);
3669 drbd_bm_unlock(device);
3670
3671 if (do_suspend_io)
3672 drbd_resume_io(device);
3673
3674 return rv;
3675 }
3676
3677 void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
3678 {
3679 if ((device->ldev->md.flags & flag) != flag) {
3680 drbd_md_mark_dirty(device);
3681 device->ldev->md.flags |= flag;
3682 }
3683 }
3684
3685 void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
3686 {
3687 if ((device->ldev->md.flags & flag) != 0) {
3688 drbd_md_mark_dirty(device);
3689 device->ldev->md.flags &= ~flag;
3690 }
3691 }
3692 int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
3693 {
3694 return (bdev->md.flags & flag) != 0;
3695 }
3696
3697 static void md_sync_timer_fn(struct timer_list *t)
3698 {
3699 struct drbd_device *device = from_timer(device, t, md_sync_timer);
3700 drbd_device_post_work(device, MD_SYNC);
3701 }
3702
3703 const char *cmdname(enum drbd_packet cmd)
3704 {
3705
3706
3707
3708 static const char *cmdnames[] = {
3709 [P_DATA] = "Data",
3710 [P_WSAME] = "WriteSame",
3711 [P_TRIM] = "Trim",
3712 [P_DATA_REPLY] = "DataReply",
3713 [P_RS_DATA_REPLY] = "RSDataReply",
3714 [P_BARRIER] = "Barrier",
3715 [P_BITMAP] = "ReportBitMap",
3716 [P_BECOME_SYNC_TARGET] = "BecomeSyncTarget",
3717 [P_BECOME_SYNC_SOURCE] = "BecomeSyncSource",
3718 [P_UNPLUG_REMOTE] = "UnplugRemote",
3719 [P_DATA_REQUEST] = "DataRequest",
3720 [P_RS_DATA_REQUEST] = "RSDataRequest",
3721 [P_SYNC_PARAM] = "SyncParam",
3722 [P_SYNC_PARAM89] = "SyncParam89",
3723 [P_PROTOCOL] = "ReportProtocol",
3724 [P_UUIDS] = "ReportUUIDs",
3725 [P_SIZES] = "ReportSizes",
3726 [P_STATE] = "ReportState",
3727 [P_SYNC_UUID] = "ReportSyncUUID",
3728 [P_AUTH_CHALLENGE] = "AuthChallenge",
3729 [P_AUTH_RESPONSE] = "AuthResponse",
3730 [P_PING] = "Ping",
3731 [P_PING_ACK] = "PingAck",
3732 [P_RECV_ACK] = "RecvAck",
3733 [P_WRITE_ACK] = "WriteAck",
3734 [P_RS_WRITE_ACK] = "RSWriteAck",
3735 [P_SUPERSEDED] = "Superseded",
3736 [P_NEG_ACK] = "NegAck",
3737 [P_NEG_DREPLY] = "NegDReply",
3738 [P_NEG_RS_DREPLY] = "NegRSDReply",
3739 [P_BARRIER_ACK] = "BarrierAck",
3740 [P_STATE_CHG_REQ] = "StateChgRequest",
3741 [P_STATE_CHG_REPLY] = "StateChgReply",
3742 [P_OV_REQUEST] = "OVRequest",
3743 [P_OV_REPLY] = "OVReply",
3744 [P_OV_RESULT] = "OVResult",
3745 [P_CSUM_RS_REQUEST] = "CsumRSRequest",
3746 [P_RS_IS_IN_SYNC] = "CsumRSIsInSync",
3747 [P_COMPRESSED_BITMAP] = "CBitmap",
3748 [P_DELAY_PROBE] = "DelayProbe",
3749 [P_OUT_OF_SYNC] = "OutOfSync",
3750 [P_RETRY_WRITE] = "RetryWrite",
3751 [P_RS_CANCEL] = "RSCancel",
3752 [P_CONN_ST_CHG_REQ] = "conn_st_chg_req",
3753 [P_CONN_ST_CHG_REPLY] = "conn_st_chg_reply",
3754 [P_RETRY_WRITE] = "retry_write",
3755 [P_PROTOCOL_UPDATE] = "protocol_update",
3756 [P_RS_THIN_REQ] = "rs_thin_req",
3757 [P_RS_DEALLOCATED] = "rs_deallocated",
3758
3759
3760
3761
3762
3763 };
3764
3765
3766 if (cmd == P_INITIAL_META)
3767 return "InitialMeta";
3768 if (cmd == P_INITIAL_DATA)
3769 return "InitialData";
3770 if (cmd == P_CONNECTION_FEATURES)
3771 return "ConnectionFeatures";
3772 if (cmd >= ARRAY_SIZE(cmdnames))
3773 return "Unknown";
3774 return cmdnames[cmd];
3775 }
3776
3777
3778
3779
3780
3781
3782
3783 int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3784 {
3785 struct net_conf *nc;
3786 DEFINE_WAIT(wait);
3787 long timeout;
3788
3789 rcu_read_lock();
3790 nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3791 if (!nc) {
3792 rcu_read_unlock();
3793 return -ETIMEDOUT;
3794 }
3795 timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
3796 rcu_read_unlock();
3797
3798
3799 i->waiting = true;
3800 prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3801 spin_unlock_irq(&device->resource->req_lock);
3802 timeout = schedule_timeout(timeout);
3803 finish_wait(&device->misc_wait, &wait);
3804 spin_lock_irq(&device->resource->req_lock);
3805 if (!timeout || device->state.conn < C_CONNECTED)
3806 return -ETIMEDOUT;
3807 if (signal_pending(current))
3808 return -ERESTARTSYS;
3809 return 0;
3810 }
3811
3812 void lock_all_resources(void)
3813 {
3814 struct drbd_resource *resource;
3815 int __maybe_unused i = 0;
3816
3817 mutex_lock(&resources_mutex);
3818 local_irq_disable();
3819 for_each_resource(resource, &drbd_resources)
3820 spin_lock_nested(&resource->req_lock, i++);
3821 }
3822
3823 void unlock_all_resources(void)
3824 {
3825 struct drbd_resource *resource;
3826
3827 for_each_resource(resource, &drbd_resources)
3828 spin_unlock(&resource->req_lock);
3829 local_irq_enable();
3830 mutex_unlock(&resources_mutex);
3831 }
3832
3833 #ifdef CONFIG_DRBD_FAULT_INJECTION
3834
3835
3836 struct fault_random_state {
3837 unsigned long state;
3838 unsigned long count;
3839 };
3840
3841 #define FAULT_RANDOM_MULT 39916801
3842 #define FAULT_RANDOM_ADD 479001701
3843 #define FAULT_RANDOM_REFRESH 10000
3844
3845
3846
3847
3848
3849 static unsigned long
3850 _drbd_fault_random(struct fault_random_state *rsp)
3851 {
3852 long refresh;
3853
3854 if (!rsp->count--) {
3855 get_random_bytes(&refresh, sizeof(refresh));
3856 rsp->state += refresh;
3857 rsp->count = FAULT_RANDOM_REFRESH;
3858 }
3859 rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
3860 return swahw32(rsp->state);
3861 }
3862
3863 static char *
3864 _drbd_fault_str(unsigned int type) {
3865 static char *_faults[] = {
3866 [DRBD_FAULT_MD_WR] = "Meta-data write",
3867 [DRBD_FAULT_MD_RD] = "Meta-data read",
3868 [DRBD_FAULT_RS_WR] = "Resync write",
3869 [DRBD_FAULT_RS_RD] = "Resync read",
3870 [DRBD_FAULT_DT_WR] = "Data write",
3871 [DRBD_FAULT_DT_RD] = "Data read",
3872 [DRBD_FAULT_DT_RA] = "Data read ahead",
3873 [DRBD_FAULT_BM_ALLOC] = "BM allocation",
3874 [DRBD_FAULT_AL_EE] = "EE allocation",
3875 [DRBD_FAULT_RECEIVE] = "receive data corruption",
3876 };
3877
3878 return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
3879 }
3880
3881 unsigned int
3882 _drbd_insert_fault(struct drbd_device *device, unsigned int type)
3883 {
3884 static struct fault_random_state rrs = {0, 0};
3885
3886 unsigned int ret = (
3887 (drbd_fault_devs == 0 ||
3888 ((1 << device_to_minor(device)) & drbd_fault_devs) != 0) &&
3889 (((_drbd_fault_random(&rrs) % 100) + 1) <= drbd_fault_rate));
3890
3891 if (ret) {
3892 drbd_fault_count++;
3893
3894 if (__ratelimit(&drbd_ratelimit_state))
3895 drbd_warn(device, "***Simulating %s failure\n",
3896 _drbd_fault_str(type));
3897 }
3898
3899 return ret;
3900 }
3901 #endif
3902
3903 const char *drbd_buildtag(void)
3904 {
3905
3906
3907
3908 static char buildtag[38] = "\0uilt-in";
3909
3910 if (buildtag[0] == 0) {
3911 #ifdef MODULE
3912 sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
3913 #else
3914 buildtag[0] = 'b';
3915 #endif
3916 }
3917
3918 return buildtag;
3919 }
3920
3921 module_init(drbd_init)
3922 module_exit(drbd_cleanup)
3923
3924 EXPORT_SYMBOL(drbd_conn_str);
3925 EXPORT_SYMBOL(drbd_role_str);
3926 EXPORT_SYMBOL(drbd_disk_str);
3927 EXPORT_SYMBOL(drbd_set_st_err_str);