Lines Matching refs:rh
102 struct dm_region_hash *rh; /* FIXME: can we get rid of this ? */ member
116 static region_t dm_rh_sector_to_region(struct dm_region_hash *rh, sector_t sector) in dm_rh_sector_to_region() argument
118 return sector >> rh->region_shift; in dm_rh_sector_to_region()
121 sector_t dm_rh_region_to_sector(struct dm_region_hash *rh, region_t region) in dm_rh_region_to_sector() argument
123 return region << rh->region_shift; in dm_rh_region_to_sector()
127 region_t dm_rh_bio_to_region(struct dm_region_hash *rh, struct bio *bio) in dm_rh_bio_to_region() argument
129 return dm_rh_sector_to_region(rh, bio->bi_iter.bi_sector - in dm_rh_bio_to_region()
130 rh->target_begin); in dm_rh_bio_to_region()
136 return reg->rh->context; in dm_rh_region_context()
146 sector_t dm_rh_get_region_size(struct dm_region_hash *rh) in dm_rh_get_region_size() argument
148 return rh->region_size; in dm_rh_get_region_size()
169 struct dm_region_hash *rh; in dm_region_hash_create() local
182 rh = kmalloc(sizeof(*rh), GFP_KERNEL); in dm_region_hash_create()
183 if (!rh) { in dm_region_hash_create()
188 rh->context = context; in dm_region_hash_create()
189 rh->dispatch_bios = dispatch_bios; in dm_region_hash_create()
190 rh->wakeup_workers = wakeup_workers; in dm_region_hash_create()
191 rh->wakeup_all_recovery_waiters = wakeup_all_recovery_waiters; in dm_region_hash_create()
192 rh->target_begin = target_begin; in dm_region_hash_create()
193 rh->max_recovery = max_recovery; in dm_region_hash_create()
194 rh->log = log; in dm_region_hash_create()
195 rh->region_size = region_size; in dm_region_hash_create()
196 rh->region_shift = __ffs(region_size); in dm_region_hash_create()
197 rwlock_init(&rh->hash_lock); in dm_region_hash_create()
198 rh->mask = nr_buckets - 1; in dm_region_hash_create()
199 rh->nr_buckets = nr_buckets; in dm_region_hash_create()
201 rh->shift = RH_HASH_SHIFT; in dm_region_hash_create()
202 rh->prime = RH_HASH_MULT; in dm_region_hash_create()
204 rh->buckets = vmalloc(nr_buckets * sizeof(*rh->buckets)); in dm_region_hash_create()
205 if (!rh->buckets) { in dm_region_hash_create()
207 kfree(rh); in dm_region_hash_create()
212 INIT_LIST_HEAD(rh->buckets + i); in dm_region_hash_create()
214 spin_lock_init(&rh->region_lock); in dm_region_hash_create()
215 sema_init(&rh->recovery_count, 0); in dm_region_hash_create()
216 atomic_set(&rh->recovery_in_flight, 0); in dm_region_hash_create()
217 INIT_LIST_HEAD(&rh->clean_regions); in dm_region_hash_create()
218 INIT_LIST_HEAD(&rh->quiesced_regions); in dm_region_hash_create()
219 INIT_LIST_HEAD(&rh->recovered_regions); in dm_region_hash_create()
220 INIT_LIST_HEAD(&rh->failed_recovered_regions); in dm_region_hash_create()
221 rh->flush_failure = 0; in dm_region_hash_create()
223 rh->region_pool = mempool_create_kmalloc_pool(MIN_REGIONS, in dm_region_hash_create()
225 if (!rh->region_pool) { in dm_region_hash_create()
226 vfree(rh->buckets); in dm_region_hash_create()
227 kfree(rh); in dm_region_hash_create()
228 rh = ERR_PTR(-ENOMEM); in dm_region_hash_create()
231 return rh; in dm_region_hash_create()
235 void dm_region_hash_destroy(struct dm_region_hash *rh) in dm_region_hash_destroy() argument
240 BUG_ON(!list_empty(&rh->quiesced_regions)); in dm_region_hash_destroy()
241 for (h = 0; h < rh->nr_buckets; h++) { in dm_region_hash_destroy()
242 list_for_each_entry_safe(reg, nreg, rh->buckets + h, in dm_region_hash_destroy()
245 mempool_free(reg, rh->region_pool); in dm_region_hash_destroy()
249 if (rh->log) in dm_region_hash_destroy()
250 dm_dirty_log_destroy(rh->log); in dm_region_hash_destroy()
252 mempool_destroy(rh->region_pool); in dm_region_hash_destroy()
253 vfree(rh->buckets); in dm_region_hash_destroy()
254 kfree(rh); in dm_region_hash_destroy()
258 struct dm_dirty_log *dm_rh_dirty_log(struct dm_region_hash *rh) in dm_rh_dirty_log() argument
260 return rh->log; in dm_rh_dirty_log()
264 static unsigned rh_hash(struct dm_region_hash *rh, region_t region) in rh_hash() argument
266 return (unsigned) ((region * rh->prime) >> rh->shift) & rh->mask; in rh_hash()
269 static struct dm_region *__rh_lookup(struct dm_region_hash *rh, region_t region) in __rh_lookup() argument
272 struct list_head *bucket = rh->buckets + rh_hash(rh, region); in __rh_lookup()
281 static void __rh_insert(struct dm_region_hash *rh, struct dm_region *reg) in __rh_insert() argument
283 list_add(®->hash_list, rh->buckets + rh_hash(rh, reg->key)); in __rh_insert()
286 static struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region) in __rh_alloc() argument
290 nreg = mempool_alloc(rh->region_pool, GFP_ATOMIC); in __rh_alloc()
294 nreg->state = rh->log->type->in_sync(rh->log, region, 1) ? in __rh_alloc()
296 nreg->rh = rh; in __rh_alloc()
302 write_lock_irq(&rh->hash_lock); in __rh_alloc()
303 reg = __rh_lookup(rh, region); in __rh_alloc()
306 mempool_free(nreg, rh->region_pool); in __rh_alloc()
308 __rh_insert(rh, nreg); in __rh_alloc()
310 spin_lock(&rh->region_lock); in __rh_alloc()
311 list_add(&nreg->list, &rh->clean_regions); in __rh_alloc()
312 spin_unlock(&rh->region_lock); in __rh_alloc()
317 write_unlock_irq(&rh->hash_lock); in __rh_alloc()
322 static struct dm_region *__rh_find(struct dm_region_hash *rh, region_t region) in __rh_find() argument
326 reg = __rh_lookup(rh, region); in __rh_find()
328 read_unlock(&rh->hash_lock); in __rh_find()
329 reg = __rh_alloc(rh, region); in __rh_find()
330 read_lock(&rh->hash_lock); in __rh_find()
336 int dm_rh_get_state(struct dm_region_hash *rh, region_t region, int may_block) in dm_rh_get_state() argument
341 read_lock(&rh->hash_lock); in dm_rh_get_state()
342 reg = __rh_lookup(rh, region); in dm_rh_get_state()
343 read_unlock(&rh->hash_lock); in dm_rh_get_state()
352 r = rh->log->type->in_sync(rh->log, region, may_block); in dm_rh_get_state()
364 struct dm_region_hash *rh = reg->rh; in complete_resync_work() local
366 rh->log->type->set_region_sync(rh->log, reg->key, success); in complete_resync_work()
377 rh->dispatch_bios(rh->context, ®->delayed_bios); in complete_resync_work()
378 if (atomic_dec_and_test(&rh->recovery_in_flight)) in complete_resync_work()
379 rh->wakeup_all_recovery_waiters(rh->context); in complete_resync_work()
380 up(&rh->recovery_count); in complete_resync_work()
393 void dm_rh_mark_nosync(struct dm_region_hash *rh, struct bio *bio) in dm_rh_mark_nosync() argument
396 struct dm_dirty_log *log = rh->log; in dm_rh_mark_nosync()
398 region_t region = dm_rh_bio_to_region(rh, bio); in dm_rh_mark_nosync()
402 rh->flush_failure = 1; in dm_rh_mark_nosync()
412 read_lock(&rh->hash_lock); in dm_rh_mark_nosync()
413 reg = __rh_find(rh, region); in dm_rh_mark_nosync()
414 read_unlock(&rh->hash_lock); in dm_rh_mark_nosync()
420 spin_lock_irqsave(&rh->region_lock, flags); in dm_rh_mark_nosync()
431 spin_unlock_irqrestore(&rh->region_lock, flags); in dm_rh_mark_nosync()
438 void dm_rh_update_states(struct dm_region_hash *rh, int errors_handled) in dm_rh_update_states() argument
449 write_lock_irq(&rh->hash_lock); in dm_rh_update_states()
450 spin_lock(&rh->region_lock); in dm_rh_update_states()
451 if (!list_empty(&rh->clean_regions)) { in dm_rh_update_states()
452 list_splice_init(&rh->clean_regions, &clean); in dm_rh_update_states()
458 if (!list_empty(&rh->recovered_regions)) { in dm_rh_update_states()
459 list_splice_init(&rh->recovered_regions, &recovered); in dm_rh_update_states()
465 if (!list_empty(&rh->failed_recovered_regions)) { in dm_rh_update_states()
466 list_splice_init(&rh->failed_recovered_regions, in dm_rh_update_states()
473 spin_unlock(&rh->region_lock); in dm_rh_update_states()
474 write_unlock_irq(&rh->hash_lock); in dm_rh_update_states()
482 rh->log->type->clear_region(rh->log, reg->key); in dm_rh_update_states()
484 mempool_free(reg, rh->region_pool); in dm_rh_update_states()
489 mempool_free(reg, rh->region_pool); in dm_rh_update_states()
493 rh->log->type->clear_region(rh->log, reg->key); in dm_rh_update_states()
494 mempool_free(reg, rh->region_pool); in dm_rh_update_states()
497 rh->log->type->flush(rh->log); in dm_rh_update_states()
501 static void rh_inc(struct dm_region_hash *rh, region_t region) in rh_inc() argument
505 read_lock(&rh->hash_lock); in rh_inc()
506 reg = __rh_find(rh, region); in rh_inc()
508 spin_lock_irq(&rh->region_lock); in rh_inc()
514 spin_unlock_irq(&rh->region_lock); in rh_inc()
516 rh->log->type->mark_region(rh->log, reg->key); in rh_inc()
518 spin_unlock_irq(&rh->region_lock); in rh_inc()
521 read_unlock(&rh->hash_lock); in rh_inc()
524 void dm_rh_inc_pending(struct dm_region_hash *rh, struct bio_list *bios) in dm_rh_inc_pending() argument
531 rh_inc(rh, dm_rh_bio_to_region(rh, bio)); in dm_rh_inc_pending()
536 void dm_rh_dec(struct dm_region_hash *rh, region_t region) in dm_rh_dec() argument
542 read_lock(&rh->hash_lock); in dm_rh_dec()
543 reg = __rh_lookup(rh, region); in dm_rh_dec()
544 read_unlock(&rh->hash_lock); in dm_rh_dec()
546 spin_lock_irqsave(&rh->region_lock, flags); in dm_rh_dec()
560 if (unlikely(rh->flush_failure)) { in dm_rh_dec()
568 list_add_tail(®->list, &rh->quiesced_regions); in dm_rh_dec()
571 list_add(®->list, &rh->clean_regions); in dm_rh_dec()
575 spin_unlock_irqrestore(&rh->region_lock, flags); in dm_rh_dec()
578 rh->wakeup_workers(rh->context); in dm_rh_dec()
585 static int __rh_recovery_prepare(struct dm_region_hash *rh) in __rh_recovery_prepare() argument
594 r = rh->log->type->get_resync_work(rh->log, ®ion); in __rh_recovery_prepare()
602 read_lock(&rh->hash_lock); in __rh_recovery_prepare()
603 reg = __rh_find(rh, region); in __rh_recovery_prepare()
604 read_unlock(&rh->hash_lock); in __rh_recovery_prepare()
606 spin_lock_irq(&rh->region_lock); in __rh_recovery_prepare()
613 list_move(®->list, &rh->quiesced_regions); in __rh_recovery_prepare()
615 spin_unlock_irq(&rh->region_lock); in __rh_recovery_prepare()
620 void dm_rh_recovery_prepare(struct dm_region_hash *rh) in dm_rh_recovery_prepare() argument
623 atomic_inc(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
625 while (!down_trylock(&rh->recovery_count)) { in dm_rh_recovery_prepare()
626 atomic_inc(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
627 if (__rh_recovery_prepare(rh) <= 0) { in dm_rh_recovery_prepare()
628 atomic_dec(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
629 up(&rh->recovery_count); in dm_rh_recovery_prepare()
635 if (atomic_dec_and_test(&rh->recovery_in_flight)) in dm_rh_recovery_prepare()
636 rh->wakeup_all_recovery_waiters(rh->context); in dm_rh_recovery_prepare()
643 struct dm_region *dm_rh_recovery_start(struct dm_region_hash *rh) in dm_rh_recovery_start() argument
647 spin_lock_irq(&rh->region_lock); in dm_rh_recovery_start()
648 if (!list_empty(&rh->quiesced_regions)) { in dm_rh_recovery_start()
649 reg = list_entry(rh->quiesced_regions.next, in dm_rh_recovery_start()
653 spin_unlock_irq(&rh->region_lock); in dm_rh_recovery_start()
661 struct dm_region_hash *rh = reg->rh; in dm_rh_recovery_end() local
663 spin_lock_irq(&rh->region_lock); in dm_rh_recovery_end()
665 list_add(®->list, ®->rh->recovered_regions); in dm_rh_recovery_end()
667 list_add(®->list, ®->rh->failed_recovered_regions); in dm_rh_recovery_end()
669 spin_unlock_irq(&rh->region_lock); in dm_rh_recovery_end()
671 rh->wakeup_workers(rh->context); in dm_rh_recovery_end()
676 int dm_rh_recovery_in_flight(struct dm_region_hash *rh) in dm_rh_recovery_in_flight() argument
678 return atomic_read(&rh->recovery_in_flight); in dm_rh_recovery_in_flight()
682 int dm_rh_flush(struct dm_region_hash *rh) in dm_rh_flush() argument
684 return rh->log->type->flush(rh->log); in dm_rh_flush()
688 void dm_rh_delay(struct dm_region_hash *rh, struct bio *bio) in dm_rh_delay() argument
692 read_lock(&rh->hash_lock); in dm_rh_delay()
693 reg = __rh_find(rh, dm_rh_bio_to_region(rh, bio)); in dm_rh_delay()
695 read_unlock(&rh->hash_lock); in dm_rh_delay()
699 void dm_rh_stop_recovery(struct dm_region_hash *rh) in dm_rh_stop_recovery() argument
704 for (i = 0; i < rh->max_recovery; i++) in dm_rh_stop_recovery()
705 down(&rh->recovery_count); in dm_rh_stop_recovery()
709 void dm_rh_start_recovery(struct dm_region_hash *rh) in dm_rh_start_recovery() argument
713 for (i = 0; i < rh->max_recovery; i++) in dm_rh_start_recovery()
714 up(&rh->recovery_count); in dm_rh_start_recovery()
716 rh->wakeup_workers(rh->context); in dm_rh_start_recovery()