74 uint8_t bch_inc_gen(struct cache *ca, struct bucket *b)  in bch_inc_gen()  argument
76 	uint8_t ret = ++b->gen;  in bch_inc_gen()
78 	ca->set->need_gc = max(ca->set->need_gc, bucket_gc_gen(b));  in bch_inc_gen()
87 	struct bucket *b;  in bch_rescale_priorities()  local
106 		for_each_bucket(b, ca)  in bch_rescale_priorities()
107 			if (b->prio &&  in bch_rescale_priorities()
108 			    b->prio != BTREE_PRIO &&  in bch_rescale_priorities()
109 			    !atomic_read(&b->pin)) {  in bch_rescale_priorities()
110 				b->prio--;  in bch_rescale_priorities()
111 				c->min_prio = min(c->min_prio, b->prio);  in bch_rescale_priorities()
124 static inline bool can_inc_bucket_gen(struct bucket *b)  in can_inc_bucket_gen()  argument
126 	return bucket_gc_gen(b) < BUCKET_GC_GEN_MAX;  in can_inc_bucket_gen()
129 bool bch_can_invalidate_bucket(struct cache *ca, struct bucket *b)  in bch_can_invalidate_bucket()  argument
133 	return (!GC_MARK(b) ||  in bch_can_invalidate_bucket()
134 		GC_MARK(b) == GC_MARK_RECLAIMABLE) &&  in bch_can_invalidate_bucket()
135 		!atomic_read(&b->pin) &&  in bch_can_invalidate_bucket()
136 		can_inc_bucket_gen(b);  in bch_can_invalidate_bucket()
139 void __bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)  in __bch_invalidate_one_bucket()  argument
142 	BUG_ON(GC_MARK(b) && GC_MARK(b) != GC_MARK_RECLAIMABLE);  in __bch_invalidate_one_bucket()
144 	if (GC_SECTORS_USED(b))  in __bch_invalidate_one_bucket()
145 		trace_bcache_invalidate(ca, b - ca->buckets);  in __bch_invalidate_one_bucket()
147 	bch_inc_gen(ca, b);  in __bch_invalidate_one_bucket()
148 	b->prio = INITIAL_PRIO;  in __bch_invalidate_one_bucket()
149 	atomic_inc(&b->pin);  in __bch_invalidate_one_bucket()
152 static void bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)  in bch_invalidate_one_bucket()  argument
154 	__bch_invalidate_one_bucket(ca, b);  in bch_invalidate_one_bucket()
156 	fifo_push(&ca->free_inc, b - ca->buckets);  in bch_invalidate_one_bucket()
168 #define bucket_prio(b)							\  argument
172 	(b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b);	\
180 	struct bucket *b;  in invalidate_buckets_lru()  local
185 	for_each_bucket(b, ca) {  in invalidate_buckets_lru()
186 		if (!bch_can_invalidate_bucket(ca, b))  in invalidate_buckets_lru()
190 			heap_add(&ca->heap, b, bucket_max_cmp);  in invalidate_buckets_lru()
191 		else if (bucket_max_cmp(b, heap_peek(&ca->heap))) {  in invalidate_buckets_lru()
192 			ca->heap.data[0] = b;  in invalidate_buckets_lru()
201 		if (!heap_pop(&ca->heap, b, bucket_min_cmp)) {  in invalidate_buckets_lru()
211 		bch_invalidate_one_bucket(ca, b);  in invalidate_buckets_lru()
217 	struct bucket *b;  in invalidate_buckets_fifo()  local
225 		b = ca->buckets + ca->fifo_last_bucket++;  in invalidate_buckets_fifo()
227 		if (bch_can_invalidate_bucket(ca, b))  in invalidate_buckets_fifo()
228 			bch_invalidate_one_bucket(ca, b);  in invalidate_buckets_fifo()
240 	struct bucket *b;  in invalidate_buckets_random()  local
250 		b = ca->buckets + n;  in invalidate_buckets_random()
252 		if (bch_can_invalidate_bucket(ca, b))  in invalidate_buckets_random()
253 			bch_invalidate_one_bucket(ca, b);  in invalidate_buckets_random()
384 	struct bucket *b;  in bch_bucket_alloc()  local
428 	b = ca->buckets + r;  in bch_bucket_alloc()
430 	BUG_ON(atomic_read(&b->pin) != 1);  in bch_bucket_alloc()
432 	SET_GC_SECTORS_USED(b, ca->sb.bucket_size);  in bch_bucket_alloc()
435 		SET_GC_MARK(b, GC_MARK_METADATA);  in bch_bucket_alloc()
436 		SET_GC_MOVE(b, 0);  in bch_bucket_alloc()
437 		b->prio = BTREE_PRIO;  in bch_bucket_alloc()
439 		SET_GC_MARK(b, GC_MARK_RECLAIMABLE);  in bch_bucket_alloc()
440 		SET_GC_MOVE(b, 0);  in bch_bucket_alloc()
441 		b->prio = INITIAL_PRIO;  in bch_bucket_alloc()
447 void __bch_bucket_free(struct cache *ca, struct bucket *b)  in __bch_bucket_free()  argument
449 	SET_GC_MARK(b, 0);  in __bch_bucket_free()
450 	SET_GC_SECTORS_USED(b, 0);  in __bch_bucket_free()
476 		long b = bch_bucket_alloc(ca, reserve, wait);  in __bch_bucket_alloc_set()  local
478 		if (b == -1)  in __bch_bucket_alloc_set()
481 		k->ptr[i] = PTR(ca->buckets[b].gen,  in __bch_bucket_alloc_set()
482 				bucket_to_sector(c, b),  in __bch_bucket_alloc_set()
574 	struct open_bucket *b;  in bch_alloc_sectors()  local
588 	while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {  in bch_alloc_sectors()
609 	for (i = 0; i < KEY_PTRS(&b->key); i++)  in bch_alloc_sectors()
610 		EBUG_ON(ptr_stale(c, &b->key, i));  in bch_alloc_sectors()
614 	for (i = 0; i < KEY_PTRS(&b->key); i++)  in bch_alloc_sectors()
615 		k->ptr[i] = b->key.ptr[i];  in bch_alloc_sectors()
617 	sectors = min(sectors, b->sectors_free);  in bch_alloc_sectors()
621 	SET_KEY_PTRS(k, KEY_PTRS(&b->key));  in bch_alloc_sectors()
627 	list_move_tail(&b->list, &c->data_buckets);  in bch_alloc_sectors()
628 	bkey_copy_key(&b->key, k);  in bch_alloc_sectors()
629 	b->last_write_point = write_point;  in bch_alloc_sectors()
631 	b->sectors_free	-= sectors;  in bch_alloc_sectors()
633 	for (i = 0; i < KEY_PTRS(&b->key); i++) {  in bch_alloc_sectors()
634 		SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);  in bch_alloc_sectors()
637 				&PTR_CACHE(c, &b->key, i)->sectors_written);  in bch_alloc_sectors()
640 	if (b->sectors_free < c->sb.block_size)  in bch_alloc_sectors()
641 		b->sectors_free = 0;  in bch_alloc_sectors()
648 	if (b->sectors_free)  in bch_alloc_sectors()
649 		for (i = 0; i < KEY_PTRS(&b->key); i++)  in bch_alloc_sectors()
650 			atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);  in bch_alloc_sectors()
660 	struct open_bucket *b;  in bch_open_buckets_free()  local
663 		b = list_first_entry(&c->data_buckets,  in bch_open_buckets_free()
665 		list_del(&b->list);  in bch_open_buckets_free()
666 		kfree(b);  in bch_open_buckets_free()
677 		struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);  in bch_open_buckets_alloc()  local
678 		if (!b)  in bch_open_buckets_alloc()
681 		list_add(&b->list, &c->data_buckets);  in bch_open_buckets_alloc()