66 static struct policy *to_policy(struct dm_cache_policy *p)  in to_policy()  argument
68 	return container_of(p, struct policy, policy);  in to_policy()
97 static int alloc_cache_blocks_with_hash(struct policy *p, dm_cblock_t cache_size)  in alloc_cache_blocks_with_hash()  argument
101 	p->cblocks = vzalloc(sizeof(*p->cblocks) * from_cblock(cache_size));  in alloc_cache_blocks_with_hash()
102 	if (p->cblocks) {  in alloc_cache_blocks_with_hash()
106 			list_add(&p->cblocks[u].list, &p->free);  in alloc_cache_blocks_with_hash()
108 		p->nr_cblocks_allocated = 0;  in alloc_cache_blocks_with_hash()
111 		r = alloc_hash(&p->chash, from_cblock(cache_size));  in alloc_cache_blocks_with_hash()
113 			vfree(p->cblocks);  in alloc_cache_blocks_with_hash()
119 static void free_cache_blocks_and_hash(struct policy *p)  in free_cache_blocks_and_hash()  argument
121 	free_hash(&p->chash);  in free_cache_blocks_and_hash()
122 	vfree(p->cblocks);  in free_cache_blocks_and_hash()
125 static struct wb_cache_entry *alloc_cache_entry(struct policy *p)  in alloc_cache_entry()  argument
129 	BUG_ON(from_cblock(p->nr_cblocks_allocated) >= from_cblock(p->cache_size));  in alloc_cache_entry()
131 	e = list_entry(list_pop(&p->free), struct wb_cache_entry, list);  in alloc_cache_entry()
132 	p->nr_cblocks_allocated = to_cblock(from_cblock(p->nr_cblocks_allocated) + 1);  in alloc_cache_entry()
140 static struct wb_cache_entry *lookup_cache_entry(struct policy *p, dm_oblock_t oblock)  in lookup_cache_entry()  argument
142 	struct hash *hash = &p->chash;  in lookup_cache_entry()
159 static void insert_cache_hash_entry(struct policy *p, struct wb_cache_entry *e)  in insert_cache_hash_entry()  argument
161 	unsigned h = hash_64(from_oblock(e->oblock), p->chash.hash_bits);  in insert_cache_hash_entry()
163 	hlist_add_head(&e->hlist, &p->chash.table[h]);  in insert_cache_hash_entry()
177 	struct policy *p = to_policy(pe);  in wb_map()  local
184 		spin_lock_irqsave(&p->lock, flags);  in wb_map()
186 	else if (!spin_trylock_irqsave(&p->lock, flags))  in wb_map()
189 	e = lookup_cache_entry(p, oblock);  in wb_map()
196 	spin_unlock_irqrestore(&p->lock, flags);  in wb_map()
204 	struct policy *p = to_policy(pe);  in wb_lookup()  local
208 	if (!spin_trylock_irqsave(&p->lock, flags))  in wb_lookup()
211 	e = lookup_cache_entry(p, oblock);  in wb_lookup()
219 	spin_unlock_irqrestore(&p->lock, flags);  in wb_lookup()
226 	struct policy *p = to_policy(pe);  in __set_clear_dirty()  local
229 	e = lookup_cache_entry(p, oblock);  in __set_clear_dirty()
235 			list_move(&e->list, &p->dirty);  in __set_clear_dirty()
242 			list_move(&e->list, &p->clean);  in __set_clear_dirty()
249 	struct policy *p = to_policy(pe);  in wb_set_dirty()  local
252 	spin_lock_irqsave(&p->lock, flags);  in wb_set_dirty()
254 	spin_unlock_irqrestore(&p->lock, flags);  in wb_set_dirty()
259 	struct policy *p = to_policy(pe);  in wb_clear_dirty()  local
262 	spin_lock_irqsave(&p->lock, flags);  in wb_clear_dirty()
264 	spin_unlock_irqrestore(&p->lock, flags);  in wb_clear_dirty()
267 static void add_cache_entry(struct policy *p, struct wb_cache_entry *e)  in add_cache_entry()  argument
269 	insert_cache_hash_entry(p, e);  in add_cache_entry()
271 		list_add(&e->list, &p->dirty);  in add_cache_entry()
273 		list_add(&e->list, &p->clean);  in add_cache_entry()
281 	struct policy *p = to_policy(pe);  in wb_load_mapping()  local
282 	struct wb_cache_entry *e = alloc_cache_entry(p);  in wb_load_mapping()
288 		add_cache_entry(p, e);  in wb_load_mapping()
299 	struct policy *p = to_policy(pe);  in wb_destroy()  local
301 	free_cache_blocks_and_hash(p);  in wb_destroy()
302 	kfree(p);  in wb_destroy()
305 static struct wb_cache_entry *__wb_force_remove_mapping(struct policy *p, dm_oblock_t oblock)  in __wb_force_remove_mapping()  argument
307 	struct wb_cache_entry *r = lookup_cache_entry(p, oblock);  in __wb_force_remove_mapping()
319 	struct policy *p = to_policy(pe);  in wb_remove_mapping()  local
323 	spin_lock_irqsave(&p->lock, flags);  in wb_remove_mapping()
324 	e = __wb_force_remove_mapping(p, oblock);  in wb_remove_mapping()
325 	list_add_tail(&e->list, &p->free);  in wb_remove_mapping()
326 	BUG_ON(!from_cblock(p->nr_cblocks_allocated));  in wb_remove_mapping()
327 	p->nr_cblocks_allocated = to_cblock(from_cblock(p->nr_cblocks_allocated) - 1);  in wb_remove_mapping()
328 	spin_unlock_irqrestore(&p->lock, flags);  in wb_remove_mapping()
334 	struct policy *p = to_policy(pe);  in wb_force_mapping()  local
338 	spin_lock_irqsave(&p->lock, flags);  in wb_force_mapping()
339 	e = __wb_force_remove_mapping(p, current_oblock);  in wb_force_mapping()
341 	add_cache_entry(p, e);  in wb_force_mapping()
342 	spin_unlock_irqrestore(&p->lock, flags);  in wb_force_mapping()
345 static struct wb_cache_entry *get_next_dirty_entry(struct policy *p)  in get_next_dirty_entry()  argument
350 	if (list_empty(&p->dirty))  in get_next_dirty_entry()
353 	l = list_pop(&p->dirty);  in get_next_dirty_entry()
355 	list_add(l, &p->clean_pending);  in get_next_dirty_entry()
365 	struct policy *p = to_policy(pe);  in wb_writeback_work()  local
369 	spin_lock_irqsave(&p->lock, flags);  in wb_writeback_work()
371 	e = get_next_dirty_entry(p);  in wb_writeback_work()
378 	spin_unlock_irqrestore(&p->lock, flags);  in wb_writeback_work()
389 static void init_policy_functions(struct policy *p)  in init_policy_functions()  argument
391 	p->policy.destroy = wb_destroy;  in init_policy_functions()
392 	p->policy.map = wb_map;  in init_policy_functions()
393 	p->policy.lookup = wb_lookup;  in init_policy_functions()
394 	p->policy.set_dirty = wb_set_dirty;  in init_policy_functions()
395 	p->policy.clear_dirty = wb_clear_dirty;  in init_policy_functions()
396 	p->policy.load_mapping = wb_load_mapping;  in init_policy_functions()
397 	p->policy.walk_mappings = NULL;  in init_policy_functions()
398 	p->policy.remove_mapping = wb_remove_mapping;  in init_policy_functions()
399 	p->policy.writeback_work = wb_writeback_work;  in init_policy_functions()
400 	p->policy.force_mapping = wb_force_mapping;  in init_policy_functions()
401 	p->policy.residency = wb_residency;  in init_policy_functions()
402 	p->policy.tick = NULL;  in init_policy_functions()
410 	struct policy *p = kzalloc(sizeof(*p), GFP_KERNEL);  in wb_create()  local
412 	if (!p)  in wb_create()
415 	init_policy_functions(p);  in wb_create()
416 	INIT_LIST_HEAD(&p->free);  in wb_create()
417 	INIT_LIST_HEAD(&p->clean);  in wb_create()
418 	INIT_LIST_HEAD(&p->clean_pending);  in wb_create()
419 	INIT_LIST_HEAD(&p->dirty);  in wb_create()
421 	p->cache_size = cache_size;  in wb_create()
422 	spin_lock_init(&p->lock);  in wb_create()
425 	r = alloc_cache_blocks_with_hash(p, cache_size);  in wb_create()
427 		return &p->policy;  in wb_create()
429 	kfree(p);  in wb_create()