287 static int create_handle_cache(struct zs_pool *pool)  in create_handle_cache()  argument
289 	pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,  in create_handle_cache()
291 	return pool->handle_cachep ? 0 : 1;  in create_handle_cache()
294 static void destroy_handle_cache(struct zs_pool *pool)  in destroy_handle_cache()  argument
296 	kmem_cache_destroy(pool->handle_cachep);  in destroy_handle_cache()
299 static unsigned long alloc_handle(struct zs_pool *pool)  in alloc_handle()  argument
301 	return (unsigned long)kmem_cache_alloc(pool->handle_cachep,  in alloc_handle()
302 		pool->flags & ~__GFP_HIGHMEM);  in alloc_handle()
305 static void free_handle(struct zs_pool *pool, unsigned long handle)  in free_handle()  argument
307 	kmem_cache_free(pool->handle_cachep, (void *)handle);  in free_handle()
331 static void zs_zpool_destroy(void *pool)  in zs_zpool_destroy()  argument
333 	zs_destroy_pool(pool);  in zs_zpool_destroy()
336 static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp,  in zs_zpool_malloc()  argument
339 	*handle = zs_malloc(pool, size);  in zs_zpool_malloc()
342 static void zs_zpool_free(void *pool, unsigned long handle)  in zs_zpool_free()  argument
344 	zs_free(pool, handle);  in zs_zpool_free()
347 static int zs_zpool_shrink(void *pool, unsigned int pages,  in zs_zpool_shrink()  argument
353 static void *zs_zpool_map(void *pool, unsigned long handle,  in zs_zpool_map()  argument
371 	return zs_map_object(pool, handle, zs_mm);  in zs_zpool_map()
373 static void zs_zpool_unmap(void *pool, unsigned long handle)  in zs_zpool_unmap()  argument
375 	zs_unmap_object(pool, handle);  in zs_zpool_unmap()
378 static u64 zs_zpool_total_size(void *pool)  in zs_zpool_total_size()  argument
380 	return zs_get_total_pages(pool) << PAGE_SHIFT;  in zs_zpool_total_size()
501 	struct zs_pool *pool = s->private;  in zs_stats_size_show()  local
515 		class = pool->size_class[i];  in zs_stats_size_show()
565 static int zs_pool_stat_create(const char *name, struct zs_pool *pool)  in zs_pool_stat_create()  argument
577 	pool->stat_dentry = entry;  in zs_pool_stat_create()
580 			pool->stat_dentry, pool, &zs_stat_size_ops);  in zs_pool_stat_create()
590 static void zs_pool_stat_destroy(struct zs_pool *pool)  in zs_pool_stat_destroy()  argument
592 	debugfs_remove_recursive(pool->stat_dentry);  in zs_pool_stat_destroy()
605 static inline int zs_pool_stat_create(const char *name, struct zs_pool *pool)  in zs_pool_stat_create()  argument
610 static inline void zs_pool_stat_destroy(struct zs_pool *pool)  in zs_pool_stat_destroy()  argument
1241 unsigned long zs_get_total_pages(struct zs_pool *pool)  in zs_get_total_pages()  argument
1243 	return atomic_long_read(&pool->pages_allocated);  in zs_get_total_pages()
1261 void *zs_map_object(struct zs_pool *pool, unsigned long handle,  in zs_map_object()  argument
1289 	class = pool->size_class[class_idx];  in zs_map_object()
1315 void zs_unmap_object(struct zs_pool *pool, unsigned long handle)  in zs_unmap_object()  argument
1330 	class = pool->size_class[class_idx];  in zs_unmap_object()
1391 unsigned long zs_malloc(struct zs_pool *pool, size_t size)  in zs_malloc()  argument
1400 	handle = alloc_handle(pool);  in zs_malloc()
1406 	class = pool->size_class[get_size_class_index(size)];  in zs_malloc()
1413 		first_page = alloc_zspage(class, pool->flags);  in zs_malloc()
1415 			free_handle(pool, handle);  in zs_malloc()
1421 					&pool->pages_allocated);  in zs_malloc()
1438 static void obj_free(struct zs_pool *pool, struct size_class *class,  in obj_free()  argument
1467 void zs_free(struct zs_pool *pool, unsigned long handle)  in zs_free()  argument
1484 	class = pool->size_class[class_idx];  in zs_free()
1487 	obj_free(pool, class, obj);  in zs_free()
1493 				&pool->pages_allocated);  in zs_free()
1499 	free_handle(pool, handle);  in zs_free()
1612 static int migrate_zspage(struct zs_pool *pool, struct size_class *class,  in migrate_zspage()  argument
1652 		obj_free(pool, class, used_obj);  in migrate_zspage()
1686 static enum fullness_group putback_zspage(struct zs_pool *pool,  in putback_zspage()  argument
1702 				&pool->pages_allocated);  in putback_zspage()
1748 static void __zs_compact(struct zs_pool *pool, struct size_class *class)  in __zs_compact()  argument
1771 			if (!migrate_zspage(pool, class, &cc))  in __zs_compact()
1774 			putback_zspage(pool, class, dst_page);  in __zs_compact()
1781 		putback_zspage(pool, class, dst_page);  in __zs_compact()
1782 		if (putback_zspage(pool, class, src_page) == ZS_EMPTY)  in __zs_compact()
1783 			pool->stats.pages_compacted += class->pages_per_zspage;  in __zs_compact()
1790 		putback_zspage(pool, class, src_page);  in __zs_compact()
1795 unsigned long zs_compact(struct zs_pool *pool)  in zs_compact()  argument
1801 		class = pool->size_class[i];  in zs_compact()
1806 		__zs_compact(pool, class);  in zs_compact()
1809 	return pool->stats.pages_compacted;  in zs_compact()
1813 void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats)  in zs_pool_stats()  argument
1815 	memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats));  in zs_pool_stats()
1823 	struct zs_pool *pool = container_of(shrinker, struct zs_pool,  in zs_shrinker_scan()  local
1826 	pages_freed = pool->stats.pages_compacted;  in zs_shrinker_scan()
1832 	pages_freed = zs_compact(pool) - pages_freed;  in zs_shrinker_scan()
1843 	struct zs_pool *pool = container_of(shrinker, struct zs_pool,  in zs_shrinker_count()  local
1847 		class = pool->size_class[i];  in zs_shrinker_count()
1859 static void zs_unregister_shrinker(struct zs_pool *pool)  in zs_unregister_shrinker()  argument
1861 	if (pool->shrinker_enabled) {  in zs_unregister_shrinker()
1862 		unregister_shrinker(&pool->shrinker);  in zs_unregister_shrinker()
1863 		pool->shrinker_enabled = false;  in zs_unregister_shrinker()
1867 static int zs_register_shrinker(struct zs_pool *pool)  in zs_register_shrinker()  argument
1869 	pool->shrinker.scan_objects = zs_shrinker_scan;  in zs_register_shrinker()
1870 	pool->shrinker.count_objects = zs_shrinker_count;  in zs_register_shrinker()
1871 	pool->shrinker.batch = 0;  in zs_register_shrinker()
1872 	pool->shrinker.seeks = DEFAULT_SEEKS;  in zs_register_shrinker()
1874 	return register_shrinker(&pool->shrinker);  in zs_register_shrinker()
1890 	struct zs_pool *pool;  in zs_create_pool()  local
1893 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);  in zs_create_pool()
1894 	if (!pool)  in zs_create_pool()
1897 	pool->size_class = kcalloc(zs_size_classes, sizeof(struct size_class *),  in zs_create_pool()
1899 	if (!pool->size_class) {  in zs_create_pool()
1900 		kfree(pool);  in zs_create_pool()
1904 	pool->name = kstrdup(name, GFP_KERNEL);  in zs_create_pool()
1905 	if (!pool->name)  in zs_create_pool()
1908 	if (create_handle_cache(pool))  in zs_create_pool()
1936 				pool->size_class[i] = prev_class;  in zs_create_pool()
1952 		pool->size_class[i] = class;  in zs_create_pool()
1957 	pool->flags = flags;  in zs_create_pool()
1959 	if (zs_pool_stat_create(name, pool))  in zs_create_pool()
1966 	if (zs_register_shrinker(pool) == 0)  in zs_create_pool()
1967 		pool->shrinker_enabled = true;  in zs_create_pool()
1968 	return pool;  in zs_create_pool()
1971 	zs_destroy_pool(pool);  in zs_create_pool()
1976 void zs_destroy_pool(struct zs_pool *pool)  in zs_destroy_pool()  argument
1980 	zs_unregister_shrinker(pool);  in zs_destroy_pool()
1981 	zs_pool_stat_destroy(pool);  in zs_destroy_pool()
1985 		struct size_class *class = pool->size_class[i];  in zs_destroy_pool()
2002 	destroy_handle_cache(pool);  in zs_destroy_pool()
2003 	kfree(pool->size_class);  in zs_destroy_pool()
2004 	kfree(pool->name);  in zs_destroy_pool()
2005 	kfree(pool);  in zs_destroy_pool()