40 static struct zram *zram_devices;
59 	struct zram *zram = dev_to_zram(d);				\
63 		(u64)atomic64_read(&zram->stats.name));			\
67 static inline bool init_done(struct zram *zram)  in init_done()  argument
69 	return zram->disksize;  in init_done()
72 static inline struct zram *dev_to_zram(struct device *dev)  in dev_to_zram()
74 	return (struct zram *)dev_to_disk(dev)->private_data;  in dev_to_zram()
81 	struct zram *zram = dev_to_zram(dev);  in compact_store()  local
84 	down_read(&zram->init_lock);  in compact_store()
85 	if (!init_done(zram)) {  in compact_store()
86 		up_read(&zram->init_lock);  in compact_store()
90 	meta = zram->meta;  in compact_store()
92 	atomic64_add(nr_migrated, &zram->stats.num_migrated);  in compact_store()
93 	up_read(&zram->init_lock);  in compact_store()
101 	struct zram *zram = dev_to_zram(dev);  in disksize_show()  local
103 	return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);  in disksize_show()
110 	struct zram *zram = dev_to_zram(dev);  in initstate_show()  local
112 	down_read(&zram->init_lock);  in initstate_show()
113 	val = init_done(zram);  in initstate_show()
114 	up_read(&zram->init_lock);  in initstate_show()
122 	struct zram *zram = dev_to_zram(dev);  in orig_data_size_show()  local
126 		(u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);  in orig_data_size_show()
133 	struct zram *zram = dev_to_zram(dev);  in mem_used_total_show()  local
136 	down_read(&zram->init_lock);  in mem_used_total_show()
137 	if (init_done(zram)) {  in mem_used_total_show()
138 		struct zram_meta *meta = zram->meta;  in mem_used_total_show()
141 	up_read(&zram->init_lock);  in mem_used_total_show()
150 	struct zram *zram = dev_to_zram(dev);  in max_comp_streams_show()  local
152 	down_read(&zram->init_lock);  in max_comp_streams_show()
153 	val = zram->max_comp_streams;  in max_comp_streams_show()
154 	up_read(&zram->init_lock);  in max_comp_streams_show()
163 	struct zram *zram = dev_to_zram(dev);  in mem_limit_show()  local
166 	down_read(&zram->init_lock);  in mem_limit_show()
167 	val = zram->limit_pages;  in mem_limit_show()
168 	up_read(&zram->init_lock);  in mem_limit_show()
178 	struct zram *zram = dev_to_zram(dev);  in mem_limit_store()  local
184 	down_write(&zram->init_lock);  in mem_limit_store()
185 	zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;  in mem_limit_store()
186 	up_write(&zram->init_lock);  in mem_limit_store()
195 	struct zram *zram = dev_to_zram(dev);  in mem_used_max_show()  local
198 	down_read(&zram->init_lock);  in mem_used_max_show()
199 	if (init_done(zram))  in mem_used_max_show()
200 		val = atomic_long_read(&zram->stats.max_used_pages);  in mem_used_max_show()
201 	up_read(&zram->init_lock);  in mem_used_max_show()
211 	struct zram *zram = dev_to_zram(dev);  in mem_used_max_store()  local
217 	down_read(&zram->init_lock);  in mem_used_max_store()
218 	if (init_done(zram)) {  in mem_used_max_store()
219 		struct zram_meta *meta = zram->meta;  in mem_used_max_store()
220 		atomic_long_set(&zram->stats.max_used_pages,  in mem_used_max_store()
223 	up_read(&zram->init_lock);  in mem_used_max_store()
232 	struct zram *zram = dev_to_zram(dev);  in max_comp_streams_store()  local
241 	down_write(&zram->init_lock);  in max_comp_streams_store()
242 	if (init_done(zram)) {  in max_comp_streams_store()
243 		if (!zcomp_set_max_streams(zram->comp, num)) {  in max_comp_streams_store()
250 	zram->max_comp_streams = num;  in max_comp_streams_store()
253 	up_write(&zram->init_lock);  in max_comp_streams_store()
261 	struct zram *zram = dev_to_zram(dev);  in comp_algorithm_show()  local
263 	down_read(&zram->init_lock);  in comp_algorithm_show()
264 	sz = zcomp_available_show(zram->compressor, buf);  in comp_algorithm_show()
265 	up_read(&zram->init_lock);  in comp_algorithm_show()
273 	struct zram *zram = dev_to_zram(dev);  in comp_algorithm_store()  local
274 	down_write(&zram->init_lock);  in comp_algorithm_store()
275 	if (init_done(zram)) {  in comp_algorithm_store()
276 		up_write(&zram->init_lock);  in comp_algorithm_store()
280 	strlcpy(zram->compressor, buf, sizeof(zram->compressor));  in comp_algorithm_store()
281 	up_write(&zram->init_lock);  in comp_algorithm_store()
325 static inline int valid_io_request(struct zram *zram,  in valid_io_request()  argument
337 	bound = zram->disksize >> SECTOR_SHIFT;  in valid_io_request()
397 static inline bool zram_meta_get(struct zram *zram)  in zram_meta_get()  argument
399 	if (atomic_inc_not_zero(&zram->refcount))  in zram_meta_get()
404 static inline void zram_meta_put(struct zram *zram)  in zram_meta_put()  argument
406 	atomic_dec(&zram->refcount);  in zram_meta_put()
452 static void zram_free_page(struct zram *zram, size_t index)  in zram_free_page()  argument
454 	struct zram_meta *meta = zram->meta;  in zram_free_page()
464 			atomic64_dec(&zram->stats.zero_pages);  in zram_free_page()
472 			&zram->stats.compr_data_size);  in zram_free_page()
473 	atomic64_dec(&zram->stats.pages_stored);  in zram_free_page()
479 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)  in zram_decompress_page()  argument
483 	struct zram_meta *meta = zram->meta;  in zram_decompress_page()
501 		ret = zcomp_decompress(zram->comp, cmem, size, mem);  in zram_decompress_page()
514 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,  in zram_bvec_read()  argument
520 	struct zram_meta *meta = zram->meta;  in zram_bvec_read()
546 	ret = zram_decompress_page(zram, uncmem, index);  in zram_bvec_read()
564 static inline void update_used_max(struct zram *zram,  in update_used_max()  argument
569 	old_max = atomic_long_read(&zram->stats.max_used_pages);  in update_used_max()
575 				&zram->stats.max_used_pages, cur_max, pages);  in update_used_max()
579 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,  in zram_bvec_write()  argument
587 	struct zram_meta *meta = zram->meta;  in zram_bvec_write()
603 		ret = zram_decompress_page(zram, uncmem, index);  in zram_bvec_write()
608 	zstrm = zcomp_strm_find(zram->comp);  in zram_bvec_write()
626 		zram_free_page(zram, index);  in zram_bvec_write()
630 		atomic64_inc(&zram->stats.zero_pages);  in zram_bvec_write()
635 	ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);  in zram_bvec_write()
662 	if (zram->limit_pages && alloced_pages > zram->limit_pages) {  in zram_bvec_write()
668 	update_used_max(zram, alloced_pages);  in zram_bvec_write()
680 	zcomp_strm_release(zram->comp, zstrm);  in zram_bvec_write()
689 	zram_free_page(zram, index);  in zram_bvec_write()
696 	atomic64_add(clen, &zram->stats.compr_data_size);  in zram_bvec_write()
697 	atomic64_inc(&zram->stats.pages_stored);  in zram_bvec_write()
700 		zcomp_strm_release(zram->comp, zstrm);  in zram_bvec_write()
706 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,  in zram_bvec_rw()  argument
713 			&zram->disk->part0);  in zram_bvec_rw()
716 		atomic64_inc(&zram->stats.num_reads);  in zram_bvec_rw()
717 		ret = zram_bvec_read(zram, bvec, index, offset);  in zram_bvec_rw()
719 		atomic64_inc(&zram->stats.num_writes);  in zram_bvec_rw()
720 		ret = zram_bvec_write(zram, bvec, index, offset);  in zram_bvec_rw()
723 	generic_end_io_acct(rw, &zram->disk->part0, start_time);  in zram_bvec_rw()
727 			atomic64_inc(&zram->stats.failed_reads);  in zram_bvec_rw()
729 			atomic64_inc(&zram->stats.failed_writes);  in zram_bvec_rw()
740 static void zram_bio_discard(struct zram *zram, u32 index,  in zram_bio_discard()  argument
744 	struct zram_meta *meta = zram->meta;  in zram_bio_discard()
766 		zram_free_page(zram, index);  in zram_bio_discard()
768 		atomic64_inc(&zram->stats.notify_free);  in zram_bio_discard()
774 static void zram_reset_device(struct zram *zram)  in zram_reset_device()  argument
780 	down_write(&zram->init_lock);  in zram_reset_device()
782 	zram->limit_pages = 0;  in zram_reset_device()
784 	if (!init_done(zram)) {  in zram_reset_device()
785 		up_write(&zram->init_lock);  in zram_reset_device()
789 	meta = zram->meta;  in zram_reset_device()
790 	comp = zram->comp;  in zram_reset_device()
791 	disksize = zram->disksize;  in zram_reset_device()
797 	zram_meta_put(zram);  in zram_reset_device()
802 	wait_event(zram->io_done, atomic_read(&zram->refcount) == 0);  in zram_reset_device()
805 	memset(&zram->stats, 0, sizeof(zram->stats));  in zram_reset_device()
806 	zram->disksize = 0;  in zram_reset_device()
807 	zram->max_comp_streams = 1;  in zram_reset_device()
809 	set_capacity(zram->disk, 0);  in zram_reset_device()
810 	part_stat_set_all(&zram->disk->part0, 0);  in zram_reset_device()
812 	up_write(&zram->init_lock);  in zram_reset_device()
824 	struct zram *zram = dev_to_zram(dev);  in disksize_store()  local
832 	meta = zram_meta_alloc(zram->disk->first_minor, disksize);  in disksize_store()
836 	comp = zcomp_create(zram->compressor, zram->max_comp_streams);  in disksize_store()
839 				zram->compressor);  in disksize_store()
844 	down_write(&zram->init_lock);  in disksize_store()
845 	if (init_done(zram)) {  in disksize_store()
851 	init_waitqueue_head(&zram->io_done);  in disksize_store()
852 	atomic_set(&zram->refcount, 1);  in disksize_store()
853 	zram->meta = meta;  in disksize_store()
854 	zram->comp = comp;  in disksize_store()
855 	zram->disksize = disksize;  in disksize_store()
856 	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);  in disksize_store()
857 	up_write(&zram->init_lock);  in disksize_store()
864 	revalidate_disk(zram->disk);  in disksize_store()
869 	up_write(&zram->init_lock);  in disksize_store()
881 	struct zram *zram;  in reset_store()  local
884 	zram = dev_to_zram(dev);  in reset_store()
885 	bdev = bdget_disk(zram->disk, 0);  in reset_store()
908 	zram_reset_device(zram);  in reset_store()
911 	revalidate_disk(zram->disk);  in reset_store()
922 static void __zram_make_request(struct zram *zram, struct bio *bio)  in __zram_make_request()  argument
934 		zram_bio_discard(zram, index, offset, bio);  in __zram_make_request()
954 			if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0)  in __zram_make_request()
959 			if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0)  in __zram_make_request()
962 			if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0)  in __zram_make_request()
981 	struct zram *zram = queue->queuedata;  in zram_make_request()  local
983 	if (unlikely(!zram_meta_get(zram)))  in zram_make_request()
986 	if (!valid_io_request(zram, bio->bi_iter.bi_sector,  in zram_make_request()
988 		atomic64_inc(&zram->stats.invalid_io);  in zram_make_request()
992 	__zram_make_request(zram, bio);  in zram_make_request()
993 	zram_meta_put(zram);  in zram_make_request()
996 	zram_meta_put(zram);  in zram_make_request()
1004 	struct zram *zram;  in zram_slot_free_notify()  local
1007 	zram = bdev->bd_disk->private_data;  in zram_slot_free_notify()
1008 	meta = zram->meta;  in zram_slot_free_notify()
1011 	zram_free_page(zram, index);  in zram_slot_free_notify()
1013 	atomic64_inc(&zram->stats.notify_free);  in zram_slot_free_notify()
1021 	struct zram *zram;  in zram_rw_page()  local
1024 	zram = bdev->bd_disk->private_data;  in zram_rw_page()
1025 	if (unlikely(!zram_meta_get(zram)))  in zram_rw_page()
1028 	if (!valid_io_request(zram, sector, PAGE_SIZE)) {  in zram_rw_page()
1029 		atomic64_inc(&zram->stats.invalid_io);  in zram_rw_page()
1041 	err = zram_bvec_rw(zram, &bv, index, offset, rw);  in zram_rw_page()
1043 	zram_meta_put(zram);  in zram_rw_page()
1078 	struct zram *zram = dev_to_zram(dev);  in io_stat_show()  local
1081 	down_read(&zram->init_lock);  in io_stat_show()
1084 			(u64)atomic64_read(&zram->stats.failed_reads),  in io_stat_show()
1085 			(u64)atomic64_read(&zram->stats.failed_writes),  in io_stat_show()
1086 			(u64)atomic64_read(&zram->stats.invalid_io),  in io_stat_show()
1087 			(u64)atomic64_read(&zram->stats.notify_free));  in io_stat_show()
1088 	up_read(&zram->init_lock);  in io_stat_show()
1096 	struct zram *zram = dev_to_zram(dev);  in mm_stat_show()  local
1101 	down_read(&zram->init_lock);  in mm_stat_show()
1102 	if (init_done(zram))  in mm_stat_show()
1103 		mem_used = zs_get_total_pages(zram->meta->mem_pool);  in mm_stat_show()
1105 	orig_size = atomic64_read(&zram->stats.pages_stored);  in mm_stat_show()
1106 	max_used = atomic_long_read(&zram->stats.max_used_pages);  in mm_stat_show()
1111 			(u64)atomic64_read(&zram->stats.compr_data_size),  in mm_stat_show()
1113 			zram->limit_pages << PAGE_SHIFT,  in mm_stat_show()
1115 			(u64)atomic64_read(&zram->stats.zero_pages),  in mm_stat_show()
1116 			(u64)atomic64_read(&zram->stats.num_migrated));  in mm_stat_show()
1117 	up_read(&zram->init_lock);  in mm_stat_show()
1161 static int create_device(struct zram *zram, int device_id)  in create_device()  argument
1166 	init_rwsem(&zram->init_lock);  in create_device()
1178 	zram->disk = alloc_disk(1);  in create_device()
1179 	if (!zram->disk) {  in create_device()
1186 	zram->disk->major = zram_major;  in create_device()
1187 	zram->disk->first_minor = device_id;  in create_device()
1188 	zram->disk->fops = &zram_devops;  in create_device()
1189 	zram->disk->queue = queue;  in create_device()
1190 	zram->disk->queue->queuedata = zram;  in create_device()
1191 	zram->disk->private_data = zram;  in create_device()
1192 	snprintf(zram->disk->disk_name, 16, "zram%d", device_id);  in create_device()
1195 	set_capacity(zram->disk, 0);  in create_device()
1197 	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);  in create_device()
1198 	queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);  in create_device()
1203 	blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);  in create_device()
1204 	blk_queue_logical_block_size(zram->disk->queue,  in create_device()
1206 	blk_queue_io_min(zram->disk->queue, PAGE_SIZE);  in create_device()
1207 	blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);  in create_device()
1208 	zram->disk->queue->limits.discard_granularity = PAGE_SIZE;  in create_device()
1209 	zram->disk->queue->limits.max_discard_sectors = UINT_MAX;  in create_device()
1219 		zram->disk->queue->limits.discard_zeroes_data = 1;  in create_device()
1221 		zram->disk->queue->limits.discard_zeroes_data = 0;  in create_device()
1222 	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);  in create_device()
1224 	add_disk(zram->disk);  in create_device()
1226 	ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,  in create_device()
1232 	strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));  in create_device()
1233 	zram->meta = NULL;  in create_device()
1234 	zram->max_comp_streams = 1;  in create_device()
1238 	del_gendisk(zram->disk);  in create_device()
1239 	put_disk(zram->disk);  in create_device()
1248 	struct zram *zram;  in destroy_devices()  local
1252 		zram = &zram_devices[i];  in destroy_devices()
1257 		sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,  in destroy_devices()
1260 		zram_reset_device(zram);  in destroy_devices()
1262 		blk_cleanup_queue(zram->disk->queue);  in destroy_devices()
1263 		del_gendisk(zram->disk);  in destroy_devices()
1264 		put_disk(zram->disk);  in destroy_devices()
1289 	zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);  in zram_init()