78 static int zram_test_flag(struct zram_meta *meta, u32 index,  in zram_test_flag()  argument
81 	return meta->table[index].value & BIT(flag);  in zram_test_flag()
84 static void zram_set_flag(struct zram_meta *meta, u32 index,  in zram_set_flag()  argument
87 	meta->table[index].value |= BIT(flag);  in zram_set_flag()
90 static void zram_clear_flag(struct zram_meta *meta, u32 index,  in zram_clear_flag()  argument
93 	meta->table[index].value &= ~BIT(flag);  in zram_clear_flag()
96 static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)  in zram_get_obj_size()  argument
98 	return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);  in zram_get_obj_size()
102 					u32 index, size_t size)  in zram_set_obj_size()  argument
104 	unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;  in zram_set_obj_size()
106 	meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;  in zram_set_obj_size()
138 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)  in update_position()  argument
141 		(*index)++;  in update_position()
485 	size_t index;  in zram_meta_free()  local
488 	for (index = 0; index < num_pages; index++) {  in zram_meta_free()
489 		unsigned long handle = meta->table[index].handle;  in zram_meta_free()
536 static void zram_free_page(struct zram *zram, size_t index)  in zram_free_page()  argument
539 	unsigned long handle = meta->table[index].handle;  in zram_free_page()
546 		if (zram_test_flag(meta, index, ZRAM_ZERO)) {  in zram_free_page()
547 			zram_clear_flag(meta, index, ZRAM_ZERO);  in zram_free_page()
555 	atomic64_sub(zram_get_obj_size(meta, index),  in zram_free_page()
559 	meta->table[index].handle = 0;  in zram_free_page()
560 	zram_set_obj_size(meta, index, 0);  in zram_free_page()
563 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)  in zram_decompress_page()  argument
571 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_decompress_page()
572 	handle = meta->table[index].handle;  in zram_decompress_page()
573 	size = zram_get_obj_size(meta, index);  in zram_decompress_page()
575 	if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {  in zram_decompress_page()
576 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_decompress_page()
587 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_decompress_page()
591 		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);  in zram_decompress_page()
599 			  u32 index, int offset)  in zram_bvec_read()  argument
607 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_read()
608 	if (unlikely(!meta->table[index].handle) ||  in zram_bvec_read()
609 			zram_test_flag(meta, index, ZRAM_ZERO)) {  in zram_bvec_read()
610 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_read()
614 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_read()
630 	ret = zram_decompress_page(zram, uncmem, index);  in zram_bvec_read()
648 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,  in zram_bvec_write()  argument
671 		ret = zram_decompress_page(zram, uncmem, index);  in zram_bvec_write()
692 		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
693 		zram_free_page(zram, index);  in zram_bvec_write()
694 		zram_set_flag(meta, index, ZRAM_ZERO);  in zram_bvec_write()
695 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
723 			index, clen);  in zram_bvec_write()
755 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
756 	zram_free_page(zram, index);  in zram_bvec_write()
758 	meta->table[index].handle = handle;  in zram_bvec_write()
759 	zram_set_obj_size(meta, index, clen);  in zram_bvec_write()
760 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
778 static void zram_bio_discard(struct zram *zram, u32 index,  in zram_bio_discard()  argument
799 		index++;  in zram_bio_discard()
803 		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bio_discard()
804 		zram_free_page(zram, index);  in zram_bio_discard()
805 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bio_discard()
807 		index++;  in zram_bio_discard()
812 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,  in zram_bvec_rw()  argument
823 		ret = zram_bvec_read(zram, bvec, index, offset);  in zram_bvec_rw()
826 		ret = zram_bvec_write(zram, bvec, index, offset);  in zram_bvec_rw()
844 	u32 index;  in __zram_make_request()  local
848 	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;  in __zram_make_request()
853 		zram_bio_discard(zram, index, offset, bio);  in __zram_make_request()
873 			if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0)  in __zram_make_request()
878 			if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0)  in __zram_make_request()
881 			if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0)  in __zram_make_request()
884 		update_position(&index, &offset, &bvec);  in __zram_make_request()
923 				unsigned long index)  in zram_slot_free_notify()  argument
931 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_slot_free_notify()
932 	zram_free_page(zram, index);  in zram_slot_free_notify()
933 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_slot_free_notify()
941 	u32 index;  in zram_rw_page()  local
955 	index = sector >> SECTORS_PER_PAGE_SHIFT;  in zram_rw_page()
962 	err = zram_bvec_rw(zram, &bv, index, offset, rw);  in zram_rw_page()