286 static int zram_test_flag(struct zram_meta *meta, u32 index,  in zram_test_flag()  argument
289 	return meta->table[index].value & BIT(flag);  in zram_test_flag()
292 static void zram_set_flag(struct zram_meta *meta, u32 index,  in zram_set_flag()  argument
295 	meta->table[index].value |= BIT(flag);  in zram_set_flag()
298 static void zram_clear_flag(struct zram_meta *meta, u32 index,  in zram_clear_flag()  argument
301 	meta->table[index].value &= ~BIT(flag);  in zram_clear_flag()
304 static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)  in zram_get_obj_size()  argument
306 	return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);  in zram_get_obj_size()
310 					u32 index, size_t size)  in zram_set_obj_size()  argument
312 	unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;  in zram_set_obj_size()
314 	meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;  in zram_set_obj_size()
349 	size_t index;  in zram_meta_free()  local
352 	for (index = 0; index < num_pages; index++) {  in zram_meta_free()
353 		unsigned long handle = meta->table[index].handle;  in zram_meta_free()
409 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)  in update_position()  argument
412 		(*index)++;  in update_position()
452 static void zram_free_page(struct zram *zram, size_t index)  in zram_free_page()  argument
455 	unsigned long handle = meta->table[index].handle;  in zram_free_page()
462 		if (zram_test_flag(meta, index, ZRAM_ZERO)) {  in zram_free_page()
463 			zram_clear_flag(meta, index, ZRAM_ZERO);  in zram_free_page()
471 	atomic64_sub(zram_get_obj_size(meta, index),  in zram_free_page()
475 	meta->table[index].handle = 0;  in zram_free_page()
476 	zram_set_obj_size(meta, index, 0);  in zram_free_page()
479 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)  in zram_decompress_page()  argument
487 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_decompress_page()
488 	handle = meta->table[index].handle;  in zram_decompress_page()
489 	size = zram_get_obj_size(meta, index);  in zram_decompress_page()
491 	if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {  in zram_decompress_page()
492 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_decompress_page()
503 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_decompress_page()
507 		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);  in zram_decompress_page()
515 			  u32 index, int offset)  in zram_bvec_read()  argument
523 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_read()
524 	if (unlikely(!meta->table[index].handle) ||  in zram_bvec_read()
525 			zram_test_flag(meta, index, ZRAM_ZERO)) {  in zram_bvec_read()
526 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_read()
530 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_read()
546 	ret = zram_decompress_page(zram, uncmem, index);  in zram_bvec_read()
579 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,  in zram_bvec_write()  argument
603 		ret = zram_decompress_page(zram, uncmem, index);  in zram_bvec_write()
625 		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
626 		zram_free_page(zram, index);  in zram_bvec_write()
627 		zram_set_flag(meta, index, ZRAM_ZERO);  in zram_bvec_write()
628 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
656 			index, clen);  in zram_bvec_write()
688 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
689 	zram_free_page(zram, index);  in zram_bvec_write()
691 	meta->table[index].handle = handle;  in zram_bvec_write()
692 	zram_set_obj_size(meta, index, clen);  in zram_bvec_write()
693 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bvec_write()
706 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,  in zram_bvec_rw()  argument
717 		ret = zram_bvec_read(zram, bvec, index, offset);  in zram_bvec_rw()
720 		ret = zram_bvec_write(zram, bvec, index, offset);  in zram_bvec_rw()
740 static void zram_bio_discard(struct zram *zram, u32 index,  in zram_bio_discard()  argument
761 		index++;  in zram_bio_discard()
765 		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bio_discard()
766 		zram_free_page(zram, index);  in zram_bio_discard()
767 		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_bio_discard()
769 		index++;  in zram_bio_discard()
925 	u32 index;  in __zram_make_request()  local
929 	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;  in __zram_make_request()
934 		zram_bio_discard(zram, index, offset, bio);  in __zram_make_request()
954 			if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0)  in __zram_make_request()
959 			if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0)  in __zram_make_request()
962 			if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0)  in __zram_make_request()
965 		update_position(&index, &offset, &bvec);  in __zram_make_request()
1002 				unsigned long index)  in zram_slot_free_notify()  argument
1010 	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);  in zram_slot_free_notify()
1011 	zram_free_page(zram, index);  in zram_slot_free_notify()
1012 	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);  in zram_slot_free_notify()
1020 	u32 index;  in zram_rw_page()  local
1034 	index = sector >> SECTORS_PER_PAGE_SHIFT;  in zram_rw_page()
1041 	err = zram_bvec_rw(zram, &bv, index, offset, rw);  in zram_rw_page()