zram 54 drivers/block/zram/zram_drv.c static void zram_free_page(struct zram *zram, size_t index); zram 55 drivers/block/zram/zram_drv.c static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, zram 59 drivers/block/zram/zram_drv.c static int zram_slot_trylock(struct zram *zram, u32 index) zram 61 drivers/block/zram/zram_drv.c return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags); zram 64 drivers/block/zram/zram_drv.c static void zram_slot_lock(struct zram *zram, u32 index) zram 66 drivers/block/zram/zram_drv.c bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags); zram 69 drivers/block/zram/zram_drv.c static void zram_slot_unlock(struct zram *zram, u32 index) zram 71 drivers/block/zram/zram_drv.c bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags); zram 74 drivers/block/zram/zram_drv.c static inline bool init_done(struct zram *zram) zram 76 drivers/block/zram/zram_drv.c return zram->disksize; zram 79 drivers/block/zram/zram_drv.c static inline struct zram *dev_to_zram(struct device *dev) zram 81 drivers/block/zram/zram_drv.c return (struct zram *)dev_to_disk(dev)->private_data; zram 84 drivers/block/zram/zram_drv.c static unsigned long zram_get_handle(struct zram *zram, u32 index) zram 86 drivers/block/zram/zram_drv.c return zram->table[index].handle; zram 89 drivers/block/zram/zram_drv.c static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle) zram 91 drivers/block/zram/zram_drv.c zram->table[index].handle = handle; zram 95 drivers/block/zram/zram_drv.c static bool zram_test_flag(struct zram *zram, u32 index, zram 98 drivers/block/zram/zram_drv.c return zram->table[index].flags & BIT(flag); zram 101 drivers/block/zram/zram_drv.c static void zram_set_flag(struct zram *zram, u32 index, zram 104 drivers/block/zram/zram_drv.c zram->table[index].flags |= BIT(flag); zram 107 drivers/block/zram/zram_drv.c static void zram_clear_flag(struct zram *zram, u32 index, zram 110 drivers/block/zram/zram_drv.c zram->table[index].flags &= ~BIT(flag); zram 113 drivers/block/zram/zram_drv.c static inline void zram_set_element(struct zram *zram, u32 index, zram 116 drivers/block/zram/zram_drv.c zram->table[index].element = element; zram 119 drivers/block/zram/zram_drv.c static unsigned long zram_get_element(struct zram *zram, u32 index) zram 121 drivers/block/zram/zram_drv.c return zram->table[index].element; zram 124 drivers/block/zram/zram_drv.c static size_t zram_get_obj_size(struct zram *zram, u32 index) zram 126 drivers/block/zram/zram_drv.c return zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1); zram 129 drivers/block/zram/zram_drv.c static void zram_set_obj_size(struct zram *zram, zram 132 drivers/block/zram/zram_drv.c unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT; zram 134 drivers/block/zram/zram_drv.c zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; zram 137 drivers/block/zram/zram_drv.c static inline bool zram_allocated(struct zram *zram, u32 index) zram 139 drivers/block/zram/zram_drv.c return zram_get_obj_size(zram, index) || zram 140 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_SAME) || zram 141 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_WB); zram 159 drivers/block/zram/zram_drv.c static inline bool valid_io_request(struct zram *zram, zram 171 drivers/block/zram/zram_drv.c bound = zram->disksize >> SECTOR_SHIFT; zram 186 drivers/block/zram/zram_drv.c static inline void update_used_max(struct zram *zram, zram 191 drivers/block/zram/zram_drv.c old_max = atomic_long_read(&zram->stats.max_used_pages); zram 197 drivers/block/zram/zram_drv.c &zram->stats.max_used_pages, cur_max, pages); zram 231 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 233 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 234 drivers/block/zram/zram_drv.c val = init_done(zram); zram 235 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 243 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 245 drivers/block/zram/zram_drv.c return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize); zram 253 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 259 drivers/block/zram/zram_drv.c down_write(&zram->init_lock); zram 260 drivers/block/zram/zram_drv.c zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT; zram 261 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 271 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 277 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 278 drivers/block/zram/zram_drv.c if (init_done(zram)) { zram 279 drivers/block/zram/zram_drv.c atomic_long_set(&zram->stats.max_used_pages, zram 280 drivers/block/zram/zram_drv.c zs_get_total_pages(zram->mem_pool)); zram 282 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 290 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 291 drivers/block/zram/zram_drv.c unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; zram 297 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 298 drivers/block/zram/zram_drv.c if (!init_done(zram)) { zram 299 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 308 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 309 drivers/block/zram/zram_drv.c if (zram_allocated(zram, index) && zram 310 drivers/block/zram/zram_drv.c !zram_test_flag(zram, index, ZRAM_UNDER_WB)) zram 311 drivers/block/zram/zram_drv.c zram_set_flag(zram, index, ZRAM_IDLE); zram 312 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 315 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 324 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 331 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 332 drivers/block/zram/zram_drv.c spin_lock(&zram->wb_limit_lock); zram 333 drivers/block/zram/zram_drv.c zram->wb_limit_enable = val; zram 334 drivers/block/zram/zram_drv.c spin_unlock(&zram->wb_limit_lock); zram 335 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 345 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 347 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 348 drivers/block/zram/zram_drv.c spin_lock(&zram->wb_limit_lock); zram 349 drivers/block/zram/zram_drv.c val = zram->wb_limit_enable; zram 350 drivers/block/zram/zram_drv.c spin_unlock(&zram->wb_limit_lock); zram 351 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 359 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 366 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 367 drivers/block/zram/zram_drv.c spin_lock(&zram->wb_limit_lock); zram 368 drivers/block/zram/zram_drv.c zram->bd_wb_limit = val; zram 369 drivers/block/zram/zram_drv.c spin_unlock(&zram->wb_limit_lock); zram 370 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 380 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 382 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 383 drivers/block/zram/zram_drv.c spin_lock(&zram->wb_limit_lock); zram 384 drivers/block/zram/zram_drv.c val = zram->bd_wb_limit; zram 385 drivers/block/zram/zram_drv.c spin_unlock(&zram->wb_limit_lock); zram 386 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 391 drivers/block/zram/zram_drv.c static void reset_bdev(struct zram *zram) zram 395 drivers/block/zram/zram_drv.c if (!zram->backing_dev) zram 398 drivers/block/zram/zram_drv.c bdev = zram->bdev; zram 399 drivers/block/zram/zram_drv.c if (zram->old_block_size) zram 400 drivers/block/zram/zram_drv.c set_blocksize(bdev, zram->old_block_size); zram 403 drivers/block/zram/zram_drv.c filp_close(zram->backing_dev, NULL); zram 404 drivers/block/zram/zram_drv.c zram->backing_dev = NULL; zram 405 drivers/block/zram/zram_drv.c zram->old_block_size = 0; zram 406 drivers/block/zram/zram_drv.c zram->bdev = NULL; zram 407 drivers/block/zram/zram_drv.c zram->disk->queue->backing_dev_info->capabilities |= zram 409 drivers/block/zram/zram_drv.c kvfree(zram->bitmap); zram 410 drivers/block/zram/zram_drv.c zram->bitmap = NULL; zram 417 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 421 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 422 drivers/block/zram/zram_drv.c file = zram->backing_dev; zram 425 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 439 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 455 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 461 drivers/block/zram/zram_drv.c down_write(&zram->init_lock); zram 462 drivers/block/zram/zram_drv.c if (init_done(zram)) { zram 491 drivers/block/zram/zram_drv.c err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram); zram 510 drivers/block/zram/zram_drv.c reset_bdev(zram); zram 512 drivers/block/zram/zram_drv.c zram->old_block_size = old_block_size; zram 513 drivers/block/zram/zram_drv.c zram->bdev = bdev; zram 514 drivers/block/zram/zram_drv.c zram->backing_dev = backing_dev; zram 515 drivers/block/zram/zram_drv.c zram->bitmap = bitmap; zram 516 drivers/block/zram/zram_drv.c zram->nr_pages = nr_pages; zram 527 drivers/block/zram/zram_drv.c zram->disk->queue->backing_dev_info->capabilities &= zram 529 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 545 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 552 drivers/block/zram/zram_drv.c static unsigned long alloc_block_bdev(struct zram *zram) zram 557 drivers/block/zram/zram_drv.c blk_idx = find_next_zero_bit(zram->bitmap, zram->nr_pages, blk_idx); zram 558 drivers/block/zram/zram_drv.c if (blk_idx == zram->nr_pages) zram 561 drivers/block/zram/zram_drv.c if (test_and_set_bit(blk_idx, zram->bitmap)) zram 564 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.bd_count); zram 568 drivers/block/zram/zram_drv.c static void free_block_bdev(struct zram *zram, unsigned long blk_idx) zram 572 drivers/block/zram/zram_drv.c was_set = test_and_clear_bit(blk_idx, zram->bitmap); zram 574 drivers/block/zram/zram_drv.c atomic64_dec(&zram->stats.bd_count); zram 589 drivers/block/zram/zram_drv.c static int read_from_bdev_async(struct zram *zram, struct bio_vec *bvec, zram 599 drivers/block/zram/zram_drv.c bio_set_dev(bio, zram->bdev); zram 623 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 624 drivers/block/zram/zram_drv.c unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; zram 640 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 641 drivers/block/zram/zram_drv.c if (!init_done(zram)) { zram 646 drivers/block/zram/zram_drv.c if (!zram->backing_dev) { zram 664 drivers/block/zram/zram_drv.c spin_lock(&zram->wb_limit_lock); zram 665 drivers/block/zram/zram_drv.c if (zram->wb_limit_enable && !zram->bd_wb_limit) { zram 666 drivers/block/zram/zram_drv.c spin_unlock(&zram->wb_limit_lock); zram 670 drivers/block/zram/zram_drv.c spin_unlock(&zram->wb_limit_lock); zram 673 drivers/block/zram/zram_drv.c blk_idx = alloc_block_bdev(zram); zram 680 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 681 drivers/block/zram/zram_drv.c if (!zram_allocated(zram, index)) zram 684 drivers/block/zram/zram_drv.c if (zram_test_flag(zram, index, ZRAM_WB) || zram 685 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_SAME) || zram 686 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_UNDER_WB)) zram 690 drivers/block/zram/zram_drv.c !zram_test_flag(zram, index, ZRAM_IDLE)) zram 693 drivers/block/zram/zram_drv.c !zram_test_flag(zram, index, ZRAM_HUGE)) zram 699 drivers/block/zram/zram_drv.c zram_set_flag(zram, index, ZRAM_UNDER_WB); zram 701 drivers/block/zram/zram_drv.c zram_set_flag(zram, index, ZRAM_IDLE); zram 702 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 703 drivers/block/zram/zram_drv.c if (zram_bvec_read(zram, &bvec, index, 0, NULL)) { zram 704 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 705 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_UNDER_WB); zram 706 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_IDLE); zram 707 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 712 drivers/block/zram/zram_drv.c bio_set_dev(&bio, zram->bdev); zram 724 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 725 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_UNDER_WB); zram 726 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_IDLE); zram 727 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 731 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.bd_writes); zram 741 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 742 drivers/block/zram/zram_drv.c if (!zram_allocated(zram, index) || zram 743 drivers/block/zram/zram_drv.c !zram_test_flag(zram, index, ZRAM_IDLE)) { zram 744 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_UNDER_WB); zram 745 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_IDLE); zram 749 drivers/block/zram/zram_drv.c zram_free_page(zram, index); zram 750 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_UNDER_WB); zram 751 drivers/block/zram/zram_drv.c zram_set_flag(zram, index, ZRAM_WB); zram 752 drivers/block/zram/zram_drv.c zram_set_element(zram, index, blk_idx); zram 754 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.pages_stored); zram 755 drivers/block/zram/zram_drv.c spin_lock(&zram->wb_limit_lock); zram 756 drivers/block/zram/zram_drv.c if (zram->wb_limit_enable && zram->bd_wb_limit > 0) zram 757 drivers/block/zram/zram_drv.c zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12); zram 758 drivers/block/zram/zram_drv.c spin_unlock(&zram->wb_limit_lock); zram 760 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 764 drivers/block/zram/zram_drv.c free_block_bdev(zram, blk_idx); zram 767 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 774 drivers/block/zram/zram_drv.c struct zram *zram; zram 784 drivers/block/zram/zram_drv.c struct zram *zram = zw->zram; zram 788 drivers/block/zram/zram_drv.c read_from_bdev_async(zram, &zw->bvec, entry, bio); zram 796 drivers/block/zram/zram_drv.c static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec, zram 802 drivers/block/zram/zram_drv.c work.zram = zram; zram 814 drivers/block/zram/zram_drv.c static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec, zram 822 drivers/block/zram/zram_drv.c static int read_from_bdev(struct zram *zram, struct bio_vec *bvec, zram 825 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.bd_reads); zram 827 drivers/block/zram/zram_drv.c return read_from_bdev_sync(zram, bvec, entry, parent); zram 829 drivers/block/zram/zram_drv.c return read_from_bdev_async(zram, bvec, entry, parent); zram 832 drivers/block/zram/zram_drv.c static inline void reset_bdev(struct zram *zram) {}; zram 833 drivers/block/zram/zram_drv.c static int read_from_bdev(struct zram *zram, struct bio_vec *bvec, zram 839 drivers/block/zram/zram_drv.c static void free_block_bdev(struct zram *zram, unsigned long blk_idx) {}; zram 856 drivers/block/zram/zram_drv.c static void zram_accessed(struct zram *zram, u32 index) zram 858 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_IDLE); zram 859 drivers/block/zram/zram_drv.c zram->table[index].ac_time = ktime_get_boottime(); zram 867 drivers/block/zram/zram_drv.c struct zram *zram = file->private_data; zram 868 drivers/block/zram/zram_drv.c unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; zram 875 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 876 drivers/block/zram/zram_drv.c if (!init_done(zram)) { zram 877 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 885 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 886 drivers/block/zram/zram_drv.c if (!zram_allocated(zram, index)) zram 889 drivers/block/zram/zram_drv.c ts = ktime_to_timespec64(zram->table[index].ac_time); zram 894 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_SAME) ? 's' : '.', zram 895 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_WB) ? 'w' : '.', zram 896 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_HUGE) ? 'h' : '.', zram 897 drivers/block/zram/zram_drv.c zram_test_flag(zram, index, ZRAM_IDLE) ? 'i' : '.'); zram 900 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 906 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 910 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 924 drivers/block/zram/zram_drv.c static void zram_debugfs_register(struct zram *zram) zram 929 drivers/block/zram/zram_drv.c zram->debugfs_dir = debugfs_create_dir(zram->disk->disk_name, zram 931 drivers/block/zram/zram_drv.c debugfs_create_file("block_state", 0400, zram->debugfs_dir, zram 932 drivers/block/zram/zram_drv.c zram, &proc_zram_block_state_op); zram 935 drivers/block/zram/zram_drv.c static void zram_debugfs_unregister(struct zram *zram) zram 937 drivers/block/zram/zram_drv.c debugfs_remove_recursive(zram->debugfs_dir); zram 942 drivers/block/zram/zram_drv.c static void zram_accessed(struct zram *zram, u32 index) zram 944 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_IDLE); zram 946 drivers/block/zram/zram_drv.c static void zram_debugfs_register(struct zram *zram) {}; zram 947 drivers/block/zram/zram_drv.c static void zram_debugfs_unregister(struct zram *zram) {}; zram 975 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 977 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 978 drivers/block/zram/zram_drv.c sz = zcomp_available_show(zram->compressor, buf); zram 979 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 987 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 988 drivers/block/zram/zram_drv.c char compressor[ARRAY_SIZE(zram->compressor)]; zram 1000 drivers/block/zram/zram_drv.c down_write(&zram->init_lock); zram 1001 drivers/block/zram/zram_drv.c if (init_done(zram)) { zram 1002 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 1007 drivers/block/zram/zram_drv.c strcpy(zram->compressor, compressor); zram 1008 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 1015 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 1017 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 1018 drivers/block/zram/zram_drv.c if (!init_done(zram)) { zram 1019 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 1023 drivers/block/zram/zram_drv.c zs_compact(zram->mem_pool); zram 1024 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 1032 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 1035 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 1038 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.failed_reads), zram 1039 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.failed_writes), zram 1040 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.invalid_io), zram 1041 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.notify_free)); zram 1042 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 1050 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 1058 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 1059 drivers/block/zram/zram_drv.c if (init_done(zram)) { zram 1060 drivers/block/zram/zram_drv.c mem_used = zs_get_total_pages(zram->mem_pool); zram 1061 drivers/block/zram/zram_drv.c zs_pool_stats(zram->mem_pool, &pool_stats); zram 1064 drivers/block/zram/zram_drv.c orig_size = atomic64_read(&zram->stats.pages_stored); zram 1065 drivers/block/zram/zram_drv.c max_used = atomic_long_read(&zram->stats.max_used_pages); zram 1070 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.compr_data_size), zram 1072 drivers/block/zram/zram_drv.c zram->limit_pages << PAGE_SHIFT, zram 1074 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.same_pages), zram 1076 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.huge_pages)); zram 1077 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 1087 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 1090 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 1093 drivers/block/zram/zram_drv.c FOUR_K((u64)atomic64_read(&zram->stats.bd_count)), zram 1094 drivers/block/zram/zram_drv.c FOUR_K((u64)atomic64_read(&zram->stats.bd_reads)), zram 1095 drivers/block/zram/zram_drv.c FOUR_K((u64)atomic64_read(&zram->stats.bd_writes))); zram 1096 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 1106 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 1109 drivers/block/zram/zram_drv.c down_read(&zram->init_lock); zram 1113 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.writestall), zram 1114 drivers/block/zram/zram_drv.c (u64)atomic64_read(&zram->stats.miss_free)); zram 1115 drivers/block/zram/zram_drv.c up_read(&zram->init_lock); zram 1127 drivers/block/zram/zram_drv.c static void zram_meta_free(struct zram *zram, u64 disksize) zram 1134 drivers/block/zram/zram_drv.c zram_free_page(zram, index); zram 1136 drivers/block/zram/zram_drv.c zs_destroy_pool(zram->mem_pool); zram 1137 drivers/block/zram/zram_drv.c vfree(zram->table); zram 1140 drivers/block/zram/zram_drv.c static bool zram_meta_alloc(struct zram *zram, u64 disksize) zram 1145 drivers/block/zram/zram_drv.c zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); zram 1146 drivers/block/zram/zram_drv.c if (!zram->table) zram 1149 drivers/block/zram/zram_drv.c zram->mem_pool = zs_create_pool(zram->disk->disk_name); zram 1150 drivers/block/zram/zram_drv.c if (!zram->mem_pool) { zram 1151 drivers/block/zram/zram_drv.c vfree(zram->table); zram 1156 drivers/block/zram/zram_drv.c huge_class_size = zs_huge_class_size(zram->mem_pool); zram 1165 drivers/block/zram/zram_drv.c static void zram_free_page(struct zram *zram, size_t index) zram 1170 drivers/block/zram/zram_drv.c zram->table[index].ac_time = 0; zram 1172 drivers/block/zram/zram_drv.c if (zram_test_flag(zram, index, ZRAM_IDLE)) zram 1173 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_IDLE); zram 1175 drivers/block/zram/zram_drv.c if (zram_test_flag(zram, index, ZRAM_HUGE)) { zram 1176 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_HUGE); zram 1177 drivers/block/zram/zram_drv.c atomic64_dec(&zram->stats.huge_pages); zram 1180 drivers/block/zram/zram_drv.c if (zram_test_flag(zram, index, ZRAM_WB)) { zram 1181 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_WB); zram 1182 drivers/block/zram/zram_drv.c free_block_bdev(zram, zram_get_element(zram, index)); zram 1190 drivers/block/zram/zram_drv.c if (zram_test_flag(zram, index, ZRAM_SAME)) { zram 1191 drivers/block/zram/zram_drv.c zram_clear_flag(zram, index, ZRAM_SAME); zram 1192 drivers/block/zram/zram_drv.c atomic64_dec(&zram->stats.same_pages); zram 1196 drivers/block/zram/zram_drv.c handle = zram_get_handle(zram, index); zram 1200 drivers/block/zram/zram_drv.c zs_free(zram->mem_pool, handle); zram 1202 drivers/block/zram/zram_drv.c atomic64_sub(zram_get_obj_size(zram, index), zram 1203 drivers/block/zram/zram_drv.c &zram->stats.compr_data_size); zram 1205 drivers/block/zram/zram_drv.c atomic64_dec(&zram->stats.pages_stored); zram 1206 drivers/block/zram/zram_drv.c zram_set_handle(zram, index, 0); zram 1207 drivers/block/zram/zram_drv.c zram_set_obj_size(zram, index, 0); zram 1208 drivers/block/zram/zram_drv.c WARN_ON_ONCE(zram->table[index].flags & zram 1212 drivers/block/zram/zram_drv.c static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index, zram 1220 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 1221 drivers/block/zram/zram_drv.c if (zram_test_flag(zram, index, ZRAM_WB)) { zram 1224 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 1229 drivers/block/zram/zram_drv.c return read_from_bdev(zram, &bvec, zram 1230 drivers/block/zram/zram_drv.c zram_get_element(zram, index), zram 1234 drivers/block/zram/zram_drv.c handle = zram_get_handle(zram, index); zram 1235 drivers/block/zram/zram_drv.c if (!handle || zram_test_flag(zram, index, ZRAM_SAME)) { zram 1239 drivers/block/zram/zram_drv.c value = handle ? zram_get_element(zram, index) : 0; zram 1243 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 1247 drivers/block/zram/zram_drv.c size = zram_get_obj_size(zram, index); zram 1249 drivers/block/zram/zram_drv.c src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO); zram 1256 drivers/block/zram/zram_drv.c struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp); zram 1261 drivers/block/zram/zram_drv.c zcomp_stream_put(zram->comp); zram 1263 drivers/block/zram/zram_drv.c zs_unmap_object(zram->mem_pool, handle); zram 1264 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 1273 drivers/block/zram/zram_drv.c static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, zram 1287 drivers/block/zram/zram_drv.c ret = __zram_bvec_read(zram, page, index, bio, is_partial_io(bvec)); zram 1306 drivers/block/zram/zram_drv.c static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, zram 1324 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.same_pages); zram 1330 drivers/block/zram/zram_drv.c zstrm = zcomp_stream_get(zram->comp); zram 1336 drivers/block/zram/zram_drv.c zcomp_stream_put(zram->comp); zram 1338 drivers/block/zram/zram_drv.c zs_free(zram->mem_pool, handle); zram 1358 drivers/block/zram/zram_drv.c handle = zs_malloc(zram->mem_pool, comp_len, zram 1364 drivers/block/zram/zram_drv.c zcomp_stream_put(zram->comp); zram 1365 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.writestall); zram 1366 drivers/block/zram/zram_drv.c handle = zs_malloc(zram->mem_pool, comp_len, zram 1374 drivers/block/zram/zram_drv.c alloced_pages = zs_get_total_pages(zram->mem_pool); zram 1375 drivers/block/zram/zram_drv.c update_used_max(zram, alloced_pages); zram 1377 drivers/block/zram/zram_drv.c if (zram->limit_pages && alloced_pages > zram->limit_pages) { zram 1378 drivers/block/zram/zram_drv.c zcomp_stream_put(zram->comp); zram 1379 drivers/block/zram/zram_drv.c zs_free(zram->mem_pool, handle); zram 1383 drivers/block/zram/zram_drv.c dst = zs_map_object(zram->mem_pool, handle, ZS_MM_WO); zram 1392 drivers/block/zram/zram_drv.c zcomp_stream_put(zram->comp); zram 1393 drivers/block/zram/zram_drv.c zs_unmap_object(zram->mem_pool, handle); zram 1394 drivers/block/zram/zram_drv.c atomic64_add(comp_len, &zram->stats.compr_data_size); zram 1400 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 1401 drivers/block/zram/zram_drv.c zram_free_page(zram, index); zram 1404 drivers/block/zram/zram_drv.c zram_set_flag(zram, index, ZRAM_HUGE); zram 1405 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.huge_pages); zram 1409 drivers/block/zram/zram_drv.c zram_set_flag(zram, index, flags); zram 1410 drivers/block/zram/zram_drv.c zram_set_element(zram, index, element); zram 1412 drivers/block/zram/zram_drv.c zram_set_handle(zram, index, handle); zram 1413 drivers/block/zram/zram_drv.c zram_set_obj_size(zram, index, comp_len); zram 1415 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 1418 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.pages_stored); zram 1422 drivers/block/zram/zram_drv.c static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, zram 1441 drivers/block/zram/zram_drv.c ret = __zram_bvec_read(zram, page, index, bio, true); zram 1456 drivers/block/zram/zram_drv.c ret = __zram_bvec_write(zram, &vec, index, bio); zram 1468 drivers/block/zram/zram_drv.c static void zram_bio_discard(struct zram *zram, u32 index, zram 1492 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 1493 drivers/block/zram/zram_drv.c zram_free_page(zram, index); zram 1494 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 1495 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.notify_free); zram 1506 drivers/block/zram/zram_drv.c static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, zram 1510 drivers/block/zram/zram_drv.c struct request_queue *q = zram->disk->queue; zram 1514 drivers/block/zram/zram_drv.c &zram->disk->part0); zram 1517 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.num_reads); zram 1518 drivers/block/zram/zram_drv.c ret = zram_bvec_read(zram, bvec, index, offset, bio); zram 1521 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.num_writes); zram 1522 drivers/block/zram/zram_drv.c ret = zram_bvec_write(zram, bvec, index, offset, bio); zram 1525 drivers/block/zram/zram_drv.c generic_end_io_acct(q, op, &zram->disk->part0, start_time); zram 1527 drivers/block/zram/zram_drv.c zram_slot_lock(zram, index); zram 1528 drivers/block/zram/zram_drv.c zram_accessed(zram, index); zram 1529 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 1533 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.failed_reads); zram 1535 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.failed_writes); zram 1541 drivers/block/zram/zram_drv.c static void __zram_make_request(struct zram *zram, struct bio *bio) zram 1555 drivers/block/zram/zram_drv.c zram_bio_discard(zram, index, offset, bio); zram 1569 drivers/block/zram/zram_drv.c if (zram_bvec_rw(zram, &bv, index, offset, zram 1592 drivers/block/zram/zram_drv.c struct zram *zram = queue->queuedata; zram 1594 drivers/block/zram/zram_drv.c if (!valid_io_request(zram, bio->bi_iter.bi_sector, zram 1596 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.invalid_io); zram 1600 drivers/block/zram/zram_drv.c __zram_make_request(zram, bio); zram 1611 drivers/block/zram/zram_drv.c struct zram *zram; zram 1613 drivers/block/zram/zram_drv.c zram = bdev->bd_disk->private_data; zram 1615 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.notify_free); zram 1616 drivers/block/zram/zram_drv.c if (!zram_slot_trylock(zram, index)) { zram 1617 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.miss_free); zram 1621 drivers/block/zram/zram_drv.c zram_free_page(zram, index); zram 1622 drivers/block/zram/zram_drv.c zram_slot_unlock(zram, index); zram 1630 drivers/block/zram/zram_drv.c struct zram *zram; zram 1635 drivers/block/zram/zram_drv.c zram = bdev->bd_disk->private_data; zram 1637 drivers/block/zram/zram_drv.c if (!valid_io_request(zram, sector, PAGE_SIZE)) { zram 1638 drivers/block/zram/zram_drv.c atomic64_inc(&zram->stats.invalid_io); zram 1650 drivers/block/zram/zram_drv.c ret = zram_bvec_rw(zram, &bv, index, offset, op, NULL); zram 1676 drivers/block/zram/zram_drv.c static void zram_reset_device(struct zram *zram) zram 1681 drivers/block/zram/zram_drv.c down_write(&zram->init_lock); zram 1683 drivers/block/zram/zram_drv.c zram->limit_pages = 0; zram 1685 drivers/block/zram/zram_drv.c if (!init_done(zram)) { zram 1686 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 1690 drivers/block/zram/zram_drv.c comp = zram->comp; zram 1691 drivers/block/zram/zram_drv.c disksize = zram->disksize; zram 1692 drivers/block/zram/zram_drv.c zram->disksize = 0; zram 1694 drivers/block/zram/zram_drv.c set_capacity(zram->disk, 0); zram 1695 drivers/block/zram/zram_drv.c part_stat_set_all(&zram->disk->part0, 0); zram 1697 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 1699 drivers/block/zram/zram_drv.c zram_meta_free(zram, disksize); zram 1700 drivers/block/zram/zram_drv.c memset(&zram->stats, 0, sizeof(zram->stats)); zram 1702 drivers/block/zram/zram_drv.c reset_bdev(zram); zram 1710 drivers/block/zram/zram_drv.c struct zram *zram = dev_to_zram(dev); zram 1717 drivers/block/zram/zram_drv.c down_write(&zram->init_lock); zram 1718 drivers/block/zram/zram_drv.c if (init_done(zram)) { zram 1725 drivers/block/zram/zram_drv.c if (!zram_meta_alloc(zram, disksize)) { zram 1730 drivers/block/zram/zram_drv.c comp = zcomp_create(zram->compressor); zram 1733 drivers/block/zram/zram_drv.c zram->compressor); zram 1738 drivers/block/zram/zram_drv.c zram->comp = comp; zram 1739 drivers/block/zram/zram_drv.c zram->disksize = disksize; zram 1740 drivers/block/zram/zram_drv.c set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); zram 1742 drivers/block/zram/zram_drv.c revalidate_disk(zram->disk); zram 1743 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 1748 drivers/block/zram/zram_drv.c zram_meta_free(zram, disksize); zram 1750 drivers/block/zram/zram_drv.c up_write(&zram->init_lock); zram 1759 drivers/block/zram/zram_drv.c struct zram *zram; zram 1769 drivers/block/zram/zram_drv.c zram = dev_to_zram(dev); zram 1770 drivers/block/zram/zram_drv.c bdev = bdget_disk(zram->disk, 0); zram 1776 drivers/block/zram/zram_drv.c if (bdev->bd_openers || zram->claim) { zram 1783 drivers/block/zram/zram_drv.c zram->claim = true; zram 1788 drivers/block/zram/zram_drv.c zram_reset_device(zram); zram 1789 drivers/block/zram/zram_drv.c revalidate_disk(zram->disk); zram 1793 drivers/block/zram/zram_drv.c zram->claim = false; zram 1802 drivers/block/zram/zram_drv.c struct zram *zram; zram 1806 drivers/block/zram/zram_drv.c zram = bdev->bd_disk->private_data; zram 1808 drivers/block/zram/zram_drv.c if (zram->claim) zram 1877 drivers/block/zram/zram_drv.c struct zram *zram; zram 1881 drivers/block/zram/zram_drv.c zram = kzalloc(sizeof(struct zram), GFP_KERNEL); zram 1882 drivers/block/zram/zram_drv.c if (!zram) zram 1885 drivers/block/zram/zram_drv.c ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL); zram 1890 drivers/block/zram/zram_drv.c init_rwsem(&zram->init_lock); zram 1892 drivers/block/zram/zram_drv.c spin_lock_init(&zram->wb_limit_lock); zram 1905 drivers/block/zram/zram_drv.c zram->disk = alloc_disk(1); zram 1906 drivers/block/zram/zram_drv.c if (!zram->disk) { zram 1913 drivers/block/zram/zram_drv.c zram->disk->major = zram_major; zram 1914 drivers/block/zram/zram_drv.c zram->disk->first_minor = device_id; zram 1915 drivers/block/zram/zram_drv.c zram->disk->fops = &zram_devops; zram 1916 drivers/block/zram/zram_drv.c zram->disk->queue = queue; zram 1917 drivers/block/zram/zram_drv.c zram->disk->queue->queuedata = zram; zram 1918 drivers/block/zram/zram_drv.c zram->disk->private_data = zram; zram 1919 drivers/block/zram/zram_drv.c snprintf(zram->disk->disk_name, 16, "zram%d", device_id); zram 1922 drivers/block/zram/zram_drv.c set_capacity(zram->disk, 0); zram 1924 drivers/block/zram/zram_drv.c blk_queue_flag_set(QUEUE_FLAG_NONROT, zram->disk->queue); zram 1925 drivers/block/zram/zram_drv.c blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue); zram 1931 drivers/block/zram/zram_drv.c blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE); zram 1932 drivers/block/zram/zram_drv.c blk_queue_logical_block_size(zram->disk->queue, zram 1934 drivers/block/zram/zram_drv.c blk_queue_io_min(zram->disk->queue, PAGE_SIZE); zram 1935 drivers/block/zram/zram_drv.c blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); zram 1936 drivers/block/zram/zram_drv.c zram->disk->queue->limits.discard_granularity = PAGE_SIZE; zram 1937 drivers/block/zram/zram_drv.c blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX); zram 1938 drivers/block/zram/zram_drv.c blk_queue_flag_set(QUEUE_FLAG_DISCARD, zram->disk->queue); zram 1949 drivers/block/zram/zram_drv.c blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX); zram 1951 drivers/block/zram/zram_drv.c zram->disk->queue->backing_dev_info->capabilities |= zram 1953 drivers/block/zram/zram_drv.c device_add_disk(NULL, zram->disk, zram_disk_attr_groups); zram 1955 drivers/block/zram/zram_drv.c strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor)); zram 1957 drivers/block/zram/zram_drv.c zram_debugfs_register(zram); zram 1958 drivers/block/zram/zram_drv.c pr_info("Added device: %s\n", zram->disk->disk_name); zram 1966 drivers/block/zram/zram_drv.c kfree(zram); zram 1970 drivers/block/zram/zram_drv.c static int zram_remove(struct zram *zram) zram 1974 drivers/block/zram/zram_drv.c bdev = bdget_disk(zram->disk, 0); zram 1979 drivers/block/zram/zram_drv.c if (bdev->bd_openers || zram->claim) { zram 1985 drivers/block/zram/zram_drv.c zram->claim = true; zram 1988 drivers/block/zram/zram_drv.c zram_debugfs_unregister(zram); zram 1992 drivers/block/zram/zram_drv.c zram_reset_device(zram); zram 1995 drivers/block/zram/zram_drv.c pr_info("Removed device: %s\n", zram->disk->disk_name); zram 1997 drivers/block/zram/zram_drv.c del_gendisk(zram->disk); zram 1998 drivers/block/zram/zram_drv.c blk_cleanup_queue(zram->disk->queue); zram 1999 drivers/block/zram/zram_drv.c put_disk(zram->disk); zram 2000 drivers/block/zram/zram_drv.c kfree(zram); zram 2033 drivers/block/zram/zram_drv.c struct zram *zram; zram 2045 drivers/block/zram/zram_drv.c zram = idr_find(&zram_index_idr, dev_id); zram 2046 drivers/block/zram/zram_drv.c if (zram) { zram 2047 drivers/block/zram/zram_drv.c ret = zram_remove(zram);