Lines Matching refs:class
457 static inline void zs_stat_inc(struct size_class *class, in zs_stat_inc() argument
461 class->stats.objs[type] += cnt; in zs_stat_inc()
464 static inline void zs_stat_dec(struct size_class *class, in zs_stat_dec() argument
468 class->stats.objs[type] -= cnt; in zs_stat_dec()
471 static inline unsigned long zs_stat_get(struct size_class *class, in zs_stat_get() argument
475 return class->stats.objs[type]; in zs_stat_get()
502 struct size_class *class; in zs_stats_size_show() local
515 class = pool->size_class[i]; in zs_stats_size_show()
517 if (class->index != i) in zs_stats_size_show()
520 spin_lock(&class->lock); in zs_stats_size_show()
521 class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL); in zs_stats_size_show()
522 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY); in zs_stats_size_show()
523 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED); in zs_stats_size_show()
524 obj_used = zs_stat_get(class, OBJ_USED); in zs_stats_size_show()
525 spin_unlock(&class->lock); in zs_stats_size_show()
527 objs_per_zspage = get_maxobj_per_zspage(class->size, in zs_stats_size_show()
528 class->pages_per_zspage); in zs_stats_size_show()
530 class->pages_per_zspage; in zs_stats_size_show()
533 i, class->size, class_almost_full, class_almost_empty, in zs_stats_size_show()
535 class->pages_per_zspage); in zs_stats_size_show()
650 static void insert_zspage(struct page *page, struct size_class *class, in insert_zspage() argument
660 zs_stat_inc(class, fullness == ZS_ALMOST_EMPTY ? in insert_zspage()
663 head = &class->fullness_list[fullness]; in insert_zspage()
682 static void remove_zspage(struct page *page, struct size_class *class, in remove_zspage() argument
692 head = &class->fullness_list[fullness]; in remove_zspage()
701 zs_stat_dec(class, fullness == ZS_ALMOST_EMPTY ? in remove_zspage()
714 static enum fullness_group fix_fullness_group(struct size_class *class, in fix_fullness_group() argument
727 remove_zspage(page, class, currfg); in fix_fullness_group()
728 insert_zspage(page, class, newfg); in fix_fullness_group()
836 static unsigned long obj_to_head(struct size_class *class, struct page *page, in obj_to_head() argument
839 if (class->huge) { in obj_to_head()
912 static void init_zspage(struct page *first_page, struct size_class *class) in init_zspage() argument
936 while ((off += class->size) < PAGE_SIZE) { in init_zspage()
938 link += class->size / sizeof(*link); in init_zspage()
957 static struct page *alloc_zspage(struct size_class *class, gfp_t flags) in alloc_zspage() argument
974 for (i = 0; i < class->pages_per_zspage; i++) { in alloc_zspage()
994 if (i == class->pages_per_zspage - 1) /* last page */ in alloc_zspage()
999 init_zspage(first_page, class); in alloc_zspage()
1003 first_page->objects = class->pages_per_zspage * PAGE_SIZE / class->size; in alloc_zspage()
1016 static struct page *find_get_zspage(struct size_class *class) in find_get_zspage() argument
1022 page = class->fullness_list[i]; in find_get_zspage()
1269 struct size_class *class; in zs_map_object() local
1289 class = pool->size_class[class_idx]; in zs_map_object()
1290 off = obj_idx_to_offset(page, obj_idx, class->size); in zs_map_object()
1294 if (off + class->size <= PAGE_SIZE) { in zs_map_object()
1306 ret = __zs_map_object(area, pages, off, class->size); in zs_map_object()
1308 if (!class->huge) in zs_map_object()
1322 struct size_class *class; in zs_unmap_object() local
1330 class = pool->size_class[class_idx]; in zs_unmap_object()
1331 off = obj_idx_to_offset(page, obj_idx, class->size); in zs_unmap_object()
1334 if (off + class->size <= PAGE_SIZE) in zs_unmap_object()
1343 __zs_unmap_object(area, pages, off, class->size); in zs_unmap_object()
1351 struct size_class *class, unsigned long handle) in obj_malloc() argument
1363 m_offset = obj_idx_to_offset(m_page, m_objidx, class->size); in obj_malloc()
1368 if (!class->huge) in obj_malloc()
1376 zs_stat_inc(class, OBJ_USED, 1); in obj_malloc()
1394 struct size_class *class; in zs_malloc() local
1406 class = pool->size_class[get_size_class_index(size)]; in zs_malloc()
1408 spin_lock(&class->lock); in zs_malloc()
1409 first_page = find_get_zspage(class); in zs_malloc()
1412 spin_unlock(&class->lock); in zs_malloc()
1413 first_page = alloc_zspage(class, pool->flags); in zs_malloc()
1419 set_zspage_mapping(first_page, class->index, ZS_EMPTY); in zs_malloc()
1420 atomic_long_add(class->pages_per_zspage, in zs_malloc()
1423 spin_lock(&class->lock); in zs_malloc()
1424 zs_stat_inc(class, OBJ_ALLOCATED, get_maxobj_per_zspage( in zs_malloc()
1425 class->size, class->pages_per_zspage)); in zs_malloc()
1428 obj = obj_malloc(first_page, class, handle); in zs_malloc()
1430 fix_fullness_group(class, first_page); in zs_malloc()
1432 spin_unlock(&class->lock); in zs_malloc()
1438 static void obj_free(struct zs_pool *pool, struct size_class *class, in obj_free() argument
1452 f_offset = obj_idx_to_offset(f_page, f_objidx, class->size); in obj_free()
1459 if (class->huge) in obj_free()
1464 zs_stat_dec(class, OBJ_USED, 1); in obj_free()
1472 struct size_class *class; in zs_free() local
1484 class = pool->size_class[class_idx]; in zs_free()
1486 spin_lock(&class->lock); in zs_free()
1487 obj_free(pool, class, obj); in zs_free()
1488 fullness = fix_fullness_group(class, first_page); in zs_free()
1490 zs_stat_dec(class, OBJ_ALLOCATED, get_maxobj_per_zspage( in zs_free()
1491 class->size, class->pages_per_zspage)); in zs_free()
1492 atomic_long_sub(class->pages_per_zspage, in zs_free()
1496 spin_unlock(&class->lock); in zs_free()
1504 struct size_class *class) in zs_object_copy() argument
1513 s_size = d_size = class->size; in zs_object_copy()
1518 s_off = obj_idx_to_offset(s_page, s_objidx, class->size); in zs_object_copy()
1519 d_off = obj_idx_to_offset(d_page, d_objidx, class->size); in zs_object_copy()
1521 if (s_off + class->size > PAGE_SIZE) in zs_object_copy()
1524 if (d_off + class->size > PAGE_SIZE) in zs_object_copy()
1535 if (written == class->size) in zs_object_copy()
1550 s_size = class->size - written; in zs_object_copy()
1559 d_size = class->size - written; in zs_object_copy()
1573 struct size_class *class) in find_alloced_obj() argument
1582 offset += class->size * index; in find_alloced_obj()
1585 head = obj_to_head(class, page, addr + offset); in find_alloced_obj()
1593 offset += class->size; in find_alloced_obj()
1612 static int migrate_zspage(struct zs_pool *pool, struct size_class *class, in migrate_zspage() argument
1623 handle = find_alloced_obj(s_page, index, class); in migrate_zspage()
1640 free_obj = obj_malloc(d_page, class, handle); in migrate_zspage()
1641 zs_object_copy(free_obj, used_obj, class); in migrate_zspage()
1652 obj_free(pool, class, used_obj); in migrate_zspage()
1662 static struct page *isolate_target_page(struct size_class *class) in isolate_target_page() argument
1668 page = class->fullness_list[i]; in isolate_target_page()
1670 remove_zspage(page, class, i); in isolate_target_page()
1687 struct size_class *class, in putback_zspage() argument
1695 insert_zspage(first_page, class, fullness); in putback_zspage()
1696 set_zspage_mapping(first_page, class->index, fullness); in putback_zspage()
1699 zs_stat_dec(class, OBJ_ALLOCATED, get_maxobj_per_zspage( in putback_zspage()
1700 class->size, class->pages_per_zspage)); in putback_zspage()
1701 atomic_long_sub(class->pages_per_zspage, in putback_zspage()
1710 static struct page *isolate_source_page(struct size_class *class) in isolate_source_page() argument
1716 page = class->fullness_list[i]; in isolate_source_page()
1720 remove_zspage(page, class, i); in isolate_source_page()
1732 static unsigned long zs_can_compact(struct size_class *class) in zs_can_compact() argument
1735 unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED); in zs_can_compact()
1736 unsigned long obj_used = zs_stat_get(class, OBJ_USED); in zs_can_compact()
1742 obj_wasted /= get_maxobj_per_zspage(class->size, in zs_can_compact()
1743 class->pages_per_zspage); in zs_can_compact()
1745 return obj_wasted * class->pages_per_zspage; in zs_can_compact()
1748 static void __zs_compact(struct zs_pool *pool, struct size_class *class) in __zs_compact() argument
1754 spin_lock(&class->lock); in __zs_compact()
1755 while ((src_page = isolate_source_page(class))) { in __zs_compact()
1759 if (!zs_can_compact(class)) in __zs_compact()
1765 while ((dst_page = isolate_target_page(class))) { in __zs_compact()
1771 if (!migrate_zspage(pool, class, &cc)) in __zs_compact()
1774 putback_zspage(pool, class, dst_page); in __zs_compact()
1781 putback_zspage(pool, class, dst_page); in __zs_compact()
1782 if (putback_zspage(pool, class, src_page) == ZS_EMPTY) in __zs_compact()
1783 pool->stats.pages_compacted += class->pages_per_zspage; in __zs_compact()
1784 spin_unlock(&class->lock); in __zs_compact()
1786 spin_lock(&class->lock); in __zs_compact()
1790 putback_zspage(pool, class, src_page); in __zs_compact()
1792 spin_unlock(&class->lock); in __zs_compact()
1798 struct size_class *class; in zs_compact() local
1801 class = pool->size_class[i]; in zs_compact()
1802 if (!class) in zs_compact()
1804 if (class->index != i) in zs_compact()
1806 __zs_compact(pool, class); in zs_compact()
1841 struct size_class *class; in zs_shrinker_count() local
1847 class = pool->size_class[i]; in zs_shrinker_count()
1848 if (!class) in zs_shrinker_count()
1850 if (class->index != i) in zs_shrinker_count()
1853 pages_to_free += zs_can_compact(class); in zs_shrinker_count()
1918 struct size_class *class; in zs_create_pool() local
1941 class = kzalloc(sizeof(struct size_class), GFP_KERNEL); in zs_create_pool()
1942 if (!class) in zs_create_pool()
1945 class->size = size; in zs_create_pool()
1946 class->index = i; in zs_create_pool()
1947 class->pages_per_zspage = pages_per_zspage; in zs_create_pool()
1950 class->huge = true; in zs_create_pool()
1951 spin_lock_init(&class->lock); in zs_create_pool()
1952 pool->size_class[i] = class; in zs_create_pool()
1954 prev_class = class; in zs_create_pool()
1985 struct size_class *class = pool->size_class[i]; in zs_destroy_pool() local
1987 if (!class) in zs_destroy_pool()
1990 if (class->index != i) in zs_destroy_pool()
1994 if (class->fullness_list[fg]) { in zs_destroy_pool()
1996 class->size, fg); in zs_destroy_pool()
1999 kfree(class); in zs_destroy_pool()