226 	struct page *fullness_list[_ZS_NR_FULLNESS_GROUPS];
402 static int is_first_page(struct page *page)  in is_first_page()  argument
404 	return PagePrivate(page);  in is_first_page()
407 static int is_last_page(struct page *page)  in is_last_page()  argument
409 	return PagePrivate2(page);  in is_last_page()
412 static void get_zspage_mapping(struct page *page, unsigned int *class_idx,  in get_zspage_mapping()  argument
416 	BUG_ON(!is_first_page(page));  in get_zspage_mapping()
418 	m = (unsigned long)page->mapping;  in get_zspage_mapping()
423 static void set_zspage_mapping(struct page *page, unsigned int class_idx,  in set_zspage_mapping()  argument
427 	BUG_ON(!is_first_page(page));  in set_zspage_mapping()
431 	page->mapping = (struct address_space *)m;  in set_zspage_mapping()
632 static enum fullness_group get_fullness_group(struct page *page)  in get_fullness_group()  argument
636 	BUG_ON(!is_first_page(page));  in get_fullness_group()
638 	inuse = page->inuse;  in get_fullness_group()
639 	max_objects = page->objects;  in get_fullness_group()
659 static void insert_zspage(struct page *page, struct size_class *class,  in insert_zspage()  argument
662 	struct page **head;  in insert_zspage()
664 	BUG_ON(!is_first_page(page));  in insert_zspage()
671 		list_add_tail(&page->lru, &(*head)->lru);  in insert_zspage()
673 	*head = page;  in insert_zspage()
682 static void remove_zspage(struct page *page, struct size_class *class,  in remove_zspage()  argument
685 	struct page **head;  in remove_zspage()
687 	BUG_ON(!is_first_page(page));  in remove_zspage()
696 	else if (*head == page)  in remove_zspage()
697 		*head = (struct page *)list_entry((*head)->lru.next,  in remove_zspage()
698 					struct page, lru);  in remove_zspage()
700 	list_del_init(&page->lru);  in remove_zspage()
715 						struct page *page)  in fix_fullness_group()  argument
720 	BUG_ON(!is_first_page(page));  in fix_fullness_group()
722 	get_zspage_mapping(page, &class_idx, &currfg);  in fix_fullness_group()
723 	newfg = get_fullness_group(page);  in fix_fullness_group()
727 	remove_zspage(page, class, currfg);  in fix_fullness_group()
728 	insert_zspage(page, class, newfg);  in fix_fullness_group()
729 	set_zspage_mapping(page, class_idx, newfg);  in fix_fullness_group()
776 static struct page *get_first_page(struct page *page)  in get_first_page()  argument
778 	if (is_first_page(page))  in get_first_page()
779 		return page;  in get_first_page()
781 		return page->first_page;  in get_first_page()
784 static struct page *get_next_page(struct page *page)  in get_next_page()  argument
786 	struct page *next;  in get_next_page()
788 	if (is_last_page(page))  in get_next_page()
790 	else if (is_first_page(page))  in get_next_page()
791 		next = (struct page *)page_private(page);  in get_next_page()
793 		next = list_entry(page->lru.next, struct page, lru);  in get_next_page()
802 static void *location_to_obj(struct page *page, unsigned long obj_idx)  in location_to_obj()  argument
806 	if (!page) {  in location_to_obj()
811 	obj = page_to_pfn(page) << OBJ_INDEX_BITS;  in location_to_obj()
823 static void obj_to_location(unsigned long obj, struct page **page,  in obj_to_location()  argument
827 	*page = pfn_to_page(obj >> OBJ_INDEX_BITS);  in obj_to_location()
836 static unsigned long obj_to_head(struct size_class *class, struct page *page,  in obj_to_head()  argument
840 		VM_BUG_ON(!is_first_page(page));  in obj_to_head()
841 		return *(unsigned long *)page_private(page);  in obj_to_head()
846 static unsigned long obj_idx_to_offset(struct page *page,  in obj_idx_to_offset()  argument
851 	if (!is_first_page(page))  in obj_idx_to_offset()
852 		off = page->index;  in obj_idx_to_offset()
876 static void reset_page(struct page *page)  in reset_page()  argument
878 	clear_bit(PG_private, &page->flags);  in reset_page()
879 	clear_bit(PG_private_2, &page->flags);  in reset_page()
880 	set_page_private(page, 0);  in reset_page()
881 	page->mapping = NULL;  in reset_page()
882 	page->freelist = NULL;  in reset_page()
883 	page_mapcount_reset(page);  in reset_page()
886 static void free_zspage(struct page *first_page)  in free_zspage()
888 	struct page *nextp, *tmp, *head_extra;  in free_zspage()
893 	head_extra = (struct page *)page_private(first_page);  in free_zspage()
912 static void init_zspage(struct page *first_page, struct size_class *class)  in init_zspage()
915 	struct page *page = first_page;  in init_zspage()  local
918 	while (page) {  in init_zspage()
919 		struct page *next_page;  in init_zspage()
930 		if (page != first_page)  in init_zspage()
931 			page->index = off;  in init_zspage()
933 		vaddr = kmap_atomic(page);  in init_zspage()
937 			link->next = location_to_obj(page, i++);  in init_zspage()
946 		next_page = get_next_page(page);  in init_zspage()
949 		page = next_page;  in init_zspage()
957 static struct page *alloc_zspage(struct size_class *class, gfp_t flags)  in alloc_zspage()
960 	struct page *first_page = NULL, *uninitialized_var(prev_page);  in alloc_zspage()
975 		struct page *page;  in alloc_zspage()  local
977 		page = alloc_page(flags);  in alloc_zspage()
978 		if (!page)  in alloc_zspage()
981 		INIT_LIST_HEAD(&page->lru);  in alloc_zspage()
983 			SetPagePrivate(page);  in alloc_zspage()
984 			set_page_private(page, 0);  in alloc_zspage()
985 			first_page = page;  in alloc_zspage()
989 			set_page_private(first_page, (unsigned long)page);  in alloc_zspage()
991 			page->first_page = first_page;  in alloc_zspage()
993 			list_add(&page->lru, &prev_page->lru);  in alloc_zspage()
995 			SetPagePrivate2(page);  in alloc_zspage()
996 		prev_page = page;  in alloc_zspage()
1016 static struct page *find_get_zspage(struct size_class *class)  in find_get_zspage()
1019 	struct page *page;  in find_get_zspage()  local
1022 		page = class->fullness_list[i];  in find_get_zspage()
1023 		if (page)  in find_get_zspage()
1027 	return page;  in find_get_zspage()
1053 				struct page *pages[2], int off, int size)  in __zs_map_object()
1061 				struct page *pages[2], int off, int size)  in __zs_unmap_object()
1091 			struct page *pages[2], int off, int size)  in __zs_map_object()
1119 			struct page *pages[2], int off, int size)  in __zs_unmap_object()
1234 static bool zspage_full(struct page *page)  in zspage_full()  argument
1236 	BUG_ON(!is_first_page(page));  in zspage_full()
1238 	return page->inuse == page->objects;  in zspage_full()
1264 	struct page *page;  in zs_map_object()  local
1271 	struct page *pages[2];  in zs_map_object()
1287 	obj_to_location(obj, &page, &obj_idx);  in zs_map_object()
1288 	get_zspage_mapping(get_first_page(page), &class_idx, &fg);  in zs_map_object()
1290 	off = obj_idx_to_offset(page, obj_idx, class->size);  in zs_map_object()
1296 		area->vm_addr = kmap_atomic(page);  in zs_map_object()
1302 	pages[0] = page;  in zs_map_object()
1303 	pages[1] = get_next_page(page);  in zs_map_object()
1317 	struct page *page;  in zs_unmap_object()  local
1328 	obj_to_location(obj, &page, &obj_idx);  in zs_unmap_object()
1329 	get_zspage_mapping(get_first_page(page), &class_idx, &fg);  in zs_unmap_object()
1331 	off = obj_idx_to_offset(page, obj_idx, class->size);  in zs_unmap_object()
1337 		struct page *pages[2];  in zs_unmap_object()
1339 		pages[0] = page;  in zs_unmap_object()
1340 		pages[1] = get_next_page(page);  in zs_unmap_object()
1350 static unsigned long obj_malloc(struct page *first_page,  in obj_malloc()
1356 	struct page *m_page;  in obj_malloc()
1395 	struct page *first_page;  in zs_malloc()
1442 	struct page *first_page, *f_page;  in obj_free()
1472 	struct page *first_page, *f_page;  in zs_free()
1509 	struct page *s_page, *d_page;  in zs_object_copy()
1575 static unsigned long find_alloced_obj(struct page *page, int index,  in find_alloced_obj()  argument
1581 	void *addr = kmap_atomic(page);  in find_alloced_obj()
1583 	if (!is_first_page(page))  in find_alloced_obj()
1584 		offset = page->index;  in find_alloced_obj()
1588 		head = obj_to_head(class, page, addr + offset);  in find_alloced_obj()
1606 	struct page *s_page;
1609 	struct page *d_page;
1622 	struct page *s_page = cc->s_page;  in migrate_zspage()
1623 	struct page *d_page = cc->d_page;  in migrate_zspage()
1670 static struct page *alloc_target_page(struct size_class *class)  in alloc_target_page()
1673 	struct page *page;  in alloc_target_page()  local
1676 		page = class->fullness_list[i];  in alloc_target_page()
1677 		if (page) {  in alloc_target_page()
1678 			remove_zspage(page, class, i);  in alloc_target_page()
1683 	return page;  in alloc_target_page()
1687 				struct page *first_page)  in putback_zspage()
1707 static struct page *isolate_source_page(struct size_class *class)  in isolate_source_page()
1709 	struct page *page;  in isolate_source_page()  local
1711 	page = class->fullness_list[ZS_ALMOST_EMPTY];  in isolate_source_page()
1712 	if (page)  in isolate_source_page()
1713 		remove_zspage(page, class, ZS_ALMOST_EMPTY);  in isolate_source_page()
1715 	return page;  in isolate_source_page()
1723 	struct page *src_page;  in __zs_compact()
1724 	struct page *dst_page = NULL;  in __zs_compact()