53 static void __page_cache_release(struct page *page)  in __page_cache_release()  argument
55 	if (PageLRU(page)) {  in __page_cache_release()
56 		struct zone *zone = page_zone(page);  in __page_cache_release()
61 		lruvec = mem_cgroup_page_lruvec(page, zone);  in __page_cache_release()
62 		VM_BUG_ON_PAGE(!PageLRU(page), page);  in __page_cache_release()
63 		__ClearPageLRU(page);  in __page_cache_release()
64 		del_page_from_lru_list(page, lruvec, page_off_lru(page));  in __page_cache_release()
67 	mem_cgroup_uncharge(page);  in __page_cache_release()
70 static void __put_single_page(struct page *page)  in __put_single_page()  argument
72 	__page_cache_release(page);  in __put_single_page()
73 	free_hot_cold_page(page, false);  in __put_single_page()
76 static void __put_compound_page(struct page *page)  in __put_compound_page()  argument
86 	if (!PageHuge(page))  in __put_compound_page()
87 		__page_cache_release(page);  in __put_compound_page()
88 	dtor = get_compound_page_dtor(page);  in __put_compound_page()
89 	(*dtor)(page);  in __put_compound_page()
119 void put_unrefcounted_compound_page(struct page *page_head, struct page *page)  in put_unrefcounted_compound_page()  argument
129 	if (likely(PageTail(page))) {  in put_unrefcounted_compound_page()
161 		if (put_page_testzero(page))  in put_unrefcounted_compound_page()
162 			__put_single_page(page);  in put_unrefcounted_compound_page()
166 void put_refcounted_compound_page(struct page *page_head, struct page *page)  in put_refcounted_compound_page()  argument
168 	if (likely(page != page_head && get_page_unless_zero(page_head))) {  in put_refcounted_compound_page()
178 		if (unlikely(!PageTail(page))) {  in put_refcounted_compound_page()
200 			if (put_page_testzero(page))  in put_refcounted_compound_page()
201 				__put_single_page(page);  in put_refcounted_compound_page()
204 		VM_BUG_ON_PAGE(page_head != compound_head(page), page);  in put_refcounted_compound_page()
214 		VM_BUG_ON_PAGE(page_mapcount(page) <= 0, page);  in put_refcounted_compound_page()
215 		atomic_dec(&page->_mapcount);  in put_refcounted_compound_page()
217 		VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);  in put_refcounted_compound_page()
228 		VM_BUG_ON_PAGE(PageTail(page), page);  in put_refcounted_compound_page()
233 static void put_compound_page(struct page *page)  in put_compound_page()  argument
235 	struct page *page_head;  in put_compound_page()
242 	if (likely(!PageTail(page))) {  in put_compound_page()
243 		if (put_page_testzero(page)) {  in put_compound_page()
248 			if (PageHead(page))  in put_compound_page()
249 				__put_compound_page(page);  in put_compound_page()
251 				__put_single_page(page);  in put_compound_page()
265 	page_head = compound_head(page);  in put_compound_page()
267 		put_unrefcounted_compound_page(page_head, page);  in put_compound_page()
269 		put_refcounted_compound_page(page_head, page);  in put_compound_page()
272 void put_page(struct page *page)  in put_page()  argument
274 	if (unlikely(PageCompound(page)))  in put_page()
275 		put_compound_page(page);  in put_page()
276 	else if (put_page_testzero(page))  in put_page()
277 		__put_single_page(page);  in put_page()
285 bool __get_page_tail(struct page *page)  in __get_page_tail()  argument
297 	struct page *page_head = compound_head(page);  in __get_page_tail()
302 		if (likely(PageTail(page))) {  in __get_page_tail()
309 			__get_page_tail_foll(page, true);  in __get_page_tail()
326 	if (likely(page != page_head && get_page_unless_zero(page_head))) {  in __get_page_tail()
335 		if (likely(PageTail(page))) {  in __get_page_tail()
336 			__get_page_tail_foll(page, false);  in __get_page_tail()
357 		struct page *victim;  in put_pages_list()
359 		victim = list_entry(pages->prev, struct page, lru);  in put_pages_list()
380 		struct page **pages)  in get_kernel_pages()
407 int get_kernel_page(unsigned long start, int write, struct page **pages)  in get_kernel_page()
419 	void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),  in pagevec_lru_move_fn()  argument
428 		struct page *page = pvec->pages[i];  in pagevec_lru_move_fn()  local
429 		struct zone *pagezone = page_zone(page);  in pagevec_lru_move_fn()
438 		lruvec = mem_cgroup_page_lruvec(page, zone);  in pagevec_lru_move_fn()
439 		(*move_fn)(page, lruvec, arg);  in pagevec_lru_move_fn()
447 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,  in pagevec_move_tail_fn()  argument
452 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {  in pagevec_move_tail_fn()
453 		enum lru_list lru = page_lru_base_type(page);  in pagevec_move_tail_fn()
454 		list_move_tail(&page->lru, &lruvec->lists[lru]);  in pagevec_move_tail_fn()
476 void rotate_reclaimable_page(struct page *page)  in rotate_reclaimable_page()  argument
478 	if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&  in rotate_reclaimable_page()
479 	    !PageUnevictable(page) && PageLRU(page)) {  in rotate_reclaimable_page()
483 		page_cache_get(page);  in rotate_reclaimable_page()
486 		if (!pagevec_add(pvec, page))  in rotate_reclaimable_page()
502 static void __activate_page(struct page *page, struct lruvec *lruvec,  in __activate_page()  argument
505 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {  in __activate_page()
506 		int file = page_is_file_cache(page);  in __activate_page()
507 		int lru = page_lru_base_type(page);  in __activate_page()
509 		del_page_from_lru_list(page, lruvec, lru);  in __activate_page()
510 		SetPageActive(page);  in __activate_page()
512 		add_page_to_lru_list(page, lruvec, lru);  in __activate_page()
513 		trace_mm_lru_activate(page);  in __activate_page()
536 void activate_page(struct page *page)  in activate_page()  argument
538 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {  in activate_page()
541 		page_cache_get(page);  in activate_page()
542 		if (!pagevec_add(pvec, page))  in activate_page()
558 void activate_page(struct page *page)  in activate_page()  argument
560 	struct zone *zone = page_zone(page);  in activate_page()
563 	__activate_page(page, mem_cgroup_page_lruvec(page, zone), NULL);  in activate_page()
568 static void __lru_cache_activate_page(struct page *page)  in __lru_cache_activate_page()  argument
584 		struct page *pagevec_page = pvec->pages[i];  in __lru_cache_activate_page()
586 		if (pagevec_page == page) {  in __lru_cache_activate_page()
587 			SetPageActive(page);  in __lru_cache_activate_page()
605 void mark_page_accessed(struct page *page)  in mark_page_accessed()  argument
607 	if (!PageActive(page) && !PageUnevictable(page) &&  in mark_page_accessed()
608 			PageReferenced(page)) {  in mark_page_accessed()
616 		if (PageLRU(page))  in mark_page_accessed()
617 			activate_page(page);  in mark_page_accessed()
619 			__lru_cache_activate_page(page);  in mark_page_accessed()
620 		ClearPageReferenced(page);  in mark_page_accessed()
621 		if (page_is_file_cache(page))  in mark_page_accessed()
622 			workingset_activation(page);  in mark_page_accessed()
623 	} else if (!PageReferenced(page)) {  in mark_page_accessed()
624 		SetPageReferenced(page);  in mark_page_accessed()
626 	if (page_is_idle(page))  in mark_page_accessed()
627 		clear_page_idle(page);  in mark_page_accessed()
631 static void __lru_cache_add(struct page *page)  in __lru_cache_add()  argument
635 	page_cache_get(page);  in __lru_cache_add()
638 	pagevec_add(pvec, page);  in __lru_cache_add()
646 void lru_cache_add_anon(struct page *page)  in lru_cache_add_anon()  argument
648 	if (PageActive(page))  in lru_cache_add_anon()
649 		ClearPageActive(page);  in lru_cache_add_anon()
650 	__lru_cache_add(page);  in lru_cache_add_anon()
653 void lru_cache_add_file(struct page *page)  in lru_cache_add_file()  argument
655 	if (PageActive(page))  in lru_cache_add_file()
656 		ClearPageActive(page);  in lru_cache_add_file()
657 	__lru_cache_add(page);  in lru_cache_add_file()
670 void lru_cache_add(struct page *page)  in lru_cache_add()  argument
672 	VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);  in lru_cache_add()
673 	VM_BUG_ON_PAGE(PageLRU(page), page);  in lru_cache_add()
674 	__lru_cache_add(page);  in lru_cache_add()
687 void add_page_to_unevictable_list(struct page *page)  in add_page_to_unevictable_list()  argument
689 	struct zone *zone = page_zone(page);  in add_page_to_unevictable_list()
693 	lruvec = mem_cgroup_page_lruvec(page, zone);  in add_page_to_unevictable_list()
694 	ClearPageActive(page);  in add_page_to_unevictable_list()
695 	SetPageUnevictable(page);  in add_page_to_unevictable_list()
696 	SetPageLRU(page);  in add_page_to_unevictable_list()
697 	add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE);  in add_page_to_unevictable_list()
711 void lru_cache_add_active_or_unevictable(struct page *page,  in lru_cache_add_active_or_unevictable()  argument
714 	VM_BUG_ON_PAGE(PageLRU(page), page);  in lru_cache_add_active_or_unevictable()
717 		SetPageActive(page);  in lru_cache_add_active_or_unevictable()
718 		lru_cache_add(page);  in lru_cache_add_active_or_unevictable()
722 	if (!TestSetPageMlocked(page)) {  in lru_cache_add_active_or_unevictable()
728 		__mod_zone_page_state(page_zone(page), NR_MLOCK,  in lru_cache_add_active_or_unevictable()
729 				    hpage_nr_pages(page));  in lru_cache_add_active_or_unevictable()
732 	add_page_to_unevictable_list(page);  in lru_cache_add_active_or_unevictable()
756 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,  in lru_deactivate_file_fn()  argument
762 	if (!PageLRU(page))  in lru_deactivate_file_fn()
765 	if (PageUnevictable(page))  in lru_deactivate_file_fn()
769 	if (page_mapped(page))  in lru_deactivate_file_fn()
772 	active = PageActive(page);  in lru_deactivate_file_fn()
773 	file = page_is_file_cache(page);  in lru_deactivate_file_fn()
774 	lru = page_lru_base_type(page);  in lru_deactivate_file_fn()
776 	del_page_from_lru_list(page, lruvec, lru + active);  in lru_deactivate_file_fn()
777 	ClearPageActive(page);  in lru_deactivate_file_fn()
778 	ClearPageReferenced(page);  in lru_deactivate_file_fn()
779 	add_page_to_lru_list(page, lruvec, lru);  in lru_deactivate_file_fn()
781 	if (PageWriteback(page) || PageDirty(page)) {  in lru_deactivate_file_fn()
787 		SetPageReclaim(page);  in lru_deactivate_file_fn()
793 		list_move_tail(&page->lru, &lruvec->lists[lru]);  in lru_deactivate_file_fn()
839 void deactivate_file_page(struct page *page)  in deactivate_file_page()  argument
845 	if (PageUnevictable(page))  in deactivate_file_page()
848 	if (likely(get_page_unless_zero(page))) {  in deactivate_file_page()
851 		if (!pagevec_add(pvec, page))  in deactivate_file_page()
909 void release_pages(struct page **pages, int nr, bool cold)  in release_pages()
919 		struct page *page = pages[i];  in release_pages()  local
921 		if (unlikely(PageCompound(page))) {  in release_pages()
926 			put_compound_page(page);  in release_pages()
940 		if (!put_page_testzero(page))  in release_pages()
943 		if (PageLRU(page)) {  in release_pages()
944 			struct zone *pagezone = page_zone(page);  in release_pages()
955 			lruvec = mem_cgroup_page_lruvec(page, zone);  in release_pages()
956 			VM_BUG_ON_PAGE(!PageLRU(page), page);  in release_pages()
957 			__ClearPageLRU(page);  in release_pages()
958 			del_page_from_lru_list(page, lruvec, page_off_lru(page));  in release_pages()
962 		__ClearPageActive(page);  in release_pages()
964 		list_add(&page->lru, &pages_to_free);  in release_pages()
994 void lru_add_page_tail(struct page *page, struct page *page_tail,  in lru_add_page_tail()  argument
999 	VM_BUG_ON_PAGE(!PageHead(page), page);  in lru_add_page_tail()
1000 	VM_BUG_ON_PAGE(PageCompound(page_tail), page);  in lru_add_page_tail()
1001 	VM_BUG_ON_PAGE(PageLRU(page_tail), page);  in lru_add_page_tail()
1008 	if (likely(PageLRU(page)))  in lru_add_page_tail()
1009 		list_add_tail(&page_tail->lru, &page->lru);  in lru_add_page_tail()
1028 	if (!PageUnevictable(page))  in lru_add_page_tail()
1033 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,  in __pagevec_lru_add_fn()  argument
1036 	int file = page_is_file_cache(page);  in __pagevec_lru_add_fn()
1037 	int active = PageActive(page);  in __pagevec_lru_add_fn()
1038 	enum lru_list lru = page_lru(page);  in __pagevec_lru_add_fn()
1040 	VM_BUG_ON_PAGE(PageLRU(page), page);  in __pagevec_lru_add_fn()
1042 	SetPageLRU(page);  in __pagevec_lru_add_fn()
1043 	add_page_to_lru_list(page, lruvec, lru);  in __pagevec_lru_add_fn()
1045 	trace_mm_lru_insertion(page, lru);  in __pagevec_lru_add_fn()
1102 		struct page *page = pvec->pages[i];  in pagevec_remove_exceptionals()  local
1103 		if (!radix_tree_exceptional_entry(page))  in pagevec_remove_exceptionals()
1104 			pvec->pages[j++] = page;  in pagevec_remove_exceptionals()