Lines Matching refs:page
52 static void __page_cache_release(struct page *page) in __page_cache_release() argument
54 if (PageLRU(page)) { in __page_cache_release()
55 struct zone *zone = page_zone(page); in __page_cache_release()
60 lruvec = mem_cgroup_page_lruvec(page, zone); in __page_cache_release()
61 VM_BUG_ON_PAGE(!PageLRU(page), page); in __page_cache_release()
62 __ClearPageLRU(page); in __page_cache_release()
63 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in __page_cache_release()
66 mem_cgroup_uncharge(page); in __page_cache_release()
69 static void __put_single_page(struct page *page) in __put_single_page() argument
71 __page_cache_release(page); in __put_single_page()
72 free_hot_cold_page(page, false); in __put_single_page()
75 static void __put_compound_page(struct page *page) in __put_compound_page() argument
85 if (!PageHuge(page)) in __put_compound_page()
86 __page_cache_release(page); in __put_compound_page()
87 dtor = get_compound_page_dtor(page); in __put_compound_page()
88 (*dtor)(page); in __put_compound_page()
118 void put_unrefcounted_compound_page(struct page *page_head, struct page *page) in put_unrefcounted_compound_page() argument
128 if (likely(PageTail(page))) { in put_unrefcounted_compound_page()
134 VM_BUG_ON_PAGE(page_mapcount(page) != 0, page); in put_unrefcounted_compound_page()
161 if (put_page_testzero(page)) in put_unrefcounted_compound_page()
162 __put_single_page(page); in put_unrefcounted_compound_page()
166 void put_refcounted_compound_page(struct page *page_head, struct page *page) in put_refcounted_compound_page() argument
168 if (likely(page != page_head && get_page_unless_zero(page_head))) { in put_refcounted_compound_page()
178 if (unlikely(!PageTail(page))) { in put_refcounted_compound_page()
200 if (put_page_testzero(page)) in put_refcounted_compound_page()
201 __put_single_page(page); in put_refcounted_compound_page()
204 VM_BUG_ON_PAGE(page_head != page->first_page, page); in put_refcounted_compound_page()
214 VM_BUG_ON_PAGE(page_mapcount(page) <= 0, page); in put_refcounted_compound_page()
215 atomic_dec(&page->_mapcount); in put_refcounted_compound_page()
217 VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page); in put_refcounted_compound_page()
228 VM_BUG_ON_PAGE(PageTail(page), page); in put_refcounted_compound_page()
233 static void put_compound_page(struct page *page) in put_compound_page() argument
235 struct page *page_head; in put_compound_page()
242 if (likely(!PageTail(page))) { in put_compound_page()
243 if (put_page_testzero(page)) { in put_compound_page()
248 if (PageHead(page)) in put_compound_page()
249 __put_compound_page(page); in put_compound_page()
251 __put_single_page(page); in put_compound_page()
265 page_head = compound_head_by_tail(page); in put_compound_page()
267 put_unrefcounted_compound_page(page_head, page); in put_compound_page()
269 put_refcounted_compound_page(page_head, page); in put_compound_page()
272 void put_page(struct page *page) in put_page() argument
274 if (unlikely(PageCompound(page))) in put_page()
275 put_compound_page(page); in put_page()
276 else if (put_page_testzero(page)) in put_page()
277 __put_single_page(page); in put_page()
285 bool __get_page_tail(struct page *page) in __get_page_tail() argument
297 struct page *page_head = compound_head(page); in __get_page_tail()
302 if (likely(PageTail(page))) { in __get_page_tail()
309 __get_page_tail_foll(page, true); in __get_page_tail()
326 if (likely(page != page_head && get_page_unless_zero(page_head))) { in __get_page_tail()
335 if (likely(PageTail(page))) { in __get_page_tail()
336 __get_page_tail_foll(page, false); in __get_page_tail()
357 struct page *victim; in put_pages_list()
359 victim = list_entry(pages->prev, struct page, lru); in put_pages_list()
380 struct page **pages) in get_kernel_pages()
407 int get_kernel_page(unsigned long start, int write, struct page **pages) in get_kernel_page()
419 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), in pagevec_lru_move_fn() argument
428 struct page *page = pvec->pages[i]; in pagevec_lru_move_fn() local
429 struct zone *pagezone = page_zone(page); in pagevec_lru_move_fn()
438 lruvec = mem_cgroup_page_lruvec(page, zone); in pagevec_lru_move_fn()
439 (*move_fn)(page, lruvec, arg); in pagevec_lru_move_fn()
447 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, in pagevec_move_tail_fn() argument
452 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in pagevec_move_tail_fn()
453 enum lru_list lru = page_lru_base_type(page); in pagevec_move_tail_fn()
454 list_move_tail(&page->lru, &lruvec->lists[lru]); in pagevec_move_tail_fn()
476 void rotate_reclaimable_page(struct page *page) in rotate_reclaimable_page() argument
478 if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) && in rotate_reclaimable_page()
479 !PageUnevictable(page) && PageLRU(page)) { in rotate_reclaimable_page()
483 page_cache_get(page); in rotate_reclaimable_page()
486 if (!pagevec_add(pvec, page)) in rotate_reclaimable_page()
502 static void __activate_page(struct page *page, struct lruvec *lruvec, in __activate_page() argument
505 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in __activate_page()
506 int file = page_is_file_cache(page); in __activate_page()
507 int lru = page_lru_base_type(page); in __activate_page()
509 del_page_from_lru_list(page, lruvec, lru); in __activate_page()
510 SetPageActive(page); in __activate_page()
512 add_page_to_lru_list(page, lruvec, lru); in __activate_page()
513 trace_mm_lru_activate(page); in __activate_page()
536 void activate_page(struct page *page) in activate_page() argument
538 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in activate_page()
541 page_cache_get(page); in activate_page()
542 if (!pagevec_add(pvec, page)) in activate_page()
558 void activate_page(struct page *page) in activate_page() argument
560 struct zone *zone = page_zone(page); in activate_page()
563 __activate_page(page, mem_cgroup_page_lruvec(page, zone), NULL); in activate_page()
568 static void __lru_cache_activate_page(struct page *page) in __lru_cache_activate_page() argument
584 struct page *pagevec_page = pvec->pages[i]; in __lru_cache_activate_page()
586 if (pagevec_page == page) { in __lru_cache_activate_page()
587 SetPageActive(page); in __lru_cache_activate_page()
605 void mark_page_accessed(struct page *page) in mark_page_accessed() argument
607 if (!PageActive(page) && !PageUnevictable(page) && in mark_page_accessed()
608 PageReferenced(page)) { in mark_page_accessed()
616 if (PageLRU(page)) in mark_page_accessed()
617 activate_page(page); in mark_page_accessed()
619 __lru_cache_activate_page(page); in mark_page_accessed()
620 ClearPageReferenced(page); in mark_page_accessed()
621 if (page_is_file_cache(page)) in mark_page_accessed()
622 workingset_activation(page); in mark_page_accessed()
623 } else if (!PageReferenced(page)) { in mark_page_accessed()
624 SetPageReferenced(page); in mark_page_accessed()
629 static void __lru_cache_add(struct page *page) in __lru_cache_add() argument
633 page_cache_get(page); in __lru_cache_add()
636 pagevec_add(pvec, page); in __lru_cache_add()
644 void lru_cache_add_anon(struct page *page) in lru_cache_add_anon() argument
646 if (PageActive(page)) in lru_cache_add_anon()
647 ClearPageActive(page); in lru_cache_add_anon()
648 __lru_cache_add(page); in lru_cache_add_anon()
651 void lru_cache_add_file(struct page *page) in lru_cache_add_file() argument
653 if (PageActive(page)) in lru_cache_add_file()
654 ClearPageActive(page); in lru_cache_add_file()
655 __lru_cache_add(page); in lru_cache_add_file()
668 void lru_cache_add(struct page *page) in lru_cache_add() argument
670 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); in lru_cache_add()
671 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add()
672 __lru_cache_add(page); in lru_cache_add()
685 void add_page_to_unevictable_list(struct page *page) in add_page_to_unevictable_list() argument
687 struct zone *zone = page_zone(page); in add_page_to_unevictable_list()
691 lruvec = mem_cgroup_page_lruvec(page, zone); in add_page_to_unevictable_list()
692 ClearPageActive(page); in add_page_to_unevictable_list()
693 SetPageUnevictable(page); in add_page_to_unevictable_list()
694 SetPageLRU(page); in add_page_to_unevictable_list()
695 add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE); in add_page_to_unevictable_list()
709 void lru_cache_add_active_or_unevictable(struct page *page, in lru_cache_add_active_or_unevictable() argument
712 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add_active_or_unevictable()
715 SetPageActive(page); in lru_cache_add_active_or_unevictable()
716 lru_cache_add(page); in lru_cache_add_active_or_unevictable()
720 if (!TestSetPageMlocked(page)) { in lru_cache_add_active_or_unevictable()
726 __mod_zone_page_state(page_zone(page), NR_MLOCK, in lru_cache_add_active_or_unevictable()
727 hpage_nr_pages(page)); in lru_cache_add_active_or_unevictable()
730 add_page_to_unevictable_list(page); in lru_cache_add_active_or_unevictable()
754 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, in lru_deactivate_file_fn() argument
760 if (!PageLRU(page)) in lru_deactivate_file_fn()
763 if (PageUnevictable(page)) in lru_deactivate_file_fn()
767 if (page_mapped(page)) in lru_deactivate_file_fn()
770 active = PageActive(page); in lru_deactivate_file_fn()
771 file = page_is_file_cache(page); in lru_deactivate_file_fn()
772 lru = page_lru_base_type(page); in lru_deactivate_file_fn()
774 del_page_from_lru_list(page, lruvec, lru + active); in lru_deactivate_file_fn()
775 ClearPageActive(page); in lru_deactivate_file_fn()
776 ClearPageReferenced(page); in lru_deactivate_file_fn()
777 add_page_to_lru_list(page, lruvec, lru); in lru_deactivate_file_fn()
779 if (PageWriteback(page) || PageDirty(page)) { in lru_deactivate_file_fn()
785 SetPageReclaim(page); in lru_deactivate_file_fn()
791 list_move_tail(&page->lru, &lruvec->lists[lru]); in lru_deactivate_file_fn()
837 void deactivate_file_page(struct page *page) in deactivate_file_page() argument
843 if (PageUnevictable(page)) in deactivate_file_page()
846 if (likely(get_page_unless_zero(page))) { in deactivate_file_page()
849 if (!pagevec_add(pvec, page)) in deactivate_file_page()
907 void release_pages(struct page **pages, int nr, bool cold) in release_pages()
917 struct page *page = pages[i]; in release_pages() local
919 if (unlikely(PageCompound(page))) { in release_pages()
924 put_compound_page(page); in release_pages()
938 if (!put_page_testzero(page)) in release_pages()
941 if (PageLRU(page)) { in release_pages()
942 struct zone *pagezone = page_zone(page); in release_pages()
953 lruvec = mem_cgroup_page_lruvec(page, zone); in release_pages()
954 VM_BUG_ON_PAGE(!PageLRU(page), page); in release_pages()
955 __ClearPageLRU(page); in release_pages()
956 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in release_pages()
960 __ClearPageActive(page); in release_pages()
962 list_add(&page->lru, &pages_to_free); in release_pages()
992 void lru_add_page_tail(struct page *page, struct page *page_tail, in lru_add_page_tail() argument
997 VM_BUG_ON_PAGE(!PageHead(page), page); in lru_add_page_tail()
998 VM_BUG_ON_PAGE(PageCompound(page_tail), page); in lru_add_page_tail()
999 VM_BUG_ON_PAGE(PageLRU(page_tail), page); in lru_add_page_tail()
1006 if (likely(PageLRU(page))) in lru_add_page_tail()
1007 list_add_tail(&page_tail->lru, &page->lru); in lru_add_page_tail()
1026 if (!PageUnevictable(page)) in lru_add_page_tail()
1031 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, in __pagevec_lru_add_fn() argument
1034 int file = page_is_file_cache(page); in __pagevec_lru_add_fn()
1035 int active = PageActive(page); in __pagevec_lru_add_fn()
1036 enum lru_list lru = page_lru(page); in __pagevec_lru_add_fn()
1038 VM_BUG_ON_PAGE(PageLRU(page), page); in __pagevec_lru_add_fn()
1040 SetPageLRU(page); in __pagevec_lru_add_fn()
1041 add_page_to_lru_list(page, lruvec, lru); in __pagevec_lru_add_fn()
1043 trace_mm_lru_insertion(page, lru); in __pagevec_lru_add_fn()
1100 struct page *page = pvec->pages[i]; in pagevec_remove_exceptionals() local
1101 if (!radix_tree_exceptional_entry(page)) in pagevec_remove_exceptionals()
1102 pvec->pages[j++] = page; in pagevec_remove_exceptionals()