84 	struct page *page;  in putback_movable_pages()  local
85 	struct page *page2;  in putback_movable_pages()
87 	list_for_each_entry_safe(page, page2, l, lru) {  in putback_movable_pages()
88 		if (unlikely(PageHuge(page))) {  in putback_movable_pages()
89 			putback_active_hugepage(page);  in putback_movable_pages()
92 		list_del(&page->lru);  in putback_movable_pages()
93 		dec_zone_page_state(page, NR_ISOLATED_ANON +  in putback_movable_pages()
94 				page_is_file_cache(page));  in putback_movable_pages()
95 		if (unlikely(isolated_balloon_page(page)))  in putback_movable_pages()
96 			balloon_page_putback(page);  in putback_movable_pages()
98 			putback_lru_page(page);  in putback_movable_pages()
105 static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,  in remove_migration_pte()
185 static void remove_migration_ptes(struct page *old, struct page *new)  in remove_migration_ptes()
205 	struct page *page;  in __migration_entry_wait()  local
216 	page = migration_entry_to_page(entry);  in __migration_entry_wait()
225 	if (!get_page_unless_zero(page))  in __migration_entry_wait()
228 	wait_on_page_locked(page);  in __migration_entry_wait()
229 	put_page(page);  in __migration_entry_wait()
309 		struct page *newpage, struct page *page,  in migrate_page_move_mapping()  argument
318 		if (page_count(page) != expected_count)  in migrate_page_move_mapping()
326  					page_index(page));  in migrate_page_move_mapping()
328 	expected_count += 1 + page_has_private(page);  in migrate_page_move_mapping()
329 	if (page_count(page) != expected_count ||  in migrate_page_move_mapping()
330 		radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {  in migrate_page_move_mapping()
335 	if (!page_freeze_refs(page, expected_count)) {  in migrate_page_move_mapping()
349 		page_unfreeze_refs(page, expected_count);  in migrate_page_move_mapping()
358 	if (PageSwapCache(page)) {  in migrate_page_move_mapping()
360 		set_page_private(newpage, page_private(page));  in migrate_page_move_mapping()
370 	page_unfreeze_refs(page, expected_count - 1);  in migrate_page_move_mapping()
382 	__dec_zone_page_state(page, NR_FILE_PAGES);  in migrate_page_move_mapping()
384 	if (!PageSwapCache(page) && PageSwapBacked(page)) {  in migrate_page_move_mapping()
385 		__dec_zone_page_state(page, NR_SHMEM);  in migrate_page_move_mapping()
398 				   struct page *newpage, struct page *page)  in migrate_huge_page_move_mapping()  argument
404 		if (page_count(page) != 1)  in migrate_huge_page_move_mapping()
412 					page_index(page));  in migrate_huge_page_move_mapping()
414 	expected_count = 2 + page_has_private(page);  in migrate_huge_page_move_mapping()
415 	if (page_count(page) != expected_count ||  in migrate_huge_page_move_mapping()
416 		radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {  in migrate_huge_page_move_mapping()
421 	if (!page_freeze_refs(page, expected_count)) {  in migrate_huge_page_move_mapping()
430 	page_unfreeze_refs(page, expected_count - 1);  in migrate_huge_page_move_mapping()
441 static void __copy_gigantic_page(struct page *dst, struct page *src,  in __copy_gigantic_page()
445 	struct page *dst_base = dst;  in __copy_gigantic_page()
446 	struct page *src_base = src;  in __copy_gigantic_page()
458 static void copy_huge_page(struct page *dst, struct page *src)  in copy_huge_page()
487 void migrate_page_copy(struct page *newpage, struct page *page)  in migrate_page_copy()  argument
491 	if (PageHuge(page) || PageTransHuge(page))  in migrate_page_copy()
492 		copy_huge_page(newpage, page);  in migrate_page_copy()
494 		copy_highpage(newpage, page);  in migrate_page_copy()
496 	if (PageError(page))  in migrate_page_copy()
498 	if (PageReferenced(page))  in migrate_page_copy()
500 	if (PageUptodate(page))  in migrate_page_copy()
502 	if (TestClearPageActive(page)) {  in migrate_page_copy()
503 		VM_BUG_ON_PAGE(PageUnevictable(page), page);  in migrate_page_copy()
505 	} else if (TestClearPageUnevictable(page))  in migrate_page_copy()
507 	if (PageChecked(page))  in migrate_page_copy()
509 	if (PageMappedToDisk(page))  in migrate_page_copy()
512 	if (PageDirty(page)) {  in migrate_page_copy()
513 		clear_page_dirty_for_io(page);  in migrate_page_copy()
521 		if (PageSwapBacked(page))  in migrate_page_copy()
531 	cpupid = page_cpupid_xchg_last(page, -1);  in migrate_page_copy()
534 	mlock_migrate_page(newpage, page);  in migrate_page_copy()
535 	ksm_migrate_page(newpage, page);  in migrate_page_copy()
540 	if (PageSwapCache(page))  in migrate_page_copy()
541 		ClearPageSwapCache(page);  in migrate_page_copy()
542 	ClearPagePrivate(page);  in migrate_page_copy()
543 	set_page_private(page, 0);  in migrate_page_copy()
564 		struct page *newpage, struct page *page,  in migrate_page()  argument
569 	BUG_ON(PageWriteback(page));	/* Writeback must be complete */  in migrate_page()
571 	rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0);  in migrate_page()
576 	migrate_page_copy(newpage, page);  in migrate_page()
588 		struct page *newpage, struct page *page, enum migrate_mode mode)  in buffer_migrate_page()  argument
593 	if (!page_has_buffers(page))  in buffer_migrate_page()
594 		return migrate_page(mapping, newpage, page, mode);  in buffer_migrate_page()
596 	head = page_buffers(page);  in buffer_migrate_page()
598 	rc = migrate_page_move_mapping(mapping, newpage, page, head, mode, 0);  in buffer_migrate_page()
611 	ClearPagePrivate(page);  in buffer_migrate_page()
612 	set_page_private(newpage, page_private(page));  in buffer_migrate_page()
613 	set_page_private(page, 0);  in buffer_migrate_page()
614 	put_page(page);  in buffer_migrate_page()
626 	migrate_page_copy(newpage, page);  in buffer_migrate_page()
644 static int writeout(struct address_space *mapping, struct page *page)  in writeout()  argument
659 	if (!clear_page_dirty_for_io(page))  in writeout()
671 	remove_migration_ptes(page, page);  in writeout()
673 	rc = mapping->a_ops->writepage(page, &wbc);  in writeout()
677 		lock_page(page);  in writeout()
686 	struct page *newpage, struct page *page, enum migrate_mode mode)  in fallback_migrate_page()  argument
688 	if (PageDirty(page)) {  in fallback_migrate_page()
692 		return writeout(mapping, page);  in fallback_migrate_page()
699 	if (page_has_private(page) &&  in fallback_migrate_page()
700 	    !try_to_release_page(page, GFP_KERNEL))  in fallback_migrate_page()
703 	return migrate_page(mapping, newpage, page, mode);  in fallback_migrate_page()
717 static int move_to_new_page(struct page *newpage, struct page *page,  in move_to_new_page()  argument
732 	newpage->index = page->index;  in move_to_new_page()
733 	newpage->mapping = page->mapping;  in move_to_new_page()
734 	if (PageSwapBacked(page))  in move_to_new_page()
737 	mapping = page_mapping(page);  in move_to_new_page()
739 		rc = migrate_page(mapping, newpage, page, mode);  in move_to_new_page()
748 						newpage, page, mode);  in move_to_new_page()
750 		rc = fallback_migrate_page(mapping, newpage, page, mode);  in move_to_new_page()
755 		mem_cgroup_migrate(page, newpage, false);  in move_to_new_page()
757 			remove_migration_ptes(page, newpage);  in move_to_new_page()
758 		page->mapping = NULL;  in move_to_new_page()
766 static int __unmap_and_move(struct page *page, struct page *newpage,  in __unmap_and_move()  argument
773 	if (!trylock_page(page)) {  in __unmap_and_move()
793 		lock_page(page);  in __unmap_and_move()
796 	if (PageWriteback(page)) {  in __unmap_and_move()
809 		wait_on_page_writeback(page);  in __unmap_and_move()
819 	if (PageAnon(page) && !PageKsm(page)) {  in __unmap_and_move()
824 		anon_vma = page_get_anon_vma(page);  in __unmap_and_move()
829 		} else if (PageSwapCache(page)) {  in __unmap_and_move()
847 	if (unlikely(isolated_balloon_page(page))) {  in __unmap_and_move()
855 		rc = balloon_page_migrate(newpage, page, mode);  in __unmap_and_move()
871 	if (!page->mapping) {  in __unmap_and_move()
872 		VM_BUG_ON_PAGE(PageAnon(page), page);  in __unmap_and_move()
873 		if (page_has_private(page)) {  in __unmap_and_move()
874 			try_to_free_buffers(page);  in __unmap_and_move()
881 	if (page_mapped(page)) {  in __unmap_and_move()
882 		try_to_unmap(page,  in __unmap_and_move()
888 	if (!page_mapped(page))  in __unmap_and_move()
889 		rc = move_to_new_page(newpage, page, page_was_mapped, mode);  in __unmap_and_move()
892 		remove_migration_ptes(page, page);  in __unmap_and_move()
899 	unlock_page(page);  in __unmap_and_move()
920 				   unsigned long private, struct page *page,  in unmap_and_move()  argument
926 	struct page *newpage = get_new_page(page, private, &result);  in unmap_and_move()
931 	if (page_count(page) == 1) {  in unmap_and_move()
936 	if (unlikely(PageTransHuge(page)))  in unmap_and_move()
937 		if (unlikely(split_huge_page(page)))  in unmap_and_move()
940 	rc = __unmap_and_move(page, newpage, force, mode);  in unmap_and_move()
950 		list_del(&page->lru);  in unmap_and_move()
951 		dec_zone_page_state(page, NR_ISOLATED_ANON +  in unmap_and_move()
952 				page_is_file_cache(page));  in unmap_and_move()
955 			put_page(page);  in unmap_and_move()
957 			putback_lru_page(page);  in unmap_and_move()
1003 				struct page *hpage, int force,  in unmap_and_move_huge_page()
1009 	struct page *new_hpage;  in unmap_and_move_huge_page()
1110 	struct page *page;  in migrate_pages()  local
1111 	struct page *page2;  in migrate_pages()
1121 		list_for_each_entry_safe(page, page2, from, lru) {  in migrate_pages()
1124 			if (PageHuge(page))  in migrate_pages()
1126 						put_new_page, private, page,  in migrate_pages()
1130 						private, page, pass > 2, mode,  in migrate_pages()
1174 	struct page *page;  member
1179 static struct page *new_page_node(struct page *p, unsigned long private,  in new_page_node()
1184 	while (pm->node != MAX_NUMNODES && pm->page != p)  in new_page_node()
1221 		struct page *page;  in do_move_page_to_node_array()  local
1228 		page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT);  in do_move_page_to_node_array()
1230 		err = PTR_ERR(page);  in do_move_page_to_node_array()
1231 		if (IS_ERR(page))  in do_move_page_to_node_array()
1235 		if (!page)  in do_move_page_to_node_array()
1239 		if (PageReserved(page))  in do_move_page_to_node_array()
1242 		pp->page = page;  in do_move_page_to_node_array()
1243 		err = page_to_nid(page);  in do_move_page_to_node_array()
1252 		if (page_mapcount(page) > 1 &&  in do_move_page_to_node_array()
1256 		if (PageHuge(page)) {  in do_move_page_to_node_array()
1257 			if (PageHead(page))  in do_move_page_to_node_array()
1258 				isolate_huge_page(page, &pagelist);  in do_move_page_to_node_array()
1262 		err = isolate_lru_page(page);  in do_move_page_to_node_array()
1264 			list_add_tail(&page->lru, &pagelist);  in do_move_page_to_node_array()
1265 			inc_zone_page_state(page, NR_ISOLATED_ANON +  in do_move_page_to_node_array()
1266 					    page_is_file_cache(page));  in do_move_page_to_node_array()
1274 		put_page(page);  in do_move_page_to_node_array()
1391 		struct page *page;  in do_pages_stat_array()  local
1398 		page = follow_page(vma, addr, 0);  in do_pages_stat_array()
1400 		err = PTR_ERR(page);  in do_pages_stat_array()
1401 		if (IS_ERR(page))  in do_pages_stat_array()
1406 		if (!page || PageReserved(page))  in do_pages_stat_array()
1409 		err = page_to_nid(page);  in do_pages_stat_array()
1555 static struct page *alloc_misplaced_dst_page(struct page *page,  in alloc_misplaced_dst_page()  argument
1560 	struct page *newpage;  in alloc_misplaced_dst_page()
1611 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)  in numamigrate_isolate_page()  argument
1615 	VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);  in numamigrate_isolate_page()
1618 	if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page)))  in numamigrate_isolate_page()
1621 	if (isolate_lru_page(page))  in numamigrate_isolate_page()
1631 	if (PageTransHuge(page) && page_count(page) != 3) {  in numamigrate_isolate_page()
1632 		putback_lru_page(page);  in numamigrate_isolate_page()
1636 	page_lru = page_is_file_cache(page);  in numamigrate_isolate_page()
1637 	mod_zone_page_state(page_zone(page), NR_ISOLATED_ANON + page_lru,  in numamigrate_isolate_page()
1638 				hpage_nr_pages(page));  in numamigrate_isolate_page()
1645 	put_page(page);  in numamigrate_isolate_page()
1651 	struct page *page = pmd_page(pmd);  in pmd_trans_migrating()  local
1652 	return PageLocked(page);  in pmd_trans_migrating()
1660 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,  in migrate_misplaced_page()  argument
1672 	if (page_mapcount(page) != 1 && page_is_file_cache(page) &&  in migrate_misplaced_page()
1684 	isolated = numamigrate_isolate_page(pgdat, page);  in migrate_misplaced_page()
1688 	list_add(&page->lru, &migratepages);  in migrate_misplaced_page()
1694 			list_del(&page->lru);  in migrate_misplaced_page()
1695 			dec_zone_page_state(page, NR_ISOLATED_ANON +  in migrate_misplaced_page()
1696 					page_is_file_cache(page));  in migrate_misplaced_page()
1697 			putback_lru_page(page);  in migrate_misplaced_page()
1706 	put_page(page);  in migrate_misplaced_page()
1720 				struct page *page, int node)  in migrate_misplaced_transhuge_page()  argument
1725 	struct page *new_page = NULL;  in migrate_misplaced_transhuge_page()
1726 	int page_lru = page_is_file_cache(page);  in migrate_misplaced_transhuge_page()
1745 	isolated = numamigrate_isolate_page(pgdat, page);  in migrate_misplaced_transhuge_page()
1759 	new_page->mapping = page->mapping;  in migrate_misplaced_transhuge_page()
1760 	new_page->index = page->index;  in migrate_misplaced_transhuge_page()
1761 	migrate_page_copy(new_page, page);  in migrate_misplaced_transhuge_page()
1767 	if (unlikely(!pmd_same(*pmd, entry) || page_count(page) != 2)) {  in migrate_misplaced_transhuge_page()
1774 			SetPageActive(page);  in migrate_misplaced_transhuge_page()
1776 			SetPageUnevictable(page);  in migrate_misplaced_transhuge_page()
1777 		mlock_migrate_page(page, new_page);  in migrate_misplaced_transhuge_page()
1783 		get_page(page);  in migrate_misplaced_transhuge_page()
1784 		putback_lru_page(page);  in migrate_misplaced_transhuge_page()
1785 		mod_zone_page_state(page_zone(page),  in migrate_misplaced_transhuge_page()
1810 	if (page_count(page) != 2) {  in migrate_misplaced_transhuge_page()
1819 	mem_cgroup_migrate(page, new_page, false);  in migrate_misplaced_transhuge_page()
1821 	page_remove_rmap(page);  in migrate_misplaced_transhuge_page()
1831 	unlock_page(page);  in migrate_misplaced_transhuge_page()
1832 	put_page(page);			/* Drop the rmap reference */  in migrate_misplaced_transhuge_page()
1833 	put_page(page);			/* Drop the LRU isolation reference */  in migrate_misplaced_transhuge_page()
1838 	mod_zone_page_state(page_zone(page),  in migrate_misplaced_transhuge_page()
1855 	unlock_page(page);  in migrate_misplaced_transhuge_page()
1856 	put_page(page);  in migrate_misplaced_transhuge_page()