root/mm/migrate.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. migrate_prep
  2. migrate_prep_local
  3. isolate_movable_page
  4. putback_movable_page
  5. putback_movable_pages
  6. remove_migration_pte
  7. remove_migration_ptes
  8. __migration_entry_wait
  9. migration_entry_wait
  10. migration_entry_wait_huge
  11. pmd_migration_entry_wait
  12. expected_page_refs
  13. migrate_page_move_mapping
  14. migrate_huge_page_move_mapping
  15. __copy_gigantic_page
  16. copy_huge_page
  17. migrate_page_states
  18. migrate_page_copy
  19. migrate_page
  20. buffer_migrate_lock_buffers
  21. __buffer_migrate_page
  22. buffer_migrate_page
  23. buffer_migrate_page_norefs
  24. writeout
  25. fallback_migrate_page
  26. move_to_new_page
  27. __unmap_and_move
  28. unmap_and_move
  29. unmap_and_move_huge_page
  30. migrate_pages
  31. store_status
  32. do_move_pages_to_node
  33. add_page_for_migration
  34. do_pages_move
  35. do_pages_stat_array
  36. do_pages_stat
  37. kernel_move_pages
  38. SYSCALL_DEFINE6
  39. COMPAT_SYSCALL_DEFINE6
  40. migrate_balanced_pgdat
  41. alloc_misplaced_dst_page
  42. numamigrate_isolate_page
  43. pmd_trans_migrating
  44. migrate_misplaced_page
  45. migrate_misplaced_transhuge_page
  46. migrate_vma_collect_hole
  47. migrate_vma_collect_skip
  48. migrate_vma_collect_pmd
  49. migrate_vma_collect
  50. migrate_vma_check_page
  51. migrate_vma_prepare
  52. migrate_vma_unmap
  53. migrate_vma_setup
  54. migrate_vma_insert_page
  55. migrate_vma_pages
  56. migrate_vma_finalize

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Memory Migration functionality - linux/mm/migrate.c
   4  *
   5  * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
   6  *
   7  * Page migration was first developed in the context of the memory hotplug
   8  * project. The main authors of the migration code are:
   9  *
  10  * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  11  * Hirokazu Takahashi <taka@valinux.co.jp>
  12  * Dave Hansen <haveblue@us.ibm.com>
  13  * Christoph Lameter
  14  */
  15 
  16 #include <linux/migrate.h>
  17 #include <linux/export.h>
  18 #include <linux/swap.h>
  19 #include <linux/swapops.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/buffer_head.h>
  22 #include <linux/mm_inline.h>
  23 #include <linux/nsproxy.h>
  24 #include <linux/pagevec.h>
  25 #include <linux/ksm.h>
  26 #include <linux/rmap.h>
  27 #include <linux/topology.h>
  28 #include <linux/cpu.h>
  29 #include <linux/cpuset.h>
  30 #include <linux/writeback.h>
  31 #include <linux/mempolicy.h>
  32 #include <linux/vmalloc.h>
  33 #include <linux/security.h>
  34 #include <linux/backing-dev.h>
  35 #include <linux/compaction.h>
  36 #include <linux/syscalls.h>
  37 #include <linux/compat.h>
  38 #include <linux/hugetlb.h>
  39 #include <linux/hugetlb_cgroup.h>
  40 #include <linux/gfp.h>
  41 #include <linux/pagewalk.h>
  42 #include <linux/pfn_t.h>
  43 #include <linux/memremap.h>
  44 #include <linux/userfaultfd_k.h>
  45 #include <linux/balloon_compaction.h>
  46 #include <linux/mmu_notifier.h>
  47 #include <linux/page_idle.h>
  48 #include <linux/page_owner.h>
  49 #include <linux/sched/mm.h>
  50 #include <linux/ptrace.h>
  51 
  52 #include <asm/tlbflush.h>
  53 
  54 #define CREATE_TRACE_POINTS
  55 #include <trace/events/migrate.h>
  56 
  57 #include "internal.h"
  58 
  59 /*
  60  * migrate_prep() needs to be called before we start compiling a list of pages
  61  * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
  62  * undesirable, use migrate_prep_local()
  63  */
  64 int migrate_prep(void)
  65 {
  66         /*
  67          * Clear the LRU lists so pages can be isolated.
  68          * Note that pages may be moved off the LRU after we have
  69          * drained them. Those pages will fail to migrate like other
  70          * pages that may be busy.
  71          */
  72         lru_add_drain_all();
  73 
  74         return 0;
  75 }
  76 
  77 /* Do the necessary work of migrate_prep but not if it involves other CPUs */
  78 int migrate_prep_local(void)
  79 {
  80         lru_add_drain();
  81 
  82         return 0;
  83 }
  84 
  85 int isolate_movable_page(struct page *page, isolate_mode_t mode)
  86 {
  87         struct address_space *mapping;
  88 
  89         /*
  90          * Avoid burning cycles with pages that are yet under __free_pages(),
  91          * or just got freed under us.
  92          *
  93          * In case we 'win' a race for a movable page being freed under us and
  94          * raise its refcount preventing __free_pages() from doing its job
  95          * the put_page() at the end of this block will take care of
  96          * release this page, thus avoiding a nasty leakage.
  97          */
  98         if (unlikely(!get_page_unless_zero(page)))
  99                 goto out;
 100 
 101         /*
 102          * Check PageMovable before holding a PG_lock because page's owner
 103          * assumes anybody doesn't touch PG_lock of newly allocated page
 104          * so unconditionally grabbing the lock ruins page's owner side.
 105          */
 106         if (unlikely(!__PageMovable(page)))
 107                 goto out_putpage;
 108         /*
 109          * As movable pages are not isolated from LRU lists, concurrent
 110          * compaction threads can race against page migration functions
 111          * as well as race against the releasing a page.
 112          *
 113          * In order to avoid having an already isolated movable page
 114          * being (wrongly) re-isolated while it is under migration,
 115          * or to avoid attempting to isolate pages being released,
 116          * lets be sure we have the page lock
 117          * before proceeding with the movable page isolation steps.
 118          */
 119         if (unlikely(!trylock_page(page)))
 120                 goto out_putpage;
 121 
 122         if (!PageMovable(page) || PageIsolated(page))
 123                 goto out_no_isolated;
 124 
 125         mapping = page_mapping(page);
 126         VM_BUG_ON_PAGE(!mapping, page);
 127 
 128         if (!mapping->a_ops->isolate_page(page, mode))
 129                 goto out_no_isolated;
 130 
 131         /* Driver shouldn't use PG_isolated bit of page->flags */
 132         WARN_ON_ONCE(PageIsolated(page));
 133         __SetPageIsolated(page);
 134         unlock_page(page);
 135 
 136         return 0;
 137 
 138 out_no_isolated:
 139         unlock_page(page);
 140 out_putpage:
 141         put_page(page);
 142 out:
 143         return -EBUSY;
 144 }
 145 
 146 /* It should be called on page which is PG_movable */
 147 void putback_movable_page(struct page *page)
 148 {
 149         struct address_space *mapping;
 150 
 151         VM_BUG_ON_PAGE(!PageLocked(page), page);
 152         VM_BUG_ON_PAGE(!PageMovable(page), page);
 153         VM_BUG_ON_PAGE(!PageIsolated(page), page);
 154 
 155         mapping = page_mapping(page);
 156         mapping->a_ops->putback_page(page);
 157         __ClearPageIsolated(page);
 158 }
 159 
 160 /*
 161  * Put previously isolated pages back onto the appropriate lists
 162  * from where they were once taken off for compaction/migration.
 163  *
 164  * This function shall be used whenever the isolated pageset has been
 165  * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
 166  * and isolate_huge_page().
 167  */
 168 void putback_movable_pages(struct list_head *l)
 169 {
 170         struct page *page;
 171         struct page *page2;
 172 
 173         list_for_each_entry_safe(page, page2, l, lru) {
 174                 if (unlikely(PageHuge(page))) {
 175                         putback_active_hugepage(page);
 176                         continue;
 177                 }
 178                 list_del(&page->lru);
 179                 /*
 180                  * We isolated non-lru movable page so here we can use
 181                  * __PageMovable because LRU page's mapping cannot have
 182                  * PAGE_MAPPING_MOVABLE.
 183                  */
 184                 if (unlikely(__PageMovable(page))) {
 185                         VM_BUG_ON_PAGE(!PageIsolated(page), page);
 186                         lock_page(page);
 187                         if (PageMovable(page))
 188                                 putback_movable_page(page);
 189                         else
 190                                 __ClearPageIsolated(page);
 191                         unlock_page(page);
 192                         put_page(page);
 193                 } else {
 194                         mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
 195                                         page_is_file_cache(page), -hpage_nr_pages(page));
 196                         putback_lru_page(page);
 197                 }
 198         }
 199 }
 200 
 201 /*
 202  * Restore a potential migration pte to a working pte entry
 203  */
 204 static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
 205                                  unsigned long addr, void *old)
 206 {
 207         struct page_vma_mapped_walk pvmw = {
 208                 .page = old,
 209                 .vma = vma,
 210                 .address = addr,
 211                 .flags = PVMW_SYNC | PVMW_MIGRATION,
 212         };
 213         struct page *new;
 214         pte_t pte;
 215         swp_entry_t entry;
 216 
 217         VM_BUG_ON_PAGE(PageTail(page), page);
 218         while (page_vma_mapped_walk(&pvmw)) {
 219                 if (PageKsm(page))
 220                         new = page;
 221                 else
 222                         new = page - pvmw.page->index +
 223                                 linear_page_index(vma, pvmw.address);
 224 
 225 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
 226                 /* PMD-mapped THP migration entry */
 227                 if (!pvmw.pte) {
 228                         VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
 229                         remove_migration_pmd(&pvmw, new);
 230                         continue;
 231                 }
 232 #endif
 233 
 234                 get_page(new);
 235                 pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
 236                 if (pte_swp_soft_dirty(*pvmw.pte))
 237                         pte = pte_mksoft_dirty(pte);
 238 
 239                 /*
 240                  * Recheck VMA as permissions can change since migration started
 241                  */
 242                 entry = pte_to_swp_entry(*pvmw.pte);
 243                 if (is_write_migration_entry(entry))
 244                         pte = maybe_mkwrite(pte, vma);
 245 
 246                 if (unlikely(is_zone_device_page(new))) {
 247                         if (is_device_private_page(new)) {
 248                                 entry = make_device_private_entry(new, pte_write(pte));
 249                                 pte = swp_entry_to_pte(entry);
 250                         }
 251                 }
 252 
 253 #ifdef CONFIG_HUGETLB_PAGE
 254                 if (PageHuge(new)) {
 255                         pte = pte_mkhuge(pte);
 256                         pte = arch_make_huge_pte(pte, vma, new, 0);
 257                         set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 258                         if (PageAnon(new))
 259                                 hugepage_add_anon_rmap(new, vma, pvmw.address);
 260                         else
 261                                 page_dup_rmap(new, true);
 262                 } else
 263 #endif
 264                 {
 265                         set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 266 
 267                         if (PageAnon(new))
 268                                 page_add_anon_rmap(new, vma, pvmw.address, false);
 269                         else
 270                                 page_add_file_rmap(new, false);
 271                 }
 272                 if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
 273                         mlock_vma_page(new);
 274 
 275                 if (PageTransHuge(page) && PageMlocked(page))
 276                         clear_page_mlock(page);
 277 
 278                 /* No need to invalidate - it was non-present before */
 279                 update_mmu_cache(vma, pvmw.address, pvmw.pte);
 280         }
 281 
 282         return true;
 283 }
 284 
 285 /*
 286  * Get rid of all migration entries and replace them by
 287  * references to the indicated page.
 288  */
 289 void remove_migration_ptes(struct page *old, struct page *new, bool locked)
 290 {
 291         struct rmap_walk_control rwc = {
 292                 .rmap_one = remove_migration_pte,
 293                 .arg = old,
 294         };
 295 
 296         if (locked)
 297                 rmap_walk_locked(new, &rwc);
 298         else
 299                 rmap_walk(new, &rwc);
 300 }
 301 
 302 /*
 303  * Something used the pte of a page under migration. We need to
 304  * get to the page and wait until migration is finished.
 305  * When we return from this function the fault will be retried.
 306  */
 307 void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
 308                                 spinlock_t *ptl)
 309 {
 310         pte_t pte;
 311         swp_entry_t entry;
 312         struct page *page;
 313 
 314         spin_lock(ptl);
 315         pte = *ptep;
 316         if (!is_swap_pte(pte))
 317                 goto out;
 318 
 319         entry = pte_to_swp_entry(pte);
 320         if (!is_migration_entry(entry))
 321                 goto out;
 322 
 323         page = migration_entry_to_page(entry);
 324 
 325         /*
 326          * Once page cache replacement of page migration started, page_count
 327          * is zero; but we must not call put_and_wait_on_page_locked() without
 328          * a ref. Use get_page_unless_zero(), and just fault again if it fails.
 329          */
 330         if (!get_page_unless_zero(page))
 331                 goto out;
 332         pte_unmap_unlock(ptep, ptl);
 333         put_and_wait_on_page_locked(page);
 334         return;
 335 out:
 336         pte_unmap_unlock(ptep, ptl);
 337 }
 338 
 339 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
 340                                 unsigned long address)
 341 {
 342         spinlock_t *ptl = pte_lockptr(mm, pmd);
 343         pte_t *ptep = pte_offset_map(pmd, address);
 344         __migration_entry_wait(mm, ptep, ptl);
 345 }
 346 
 347 void migration_entry_wait_huge(struct vm_area_struct *vma,
 348                 struct mm_struct *mm, pte_t *pte)
 349 {
 350         spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
 351         __migration_entry_wait(mm, pte, ptl);
 352 }
 353 
 354 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
 355 void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
 356 {
 357         spinlock_t *ptl;
 358         struct page *page;
 359 
 360         ptl = pmd_lock(mm, pmd);
 361         if (!is_pmd_migration_entry(*pmd))
 362                 goto unlock;
 363         page = migration_entry_to_page(pmd_to_swp_entry(*pmd));
 364         if (!get_page_unless_zero(page))
 365                 goto unlock;
 366         spin_unlock(ptl);
 367         put_and_wait_on_page_locked(page);
 368         return;
 369 unlock:
 370         spin_unlock(ptl);
 371 }
 372 #endif
 373 
 374 static int expected_page_refs(struct address_space *mapping, struct page *page)
 375 {
 376         int expected_count = 1;
 377 
 378         /*
 379          * Device public or private pages have an extra refcount as they are
 380          * ZONE_DEVICE pages.
 381          */
 382         expected_count += is_device_private_page(page);
 383         if (mapping)
 384                 expected_count += hpage_nr_pages(page) + page_has_private(page);
 385 
 386         return expected_count;
 387 }
 388 
 389 /*
 390  * Replace the page in the mapping.
 391  *
 392  * The number of remaining references must be:
 393  * 1 for anonymous pages without a mapping
 394  * 2 for pages with a mapping
 395  * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
 396  */
 397 int migrate_page_move_mapping(struct address_space *mapping,
 398                 struct page *newpage, struct page *page, int extra_count)
 399 {
 400         XA_STATE(xas, &mapping->i_pages, page_index(page));
 401         struct zone *oldzone, *newzone;
 402         int dirty;
 403         int expected_count = expected_page_refs(mapping, page) + extra_count;
 404 
 405         if (!mapping) {
 406                 /* Anonymous page without mapping */
 407                 if (page_count(page) != expected_count)
 408                         return -EAGAIN;
 409 
 410                 /* No turning back from here */
 411                 newpage->index = page->index;
 412                 newpage->mapping = page->mapping;
 413                 if (PageSwapBacked(page))
 414                         __SetPageSwapBacked(newpage);
 415 
 416                 return MIGRATEPAGE_SUCCESS;
 417         }
 418 
 419         oldzone = page_zone(page);
 420         newzone = page_zone(newpage);
 421 
 422         xas_lock_irq(&xas);
 423         if (page_count(page) != expected_count || xas_load(&xas) != page) {
 424                 xas_unlock_irq(&xas);
 425                 return -EAGAIN;
 426         }
 427 
 428         if (!page_ref_freeze(page, expected_count)) {
 429                 xas_unlock_irq(&xas);
 430                 return -EAGAIN;
 431         }
 432 
 433         /*
 434          * Now we know that no one else is looking at the page:
 435          * no turning back from here.
 436          */
 437         newpage->index = page->index;
 438         newpage->mapping = page->mapping;
 439         page_ref_add(newpage, hpage_nr_pages(page)); /* add cache reference */
 440         if (PageSwapBacked(page)) {
 441                 __SetPageSwapBacked(newpage);
 442                 if (PageSwapCache(page)) {
 443                         SetPageSwapCache(newpage);
 444                         set_page_private(newpage, page_private(page));
 445                 }
 446         } else {
 447                 VM_BUG_ON_PAGE(PageSwapCache(page), page);
 448         }
 449 
 450         /* Move dirty while page refs frozen and newpage not yet exposed */
 451         dirty = PageDirty(page);
 452         if (dirty) {
 453                 ClearPageDirty(page);
 454                 SetPageDirty(newpage);
 455         }
 456 
 457         xas_store(&xas, newpage);
 458         if (PageTransHuge(page)) {
 459                 int i;
 460 
 461                 for (i = 1; i < HPAGE_PMD_NR; i++) {
 462                         xas_next(&xas);
 463                         xas_store(&xas, newpage);
 464                 }
 465         }
 466 
 467         /*
 468          * Drop cache reference from old page by unfreezing
 469          * to one less reference.
 470          * We know this isn't the last reference.
 471          */
 472         page_ref_unfreeze(page, expected_count - hpage_nr_pages(page));
 473 
 474         xas_unlock(&xas);
 475         /* Leave irq disabled to prevent preemption while updating stats */
 476 
 477         /*
 478          * If moved to a different zone then also account
 479          * the page for that zone. Other VM counters will be
 480          * taken care of when we establish references to the
 481          * new page and drop references to the old page.
 482          *
 483          * Note that anonymous pages are accounted for
 484          * via NR_FILE_PAGES and NR_ANON_MAPPED if they
 485          * are mapped to swap space.
 486          */
 487         if (newzone != oldzone) {
 488                 __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES);
 489                 __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES);
 490                 if (PageSwapBacked(page) && !PageSwapCache(page)) {
 491                         __dec_node_state(oldzone->zone_pgdat, NR_SHMEM);
 492                         __inc_node_state(newzone->zone_pgdat, NR_SHMEM);
 493                 }
 494                 if (dirty && mapping_cap_account_dirty(mapping)) {
 495                         __dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);
 496                         __dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);
 497                         __inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);
 498                         __inc_zone_state(newzone, NR_ZONE_WRITE_PENDING);
 499                 }
 500         }
 501         local_irq_enable();
 502 
 503         return MIGRATEPAGE_SUCCESS;
 504 }
 505 EXPORT_SYMBOL(migrate_page_move_mapping);
 506 
 507 /*
 508  * The expected number of remaining references is the same as that
 509  * of migrate_page_move_mapping().
 510  */
 511 int migrate_huge_page_move_mapping(struct address_space *mapping,
 512                                    struct page *newpage, struct page *page)
 513 {
 514         XA_STATE(xas, &mapping->i_pages, page_index(page));
 515         int expected_count;
 516 
 517         xas_lock_irq(&xas);
 518         expected_count = 2 + page_has_private(page);
 519         if (page_count(page) != expected_count || xas_load(&xas) != page) {
 520                 xas_unlock_irq(&xas);
 521                 return -EAGAIN;
 522         }
 523 
 524         if (!page_ref_freeze(page, expected_count)) {
 525                 xas_unlock_irq(&xas);
 526                 return -EAGAIN;
 527         }
 528 
 529         newpage->index = page->index;
 530         newpage->mapping = page->mapping;
 531 
 532         get_page(newpage);
 533 
 534         xas_store(&xas, newpage);
 535 
 536         page_ref_unfreeze(page, expected_count - 1);
 537 
 538         xas_unlock_irq(&xas);
 539 
 540         return MIGRATEPAGE_SUCCESS;
 541 }
 542 
 543 /*
 544  * Gigantic pages are so large that we do not guarantee that page++ pointer
 545  * arithmetic will work across the entire page.  We need something more
 546  * specialized.
 547  */
 548 static void __copy_gigantic_page(struct page *dst, struct page *src,
 549                                 int nr_pages)
 550 {
 551         int i;
 552         struct page *dst_base = dst;
 553         struct page *src_base = src;
 554 
 555         for (i = 0; i < nr_pages; ) {
 556                 cond_resched();
 557                 copy_highpage(dst, src);
 558 
 559                 i++;
 560                 dst = mem_map_next(dst, dst_base, i);
 561                 src = mem_map_next(src, src_base, i);
 562         }
 563 }
 564 
 565 static void copy_huge_page(struct page *dst, struct page *src)
 566 {
 567         int i;
 568         int nr_pages;
 569 
 570         if (PageHuge(src)) {
 571                 /* hugetlbfs page */
 572                 struct hstate *h = page_hstate(src);
 573                 nr_pages = pages_per_huge_page(h);
 574 
 575                 if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
 576                         __copy_gigantic_page(dst, src, nr_pages);
 577                         return;
 578                 }
 579         } else {
 580                 /* thp page */
 581                 BUG_ON(!PageTransHuge(src));
 582                 nr_pages = hpage_nr_pages(src);
 583         }
 584 
 585         for (i = 0; i < nr_pages; i++) {
 586                 cond_resched();
 587                 copy_highpage(dst + i, src + i);
 588         }
 589 }
 590 
 591 /*
 592  * Copy the page to its new location
 593  */
 594 void migrate_page_states(struct page *newpage, struct page *page)
 595 {
 596         int cpupid;
 597 
 598         if (PageError(page))
 599                 SetPageError(newpage);
 600         if (PageReferenced(page))
 601                 SetPageReferenced(newpage);
 602         if (PageUptodate(page))
 603                 SetPageUptodate(newpage);
 604         if (TestClearPageActive(page)) {
 605                 VM_BUG_ON_PAGE(PageUnevictable(page), page);
 606                 SetPageActive(newpage);
 607         } else if (TestClearPageUnevictable(page))
 608                 SetPageUnevictable(newpage);
 609         if (PageWorkingset(page))
 610                 SetPageWorkingset(newpage);
 611         if (PageChecked(page))
 612                 SetPageChecked(newpage);
 613         if (PageMappedToDisk(page))
 614                 SetPageMappedToDisk(newpage);
 615 
 616         /* Move dirty on pages not done by migrate_page_move_mapping() */
 617         if (PageDirty(page))
 618                 SetPageDirty(newpage);
 619 
 620         if (page_is_young(page))
 621                 set_page_young(newpage);
 622         if (page_is_idle(page))
 623                 set_page_idle(newpage);
 624 
 625         /*
 626          * Copy NUMA information to the new page, to prevent over-eager
 627          * future migrations of this same page.
 628          */
 629         cpupid = page_cpupid_xchg_last(page, -1);
 630         page_cpupid_xchg_last(newpage, cpupid);
 631 
 632         ksm_migrate_page(newpage, page);
 633         /*
 634          * Please do not reorder this without considering how mm/ksm.c's
 635          * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
 636          */
 637         if (PageSwapCache(page))
 638                 ClearPageSwapCache(page);
 639         ClearPagePrivate(page);
 640         set_page_private(page, 0);
 641 
 642         /*
 643          * If any waiters have accumulated on the new page then
 644          * wake them up.
 645          */
 646         if (PageWriteback(newpage))
 647                 end_page_writeback(newpage);
 648 
 649         copy_page_owner(page, newpage);
 650 
 651         mem_cgroup_migrate(page, newpage);
 652 }
 653 EXPORT_SYMBOL(migrate_page_states);
 654 
 655 void migrate_page_copy(struct page *newpage, struct page *page)
 656 {
 657         if (PageHuge(page) || PageTransHuge(page))
 658                 copy_huge_page(newpage, page);
 659         else
 660                 copy_highpage(newpage, page);
 661 
 662         migrate_page_states(newpage, page);
 663 }
 664 EXPORT_SYMBOL(migrate_page_copy);
 665 
 666 /************************************************************
 667  *                    Migration functions
 668  ***********************************************************/
 669 
 670 /*
 671  * Common logic to directly migrate a single LRU page suitable for
 672  * pages that do not use PagePrivate/PagePrivate2.
 673  *
 674  * Pages are locked upon entry and exit.
 675  */
 676 int migrate_page(struct address_space *mapping,
 677                 struct page *newpage, struct page *page,
 678                 enum migrate_mode mode)
 679 {
 680         int rc;
 681 
 682         BUG_ON(PageWriteback(page));    /* Writeback must be complete */
 683 
 684         rc = migrate_page_move_mapping(mapping, newpage, page, 0);
 685 
 686         if (rc != MIGRATEPAGE_SUCCESS)
 687                 return rc;
 688 
 689         if (mode != MIGRATE_SYNC_NO_COPY)
 690                 migrate_page_copy(newpage, page);
 691         else
 692                 migrate_page_states(newpage, page);
 693         return MIGRATEPAGE_SUCCESS;
 694 }
 695 EXPORT_SYMBOL(migrate_page);
 696 
 697 #ifdef CONFIG_BLOCK
 698 /* Returns true if all buffers are successfully locked */
 699 static bool buffer_migrate_lock_buffers(struct buffer_head *head,
 700                                                         enum migrate_mode mode)
 701 {
 702         struct buffer_head *bh = head;
 703 
 704         /* Simple case, sync compaction */
 705         if (mode != MIGRATE_ASYNC) {
 706                 do {
 707                         lock_buffer(bh);
 708                         bh = bh->b_this_page;
 709 
 710                 } while (bh != head);
 711 
 712                 return true;
 713         }
 714 
 715         /* async case, we cannot block on lock_buffer so use trylock_buffer */
 716         do {
 717                 if (!trylock_buffer(bh)) {
 718                         /*
 719                          * We failed to lock the buffer and cannot stall in
 720                          * async migration. Release the taken locks
 721                          */
 722                         struct buffer_head *failed_bh = bh;
 723                         bh = head;
 724                         while (bh != failed_bh) {
 725                                 unlock_buffer(bh);
 726                                 bh = bh->b_this_page;
 727                         }
 728                         return false;
 729                 }
 730 
 731                 bh = bh->b_this_page;
 732         } while (bh != head);
 733         return true;
 734 }
 735 
 736 static int __buffer_migrate_page(struct address_space *mapping,
 737                 struct page *newpage, struct page *page, enum migrate_mode mode,
 738                 bool check_refs)
 739 {
 740         struct buffer_head *bh, *head;
 741         int rc;
 742         int expected_count;
 743 
 744         if (!page_has_buffers(page))
 745                 return migrate_page(mapping, newpage, page, mode);
 746 
 747         /* Check whether page does not have extra refs before we do more work */
 748         expected_count = expected_page_refs(mapping, page);
 749         if (page_count(page) != expected_count)
 750                 return -EAGAIN;
 751 
 752         head = page_buffers(page);
 753         if (!buffer_migrate_lock_buffers(head, mode))
 754                 return -EAGAIN;
 755 
 756         if (check_refs) {
 757                 bool busy;
 758                 bool invalidated = false;
 759 
 760 recheck_buffers:
 761                 busy = false;
 762                 spin_lock(&mapping->private_lock);
 763                 bh = head;
 764                 do {
 765                         if (atomic_read(&bh->b_count)) {
 766                                 busy = true;
 767                                 break;
 768                         }
 769                         bh = bh->b_this_page;
 770                 } while (bh != head);
 771                 if (busy) {
 772                         if (invalidated) {
 773                                 rc = -EAGAIN;
 774                                 goto unlock_buffers;
 775                         }
 776                         spin_unlock(&mapping->private_lock);
 777                         invalidate_bh_lrus();
 778                         invalidated = true;
 779                         goto recheck_buffers;
 780                 }
 781         }
 782 
 783         rc = migrate_page_move_mapping(mapping, newpage, page, 0);
 784         if (rc != MIGRATEPAGE_SUCCESS)
 785                 goto unlock_buffers;
 786 
 787         ClearPagePrivate(page);
 788         set_page_private(newpage, page_private(page));
 789         set_page_private(page, 0);
 790         put_page(page);
 791         get_page(newpage);
 792 
 793         bh = head;
 794         do {
 795                 set_bh_page(bh, newpage, bh_offset(bh));
 796                 bh = bh->b_this_page;
 797 
 798         } while (bh != head);
 799 
 800         SetPagePrivate(newpage);
 801 
 802         if (mode != MIGRATE_SYNC_NO_COPY)
 803                 migrate_page_copy(newpage, page);
 804         else
 805                 migrate_page_states(newpage, page);
 806 
 807         rc = MIGRATEPAGE_SUCCESS;
 808 unlock_buffers:
 809         if (check_refs)
 810                 spin_unlock(&mapping->private_lock);
 811         bh = head;
 812         do {
 813                 unlock_buffer(bh);
 814                 bh = bh->b_this_page;
 815 
 816         } while (bh != head);
 817 
 818         return rc;
 819 }
 820 
 821 /*
 822  * Migration function for pages with buffers. This function can only be used
 823  * if the underlying filesystem guarantees that no other references to "page"
 824  * exist. For example attached buffer heads are accessed only under page lock.
 825  */
 826 int buffer_migrate_page(struct address_space *mapping,
 827                 struct page *newpage, struct page *page, enum migrate_mode mode)
 828 {
 829         return __buffer_migrate_page(mapping, newpage, page, mode, false);
 830 }
 831 EXPORT_SYMBOL(buffer_migrate_page);
 832 
 833 /*
 834  * Same as above except that this variant is more careful and checks that there
 835  * are also no buffer head references. This function is the right one for
 836  * mappings where buffer heads are directly looked up and referenced (such as
 837  * block device mappings).
 838  */
 839 int buffer_migrate_page_norefs(struct address_space *mapping,
 840                 struct page *newpage, struct page *page, enum migrate_mode mode)
 841 {
 842         return __buffer_migrate_page(mapping, newpage, page, mode, true);
 843 }
 844 #endif
 845 
 846 /*
 847  * Writeback a page to clean the dirty state
 848  */
 849 static int writeout(struct address_space *mapping, struct page *page)
 850 {
 851         struct writeback_control wbc = {
 852                 .sync_mode = WB_SYNC_NONE,
 853                 .nr_to_write = 1,
 854                 .range_start = 0,
 855                 .range_end = LLONG_MAX,
 856                 .for_reclaim = 1
 857         };
 858         int rc;
 859 
 860         if (!mapping->a_ops->writepage)
 861                 /* No write method for the address space */
 862                 return -EINVAL;
 863 
 864         if (!clear_page_dirty_for_io(page))
 865                 /* Someone else already triggered a write */
 866                 return -EAGAIN;
 867 
 868         /*
 869          * A dirty page may imply that the underlying filesystem has
 870          * the page on some queue. So the page must be clean for
 871          * migration. Writeout may mean we loose the lock and the
 872          * page state is no longer what we checked for earlier.
 873          * At this point we know that the migration attempt cannot
 874          * be successful.
 875          */
 876         remove_migration_ptes(page, page, false);
 877 
 878         rc = mapping->a_ops->writepage(page, &wbc);
 879 
 880         if (rc != AOP_WRITEPAGE_ACTIVATE)
 881                 /* unlocked. Relock */
 882                 lock_page(page);
 883 
 884         return (rc < 0) ? -EIO : -EAGAIN;
 885 }
 886 
 887 /*
 888  * Default handling if a filesystem does not provide a migration function.
 889  */
 890 static int fallback_migrate_page(struct address_space *mapping,
 891         struct page *newpage, struct page *page, enum migrate_mode mode)
 892 {
 893         if (PageDirty(page)) {
 894                 /* Only writeback pages in full synchronous migration */
 895                 switch (mode) {
 896                 case MIGRATE_SYNC:
 897                 case MIGRATE_SYNC_NO_COPY:
 898                         break;
 899                 default:
 900                         return -EBUSY;
 901                 }
 902                 return writeout(mapping, page);
 903         }
 904 
 905         /*
 906          * Buffers may be managed in a filesystem specific way.
 907          * We must have no buffers or drop them.
 908          */
 909         if (page_has_private(page) &&
 910             !try_to_release_page(page, GFP_KERNEL))
 911                 return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
 912 
 913         return migrate_page(mapping, newpage, page, mode);
 914 }
 915 
 916 /*
 917  * Move a page to a newly allocated page
 918  * The page is locked and all ptes have been successfully removed.
 919  *
 920  * The new page will have replaced the old page if this function
 921  * is successful.
 922  *
 923  * Return value:
 924  *   < 0 - error code
 925  *  MIGRATEPAGE_SUCCESS - success
 926  */
 927 static int move_to_new_page(struct page *newpage, struct page *page,
 928                                 enum migrate_mode mode)
 929 {
 930         struct address_space *mapping;
 931         int rc = -EAGAIN;
 932         bool is_lru = !__PageMovable(page);
 933 
 934         VM_BUG_ON_PAGE(!PageLocked(page), page);
 935         VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
 936 
 937         mapping = page_mapping(page);
 938 
 939         if (likely(is_lru)) {
 940                 if (!mapping)
 941                         rc = migrate_page(mapping, newpage, page, mode);
 942                 else if (mapping->a_ops->migratepage)
 943                         /*
 944                          * Most pages have a mapping and most filesystems
 945                          * provide a migratepage callback. Anonymous pages
 946                          * are part of swap space which also has its own
 947                          * migratepage callback. This is the most common path
 948                          * for page migration.
 949                          */
 950                         rc = mapping->a_ops->migratepage(mapping, newpage,
 951                                                         page, mode);
 952                 else
 953                         rc = fallback_migrate_page(mapping, newpage,
 954                                                         page, mode);
 955         } else {
 956                 /*
 957                  * In case of non-lru page, it could be released after
 958                  * isolation step. In that case, we shouldn't try migration.
 959                  */
 960                 VM_BUG_ON_PAGE(!PageIsolated(page), page);
 961                 if (!PageMovable(page)) {
 962                         rc = MIGRATEPAGE_SUCCESS;
 963                         __ClearPageIsolated(page);
 964                         goto out;
 965                 }
 966 
 967                 rc = mapping->a_ops->migratepage(mapping, newpage,
 968                                                 page, mode);
 969                 WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
 970                         !PageIsolated(page));
 971         }
 972 
 973         /*
 974          * When successful, old pagecache page->mapping must be cleared before
 975          * page is freed; but stats require that PageAnon be left as PageAnon.
 976          */
 977         if (rc == MIGRATEPAGE_SUCCESS) {
 978                 if (__PageMovable(page)) {
 979                         VM_BUG_ON_PAGE(!PageIsolated(page), page);
 980 
 981                         /*
 982                          * We clear PG_movable under page_lock so any compactor
 983                          * cannot try to migrate this page.
 984                          */
 985                         __ClearPageIsolated(page);
 986                 }
 987 
 988                 /*
 989                  * Anonymous and movable page->mapping will be cleard by
 990                  * free_pages_prepare so don't reset it here for keeping
 991                  * the type to work PageAnon, for example.
 992                  */
 993                 if (!PageMappingFlags(page))
 994                         page->mapping = NULL;
 995 
 996                 if (likely(!is_zone_device_page(newpage)))
 997                         flush_dcache_page(newpage);
 998 
 999         }
1000 out:
1001         return rc;
1002 }
1003 
1004 static int __unmap_and_move(struct page *page, struct page *newpage,
1005                                 int force, enum migrate_mode mode)
1006 {
1007         int rc = -EAGAIN;
1008         int page_was_mapped = 0;
1009         struct anon_vma *anon_vma = NULL;
1010         bool is_lru = !__PageMovable(page);
1011 
1012         if (!trylock_page(page)) {
1013                 if (!force || mode == MIGRATE_ASYNC)
1014                         goto out;
1015 
1016                 /*
1017                  * It's not safe for direct compaction to call lock_page.
1018                  * For example, during page readahead pages are added locked
1019                  * to the LRU. Later, when the IO completes the pages are
1020                  * marked uptodate and unlocked. However, the queueing
1021                  * could be merging multiple pages for one bio (e.g.
1022                  * mpage_readpages). If an allocation happens for the
1023                  * second or third page, the process can end up locking
1024                  * the same page twice and deadlocking. Rather than
1025                  * trying to be clever about what pages can be locked,
1026                  * avoid the use of lock_page for direct compaction
1027                  * altogether.
1028                  */
1029                 if (current->flags & PF_MEMALLOC)
1030                         goto out;
1031 
1032                 lock_page(page);
1033         }
1034 
1035         if (PageWriteback(page)) {
1036                 /*
1037                  * Only in the case of a full synchronous migration is it
1038                  * necessary to wait for PageWriteback. In the async case,
1039                  * the retry loop is too short and in the sync-light case,
1040                  * the overhead of stalling is too much
1041                  */
1042                 switch (mode) {
1043                 case MIGRATE_SYNC:
1044                 case MIGRATE_SYNC_NO_COPY:
1045                         break;
1046                 default:
1047                         rc = -EBUSY;
1048                         goto out_unlock;
1049                 }
1050                 if (!force)
1051                         goto out_unlock;
1052                 wait_on_page_writeback(page);
1053         }
1054 
1055         /*
1056          * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
1057          * we cannot notice that anon_vma is freed while we migrates a page.
1058          * This get_anon_vma() delays freeing anon_vma pointer until the end
1059          * of migration. File cache pages are no problem because of page_lock()
1060          * File Caches may use write_page() or lock_page() in migration, then,
1061          * just care Anon page here.
1062          *
1063          * Only page_get_anon_vma() understands the subtleties of
1064          * getting a hold on an anon_vma from outside one of its mms.
1065          * But if we cannot get anon_vma, then we won't need it anyway,
1066          * because that implies that the anon page is no longer mapped
1067          * (and cannot be remapped so long as we hold the page lock).
1068          */
1069         if (PageAnon(page) && !PageKsm(page))
1070                 anon_vma = page_get_anon_vma(page);
1071 
1072         /*
1073          * Block others from accessing the new page when we get around to
1074          * establishing additional references. We are usually the only one
1075          * holding a reference to newpage at this point. We used to have a BUG
1076          * here if trylock_page(newpage) fails, but would like to allow for
1077          * cases where there might be a race with the previous use of newpage.
1078          * This is much like races on refcount of oldpage: just don't BUG().
1079          */
1080         if (unlikely(!trylock_page(newpage)))
1081                 goto out_unlock;
1082 
1083         if (unlikely(!is_lru)) {
1084                 rc = move_to_new_page(newpage, page, mode);
1085                 goto out_unlock_both;
1086         }
1087 
1088         /*
1089          * Corner case handling:
1090          * 1. When a new swap-cache page is read into, it is added to the LRU
1091          * and treated as swapcache but it has no rmap yet.
1092          * Calling try_to_unmap() against a page->mapping==NULL page will
1093          * trigger a BUG.  So handle it here.
1094          * 2. An orphaned page (see truncate_complete_page) might have
1095          * fs-private metadata. The page can be picked up due to memory
1096          * offlining.  Everywhere else except page reclaim, the page is
1097          * invisible to the vm, so the page can not be migrated.  So try to
1098          * free the metadata, so the page can be freed.
1099          */
1100         if (!page->mapping) {
1101                 VM_BUG_ON_PAGE(PageAnon(page), page);
1102                 if (page_has_private(page)) {
1103                         try_to_free_buffers(page);
1104                         goto out_unlock_both;
1105                 }
1106         } else if (page_mapped(page)) {
1107                 /* Establish migration ptes */
1108                 VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
1109                                 page);
1110                 try_to_unmap(page,
1111                         TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1112                 page_was_mapped = 1;
1113         }
1114 
1115         if (!page_mapped(page))
1116                 rc = move_to_new_page(newpage, page, mode);
1117 
1118         if (page_was_mapped)
1119                 remove_migration_ptes(page,
1120                         rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
1121 
1122 out_unlock_both:
1123         unlock_page(newpage);
1124 out_unlock:
1125         /* Drop an anon_vma reference if we took one */
1126         if (anon_vma)
1127                 put_anon_vma(anon_vma);
1128         unlock_page(page);
1129 out:
1130         /*
1131          * If migration is successful, decrease refcount of the newpage
1132          * which will not free the page because new page owner increased
1133          * refcounter. As well, if it is LRU page, add the page to LRU
1134          * list in here. Use the old state of the isolated source page to
1135          * determine if we migrated a LRU page. newpage was already unlocked
1136          * and possibly modified by its owner - don't rely on the page
1137          * state.
1138          */
1139         if (rc == MIGRATEPAGE_SUCCESS) {
1140                 if (unlikely(!is_lru))
1141                         put_page(newpage);
1142                 else
1143                         putback_lru_page(newpage);
1144         }
1145 
1146         return rc;
1147 }
1148 
1149 /*
1150  * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move().  Work
1151  * around it.
1152  */
1153 #if defined(CONFIG_ARM) && \
1154         defined(GCC_VERSION) && GCC_VERSION < 40900 && GCC_VERSION >= 40700
1155 #define ICE_noinline noinline
1156 #else
1157 #define ICE_noinline
1158 #endif
1159 
1160 /*
1161  * Obtain the lock on page, remove all ptes and migrate the page
1162  * to the newly allocated page in newpage.
1163  */
1164 static ICE_noinline int unmap_and_move(new_page_t get_new_page,
1165                                    free_page_t put_new_page,
1166                                    unsigned long private, struct page *page,
1167                                    int force, enum migrate_mode mode,
1168                                    enum migrate_reason reason)
1169 {
1170         int rc = MIGRATEPAGE_SUCCESS;
1171         struct page *newpage;
1172 
1173         if (!thp_migration_supported() && PageTransHuge(page))
1174                 return -ENOMEM;
1175 
1176         newpage = get_new_page(page, private);
1177         if (!newpage)
1178                 return -ENOMEM;
1179 
1180         if (page_count(page) == 1) {
1181                 /* page was freed from under us. So we are done. */
1182                 ClearPageActive(page);
1183                 ClearPageUnevictable(page);
1184                 if (unlikely(__PageMovable(page))) {
1185                         lock_page(page);
1186                         if (!PageMovable(page))
1187                                 __ClearPageIsolated(page);
1188                         unlock_page(page);
1189                 }
1190                 if (put_new_page)
1191                         put_new_page(newpage, private);
1192                 else
1193                         put_page(newpage);
1194                 goto out;
1195         }
1196 
1197         rc = __unmap_and_move(page, newpage, force, mode);
1198         if (rc == MIGRATEPAGE_SUCCESS)
1199                 set_page_owner_migrate_reason(newpage, reason);
1200 
1201 out:
1202         if (rc != -EAGAIN) {
1203                 /*
1204                  * A page that has been migrated has all references
1205                  * removed and will be freed. A page that has not been
1206                  * migrated will have kepts its references and be
1207                  * restored.
1208                  */
1209                 list_del(&page->lru);
1210 
1211                 /*
1212                  * Compaction can migrate also non-LRU pages which are
1213                  * not accounted to NR_ISOLATED_*. They can be recognized
1214                  * as __PageMovable
1215                  */
1216                 if (likely(!__PageMovable(page)))
1217                         mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
1218                                         page_is_file_cache(page), -hpage_nr_pages(page));
1219         }
1220 
1221         /*
1222          * If migration is successful, releases reference grabbed during
1223          * isolation. Otherwise, restore the page to right list unless
1224          * we want to retry.
1225          */
1226         if (rc == MIGRATEPAGE_SUCCESS) {
1227                 put_page(page);
1228                 if (reason == MR_MEMORY_FAILURE) {
1229                         /*
1230                          * Set PG_HWPoison on just freed page
1231                          * intentionally. Although it's rather weird,
1232                          * it's how HWPoison flag works at the moment.
1233                          */
1234                         if (set_hwpoison_free_buddy_page(page))
1235                                 num_poisoned_pages_inc();
1236                 }
1237         } else {
1238                 if (rc != -EAGAIN) {
1239                         if (likely(!__PageMovable(page))) {
1240                                 putback_lru_page(page);
1241                                 goto put_new;
1242                         }
1243 
1244                         lock_page(page);
1245                         if (PageMovable(page))
1246                                 putback_movable_page(page);
1247                         else
1248                                 __ClearPageIsolated(page);
1249                         unlock_page(page);
1250                         put_page(page);
1251                 }
1252 put_new:
1253                 if (put_new_page)
1254                         put_new_page(newpage, private);
1255                 else
1256                         put_page(newpage);
1257         }
1258 
1259         return rc;
1260 }
1261 
1262 /*
1263  * Counterpart of unmap_and_move_page() for hugepage migration.
1264  *
1265  * This function doesn't wait the completion of hugepage I/O
1266  * because there is no race between I/O and migration for hugepage.
1267  * Note that currently hugepage I/O occurs only in direct I/O
1268  * where no lock is held and PG_writeback is irrelevant,
1269  * and writeback status of all subpages are counted in the reference
1270  * count of the head page (i.e. if all subpages of a 2MB hugepage are
1271  * under direct I/O, the reference of the head page is 512 and a bit more.)
1272  * This means that when we try to migrate hugepage whose subpages are
1273  * doing direct I/O, some references remain after try_to_unmap() and
1274  * hugepage migration fails without data corruption.
1275  *
1276  * There is also no race when direct I/O is issued on the page under migration,
1277  * because then pte is replaced with migration swap entry and direct I/O code
1278  * will wait in the page fault for migration to complete.
1279  */
1280 static int unmap_and_move_huge_page(new_page_t get_new_page,
1281                                 free_page_t put_new_page, unsigned long private,
1282                                 struct page *hpage, int force,
1283                                 enum migrate_mode mode, int reason)
1284 {
1285         int rc = -EAGAIN;
1286         int page_was_mapped = 0;
1287         struct page *new_hpage;
1288         struct anon_vma *anon_vma = NULL;
1289 
1290         /*
1291          * Migratability of hugepages depends on architectures and their size.
1292          * This check is necessary because some callers of hugepage migration
1293          * like soft offline and memory hotremove don't walk through page
1294          * tables or check whether the hugepage is pmd-based or not before
1295          * kicking migration.
1296          */
1297         if (!hugepage_migration_supported(page_hstate(hpage))) {
1298                 putback_active_hugepage(hpage);
1299                 return -ENOSYS;
1300         }
1301 
1302         new_hpage = get_new_page(hpage, private);
1303         if (!new_hpage)
1304                 return -ENOMEM;
1305 
1306         if (!trylock_page(hpage)) {
1307                 if (!force)
1308                         goto out;
1309                 switch (mode) {
1310                 case MIGRATE_SYNC:
1311                 case MIGRATE_SYNC_NO_COPY:
1312                         break;
1313                 default:
1314                         goto out;
1315                 }
1316                 lock_page(hpage);
1317         }
1318 
1319         /*
1320          * Check for pages which are in the process of being freed.  Without
1321          * page_mapping() set, hugetlbfs specific move page routine will not
1322          * be called and we could leak usage counts for subpools.
1323          */
1324         if (page_private(hpage) && !page_mapping(hpage)) {
1325                 rc = -EBUSY;
1326                 goto out_unlock;
1327         }
1328 
1329         if (PageAnon(hpage))
1330                 anon_vma = page_get_anon_vma(hpage);
1331 
1332         if (unlikely(!trylock_page(new_hpage)))
1333                 goto put_anon;
1334 
1335         if (page_mapped(hpage)) {
1336                 try_to_unmap(hpage,
1337                         TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1338                 page_was_mapped = 1;
1339         }
1340 
1341         if (!page_mapped(hpage))
1342                 rc = move_to_new_page(new_hpage, hpage, mode);
1343 
1344         if (page_was_mapped)
1345                 remove_migration_ptes(hpage,
1346                         rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1347 
1348         unlock_page(new_hpage);
1349 
1350 put_anon:
1351         if (anon_vma)
1352                 put_anon_vma(anon_vma);
1353 
1354         if (rc == MIGRATEPAGE_SUCCESS) {
1355                 move_hugetlb_state(hpage, new_hpage, reason);
1356                 put_new_page = NULL;
1357         }
1358 
1359 out_unlock:
1360         unlock_page(hpage);
1361 out:
1362         if (rc != -EAGAIN)
1363                 putback_active_hugepage(hpage);
1364 
1365         /*
1366          * If migration was not successful and there's a freeing callback, use
1367          * it.  Otherwise, put_page() will drop the reference grabbed during
1368          * isolation.
1369          */
1370         if (put_new_page)
1371                 put_new_page(new_hpage, private);
1372         else
1373                 putback_active_hugepage(new_hpage);
1374 
1375         return rc;
1376 }
1377 
1378 /*
1379  * migrate_pages - migrate the pages specified in a list, to the free pages
1380  *                 supplied as the target for the page migration
1381  *
1382  * @from:               The list of pages to be migrated.
1383  * @get_new_page:       The function used to allocate free pages to be used
1384  *                      as the target of the page migration.
1385  * @put_new_page:       The function used to free target pages if migration
1386  *                      fails, or NULL if no special handling is necessary.
1387  * @private:            Private data to be passed on to get_new_page()
1388  * @mode:               The migration mode that specifies the constraints for
1389  *                      page migration, if any.
1390  * @reason:             The reason for page migration.
1391  *
1392  * The function returns after 10 attempts or if no pages are movable any more
1393  * because the list has become empty or no retryable pages exist any more.
1394  * The caller should call putback_movable_pages() to return pages to the LRU
1395  * or free list only if ret != 0.
1396  *
1397  * Returns the number of pages that were not migrated, or an error code.
1398  */
1399 int migrate_pages(struct list_head *from, new_page_t get_new_page,
1400                 free_page_t put_new_page, unsigned long private,
1401                 enum migrate_mode mode, int reason)
1402 {
1403         int retry = 1;
1404         int nr_failed = 0;
1405         int nr_succeeded = 0;
1406         int pass = 0;
1407         struct page *page;
1408         struct page *page2;
1409         int swapwrite = current->flags & PF_SWAPWRITE;
1410         int rc;
1411 
1412         if (!swapwrite)
1413                 current->flags |= PF_SWAPWRITE;
1414 
1415         for(pass = 0; pass < 10 && retry; pass++) {
1416                 retry = 0;
1417 
1418                 list_for_each_entry_safe(page, page2, from, lru) {
1419 retry:
1420                         cond_resched();
1421 
1422                         if (PageHuge(page))
1423                                 rc = unmap_and_move_huge_page(get_new_page,
1424                                                 put_new_page, private, page,
1425                                                 pass > 2, mode, reason);
1426                         else
1427                                 rc = unmap_and_move(get_new_page, put_new_page,
1428                                                 private, page, pass > 2, mode,
1429                                                 reason);
1430 
1431                         switch(rc) {
1432                         case -ENOMEM:
1433                                 /*
1434                                  * THP migration might be unsupported or the
1435                                  * allocation could've failed so we should
1436                                  * retry on the same page with the THP split
1437                                  * to base pages.
1438                                  *
1439                                  * Head page is retried immediately and tail
1440                                  * pages are added to the tail of the list so
1441                                  * we encounter them after the rest of the list
1442                                  * is processed.
1443                                  */
1444                                 if (PageTransHuge(page) && !PageHuge(page)) {
1445                                         lock_page(page);
1446                                         rc = split_huge_page_to_list(page, from);
1447                                         unlock_page(page);
1448                                         if (!rc) {
1449                                                 list_safe_reset_next(page, page2, lru);
1450                                                 goto retry;
1451                                         }
1452                                 }
1453                                 nr_failed++;
1454                                 goto out;
1455                         case -EAGAIN:
1456                                 retry++;
1457                                 break;
1458                         case MIGRATEPAGE_SUCCESS:
1459                                 nr_succeeded++;
1460                                 break;
1461                         default:
1462                                 /*
1463                                  * Permanent failure (-EBUSY, -ENOSYS, etc.):
1464                                  * unlike -EAGAIN case, the failed page is
1465                                  * removed from migration page list and not
1466                                  * retried in the next outer loop.
1467                                  */
1468                                 nr_failed++;
1469                                 break;
1470                         }
1471                 }
1472         }
1473         nr_failed += retry;
1474         rc = nr_failed;
1475 out:
1476         if (nr_succeeded)
1477                 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1478         if (nr_failed)
1479                 count_vm_events(PGMIGRATE_FAIL, nr_failed);
1480         trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
1481 
1482         if (!swapwrite)
1483                 current->flags &= ~PF_SWAPWRITE;
1484 
1485         return rc;
1486 }
1487 
1488 #ifdef CONFIG_NUMA
1489 
1490 static int store_status(int __user *status, int start, int value, int nr)
1491 {
1492         while (nr-- > 0) {
1493                 if (put_user(value, status + start))
1494                         return -EFAULT;
1495                 start++;
1496         }
1497 
1498         return 0;
1499 }
1500 
1501 static int do_move_pages_to_node(struct mm_struct *mm,
1502                 struct list_head *pagelist, int node)
1503 {
1504         int err;
1505 
1506         if (list_empty(pagelist))
1507                 return 0;
1508 
1509         err = migrate_pages(pagelist, alloc_new_node_page, NULL, node,
1510                         MIGRATE_SYNC, MR_SYSCALL);
1511         if (err)
1512                 putback_movable_pages(pagelist);
1513         return err;
1514 }
1515 
1516 /*
1517  * Resolves the given address to a struct page, isolates it from the LRU and
1518  * puts it to the given pagelist.
1519  * Returns:
1520  *     errno - if the page cannot be found/isolated
1521  *     0 - when it doesn't have to be migrated because it is already on the
1522  *         target node
1523  *     1 - when it has been queued
1524  */
1525 static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
1526                 int node, struct list_head *pagelist, bool migrate_all)
1527 {
1528         struct vm_area_struct *vma;
1529         struct page *page;
1530         unsigned int follflags;
1531         int err;
1532 
1533         down_read(&mm->mmap_sem);
1534         err = -EFAULT;
1535         vma = find_vma(mm, addr);
1536         if (!vma || addr < vma->vm_start || !vma_migratable(vma))
1537                 goto out;
1538 
1539         /* FOLL_DUMP to ignore special (like zero) pages */
1540         follflags = FOLL_GET | FOLL_DUMP;
1541         page = follow_page(vma, addr, follflags);
1542 
1543         err = PTR_ERR(page);
1544         if (IS_ERR(page))
1545                 goto out;
1546 
1547         err = -ENOENT;
1548         if (!page)
1549                 goto out;
1550 
1551         err = 0;
1552         if (page_to_nid(page) == node)
1553                 goto out_putpage;
1554 
1555         err = -EACCES;
1556         if (page_mapcount(page) > 1 && !migrate_all)
1557                 goto out_putpage;
1558 
1559         if (PageHuge(page)) {
1560                 if (PageHead(page)) {
1561                         isolate_huge_page(page, pagelist);
1562                         err = 1;
1563                 }
1564         } else {
1565                 struct page *head;
1566 
1567                 head = compound_head(page);
1568                 err = isolate_lru_page(head);
1569                 if (err)
1570                         goto out_putpage;
1571 
1572                 err = 1;
1573                 list_add_tail(&head->lru, pagelist);
1574                 mod_node_page_state(page_pgdat(head),
1575                         NR_ISOLATED_ANON + page_is_file_cache(head),
1576                         hpage_nr_pages(head));
1577         }
1578 out_putpage:
1579         /*
1580          * Either remove the duplicate refcount from
1581          * isolate_lru_page() or drop the page ref if it was
1582          * not isolated.
1583          */
1584         put_page(page);
1585 out:
1586         up_read(&mm->mmap_sem);
1587         return err;
1588 }
1589 
1590 /*
1591  * Migrate an array of page address onto an array of nodes and fill
1592  * the corresponding array of status.
1593  */
1594 static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1595                          unsigned long nr_pages,
1596                          const void __user * __user *pages,
1597                          const int __user *nodes,
1598                          int __user *status, int flags)
1599 {
1600         int current_node = NUMA_NO_NODE;
1601         LIST_HEAD(pagelist);
1602         int start, i;
1603         int err = 0, err1;
1604 
1605         migrate_prep();
1606 
1607         for (i = start = 0; i < nr_pages; i++) {
1608                 const void __user *p;
1609                 unsigned long addr;
1610                 int node;
1611 
1612                 err = -EFAULT;
1613                 if (get_user(p, pages + i))
1614                         goto out_flush;
1615                 if (get_user(node, nodes + i))
1616                         goto out_flush;
1617                 addr = (unsigned long)untagged_addr(p);
1618 
1619                 err = -ENODEV;
1620                 if (node < 0 || node >= MAX_NUMNODES)
1621                         goto out_flush;
1622                 if (!node_state(node, N_MEMORY))
1623                         goto out_flush;
1624 
1625                 err = -EACCES;
1626                 if (!node_isset(node, task_nodes))
1627                         goto out_flush;
1628 
1629                 if (current_node == NUMA_NO_NODE) {
1630                         current_node = node;
1631                         start = i;
1632                 } else if (node != current_node) {
1633                         err = do_move_pages_to_node(mm, &pagelist, current_node);
1634                         if (err) {
1635                                 /*
1636                                  * Positive err means the number of failed
1637                                  * pages to migrate.  Since we are going to
1638                                  * abort and return the number of non-migrated
1639                                  * pages, so need to incude the rest of the
1640                                  * nr_pages that have not been attempted as
1641                                  * well.
1642                                  */
1643                                 if (err > 0)
1644                                         err += nr_pages - i - 1;
1645                                 goto out;
1646                         }
1647                         err = store_status(status, start, current_node, i - start);
1648                         if (err)
1649                                 goto out;
1650                         start = i;
1651                         current_node = node;
1652                 }
1653 
1654                 /*
1655                  * Errors in the page lookup or isolation are not fatal and we simply
1656                  * report them via status
1657                  */
1658                 err = add_page_for_migration(mm, addr, current_node,
1659                                 &pagelist, flags & MPOL_MF_MOVE_ALL);
1660 
1661                 if (!err) {
1662                         /* The page is already on the target node */
1663                         err = store_status(status, i, current_node, 1);
1664                         if (err)
1665                                 goto out_flush;
1666                         continue;
1667                 } else if (err > 0) {
1668                         /* The page is successfully queued for migration */
1669                         continue;
1670                 }
1671 
1672                 err = store_status(status, i, err, 1);
1673                 if (err)
1674                         goto out_flush;
1675 
1676                 err = do_move_pages_to_node(mm, &pagelist, current_node);
1677                 if (err) {
1678                         if (err > 0)
1679                                 err += nr_pages - i - 1;
1680                         goto out;
1681                 }
1682                 if (i > start) {
1683                         err = store_status(status, start, current_node, i - start);
1684                         if (err)
1685                                 goto out;
1686                 }
1687                 current_node = NUMA_NO_NODE;
1688         }
1689 out_flush:
1690         if (list_empty(&pagelist))
1691                 return err;
1692 
1693         /* Make sure we do not overwrite the existing error */
1694         err1 = do_move_pages_to_node(mm, &pagelist, current_node);
1695         /*
1696          * Don't have to report non-attempted pages here since:
1697          *     - If the above loop is done gracefully all pages have been
1698          *       attempted.
1699          *     - If the above loop is aborted it means a fatal error
1700          *       happened, should return ret.
1701          */
1702         if (!err1)
1703                 err1 = store_status(status, start, current_node, i - start);
1704         if (err >= 0)
1705                 err = err1;
1706 out:
1707         return err;
1708 }
1709 
1710 /*
1711  * Determine the nodes of an array of pages and store it in an array of status.
1712  */
1713 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1714                                 const void __user **pages, int *status)
1715 {
1716         unsigned long i;
1717 
1718         down_read(&mm->mmap_sem);
1719 
1720         for (i = 0; i < nr_pages; i++) {
1721                 unsigned long addr = (unsigned long)(*pages);
1722                 struct vm_area_struct *vma;
1723                 struct page *page;
1724                 int err = -EFAULT;
1725 
1726                 vma = find_vma(mm, addr);
1727                 if (!vma || addr < vma->vm_start)
1728                         goto set_status;
1729 
1730                 /* FOLL_DUMP to ignore special (like zero) pages */
1731                 page = follow_page(vma, addr, FOLL_DUMP);
1732 
1733                 err = PTR_ERR(page);
1734                 if (IS_ERR(page))
1735                         goto set_status;
1736 
1737                 err = page ? page_to_nid(page) : -ENOENT;
1738 set_status:
1739                 *status = err;
1740 
1741                 pages++;
1742                 status++;
1743         }
1744 
1745         up_read(&mm->mmap_sem);
1746 }
1747 
1748 /*
1749  * Determine the nodes of a user array of pages and store it in
1750  * a user array of status.
1751  */
1752 static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1753                          const void __user * __user *pages,
1754                          int __user *status)
1755 {
1756 #define DO_PAGES_STAT_CHUNK_NR 16
1757         const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1758         int chunk_status[DO_PAGES_STAT_CHUNK_NR];
1759 
1760         while (nr_pages) {
1761                 unsigned long chunk_nr;
1762 
1763                 chunk_nr = nr_pages;
1764                 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1765                         chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1766 
1767                 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1768                         break;
1769 
1770                 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1771 
1772                 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1773                         break;
1774 
1775                 pages += chunk_nr;
1776                 status += chunk_nr;
1777                 nr_pages -= chunk_nr;
1778         }
1779         return nr_pages ? -EFAULT : 0;
1780 }
1781 
1782 /*
1783  * Move a list of pages in the address space of the currently executing
1784  * process.
1785  */
1786 static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
1787                              const void __user * __user *pages,
1788                              const int __user *nodes,
1789                              int __user *status, int flags)
1790 {
1791         struct task_struct *task;
1792         struct mm_struct *mm;
1793         int err;
1794         nodemask_t task_nodes;
1795 
1796         /* Check flags */
1797         if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1798                 return -EINVAL;
1799 
1800         if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1801                 return -EPERM;
1802 
1803         /* Find the mm_struct */
1804         rcu_read_lock();
1805         task = pid ? find_task_by_vpid(pid) : current;
1806         if (!task) {
1807                 rcu_read_unlock();
1808                 return -ESRCH;
1809         }
1810         get_task_struct(task);
1811 
1812         /*
1813          * Check if this process has the right to modify the specified
1814          * process. Use the regular "ptrace_may_access()" checks.
1815          */
1816         if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1817                 rcu_read_unlock();
1818                 err = -EPERM;
1819                 goto out;
1820         }
1821         rcu_read_unlock();
1822 
1823         err = security_task_movememory(task);
1824         if (err)
1825                 goto out;
1826 
1827         task_nodes = cpuset_mems_allowed(task);
1828         mm = get_task_mm(task);
1829         put_task_struct(task);
1830 
1831         if (!mm)
1832                 return -EINVAL;
1833 
1834         if (nodes)
1835                 err = do_pages_move(mm, task_nodes, nr_pages, pages,
1836                                     nodes, status, flags);
1837         else
1838                 err = do_pages_stat(mm, nr_pages, pages, status);
1839 
1840         mmput(mm);
1841         return err;
1842 
1843 out:
1844         put_task_struct(task);
1845         return err;
1846 }
1847 
1848 SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1849                 const void __user * __user *, pages,
1850                 const int __user *, nodes,
1851                 int __user *, status, int, flags)
1852 {
1853         return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1854 }
1855 
1856 #ifdef CONFIG_COMPAT
1857 COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
1858                        compat_uptr_t __user *, pages32,
1859                        const int __user *, nodes,
1860                        int __user *, status,
1861                        int, flags)
1862 {
1863         const void __user * __user *pages;
1864         int i;
1865 
1866         pages = compat_alloc_user_space(nr_pages * sizeof(void *));
1867         for (i = 0; i < nr_pages; i++) {
1868                 compat_uptr_t p;
1869 
1870                 if (get_user(p, pages32 + i) ||
1871                         put_user(compat_ptr(p), pages + i))
1872                         return -EFAULT;
1873         }
1874         return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1875 }
1876 #endif /* CONFIG_COMPAT */
1877 
1878 #ifdef CONFIG_NUMA_BALANCING
1879 /*
1880  * Returns true if this is a safe migration target node for misplaced NUMA
1881  * pages. Currently it only checks the watermarks which crude
1882  */
1883 static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1884                                    unsigned long nr_migrate_pages)
1885 {
1886         int z;
1887 
1888         for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1889                 struct zone *zone = pgdat->node_zones + z;
1890 
1891                 if (!populated_zone(zone))
1892                         continue;
1893 
1894                 /* Avoid waking kswapd by allocating pages_to_migrate pages. */
1895                 if (!zone_watermark_ok(zone, 0,
1896                                        high_wmark_pages(zone) +
1897                                        nr_migrate_pages,
1898                                        0, 0))
1899                         continue;
1900                 return true;
1901         }
1902         return false;
1903 }
1904 
1905 static struct page *alloc_misplaced_dst_page(struct page *page,
1906                                            unsigned long data)
1907 {
1908         int nid = (int) data;
1909         struct page *newpage;
1910 
1911         newpage = __alloc_pages_node(nid,
1912                                          (GFP_HIGHUSER_MOVABLE |
1913                                           __GFP_THISNODE | __GFP_NOMEMALLOC |
1914                                           __GFP_NORETRY | __GFP_NOWARN) &
1915                                          ~__GFP_RECLAIM, 0);
1916 
1917         return newpage;
1918 }
1919 
1920 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
1921 {
1922         int page_lru;
1923 
1924         VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
1925 
1926         /* Avoid migrating to a node that is nearly full */
1927         if (!migrate_balanced_pgdat(pgdat, compound_nr(page)))
1928                 return 0;
1929 
1930         if (isolate_lru_page(page))
1931                 return 0;
1932 
1933         /*
1934          * migrate_misplaced_transhuge_page() skips page migration's usual
1935          * check on page_count(), so we must do it here, now that the page
1936          * has been isolated: a GUP pin, or any other pin, prevents migration.
1937          * The expected page count is 3: 1 for page's mapcount and 1 for the
1938          * caller's pin and 1 for the reference taken by isolate_lru_page().
1939          */
1940         if (PageTransHuge(page) && page_count(page) != 3) {
1941                 putback_lru_page(page);
1942                 return 0;
1943         }
1944 
1945         page_lru = page_is_file_cache(page);
1946         mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
1947                                 hpage_nr_pages(page));
1948 
1949         /*
1950          * Isolating the page has taken another reference, so the
1951          * caller's reference can be safely dropped without the page
1952          * disappearing underneath us during migration.
1953          */
1954         put_page(page);
1955         return 1;
1956 }
1957 
1958 bool pmd_trans_migrating(pmd_t pmd)
1959 {
1960         struct page *page = pmd_page(pmd);
1961         return PageLocked(page);
1962 }
1963 
1964 /*
1965  * Attempt to migrate a misplaced page to the specified destination
1966  * node. Caller is expected to have an elevated reference count on
1967  * the page that will be dropped by this function before returning.
1968  */
1969 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
1970                            int node)
1971 {
1972         pg_data_t *pgdat = NODE_DATA(node);
1973         int isolated;
1974         int nr_remaining;
1975         LIST_HEAD(migratepages);
1976 
1977         /*
1978          * Don't migrate file pages that are mapped in multiple processes
1979          * with execute permissions as they are probably shared libraries.
1980          */
1981         if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
1982             (vma->vm_flags & VM_EXEC))
1983                 goto out;
1984 
1985         /*
1986          * Also do not migrate dirty pages as not all filesystems can move
1987          * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
1988          */
1989         if (page_is_file_cache(page) && PageDirty(page))
1990                 goto out;
1991 
1992         isolated = numamigrate_isolate_page(pgdat, page);
1993         if (!isolated)
1994                 goto out;
1995 
1996         list_add(&page->lru, &migratepages);
1997         nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
1998                                      NULL, node, MIGRATE_ASYNC,
1999                                      MR_NUMA_MISPLACED);
2000         if (nr_remaining) {
2001                 if (!list_empty(&migratepages)) {
2002                         list_del(&page->lru);
2003                         dec_node_page_state(page, NR_ISOLATED_ANON +
2004                                         page_is_file_cache(page));
2005                         putback_lru_page(page);
2006                 }
2007                 isolated = 0;
2008         } else
2009                 count_vm_numa_event(NUMA_PAGE_MIGRATE);
2010         BUG_ON(!list_empty(&migratepages));
2011         return isolated;
2012 
2013 out:
2014         put_page(page);
2015         return 0;
2016 }
2017 #endif /* CONFIG_NUMA_BALANCING */
2018 
2019 #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
2020 /*
2021  * Migrates a THP to a given target node. page must be locked and is unlocked
2022  * before returning.
2023  */
2024 int migrate_misplaced_transhuge_page(struct mm_struct *mm,
2025                                 struct vm_area_struct *vma,
2026                                 pmd_t *pmd, pmd_t entry,
2027                                 unsigned long address,
2028                                 struct page *page, int node)
2029 {
2030         spinlock_t *ptl;
2031         pg_data_t *pgdat = NODE_DATA(node);
2032         int isolated = 0;
2033         struct page *new_page = NULL;
2034         int page_lru = page_is_file_cache(page);
2035         unsigned long start = address & HPAGE_PMD_MASK;
2036 
2037         new_page = alloc_pages_node(node,
2038                 (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
2039                 HPAGE_PMD_ORDER);
2040         if (!new_page)
2041                 goto out_fail;
2042         prep_transhuge_page(new_page);
2043 
2044         isolated = numamigrate_isolate_page(pgdat, page);
2045         if (!isolated) {
2046                 put_page(new_page);
2047                 goto out_fail;
2048         }
2049 
2050         /* Prepare a page as a migration target */
2051         __SetPageLocked(new_page);
2052         if (PageSwapBacked(page))
2053                 __SetPageSwapBacked(new_page);
2054 
2055         /* anon mapping, we can simply copy page->mapping to the new page: */
2056         new_page->mapping = page->mapping;
2057         new_page->index = page->index;
2058         /* flush the cache before copying using the kernel virtual address */
2059         flush_cache_range(vma, start, start + HPAGE_PMD_SIZE);
2060         migrate_page_copy(new_page, page);
2061         WARN_ON(PageLRU(new_page));
2062 
2063         /* Recheck the target PMD */
2064         ptl = pmd_lock(mm, pmd);
2065         if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) {
2066                 spin_unlock(ptl);
2067 
2068                 /* Reverse changes made by migrate_page_copy() */
2069                 if (TestClearPageActive(new_page))
2070                         SetPageActive(page);
2071                 if (TestClearPageUnevictable(new_page))
2072                         SetPageUnevictable(page);
2073 
2074                 unlock_page(new_page);
2075                 put_page(new_page);             /* Free it */
2076 
2077                 /* Retake the callers reference and putback on LRU */
2078                 get_page(page);
2079                 putback_lru_page(page);
2080                 mod_node_page_state(page_pgdat(page),
2081                          NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
2082 
2083                 goto out_unlock;
2084         }
2085 
2086         entry = mk_huge_pmd(new_page, vma->vm_page_prot);
2087         entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
2088 
2089         /*
2090          * Overwrite the old entry under pagetable lock and establish
2091          * the new PTE. Any parallel GUP will either observe the old
2092          * page blocking on the page lock, block on the page table
2093          * lock or observe the new page. The SetPageUptodate on the
2094          * new page and page_add_new_anon_rmap guarantee the copy is
2095          * visible before the pagetable update.
2096          */
2097         page_add_anon_rmap(new_page, vma, start, true);
2098         /*
2099          * At this point the pmd is numa/protnone (i.e. non present) and the TLB
2100          * has already been flushed globally.  So no TLB can be currently
2101          * caching this non present pmd mapping.  There's no need to clear the
2102          * pmd before doing set_pmd_at(), nor to flush the TLB after
2103          * set_pmd_at().  Clearing the pmd here would introduce a race
2104          * condition against MADV_DONTNEED, because MADV_DONTNEED only holds the
2105          * mmap_sem for reading.  If the pmd is set to NULL at any given time,
2106          * MADV_DONTNEED won't wait on the pmd lock and it'll skip clearing this
2107          * pmd.
2108          */
2109         set_pmd_at(mm, start, pmd, entry);
2110         update_mmu_cache_pmd(vma, address, &entry);
2111 
2112         page_ref_unfreeze(page, 2);
2113         mlock_migrate_page(new_page, page);
2114         page_remove_rmap(page, true);
2115         set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
2116 
2117         spin_unlock(ptl);
2118 
2119         /* Take an "isolate" reference and put new page on the LRU. */
2120         get_page(new_page);
2121         putback_lru_page(new_page);
2122 
2123         unlock_page(new_page);
2124         unlock_page(page);
2125         put_page(page);                 /* Drop the rmap reference */
2126         put_page(page);                 /* Drop the LRU isolation reference */
2127 
2128         count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
2129         count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
2130 
2131         mod_node_page_state(page_pgdat(page),
2132                         NR_ISOLATED_ANON + page_lru,
2133                         -HPAGE_PMD_NR);
2134         return isolated;
2135 
2136 out_fail:
2137         count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
2138         ptl = pmd_lock(mm, pmd);
2139         if (pmd_same(*pmd, entry)) {
2140                 entry = pmd_modify(entry, vma->vm_page_prot);
2141                 set_pmd_at(mm, start, pmd, entry);
2142                 update_mmu_cache_pmd(vma, address, &entry);
2143         }
2144         spin_unlock(ptl);
2145 
2146 out_unlock:
2147         unlock_page(page);
2148         put_page(page);
2149         return 0;
2150 }
2151 #endif /* CONFIG_NUMA_BALANCING */
2152 
2153 #endif /* CONFIG_NUMA */
2154 
2155 #ifdef CONFIG_DEVICE_PRIVATE
2156 static int migrate_vma_collect_hole(unsigned long start,
2157                                     unsigned long end,
2158                                     struct mm_walk *walk)
2159 {
2160         struct migrate_vma *migrate = walk->private;
2161         unsigned long addr;
2162 
2163         for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
2164                 migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
2165                 migrate->dst[migrate->npages] = 0;
2166                 migrate->npages++;
2167                 migrate->cpages++;
2168         }
2169 
2170         return 0;
2171 }
2172 
2173 static int migrate_vma_collect_skip(unsigned long start,
2174                                     unsigned long end,
2175                                     struct mm_walk *walk)
2176 {
2177         struct migrate_vma *migrate = walk->private;
2178         unsigned long addr;
2179 
2180         for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
2181                 migrate->dst[migrate->npages] = 0;
2182                 migrate->src[migrate->npages++] = 0;
2183         }
2184 
2185         return 0;
2186 }
2187 
2188 static int migrate_vma_collect_pmd(pmd_t *pmdp,
2189                                    unsigned long start,
2190                                    unsigned long end,
2191                                    struct mm_walk *walk)
2192 {
2193         struct migrate_vma *migrate = walk->private;
2194         struct vm_area_struct *vma = walk->vma;
2195         struct mm_struct *mm = vma->vm_mm;
2196         unsigned long addr = start, unmapped = 0;
2197         spinlock_t *ptl;
2198         pte_t *ptep;
2199 
2200 again:
2201         if (pmd_none(*pmdp))
2202                 return migrate_vma_collect_hole(start, end, walk);
2203 
2204         if (pmd_trans_huge(*pmdp)) {
2205                 struct page *page;
2206 
2207                 ptl = pmd_lock(mm, pmdp);
2208                 if (unlikely(!pmd_trans_huge(*pmdp))) {
2209                         spin_unlock(ptl);
2210                         goto again;
2211                 }
2212 
2213                 page = pmd_page(*pmdp);
2214                 if (is_huge_zero_page(page)) {
2215                         spin_unlock(ptl);
2216                         split_huge_pmd(vma, pmdp, addr);
2217                         if (pmd_trans_unstable(pmdp))
2218                                 return migrate_vma_collect_skip(start, end,
2219                                                                 walk);
2220                 } else {
2221                         int ret;
2222 
2223                         get_page(page);
2224                         spin_unlock(ptl);
2225                         if (unlikely(!trylock_page(page)))
2226                                 return migrate_vma_collect_skip(start, end,
2227                                                                 walk);
2228                         ret = split_huge_page(page);
2229                         unlock_page(page);
2230                         put_page(page);
2231                         if (ret)
2232                                 return migrate_vma_collect_skip(start, end,
2233                                                                 walk);
2234                         if (pmd_none(*pmdp))
2235                                 return migrate_vma_collect_hole(start, end,
2236                                                                 walk);
2237                 }
2238         }
2239 
2240         if (unlikely(pmd_bad(*pmdp)))
2241                 return migrate_vma_collect_skip(start, end, walk);
2242 
2243         ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2244         arch_enter_lazy_mmu_mode();
2245 
2246         for (; addr < end; addr += PAGE_SIZE, ptep++) {
2247                 unsigned long mpfn, pfn;
2248                 struct page *page;
2249                 swp_entry_t entry;
2250                 pte_t pte;
2251 
2252                 pte = *ptep;
2253 
2254                 if (pte_none(pte)) {
2255                         mpfn = MIGRATE_PFN_MIGRATE;
2256                         migrate->cpages++;
2257                         goto next;
2258                 }
2259 
2260                 if (!pte_present(pte)) {
2261                         mpfn = 0;
2262 
2263                         /*
2264                          * Only care about unaddressable device page special
2265                          * page table entry. Other special swap entries are not
2266                          * migratable, and we ignore regular swapped page.
2267                          */
2268                         entry = pte_to_swp_entry(pte);
2269                         if (!is_device_private_entry(entry))
2270                                 goto next;
2271 
2272                         page = device_private_entry_to_page(entry);
2273                         mpfn = migrate_pfn(page_to_pfn(page)) |
2274                                         MIGRATE_PFN_MIGRATE;
2275                         if (is_write_device_private_entry(entry))
2276                                 mpfn |= MIGRATE_PFN_WRITE;
2277                 } else {
2278                         pfn = pte_pfn(pte);
2279                         if (is_zero_pfn(pfn)) {
2280                                 mpfn = MIGRATE_PFN_MIGRATE;
2281                                 migrate->cpages++;
2282                                 goto next;
2283                         }
2284                         page = vm_normal_page(migrate->vma, addr, pte);
2285                         mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
2286                         mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2287                 }
2288 
2289                 /* FIXME support THP */
2290                 if (!page || !page->mapping || PageTransCompound(page)) {
2291                         mpfn = 0;
2292                         goto next;
2293                 }
2294 
2295                 /*
2296                  * By getting a reference on the page we pin it and that blocks
2297                  * any kind of migration. Side effect is that it "freezes" the
2298                  * pte.
2299                  *
2300                  * We drop this reference after isolating the page from the lru
2301                  * for non device page (device page are not on the lru and thus
2302                  * can't be dropped from it).
2303                  */
2304                 get_page(page);
2305                 migrate->cpages++;
2306 
2307                 /*
2308                  * Optimize for the common case where page is only mapped once
2309                  * in one process. If we can lock the page, then we can safely
2310                  * set up a special migration page table entry now.
2311                  */
2312                 if (trylock_page(page)) {
2313                         pte_t swp_pte;
2314 
2315                         mpfn |= MIGRATE_PFN_LOCKED;
2316                         ptep_get_and_clear(mm, addr, ptep);
2317 
2318                         /* Setup special migration page table entry */
2319                         entry = make_migration_entry(page, mpfn &
2320                                                      MIGRATE_PFN_WRITE);
2321                         swp_pte = swp_entry_to_pte(entry);
2322                         if (pte_soft_dirty(pte))
2323                                 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2324                         set_pte_at(mm, addr, ptep, swp_pte);
2325 
2326                         /*
2327                          * This is like regular unmap: we remove the rmap and
2328                          * drop page refcount. Page won't be freed, as we took
2329                          * a reference just above.
2330                          */
2331                         page_remove_rmap(page, false);
2332                         put_page(page);
2333 
2334                         if (pte_present(pte))
2335                                 unmapped++;
2336                 }
2337 
2338 next:
2339                 migrate->dst[migrate->npages] = 0;
2340                 migrate->src[migrate->npages++] = mpfn;
2341         }
2342         arch_leave_lazy_mmu_mode();
2343         pte_unmap_unlock(ptep - 1, ptl);
2344 
2345         /* Only flush the TLB if we actually modified any entries */
2346         if (unmapped)
2347                 flush_tlb_range(walk->vma, start, end);
2348 
2349         return 0;
2350 }
2351 
2352 static const struct mm_walk_ops migrate_vma_walk_ops = {
2353         .pmd_entry              = migrate_vma_collect_pmd,
2354         .pte_hole               = migrate_vma_collect_hole,
2355 };
2356 
2357 /*
2358  * migrate_vma_collect() - collect pages over a range of virtual addresses
2359  * @migrate: migrate struct containing all migration information
2360  *
2361  * This will walk the CPU page table. For each virtual address backed by a
2362  * valid page, it updates the src array and takes a reference on the page, in
2363  * order to pin the page until we lock it and unmap it.
2364  */
2365 static void migrate_vma_collect(struct migrate_vma *migrate)
2366 {
2367         struct mmu_notifier_range range;
2368 
2369         mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL,
2370                         migrate->vma->vm_mm, migrate->start, migrate->end);
2371         mmu_notifier_invalidate_range_start(&range);
2372 
2373         walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
2374                         &migrate_vma_walk_ops, migrate);
2375 
2376         mmu_notifier_invalidate_range_end(&range);
2377         migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
2378 }
2379 
2380 /*
2381  * migrate_vma_check_page() - check if page is pinned or not
2382  * @page: struct page to check
2383  *
2384  * Pinned pages cannot be migrated. This is the same test as in
2385  * migrate_page_move_mapping(), except that here we allow migration of a
2386  * ZONE_DEVICE page.
2387  */
2388 static bool migrate_vma_check_page(struct page *page)
2389 {
2390         /*
2391          * One extra ref because caller holds an extra reference, either from
2392          * isolate_lru_page() for a regular page, or migrate_vma_collect() for
2393          * a device page.
2394          */
2395         int extra = 1;
2396 
2397         /*
2398          * FIXME support THP (transparent huge page), it is bit more complex to
2399          * check them than regular pages, because they can be mapped with a pmd
2400          * or with a pte (split pte mapping).
2401          */
2402         if (PageCompound(page))
2403                 return false;
2404 
2405         /* Page from ZONE_DEVICE have one extra reference */
2406         if (is_zone_device_page(page)) {
2407                 /*
2408                  * Private page can never be pin as they have no valid pte and
2409                  * GUP will fail for those. Yet if there is a pending migration
2410                  * a thread might try to wait on the pte migration entry and
2411                  * will bump the page reference count. Sadly there is no way to
2412                  * differentiate a regular pin from migration wait. Hence to
2413                  * avoid 2 racing thread trying to migrate back to CPU to enter
2414                  * infinite loop (one stoping migration because the other is
2415                  * waiting on pte migration entry). We always return true here.
2416                  *
2417                  * FIXME proper solution is to rework migration_entry_wait() so
2418                  * it does not need to take a reference on page.
2419                  */
2420                 return is_device_private_page(page);
2421         }
2422 
2423         /* For file back page */
2424         if (page_mapping(page))
2425                 extra += 1 + page_has_private(page);
2426 
2427         if ((page_count(page) - extra) > page_mapcount(page))
2428                 return false;
2429 
2430         return true;
2431 }
2432 
2433 /*
2434  * migrate_vma_prepare() - lock pages and isolate them from the lru
2435  * @migrate: migrate struct containing all migration information
2436  *
2437  * This locks pages that have been collected by migrate_vma_collect(). Once each
2438  * page is locked it is isolated from the lru (for non-device pages). Finally,
2439  * the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be
2440  * migrated by concurrent kernel threads.
2441  */
2442 static void migrate_vma_prepare(struct migrate_vma *migrate)
2443 {
2444         const unsigned long npages = migrate->npages;
2445         const unsigned long start = migrate->start;
2446         unsigned long addr, i, restore = 0;
2447         bool allow_drain = true;
2448 
2449         lru_add_drain();
2450 
2451         for (i = 0; (i < npages) && migrate->cpages; i++) {
2452                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2453                 bool remap = true;
2454 
2455                 if (!page)
2456                         continue;
2457 
2458                 if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
2459                         /*
2460                          * Because we are migrating several pages there can be
2461                          * a deadlock between 2 concurrent migration where each
2462                          * are waiting on each other page lock.
2463                          *
2464                          * Make migrate_vma() a best effort thing and backoff
2465                          * for any page we can not lock right away.
2466                          */
2467                         if (!trylock_page(page)) {
2468                                 migrate->src[i] = 0;
2469                                 migrate->cpages--;
2470                                 put_page(page);
2471                                 continue;
2472                         }
2473                         remap = false;
2474                         migrate->src[i] |= MIGRATE_PFN_LOCKED;
2475                 }
2476 
2477                 /* ZONE_DEVICE pages are not on LRU */
2478                 if (!is_zone_device_page(page)) {
2479                         if (!PageLRU(page) && allow_drain) {
2480                                 /* Drain CPU's pagevec */
2481                                 lru_add_drain_all();
2482                                 allow_drain = false;
2483                         }
2484 
2485                         if (isolate_lru_page(page)) {
2486                                 if (remap) {
2487                                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2488                                         migrate->cpages--;
2489                                         restore++;
2490                                 } else {
2491                                         migrate->src[i] = 0;
2492                                         unlock_page(page);
2493                                         migrate->cpages--;
2494                                         put_page(page);
2495                                 }
2496                                 continue;
2497                         }
2498 
2499                         /* Drop the reference we took in collect */
2500                         put_page(page);
2501                 }
2502 
2503                 if (!migrate_vma_check_page(page)) {
2504                         if (remap) {
2505                                 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2506                                 migrate->cpages--;
2507                                 restore++;
2508 
2509                                 if (!is_zone_device_page(page)) {
2510                                         get_page(page);
2511                                         putback_lru_page(page);
2512                                 }
2513                         } else {
2514                                 migrate->src[i] = 0;
2515                                 unlock_page(page);
2516                                 migrate->cpages--;
2517 
2518                                 if (!is_zone_device_page(page))
2519                                         putback_lru_page(page);
2520                                 else
2521                                         put_page(page);
2522                         }
2523                 }
2524         }
2525 
2526         for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
2527                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2528 
2529                 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2530                         continue;
2531 
2532                 remove_migration_pte(page, migrate->vma, addr, page);
2533 
2534                 migrate->src[i] = 0;
2535                 unlock_page(page);
2536                 put_page(page);
2537                 restore--;
2538         }
2539 }
2540 
2541 /*
2542  * migrate_vma_unmap() - replace page mapping with special migration pte entry
2543  * @migrate: migrate struct containing all migration information
2544  *
2545  * Replace page mapping (CPU page table pte) with a special migration pte entry
2546  * and check again if it has been pinned. Pinned pages are restored because we
2547  * cannot migrate them.
2548  *
2549  * This is the last step before we call the device driver callback to allocate
2550  * destination memory and copy contents of original page over to new page.
2551  */
2552 static void migrate_vma_unmap(struct migrate_vma *migrate)
2553 {
2554         int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
2555         const unsigned long npages = migrate->npages;
2556         const unsigned long start = migrate->start;
2557         unsigned long addr, i, restore = 0;
2558 
2559         for (i = 0; i < npages; i++) {
2560                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2561 
2562                 if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2563                         continue;
2564 
2565                 if (page_mapped(page)) {
2566                         try_to_unmap(page, flags);
2567                         if (page_mapped(page))
2568                                 goto restore;
2569                 }
2570 
2571                 if (migrate_vma_check_page(page))
2572                         continue;
2573 
2574 restore:
2575                 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2576                 migrate->cpages--;
2577                 restore++;
2578         }
2579 
2580         for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
2581                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2582 
2583                 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2584                         continue;
2585 
2586                 remove_migration_ptes(page, page, false);
2587 
2588                 migrate->src[i] = 0;
2589                 unlock_page(page);
2590                 restore--;
2591 
2592                 if (is_zone_device_page(page))
2593                         put_page(page);
2594                 else
2595                         putback_lru_page(page);
2596         }
2597 }
2598 
2599 /**
2600  * migrate_vma_setup() - prepare to migrate a range of memory
2601  * @args: contains the vma, start, and and pfns arrays for the migration
2602  *
2603  * Returns: negative errno on failures, 0 when 0 or more pages were migrated
2604  * without an error.
2605  *
2606  * Prepare to migrate a range of memory virtual address range by collecting all
2607  * the pages backing each virtual address in the range, saving them inside the
2608  * src array.  Then lock those pages and unmap them. Once the pages are locked
2609  * and unmapped, check whether each page is pinned or not.  Pages that aren't
2610  * pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
2611  * corresponding src array entry.  Then restores any pages that are pinned, by
2612  * remapping and unlocking those pages.
2613  *
2614  * The caller should then allocate destination memory and copy source memory to
2615  * it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
2616  * flag set).  Once these are allocated and copied, the caller must update each
2617  * corresponding entry in the dst array with the pfn value of the destination
2618  * page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_LOCKED flags set
2619  * (destination pages must have their struct pages locked, via lock_page()).
2620  *
2621  * Note that the caller does not have to migrate all the pages that are marked
2622  * with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
2623  * device memory to system memory.  If the caller cannot migrate a device page
2624  * back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
2625  * consequences for the userspace process, so it must be avoided if at all
2626  * possible.
2627  *
2628  * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
2629  * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
2630  * allowing the caller to allocate device memory for those unback virtual
2631  * address.  For this the caller simply has to allocate device memory and
2632  * properly set the destination entry like for regular migration.  Note that
2633  * this can still fails and thus inside the device driver must check if the
2634  * migration was successful for those entries after calling migrate_vma_pages()
2635  * just like for regular migration.
2636  *
2637  * After that, the callers must call migrate_vma_pages() to go over each entry
2638  * in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
2639  * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
2640  * then migrate_vma_pages() to migrate struct page information from the source
2641  * struct page to the destination struct page.  If it fails to migrate the
2642  * struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
2643  * src array.
2644  *
2645  * At this point all successfully migrated pages have an entry in the src
2646  * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
2647  * array entry with MIGRATE_PFN_VALID flag set.
2648  *
2649  * Once migrate_vma_pages() returns the caller may inspect which pages were
2650  * successfully migrated, and which were not.  Successfully migrated pages will
2651  * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
2652  *
2653  * It is safe to update device page table after migrate_vma_pages() because
2654  * both destination and source page are still locked, and the mmap_sem is held
2655  * in read mode (hence no one can unmap the range being migrated).
2656  *
2657  * Once the caller is done cleaning up things and updating its page table (if it
2658  * chose to do so, this is not an obligation) it finally calls
2659  * migrate_vma_finalize() to update the CPU page table to point to new pages
2660  * for successfully migrated pages or otherwise restore the CPU page table to
2661  * point to the original source pages.
2662  */
2663 int migrate_vma_setup(struct migrate_vma *args)
2664 {
2665         long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
2666 
2667         args->start &= PAGE_MASK;
2668         args->end &= PAGE_MASK;
2669         if (!args->vma || is_vm_hugetlb_page(args->vma) ||
2670             (args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
2671                 return -EINVAL;
2672         if (nr_pages <= 0)
2673                 return -EINVAL;
2674         if (args->start < args->vma->vm_start ||
2675             args->start >= args->vma->vm_end)
2676                 return -EINVAL;
2677         if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
2678                 return -EINVAL;
2679         if (!args->src || !args->dst)
2680                 return -EINVAL;
2681 
2682         memset(args->src, 0, sizeof(*args->src) * nr_pages);
2683         args->cpages = 0;
2684         args->npages = 0;
2685 
2686         migrate_vma_collect(args);
2687 
2688         if (args->cpages)
2689                 migrate_vma_prepare(args);
2690         if (args->cpages)
2691                 migrate_vma_unmap(args);
2692 
2693         /*
2694          * At this point pages are locked and unmapped, and thus they have
2695          * stable content and can safely be copied to destination memory that
2696          * is allocated by the drivers.
2697          */
2698         return 0;
2699 
2700 }
2701 EXPORT_SYMBOL(migrate_vma_setup);
2702 
2703 static void migrate_vma_insert_page(struct migrate_vma *migrate,
2704                                     unsigned long addr,
2705                                     struct page *page,
2706                                     unsigned long *src,
2707                                     unsigned long *dst)
2708 {
2709         struct vm_area_struct *vma = migrate->vma;
2710         struct mm_struct *mm = vma->vm_mm;
2711         struct mem_cgroup *memcg;
2712         bool flush = false;
2713         spinlock_t *ptl;
2714         pte_t entry;
2715         pgd_t *pgdp;
2716         p4d_t *p4dp;
2717         pud_t *pudp;
2718         pmd_t *pmdp;
2719         pte_t *ptep;
2720 
2721         /* Only allow populating anonymous memory */
2722         if (!vma_is_anonymous(vma))
2723                 goto abort;
2724 
2725         pgdp = pgd_offset(mm, addr);
2726         p4dp = p4d_alloc(mm, pgdp, addr);
2727         if (!p4dp)
2728                 goto abort;
2729         pudp = pud_alloc(mm, p4dp, addr);
2730         if (!pudp)
2731                 goto abort;
2732         pmdp = pmd_alloc(mm, pudp, addr);
2733         if (!pmdp)
2734                 goto abort;
2735 
2736         if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
2737                 goto abort;
2738 
2739         /*
2740          * Use pte_alloc() instead of pte_alloc_map().  We can't run
2741          * pte_offset_map() on pmds where a huge pmd might be created
2742          * from a different thread.
2743          *
2744          * pte_alloc_map() is safe to use under down_write(mmap_sem) or when
2745          * parallel threads are excluded by other means.
2746          *
2747          * Here we only have down_read(mmap_sem).
2748          */
2749         if (pte_alloc(mm, pmdp))
2750                 goto abort;
2751 
2752         /* See the comment in pte_alloc_one_map() */
2753         if (unlikely(pmd_trans_unstable(pmdp)))
2754                 goto abort;
2755 
2756         if (unlikely(anon_vma_prepare(vma)))
2757                 goto abort;
2758         if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
2759                 goto abort;
2760 
2761         /*
2762          * The memory barrier inside __SetPageUptodate makes sure that
2763          * preceding stores to the page contents become visible before
2764          * the set_pte_at() write.
2765          */
2766         __SetPageUptodate(page);
2767 
2768         if (is_zone_device_page(page)) {
2769                 if (is_device_private_page(page)) {
2770                         swp_entry_t swp_entry;
2771 
2772                         swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
2773                         entry = swp_entry_to_pte(swp_entry);
2774                 }
2775         } else {
2776                 entry = mk_pte(page, vma->vm_page_prot);
2777                 if (vma->vm_flags & VM_WRITE)
2778                         entry = pte_mkwrite(pte_mkdirty(entry));
2779         }
2780 
2781         ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2782 
2783         if (pte_present(*ptep)) {
2784                 unsigned long pfn = pte_pfn(*ptep);
2785 
2786                 if (!is_zero_pfn(pfn)) {
2787                         pte_unmap_unlock(ptep, ptl);
2788                         mem_cgroup_cancel_charge(page, memcg, false);
2789                         goto abort;
2790                 }
2791                 flush = true;
2792         } else if (!pte_none(*ptep)) {
2793                 pte_unmap_unlock(ptep, ptl);
2794                 mem_cgroup_cancel_charge(page, memcg, false);
2795                 goto abort;
2796         }
2797 
2798         /*
2799          * Check for usefaultfd but do not deliver the fault. Instead,
2800          * just back off.
2801          */
2802         if (userfaultfd_missing(vma)) {
2803                 pte_unmap_unlock(ptep, ptl);
2804                 mem_cgroup_cancel_charge(page, memcg, false);
2805                 goto abort;
2806         }
2807 
2808         inc_mm_counter(mm, MM_ANONPAGES);
2809         page_add_new_anon_rmap(page, vma, addr, false);
2810         mem_cgroup_commit_charge(page, memcg, false, false);
2811         if (!is_zone_device_page(page))
2812                 lru_cache_add_active_or_unevictable(page, vma);
2813         get_page(page);
2814 
2815         if (flush) {
2816                 flush_cache_page(vma, addr, pte_pfn(*ptep));
2817                 ptep_clear_flush_notify(vma, addr, ptep);
2818                 set_pte_at_notify(mm, addr, ptep, entry);
2819                 update_mmu_cache(vma, addr, ptep);
2820         } else {
2821                 /* No need to invalidate - it was non-present before */
2822                 set_pte_at(mm, addr, ptep, entry);
2823                 update_mmu_cache(vma, addr, ptep);
2824         }
2825 
2826         pte_unmap_unlock(ptep, ptl);
2827         *src = MIGRATE_PFN_MIGRATE;
2828         return;
2829 
2830 abort:
2831         *src &= ~MIGRATE_PFN_MIGRATE;
2832 }
2833 
2834 /**
2835  * migrate_vma_pages() - migrate meta-data from src page to dst page
2836  * @migrate: migrate struct containing all migration information
2837  *
2838  * This migrates struct page meta-data from source struct page to destination
2839  * struct page. This effectively finishes the migration from source page to the
2840  * destination page.
2841  */
2842 void migrate_vma_pages(struct migrate_vma *migrate)
2843 {
2844         const unsigned long npages = migrate->npages;
2845         const unsigned long start = migrate->start;
2846         struct mmu_notifier_range range;
2847         unsigned long addr, i;
2848         bool notified = false;
2849 
2850         for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
2851                 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2852                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2853                 struct address_space *mapping;
2854                 int r;
2855 
2856                 if (!newpage) {
2857                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2858                         continue;
2859                 }
2860 
2861                 if (!page) {
2862                         if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
2863                                 continue;
2864                         }
2865                         if (!notified) {
2866                                 notified = true;
2867 
2868                                 mmu_notifier_range_init(&range,
2869                                                         MMU_NOTIFY_CLEAR, 0,
2870                                                         NULL,
2871                                                         migrate->vma->vm_mm,
2872                                                         addr, migrate->end);
2873                                 mmu_notifier_invalidate_range_start(&range);
2874                         }
2875                         migrate_vma_insert_page(migrate, addr, newpage,
2876                                                 &migrate->src[i],
2877                                                 &migrate->dst[i]);
2878                         continue;
2879                 }
2880 
2881                 mapping = page_mapping(page);
2882 
2883                 if (is_zone_device_page(newpage)) {
2884                         if (is_device_private_page(newpage)) {
2885                                 /*
2886                                  * For now only support private anonymous when
2887                                  * migrating to un-addressable device memory.
2888                                  */
2889                                 if (mapping) {
2890                                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2891                                         continue;
2892                                 }
2893                         } else {
2894                                 /*
2895                                  * Other types of ZONE_DEVICE page are not
2896                                  * supported.
2897                                  */
2898                                 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2899                                 continue;
2900                         }
2901                 }
2902 
2903                 r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
2904                 if (r != MIGRATEPAGE_SUCCESS)
2905                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2906         }
2907 
2908         /*
2909          * No need to double call mmu_notifier->invalidate_range() callback as
2910          * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
2911          * did already call it.
2912          */
2913         if (notified)
2914                 mmu_notifier_invalidate_range_only_end(&range);
2915 }
2916 EXPORT_SYMBOL(migrate_vma_pages);
2917 
2918 /**
2919  * migrate_vma_finalize() - restore CPU page table entry
2920  * @migrate: migrate struct containing all migration information
2921  *
2922  * This replaces the special migration pte entry with either a mapping to the
2923  * new page if migration was successful for that page, or to the original page
2924  * otherwise.
2925  *
2926  * This also unlocks the pages and puts them back on the lru, or drops the extra
2927  * refcount, for device pages.
2928  */
2929 void migrate_vma_finalize(struct migrate_vma *migrate)
2930 {
2931         const unsigned long npages = migrate->npages;
2932         unsigned long i;
2933 
2934         for (i = 0; i < npages; i++) {
2935                 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2936                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2937 
2938                 if (!page) {
2939                         if (newpage) {
2940                                 unlock_page(newpage);
2941                                 put_page(newpage);
2942                         }
2943                         continue;
2944                 }
2945 
2946                 if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
2947                         if (newpage) {
2948                                 unlock_page(newpage);
2949                                 put_page(newpage);
2950                         }
2951                         newpage = page;
2952                 }
2953 
2954                 remove_migration_ptes(page, newpage, false);
2955                 unlock_page(page);
2956                 migrate->cpages--;
2957 
2958                 if (is_zone_device_page(page))
2959                         put_page(page);
2960                 else
2961                         putback_lru_page(page);
2962 
2963                 if (newpage != page) {
2964                         unlock_page(newpage);
2965                         if (is_zone_device_page(newpage))
2966                                 put_page(newpage);
2967                         else
2968                                 putback_lru_page(newpage);
2969                 }
2970         }
2971 }
2972 EXPORT_SYMBOL(migrate_vma_finalize);
2973 #endif /* CONFIG_DEVICE_PRIVATE */

/* [<][>][^][v][top][bottom][index][help] */