mm/hugetlb.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
unlock_or_release_subpool
hugepage_new_subpool
hugepage_put_subpool
hugepage_subpool_get_pages
hugepage_subpool_put_pages
subpool_inode
subpool_vma
region_add
region_chg
region_abort
region_del
hugetlb_fix_reserve_counts
region_count
vma_hugecache_offset
linear_hugepage_index
vma_kernel_pagesize
vma_mmu_pagesize
get_vma_private_data
set_vma_private_data
resv_map_alloc
resv_map_release
inode_resv_map
vma_resv_map
set_vma_resv_map
set_vma_resv_flags
is_vma_resv_set
reset_vma_resv_huge_pages
vma_has_reserves
enqueue_huge_page
dequeue_huge_page_node_exact
dequeue_huge_page_nodemask
htlb_alloc_mask
dequeue_huge_page_vma
next_node_allowed
get_valid_node_allowed
hstate_next_node_to_alloc
hstate_next_node_to_free
destroy_compound_gigantic_page
free_gigantic_page
__alloc_gigantic_page
pfn_range_valid_gigantic
zone_spans_last_pfn
alloc_gigantic_page
alloc_gigantic_page
alloc_gigantic_page
free_gigantic_page
destroy_compound_gigantic_page
update_and_free_page
size_to_hstate
page_huge_active
set_page_huge_active
clear_page_huge_active
PageHugeTemporary
SetPageHugeTemporary
ClearPageHugeTemporary
__free_huge_page
free_hpage_workfn
free_huge_page
prep_new_huge_page
prep_compound_gigantic_page
PageHuge
PageHeadHuge
__basepage_index
alloc_buddy_huge_page
alloc_fresh_huge_page
alloc_pool_huge_page
free_pool_huge_page
dissolve_free_huge_page
dissolve_free_huge_pages
alloc_surplus_huge_page
alloc_migrate_huge_page
alloc_buddy_huge_page_with_mpol
alloc_huge_page_node
alloc_huge_page_nodemask
alloc_huge_page_vma
gather_surplus_pages
return_unused_surplus_pages
__vma_reservation_common
vma_needs_reservation
vma_commit_reservation
vma_end_reservation
vma_add_reservation
restore_reserve_on_error
alloc_huge_page
__alloc_bootmem_huge_page
prep_compound_huge_page
gather_bootmem_prealloc
hugetlb_hstate_alloc_pages
hugetlb_init_hstates
report_hugepages
try_to_free_low
try_to_free_low
adjust_pool_surplus
set_max_huge_pages
kobj_to_hstate
nr_hugepages_show_common
__nr_hugepages_store_common
nr_hugepages_store_common
nr_hugepages_show
nr_hugepages_store
nr_hugepages_mempolicy_show
nr_hugepages_mempolicy_store
nr_overcommit_hugepages_show
nr_overcommit_hugepages_store
free_hugepages_show
resv_hugepages_show
surplus_hugepages_show
hugetlb_sysfs_add_hstate
hugetlb_sysfs_init
kobj_to_node_hstate
hugetlb_unregister_node
hugetlb_register_node
hugetlb_register_all_nodes
kobj_to_node_hstate
hugetlb_register_all_nodes
hugetlb_init
hugetlb_bad_size
hugetlb_add_hstate
hugetlb_nrpages_setup
hugetlb_default_setup
cpuset_mems_nr
hugetlb_sysctl_handler_common
hugetlb_sysctl_handler
hugetlb_mempolicy_sysctl_handler
hugetlb_overcommit_handler
hugetlb_report_meminfo
hugetlb_report_node_meminfo
hugetlb_show_meminfo
hugetlb_report_usage
hugetlb_total_pages
hugetlb_acct_memory
hugetlb_vm_op_open
hugetlb_vm_op_close
hugetlb_vm_op_split
hugetlb_vm_op_pagesize
hugetlb_vm_op_fault
make_huge_pte
set_huge_ptep_writable
is_hugetlb_entry_migration
is_hugetlb_entry_hwpoisoned
copy_hugetlb_page_range
__unmap_hugepage_range
__unmap_hugepage_range_final
unmap_hugepage_range
unmap_ref_private
hugetlb_cow
hugetlbfs_pagecache_page
hugetlbfs_pagecache_present
huge_add_to_page_cache
hugetlb_no_page
hugetlb_fault_mutex_hash
hugetlb_fault_mutex_hash
hugetlb_fault
hugetlb_mcopy_atomic_pte
follow_hugetlb_page
hugetlb_change_protection
hugetlb_reserve_pages
hugetlb_unreserve_pages
page_table_shareable
vma_shareable
adjust_range_if_pmd_sharing_possible
huge_pmd_share
huge_pmd_unshare
huge_pmd_share
huge_pmd_unshare
adjust_range_if_pmd_sharing_possible
huge_pte_alloc
huge_pte_offset
follow_huge_addr
follow_huge_pd
follow_huge_pmd
follow_huge_pud
follow_huge_pgd
isolate_huge_page
putback_active_hugepage
move_hugetlb_state
   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Generic hugetlb support.
   4  * (C) Nadia Yvette Chambers, April 2004
   5  */
   6 #include <linux/list.h>
   7 #include <linux/init.h>
   8 #include <linux/mm.h>
   9 #include <linux/seq_file.h>
  10 #include <linux/sysctl.h>
  11 #include <linux/highmem.h>
  12 #include <linux/mmu_notifier.h>
  13 #include <linux/nodemask.h>
  14 #include <linux/pagemap.h>
  15 #include <linux/mempolicy.h>
  16 #include <linux/compiler.h>
  17 #include <linux/cpuset.h>
  18 #include <linux/mutex.h>
  19 #include <linux/memblock.h>
  20 #include <linux/sysfs.h>
  21 #include <linux/slab.h>
  22 #include <linux/mmdebug.h>
  23 #include <linux/sched/signal.h>
  24 #include <linux/rmap.h>
  25 #include <linux/string_helpers.h>
  26 #include <linux/swap.h>
  27 #include <linux/swapops.h>
  28 #include <linux/jhash.h>
  29 #include <linux/numa.h>
  30 #include <linux/llist.h>
  31 
  32 #include <asm/page.h>
  33 #include <asm/pgtable.h>
  34 #include <asm/tlb.h>
  35 
  36 #include <linux/io.h>
  37 #include <linux/hugetlb.h>
  38 #include <linux/hugetlb_cgroup.h>
  39 #include <linux/node.h>
  40 #include <linux/userfaultfd_k.h>
  41 #include <linux/page_owner.h>
  42 #include "internal.h"
  43 
  44 int hugetlb_max_hstate __read_mostly;
  45 unsigned int default_hstate_idx;
  46 struct hstate hstates[HUGE_MAX_HSTATE];
  47 /*
  48  * Minimum page order among possible hugepage sizes, set to a proper value
  49  * at boot time.
  50  */
  51 static unsigned int minimum_order __read_mostly = UINT_MAX;
  52 
  53 __initdata LIST_HEAD(huge_boot_pages);
  54 
  55 /* for command line parsing */
  56 static struct hstate * __initdata parsed_hstate;
  57 static unsigned long __initdata default_hstate_max_huge_pages;
  58 static unsigned long __initdata default_hstate_size;
  59 static bool __initdata parsed_valid_hugepagesz = true;
  60 
  61 /*
  62  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
  63  * free_huge_pages, and surplus_huge_pages.
  64  */
  65 DEFINE_SPINLOCK(hugetlb_lock);
  66 
  67 /*
  68  * Serializes faults on the same logical page.  This is used to
  69  * prevent spurious OOMs when the hugepage pool is fully utilized.
  70  */
  71 static int num_fault_mutexes;
  72 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
  73 
  74 /* Forward declaration */
  75 static int hugetlb_acct_memory(struct hstate *h, long delta);
  76 
  77 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
  78 {
  79         bool free = (spool->count == 0) && (spool->used_hpages == 0);
  80 
  81         spin_unlock(&spool->lock);
  82 
  83         /* If no pages are used, and no other handles to the subpool
  84          * remain, give up any reservations mased on minimum size and
  85          * free the subpool */
  86         if (free) {
  87                 if (spool->min_hpages != -1)
  88                         hugetlb_acct_memory(spool->hstate,
  89                                                 -spool->min_hpages);
  90                 kfree(spool);
  91         }
  92 }
  93 
  94 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
  95                                                 long min_hpages)
  96 {
  97         struct hugepage_subpool *spool;
  98 
  99         spool = kzalloc(sizeof(*spool), GFP_KERNEL);
 100         if (!spool)
 101                 return NULL;
 102 
 103         spin_lock_init(&spool->lock);
 104         spool->count = 1;
 105         spool->max_hpages = max_hpages;
 106         spool->hstate = h;
 107         spool->min_hpages = min_hpages;
 108 
 109         if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
 110                 kfree(spool);
 111                 return NULL;
 112         }
 113         spool->rsv_hpages = min_hpages;
 114 
 115         return spool;
 116 }
 117 
 118 void hugepage_put_subpool(struct hugepage_subpool *spool)
 119 {
 120         spin_lock(&spool->lock);
 121         BUG_ON(!spool->count);
 122         spool->count--;
 123         unlock_or_release_subpool(spool);
 124 }
 125 
 126 /*
 127  * Subpool accounting for allocating and reserving pages.
 128  * Return -ENOMEM if there are not enough resources to satisfy the
 129  * the request.  Otherwise, return the number of pages by which the
 130  * global pools must be adjusted (upward).  The returned value may
 131  * only be different than the passed value (delta) in the case where
 132  * a subpool minimum size must be manitained.
 133  */
 134 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
 135                                       long delta)
 136 {
 137         long ret = delta;
 138 
 139         if (!spool)
 140                 return ret;
 141 
 142         spin_lock(&spool->lock);
 143 
 144         if (spool->max_hpages != -1) {          /* maximum size accounting */
 145                 if ((spool->used_hpages + delta) <= spool->max_hpages)
 146                         spool->used_hpages += delta;
 147                 else {
 148                         ret = -ENOMEM;
 149                         goto unlock_ret;
 150                 }
 151         }
 152 
 153         /* minimum size accounting */
 154         if (spool->min_hpages != -1 && spool->rsv_hpages) {
 155                 if (delta > spool->rsv_hpages) {
 156                         /*
 157                          * Asking for more reserves than those already taken on
 158                          * behalf of subpool.  Return difference.
 159                          */
 160                         ret = delta - spool->rsv_hpages;
 161                         spool->rsv_hpages = 0;
 162                 } else {
 163                         ret = 0;        /* reserves already accounted for */
 164                         spool->rsv_hpages -= delta;
 165                 }
 166         }
 167 
 168 unlock_ret:
 169         spin_unlock(&spool->lock);
 170         return ret;
 171 }
 172 
 173 /*
 174  * Subpool accounting for freeing and unreserving pages.
 175  * Return the number of global page reservations that must be dropped.
 176  * The return value may only be different than the passed value (delta)
 177  * in the case where a subpool minimum size must be maintained.
 178  */
 179 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
 180                                        long delta)
 181 {
 182         long ret = delta;
 183 
 184         if (!spool)
 185                 return delta;
 186 
 187         spin_lock(&spool->lock);
 188 
 189         if (spool->max_hpages != -1)            /* maximum size accounting */
 190                 spool->used_hpages -= delta;
 191 
 192          /* minimum size accounting */
 193         if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
 194                 if (spool->rsv_hpages + delta <= spool->min_hpages)
 195                         ret = 0;
 196                 else
 197                         ret = spool->rsv_hpages + delta - spool->min_hpages;
 198 
 199                 spool->rsv_hpages += delta;
 200                 if (spool->rsv_hpages > spool->min_hpages)
 201                         spool->rsv_hpages = spool->min_hpages;
 202         }
 203 
 204         /*
 205          * If hugetlbfs_put_super couldn't free spool due to an outstanding
 206          * quota reference, free it now.
 207          */
 208         unlock_or_release_subpool(spool);
 209 
 210         return ret;
 211 }
 212 
 213 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 214 {
 215         return HUGETLBFS_SB(inode->i_sb)->spool;
 216 }
 217 
 218 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
 219 {
 220         return subpool_inode(file_inode(vma->vm_file));
 221 }
 222 
 223 /*
 224  * Region tracking -- allows tracking of reservations and instantiated pages
 225  *                    across the pages in a mapping.
 226  *
 227  * The region data structures are embedded into a resv_map and protected
 228  * by a resv_map's lock.  The set of regions within the resv_map represent
 229  * reservations for huge pages, or huge pages that have already been
 230  * instantiated within the map.  The from and to elements are huge page
 231  * indicies into the associated mapping.  from indicates the starting index
 232  * of the region.  to represents the first index past the end of  the region.
 233  *
 234  * For example, a file region structure with from == 0 and to == 4 represents
 235  * four huge pages in a mapping.  It is important to note that the to element
 236  * represents the first element past the end of the region. This is used in
 237  * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
 238  *
 239  * Interval notation of the form [from, to) will be used to indicate that
 240  * the endpoint from is inclusive and to is exclusive.
 241  */
 242 struct file_region {
 243         struct list_head link;
 244         long from;
 245         long to;
 246 };
 247 
 248 /*
 249  * Add the huge page range represented by [f, t) to the reserve
 250  * map.  In the normal case, existing regions will be expanded
 251  * to accommodate the specified range.  Sufficient regions should
 252  * exist for expansion due to the previous call to region_chg
 253  * with the same range.  However, it is possible that region_del
 254  * could have been called after region_chg and modifed the map
 255  * in such a way that no region exists to be expanded.  In this
 256  * case, pull a region descriptor from the cache associated with
 257  * the map and use that for the new range.
 258  *
 259  * Return the number of new huge pages added to the map.  This
 260  * number is greater than or equal to zero.
 261  */
 262 static long region_add(struct resv_map *resv, long f, long t)
 263 {
 264         struct list_head *head = &resv->regions;
 265         struct file_region *rg, *nrg, *trg;
 266         long add = 0;
 267 
 268         spin_lock(&resv->lock);
 269         /* Locate the region we are either in or before. */
 270         list_for_each_entry(rg, head, link)
 271                 if (f <= rg->to)
 272                         break;
 273 
 274         /*
 275          * If no region exists which can be expanded to include the
 276          * specified range, the list must have been modified by an
 277          * interleving call to region_del().  Pull a region descriptor
 278          * from the cache and use it for this range.
 279          */
 280         if (&rg->link == head || t < rg->from) {
 281                 VM_BUG_ON(resv->region_cache_count <= 0);
 282 
 283                 resv->region_cache_count--;
 284                 nrg = list_first_entry(&resv->region_cache, struct file_region,
 285                                         link);
 286                 list_del(&nrg->link);
 287 
 288                 nrg->from = f;
 289                 nrg->to = t;
 290                 list_add(&nrg->link, rg->link.prev);
 291 
 292                 add += t - f;
 293                 goto out_locked;
 294         }
 295 
 296         /* Round our left edge to the current segment if it encloses us. */
 297         if (f > rg->from)
 298                 f = rg->from;
 299 
 300         /* Check for and consume any regions we now overlap with. */
 301         nrg = rg;
 302         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
 303                 if (&rg->link == head)
 304                         break;
 305                 if (rg->from > t)
 306                         break;
 307 
 308                 /* If this area reaches higher then extend our area to
 309                  * include it completely.  If this is not the first area
 310                  * which we intend to reuse, free it. */
 311                 if (rg->to > t)
 312                         t = rg->to;
 313                 if (rg != nrg) {
 314                         /* Decrement return value by the deleted range.
 315                          * Another range will span this area so that by
 316                          * end of routine add will be >= zero
 317                          */
 318                         add -= (rg->to - rg->from);
 319                         list_del(&rg->link);
 320                         kfree(rg);
 321                 }
 322         }
 323 
 324         add += (nrg->from - f);         /* Added to beginning of region */
 325         nrg->from = f;
 326         add += t - nrg->to;             /* Added to end of region */
 327         nrg->to = t;
 328 
 329 out_locked:
 330         resv->adds_in_progress--;
 331         spin_unlock(&resv->lock);
 332         VM_BUG_ON(add < 0);
 333         return add;
 334 }
 335 
 336 /*
 337  * Examine the existing reserve map and determine how many
 338  * huge pages in the specified range [f, t) are NOT currently
 339  * represented.  This routine is called before a subsequent
 340  * call to region_add that will actually modify the reserve
 341  * map to add the specified range [f, t).  region_chg does
 342  * not change the number of huge pages represented by the
 343  * map.  However, if the existing regions in the map can not
 344  * be expanded to represent the new range, a new file_region
 345  * structure is added to the map as a placeholder.  This is
 346  * so that the subsequent region_add call will have all the
 347  * regions it needs and will not fail.
 348  *
 349  * Upon entry, region_chg will also examine the cache of region descriptors
 350  * associated with the map.  If there are not enough descriptors cached, one
 351  * will be allocated for the in progress add operation.
 352  *
 353  * Returns the number of huge pages that need to be added to the existing
 354  * reservation map for the range [f, t).  This number is greater or equal to
 355  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
 356  * is needed and can not be allocated.
 357  */
 358 static long region_chg(struct resv_map *resv, long f, long t)
 359 {
 360         struct list_head *head = &resv->regions;
 361         struct file_region *rg, *nrg = NULL;
 362         long chg = 0;
 363 
 364 retry:
 365         spin_lock(&resv->lock);
 366 retry_locked:
 367         resv->adds_in_progress++;
 368 
 369         /*
 370          * Check for sufficient descriptors in the cache to accommodate
 371          * the number of in progress add operations.
 372          */
 373         if (resv->adds_in_progress > resv->region_cache_count) {
 374                 struct file_region *trg;
 375 
 376                 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
 377                 /* Must drop lock to allocate a new descriptor. */
 378                 resv->adds_in_progress--;
 379                 spin_unlock(&resv->lock);
 380 
 381                 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
 382                 if (!trg) {
 383                         kfree(nrg);
 384                         return -ENOMEM;
 385                 }
 386 
 387                 spin_lock(&resv->lock);
 388                 list_add(&trg->link, &resv->region_cache);
 389                 resv->region_cache_count++;
 390                 goto retry_locked;
 391         }
 392 
 393         /* Locate the region we are before or in. */
 394         list_for_each_entry(rg, head, link)
 395                 if (f <= rg->to)
 396                         break;
 397 
 398         /* If we are below the current region then a new region is required.
 399          * Subtle, allocate a new region at the position but make it zero
 400          * size such that we can guarantee to record the reservation. */
 401         if (&rg->link == head || t < rg->from) {
 402                 if (!nrg) {
 403                         resv->adds_in_progress--;
 404                         spin_unlock(&resv->lock);
 405                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 406                         if (!nrg)
 407                                 return -ENOMEM;
 408 
 409                         nrg->from = f;
 410                         nrg->to   = f;
 411                         INIT_LIST_HEAD(&nrg->link);
 412                         goto retry;
 413                 }
 414 
 415                 list_add(&nrg->link, rg->link.prev);
 416                 chg = t - f;
 417                 goto out_nrg;
 418         }
 419 
 420         /* Round our left edge to the current segment if it encloses us. */
 421         if (f > rg->from)
 422                 f = rg->from;
 423         chg = t - f;
 424 
 425         /* Check for and consume any regions we now overlap with. */
 426         list_for_each_entry(rg, rg->link.prev, link) {
 427                 if (&rg->link == head)
 428                         break;
 429                 if (rg->from > t)
 430                         goto out;
 431 
 432                 /* We overlap with this area, if it extends further than
 433                  * us then we must extend ourselves.  Account for its
 434                  * existing reservation. */
 435                 if (rg->to > t) {
 436                         chg += rg->to - t;
 437                         t = rg->to;
 438                 }
 439                 chg -= rg->to - rg->from;
 440         }
 441 
 442 out:
 443         spin_unlock(&resv->lock);
 444         /*  We already know we raced and no longer need the new region */
 445         kfree(nrg);
 446         return chg;
 447 out_nrg:
 448         spin_unlock(&resv->lock);
 449         return chg;
 450 }
 451 
 452 /*
 453  * Abort the in progress add operation.  The adds_in_progress field
 454  * of the resv_map keeps track of the operations in progress between
 455  * calls to region_chg and region_add.  Operations are sometimes
 456  * aborted after the call to region_chg.  In such cases, region_abort
 457  * is called to decrement the adds_in_progress counter.
 458  *
 459  * NOTE: The range arguments [f, t) are not needed or used in this
 460  * routine.  They are kept to make reading the calling code easier as
 461  * arguments will match the associated region_chg call.
 462  */
 463 static void region_abort(struct resv_map *resv, long f, long t)
 464 {
 465         spin_lock(&resv->lock);
 466         VM_BUG_ON(!resv->region_cache_count);
 467         resv->adds_in_progress--;
 468         spin_unlock(&resv->lock);
 469 }
 470 
 471 /*
 472  * Delete the specified range [f, t) from the reserve map.  If the
 473  * t parameter is LONG_MAX, this indicates that ALL regions after f
 474  * should be deleted.  Locate the regions which intersect [f, t)
 475  * and either trim, delete or split the existing regions.
 476  *
 477  * Returns the number of huge pages deleted from the reserve map.
 478  * In the normal case, the return value is zero or more.  In the
 479  * case where a region must be split, a new region descriptor must
 480  * be allocated.  If the allocation fails, -ENOMEM will be returned.
 481  * NOTE: If the parameter t == LONG_MAX, then we will never split
 482  * a region and possibly return -ENOMEM.  Callers specifying
 483  * t == LONG_MAX do not need to check for -ENOMEM error.
 484  */
 485 static long region_del(struct resv_map *resv, long f, long t)
 486 {
 487         struct list_head *head = &resv->regions;
 488         struct file_region *rg, *trg;
 489         struct file_region *nrg = NULL;
 490         long del = 0;
 491 
 492 retry:
 493         spin_lock(&resv->lock);
 494         list_for_each_entry_safe(rg, trg, head, link) {
 495                 /*
 496                  * Skip regions before the range to be deleted.  file_region
 497                  * ranges are normally of the form [from, to).  However, there
 498                  * may be a "placeholder" entry in the map which is of the form
 499                  * (from, to) with from == to.  Check for placeholder entries
 500                  * at the beginning of the range to be deleted.
 501                  */
 502                 if (rg->to <= f && (rg->to != rg->from || rg->to != f))
 503                         continue;
 504 
 505                 if (rg->from >= t)
 506                         break;
 507 
 508                 if (f > rg->from && t < rg->to) { /* Must split region */
 509                         /*
 510                          * Check for an entry in the cache before dropping
 511                          * lock and attempting allocation.
 512                          */
 513                         if (!nrg &&
 514                             resv->region_cache_count > resv->adds_in_progress) {
 515                                 nrg = list_first_entry(&resv->region_cache,
 516                                                         struct file_region,
 517                                                         link);
 518                                 list_del(&nrg->link);
 519                                 resv->region_cache_count--;
 520                         }
 521 
 522                         if (!nrg) {
 523                                 spin_unlock(&resv->lock);
 524                                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 525                                 if (!nrg)
 526                                         return -ENOMEM;
 527                                 goto retry;
 528                         }
 529 
 530                         del += t - f;
 531 
 532                         /* New entry for end of split region */
 533                         nrg->from = t;
 534                         nrg->to = rg->to;
 535                         INIT_LIST_HEAD(&nrg->link);
 536 
 537                         /* Original entry is trimmed */
 538                         rg->to = f;
 539 
 540                         list_add(&nrg->link, &rg->link);
 541                         nrg = NULL;
 542                         break;
 543                 }
 544 
 545                 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
 546                         del += rg->to - rg->from;
 547                         list_del(&rg->link);
 548                         kfree(rg);
 549                         continue;
 550                 }
 551 
 552                 if (f <= rg->from) {    /* Trim beginning of region */
 553                         del += t - rg->from;
 554                         rg->from = t;
 555                 } else {                /* Trim end of region */
 556                         del += rg->to - f;
 557                         rg->to = f;
 558                 }
 559         }
 560 
 561         spin_unlock(&resv->lock);
 562         kfree(nrg);
 563         return del;
 564 }
 565 
 566 /*
 567  * A rare out of memory error was encountered which prevented removal of
 568  * the reserve map region for a page.  The huge page itself was free'ed
 569  * and removed from the page cache.  This routine will adjust the subpool
 570  * usage count, and the global reserve count if needed.  By incrementing
 571  * these counts, the reserve map entry which could not be deleted will
 572  * appear as a "reserved" entry instead of simply dangling with incorrect
 573  * counts.
 574  */
 575 void hugetlb_fix_reserve_counts(struct inode *inode)
 576 {
 577         struct hugepage_subpool *spool = subpool_inode(inode);
 578         long rsv_adjust;
 579 
 580         rsv_adjust = hugepage_subpool_get_pages(spool, 1);
 581         if (rsv_adjust) {
 582                 struct hstate *h = hstate_inode(inode);
 583 
 584                 hugetlb_acct_memory(h, 1);
 585         }
 586 }
 587 
 588 /*
 589  * Count and return the number of huge pages in the reserve map
 590  * that intersect with the range [f, t).
 591  */
 592 static long region_count(struct resv_map *resv, long f, long t)
 593 {
 594         struct list_head *head = &resv->regions;
 595         struct file_region *rg;
 596         long chg = 0;
 597 
 598         spin_lock(&resv->lock);
 599         /* Locate each segment we overlap with, and count that overlap. */
 600         list_for_each_entry(rg, head, link) {
 601                 long seg_from;
 602                 long seg_to;
 603 
 604                 if (rg->to <= f)
 605                         continue;
 606                 if (rg->from >= t)
 607                         break;
 608 
 609                 seg_from = max(rg->from, f);
 610                 seg_to = min(rg->to, t);
 611 
 612                 chg += seg_to - seg_from;
 613         }
 614         spin_unlock(&resv->lock);
 615 
 616         return chg;
 617 }
 618 
 619 /*
 620  * Convert the address within this vma to the page offset within
 621  * the mapping, in pagecache page units; huge pages here.
 622  */
 623 static pgoff_t vma_hugecache_offset(struct hstate *h,
 624                         struct vm_area_struct *vma, unsigned long address)
 625 {
 626         return ((address - vma->vm_start) >> huge_page_shift(h)) +
 627                         (vma->vm_pgoff >> huge_page_order(h));
 628 }
 629 
 630 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
 631                                      unsigned long address)
 632 {
 633         return vma_hugecache_offset(hstate_vma(vma), vma, address);
 634 }
 635 EXPORT_SYMBOL_GPL(linear_hugepage_index);
 636 
 637 /*
 638  * Return the size of the pages allocated when backing a VMA. In the majority
 639  * cases this will be same size as used by the page table entries.
 640  */
 641 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
 642 {
 643         if (vma->vm_ops && vma->vm_ops->pagesize)
 644                 return vma->vm_ops->pagesize(vma);
 645         return PAGE_SIZE;
 646 }
 647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
 648 
 649 /*
 650  * Return the page size being used by the MMU to back a VMA. In the majority
 651  * of cases, the page size used by the kernel matches the MMU size. On
 652  * architectures where it differs, an architecture-specific 'strong'
 653  * version of this symbol is required.
 654  */
 655 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 656 {
 657         return vma_kernel_pagesize(vma);
 658 }
 659 
 660 /*
 661  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
 662  * bits of the reservation map pointer, which are always clear due to
 663  * alignment.
 664  */
 665 #define HPAGE_RESV_OWNER    (1UL << 0)
 666 #define HPAGE_RESV_UNMAPPED (1UL << 1)
 667 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
 668 
 669 /*
 670  * These helpers are used to track how many pages are reserved for
 671  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
 672  * is guaranteed to have their future faults succeed.
 673  *
 674  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
 675  * the reserve counters are updated with the hugetlb_lock held. It is safe
 676  * to reset the VMA at fork() time as it is not in use yet and there is no
 677  * chance of the global counters getting corrupted as a result of the values.
 678  *
 679  * The private mapping reservation is represented in a subtly different
 680  * manner to a shared mapping.  A shared mapping has a region map associated
 681  * with the underlying file, this region map represents the backing file
 682  * pages which have ever had a reservation assigned which this persists even
 683  * after the page is instantiated.  A private mapping has a region map
 684  * associated with the original mmap which is attached to all VMAs which
 685  * reference it, this region map represents those offsets which have consumed
 686  * reservation ie. where pages have been instantiated.
 687  */
 688 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
 689 {
 690         return (unsigned long)vma->vm_private_data;
 691 }
 692 
 693 static void set_vma_private_data(struct vm_area_struct *vma,
 694                                                         unsigned long value)
 695 {
 696         vma->vm_private_data = (void *)value;
 697 }
 698 
 699 struct resv_map *resv_map_alloc(void)
 700 {
 701         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
 702         struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
 703 
 704         if (!resv_map || !rg) {
 705                 kfree(resv_map);
 706                 kfree(rg);
 707                 return NULL;
 708         }
 709 
 710         kref_init(&resv_map->refs);
 711         spin_lock_init(&resv_map->lock);
 712         INIT_LIST_HEAD(&resv_map->regions);
 713 
 714         resv_map->adds_in_progress = 0;
 715 
 716         INIT_LIST_HEAD(&resv_map->region_cache);
 717         list_add(&rg->link, &resv_map->region_cache);
 718         resv_map->region_cache_count = 1;
 719 
 720         return resv_map;
 721 }
 722 
 723 void resv_map_release(struct kref *ref)
 724 {
 725         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
 726         struct list_head *head = &resv_map->region_cache;
 727         struct file_region *rg, *trg;
 728 
 729         /* Clear out any active regions before we release the map. */
 730         region_del(resv_map, 0, LONG_MAX);
 731 
 732         /* ... and any entries left in the cache */
 733         list_for_each_entry_safe(rg, trg, head, link) {
 734                 list_del(&rg->link);
 735                 kfree(rg);
 736         }
 737 
 738         VM_BUG_ON(resv_map->adds_in_progress);
 739 
 740         kfree(resv_map);
 741 }
 742 
 743 static inline struct resv_map *inode_resv_map(struct inode *inode)
 744 {
 745         /*
 746          * At inode evict time, i_mapping may not point to the original
 747          * address space within the inode.  This original address space
 748          * contains the pointer to the resv_map.  So, always use the
 749          * address space embedded within the inode.
 750          * The VERY common case is inode->mapping == &inode->i_data but,
 751          * this may not be true for device special inodes.
 752          */
 753         return (struct resv_map *)(&inode->i_data)->private_data;
 754 }
 755 
 756 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 757 {
 758         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 759         if (vma->vm_flags & VM_MAYSHARE) {
 760                 struct address_space *mapping = vma->vm_file->f_mapping;
 761                 struct inode *inode = mapping->host;
 762 
 763                 return inode_resv_map(inode);
 764 
 765         } else {
 766                 return (struct resv_map *)(get_vma_private_data(vma) &
 767                                                         ~HPAGE_RESV_MASK);
 768         }
 769 }
 770 
 771 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 772 {
 773         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 774         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
 775 
 776         set_vma_private_data(vma, (get_vma_private_data(vma) &
 777                                 HPAGE_RESV_MASK) | (unsigned long)map);
 778 }
 779 
 780 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 781 {
 782         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 783         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
 784 
 785         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
 786 }
 787 
 788 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
 789 {
 790         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 791 
 792         return (get_vma_private_data(vma) & flag) != 0;
 793 }
 794 
 795 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
 796 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
 797 {
 798         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 799         if (!(vma->vm_flags & VM_MAYSHARE))
 800                 vma->vm_private_data = (void *)0;
 801 }
 802 
 803 /* Returns true if the VMA has associated reserve pages */
 804 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
 805 {
 806         if (vma->vm_flags & VM_NORESERVE) {
 807                 /*
 808                  * This address is already reserved by other process(chg == 0),
 809                  * so, we should decrement reserved count. Without decrementing,
 810                  * reserve count remains after releasing inode, because this
 811                  * allocated page will go into page cache and is regarded as
 812                  * coming from reserved pool in releasing step.  Currently, we
 813                  * don't have any other solution to deal with this situation
 814                  * properly, so add work-around here.
 815                  */
 816                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
 817                         return true;
 818                 else
 819                         return false;
 820         }
 821 
 822         /* Shared mappings always use reserves */
 823         if (vma->vm_flags & VM_MAYSHARE) {
 824                 /*
 825                  * We know VM_NORESERVE is not set.  Therefore, there SHOULD
 826                  * be a region map for all pages.  The only situation where
 827                  * there is no region map is if a hole was punched via
 828                  * fallocate.  In this case, there really are no reverves to
 829                  * use.  This situation is indicated if chg != 0.
 830                  */
 831                 if (chg)
 832                         return false;
 833                 else
 834                         return true;
 835         }
 836 
 837         /*
 838          * Only the process that called mmap() has reserves for
 839          * private mappings.
 840          */
 841         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
 842                 /*
 843                  * Like the shared case above, a hole punch or truncate
 844                  * could have been performed on the private mapping.
 845                  * Examine the value of chg to determine if reserves
 846                  * actually exist or were previously consumed.
 847                  * Very Subtle - The value of chg comes from a previous
 848                  * call to vma_needs_reserves().  The reserve map for
 849                  * private mappings has different (opposite) semantics
 850                  * than that of shared mappings.  vma_needs_reserves()
 851                  * has already taken this difference in semantics into
 852                  * account.  Therefore, the meaning of chg is the same
 853                  * as in the shared case above.  Code could easily be
 854                  * combined, but keeping it separate draws attention to
 855                  * subtle differences.
 856                  */
 857                 if (chg)
 858                         return false;
 859                 else
 860                         return true;
 861         }
 862 
 863         return false;
 864 }
 865 
 866 static void enqueue_huge_page(struct hstate *h, struct page *page)
 867 {
 868         int nid = page_to_nid(page);
 869         list_move(&page->lru, &h->hugepage_freelists[nid]);
 870         h->free_huge_pages++;
 871         h->free_huge_pages_node[nid]++;
 872 }
 873 
 874 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
 875 {
 876         struct page *page;
 877 
 878         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
 879                 if (!PageHWPoison(page))
 880                         break;
 881         /*
 882          * if 'non-isolated free hugepage' not found on the list,
 883          * the allocation fails.
 884          */
 885         if (&h->hugepage_freelists[nid] == &page->lru)
 886                 return NULL;
 887         list_move(&page->lru, &h->hugepage_activelist);
 888         set_page_refcounted(page);
 889         h->free_huge_pages--;
 890         h->free_huge_pages_node[nid]--;
 891         return page;
 892 }
 893 
 894 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
 895                 nodemask_t *nmask)
 896 {
 897         unsigned int cpuset_mems_cookie;
 898         struct zonelist *zonelist;
 899         struct zone *zone;
 900         struct zoneref *z;
 901         int node = NUMA_NO_NODE;
 902 
 903         zonelist = node_zonelist(nid, gfp_mask);
 904 
 905 retry_cpuset:
 906         cpuset_mems_cookie = read_mems_allowed_begin();
 907         for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
 908                 struct page *page;
 909 
 910                 if (!cpuset_zone_allowed(zone, gfp_mask))
 911                         continue;
 912                 /*
 913                  * no need to ask again on the same node. Pool is node rather than
 914                  * zone aware
 915                  */
 916                 if (zone_to_nid(zone) == node)
 917                         continue;
 918                 node = zone_to_nid(zone);
 919 
 920                 page = dequeue_huge_page_node_exact(h, node);
 921                 if (page)
 922                         return page;
 923         }
 924         if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
 925                 goto retry_cpuset;
 926 
 927         return NULL;
 928 }
 929 
 930 /* Movability of hugepages depends on migration support. */
 931 static inline gfp_t htlb_alloc_mask(struct hstate *h)
 932 {
 933         if (hugepage_movable_supported(h))
 934                 return GFP_HIGHUSER_MOVABLE;
 935         else
 936                 return GFP_HIGHUSER;
 937 }
 938 
 939 static struct page *dequeue_huge_page_vma(struct hstate *h,
 940                                 struct vm_area_struct *vma,
 941                                 unsigned long address, int avoid_reserve,
 942                                 long chg)
 943 {
 944         struct page *page;
 945         struct mempolicy *mpol;
 946         gfp_t gfp_mask;
 947         nodemask_t *nodemask;
 948         int nid;
 949 
 950         /*
 951          * A child process with MAP_PRIVATE mappings created by their parent
 952          * have no page reserves. This check ensures that reservations are
 953          * not "stolen". The child may still get SIGKILLed
 954          */
 955         if (!vma_has_reserves(vma, chg) &&
 956                         h->free_huge_pages - h->resv_huge_pages == 0)
 957                 goto err;
 958 
 959         /* If reserves cannot be used, ensure enough pages are in the pool */
 960         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
 961                 goto err;
 962 
 963         gfp_mask = htlb_alloc_mask(h);
 964         nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
 965         page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
 966         if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
 967                 SetPagePrivate(page);
 968                 h->resv_huge_pages--;
 969         }
 970 
 971         mpol_cond_put(mpol);
 972         return page;
 973 
 974 err:
 975         return NULL;
 976 }
 977 
 978 /*
 979  * common helper functions for hstate_next_node_to_{alloc|free}.
 980  * We may have allocated or freed a huge page based on a different
 981  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
 982  * be outside of *nodes_allowed.  Ensure that we use an allowed
 983  * node for alloc or free.
 984  */
 985 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
 986 {
 987         nid = next_node_in(nid, *nodes_allowed);
 988         VM_BUG_ON(nid >= MAX_NUMNODES);
 989 
 990         return nid;
 991 }
 992 
 993 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
 994 {
 995         if (!node_isset(nid, *nodes_allowed))
 996                 nid = next_node_allowed(nid, nodes_allowed);
 997         return nid;
 998 }
 999 
1000 /*
1001  * returns the previously saved node ["this node"] from which to
1002  * allocate a persistent huge page for the pool and advance the
1003  * next node from which to allocate, handling wrap at end of node
1004  * mask.
1005  */
1006 static int hstate_next_node_to_alloc(struct hstate *h,
1007                                         nodemask_t *nodes_allowed)
1008 {
1009         int nid;
1010 
1011         VM_BUG_ON(!nodes_allowed);
1012 
1013         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1014         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1015 
1016         return nid;
1017 }
1018 
1019 /*
1020  * helper for free_pool_huge_page() - return the previously saved
1021  * node ["this node"] from which to free a huge page.  Advance the
1022  * next node id whether or not we find a free huge page to free so
1023  * that the next attempt to free addresses the next node.
1024  */
1025 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1026 {
1027         int nid;
1028 
1029         VM_BUG_ON(!nodes_allowed);
1030 
1031         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1032         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1033 
1034         return nid;
1035 }
1036 
1037 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
1038         for (nr_nodes = nodes_weight(*mask);                            \
1039                 nr_nodes > 0 &&                                         \
1040                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
1041                 nr_nodes--)
1042 
1043 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
1044         for (nr_nodes = nodes_weight(*mask);                            \
1045                 nr_nodes > 0 &&                                         \
1046                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
1047                 nr_nodes--)
1048 
1049 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1050 static void destroy_compound_gigantic_page(struct page *page,
1051                                         unsigned int order)
1052 {
1053         int i;
1054         int nr_pages = 1 << order;
1055         struct page *p = page + 1;
1056 
1057         atomic_set(compound_mapcount_ptr(page), 0);
1058         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1059                 clear_compound_head(p);
1060                 set_page_refcounted(p);
1061         }
1062 
1063         set_compound_order(page, 0);
1064         __ClearPageHead(page);
1065 }
1066 
1067 static void free_gigantic_page(struct page *page, unsigned int order)
1068 {
1069         free_contig_range(page_to_pfn(page), 1 << order);
1070 }
1071 
1072 #ifdef CONFIG_CONTIG_ALLOC
1073 static int __alloc_gigantic_page(unsigned long start_pfn,
1074                                 unsigned long nr_pages, gfp_t gfp_mask)
1075 {
1076         unsigned long end_pfn = start_pfn + nr_pages;
1077         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
1078                                   gfp_mask);
1079 }
1080 
1081 static bool pfn_range_valid_gigantic(struct zone *z,
1082                         unsigned long start_pfn, unsigned long nr_pages)
1083 {
1084         unsigned long i, end_pfn = start_pfn + nr_pages;
1085         struct page *page;
1086 
1087         for (i = start_pfn; i < end_pfn; i++) {
1088                 page = pfn_to_online_page(i);
1089                 if (!page)
1090                         return false;
1091 
1092                 if (page_zone(page) != z)
1093                         return false;
1094 
1095                 if (PageReserved(page))
1096                         return false;
1097 
1098                 if (page_count(page) > 0)
1099                         return false;
1100 
1101                 if (PageHuge(page))
1102                         return false;
1103         }
1104 
1105         return true;
1106 }
1107 
1108 static bool zone_spans_last_pfn(const struct zone *zone,
1109                         unsigned long start_pfn, unsigned long nr_pages)
1110 {
1111         unsigned long last_pfn = start_pfn + nr_pages - 1;
1112         return zone_spans_pfn(zone, last_pfn);
1113 }
1114 
1115 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1116                 int nid, nodemask_t *nodemask)
1117 {
1118         unsigned int order = huge_page_order(h);
1119         unsigned long nr_pages = 1 << order;
1120         unsigned long ret, pfn, flags;
1121         struct zonelist *zonelist;
1122         struct zone *zone;
1123         struct zoneref *z;
1124 
1125         zonelist = node_zonelist(nid, gfp_mask);
1126         for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nodemask) {
1127                 spin_lock_irqsave(&zone->lock, flags);
1128 
1129                 pfn = ALIGN(zone->zone_start_pfn, nr_pages);
1130                 while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
1131                         if (pfn_range_valid_gigantic(zone, pfn, nr_pages)) {
1132                                 /*
1133                                  * We release the zone lock here because
1134                                  * alloc_contig_range() will also lock the zone
1135                                  * at some point. If there's an allocation
1136                                  * spinning on this lock, it may win the race
1137                                  * and cause alloc_contig_range() to fail...
1138                                  */
1139                                 spin_unlock_irqrestore(&zone->lock, flags);
1140                                 ret = __alloc_gigantic_page(pfn, nr_pages, gfp_mask);
1141                                 if (!ret)
1142                                         return pfn_to_page(pfn);
1143                                 spin_lock_irqsave(&zone->lock, flags);
1144                         }
1145                         pfn += nr_pages;
1146                 }
1147 
1148                 spin_unlock_irqrestore(&zone->lock, flags);
1149         }
1150 
1151         return NULL;
1152 }
1153 
1154 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1155 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1156 #else /* !CONFIG_CONTIG_ALLOC */
1157 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1158                                         int nid, nodemask_t *nodemask)
1159 {
1160         return NULL;
1161 }
1162 #endif /* CONFIG_CONTIG_ALLOC */
1163 
1164 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1165 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1166                                         int nid, nodemask_t *nodemask)
1167 {
1168         return NULL;
1169 }
1170 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1171 static inline void destroy_compound_gigantic_page(struct page *page,
1172                                                 unsigned int order) { }
1173 #endif
1174 
1175 static void update_and_free_page(struct hstate *h, struct page *page)
1176 {
1177         int i;
1178 
1179         if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1180                 return;
1181 
1182         h->nr_huge_pages--;
1183         h->nr_huge_pages_node[page_to_nid(page)]--;
1184         for (i = 0; i < pages_per_huge_page(h); i++) {
1185                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1186                                 1 << PG_referenced | 1 << PG_dirty |
1187                                 1 << PG_active | 1 << PG_private |
1188                                 1 << PG_writeback);
1189         }
1190         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1191         set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1192         set_page_refcounted(page);
1193         if (hstate_is_gigantic(h)) {
1194                 destroy_compound_gigantic_page(page, huge_page_order(h));
1195                 free_gigantic_page(page, huge_page_order(h));
1196         } else {
1197                 __free_pages(page, huge_page_order(h));
1198         }
1199 }
1200 
1201 struct hstate *size_to_hstate(unsigned long size)
1202 {
1203         struct hstate *h;
1204 
1205         for_each_hstate(h) {
1206                 if (huge_page_size(h) == size)
1207                         return h;
1208         }
1209         return NULL;
1210 }
1211 
1212 /*
1213  * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1214  * to hstate->hugepage_activelist.)
1215  *
1216  * This function can be called for tail pages, but never returns true for them.
1217  */
1218 bool page_huge_active(struct page *page)
1219 {
1220         VM_BUG_ON_PAGE(!PageHuge(page), page);
1221         return PageHead(page) && PagePrivate(&page[1]);
1222 }
1223 
1224 /* never called for tail page */
1225 static void set_page_huge_active(struct page *page)
1226 {
1227         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1228         SetPagePrivate(&page[1]);
1229 }
1230 
1231 static void clear_page_huge_active(struct page *page)
1232 {
1233         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1234         ClearPagePrivate(&page[1]);
1235 }
1236 
1237 /*
1238  * Internal hugetlb specific page flag. Do not use outside of the hugetlb
1239  * code
1240  */
1241 static inline bool PageHugeTemporary(struct page *page)
1242 {
1243         if (!PageHuge(page))
1244                 return false;
1245 
1246         return (unsigned long)page[2].mapping == -1U;
1247 }
1248 
1249 static inline void SetPageHugeTemporary(struct page *page)
1250 {
1251         page[2].mapping = (void *)-1U;
1252 }
1253 
1254 static inline void ClearPageHugeTemporary(struct page *page)
1255 {
1256         page[2].mapping = NULL;
1257 }
1258 
1259 static void __free_huge_page(struct page *page)
1260 {
1261         /*
1262          * Can't pass hstate in here because it is called from the
1263          * compound page destructor.
1264          */
1265         struct hstate *h = page_hstate(page);
1266         int nid = page_to_nid(page);
1267         struct hugepage_subpool *spool =
1268                 (struct hugepage_subpool *)page_private(page);
1269         bool restore_reserve;
1270 
1271         VM_BUG_ON_PAGE(page_count(page), page);
1272         VM_BUG_ON_PAGE(page_mapcount(page), page);
1273 
1274         set_page_private(page, 0);
1275         page->mapping = NULL;
1276         restore_reserve = PagePrivate(page);
1277         ClearPagePrivate(page);
1278 
1279         /*
1280          * If PagePrivate() was set on page, page allocation consumed a
1281          * reservation.  If the page was associated with a subpool, there
1282          * would have been a page reserved in the subpool before allocation
1283          * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1284          * reservtion, do not call hugepage_subpool_put_pages() as this will
1285          * remove the reserved page from the subpool.
1286          */
1287         if (!restore_reserve) {
1288                 /*
1289                  * A return code of zero implies that the subpool will be
1290                  * under its minimum size if the reservation is not restored
1291                  * after page is free.  Therefore, force restore_reserve
1292                  * operation.
1293                  */
1294                 if (hugepage_subpool_put_pages(spool, 1) == 0)
1295                         restore_reserve = true;
1296         }
1297 
1298         spin_lock(&hugetlb_lock);
1299         clear_page_huge_active(page);
1300         hugetlb_cgroup_uncharge_page(hstate_index(h),
1301                                      pages_per_huge_page(h), page);
1302         if (restore_reserve)
1303                 h->resv_huge_pages++;
1304 
1305         if (PageHugeTemporary(page)) {
1306                 list_del(&page->lru);
1307                 ClearPageHugeTemporary(page);
1308                 update_and_free_page(h, page);
1309         } else if (h->surplus_huge_pages_node[nid]) {
1310                 /* remove the page from active list */
1311                 list_del(&page->lru);
1312                 update_and_free_page(h, page);
1313                 h->surplus_huge_pages--;
1314                 h->surplus_huge_pages_node[nid]--;
1315         } else {
1316                 arch_clear_hugepage_flags(page);
1317                 enqueue_huge_page(h, page);
1318         }
1319         spin_unlock(&hugetlb_lock);
1320 }
1321 
1322 /*
1323  * As free_huge_page() can be called from a non-task context, we have
1324  * to defer the actual freeing in a workqueue to prevent potential
1325  * hugetlb_lock deadlock.
1326  *
1327  * free_hpage_workfn() locklessly retrieves the linked list of pages to
1328  * be freed and frees them one-by-one. As the page->mapping pointer is
1329  * going to be cleared in __free_huge_page() anyway, it is reused as the
1330  * llist_node structure of a lockless linked list of huge pages to be freed.
1331  */
1332 static LLIST_HEAD(hpage_freelist);
1333 
1334 static void free_hpage_workfn(struct work_struct *work)
1335 {
1336         struct llist_node *node;
1337         struct page *page;
1338 
1339         node = llist_del_all(&hpage_freelist);
1340 
1341         while (node) {
1342                 page = container_of((struct address_space **)node,
1343                                      struct page, mapping);
1344                 node = node->next;
1345                 __free_huge_page(page);
1346         }
1347 }
1348 static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
1349 
1350 void free_huge_page(struct page *page)
1351 {
1352         /*
1353          * Defer freeing if in non-task context to avoid hugetlb_lock deadlock.
1354          */
1355         if (!in_task()) {
1356                 /*
1357                  * Only call schedule_work() if hpage_freelist is previously
1358                  * empty. Otherwise, schedule_work() had been called but the
1359                  * workfn hasn't retrieved the list yet.
1360                  */
1361                 if (llist_add((struct llist_node *)&page->mapping,
1362                               &hpage_freelist))
1363                         schedule_work(&free_hpage_work);
1364                 return;
1365         }
1366 
1367         __free_huge_page(page);
1368 }
1369 
1370 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1371 {
1372         INIT_LIST_HEAD(&page->lru);
1373         set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1374         spin_lock(&hugetlb_lock);
1375         set_hugetlb_cgroup(page, NULL);
1376         h->nr_huge_pages++;
1377         h->nr_huge_pages_node[nid]++;
1378         spin_unlock(&hugetlb_lock);
1379 }
1380 
1381 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1382 {
1383         int i;
1384         int nr_pages = 1 << order;
1385         struct page *p = page + 1;
1386 
1387         /* we rely on prep_new_huge_page to set the destructor */
1388         set_compound_order(page, order);
1389         __ClearPageReserved(page);
1390         __SetPageHead(page);
1391         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1392                 /*
1393                  * For gigantic hugepages allocated through bootmem at
1394                  * boot, it's safer to be consistent with the not-gigantic
1395                  * hugepages and clear the PG_reserved bit from all tail pages
1396                  * too.  Otherwse drivers using get_user_pages() to access tail
1397                  * pages may get the reference counting wrong if they see
1398                  * PG_reserved set on a tail page (despite the head page not
1399                  * having PG_reserved set).  Enforcing this consistency between
1400                  * head and tail pages allows drivers to optimize away a check
1401                  * on the head page when they need know if put_page() is needed
1402                  * after get_user_pages().
1403                  */
1404                 __ClearPageReserved(p);
1405                 set_page_count(p, 0);
1406                 set_compound_head(p, page);
1407         }
1408         atomic_set(compound_mapcount_ptr(page), -1);
1409 }
1410 
1411 /*
1412  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1413  * transparent huge pages.  See the PageTransHuge() documentation for more
1414  * details.
1415  */
1416 int PageHuge(struct page *page)
1417 {
1418         if (!PageCompound(page))
1419                 return 0;
1420 
1421         page = compound_head(page);
1422         return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1423 }
1424 EXPORT_SYMBOL_GPL(PageHuge);
1425 
1426 /*
1427  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1428  * normal or transparent huge pages.
1429  */
1430 int PageHeadHuge(struct page *page_head)
1431 {
1432         if (!PageHead(page_head))
1433                 return 0;
1434 
1435         return get_compound_page_dtor(page_head) == free_huge_page;
1436 }
1437 
1438 pgoff_t __basepage_index(struct page *page)
1439 {
1440         struct page *page_head = compound_head(page);
1441         pgoff_t index = page_index(page_head);
1442         unsigned long compound_idx;
1443 
1444         if (!PageHuge(page_head))
1445                 return page_index(page);
1446 
1447         if (compound_order(page_head) >= MAX_ORDER)
1448                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1449         else
1450                 compound_idx = page - page_head;
1451 
1452         return (index << compound_order(page_head)) + compound_idx;
1453 }
1454 
1455 static struct page *alloc_buddy_huge_page(struct hstate *h,
1456                 gfp_t gfp_mask, int nid, nodemask_t *nmask,
1457                 nodemask_t *node_alloc_noretry)
1458 {
1459         int order = huge_page_order(h);
1460         struct page *page;
1461         bool alloc_try_hard = true;
1462 
1463         /*
1464          * By default we always try hard to allocate the page with
1465          * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
1466          * a loop (to adjust global huge page counts) and previous allocation
1467          * failed, do not continue to try hard on the same node.  Use the
1468          * node_alloc_noretry bitmap to manage this state information.
1469          */
1470         if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
1471                 alloc_try_hard = false;
1472         gfp_mask |= __GFP_COMP|__GFP_NOWARN;
1473         if (alloc_try_hard)
1474                 gfp_mask |= __GFP_RETRY_MAYFAIL;
1475         if (nid == NUMA_NO_NODE)
1476                 nid = numa_mem_id();
1477         page = __alloc_pages_nodemask(gfp_mask, order, nid, nmask);
1478         if (page)
1479                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1480         else
1481                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1482 
1483         /*
1484          * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
1485          * indicates an overall state change.  Clear bit so that we resume
1486          * normal 'try hard' allocations.
1487          */
1488         if (node_alloc_noretry && page && !alloc_try_hard)
1489                 node_clear(nid, *node_alloc_noretry);
1490 
1491         /*
1492          * If we tried hard to get a page but failed, set bit so that
1493          * subsequent attempts will not try as hard until there is an
1494          * overall state change.
1495          */
1496         if (node_alloc_noretry && !page && alloc_try_hard)
1497                 node_set(nid, *node_alloc_noretry);
1498 
1499         return page;
1500 }
1501 
1502 /*
1503  * Common helper to allocate a fresh hugetlb page. All specific allocators
1504  * should use this function to get new hugetlb pages
1505  */
1506 static struct page *alloc_fresh_huge_page(struct hstate *h,
1507                 gfp_t gfp_mask, int nid, nodemask_t *nmask,
1508                 nodemask_t *node_alloc_noretry)
1509 {
1510         struct page *page;
1511 
1512         if (hstate_is_gigantic(h))
1513                 page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
1514         else
1515                 page = alloc_buddy_huge_page(h, gfp_mask,
1516                                 nid, nmask, node_alloc_noretry);
1517         if (!page)
1518                 return NULL;
1519 
1520         if (hstate_is_gigantic(h))
1521                 prep_compound_gigantic_page(page, huge_page_order(h));
1522         prep_new_huge_page(h, page, page_to_nid(page));
1523 
1524         return page;
1525 }
1526 
1527 /*
1528  * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
1529  * manner.
1530  */
1531 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1532                                 nodemask_t *node_alloc_noretry)
1533 {
1534         struct page *page;
1535         int nr_nodes, node;
1536         gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
1537 
1538         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1539                 page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
1540                                                 node_alloc_noretry);
1541                 if (page)
1542                         break;
1543         }
1544 
1545         if (!page)
1546                 return 0;
1547 
1548         put_page(page); /* free it into the hugepage allocator */
1549 
1550         return 1;
1551 }
1552 
1553 /*
1554  * Free huge page from pool from next node to free.
1555  * Attempt to keep persistent huge pages more or less
1556  * balanced over allowed nodes.
1557  * Called with hugetlb_lock locked.
1558  */
1559 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1560                                                          bool acct_surplus)
1561 {
1562         int nr_nodes, node;
1563         int ret = 0;
1564 
1565         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1566                 /*
1567                  * If we're returning unused surplus pages, only examine
1568                  * nodes with surplus pages.
1569                  */
1570                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1571                     !list_empty(&h->hugepage_freelists[node])) {
1572                         struct page *page =
1573                                 list_entry(h->hugepage_freelists[node].next,
1574                                           struct page, lru);
1575                         list_del(&page->lru);
1576                         h->free_huge_pages--;
1577                         h->free_huge_pages_node[node]--;
1578                         if (acct_surplus) {
1579                                 h->surplus_huge_pages--;
1580                                 h->surplus_huge_pages_node[node]--;
1581                         }
1582                         update_and_free_page(h, page);
1583                         ret = 1;
1584                         break;
1585                 }
1586         }
1587 
1588         return ret;
1589 }
1590 
1591 /*
1592  * Dissolve a given free hugepage into free buddy pages. This function does
1593  * nothing for in-use hugepages and non-hugepages.
1594  * This function returns values like below:
1595  *
1596  *  -EBUSY: failed to dissolved free hugepages or the hugepage is in-use
1597  *          (allocated or reserved.)
1598  *       0: successfully dissolved free hugepages or the page is not a
1599  *          hugepage (considered as already dissolved)
1600  */
1601 int dissolve_free_huge_page(struct page *page)
1602 {
1603         int rc = -EBUSY;
1604 
1605         /* Not to disrupt normal path by vainly holding hugetlb_lock */
1606         if (!PageHuge(page))
1607                 return 0;
1608 
1609         spin_lock(&hugetlb_lock);
1610         if (!PageHuge(page)) {
1611                 rc = 0;
1612                 goto out;
1613         }
1614 
1615         if (!page_count(page)) {
1616                 struct page *head = compound_head(page);
1617                 struct hstate *h = page_hstate(head);
1618                 int nid = page_to_nid(head);
1619                 if (h->free_huge_pages - h->resv_huge_pages == 0)
1620                         goto out;
1621                 /*
1622                  * Move PageHWPoison flag from head page to the raw error page,
1623                  * which makes any subpages rather than the error page reusable.
1624                  */
1625                 if (PageHWPoison(head) && page != head) {
1626                         SetPageHWPoison(page);
1627                         ClearPageHWPoison(head);
1628                 }
1629                 list_del(&head->lru);
1630                 h->free_huge_pages--;
1631                 h->free_huge_pages_node[nid]--;
1632                 h->max_huge_pages--;
1633                 update_and_free_page(h, head);
1634                 rc = 0;
1635         }
1636 out:
1637         spin_unlock(&hugetlb_lock);
1638         return rc;
1639 }
1640 
1641 /*
1642  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1643  * make specified memory blocks removable from the system.
1644  * Note that this will dissolve a free gigantic hugepage completely, if any
1645  * part of it lies within the given range.
1646  * Also note that if dissolve_free_huge_page() returns with an error, all
1647  * free hugepages that were dissolved before that error are lost.
1648  */
1649 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1650 {
1651         unsigned long pfn;
1652         struct page *page;
1653         int rc = 0;
1654 
1655         if (!hugepages_supported())
1656                 return rc;
1657 
1658         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
1659                 page = pfn_to_page(pfn);
1660                 rc = dissolve_free_huge_page(page);
1661                 if (rc)
1662                         break;
1663         }
1664 
1665         return rc;
1666 }
1667 
1668 /*
1669  * Allocates a fresh surplus page from the page allocator.
1670  */
1671 static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
1672                 int nid, nodemask_t *nmask)
1673 {
1674         struct page *page = NULL;
1675 
1676         if (hstate_is_gigantic(h))
1677                 return NULL;
1678 
1679         spin_lock(&hugetlb_lock);
1680         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
1681                 goto out_unlock;
1682         spin_unlock(&hugetlb_lock);
1683 
1684         page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
1685         if (!page)
1686                 return NULL;
1687 
1688         spin_lock(&hugetlb_lock);
1689         /*
1690          * We could have raced with the pool size change.
1691          * Double check that and simply deallocate the new page
1692          * if we would end up overcommiting the surpluses. Abuse
1693          * temporary page to workaround the nasty free_huge_page
1694          * codeflow
1695          */
1696         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1697                 SetPageHugeTemporary(page);
1698                 spin_unlock(&hugetlb_lock);
1699                 put_page(page);
1700                 return NULL;
1701         } else {
1702                 h->surplus_huge_pages++;
1703                 h->surplus_huge_pages_node[page_to_nid(page)]++;
1704         }
1705 
1706 out_unlock:
1707         spin_unlock(&hugetlb_lock);
1708 
1709         return page;
1710 }
1711 
1712 struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
1713                                      int nid, nodemask_t *nmask)
1714 {
1715         struct page *page;
1716 
1717         if (hstate_is_gigantic(h))
1718                 return NULL;
1719 
1720         page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
1721         if (!page)
1722                 return NULL;
1723 
1724         /*
1725          * We do not account these pages as surplus because they are only
1726          * temporary and will be released properly on the last reference
1727          */
1728         SetPageHugeTemporary(page);
1729 
1730         return page;
1731 }
1732 
1733 /*
1734  * Use the VMA's mpolicy to allocate a huge page from the buddy.
1735  */
1736 static
1737 struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,
1738                 struct vm_area_struct *vma, unsigned long addr)
1739 {
1740         struct page *page;
1741         struct mempolicy *mpol;
1742         gfp_t gfp_mask = htlb_alloc_mask(h);
1743         int nid;
1744         nodemask_t *nodemask;
1745 
1746         nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
1747         page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask);
1748         mpol_cond_put(mpol);
1749 
1750         return page;
1751 }
1752 
1753 /* page migration callback function */
1754 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1755 {
1756         gfp_t gfp_mask = htlb_alloc_mask(h);
1757         struct page *page = NULL;
1758 
1759         if (nid != NUMA_NO_NODE)
1760                 gfp_mask |= __GFP_THISNODE;
1761 
1762         spin_lock(&hugetlb_lock);
1763         if (h->free_huge_pages - h->resv_huge_pages > 0)
1764                 page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL);
1765         spin_unlock(&hugetlb_lock);
1766 
1767         if (!page)
1768                 page = alloc_migrate_huge_page(h, gfp_mask, nid, NULL);
1769 
1770         return page;
1771 }
1772 
1773 /* page migration callback function */
1774 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
1775                 nodemask_t *nmask)
1776 {
1777         gfp_t gfp_mask = htlb_alloc_mask(h);
1778 
1779         spin_lock(&hugetlb_lock);
1780         if (h->free_huge_pages - h->resv_huge_pages > 0) {
1781                 struct page *page;
1782 
1783                 page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
1784                 if (page) {
1785                         spin_unlock(&hugetlb_lock);
1786                         return page;
1787                 }
1788         }
1789         spin_unlock(&hugetlb_lock);
1790 
1791         return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);
1792 }
1793 
1794 /* mempolicy aware migration callback */
1795 struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
1796                 unsigned long address)
1797 {
1798         struct mempolicy *mpol;
1799         nodemask_t *nodemask;
1800         struct page *page;
1801         gfp_t gfp_mask;
1802         int node;
1803 
1804         gfp_mask = htlb_alloc_mask(h);
1805         node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1806         page = alloc_huge_page_nodemask(h, node, nodemask);
1807         mpol_cond_put(mpol);
1808 
1809         return page;
1810 }
1811 
1812 /*
1813  * Increase the hugetlb pool such that it can accommodate a reservation
1814  * of size 'delta'.
1815  */
1816 static int gather_surplus_pages(struct hstate *h, int delta)
1817 {
1818         struct list_head surplus_list;
1819         struct page *page, *tmp;
1820         int ret, i;
1821         int needed, allocated;
1822         bool alloc_ok = true;
1823 
1824         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1825         if (needed <= 0) {
1826                 h->resv_huge_pages += delta;
1827                 return 0;
1828         }
1829 
1830         allocated = 0;
1831         INIT_LIST_HEAD(&surplus_list);
1832 
1833         ret = -ENOMEM;
1834 retry:
1835         spin_unlock(&hugetlb_lock);
1836         for (i = 0; i < needed; i++) {
1837                 page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
1838                                 NUMA_NO_NODE, NULL);
1839                 if (!page) {
1840                         alloc_ok = false;
1841                         break;
1842                 }
1843                 list_add(&page->lru, &surplus_list);
1844                 cond_resched();
1845         }
1846         allocated += i;
1847 
1848         /*
1849          * After retaking hugetlb_lock, we need to recalculate 'needed'
1850          * because either resv_huge_pages or free_huge_pages may have changed.
1851          */
1852         spin_lock(&hugetlb_lock);
1853         needed = (h->resv_huge_pages + delta) -
1854                         (h->free_huge_pages + allocated);
1855         if (needed > 0) {
1856                 if (alloc_ok)
1857                         goto retry;
1858                 /*
1859                  * We were not able to allocate enough pages to
1860                  * satisfy the entire reservation so we free what
1861                  * we've allocated so far.
1862                  */
1863                 goto free;
1864         }
1865         /*
1866          * The surplus_list now contains _at_least_ the number of extra pages
1867          * needed to accommodate the reservation.  Add the appropriate number
1868          * of pages to the hugetlb pool and free the extras back to the buddy
1869          * allocator.  Commit the entire reservation here to prevent another
1870          * process from stealing the pages as they are added to the pool but
1871          * before they are reserved.
1872          */
1873         needed += allocated;
1874         h->resv_huge_pages += delta;
1875         ret = 0;
1876 
1877         /* Free the needed pages to the hugetlb pool */
1878         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1879                 if ((--needed) < 0)
1880                         break;
1881                 /*
1882                  * This page is now managed by the hugetlb allocator and has
1883                  * no users -- drop the buddy allocator's reference.
1884                  */
1885                 put_page_testzero(page);
1886                 VM_BUG_ON_PAGE(page_count(page), page);
1887                 enqueue_huge_page(h, page);
1888         }
1889 free:
1890         spin_unlock(&hugetlb_lock);
1891 
1892         /* Free unnecessary surplus pages to the buddy allocator */
1893         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1894                 put_page(page);
1895         spin_lock(&hugetlb_lock);
1896 
1897         return ret;
1898 }
1899 
1900 /*
1901  * This routine has two main purposes:
1902  * 1) Decrement the reservation count (resv_huge_pages) by the value passed
1903  *    in unused_resv_pages.  This corresponds to the prior adjustments made
1904  *    to the associated reservation map.
1905  * 2) Free any unused surplus pages that may have been allocated to satisfy
1906  *    the reservation.  As many as unused_resv_pages may be freed.
1907  *
1908  * Called with hugetlb_lock held.  However, the lock could be dropped (and
1909  * reacquired) during calls to cond_resched_lock.  Whenever dropping the lock,
1910  * we must make sure nobody else can claim pages we are in the process of
1911  * freeing.  Do this by ensuring resv_huge_page always is greater than the
1912  * number of huge pages we plan to free when dropping the lock.
1913  */
1914 static void return_unused_surplus_pages(struct hstate *h,
1915                                         unsigned long unused_resv_pages)
1916 {
1917         unsigned long nr_pages;
1918 
1919         /* Cannot return gigantic pages currently */
1920         if (hstate_is_gigantic(h))
1921                 goto out;
1922 
1923         /*
1924          * Part (or even all) of the reservation could have been backed
1925          * by pre-allocated pages. Only free surplus pages.
1926          */
1927         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1928 
1929         /*
1930          * We want to release as many surplus pages as possible, spread
1931          * evenly across all nodes with memory. Iterate across these nodes
1932          * until we can no longer free unreserved surplus pages. This occurs
1933          * when the nodes with surplus pages have no free pages.
1934          * free_pool_huge_page() will balance the the freed pages across the
1935          * on-line nodes with memory and will handle the hstate accounting.
1936          *
1937          * Note that we decrement resv_huge_pages as we free the pages.  If
1938          * we drop the lock, resv_huge_pages will still be sufficiently large
1939          * to cover subsequent pages we may free.
1940          */
1941         while (nr_pages--) {
1942                 h->resv_huge_pages--;
1943                 unused_resv_pages--;
1944                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1945                         goto out;
1946                 cond_resched_lock(&hugetlb_lock);
1947         }
1948 
1949 out:
1950         /* Fully uncommit the reservation */
1951         h->resv_huge_pages -= unused_resv_pages;
1952 }
1953 
1954 
1955 /*
1956  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1957  * are used by the huge page allocation routines to manage reservations.
1958  *
1959  * vma_needs_reservation is called to determine if the huge page at addr
1960  * within the vma has an associated reservation.  If a reservation is
1961  * needed, the value 1 is returned.  The caller is then responsible for
1962  * managing the global reservation and subpool usage counts.  After
1963  * the huge page has been allocated, vma_commit_reservation is called
1964  * to add the page to the reservation map.  If the page allocation fails,
1965  * the reservation must be ended instead of committed.  vma_end_reservation
1966  * is called in such cases.
1967  *
1968  * In the normal case, vma_commit_reservation returns the same value
1969  * as the preceding vma_needs_reservation call.  The only time this
1970  * is not the case is if a reserve map was changed between calls.  It
1971  * is the responsibility of the caller to notice the difference and
1972  * take appropriate action.
1973  *
1974  * vma_add_reservation is used in error paths where a reservation must
1975  * be restored when a newly allocated huge page must be freed.  It is
1976  * to be called after calling vma_needs_reservation to determine if a
1977  * reservation exists.
1978  */
1979 enum vma_resv_mode {
1980         VMA_NEEDS_RESV,
1981         VMA_COMMIT_RESV,
1982         VMA_END_RESV,
1983         VMA_ADD_RESV,
1984 };
1985 static long __vma_reservation_common(struct hstate *h,
1986                                 struct vm_area_struct *vma, unsigned long addr,
1987                                 enum vma_resv_mode mode)
1988 {
1989         struct resv_map *resv;
1990         pgoff_t idx;
1991         long ret;
1992 
1993         resv = vma_resv_map(vma);
1994         if (!resv)
1995                 return 1;
1996 
1997         idx = vma_hugecache_offset(h, vma, addr);
1998         switch (mode) {
1999         case VMA_NEEDS_RESV:
2000                 ret = region_chg(resv, idx, idx + 1);
2001                 break;
2002         case VMA_COMMIT_RESV:
2003                 ret = region_add(resv, idx, idx + 1);
2004                 break;
2005         case VMA_END_RESV:
2006                 region_abort(resv, idx, idx + 1);
2007                 ret = 0;
2008                 break;
2009         case VMA_ADD_RESV:
2010                 if (vma->vm_flags & VM_MAYSHARE)
2011                         ret = region_add(resv, idx, idx + 1);
2012                 else {
2013                         region_abort(resv, idx, idx + 1);
2014                         ret = region_del(resv, idx, idx + 1);
2015                 }
2016                 break;
2017         default:
2018                 BUG();
2019         }
2020 
2021         if (vma->vm_flags & VM_MAYSHARE)
2022                 return ret;
2023         else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
2024                 /*
2025                  * In most cases, reserves always exist for private mappings.
2026                  * However, a file associated with mapping could have been
2027                  * hole punched or truncated after reserves were consumed.
2028                  * As subsequent fault on such a range will not use reserves.
2029                  * Subtle - The reserve map for private mappings has the
2030                  * opposite meaning than that of shared mappings.  If NO
2031                  * entry is in the reserve map, it means a reservation exists.
2032                  * If an entry exists in the reserve map, it means the
2033                  * reservation has already been consumed.  As a result, the
2034                  * return value of this routine is the opposite of the
2035                  * value returned from reserve map manipulation routines above.
2036                  */
2037                 if (ret)
2038                         return 0;
2039                 else
2040                         return 1;
2041         }
2042         else
2043                 return ret < 0 ? ret : 0;
2044 }
2045 
2046 static long vma_needs_reservation(struct hstate *h,
2047                         struct vm_area_struct *vma, unsigned long addr)
2048 {
2049         return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2050 }
2051 
2052 static long vma_commit_reservation(struct hstate *h,
2053                         struct vm_area_struct *vma, unsigned long addr)
2054 {
2055         return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2056 }
2057 
2058 static void vma_end_reservation(struct hstate *h,
2059                         struct vm_area_struct *vma, unsigned long addr)
2060 {
2061         (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2062 }
2063 
2064 static long vma_add_reservation(struct hstate *h,
2065                         struct vm_area_struct *vma, unsigned long addr)
2066 {
2067         return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2068 }
2069 
2070 /*
2071  * This routine is called to restore a reservation on error paths.  In the
2072  * specific error paths, a huge page was allocated (via alloc_huge_page)
2073  * and is about to be freed.  If a reservation for the page existed,
2074  * alloc_huge_page would have consumed the reservation and set PagePrivate
2075  * in the newly allocated page.  When the page is freed via free_huge_page,
2076  * the global reservation count will be incremented if PagePrivate is set.
2077  * However, free_huge_page can not adjust the reserve map.  Adjust the
2078  * reserve map here to be consistent with global reserve count adjustments
2079  * to be made by free_huge_page.
2080  */
2081 static void restore_reserve_on_error(struct hstate *h,
2082                         struct vm_area_struct *vma, unsigned long address,
2083                         struct page *page)
2084 {
2085         if (unlikely(PagePrivate(page))) {
2086                 long rc = vma_needs_reservation(h, vma, address);
2087 
2088                 if (unlikely(rc < 0)) {
2089                         /*
2090                          * Rare out of memory condition in reserve map
2091                          * manipulation.  Clear PagePrivate so that
2092                          * global reserve count will not be incremented
2093                          * by free_huge_page.  This will make it appear
2094                          * as though the reservation for this page was
2095                          * consumed.  This may prevent the task from
2096                          * faulting in the page at a later time.  This
2097                          * is better than inconsistent global huge page
2098                          * accounting of reserve counts.
2099                          */
2100                         ClearPagePrivate(page);
2101                 } else if (rc) {
2102                         rc = vma_add_reservation(h, vma, address);
2103                         if (unlikely(rc < 0))
2104                                 /*
2105                                  * See above comment about rare out of
2106                                  * memory condition.
2107                                  */
2108                                 ClearPagePrivate(page);
2109                 } else
2110                         vma_end_reservation(h, vma, address);
2111         }
2112 }
2113 
2114 struct page *alloc_huge_page(struct vm_area_struct *vma,
2115                                     unsigned long addr, int avoid_reserve)
2116 {
2117         struct hugepage_subpool *spool = subpool_vma(vma);
2118         struct hstate *h = hstate_vma(vma);
2119         struct page *page;
2120         long map_chg, map_commit;
2121         long gbl_chg;
2122         int ret, idx;
2123         struct hugetlb_cgroup *h_cg;
2124 
2125         idx = hstate_index(h);
2126         /*
2127          * Examine the region/reserve map to determine if the process
2128          * has a reservation for the page to be allocated.  A return
2129          * code of zero indicates a reservation exists (no change).
2130          */
2131         map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2132         if (map_chg < 0)
2133                 return ERR_PTR(-ENOMEM);
2134 
2135         /*
2136          * Processes that did not create the mapping will have no
2137          * reserves as indicated by the region/reserve map. Check
2138          * that the allocation will not exceed the subpool limit.
2139          * Allocations for MAP_NORESERVE mappings also need to be
2140          * checked against any subpool limit.
2141          */
2142         if (map_chg || avoid_reserve) {
2143                 gbl_chg = hugepage_subpool_get_pages(spool, 1);
2144                 if (gbl_chg < 0) {
2145                         vma_end_reservation(h, vma, addr);
2146                         return ERR_PTR(-ENOSPC);
2147                 }
2148 
2149                 /*
2150                  * Even though there was no reservation in the region/reserve
2151                  * map, there could be reservations associated with the
2152                  * subpool that can be used.  This would be indicated if the
2153                  * return value of hugepage_subpool_get_pages() is zero.
2154                  * However, if avoid_reserve is specified we still avoid even
2155                  * the subpool reservations.
2156                  */
2157                 if (avoid_reserve)
2158                         gbl_chg = 1;
2159         }
2160 
2161         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2162         if (ret)
2163                 goto out_subpool_put;
2164 
2165         spin_lock(&hugetlb_lock);
2166         /*
2167          * glb_chg is passed to indicate whether or not a page must be taken
2168          * from the global free pool (global change).  gbl_chg == 0 indicates
2169          * a reservation exists for the allocation.
2170          */
2171         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2172         if (!page) {
2173                 spin_unlock(&hugetlb_lock);
2174                 page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
2175                 if (!page)
2176                         goto out_uncharge_cgroup;
2177                 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2178                         SetPagePrivate(page);
2179                         h->resv_huge_pages--;
2180                 }
2181                 spin_lock(&hugetlb_lock);
2182                 list_move(&page->lru, &h->hugepage_activelist);
2183                 /* Fall through */
2184         }
2185         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2186         spin_unlock(&hugetlb_lock);
2187 
2188         set_page_private(page, (unsigned long)spool);
2189 
2190         map_commit = vma_commit_reservation(h, vma, addr);
2191         if (unlikely(map_chg > map_commit)) {
2192                 /*
2193                  * The page was added to the reservation map between
2194                  * vma_needs_reservation and vma_commit_reservation.
2195                  * This indicates a race with hugetlb_reserve_pages.
2196                  * Adjust for the subpool count incremented above AND
2197                  * in hugetlb_reserve_pages for the same page.  Also,
2198                  * the reservation count added in hugetlb_reserve_pages
2199                  * no longer applies.
2200                  */
2201                 long rsv_adjust;
2202 
2203                 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2204                 hugetlb_acct_memory(h, -rsv_adjust);
2205         }
2206         return page;
2207 
2208 out_uncharge_cgroup:
2209         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2210 out_subpool_put:
2211         if (map_chg || avoid_reserve)
2212                 hugepage_subpool_put_pages(spool, 1);
2213         vma_end_reservation(h, vma, addr);
2214         return ERR_PTR(-ENOSPC);
2215 }
2216 
2217 int alloc_bootmem_huge_page(struct hstate *h)
2218         __attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
2219 int __alloc_bootmem_huge_page(struct hstate *h)
2220 {
2221         struct huge_bootmem_page *m;
2222         int nr_nodes, node;
2223 
2224         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2225                 void *addr;
2226 
2227                 addr = memblock_alloc_try_nid_raw(
2228                                 huge_page_size(h), huge_page_size(h),
2229                                 0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
2230                 if (addr) {
2231                         /*
2232                          * Use the beginning of the huge page to store the
2233                          * huge_bootmem_page struct (until gather_bootmem
2234                          * puts them into the mem_map).
2235                          */
2236                         m = addr;
2237                         goto found;
2238                 }
2239         }
2240         return 0;
2241 
2242 found:
2243         BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2244         /* Put them into a private list first because mem_map is not up yet */
2245         INIT_LIST_HEAD(&m->list);
2246         list_add(&m->list, &huge_boot_pages);
2247         m->hstate = h;
2248         return 1;
2249 }
2250 
2251 static void __init prep_compound_huge_page(struct page *page,
2252                 unsigned int order)
2253 {
2254         if (unlikely(order > (MAX_ORDER - 1)))
2255                 prep_compound_gigantic_page(page, order);
2256         else
2257                 prep_compound_page(page, order);
2258 }
2259 
2260 /* Put bootmem huge pages into the standard lists after mem_map is up */
2261 static void __init gather_bootmem_prealloc(void)
2262 {
2263         struct huge_bootmem_page *m;
2264 
2265         list_for_each_entry(m, &huge_boot_pages, list) {
2266                 struct page *page = virt_to_page(m);
2267                 struct hstate *h = m->hstate;
2268 
2269                 WARN_ON(page_count(page) != 1);
2270                 prep_compound_huge_page(page, h->order);
2271                 WARN_ON(PageReserved(page));
2272                 prep_new_huge_page(h, page, page_to_nid(page));
2273                 put_page(page); /* free it into the hugepage allocator */
2274 
2275                 /*
2276                  * If we had gigantic hugepages allocated at boot time, we need
2277                  * to restore the 'stolen' pages to totalram_pages in order to
2278                  * fix confusing memory reports from free(1) and another
2279                  * side-effects, like CommitLimit going negative.
2280                  */
2281                 if (hstate_is_gigantic(h))
2282                         adjust_managed_page_count(page, 1 << h->order);
2283                 cond_resched();
2284         }
2285 }
2286 
2287 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2288 {
2289         unsigned long i;
2290         nodemask_t *node_alloc_noretry;
2291 
2292         if (!hstate_is_gigantic(h)) {
2293                 /*
2294                  * Bit mask controlling how hard we retry per-node allocations.
2295                  * Ignore errors as lower level routines can deal with
2296                  * node_alloc_noretry == NULL.  If this kmalloc fails at boot
2297                  * time, we are likely in bigger trouble.
2298                  */
2299                 node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
2300                                                 GFP_KERNEL);
2301         } else {
2302                 /* allocations done at boot time */
2303                 node_alloc_noretry = NULL;
2304         }
2305 
2306         /* bit mask controlling how hard we retry per-node allocations */
2307         if (node_alloc_noretry)
2308                 nodes_clear(*node_alloc_noretry);
2309 
2310         for (i = 0; i < h->max_huge_pages; ++i) {
2311                 if (hstate_is_gigantic(h)) {
2312                         if (!alloc_bootmem_huge_page(h))
2313                                 break;
2314                 } else if (!alloc_pool_huge_page(h,
2315                                          &node_states[N_MEMORY],
2316                                          node_alloc_noretry))
2317                         break;
2318                 cond_resched();
2319         }
2320         if (i < h->max_huge_pages) {
2321                 char buf[32];
2322 
2323                 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2324                 pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
2325                         h->max_huge_pages, buf, i);
2326                 h->max_huge_pages = i;
2327         }
2328 
2329         kfree(node_alloc_noretry);
2330 }
2331 
2332 static void __init hugetlb_init_hstates(void)
2333 {
2334         struct hstate *h;
2335 
2336         for_each_hstate(h) {
2337                 if (minimum_order > huge_page_order(h))
2338                         minimum_order = huge_page_order(h);
2339 
2340                 /* oversize hugepages were init'ed in early boot */
2341                 if (!hstate_is_gigantic(h))
2342                         hugetlb_hstate_alloc_pages(h);
2343         }
2344         VM_BUG_ON(minimum_order == UINT_MAX);
2345 }
2346 
2347 static void __init report_hugepages(void)
2348 {
2349         struct hstate *h;
2350 
2351         for_each_hstate(h) {
2352                 char buf[32];
2353 
2354                 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2355                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2356                         buf, h->free_huge_pages);
2357         }
2358 }
2359 
2360 #ifdef CONFIG_HIGHMEM
2361 static void try_to_free_low(struct hstate *h, unsigned long count,
2362                                                 nodemask_t *nodes_allowed)
2363 {
2364         int i;
2365 
2366         if (hstate_is_gigantic(h))
2367                 return;
2368 
2369         for_each_node_mask(i, *nodes_allowed) {
2370                 struct page *page, *next;
2371                 struct list_head *freel = &h->hugepage_freelists[i];
2372                 list_for_each_entry_safe(page, next, freel, lru) {
2373                         if (count >= h->nr_huge_pages)
2374                                 return;
2375                         if (PageHighMem(page))
2376                                 continue;
2377                         list_del(&page->lru);
2378                         update_and_free_page(h, page);
2379                         h->free_huge_pages--;
2380                         h->free_huge_pages_node[page_to_nid(page)]--;
2381                 }
2382         }
2383 }
2384 #else
2385 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2386                                                 nodemask_t *nodes_allowed)
2387 {
2388 }
2389 #endif
2390 
2391 /*
2392  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2393  * balanced by operating on them in a round-robin fashion.
2394  * Returns 1 if an adjustment was made.
2395  */
2396 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2397                                 int delta)
2398 {
2399         int nr_nodes, node;
2400 
2401         VM_BUG_ON(delta != -1 && delta != 1);
2402 
2403         if (delta < 0) {
2404                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2405                         if (h->surplus_huge_pages_node[node])
2406                                 goto found;
2407                 }
2408         } else {
2409                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2410                         if (h->surplus_huge_pages_node[node] <
2411                                         h->nr_huge_pages_node[node])
2412                                 goto found;
2413                 }
2414         }
2415         return 0;
2416 
2417 found:
2418         h->surplus_huge_pages += delta;
2419         h->surplus_huge_pages_node[node] += delta;
2420         return 1;
2421 }
2422 
2423 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2424 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
2425                               nodemask_t *nodes_allowed)
2426 {
2427         unsigned long min_count, ret;
2428         NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
2429 
2430         /*
2431          * Bit mask controlling how hard we retry per-node allocations.
2432          * If we can not allocate the bit mask, do not attempt to allocate
2433          * the requested huge pages.
2434          */
2435         if (node_alloc_noretry)
2436                 nodes_clear(*node_alloc_noretry);
2437         else
2438                 return -ENOMEM;
2439 
2440         spin_lock(&hugetlb_lock);
2441 
2442         /*
2443          * Check for a node specific request.
2444          * Changing node specific huge page count may require a corresponding
2445          * change to the global count.  In any case, the passed node mask
2446          * (nodes_allowed) will restrict alloc/free to the specified node.
2447          */
2448         if (nid != NUMA_NO_NODE) {
2449                 unsigned long old_count = count;
2450 
2451                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2452                 /*
2453                  * User may have specified a large count value which caused the
2454                  * above calculation to overflow.  In this case, they wanted
2455                  * to allocate as many huge pages as possible.  Set count to
2456                  * largest possible value to align with their intention.
2457                  */
2458                 if (count < old_count)
2459                         count = ULONG_MAX;
2460         }
2461 
2462         /*
2463          * Gigantic pages runtime allocation depend on the capability for large
2464          * page range allocation.
2465          * If the system does not provide this feature, return an error when
2466          * the user tries to allocate gigantic pages but let the user free the
2467          * boottime allocated gigantic pages.
2468          */
2469         if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
2470                 if (count > persistent_huge_pages(h)) {
2471                         spin_unlock(&hugetlb_lock);
2472                         NODEMASK_FREE(node_alloc_noretry);
2473                         return -EINVAL;
2474                 }
2475                 /* Fall through to decrease pool */
2476         }
2477 
2478         /*
2479          * Increase the pool size
2480          * First take pages out of surplus state.  Then make up the
2481          * remaining difference by allocating fresh huge pages.
2482          *
2483          * We might race with alloc_surplus_huge_page() here and be unable
2484          * to convert a surplus huge page to a normal huge page. That is
2485          * not critical, though, it just means the overall size of the
2486          * pool might be one hugepage larger than it needs to be, but
2487          * within all the constraints specified by the sysctls.
2488          */
2489         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2490                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2491                         break;
2492         }
2493 
2494         while (count > persistent_huge_pages(h)) {
2495                 /*
2496                  * If this allocation races such that we no longer need the
2497                  * page, free_huge_page will handle it by freeing the page
2498                  * and reducing the surplus.
2499                  */
2500                 spin_unlock(&hugetlb_lock);
2501 
2502                 /* yield cpu to avoid soft lockup */
2503                 cond_resched();
2504 
2505                 ret = alloc_pool_huge_page(h, nodes_allowed,
2506                                                 node_alloc_noretry);
2507                 spin_lock(&hugetlb_lock);
2508                 if (!ret)
2509                         goto out;
2510 
2511                 /* Bail for signals. Probably ctrl-c from user */
2512                 if (signal_pending(current))
2513                         goto out;
2514         }
2515 
2516         /*
2517          * Decrease the pool size
2518          * First return free pages to the buddy allocator (being careful
2519          * to keep enough around to satisfy reservations).  Then place
2520          * pages into surplus state as needed so the pool will shrink
2521          * to the desired size as pages become free.
2522          *
2523          * By placing pages into the surplus state independent of the
2524          * overcommit value, we are allowing the surplus pool size to
2525          * exceed overcommit. There are few sane options here. Since
2526          * alloc_surplus_huge_page() is checking the global counter,
2527          * though, we'll note that we're not allowed to exceed surplus
2528          * and won't grow the pool anywhere else. Not until one of the
2529          * sysctls are changed, or the surplus pages go out of use.
2530          */
2531         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2532         min_count = max(count, min_count);
2533         try_to_free_low(h, min_count, nodes_allowed);
2534         while (min_count < persistent_huge_pages(h)) {
2535                 if (!free_pool_huge_page(h, nodes_allowed, 0))
2536                         break;
2537                 cond_resched_lock(&hugetlb_lock);
2538         }
2539         while (count < persistent_huge_pages(h)) {
2540                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2541                         break;
2542         }
2543 out:
2544         h->max_huge_pages = persistent_huge_pages(h);
2545         spin_unlock(&hugetlb_lock);
2546 
2547         NODEMASK_FREE(node_alloc_noretry);
2548 
2549         return 0;
2550 }
2551 
2552 #define HSTATE_ATTR_RO(_name) \
2553         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2554 
2555 #define HSTATE_ATTR(_name) \
2556         static struct kobj_attribute _name##_attr = \
2557                 __ATTR(_name, 0644, _name##_show, _name##_store)
2558 
2559 static struct kobject *hugepages_kobj;
2560 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2561 
2562 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2563 
2564 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2565 {
2566         int i;
2567 
2568         for (i = 0; i < HUGE_MAX_HSTATE; i++)
2569                 if (hstate_kobjs[i] == kobj) {
2570                         if (nidp)
2571                                 *nidp = NUMA_NO_NODE;
2572                         return &hstates[i];
2573                 }
2574 
2575         return kobj_to_node_hstate(kobj, nidp);
2576 }
2577 
2578 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2579                                         struct kobj_attribute *attr, char *buf)
2580 {
2581         struct hstate *h;
2582         unsigned long nr_huge_pages;
2583         int nid;
2584 
2585         h = kobj_to_hstate(kobj, &nid);
2586         if (nid == NUMA_NO_NODE)
2587                 nr_huge_pages = h->nr_huge_pages;
2588         else
2589                 nr_huge_pages = h->nr_huge_pages_node[nid];
2590 
2591         return sprintf(buf, "%lu\n", nr_huge_pages);
2592 }
2593 
2594 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2595                                            struct hstate *h, int nid,
2596                                            unsigned long count, size_t len)
2597 {
2598         int err;
2599         nodemask_t nodes_allowed, *n_mask;
2600 
2601         if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
2602                 return -EINVAL;
2603 
2604         if (nid == NUMA_NO_NODE) {
2605                 /*
2606                  * global hstate attribute
2607                  */
2608                 if (!(obey_mempolicy &&
2609                                 init_nodemask_of_mempolicy(&nodes_allowed)))
2610                         n_mask = &node_states[N_MEMORY];
2611                 else
2612                         n_mask = &nodes_allowed;
2613         } else {
2614                 /*
2615                  * Node specific request.  count adjustment happens in
2616                  * set_max_huge_pages() after acquiring hugetlb_lock.
2617                  */
2618                 init_nodemask_of_node(&nodes_allowed, nid);
2619                 n_mask = &nodes_allowed;
2620         }
2621 
2622         err = set_max_huge_pages(h, count, nid, n_mask);
2623 
2624         return err ? err : len;
2625 }
2626 
2627 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2628                                          struct kobject *kobj, const char *buf,
2629                                          size_t len)
2630 {
2631         struct hstate *h;
2632         unsigned long count;
2633         int nid;
2634         int err;
2635 
2636         err = kstrtoul(buf, 10, &count);
2637         if (err)
2638                 return err;
2639 
2640         h = kobj_to_hstate(kobj, &nid);
2641         return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2642 }
2643 
2644 static ssize_t nr_hugepages_show(struct kobject *kobj,
2645                                        struct kobj_attribute *attr, char *buf)
2646 {
2647         return nr_hugepages_show_common(kobj, attr, buf);
2648 }
2649 
2650 static ssize_t nr_hugepages_store(struct kobject *kobj,
2651                struct kobj_attribute *attr, const char *buf, size_t len)
2652 {
2653         return nr_hugepages_store_common(false, kobj, buf, len);
2654 }
2655 HSTATE_ATTR(nr_hugepages);
2656 
2657 #ifdef CONFIG_NUMA
2658 
2659 /*
2660  * hstate attribute for optionally mempolicy-based constraint on persistent
2661  * huge page alloc/free.
2662  */
2663 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2664                                        struct kobj_attribute *attr, char *buf)
2665 {
2666         return nr_hugepages_show_common(kobj, attr, buf);
2667 }
2668 
2669 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2670                struct kobj_attribute *attr, const char *buf, size_t len)
2671 {
2672         return nr_hugepages_store_common(true, kobj, buf, len);
2673 }
2674 HSTATE_ATTR(nr_hugepages_mempolicy);
2675 #endif
2676 
2677 
2678 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2679                                         struct kobj_attribute *attr, char *buf)
2680 {
2681         struct hstate *h = kobj_to_hstate(kobj, NULL);
2682         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2683 }
2684 
2685 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2686                 struct kobj_attribute *attr, const char *buf, size_t count)
2687 {
2688         int err;
2689         unsigned long input;
2690         struct hstate *h = kobj_to_hstate(kobj, NULL);
2691 
2692         if (hstate_is_gigantic(h))
2693                 return -EINVAL;
2694 
2695         err = kstrtoul(buf, 10, &input);
2696         if (err)
2697                 return err;
2698 
2699         spin_lock(&hugetlb_lock);
2700         h->nr_overcommit_huge_pages = input;
2701         spin_unlock(&hugetlb_lock);
2702 
2703         return count;
2704 }
2705 HSTATE_ATTR(nr_overcommit_hugepages);
2706 
2707 static ssize_t free_hugepages_show(struct kobject *kobj,
2708                                         struct kobj_attribute *attr, char *buf)
2709 {
2710         struct hstate *h;
2711         unsigned long free_huge_pages;
2712         int nid;
2713 
2714         h = kobj_to_hstate(kobj, &nid);
2715         if (nid == NUMA_NO_NODE)
2716                 free_huge_pages = h->free_huge_pages;
2717         else
2718                 free_huge_pages = h->free_huge_pages_node[nid];
2719 
2720         return sprintf(buf, "%lu\n", free_huge_pages);
2721 }
2722 HSTATE_ATTR_RO(free_hugepages);
2723 
2724 static ssize_t resv_hugepages_show(struct kobject *kobj,
2725                                         struct kobj_attribute *attr, char *buf)
2726 {
2727         struct hstate *h = kobj_to_hstate(kobj, NULL);
2728         return sprintf(buf, "%lu\n", h->resv_huge_pages);
2729 }
2730 HSTATE_ATTR_RO(resv_hugepages);
2731 
2732 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2733                                         struct kobj_attribute *attr, char *buf)
2734 {
2735         struct hstate *h;
2736         unsigned long surplus_huge_pages;
2737         int nid;
2738 
2739         h = kobj_to_hstate(kobj, &nid);
2740         if (nid == NUMA_NO_NODE)
2741                 surplus_huge_pages = h->surplus_huge_pages;
2742         else
2743                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2744 
2745         return sprintf(buf, "%lu\n", surplus_huge_pages);
2746 }
2747 HSTATE_ATTR_RO(surplus_hugepages);
2748 
2749 static struct attribute *hstate_attrs[] = {
2750         &nr_hugepages_attr.attr,
2751         &nr_overcommit_hugepages_attr.attr,
2752         &free_hugepages_attr.attr,
2753         &resv_hugepages_attr.attr,
2754         &surplus_hugepages_attr.attr,
2755 #ifdef CONFIG_NUMA
2756         &nr_hugepages_mempolicy_attr.attr,
2757 #endif
2758         NULL,
2759 };
2760 
2761 static const struct attribute_group hstate_attr_group = {
2762         .attrs = hstate_attrs,
2763 };
2764 
2765 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2766                                     struct kobject **hstate_kobjs,
2767                                     const struct attribute_group *hstate_attr_group)
2768 {
2769         int retval;
2770         int hi = hstate_index(h);
2771 
2772         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2773         if (!hstate_kobjs[hi])
2774                 return -ENOMEM;
2775 
2776         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2777         if (retval)
2778                 kobject_put(hstate_kobjs[hi]);
2779 
2780         return retval;
2781 }
2782 
2783 static void __init hugetlb_sysfs_init(void)
2784 {
2785         struct hstate *h;
2786         int err;
2787 
2788         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2789         if (!hugepages_kobj)
2790                 return;
2791 
2792         for_each_hstate(h) {
2793                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2794                                          hstate_kobjs, &hstate_attr_group);
2795                 if (err)
2796                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
2797         }
2798 }
2799 
2800 #ifdef CONFIG_NUMA
2801 
2802 /*
2803  * node_hstate/s - associate per node hstate attributes, via their kobjects,
2804  * with node devices in node_devices[] using a parallel array.  The array
2805  * index of a node device or _hstate == node id.
2806  * This is here to avoid any static dependency of the node device driver, in
2807  * the base kernel, on the hugetlb module.
2808  */
2809 struct node_hstate {
2810         struct kobject          *hugepages_kobj;
2811         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
2812 };
2813 static struct node_hstate node_hstates[MAX_NUMNODES];
2814 
2815 /*
2816  * A subset of global hstate attributes for node devices
2817  */
2818 static struct attribute *per_node_hstate_attrs[] = {
2819         &nr_hugepages_attr.attr,
2820         &free_hugepages_attr.attr,
2821         &surplus_hugepages_attr.attr,
2822         NULL,
2823 };
2824 
2825 static const struct attribute_group per_node_hstate_attr_group = {
2826         .attrs = per_node_hstate_attrs,
2827 };
2828 
2829 /*
2830  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2831  * Returns node id via non-NULL nidp.
2832  */
2833 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2834 {
2835         int nid;
2836 
2837         for (nid = 0; nid < nr_node_ids; nid++) {
2838                 struct node_hstate *nhs = &node_hstates[nid];
2839                 int i;
2840                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2841                         if (nhs->hstate_kobjs[i] == kobj) {
2842                                 if (nidp)
2843                                         *nidp = nid;
2844                                 return &hstates[i];
2845                         }
2846         }
2847 
2848         BUG();
2849         return NULL;
2850 }
2851 
2852 /*
2853  * Unregister hstate attributes from a single node device.
2854  * No-op if no hstate attributes attached.
2855  */
2856 static void hugetlb_unregister_node(struct node *node)
2857 {
2858         struct hstate *h;
2859         struct node_hstate *nhs = &node_hstates[node->dev.id];
2860 
2861         if (!nhs->hugepages_kobj)
2862                 return;         /* no hstate attributes */
2863 
2864         for_each_hstate(h) {
2865                 int idx = hstate_index(h);
2866                 if (nhs->hstate_kobjs[idx]) {
2867                         kobject_put(nhs->hstate_kobjs[idx]);
2868                         nhs->hstate_kobjs[idx] = NULL;
2869                 }
2870         }
2871 
2872         kobject_put(nhs->hugepages_kobj);
2873         nhs->hugepages_kobj = NULL;
2874 }
2875 
2876 
2877 /*
2878  * Register hstate attributes for a single node device.
2879  * No-op if attributes already registered.
2880  */
2881 static void hugetlb_register_node(struct node *node)
2882 {
2883         struct hstate *h;
2884         struct node_hstate *nhs = &node_hstates[node->dev.id];
2885         int err;
2886 
2887         if (nhs->hugepages_kobj)
2888                 return;         /* already allocated */
2889 
2890         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2891                                                         &node->dev.kobj);
2892         if (!nhs->hugepages_kobj)
2893                 return;
2894 
2895         for_each_hstate(h) {
2896                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2897                                                 nhs->hstate_kobjs,
2898                                                 &per_node_hstate_attr_group);
2899                 if (err) {
2900                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2901                                 h->name, node->dev.id);
2902                         hugetlb_unregister_node(node);
2903                         break;
2904                 }
2905         }
2906 }
2907 
2908 /*
2909  * hugetlb init time:  register hstate attributes for all registered node
2910  * devices of nodes that have memory.  All on-line nodes should have
2911  * registered their associated device by this time.
2912  */
2913 static void __init hugetlb_register_all_nodes(void)
2914 {
2915         int nid;
2916 
2917         for_each_node_state(nid, N_MEMORY) {
2918                 struct node *node = node_devices[nid];
2919                 if (node->dev.id == nid)
2920                         hugetlb_register_node(node);
2921         }
2922 
2923         /*
2924          * Let the node device driver know we're here so it can
2925          * [un]register hstate attributes on node hotplug.
2926          */
2927         register_hugetlbfs_with_node(hugetlb_register_node,
2928                                      hugetlb_unregister_node);
2929 }
2930 #else   /* !CONFIG_NUMA */
2931 
2932 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2933 {
2934         BUG();
2935         if (nidp)
2936                 *nidp = -1;
2937         return NULL;
2938 }
2939 
2940 static void hugetlb_register_all_nodes(void) { }
2941 
2942 #endif
2943 
2944 static int __init hugetlb_init(void)
2945 {
2946         int i;
2947 
2948         if (!hugepages_supported())
2949                 return 0;
2950 
2951         if (!size_to_hstate(default_hstate_size)) {
2952                 if (default_hstate_size != 0) {
2953                         pr_err("HugeTLB: unsupported default_hugepagesz %lu. Reverting to %lu\n",
2954                                default_hstate_size, HPAGE_SIZE);
2955                 }
2956 
2957                 default_hstate_size = HPAGE_SIZE;
2958                 if (!size_to_hstate(default_hstate_size))
2959                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2960         }
2961         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2962         if (default_hstate_max_huge_pages) {
2963                 if (!default_hstate.max_huge_pages)
2964                         default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2965         }
2966 
2967         hugetlb_init_hstates();
2968         gather_bootmem_prealloc();
2969         report_hugepages();
2970 
2971         hugetlb_sysfs_init();
2972         hugetlb_register_all_nodes();
2973         hugetlb_cgroup_file_init();
2974 
2975 #ifdef CONFIG_SMP
2976         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2977 #else
2978         num_fault_mutexes = 1;
2979 #endif
2980         hugetlb_fault_mutex_table =
2981                 kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
2982                               GFP_KERNEL);
2983         BUG_ON(!hugetlb_fault_mutex_table);
2984 
2985         for (i = 0; i < num_fault_mutexes; i++)
2986                 mutex_init(&hugetlb_fault_mutex_table[i]);
2987         return 0;
2988 }
2989 subsys_initcall(hugetlb_init);
2990 
2991 /* Should be called on processing a hugepagesz=... option */
2992 void __init hugetlb_bad_size(void)
2993 {
2994         parsed_valid_hugepagesz = false;
2995 }
2996 
2997 void __init hugetlb_add_hstate(unsigned int order)
2998 {
2999         struct hstate *h;
3000         unsigned long i;
3001 
3002         if (size_to_hstate(PAGE_SIZE << order)) {
3003                 pr_warn("hugepagesz= specified twice, ignoring\n");
3004                 return;
3005         }
3006         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
3007         BUG_ON(order == 0);
3008         h = &hstates[hugetlb_max_hstate++];
3009         h->order = order;
3010         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
3011         h->nr_huge_pages = 0;
3012         h->free_huge_pages = 0;
3013         for (i = 0; i < MAX_NUMNODES; ++i)
3014                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
3015         INIT_LIST_HEAD(&h->hugepage_activelist);
3016         h->next_nid_to_alloc = first_memory_node;
3017         h->next_nid_to_free = first_memory_node;
3018         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
3019                                         huge_page_size(h)/1024);
3020 
3021         parsed_hstate = h;
3022 }
3023 
3024 static int __init hugetlb_nrpages_setup(char *s)
3025 {
3026         unsigned long *mhp;
3027         static unsigned long *last_mhp;
3028 
3029         if (!parsed_valid_hugepagesz) {
3030                 pr_warn("hugepages = %s preceded by "
3031                         "an unsupported hugepagesz, ignoring\n", s);
3032                 parsed_valid_hugepagesz = true;
3033                 return 1;
3034         }
3035         /*
3036          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
3037          * so this hugepages= parameter goes to the "default hstate".
3038          */
3039         else if (!hugetlb_max_hstate)
3040                 mhp = &default_hstate_max_huge_pages;
3041         else
3042                 mhp = &parsed_hstate->max_huge_pages;
3043 
3044         if (mhp == last_mhp) {
3045                 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
3046                 return 1;
3047         }
3048 
3049         if (sscanf(s, "%lu", mhp) <= 0)
3050                 *mhp = 0;
3051 
3052         /*
3053          * Global state is always initialized later in hugetlb_init.
3054          * But we need to allocate >= MAX_ORDER hstates here early to still
3055          * use the bootmem allocator.
3056          */
3057         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
3058                 hugetlb_hstate_alloc_pages(parsed_hstate);
3059 
3060         last_mhp = mhp;
3061 
3062         return 1;
3063 }
3064 __setup("hugepages=", hugetlb_nrpages_setup);
3065 
3066 static int __init hugetlb_default_setup(char *s)
3067 {
3068         default_hstate_size = memparse(s, &s);
3069         return 1;
3070 }
3071 __setup("default_hugepagesz=", hugetlb_default_setup);
3072 
3073 static unsigned int cpuset_mems_nr(unsigned int *array)
3074 {
3075         int node;
3076         unsigned int nr = 0;
3077 
3078         for_each_node_mask(node, cpuset_current_mems_allowed)
3079                 nr += array[node];
3080 
3081         return nr;
3082 }
3083 
3084 #ifdef CONFIG_SYSCTL
3085 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
3086                          struct ctl_table *table, int write,
3087                          void __user *buffer, size_t *length, loff_t *ppos)
3088 {
3089         struct hstate *h = &default_hstate;
3090         unsigned long tmp = h->max_huge_pages;
3091         int ret;
3092 
3093         if (!hugepages_supported())
3094                 return -EOPNOTSUPP;
3095 
3096         table->data = &tmp;
3097         table->maxlen = sizeof(unsigned long);
3098         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
3099         if (ret)
3100                 goto out;
3101 
3102         if (write)
3103                 ret = __nr_hugepages_store_common(obey_mempolicy, h,
3104                                                   NUMA_NO_NODE, tmp, *length);
3105 out:
3106         return ret;
3107 }
3108 
3109 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
3110                           void __user *buffer, size_t *length, loff_t *ppos)
3111 {
3112 
3113         return hugetlb_sysctl_handler_common(false, table, write,
3114                                                         buffer, length, ppos);
3115 }
3116 
3117 #ifdef CONFIG_NUMA
3118 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
3119                           void __user *buffer, size_t *length, loff_t *ppos)
3120 {
3121         return hugetlb_sysctl_handler_common(true, table, write,
3122                                                         buffer, length, ppos);
3123 }
3124 #endif /* CONFIG_NUMA */
3125 
3126 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
3127                         void __user *buffer,
3128                         size_t *length, loff_t *ppos)
3129 {
3130         struct hstate *h = &default_hstate;
3131         unsigned long tmp;
3132         int ret;
3133 
3134         if (!hugepages_supported())
3135                 return -EOPNOTSUPP;
3136 
3137         tmp = h->nr_overcommit_huge_pages;
3138 
3139         if (write && hstate_is_gigantic(h))
3140                 return -EINVAL;
3141 
3142         table->data = &tmp;
3143         table->maxlen = sizeof(unsigned long);
3144         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
3145         if (ret)
3146                 goto out;
3147 
3148         if (write) {
3149                 spin_lock(&hugetlb_lock);
3150                 h->nr_overcommit_huge_pages = tmp;
3151                 spin_unlock(&hugetlb_lock);
3152         }
3153 out:
3154         return ret;
3155 }
3156 
3157 #endif /* CONFIG_SYSCTL */
3158 
3159 void hugetlb_report_meminfo(struct seq_file *m)
3160 {
3161         struct hstate *h;
3162         unsigned long total = 0;
3163 
3164         if (!hugepages_supported())
3165                 return;
3166 
3167         for_each_hstate(h) {
3168                 unsigned long count = h->nr_huge_pages;
3169 
3170                 total += (PAGE_SIZE << huge_page_order(h)) * count;
3171 
3172                 if (h == &default_hstate)
3173                         seq_printf(m,
3174                                    "HugePages_Total:   %5lu\n"
3175                                    "HugePages_Free:    %5lu\n"
3176                                    "HugePages_Rsvd:    %5lu\n"
3177                                    "HugePages_Surp:    %5lu\n"
3178                                    "Hugepagesize:   %8lu kB\n",
3179                                    count,
3180                                    h->free_huge_pages,
3181                                    h->resv_huge_pages,
3182                                    h->surplus_huge_pages,
3183                                    (PAGE_SIZE << huge_page_order(h)) / 1024);
3184         }
3185 
3186         seq_printf(m, "Hugetlb:        %8lu kB\n", total / 1024);
3187 }
3188 
3189 int hugetlb_report_node_meminfo(int nid, char *buf)
3190 {
3191         struct hstate *h = &default_hstate;
3192         if (!hugepages_supported())
3193                 return 0;
3194         return sprintf(buf,
3195                 "Node %d HugePages_Total: %5u\n"
3196                 "Node %d HugePages_Free:  %5u\n"
3197                 "Node %d HugePages_Surp:  %5u\n",
3198                 nid, h->nr_huge_pages_node[nid],
3199                 nid, h->free_huge_pages_node[nid],
3200                 nid, h->surplus_huge_pages_node[nid]);
3201 }
3202 
3203 void hugetlb_show_meminfo(void)
3204 {
3205         struct hstate *h;
3206         int nid;
3207 
3208         if (!hugepages_supported())
3209                 return;
3210 
3211         for_each_node_state(nid, N_MEMORY)
3212                 for_each_hstate(h)
3213                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3214                                 nid,
3215                                 h->nr_huge_pages_node[nid],
3216                                 h->free_huge_pages_node[nid],
3217                                 h->surplus_huge_pages_node[nid],
3218                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3219 }
3220 
3221 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3222 {
3223         seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3224                    atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3225 }
3226 
3227 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3228 unsigned long hugetlb_total_pages(void)
3229 {
3230         struct hstate *h;
3231         unsigned long nr_total_pages = 0;
3232 
3233         for_each_hstate(h)
3234                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3235         return nr_total_pages;
3236 }
3237 
3238 static int hugetlb_acct_memory(struct hstate *h, long delta)
3239 {
3240         int ret = -ENOMEM;
3241 
3242         spin_lock(&hugetlb_lock);
3243         /*
3244          * When cpuset is configured, it breaks the strict hugetlb page
3245          * reservation as the accounting is done on a global variable. Such
3246          * reservation is completely rubbish in the presence of cpuset because
3247          * the reservation is not checked against page availability for the
3248          * current cpuset. Application can still potentially OOM'ed by kernel
3249          * with lack of free htlb page in cpuset that the task is in.
3250          * Attempt to enforce strict accounting with cpuset is almost
3251          * impossible (or too ugly) because cpuset is too fluid that
3252          * task or memory node can be dynamically moved between cpusets.
3253          *
3254          * The change of semantics for shared hugetlb mapping with cpuset is
3255          * undesirable. However, in order to preserve some of the semantics,
3256          * we fall back to check against current free page availability as
3257          * a best attempt and hopefully to minimize the impact of changing
3258          * semantics that cpuset has.
3259          */
3260         if (delta > 0) {
3261                 if (gather_surplus_pages(h, delta) < 0)
3262                         goto out;
3263 
3264                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
3265                         return_unused_surplus_pages(h, delta);
3266                         goto out;
3267                 }
3268         }
3269 
3270         ret = 0;
3271         if (delta < 0)
3272                 return_unused_surplus_pages(h, (unsigned long) -delta);
3273 
3274 out:
3275         spin_unlock(&hugetlb_lock);
3276         return ret;
3277 }
3278 
3279 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
3280 {
3281         struct resv_map *resv = vma_resv_map(vma);
3282 
3283         /*
3284          * This new VMA should share its siblings reservation map if present.
3285          * The VMA will only ever have a valid reservation map pointer where
3286          * it is being copied for another still existing VMA.  As that VMA
3287          * has a reference to the reservation map it cannot disappear until
3288          * after this open call completes.  It is therefore safe to take a
3289          * new reference here without additional locking.
3290          */
3291         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3292                 kref_get(&resv->refs);
3293 }
3294 
3295 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3296 {
3297         struct hstate *h = hstate_vma(vma);
3298         struct resv_map *resv = vma_resv_map(vma);
3299         struct hugepage_subpool *spool = subpool_vma(vma);
3300         unsigned long reserve, start, end;
3301         long gbl_reserve;
3302 
3303         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3304                 return;
3305 
3306         start = vma_hugecache_offset(h, vma, vma->vm_start);
3307         end = vma_hugecache_offset(h, vma, vma->vm_end);
3308 
3309         reserve = (end - start) - region_count(resv, start, end);
3310 
3311         kref_put(&resv->refs, resv_map_release);
3312 
3313         if (reserve) {
3314                 /*
3315                  * Decrement reserve counts.  The global reserve count may be
3316                  * adjusted if the subpool has a minimum size.
3317                  */
3318                 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3319                 hugetlb_acct_memory(h, -gbl_reserve);
3320         }
3321 }
3322 
3323 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
3324 {
3325         if (addr & ~(huge_page_mask(hstate_vma(vma))))
3326                 return -EINVAL;
3327         return 0;
3328 }
3329 
3330 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
3331 {
3332         struct hstate *hstate = hstate_vma(vma);
3333 
3334         return 1UL << huge_page_shift(hstate);
3335 }
3336 
3337 /*
3338  * We cannot handle pagefaults against hugetlb pages at all.  They cause
3339  * handle_mm_fault() to try to instantiate regular-sized pages in the
3340  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
3341  * this far.
3342  */
3343 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
3344 {
3345         BUG();
3346         return 0;
3347 }
3348 
3349 /*
3350  * When a new function is introduced to vm_operations_struct and added
3351  * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
3352  * This is because under System V memory model, mappings created via
3353  * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
3354  * their original vm_ops are overwritten with shm_vm_ops.
3355  */
3356 const struct vm_operations_struct hugetlb_vm_ops = {
3357         .fault = hugetlb_vm_op_fault,
3358         .open = hugetlb_vm_op_open,
3359         .close = hugetlb_vm_op_close,
3360         .split = hugetlb_vm_op_split,
3361         .pagesize = hugetlb_vm_op_pagesize,
3362 };
3363 
3364 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3365                                 int writable)
3366 {
3367         pte_t entry;
3368 
3369         if (writable) {
3370                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3371                                          vma->vm_page_prot)));
3372         } else {
3373                 entry = huge_pte_wrprotect(mk_huge_pte(page,
3374                                            vma->vm_page_prot));
3375         }
3376         entry = pte_mkyoung(entry);
3377         entry = pte_mkhuge(entry);
3378         entry = arch_make_huge_pte(entry, vma, page, writable);
3379 
3380         return entry;
3381 }
3382 
3383 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3384                                    unsigned long address, pte_t *ptep)
3385 {
3386         pte_t entry;
3387 
3388         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3389         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3390                 update_mmu_cache(vma, address, ptep);
3391 }
3392 
3393 bool is_hugetlb_entry_migration(pte_t pte)
3394 {
3395         swp_entry_t swp;
3396 
3397         if (huge_pte_none(pte) || pte_present(pte))
3398                 return false;
3399         swp = pte_to_swp_entry(pte);
3400         if (non_swap_entry(swp) && is_migration_entry(swp))
3401                 return true;
3402         else
3403                 return false;
3404 }
3405 
3406 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3407 {
3408         swp_entry_t swp;
3409 
3410         if (huge_pte_none(pte) || pte_present(pte))
3411                 return 0;
3412         swp = pte_to_swp_entry(pte);
3413         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3414                 return 1;
3415         else
3416                 return 0;
3417 }
3418 
3419 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3420                             struct vm_area_struct *vma)
3421 {
3422         pte_t *src_pte, *dst_pte, entry, dst_entry;
3423         struct page *ptepage;
3424         unsigned long addr;
3425         int cow;
3426         struct hstate *h = hstate_vma(vma);
3427         unsigned long sz = huge_page_size(h);
3428         struct mmu_notifier_range range;
3429         int ret = 0;
3430 
3431         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3432 
3433         if (cow) {
3434                 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src,
3435                                         vma->vm_start,
3436                                         vma->vm_end);
3437                 mmu_notifier_invalidate_range_start(&range);
3438         }
3439 
3440         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3441                 spinlock_t *src_ptl, *dst_ptl;
3442                 src_pte = huge_pte_offset(src, addr, sz);
3443                 if (!src_pte)
3444                         continue;
3445                 dst_pte = huge_pte_alloc(dst, addr, sz);
3446                 if (!dst_pte) {
3447                         ret = -ENOMEM;
3448                         break;
3449                 }
3450 
3451                 /*
3452                  * If the pagetables are shared don't copy or take references.
3453                  * dst_pte == src_pte is the common case of src/dest sharing.
3454                  *
3455                  * However, src could have 'unshared' and dst shares with
3456                  * another vma.  If dst_pte !none, this implies sharing.
3457                  * Check here before taking page table lock, and once again
3458                  * after taking the lock below.
3459                  */
3460                 dst_entry = huge_ptep_get(dst_pte);
3461                 if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
3462                         continue;
3463 
3464                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
3465                 src_ptl = huge_pte_lockptr(h, src, src_pte);
3466                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3467                 entry = huge_ptep_get(src_pte);
3468                 dst_entry = huge_ptep_get(dst_pte);
3469                 if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
3470                         /*
3471                          * Skip if src entry none.  Also, skip in the
3472                          * unlikely case dst entry !none as this implies
3473                          * sharing with another vma.
3474                          */
3475                         ;
3476                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3477                                     is_hugetlb_entry_hwpoisoned(entry))) {
3478                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
3479 
3480                         if (is_write_migration_entry(swp_entry) && cow) {
3481                                 /*
3482                                  * COW mappings require pages in both
3483                                  * parent and child to be set to read.
3484                                  */
3485                                 make_migration_entry_read(&swp_entry);
3486                                 entry = swp_entry_to_pte(swp_entry);
3487                                 set_huge_swap_pte_at(src, addr, src_pte,
3488                                                      entry, sz);
3489                         }
3490                         set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz);
3491                 } else {
3492                         if (cow) {
3493                                 /*
3494                                  * No need to notify as we are downgrading page
3495                                  * table protection not changing it to point
3496                                  * to a new page.
3497                                  *
3498                                  * See Documentation/vm/mmu_notifier.rst
3499                                  */
3500                                 huge_ptep_set_wrprotect(src, addr, src_pte);
3501                         }
3502                         entry = huge_ptep_get(src_pte);
3503                         ptepage = pte_page(entry);
3504                         get_page(ptepage);
3505                         page_dup_rmap(ptepage, true);
3506                         set_huge_pte_at(dst, addr, dst_pte, entry);
3507                         hugetlb_count_add(pages_per_huge_page(h), dst);
3508                 }
3509                 spin_unlock(src_ptl);
3510                 spin_unlock(dst_ptl);
3511         }
3512 
3513         if (cow)
3514                 mmu_notifier_invalidate_range_end(&range);
3515 
3516         return ret;
3517 }
3518 
3519 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3520                             unsigned long start, unsigned long end,
3521                             struct page *ref_page)
3522 {
3523         struct mm_struct *mm = vma->vm_mm;
3524         unsigned long address;
3525         pte_t *ptep;
3526         pte_t pte;
3527         spinlock_t *ptl;
3528         struct page *page;
3529         struct hstate *h = hstate_vma(vma);
3530         unsigned long sz = huge_page_size(h);
3531         struct mmu_notifier_range range;
3532 
3533         WARN_ON(!is_vm_hugetlb_page(vma));
3534         BUG_ON(start & ~huge_page_mask(h));
3535         BUG_ON(end & ~huge_page_mask(h));
3536 
3537         /*
3538          * This is a hugetlb vma, all the pte entries should point
3539          * to huge page.
3540          */
3541         tlb_change_page_size(tlb, sz);
3542         tlb_start_vma(tlb, vma);
3543 
3544         /*
3545          * If sharing possible, alert mmu notifiers of worst case.
3546          */
3547         mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start,
3548                                 end);
3549         adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
3550         mmu_notifier_invalidate_range_start(&range);
3551         address = start;
3552         for (; address < end; address += sz) {
3553                 ptep = huge_pte_offset(mm, address, sz);
3554                 if (!ptep)
3555                         continue;
3556 
3557                 ptl = huge_pte_lock(h, mm, ptep);
3558                 if (huge_pmd_unshare(mm, &address, ptep)) {
3559                         spin_unlock(ptl);
3560                         /*
3561                          * We just unmapped a page of PMDs by clearing a PUD.
3562                          * The caller's TLB flush range should cover this area.
3563                          */
3564                         continue;
3565                 }
3566 
3567                 pte = huge_ptep_get(ptep);
3568                 if (huge_pte_none(pte)) {
3569                         spin_unlock(ptl);
3570                         continue;
3571                 }
3572 
3573                 /*
3574                  * Migrating hugepage or HWPoisoned hugepage is already
3575                  * unmapped and its refcount is dropped, so just clear pte here.
3576                  */
3577                 if (unlikely(!pte_present(pte))) {
3578                         huge_pte_clear(mm, address, ptep, sz);
3579                         spin_unlock(ptl);
3580                         continue;
3581                 }
3582 
3583                 page = pte_page(pte);
3584                 /*
3585                  * If a reference page is supplied, it is because a specific
3586                  * page is being unmapped, not a range. Ensure the page we
3587                  * are about to unmap is the actual page of interest.
3588                  */
3589                 if (ref_page) {
3590                         if (page != ref_page) {
3591                                 spin_unlock(ptl);
3592                                 continue;
3593                         }
3594                         /*
3595                          * Mark the VMA as having unmapped its page so that
3596                          * future faults in this VMA will fail rather than
3597                          * looking like data was lost
3598                          */
3599                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3600                 }
3601 
3602                 pte = huge_ptep_get_and_clear(mm, address, ptep);
3603                 tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
3604                 if (huge_pte_dirty(pte))
3605                         set_page_dirty(page);
3606 
3607                 hugetlb_count_sub(pages_per_huge_page(h), mm);
3608                 page_remove_rmap(page, true);
3609 
3610                 spin_unlock(ptl);
3611                 tlb_remove_page_size(tlb, page, huge_page_size(h));
3612                 /*
3613                  * Bail out after unmapping reference page if supplied
3614                  */
3615                 if (ref_page)
3616                         break;
3617         }
3618         mmu_notifier_invalidate_range_end(&range);
3619         tlb_end_vma(tlb, vma);
3620 }
3621 
3622 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3623                           struct vm_area_struct *vma, unsigned long start,
3624                           unsigned long end, struct page *ref_page)
3625 {
3626         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
3627 
3628         /*
3629          * Clear this flag so that x86's huge_pmd_share page_table_shareable
3630          * test will fail on a vma being torn down, and not grab a page table
3631          * on its way out.  We're lucky that the flag has such an appropriate
3632          * name, and can in fact be safely cleared here. We could clear it
3633          * before the __unmap_hugepage_range above, but all that's necessary
3634          * is to clear it before releasing the i_mmap_rwsem. This works
3635          * because in the context this is called, the VMA is about to be
3636          * destroyed and the i_mmap_rwsem is held.
3637          */
3638         vma->vm_flags &= ~VM_MAYSHARE;
3639 }
3640 
3641 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3642                           unsigned long end, struct page *ref_page)
3643 {
3644         struct mm_struct *mm;
3645         struct mmu_gather tlb;
3646         unsigned long tlb_start = start;
3647         unsigned long tlb_end = end;
3648 
3649         /*
3650          * If shared PMDs were possibly used within this vma range, adjust
3651          * start/end for worst case tlb flushing.
3652          * Note that we can not be sure if PMDs are shared until we try to
3653          * unmap pages.  However, we want to make sure TLB flushing covers
3654          * the largest possible range.
3655          */
3656         adjust_range_if_pmd_sharing_possible(vma, &tlb_start, &tlb_end);
3657 
3658         mm = vma->vm_mm;
3659 
3660         tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end);
3661         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3662         tlb_finish_mmu(&tlb, tlb_start, tlb_end);
3663 }
3664 
3665 /*
3666  * This is called when the original mapper is failing to COW a MAP_PRIVATE
3667  * mappping it owns the reserve page for. The intention is to unmap the page
3668  * from other VMAs and let the children be SIGKILLed if they are faulting the
3669  * same region.
3670  */
3671 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3672                               struct page *page, unsigned long address)
3673 {
3674         struct hstate *h = hstate_vma(vma);
3675         struct vm_area_struct *iter_vma;
3676         struct address_space *mapping;
3677         pgoff_t pgoff;
3678 
3679         /*
3680          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3681          * from page cache lookup which is in HPAGE_SIZE units.
3682          */
3683         address = address & huge_page_mask(h);
3684         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3685                         vma->vm_pgoff;
3686         mapping = vma->vm_file->f_mapping;
3687 
3688         /*
3689          * Take the mapping lock for the duration of the table walk. As
3690          * this mapping should be shared between all the VMAs,
3691          * __unmap_hugepage_range() is called as the lock is already held
3692          */
3693         i_mmap_lock_write(mapping);
3694         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3695                 /* Do not unmap the current VMA */
3696                 if (iter_vma == vma)
3697                         continue;
3698 
3699                 /*
3700                  * Shared VMAs have their own reserves and do not affect
3701                  * MAP_PRIVATE accounting but it is possible that a shared
3702                  * VMA is using the same page so check and skip such VMAs.
3703                  */
3704                 if (iter_vma->vm_flags & VM_MAYSHARE)
3705                         continue;
3706 
3707                 /*
3708                  * Unmap the page from other VMAs without their own reserves.
3709                  * They get marked to be SIGKILLed if they fault in these
3710                  * areas. This is because a future no-page fault on this VMA
3711                  * could insert a zeroed page instead of the data existing
3712                  * from the time of fork. This would look like data corruption
3713                  */
3714                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3715                         unmap_hugepage_range(iter_vma, address,
3716                                              address + huge_page_size(h), page);
3717         }
3718         i_mmap_unlock_write(mapping);
3719 }
3720 
3721 /*
3722  * Hugetlb_cow() should be called with page lock of the original hugepage held.
3723  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3724  * cannot race with other handlers or page migration.
3725  * Keep the pte_same checks anyway to make transition from the mutex easier.
3726  */
3727 static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3728                        unsigned long address, pte_t *ptep,
3729                        struct page *pagecache_page, spinlock_t *ptl)
3730 {
3731         pte_t pte;
3732         struct hstate *h = hstate_vma(vma);
3733         struct page *old_page, *new_page;
3734         int outside_reserve = 0;
3735         vm_fault_t ret = 0;
3736         unsigned long haddr = address & huge_page_mask(h);
3737         struct mmu_notifier_range range;
3738 
3739         pte = huge_ptep_get(ptep);
3740         old_page = pte_page(pte);
3741 
3742 retry_avoidcopy:
3743         /* If no-one else is actually using this page, avoid the copy
3744          * and just make the page writable */
3745         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3746                 page_move_anon_rmap(old_page, vma);
3747                 set_huge_ptep_writable(vma, haddr, ptep);
3748                 return 0;
3749         }
3750 
3751         /*
3752          * If the process that created a MAP_PRIVATE mapping is about to
3753          * perform a COW due to a shared page count, attempt to satisfy
3754          * the allocation without using the existing reserves. The pagecache
3755          * page is used to determine if the reserve at this address was
3756          * consumed or not. If reserves were used, a partial faulted mapping
3757          * at the time of fork() could consume its reserves on COW instead
3758          * of the full address range.
3759          */
3760         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3761                         old_page != pagecache_page)
3762                 outside_reserve = 1;
3763 
3764         get_page(old_page);
3765 
3766         /*
3767          * Drop page table lock as buddy allocator may be called. It will
3768          * be acquired again before returning to the caller, as expected.
3769          */
3770         spin_unlock(ptl);
3771         new_page = alloc_huge_page(vma, haddr, outside_reserve);
3772 
3773         if (IS_ERR(new_page)) {
3774                 /*
3775                  * If a process owning a MAP_PRIVATE mapping fails to COW,
3776                  * it is due to references held by a child and an insufficient
3777                  * huge page pool. To guarantee the original mappers
3778                  * reliability, unmap the page from child processes. The child
3779                  * may get SIGKILLed if it later faults.
3780                  */
3781                 if (outside_reserve) {
3782                         put_page(old_page);
3783                         BUG_ON(huge_pte_none(pte));
3784                         unmap_ref_private(mm, vma, old_page, haddr);
3785                         BUG_ON(huge_pte_none(pte));
3786                         spin_lock(ptl);
3787                         ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
3788                         if (likely(ptep &&
3789                                    pte_same(huge_ptep_get(ptep), pte)))
3790                                 goto retry_avoidcopy;
3791                         /*
3792                          * race occurs while re-acquiring page table
3793                          * lock, and our job is done.
3794                          */
3795                         return 0;
3796                 }
3797 
3798                 ret = vmf_error(PTR_ERR(new_page));
3799                 goto out_release_old;
3800         }
3801 
3802         /*
3803          * When the original hugepage is shared one, it does not have
3804          * anon_vma prepared.
3805          */
3806         if (unlikely(anon_vma_prepare(vma))) {
3807                 ret = VM_FAULT_OOM;
3808                 goto out_release_all;
3809         }
3810 
3811         copy_user_huge_page(new_page, old_page, address, vma,
3812                             pages_per_huge_page(h));
3813         __SetPageUptodate(new_page);
3814 
3815         mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr,
3816                                 haddr + huge_page_size(h));
3817         mmu_notifier_invalidate_range_start(&range);
3818 
3819         /*
3820          * Retake the page table lock to check for racing updates
3821          * before the page tables are altered
3822          */
3823         spin_lock(ptl);
3824         ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
3825         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3826                 ClearPagePrivate(new_page);
3827 
3828                 /* Break COW */
3829                 huge_ptep_clear_flush(vma, haddr, ptep);
3830                 mmu_notifier_invalidate_range(mm, range.start, range.end);
3831                 set_huge_pte_at(mm, haddr, ptep,
3832                                 make_huge_pte(vma, new_page, 1));
3833                 page_remove_rmap(old_page, true);
3834                 hugepage_add_new_anon_rmap(new_page, vma, haddr);
3835                 set_page_huge_active(new_page);
3836                 /* Make the old page be freed below */
3837                 new_page = old_page;
3838         }
3839         spin_unlock(ptl);
3840         mmu_notifier_invalidate_range_end(&range);
3841 out_release_all:
3842         restore_reserve_on_error(h, vma, haddr, new_page);
3843         put_page(new_page);
3844 out_release_old:
3845         put_page(old_page);
3846 
3847         spin_lock(ptl); /* Caller expects lock to be held */
3848         return ret;
3849 }
3850 
3851 /* Return the pagecache page at a given address within a VMA */
3852 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3853                         struct vm_area_struct *vma, unsigned long address)
3854 {
3855         struct address_space *mapping;
3856         pgoff_t idx;
3857 
3858         mapping = vma->vm_file->f_mapping;
3859         idx = vma_hugecache_offset(h, vma, address);
3860 
3861         return find_lock_page(mapping, idx);
3862 }
3863 
3864 /*
3865  * Return whether there is a pagecache page to back given address within VMA.
3866  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3867  */
3868 static bool hugetlbfs_pagecache_present(struct hstate *h,
3869                         struct vm_area_struct *vma, unsigned long address)
3870 {
3871         struct address_space *mapping;
3872         pgoff_t idx;
3873         struct page *page;
3874 
3875         mapping = vma->vm_file->f_mapping;
3876         idx = vma_hugecache_offset(h, vma, address);
3877 
3878         page = find_get_page(mapping, idx);
3879         if (page)
3880                 put_page(page);
3881         return page != NULL;
3882 }
3883 
3884 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3885                            pgoff_t idx)
3886 {
3887         struct inode *inode = mapping->host;
3888         struct hstate *h = hstate_inode(inode);
3889         int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3890 
3891         if (err)
3892                 return err;
3893         ClearPagePrivate(page);
3894 
3895         /*
3896          * set page dirty so that it will not be removed from cache/file
3897          * by non-hugetlbfs specific code paths.
3898          */
3899         set_page_dirty(page);
3900 
3901         spin_lock(&inode->i_lock);
3902         inode->i_blocks += blocks_per_huge_page(h);
3903         spin_unlock(&inode->i_lock);
3904         return 0;
3905 }
3906 
3907 static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
3908                         struct vm_area_struct *vma,
3909                         struct address_space *mapping, pgoff_t idx,
3910                         unsigned long address, pte_t *ptep, unsigned int flags)
3911 {
3912         struct hstate *h = hstate_vma(vma);
3913         vm_fault_t ret = VM_FAULT_SIGBUS;
3914         int anon_rmap = 0;
3915         unsigned long size;
3916         struct page *page;
3917         pte_t new_pte;
3918         spinlock_t *ptl;
3919         unsigned long haddr = address & huge_page_mask(h);
3920         bool new_page = false;
3921 
3922         /*
3923          * Currently, we are forced to kill the process in the event the
3924          * original mapper has unmapped pages from the child due to a failed
3925          * COW. Warn that such a situation has occurred as it may not be obvious
3926          */
3927         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3928                 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3929                            current->pid);
3930                 return ret;
3931         }
3932 
3933         /*
3934          * Use page lock to guard against racing truncation
3935          * before we get page_table_lock.
3936          */
3937 retry:
3938         page = find_lock_page(mapping, idx);
3939         if (!page) {
3940                 size = i_size_read(mapping->host) >> huge_page_shift(h);
3941                 if (idx >= size)
3942                         goto out;
3943 
3944                 /*
3945                  * Check for page in userfault range
3946                  */
3947                 if (userfaultfd_missing(vma)) {
3948                         u32 hash;
3949                         struct vm_fault vmf = {
3950                                 .vma = vma,
3951                                 .address = haddr,
3952                                 .flags = flags,
3953                                 /*
3954                                  * Hard to debug if it ends up being
3955                                  * used by a callee that assumes
3956                                  * something about the other
3957                                  * uninitialized fields... same as in
3958                                  * memory.c
3959                                  */
3960                         };
3961 
3962                         /*
3963                          * hugetlb_fault_mutex must be dropped before
3964                          * handling userfault.  Reacquire after handling
3965                          * fault to make calling code simpler.
3966                          */
3967                         hash = hugetlb_fault_mutex_hash(h, mapping, idx, haddr);
3968                         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3969                         ret = handle_userfault(&vmf, VM_UFFD_MISSING);
3970                         mutex_lock(&hugetlb_fault_mutex_table[hash]);
3971                         goto out;
3972                 }
3973 
3974                 page = alloc_huge_page(vma, haddr, 0);
3975                 if (IS_ERR(page)) {
3976                         /*
3977                          * Returning error will result in faulting task being
3978                          * sent SIGBUS.  The hugetlb fault mutex prevents two
3979                          * tasks from racing to fault in the same page which
3980                          * could result in false unable to allocate errors.
3981                          * Page migration does not take the fault mutex, but
3982                          * does a clear then write of pte's under page table
3983                          * lock.  Page fault code could race with migration,
3984                          * notice the clear pte and try to allocate a page
3985                          * here.  Before returning error, get ptl and make
3986                          * sure there really is no pte entry.
3987                          */
3988                         ptl = huge_pte_lock(h, mm, ptep);
3989                         if (!huge_pte_none(huge_ptep_get(ptep))) {
3990                                 ret = 0;
3991                                 spin_unlock(ptl);
3992                                 goto out;
3993                         }
3994                         spin_unlock(ptl);
3995                         ret = vmf_error(PTR_ERR(page));
3996                         goto out;
3997                 }
3998                 clear_huge_page(page, address, pages_per_huge_page(h));
3999                 __SetPageUptodate(page);
4000                 new_page = true;
4001 
4002                 if (vma->vm_flags & VM_MAYSHARE) {
4003                         int err = huge_add_to_page_cache(page, mapping, idx);
4004                         if (err) {
4005                                 put_page(page);
4006                                 if (err == -EEXIST)
4007                                         goto retry;
4008                                 goto out;
4009                         }
4010                 } else {
4011                         lock_page(page);
4012                         if (unlikely(anon_vma_prepare(vma))) {
4013                                 ret = VM_FAULT_OOM;
4014                                 goto backout_unlocked;
4015                         }
4016                         anon_rmap = 1;
4017                 }
4018         } else {
4019                 /*
4020                  * If memory error occurs between mmap() and fault, some process
4021                  * don't have hwpoisoned swap entry for errored virtual address.
4022                  * So we need to block hugepage fault by PG_hwpoison bit check.
4023                  */
4024                 if (unlikely(PageHWPoison(page))) {
4025                         ret = VM_FAULT_HWPOISON |
4026                                 VM_FAULT_SET_HINDEX(hstate_index(h));
4027                         goto backout_unlocked;
4028                 }
4029         }
4030 
4031         /*
4032          * If we are going to COW a private mapping later, we examine the
4033          * pending reservations for this page now. This will ensure that
4034          * any allocations necessary to record that reservation occur outside
4035          * the spinlock.
4036          */
4037         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4038                 if (vma_needs_reservation(h, vma, haddr) < 0) {
4039                         ret = VM_FAULT_OOM;
4040                         goto backout_unlocked;
4041                 }
4042                 /* Just decrements count, does not deallocate */
4043                 vma_end_reservation(h, vma, haddr);
4044         }
4045 
4046         ptl = huge_pte_lock(h, mm, ptep);
4047         size = i_size_read(mapping->host) >> huge_page_shift(h);
4048         if (idx >= size)
4049                 goto backout;
4050 
4051         ret = 0;
4052         if (!huge_pte_none(huge_ptep_get(ptep)))
4053                 goto backout;
4054 
4055         if (anon_rmap) {
4056                 ClearPagePrivate(page);
4057                 hugepage_add_new_anon_rmap(page, vma, haddr);
4058         } else
4059                 page_dup_rmap(page, true);
4060         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
4061                                 && (vma->vm_flags & VM_SHARED)));
4062         set_huge_pte_at(mm, haddr, ptep, new_pte);
4063 
4064         hugetlb_count_add(pages_per_huge_page(h), mm);
4065         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4066                 /* Optimization, do the COW without a second fault */
4067                 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
4068         }
4069 
4070         spin_unlock(ptl);
4071 
4072         /*
4073          * Only make newly allocated pages active.  Existing pages found
4074          * in the pagecache could be !page_huge_active() if they have been
4075          * isolated for migration.
4076          */
4077         if (new_page)
4078                 set_page_huge_active(page);
4079 
4080         unlock_page(page);
4081 out:
4082         return ret;
4083 
4084 backout:
4085         spin_unlock(ptl);
4086 backout_unlocked:
4087         unlock_page(page);
4088         restore_reserve_on_error(h, vma, haddr, page);
4089         put_page(page);
4090         goto out;
4091 }
4092 
4093 #ifdef CONFIG_SMP
4094 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
4095                             pgoff_t idx, unsigned long address)
4096 {
4097         unsigned long key[2];
4098         u32 hash;
4099 
4100         key[0] = (unsigned long) mapping;
4101         key[1] = idx;
4102 
4103         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
4104 
4105         return hash & (num_fault_mutexes - 1);
4106 }
4107 #else
4108 /*
4109  * For uniprocesor systems we always use a single mutex, so just
4110  * return 0 and avoid the hashing overhead.
4111  */
4112 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
4113                             pgoff_t idx, unsigned long address)
4114 {
4115         return 0;
4116 }
4117 #endif
4118 
4119 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
4120                         unsigned long address, unsigned int flags)
4121 {
4122         pte_t *ptep, entry;
4123         spinlock_t *ptl;
4124         vm_fault_t ret;
4125         u32 hash;
4126         pgoff_t idx;
4127         struct page *page = NULL;
4128         struct page *pagecache_page = NULL;
4129         struct hstate *h = hstate_vma(vma);
4130         struct address_space *mapping;
4131         int need_wait_lock = 0;
4132         unsigned long haddr = address & huge_page_mask(h);
4133 
4134         ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4135         if (ptep) {
4136                 entry = huge_ptep_get(ptep);
4137                 if (unlikely(is_hugetlb_entry_migration(entry))) {
4138                         migration_entry_wait_huge(vma, mm, ptep);
4139                         return 0;
4140                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
4141                         return VM_FAULT_HWPOISON_LARGE |
4142                                 VM_FAULT_SET_HINDEX(hstate_index(h));
4143         } else {
4144                 ptep = huge_pte_alloc(mm, haddr, huge_page_size(h));
4145                 if (!ptep)
4146                         return VM_FAULT_OOM;
4147         }
4148 
4149         mapping = vma->vm_file->f_mapping;
4150         idx = vma_hugecache_offset(h, vma, haddr);
4151 
4152         /*
4153          * Serialize hugepage allocation and instantiation, so that we don't
4154          * get spurious allocation failures if two CPUs race to instantiate
4155          * the same page in the page cache.
4156          */
4157         hash = hugetlb_fault_mutex_hash(h, mapping, idx, haddr);
4158         mutex_lock(&hugetlb_fault_mutex_table[hash]);
4159 
4160         entry = huge_ptep_get(ptep);
4161         if (huge_pte_none(entry)) {
4162                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
4163                 goto out_mutex;
4164         }
4165 
4166         ret = 0;
4167 
4168         /*
4169          * entry could be a migration/hwpoison entry at this point, so this
4170          * check prevents the kernel from going below assuming that we have
4171          * a active hugepage in pagecache. This goto expects the 2nd page fault,
4172          * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
4173          * handle it.
4174          */
4175         if (!pte_present(entry))
4176                 goto out_mutex;
4177 
4178         /*
4179          * If we are going to COW the mapping later, we examine the pending
4180          * reservations for this page now. This will ensure that any
4181          * allocations necessary to record that reservation occur outside the
4182          * spinlock. For private mappings, we also lookup the pagecache
4183          * page now as it is used to determine if a reservation has been
4184          * consumed.
4185          */
4186         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
4187                 if (vma_needs_reservation(h, vma, haddr) < 0) {
4188                         ret = VM_FAULT_OOM;
4189                         goto out_mutex;
4190                 }
4191                 /* Just decrements count, does not deallocate */
4192                 vma_end_reservation(h, vma, haddr);
4193 
4194                 if (!(vma->vm_flags & VM_MAYSHARE))
4195                         pagecache_page = hugetlbfs_pagecache_page(h,
4196                                                                 vma, haddr);
4197         }
4198 
4199         ptl = huge_pte_lock(h, mm, ptep);
4200 
4201         /* Check for a racing update before calling hugetlb_cow */
4202         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
4203                 goto out_ptl;
4204 
4205         /*
4206          * hugetlb_cow() requires page locks of pte_page(entry) and
4207          * pagecache_page, so here we need take the former one
4208          * when page != pagecache_page or !pagecache_page.
4209          */
4210         page = pte_page(entry);
4211         if (page != pagecache_page)
4212                 if (!trylock_page(page)) {
4213                         need_wait_lock = 1;
4214                         goto out_ptl;
4215                 }
4216 
4217         get_page(page);
4218 
4219         if (flags & FAULT_FLAG_WRITE) {
4220                 if (!huge_pte_write(entry)) {
4221                         ret = hugetlb_cow(mm, vma, address, ptep,
4222                                           pagecache_page, ptl);
4223                         goto out_put_page;
4224                 }
4225                 entry = huge_pte_mkdirty(entry);
4226         }
4227         entry = pte_mkyoung(entry);
4228         if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
4229                                                 flags & FAULT_FLAG_WRITE))
4230                 update_mmu_cache(vma, haddr, ptep);
4231 out_put_page:
4232         if (page != pagecache_page)
4233                 unlock_page(page);
4234         put_page(page);
4235 out_ptl:
4236         spin_unlock(ptl);
4237 
4238         if (pagecache_page) {
4239                 unlock_page(pagecache_page);
4240                 put_page(pagecache_page);
4241         }
4242 out_mutex:
4243         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4244         /*
4245          * Generally it's safe to hold refcount during waiting page lock. But
4246          * here we just wait to defer the next page fault to avoid busy loop and
4247          * the page is not used after unlocked before returning from the current
4248          * page fault. So we are safe from accessing freed page, even if we wait
4249          * here without taking refcount.
4250          */
4251         if (need_wait_lock)
4252                 wait_on_page_locked(page);
4253         return ret;
4254 }
4255 
4256 /*
4257  * Used by userfaultfd UFFDIO_COPY.  Based on mcopy_atomic_pte with
4258  * modifications for huge pages.
4259  */
4260 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
4261                             pte_t *dst_pte,
4262                             struct vm_area_struct *dst_vma,
4263                             unsigned long dst_addr,
4264                             unsigned long src_addr,
4265                             struct page **pagep)
4266 {
4267         struct address_space *mapping;
4268         pgoff_t idx;
4269         unsigned long size;
4270         int vm_shared = dst_vma->vm_flags & VM_SHARED;
4271         struct hstate *h = hstate_vma(dst_vma);
4272         pte_t _dst_pte;
4273         spinlock_t *ptl;
4274         int ret;
4275         struct page *page;
4276 
4277         if (!*pagep) {
4278                 ret = -ENOMEM;
4279                 page = alloc_huge_page(dst_vma, dst_addr, 0);
4280                 if (IS_ERR(page))
4281                         goto out;
4282 
4283                 ret = copy_huge_page_from_user(page,
4284                                                 (const void __user *) src_addr,
4285                                                 pages_per_huge_page(h), false);
4286 
4287                 /* fallback to copy_from_user outside mmap_sem */
4288                 if (unlikely(ret)) {
4289                         ret = -ENOENT;
4290                         *pagep = page;
4291                         /* don't free the page */
4292                         goto out;
4293                 }
4294         } else {
4295                 page = *pagep;
4296                 *pagep = NULL;
4297         }
4298 
4299         /*
4300          * The memory barrier inside __SetPageUptodate makes sure that
4301          * preceding stores to the page contents become visible before
4302          * the set_pte_at() write.
4303          */
4304         __SetPageUptodate(page);
4305 
4306         mapping = dst_vma->vm_file->f_mapping;
4307         idx = vma_hugecache_offset(h, dst_vma, dst_addr);
4308 
4309         /*
4310          * If shared, add to page cache
4311          */
4312         if (vm_shared) {
4313                 size = i_size_read(mapping->host) >> huge_page_shift(h);
4314                 ret = -EFAULT;
4315                 if (idx >= size)
4316                         goto out_release_nounlock;
4317 
4318                 /*
4319                  * Serialization between remove_inode_hugepages() and
4320                  * huge_add_to_page_cache() below happens through the
4321                  * hugetlb_fault_mutex_table that here must be hold by
4322                  * the caller.
4323                  */
4324                 ret = huge_add_to_page_cache(page, mapping, idx);
4325                 if (ret)
4326                         goto out_release_nounlock;
4327         }
4328 
4329         ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
4330         spin_lock(ptl);
4331 
4332         /*
4333          * Recheck the i_size after holding PT lock to make sure not
4334          * to leave any page mapped (as page_mapped()) beyond the end
4335          * of the i_size (remove_inode_hugepages() is strict about
4336          * enforcing that). If we bail out here, we'll also leave a
4337          * page in the radix tree in the vm_shared case beyond the end
4338          * of the i_size, but remove_inode_hugepages() will take care
4339          * of it as soon as we drop the hugetlb_fault_mutex_table.
4340          */
4341         size = i_size_read(mapping->host) >> huge_page_shift(h);
4342         ret = -EFAULT;
4343         if (idx >= size)
4344                 goto out_release_unlock;
4345 
4346         ret = -EEXIST;
4347         if (!huge_pte_none(huge_ptep_get(dst_pte)))
4348                 goto out_release_unlock;
4349 
4350         if (vm_shared) {
4351                 page_dup_rmap(page, true);
4352         } else {
4353                 ClearPagePrivate(page);
4354                 hugepage_add_new_anon_rmap(page, dst_vma, dst_addr);
4355         }
4356 
4357         _dst_pte = make_huge_pte(dst_vma, page, dst_vma->vm_flags & VM_WRITE);
4358         if (dst_vma->vm_flags & VM_WRITE)
4359                 _dst_pte = huge_pte_mkdirty(_dst_pte);
4360         _dst_pte = pte_mkyoung(_dst_pte);
4361 
4362         set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
4363 
4364         (void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte,
4365                                         dst_vma->vm_flags & VM_WRITE);
4366         hugetlb_count_add(pages_per_huge_page(h), dst_mm);
4367 
4368         /* No need to invalidate - it was non-present before */
4369         update_mmu_cache(dst_vma, dst_addr, dst_pte);
4370 
4371         spin_unlock(ptl);
4372         set_page_huge_active(page);
4373         if (vm_shared)
4374                 unlock_page(page);
4375         ret = 0;
4376 out:
4377         return ret;
4378 out_release_unlock:
4379         spin_unlock(ptl);
4380         if (vm_shared)
4381                 unlock_page(page);
4382 out_release_nounlock:
4383         put_page(page);
4384         goto out;
4385 }
4386 
4387 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
4388                          struct page **pages, struct vm_area_struct **vmas,
4389                          unsigned long *position, unsigned long *nr_pages,
4390                          long i, unsigned int flags, int *nonblocking)
4391 {
4392         unsigned long pfn_offset;
4393         unsigned long vaddr = *position;
4394         unsigned long remainder = *nr_pages;
4395         struct hstate *h = hstate_vma(vma);
4396         int err = -EFAULT;
4397 
4398         while (vaddr < vma->vm_end && remainder) {
4399                 pte_t *pte;
4400                 spinlock_t *ptl = NULL;
4401                 int absent;
4402                 struct page *page;
4403 
4404                 /*
4405                  * If we have a pending SIGKILL, don't keep faulting pages and
4406                  * potentially allocating memory.
4407                  */
4408                 if (fatal_signal_pending(current)) {
4409                         remainder = 0;
4410                         break;
4411                 }
4412 
4413                 /*
4414                  * Some archs (sparc64, sh*) have multiple pte_ts to
4415                  * each hugepage.  We have to make sure we get the
4416                  * first, for the page indexing below to work.
4417                  *
4418                  * Note that page table lock is not held when pte is null.
4419                  */
4420                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h),
4421                                       huge_page_size(h));
4422                 if (pte)
4423                         ptl = huge_pte_lock(h, mm, pte);
4424                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
4425 
4426                 /*
4427                  * When coredumping, it suits get_dump_page if we just return
4428                  * an error where there's an empty slot with no huge pagecache
4429                  * to back it.  This way, we avoid allocating a hugepage, and
4430                  * the sparse dumpfile avoids allocating disk blocks, but its
4431                  * huge holes still show up with zeroes where they need to be.
4432                  */
4433                 if (absent && (flags & FOLL_DUMP) &&
4434                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
4435                         if (pte)
4436                                 spin_unlock(ptl);
4437                         remainder = 0;
4438                         break;
4439                 }
4440 
4441                 /*
4442                  * We need call hugetlb_fault for both hugepages under migration
4443                  * (in which case hugetlb_fault waits for the migration,) and
4444                  * hwpoisoned hugepages (in which case we need to prevent the
4445                  * caller from accessing to them.) In order to do this, we use
4446                  * here is_swap_pte instead of is_hugetlb_entry_migration and
4447                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
4448                  * both cases, and because we can't follow correct pages
4449                  * directly from any kind of swap entries.
4450                  */
4451                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
4452                     ((flags & FOLL_WRITE) &&
4453                       !huge_pte_write(huge_ptep_get(pte)))) {
4454                         vm_fault_t ret;
4455                         unsigned int fault_flags = 0;
4456 
4457                         if (pte)
4458                                 spin_unlock(ptl);
4459                         if (flags & FOLL_WRITE)
4460                                 fault_flags |= FAULT_FLAG_WRITE;
4461                         if (nonblocking)
4462                                 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
4463                         if (flags & FOLL_NOWAIT)
4464                                 fault_flags |= FAULT_FLAG_ALLOW_RETRY |
4465                                         FAULT_FLAG_RETRY_NOWAIT;
4466                         if (flags & FOLL_TRIED) {
4467                                 VM_WARN_ON_ONCE(fault_flags &
4468                                                 FAULT_FLAG_ALLOW_RETRY);
4469                                 fault_flags |= FAULT_FLAG_TRIED;
4470                         }
4471                         ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
4472                         if (ret & VM_FAULT_ERROR) {
4473                                 err = vm_fault_to_errno(ret, flags);
4474                                 remainder = 0;
4475                                 break;
4476                         }
4477                         if (ret & VM_FAULT_RETRY) {
4478                                 if (nonblocking &&
4479                                     !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
4480                                         *nonblocking = 0;
4481                                 *nr_pages = 0;
4482                                 /*
4483                                  * VM_FAULT_RETRY must not return an
4484                                  * error, it will return zero
4485                                  * instead.
4486                                  *
4487                                  * No need to update "position" as the
4488                                  * caller will not check it after
4489                                  * *nr_pages is set to 0.
4490                                  */
4491                                 return i;
4492                         }
4493                         continue;
4494                 }
4495 
4496                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
4497                 page = pte_page(huge_ptep_get(pte));
4498 
4499                 /*
4500                  * Instead of doing 'try_get_page()' below in the same_page
4501                  * loop, just check the count once here.
4502                  */
4503                 if (unlikely(page_count(page) <= 0)) {
4504                         if (pages) {
4505                                 spin_unlock(ptl);
4506                                 remainder = 0;
4507                                 err = -ENOMEM;
4508                                 break;
4509                         }
4510                 }
4511 same_page:
4512                 if (pages) {
4513                         pages[i] = mem_map_offset(page, pfn_offset);
4514                         get_page(pages[i]);
4515                 }
4516 
4517                 if (vmas)
4518                         vmas[i] = vma;
4519 
4520                 vaddr += PAGE_SIZE;
4521                 ++pfn_offset;
4522                 --remainder;
4523                 ++i;
4524                 if (vaddr < vma->vm_end && remainder &&
4525                                 pfn_offset < pages_per_huge_page(h)) {
4526                         /*
4527                          * We use pfn_offset to avoid touching the pageframes
4528                          * of this compound page.
4529                          */
4530                         goto same_page;
4531                 }
4532                 spin_unlock(ptl);
4533         }
4534         *nr_pages = remainder;
4535         /*
4536          * setting position is actually required only if remainder is
4537          * not zero but it's faster not to add a "if (remainder)"
4538          * branch.
4539          */
4540         *position = vaddr;
4541 
4542         return i ? i : err;
4543 }
4544 
4545 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
4546 /*
4547  * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
4548  * implement this.
4549  */
4550 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
4551 #endif
4552 
4553 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
4554                 unsigned long address, unsigned long end, pgprot_t newprot)
4555 {
4556         struct mm_struct *mm = vma->vm_mm;
4557         unsigned long start = address;
4558         pte_t *ptep;
4559         pte_t pte;
4560         struct hstate *h = hstate_vma(vma);
4561         unsigned long pages = 0;
4562         bool shared_pmd = false;
4563         struct mmu_notifier_range range;
4564 
4565         /*
4566          * In the case of shared PMDs, the area to flush could be beyond
4567          * start/end.  Set range.start/range.end to cover the maximum possible
4568          * range if PMD sharing is possible.
4569          */
4570         mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
4571                                 0, vma, mm, start, end);
4572         adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
4573 
4574         BUG_ON(address >= end);
4575         flush_cache_range(vma, range.start, range.end);
4576 
4577         mmu_notifier_invalidate_range_start(&range);
4578         i_mmap_lock_write(vma->vm_file->f_mapping);
4579         for (; address < end; address += huge_page_size(h)) {
4580                 spinlock_t *ptl;
4581                 ptep = huge_pte_offset(mm, address, huge_page_size(h));
4582                 if (!ptep)
4583                         continue;
4584                 ptl = huge_pte_lock(h, mm, ptep);
4585                 if (huge_pmd_unshare(mm, &address, ptep)) {
4586                         pages++;
4587                         spin_unlock(ptl);
4588                         shared_pmd = true;
4589                         continue;
4590                 }
4591                 pte = huge_ptep_get(ptep);
4592                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
4593                         spin_unlock(ptl);
4594                         continue;
4595                 }
4596                 if (unlikely(is_hugetlb_entry_migration(pte))) {
4597                         swp_entry_t entry = pte_to_swp_entry(pte);
4598 
4599                         if (is_write_migration_entry(entry)) {
4600                                 pte_t newpte;
4601 
4602                                 make_migration_entry_read(&entry);
4603                                 newpte = swp_entry_to_pte(entry);
4604                                 set_huge_swap_pte_at(mm, address, ptep,
4605                                                      newpte, huge_page_size(h));
4606                                 pages++;
4607                         }
4608                         spin_unlock(ptl);
4609                         continue;
4610                 }
4611                 if (!huge_pte_none(pte)) {
4612                         pte_t old_pte;
4613 
4614                         old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
4615                         pte = pte_mkhuge(huge_pte_modify(old_pte, newprot));
4616                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
4617                         huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
4618                         pages++;
4619                 }
4620                 spin_unlock(ptl);
4621         }
4622         /*
4623          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4624          * may have cleared our pud entry and done put_page on the page table:
4625          * once we release i_mmap_rwsem, another task can do the final put_page
4626          * and that page table be reused and filled with junk.  If we actually
4627          * did unshare a page of pmds, flush the range corresponding to the pud.
4628          */
4629         if (shared_pmd)
4630                 flush_hugetlb_tlb_range(vma, range.start, range.end);
4631         else
4632                 flush_hugetlb_tlb_range(vma, start, end);
4633         /*
4634          * No need to call mmu_notifier_invalidate_range() we are downgrading
4635          * page table protection not changing it to point to a new page.
4636          *
4637          * See Documentation/vm/mmu_notifier.rst
4638          */
4639         i_mmap_unlock_write(vma->vm_file->f_mapping);
4640         mmu_notifier_invalidate_range_end(&range);
4641 
4642         return pages << h->order;
4643 }
4644 
4645 int hugetlb_reserve_pages(struct inode *inode,
4646                                         long from, long to,
4647                                         struct vm_area_struct *vma,
4648                                         vm_flags_t vm_flags)
4649 {
4650         long ret, chg;
4651         struct hstate *h = hstate_inode(inode);
4652         struct hugepage_subpool *spool = subpool_inode(inode);
4653         struct resv_map *resv_map;
4654         long gbl_reserve;
4655 
4656         /* This should never happen */
4657         if (from > to) {
4658                 VM_WARN(1, "%s called with a negative range\n", __func__);
4659                 return -EINVAL;
4660         }
4661 
4662         /*
4663          * Only apply hugepage reservation if asked. At fault time, an
4664          * attempt will be made for VM_NORESERVE to allocate a page
4665          * without using reserves
4666          */
4667         if (vm_flags & VM_NORESERVE)
4668                 return 0;
4669 
4670         /*
4671          * Shared mappings base their reservation on the number of pages that
4672          * are already allocated on behalf of the file. Private mappings need
4673          * to reserve the full area even if read-only as mprotect() may be
4674          * called to make the mapping read-write. Assume !vma is a shm mapping
4675          */
4676         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4677                 /*
4678                  * resv_map can not be NULL as hugetlb_reserve_pages is only
4679                  * called for inodes for which resv_maps were created (see
4680                  * hugetlbfs_get_inode).
4681                  */
4682                 resv_map = inode_resv_map(inode);
4683 
4684                 chg = region_chg(resv_map, from, to);
4685 
4686         } else {
4687                 resv_map = resv_map_alloc();
4688                 if (!resv_map)
4689                         return -ENOMEM;
4690 
4691                 chg = to - from;
4692 
4693                 set_vma_resv_map(vma, resv_map);
4694                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4695         }
4696 
4697         if (chg < 0) {
4698                 ret = chg;
4699                 goto out_err;
4700         }
4701 
4702         /*
4703          * There must be enough pages in the subpool for the mapping. If
4704          * the subpool has a minimum size, there may be some global
4705          * reservations already in place (gbl_reserve).
4706          */
4707         gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4708         if (gbl_reserve < 0) {
4709                 ret = -ENOSPC;
4710                 goto out_err;
4711         }
4712 
4713         /*
4714          * Check enough hugepages are available for the reservation.
4715          * Hand the pages back to the subpool if there are not
4716          */
4717         ret = hugetlb_acct_memory(h, gbl_reserve);
4718         if (ret < 0) {
4719                 /* put back original number of pages, chg */
4720                 (void)hugepage_subpool_put_pages(spool, chg);
4721                 goto out_err;
4722         }
4723 
4724         /*
4725          * Account for the reservations made. Shared mappings record regions
4726          * that have reservations as they are shared by multiple VMAs.
4727          * When the last VMA disappears, the region map says how much
4728          * the reservation was and the page cache tells how much of
4729          * the reservation was consumed. Private mappings are per-VMA and
4730          * only the consumed reservations are tracked. When the VMA
4731          * disappears, the original reservation is the VMA size and the
4732          * consumed reservations are stored in the map. Hence, nothing
4733          * else has to be done for private mappings here
4734          */
4735         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4736                 long add = region_add(resv_map, from, to);
4737 
4738                 if (unlikely(chg > add)) {
4739                         /*
4740                          * pages in this range were added to the reserve
4741                          * map between region_chg and region_add.  This
4742                          * indicates a race with alloc_huge_page.  Adjust
4743                          * the subpool and reserve counts modified above
4744                          * based on the difference.
4745                          */
4746                         long rsv_adjust;
4747 
4748                         rsv_adjust = hugepage_subpool_put_pages(spool,
4749                                                                 chg - add);
4750                         hugetlb_acct_memory(h, -rsv_adjust);
4751                 }
4752         }
4753         return 0;
4754 out_err:
4755         if (!vma || vma->vm_flags & VM_MAYSHARE)
4756                 /* Don't call region_abort if region_chg failed */
4757                 if (chg >= 0)
4758                         region_abort(resv_map, from, to);
4759         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4760                 kref_put(&resv_map->refs, resv_map_release);
4761         return ret;
4762 }
4763 
4764 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4765                                                                 long freed)
4766 {
4767         struct hstate *h = hstate_inode(inode);
4768         struct resv_map *resv_map = inode_resv_map(inode);
4769         long chg = 0;
4770         struct hugepage_subpool *spool = subpool_inode(inode);
4771         long gbl_reserve;
4772 
4773         /*
4774          * Since this routine can be called in the evict inode path for all
4775          * hugetlbfs inodes, resv_map could be NULL.
4776          */
4777         if (resv_map) {
4778                 chg = region_del(resv_map, start, end);
4779                 /*
4780                  * region_del() can fail in the rare case where a region
4781                  * must be split and another region descriptor can not be
4782                  * allocated.  If end == LONG_MAX, it will not fail.
4783                  */
4784                 if (chg < 0)
4785                         return chg;
4786         }
4787 
4788         spin_lock(&inode->i_lock);
4789         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4790         spin_unlock(&inode->i_lock);
4791 
4792         /*
4793          * If the subpool has a minimum size, the number of global
4794          * reservations to be released may be adjusted.
4795          */
4796         gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4797         hugetlb_acct_memory(h, -gbl_reserve);
4798 
4799         return 0;
4800 }
4801 
4802 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4803 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4804                                 struct vm_area_struct *vma,
4805                                 unsigned long addr, pgoff_t idx)
4806 {
4807         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4808                                 svma->vm_start;
4809         unsigned long sbase = saddr & PUD_MASK;
4810         unsigned long s_end = sbase + PUD_SIZE;
4811 
4812         /* Allow segments to share if only one is marked locked */
4813         unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4814         unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4815 
4816         /*
4817          * match the virtual addresses, permission and the alignment of the
4818          * page table page.
4819          */
4820         if (pmd_index(addr) != pmd_index(saddr) ||
4821             vm_flags != svm_flags ||
4822             sbase < svma->vm_start || svma->vm_end < s_end)
4823                 return 0;
4824 
4825         return saddr;
4826 }
4827 
4828 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4829 {
4830         unsigned long base = addr & PUD_MASK;
4831         unsigned long end = base + PUD_SIZE;
4832 
4833         /*
4834          * check on proper vm_flags and page table alignment
4835          */
4836         if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
4837                 return true;
4838         return false;
4839 }
4840 
4841 /*
4842  * Determine if start,end range within vma could be mapped by shared pmd.
4843  * If yes, adjust start and end to cover range associated with possible
4844  * shared pmd mappings.
4845  */
4846 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4847                                 unsigned long *start, unsigned long *end)
4848 {
4849         unsigned long check_addr = *start;
4850 
4851         if (!(vma->vm_flags & VM_MAYSHARE))
4852                 return;
4853 
4854         for (check_addr = *start; check_addr < *end; check_addr += PUD_SIZE) {
4855                 unsigned long a_start = check_addr & PUD_MASK;
4856                 unsigned long a_end = a_start + PUD_SIZE;
4857 
4858                 /*
4859                  * If sharing is possible, adjust start/end if necessary.
4860                  */
4861                 if (range_in_vma(vma, a_start, a_end)) {
4862                         if (a_start < *start)
4863                                 *start = a_start;
4864                         if (a_end > *end)
4865                                 *end = a_end;
4866                 }
4867         }
4868 }
4869 
4870 /*
4871  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4872  * and returns the corresponding pte. While this is not necessary for the
4873  * !shared pmd case because we can allocate the pmd later as well, it makes the
4874  * code much cleaner. pmd allocation is essential for the shared case because
4875  * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4876  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4877  * bad pmd for sharing.
4878  */
4879 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4880 {
4881         struct vm_area_struct *vma = find_vma(mm, addr);
4882         struct address_space *mapping = vma->vm_file->f_mapping;
4883         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4884                         vma->vm_pgoff;
4885         struct vm_area_struct *svma;
4886         unsigned long saddr;
4887         pte_t *spte = NULL;
4888         pte_t *pte;
4889         spinlock_t *ptl;
4890 
4891         if (!vma_shareable(vma, addr))
4892                 return (pte_t *)pmd_alloc(mm, pud, addr);
4893 
4894         i_mmap_lock_write(mapping);
4895         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4896                 if (svma == vma)
4897                         continue;
4898 
4899                 saddr = page_table_shareable(svma, vma, addr, idx);
4900                 if (saddr) {
4901                         spte = huge_pte_offset(svma->vm_mm, saddr,
4902                                                vma_mmu_pagesize(svma));
4903                         if (spte) {
4904                                 get_page(virt_to_page(spte));
4905                                 break;
4906                         }
4907                 }
4908         }
4909 
4910         if (!spte)
4911                 goto out;
4912 
4913         ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
4914         if (pud_none(*pud)) {
4915                 pud_populate(mm, pud,
4916                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
4917                 mm_inc_nr_pmds(mm);
4918         } else {
4919                 put_page(virt_to_page(spte));
4920         }
4921         spin_unlock(ptl);
4922 out:
4923         pte = (pte_t *)pmd_alloc(mm, pud, addr);
4924         i_mmap_unlock_write(mapping);
4925         return pte;
4926 }
4927 
4928 /*
4929  * unmap huge page backed by shared pte.
4930  *
4931  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
4932  * indicated by page_count > 1, unmap is achieved by clearing pud and
4933  * decrementing the ref count. If count == 1, the pte page is not shared.
4934  *
4935  * called with page table lock held.
4936  *
4937  * returns: 1 successfully unmapped a shared pte page
4938  *          0 the underlying pte page is not shared, or it is the last user
4939  */
4940 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4941 {
4942         pgd_t *pgd = pgd_offset(mm, *addr);
4943         p4d_t *p4d = p4d_offset(pgd, *addr);
4944         pud_t *pud = pud_offset(p4d, *addr);
4945 
4946         BUG_ON(page_count(virt_to_page(ptep)) == 0);
4947         if (page_count(virt_to_page(ptep)) == 1)
4948                 return 0;
4949 
4950         pud_clear(pud);
4951         put_page(virt_to_page(ptep));
4952         mm_dec_nr_pmds(mm);
4953         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4954         return 1;
4955 }
4956 #define want_pmd_share()        (1)
4957 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4958 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4959 {
4960         return NULL;
4961 }
4962 
4963 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4964 {
4965         return 0;
4966 }
4967 
4968 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4969                                 unsigned long *start, unsigned long *end)
4970 {
4971 }
4972 #define want_pmd_share()        (0)
4973 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4974 
4975 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4976 pte_t *huge_pte_alloc(struct mm_struct *mm,
4977                         unsigned long addr, unsigned long sz)
4978 {
4979         pgd_t *pgd;
4980         p4d_t *p4d;
4981         pud_t *pud;
4982         pte_t *pte = NULL;
4983 
4984         pgd = pgd_offset(mm, addr);
4985         p4d = p4d_alloc(mm, pgd, addr);
4986         if (!p4d)
4987                 return NULL;
4988         pud = pud_alloc(mm, p4d, addr);
4989         if (pud) {
4990                 if (sz == PUD_SIZE) {
4991                         pte = (pte_t *)pud;
4992                 } else {
4993                         BUG_ON(sz != PMD_SIZE);
4994                         if (want_pmd_share() && pud_none(*pud))
4995                                 pte = huge_pmd_share(mm, addr, pud);
4996                         else
4997                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4998                 }
4999         }
5000         BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
5001 
5002         return pte;
5003 }
5004 
5005 /*
5006  * huge_pte_offset() - Walk the page table to resolve the hugepage
5007  * entry at address @addr
5008  *
5009  * Return: Pointer to page table or swap entry (PUD or PMD) for
5010  * address @addr, or NULL if a p*d_none() entry is encountered and the
5011  * size @sz doesn't match the hugepage size at this level of the page
5012  * table.
5013  */
5014 pte_t *huge_pte_offset(struct mm_struct *mm,
5015                        unsigned long addr, unsigned long sz)
5016 {
5017         pgd_t *pgd;
5018         p4d_t *p4d;
5019         pud_t *pud, pud_entry;
5020         pmd_t *pmd, pmd_entry;
5021 
5022         pgd = pgd_offset(mm, addr);
5023         if (!pgd_present(*pgd))
5024                 return NULL;
5025         p4d = p4d_offset(pgd, addr);
5026         if (!p4d_present(*p4d))
5027                 return NULL;
5028 
5029         pud = pud_offset(p4d, addr);
5030         pud_entry = READ_ONCE(*pud);
5031         if (sz != PUD_SIZE && pud_none(pud_entry))
5032                 return NULL;
5033         /* hugepage or swap? */
5034         if (pud_huge(pud_entry) || !pud_present(pud_entry))
5035                 return (pte_t *)pud;
5036 
5037         pmd = pmd_offset(pud, addr);
5038         pmd_entry = READ_ONCE(*pmd);
5039         if (sz != PMD_SIZE && pmd_none(pmd_entry))
5040                 return NULL;
5041         /* hugepage or swap? */
5042         if (pmd_huge(pmd_entry) || !pmd_present(pmd_entry))
5043                 return (pte_t *)pmd;
5044 
5045         return NULL;
5046 }
5047 
5048 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
5049 
5050 /*
5051  * These functions are overwritable if your architecture needs its own
5052  * behavior.
5053  */
5054 struct page * __weak
5055 follow_huge_addr(struct mm_struct *mm, unsigned long address,
5056                               int write)
5057 {
5058         return ERR_PTR(-EINVAL);
5059 }
5060 
5061 struct page * __weak
5062 follow_huge_pd(struct vm_area_struct *vma,
5063                unsigned long address, hugepd_t hpd, int flags, int pdshift)
5064 {
5065         WARN(1, "hugepd follow called with no support for hugepage directory format\n");
5066         return NULL;
5067 }
5068 
5069 struct page * __weak
5070 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
5071                 pmd_t *pmd, int flags)
5072 {
5073         struct page *page = NULL;
5074         spinlock_t *ptl;
5075         pte_t pte;
5076 retry:
5077         ptl = pmd_lockptr(mm, pmd);
5078         spin_lock(ptl);
5079         /*
5080          * make sure that the address range covered by this pmd is not
5081          * unmapped from other threads.
5082          */
5083         if (!pmd_huge(*pmd))
5084                 goto out;
5085         pte = huge_ptep_get((pte_t *)pmd);
5086         if (pte_present(pte)) {
5087                 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
5088                 if (flags & FOLL_GET)
5089                         get_page(page);
5090         } else {
5091                 if (is_hugetlb_entry_migration(pte)) {
5092                         spin_unlock(ptl);
5093                         __migration_entry_wait(mm, (pte_t *)pmd, ptl);
5094                         goto retry;
5095                 }
5096                 /*
5097                  * hwpoisoned entry is treated as no_page_table in
5098                  * follow_page_mask().
5099                  */
5100         }
5101 out:
5102         spin_unlock(ptl);
5103         return page;
5104 }
5105 
5106 struct page * __weak
5107 follow_huge_pud(struct mm_struct *mm, unsigned long address,
5108                 pud_t *pud, int flags)
5109 {
5110         if (flags & FOLL_GET)
5111                 return NULL;
5112 
5113         return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
5114 }
5115 
5116 struct page * __weak
5117 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags)
5118 {
5119         if (flags & FOLL_GET)
5120                 return NULL;
5121 
5122         return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT);
5123 }
5124 
5125 bool isolate_huge_page(struct page *page, struct list_head *list)
5126 {
5127         bool ret = true;
5128 
5129         VM_BUG_ON_PAGE(!PageHead(page), page);
5130         spin_lock(&hugetlb_lock);
5131         if (!page_huge_active(page) || !get_page_unless_zero(page)) {
5132                 ret = false;
5133                 goto unlock;
5134         }
5135         clear_page_huge_active(page);
5136         list_move_tail(&page->lru, list);
5137 unlock:
5138         spin_unlock(&hugetlb_lock);
5139         return ret;
5140 }
5141 
5142 void putback_active_hugepage(struct page *page)
5143 {
5144         VM_BUG_ON_PAGE(!PageHead(page), page);
5145         spin_lock(&hugetlb_lock);
5146         set_page_huge_active(page);
5147         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
5148         spin_unlock(&hugetlb_lock);
5149         put_page(page);
5150 }
5151 
5152 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason)
5153 {
5154         struct hstate *h = page_hstate(oldpage);
5155 
5156         hugetlb_cgroup_migrate(oldpage, newpage);
5157         set_page_owner_migrate_reason(newpage, reason);
5158 
5159         /*
5160          * transfer temporary state of the new huge page. This is
5161          * reverse to other transitions because the newpage is going to
5162          * be final while the old one will be freed so it takes over
5163          * the temporary status.
5164          *
5165          * Also note that we have to transfer the per-node surplus state
5166          * here as well otherwise the global surplus count will not match
5167          * the per-node's.
5168          */
5169         if (PageHugeTemporary(newpage)) {
5170                 int old_nid = page_to_nid(oldpage);
5171                 int new_nid = page_to_nid(newpage);
5172 
5173                 SetPageHugeTemporary(oldpage);
5174                 ClearPageHugeTemporary(newpage);
5175 
5176                 spin_lock(&hugetlb_lock);
5177                 if (h->surplus_huge_pages_node[old_nid]) {
5178                         h->surplus_huge_pages_node[old_nid]--;
5179                         h->surplus_huge_pages_node[new_nid]++;
5180                 }
5181                 spin_unlock(&hugetlb_lock);
5182         }
5183 }
/* [<][>][^][v][top][bottom][index][help] */
root/mm/hugetlb.c

DEFINITIONS
