83 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))  argument
227 	struct page *cow_page;		/* Handler may choose to COW */
228 	struct page *page;		/* ->fault handlers should return a  member
296 	struct page *(*find_special_page)(struct vm_area_struct *vma,
303 #define page_private(page)		((page)->private)  argument
304 #define set_page_private(page, v)	((page)->private = (v))  argument
307 static inline void set_freepage_migratetype(struct page *page, int migratetype)  in set_freepage_migratetype()  argument
309 	page->index = migratetype;  in set_freepage_migratetype()
313 static inline int get_freepage_migratetype(struct page *page)  in get_freepage_migratetype()  argument
315 	return page->index;  in get_freepage_migratetype()
341 static inline int put_page_testzero(struct page *page)  in put_page_testzero()  argument
343 	VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);  in put_page_testzero()
344 	return atomic_dec_and_test(&page->_count);  in put_page_testzero()
353 static inline int get_page_unless_zero(struct page *page)  in get_page_unless_zero()  argument
355 	return atomic_inc_not_zero(&page->_count);  in get_page_unless_zero()
365 static inline int put_page_unless_one(struct page *page)  in put_page_unless_one()  argument
367 	return atomic_add_unless(&page->_count, -1, 1);  in put_page_unless_one()
374 struct page *vmalloc_to_page(const void *addr);
404 static inline void compound_lock(struct page *page)  in compound_lock()  argument
407 	VM_BUG_ON_PAGE(PageSlab(page), page);  in compound_lock()
408 	bit_spin_lock(PG_compound_lock, &page->flags);  in compound_lock()
412 static inline void compound_unlock(struct page *page)  in compound_unlock()  argument
415 	VM_BUG_ON_PAGE(PageSlab(page), page);  in compound_unlock()
416 	bit_spin_unlock(PG_compound_lock, &page->flags);  in compound_unlock()
420 static inline unsigned long compound_lock_irqsave(struct page *page)  in compound_lock_irqsave()  argument
425 	compound_lock(page);  in compound_lock_irqsave()
430 static inline void compound_unlock_irqrestore(struct page *page,  in compound_unlock_irqrestore()  argument
434 	compound_unlock(page);  in compound_unlock_irqrestore()
439 static inline struct page *compound_head_by_tail(struct page *tail)  in compound_head_by_tail()
441 	struct page *head = tail->first_page;  in compound_head_by_tail()
460 static inline struct page *compound_head(struct page *page)  in compound_head()  argument
462 	if (unlikely(PageTail(page)))  in compound_head()
463 		return compound_head_by_tail(page);  in compound_head()
464 	return page;  in compound_head()
472 static inline struct page *compound_head_fast(struct page *page)  in compound_head_fast()  argument
474 	if (unlikely(PageTail(page)))  in compound_head_fast()
475 		return page->first_page;  in compound_head_fast()
476 	return page;  in compound_head_fast()
484 static inline void page_mapcount_reset(struct page *page)  in page_mapcount_reset()  argument
486 	atomic_set(&(page)->_mapcount, -1);  in page_mapcount_reset()
489 static inline int page_mapcount(struct page *page)  in page_mapcount()  argument
491 	VM_BUG_ON_PAGE(PageSlab(page), page);  in page_mapcount()
492 	return atomic_read(&page->_mapcount) + 1;  in page_mapcount()
495 static inline int page_count(struct page *page)  in page_count()  argument
497 	return atomic_read(&compound_head(page)->_count);  in page_count()
500 static inline bool __compound_tail_refcounted(struct page *page)  in __compound_tail_refcounted()  argument
502 	return !PageSlab(page) && !PageHeadHuge(page);  in __compound_tail_refcounted()
513 static inline bool compound_tail_refcounted(struct page *page)  in compound_tail_refcounted()  argument
515 	VM_BUG_ON_PAGE(!PageHead(page), page);  in compound_tail_refcounted()
516 	return __compound_tail_refcounted(page);  in compound_tail_refcounted()
519 static inline void get_huge_page_tail(struct page *page)  in get_huge_page_tail()  argument
524 	VM_BUG_ON_PAGE(!PageTail(page), page);  in get_huge_page_tail()
525 	VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);  in get_huge_page_tail()
526 	VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);  in get_huge_page_tail()
527 	if (compound_tail_refcounted(page->first_page))  in get_huge_page_tail()
528 		atomic_inc(&page->_mapcount);  in get_huge_page_tail()
531 extern bool __get_page_tail(struct page *page);
533 static inline void get_page(struct page *page)  in get_page()  argument
535 	if (unlikely(PageTail(page)))  in get_page()
536 		if (likely(__get_page_tail(page)))  in get_page()
542 	VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);  in get_page()
543 	atomic_inc(&page->_count);  in get_page()
546 static inline struct page *virt_to_head_page(const void *x)  in virt_to_head_page()
548 	struct page *page = virt_to_page(x);  in virt_to_head_page()  local
556 	return compound_head_fast(page);  in virt_to_head_page()
563 static inline void init_page_count(struct page *page)  in init_page_count()  argument
565 	atomic_set(&page->_count, 1);  in init_page_count()
568 void put_page(struct page *page);
571 void split_page(struct page *page, unsigned int order);
572 int split_free_page(struct page *page);
580 static inline void set_compound_page_dtor(struct page *page,  in set_compound_page_dtor()  argument
583 	page[1].compound_dtor = dtor;  in set_compound_page_dtor()
586 static inline compound_page_dtor *get_compound_page_dtor(struct page *page)  in get_compound_page_dtor()  argument
588 	return page[1].compound_dtor;  in get_compound_page_dtor()
591 static inline unsigned int compound_order(struct page *page)  in compound_order()  argument
593 	if (!PageHead(page))  in compound_order()
595 	return page[1].compound_order;  in compound_order()
598 static inline void set_compound_order(struct page *page, unsigned long order)  in set_compound_order()  argument
600 	page[1].compound_order = order;  in set_compound_order()
618 		struct page *page, pte_t *pte, bool write, bool anon);
725 static inline enum zone_type page_zonenum(const struct page *page)  in page_zonenum()  argument
727 	return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;  in page_zonenum()
742 static inline int page_zone_id(struct page *page)  in page_zone_id()  argument
744 	return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;  in page_zone_id()
757 extern int page_to_nid(const struct page *page);
759 static inline int page_to_nid(const struct page *page)  in page_to_nid()  argument
761 	return (page->flags >> NODES_PGSHIFT) & NODES_MASK;  in page_to_nid()
803 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
805 	return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);  in page_cpupid_xchg_last()
808 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
810 	return page->_last_cpupid;  in page_cpupid_last()
812 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
814 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;  in page_cpupid_reset_last()
817 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
819 	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;  in page_cpupid_last()
822 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
824 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
828 	page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);  in page_cpupid_reset_last()
829 	page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;  in page_cpupid_reset_last()
833 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
835 	return page_to_nid(page); /* XXX */  in page_cpupid_xchg_last()
838 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
840 	return page_to_nid(page); /* XXX */  in page_cpupid_last()
868 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
878 static inline struct zone *page_zone(const struct page *page)  in page_zone()  argument
880 	return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];  in page_zone()
884 static inline void set_page_section(struct page *page, unsigned long section)  in set_page_section()  argument
886 	page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);  in set_page_section()
887 	page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;  in set_page_section()
890 static inline unsigned long page_to_section(const struct page *page)  in page_to_section()  argument
892 	return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;  in page_to_section()
896 static inline void set_page_zone(struct page *page, enum zone_type zone)  in set_page_zone()  argument
898 	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);  in set_page_zone()
899 	page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;  in set_page_zone()
902 static inline void set_page_node(struct page *page, unsigned long node)  in set_page_node()  argument
904 	page->flags &= ~(NODES_MASK << NODES_PGSHIFT);  in set_page_node()
905 	page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;  in set_page_node()
908 static inline void set_page_links(struct page *page, enum zone_type zone,  in set_page_links()  argument
911 	set_page_zone(page, zone);  in set_page_links()
912 	set_page_node(page, node);  in set_page_links()
914 	set_page_section(page, pfn_to_section_nr(pfn));  in set_page_links()
923 static __always_inline void *lowmem_page_address(const struct page *page)  in lowmem_page_address()  argument
925 	return __va(PFN_PHYS(page_to_pfn(page)));  in lowmem_page_address()
933 static inline void *page_address(const struct page *page)  in page_address()  argument
935 	return page->virtual;  in page_address()
937 static inline void set_page_address(struct page *page, void *address)  in set_page_address()  argument
939 	page->virtual = address;  in set_page_address()
945 void *page_address(const struct page *page);
946 void set_page_address(struct page *page, void *virtual);
951 #define page_address(page) lowmem_page_address(page)  argument
952 #define set_page_address(page, address)  do { } while(0)  argument
956 extern void *page_rmapping(struct page *page);
957 extern struct anon_vma *page_anon_vma(struct page *page);
958 extern struct address_space *page_mapping(struct page *page);
960 extern struct address_space *__page_file_mapping(struct page *);
963 struct address_space *page_file_mapping(struct page *page)  in page_file_mapping()  argument
965 	if (unlikely(PageSwapCache(page)))  in page_file_mapping()
966 		return __page_file_mapping(page);  in page_file_mapping()
968 	return page->mapping;  in page_file_mapping()
975 static inline pgoff_t page_index(struct page *page)  in page_index()  argument
977 	if (unlikely(PageSwapCache(page)))  in page_index()
978 		return page_private(page);  in page_index()
979 	return page->index;  in page_index()
982 extern pgoff_t __page_file_index(struct page *page);
988 static inline pgoff_t page_file_index(struct page *page)  in page_file_index()  argument
990 	if (unlikely(PageSwapCache(page)))  in page_file_index()
991 		return __page_file_index(page);  in page_file_index()
993 	return page->index;  in page_file_index()
999 static inline int page_mapped(struct page *page)  in page_mapped()  argument
1001 	return atomic_read(&(page)->_mapcount) >= 0;  in page_mapped()
1009 static inline bool page_is_pfmemalloc(struct page *page)  in page_is_pfmemalloc()  argument
1015 	return page->index == -1UL;  in page_is_pfmemalloc()
1022 static inline void set_page_pfmemalloc(struct page *page)  in set_page_pfmemalloc()  argument
1024 	page->index = -1UL;  in set_page_pfmemalloc()
1027 static inline void clear_page_pfmemalloc(struct page *page)  in clear_page_pfmemalloc()  argument
1029 	page->index = 0;  in clear_page_pfmemalloc()
1102 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1175 int truncate_inode_page(struct address_space *mapping, struct page *page);
1176 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1177 int invalidate_inode_page(struct page *page);
1209 		      unsigned int foll_flags, struct page **pages,
1213 		    int write, int force, struct page **pages,
1217 		    int write, int force, struct page **pages,
1221 			       int write, int force, struct page **pages,
1225 		    int write, int force, struct page **pages);
1227 			struct page **pages);
1230 			struct page **pages);
1231 int get_kernel_page(unsigned long start, int write, struct page **pages);
1232 struct page *get_dump_page(unsigned long addr);
1234 extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1235 extern void do_invalidatepage(struct page *page, unsigned int offset,
1238 int __set_page_dirty_nobuffers(struct page *page);
1239 int __set_page_dirty_no_writeback(struct page *page);
1241 				struct page *page);
1242 void account_page_dirtied(struct page *page, struct address_space *mapping);
1243 void account_page_cleaned(struct page *page, struct address_space *mapping);
1244 int set_page_dirty(struct page *page);
1245 int set_page_dirty_lock(struct page *page);
1246 int clear_page_dirty_for_io(struct page *page);
1296 			  struct page **pages);
1470 extern bool ptlock_alloc(struct page *page);
1471 extern void ptlock_free(struct page *page);
1473 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
1475 	return page->ptl;  in ptlock_ptr()
1482 static inline bool ptlock_alloc(struct page *page)  in ptlock_alloc()  argument
1487 static inline void ptlock_free(struct page *page)  in ptlock_free()  argument
1491 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
1493 	return &page->ptl;  in ptlock_ptr()
1502 static inline bool ptlock_init(struct page *page)  in ptlock_init()  argument
1512 	VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);  in ptlock_init()
1513 	if (!ptlock_alloc(page))  in ptlock_init()
1515 	spin_lock_init(ptlock_ptr(page));  in ptlock_init()
1520 static inline void pte_lock_deinit(struct page *page)  in pte_lock_deinit()  argument
1522 	page->mapping = NULL;  in pte_lock_deinit()
1523 	ptlock_free(page);  in pte_lock_deinit()
1535 static inline bool ptlock_init(struct page *page) { return true; }  in ptlock_init()  argument
1536 static inline void pte_lock_deinit(struct page *page) {}  in pte_lock_deinit()  argument
1545 static inline bool pgtable_page_ctor(struct page *page)  in pgtable_page_ctor()  argument
1547 	inc_zone_page_state(page, NR_PAGETABLE);  in pgtable_page_ctor()
1548 	return ptlock_init(page);  in pgtable_page_ctor()
1551 static inline void pgtable_page_dtor(struct page *page)  in pgtable_page_dtor()  argument
1553 	pte_lock_deinit(page);  in pgtable_page_dtor()
1554 	dec_zone_page_state(page, NR_PAGETABLE);  in pgtable_page_dtor()
1587 static struct page *pmd_to_page(pmd_t *pmd)  in pmd_to_page()
1598 static inline bool pgtable_pmd_page_ctor(struct page *page)  in pgtable_pmd_page_ctor()  argument
1601 	page->pmd_huge_pte = NULL;  in pgtable_pmd_page_ctor()
1603 	return ptlock_init(page);  in pgtable_pmd_page_ctor()
1606 static inline void pgtable_pmd_page_dtor(struct page *page)  in pgtable_pmd_page_dtor()  argument
1609 	VM_BUG_ON_PAGE(page->pmd_huge_pte, page);  in pgtable_pmd_page_dtor()
1611 	ptlock_free(page);  in pgtable_pmd_page_dtor()
1623 static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }  in pgtable_pmd_page_ctor()  argument
1624 static inline void pgtable_pmd_page_dtor(struct page *page) {}  in pgtable_pmd_page_dtor()  argument
1656 extern void free_highmem_page(struct page *page);
1659 extern void adjust_managed_page_count(struct page *page, long count);
1663 static inline void __free_reserved_page(struct page *page)  in __free_reserved_page()  argument
1665 	ClearPageReserved(page);  in __free_reserved_page()
1666 	init_page_count(page);  in __free_reserved_page()
1667 	__free_page(page);  in __free_reserved_page()
1670 static inline void free_reserved_page(struct page *page)  in free_reserved_page()  argument
1672 	__free_reserved_page(page);  in free_reserved_page()
1673 	adjust_managed_page_count(page, 1);  in free_reserved_page()
1676 static inline void mark_page_reserved(struct page *page)  in mark_page_reserved()  argument
1678 	SetPageReserved(page);  in mark_page_reserved()
1679 	adjust_managed_page_count(page, -1);  in mark_page_reserved()
1870 				   unsigned long flags, struct page **pages);
1943 int write_one_page(struct page *page, int wait);
1962 				struct page *pg,
2035 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2043 struct page *follow_page_mask(struct vm_area_struct *vma,
2047 static inline struct page *follow_page(struct vm_area_struct *vma,  in follow_page()
2085 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2093 kernel_map_pages(struct page *page, int numpages, int enable)  in kernel_map_pages()  argument
2098 	__kernel_map_pages(page, numpages, enable);  in kernel_map_pages()
2101 extern bool kernel_page_present(struct page *page);
2105 kernel_map_pages(struct page *page, int numpages, int enable) {}  in kernel_map_pages()  argument
2107 static inline bool kernel_page_present(struct page *page) { return true; }  in kernel_page_present()  argument
2145 void sparse_mem_maps_populate_node(struct page **map_map,
2151 struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2166 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2180 extern void shake_page(struct page *p, int access);
2182 extern int soft_offline_page(struct page *page, int flags);
2185 extern void clear_huge_page(struct page *page,
2188 extern void copy_user_huge_page(struct page *dst, struct page *src,
2210 static inline bool page_is_guard(struct page *page)  in page_is_guard()  argument
2217 	page_ext = lookup_page_ext(page);  in page_is_guard()
2223 static inline bool page_is_guard(struct page *page) { return false; }  in page_is_guard()  argument