85 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))  argument
235 	struct page *cow_page;		/* Handler may choose to COW */
236 	struct page *page;		/* ->fault handlers should return a  member
307 	struct page *(*find_special_page)(struct vm_area_struct *vma,
314 #define page_private(page)		((page)->private)  argument
315 #define set_page_private(page, v)	((page)->private = (v))  argument
340 static inline int put_page_testzero(struct page *page)  in put_page_testzero()  argument
342 	VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);  in put_page_testzero()
343 	return atomic_dec_and_test(&page->_count);  in put_page_testzero()
352 static inline int get_page_unless_zero(struct page *page)  in get_page_unless_zero()  argument
354 	return atomic_inc_not_zero(&page->_count);  in get_page_unless_zero()
368 struct page *vmalloc_to_page(const void *addr);
398 static inline void compound_lock(struct page *page)  in compound_lock()  argument
401 	VM_BUG_ON_PAGE(PageSlab(page), page);  in compound_lock()
402 	bit_spin_lock(PG_compound_lock, &page->flags);  in compound_lock()
406 static inline void compound_unlock(struct page *page)  in compound_unlock()  argument
409 	VM_BUG_ON_PAGE(PageSlab(page), page);  in compound_unlock()
410 	bit_spin_unlock(PG_compound_lock, &page->flags);  in compound_unlock()
414 static inline unsigned long compound_lock_irqsave(struct page *page)  in compound_lock_irqsave()  argument
419 	compound_lock(page);  in compound_lock_irqsave()
424 static inline void compound_unlock_irqrestore(struct page *page,  in compound_unlock_irqrestore()  argument
428 	compound_unlock(page);  in compound_unlock_irqrestore()
438 static inline void page_mapcount_reset(struct page *page)  in page_mapcount_reset()  argument
440 	atomic_set(&(page)->_mapcount, -1);  in page_mapcount_reset()
443 static inline int page_mapcount(struct page *page)  in page_mapcount()  argument
445 	VM_BUG_ON_PAGE(PageSlab(page), page);  in page_mapcount()
446 	return atomic_read(&page->_mapcount) + 1;  in page_mapcount()
449 static inline int page_count(struct page *page)  in page_count()  argument
451 	return atomic_read(&compound_head(page)->_count);  in page_count()
454 static inline bool __compound_tail_refcounted(struct page *page)  in __compound_tail_refcounted()  argument
456 	return PageAnon(page) && !PageSlab(page) && !PageHeadHuge(page);  in __compound_tail_refcounted()
467 static inline bool compound_tail_refcounted(struct page *page)  in compound_tail_refcounted()  argument
469 	VM_BUG_ON_PAGE(!PageHead(page), page);  in compound_tail_refcounted()
470 	return __compound_tail_refcounted(page);  in compound_tail_refcounted()
473 static inline void get_huge_page_tail(struct page *page)  in get_huge_page_tail()  argument
478 	VM_BUG_ON_PAGE(!PageTail(page), page);  in get_huge_page_tail()
479 	VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);  in get_huge_page_tail()
480 	VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);  in get_huge_page_tail()
481 	if (compound_tail_refcounted(compound_head(page)))  in get_huge_page_tail()
482 		atomic_inc(&page->_mapcount);  in get_huge_page_tail()
485 extern bool __get_page_tail(struct page *page);
487 static inline void get_page(struct page *page)  in get_page()  argument
489 	if (unlikely(PageTail(page)))  in get_page()
490 		if (likely(__get_page_tail(page)))  in get_page()
496 	VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);  in get_page()
497 	atomic_inc(&page->_count);  in get_page()
500 static inline struct page *virt_to_head_page(const void *x)  in virt_to_head_page()
502 	struct page *page = virt_to_page(x);  in virt_to_head_page()  local
504 	return compound_head(page);  in virt_to_head_page()
511 static inline void init_page_count(struct page *page)  in init_page_count()  argument
513 	atomic_set(&page->_count, 1);  in init_page_count()
516 void put_page(struct page *page);
519 void split_page(struct page *page, unsigned int order);
520 int split_free_page(struct page *page);
527 typedef void compound_page_dtor(struct page *);
540 static inline void set_compound_page_dtor(struct page *page,  in set_compound_page_dtor()  argument
543 	VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);  in set_compound_page_dtor()
544 	page[1].compound_dtor = compound_dtor;  in set_compound_page_dtor()
547 static inline compound_page_dtor *get_compound_page_dtor(struct page *page)  in get_compound_page_dtor()  argument
549 	VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);  in get_compound_page_dtor()
550 	return compound_page_dtors[page[1].compound_dtor];  in get_compound_page_dtor()
553 static inline unsigned int compound_order(struct page *page)  in compound_order()  argument
555 	if (!PageHead(page))  in compound_order()
557 	return page[1].compound_order;  in compound_order()
560 static inline void set_compound_order(struct page *page, unsigned int order)  in set_compound_order()  argument
562 	page[1].compound_order = order;  in set_compound_order()
580 		struct page *page, pte_t *pte, bool write, bool anon);
687 static inline enum zone_type page_zonenum(const struct page *page)  in page_zonenum()  argument
689 	return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;  in page_zonenum()
704 static inline int page_zone_id(struct page *page)  in page_zone_id()  argument
706 	return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;  in page_zone_id()
719 extern int page_to_nid(const struct page *page);
721 static inline int page_to_nid(const struct page *page)  in page_to_nid()  argument
723 	return (page->flags >> NODES_PGSHIFT) & NODES_MASK;  in page_to_nid()
765 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
767 	return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);  in page_cpupid_xchg_last()
770 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
772 	return page->_last_cpupid;  in page_cpupid_last()
774 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
776 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;  in page_cpupid_reset_last()
779 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
781 	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;  in page_cpupid_last()
784 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
786 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
790 	page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);  in page_cpupid_reset_last()
791 	page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;  in page_cpupid_reset_last()
795 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
797 	return page_to_nid(page); /* XXX */  in page_cpupid_xchg_last()
800 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
802 	return page_to_nid(page); /* XXX */  in page_cpupid_last()
830 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
840 static inline struct zone *page_zone(const struct page *page)  in page_zone()  argument
842 	return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];  in page_zone()
846 static inline void set_page_section(struct page *page, unsigned long section)  in set_page_section()  argument
848 	page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);  in set_page_section()
849 	page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;  in set_page_section()
852 static inline unsigned long page_to_section(const struct page *page)  in page_to_section()  argument
854 	return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;  in page_to_section()
858 static inline void set_page_zone(struct page *page, enum zone_type zone)  in set_page_zone()  argument
860 	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);  in set_page_zone()
861 	page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;  in set_page_zone()
864 static inline void set_page_node(struct page *page, unsigned long node)  in set_page_node()  argument
866 	page->flags &= ~(NODES_MASK << NODES_PGSHIFT);  in set_page_node()
867 	page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;  in set_page_node()
870 static inline void set_page_links(struct page *page, enum zone_type zone,  in set_page_links()  argument
873 	set_page_zone(page, zone);  in set_page_links()
874 	set_page_node(page, node);  in set_page_links()
876 	set_page_section(page, pfn_to_section_nr(pfn));  in set_page_links()
881 static inline struct mem_cgroup *page_memcg(struct page *page)  in page_memcg()  argument
883 	return page->mem_cgroup;  in page_memcg()
886 static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)  in set_page_memcg()  argument
888 	page->mem_cgroup = memcg;  in set_page_memcg()
891 static inline struct mem_cgroup *page_memcg(struct page *page)  in page_memcg()  argument
896 static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)  in set_page_memcg()  argument
906 static __always_inline void *lowmem_page_address(const struct page *page)  in lowmem_page_address()  argument
908 	return __va(PFN_PHYS(page_to_pfn(page)));  in lowmem_page_address()
916 static inline void *page_address(const struct page *page)  in page_address()  argument
918 	return page->virtual;  in page_address()
920 static inline void set_page_address(struct page *page, void *address)  in set_page_address()  argument
922 	page->virtual = address;  in set_page_address()
928 void *page_address(const struct page *page);
929 void set_page_address(struct page *page, void *virtual);
934 #define page_address(page) lowmem_page_address(page)  argument
935 #define set_page_address(page, address)  do { } while(0)  argument
939 extern void *page_rmapping(struct page *page);
940 extern struct anon_vma *page_anon_vma(struct page *page);
941 extern struct address_space *page_mapping(struct page *page);
943 extern struct address_space *__page_file_mapping(struct page *);
946 struct address_space *page_file_mapping(struct page *page)  in page_file_mapping()  argument
948 	if (unlikely(PageSwapCache(page)))  in page_file_mapping()
949 		return __page_file_mapping(page);  in page_file_mapping()
951 	return page->mapping;  in page_file_mapping()
958 static inline pgoff_t page_index(struct page *page)  in page_index()  argument
960 	if (unlikely(PageSwapCache(page)))  in page_index()
961 		return page_private(page);  in page_index()
962 	return page->index;  in page_index()
965 extern pgoff_t __page_file_index(struct page *page);
971 static inline pgoff_t page_file_index(struct page *page)  in page_file_index()  argument
973 	if (unlikely(PageSwapCache(page)))  in page_file_index()
974 		return __page_file_index(page);  in page_file_index()
976 	return page->index;  in page_file_index()
982 static inline int page_mapped(struct page *page)  in page_mapped()  argument
984 	return atomic_read(&(page)->_mapcount) >= 0;  in page_mapped()
992 static inline bool page_is_pfmemalloc(struct page *page)  in page_is_pfmemalloc()  argument
998 	return page->index == -1UL;  in page_is_pfmemalloc()
1005 static inline void set_page_pfmemalloc(struct page *page)  in set_page_pfmemalloc()  argument
1007 	page->index = -1UL;  in set_page_pfmemalloc()
1010 static inline void clear_page_pfmemalloc(struct page *page)  in clear_page_pfmemalloc()  argument
1012 	page->index = 0;  in clear_page_pfmemalloc()
1085 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1087 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1160 int truncate_inode_page(struct address_space *mapping, struct page *page);
1161 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1162 int invalidate_inode_page(struct page *page);
1194 		      unsigned int foll_flags, struct page **pages,
1198 		    int write, int force, struct page **pages,
1202 		    int write, int force, struct page **pages,
1206 			       int write, int force, struct page **pages,
1210 		    int write, int force, struct page **pages);
1212 			struct page **pages);
1238 static inline struct page **frame_vector_pages(struct frame_vector *vec)  in frame_vector_pages()
1246 	return (struct page **)(vec->ptrs);  in frame_vector_pages()
1258 			struct page **pages);
1259 int get_kernel_page(unsigned long start, int write, struct page **pages);
1260 struct page *get_dump_page(unsigned long addr);
1262 extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1263 extern void do_invalidatepage(struct page *page, unsigned int offset,
1266 int __set_page_dirty_nobuffers(struct page *page);
1267 int __set_page_dirty_no_writeback(struct page *page);
1269 				struct page *page);
1270 void account_page_dirtied(struct page *page, struct address_space *mapping,
1272 void account_page_cleaned(struct page *page, struct address_space *mapping,
1274 int set_page_dirty(struct page *page);
1275 int set_page_dirty_lock(struct page *page);
1276 void cancel_dirty_page(struct page *page);
1277 int clear_page_dirty_for_io(struct page *page);
1332 			  struct page **pages);
1506 extern bool ptlock_alloc(struct page *page);
1507 extern void ptlock_free(struct page *page);
1509 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
1511 	return page->ptl;  in ptlock_ptr()
1518 static inline bool ptlock_alloc(struct page *page)  in ptlock_alloc()  argument
1523 static inline void ptlock_free(struct page *page)  in ptlock_free()  argument
1527 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
1529 	return &page->ptl;  in ptlock_ptr()
1538 static inline bool ptlock_init(struct page *page)  in ptlock_init()  argument
1547 	VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);  in ptlock_init()
1548 	if (!ptlock_alloc(page))  in ptlock_init()
1550 	spin_lock_init(ptlock_ptr(page));  in ptlock_init()
1555 static inline void pte_lock_deinit(struct page *page)  in pte_lock_deinit()  argument
1557 	page->mapping = NULL;  in pte_lock_deinit()
1558 	ptlock_free(page);  in pte_lock_deinit()
1570 static inline bool ptlock_init(struct page *page) { return true; }  in ptlock_init()  argument
1571 static inline void pte_lock_deinit(struct page *page) {}  in pte_lock_deinit()  argument
1580 static inline bool pgtable_page_ctor(struct page *page)  in pgtable_page_ctor()  argument
1582 	if (!ptlock_init(page))  in pgtable_page_ctor()
1584 	inc_zone_page_state(page, NR_PAGETABLE);  in pgtable_page_ctor()
1588 static inline void pgtable_page_dtor(struct page *page)  in pgtable_page_dtor()  argument
1590 	pte_lock_deinit(page);  in pgtable_page_dtor()
1591 	dec_zone_page_state(page, NR_PAGETABLE);  in pgtable_page_dtor()
1624 static struct page *pmd_to_page(pmd_t *pmd)  in pmd_to_page()
1635 static inline bool pgtable_pmd_page_ctor(struct page *page)  in pgtable_pmd_page_ctor()  argument
1638 	page->pmd_huge_pte = NULL;  in pgtable_pmd_page_ctor()
1640 	return ptlock_init(page);  in pgtable_pmd_page_ctor()
1643 static inline void pgtable_pmd_page_dtor(struct page *page)  in pgtable_pmd_page_dtor()  argument
1646 	VM_BUG_ON_PAGE(page->pmd_huge_pte, page);  in pgtable_pmd_page_dtor()
1648 	ptlock_free(page);  in pgtable_pmd_page_dtor()
1660 static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }  in pgtable_pmd_page_ctor()  argument
1661 static inline void pgtable_pmd_page_dtor(struct page *page) {}  in pgtable_pmd_page_dtor()  argument
1693 extern void free_highmem_page(struct page *page);
1696 extern void adjust_managed_page_count(struct page *page, long count);
1702 static inline void __free_reserved_page(struct page *page)  in __free_reserved_page()  argument
1704 	ClearPageReserved(page);  in __free_reserved_page()
1705 	init_page_count(page);  in __free_reserved_page()
1706 	__free_page(page);  in __free_reserved_page()
1709 static inline void free_reserved_page(struct page *page)  in free_reserved_page()  argument
1711 	__free_reserved_page(page);  in free_reserved_page()
1712 	adjust_managed_page_count(page, 1);  in free_reserved_page()
1715 static inline void mark_page_reserved(struct page *page)  in mark_page_reserved()  argument
1717 	SetPageReserved(page);  in mark_page_reserved()
1718 	adjust_managed_page_count(page, -1);  in mark_page_reserved()
1911 				   unsigned long flags, struct page **pages);
1992 int write_one_page(struct page *page, int wait);
2011 				struct page *pg,
2082 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2090 struct page *follow_page_mask(struct vm_area_struct *vma,
2094 static inline struct page *follow_page(struct vm_area_struct *vma,  in follow_page()
2133 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2141 kernel_map_pages(struct page *page, int numpages, int enable)  in kernel_map_pages()  argument
2146 	__kernel_map_pages(page, numpages, enable);  in kernel_map_pages()
2149 extern bool kernel_page_present(struct page *page);
2153 kernel_map_pages(struct page *page, int numpages, int enable) {}  in kernel_map_pages()  argument
2155 static inline bool kernel_page_present(struct page *page) { return true; }  in kernel_page_present()  argument
2193 void sparse_mem_maps_populate_node(struct page **map_map,
2199 struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2214 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2226 extern int get_hwpoison_page(struct page *page);
2227 extern void put_hwpoison_page(struct page *page);
2230 extern void shake_page(struct page *p, int access);
2232 extern int soft_offline_page(struct page *page, int flags);
2269 extern void clear_huge_page(struct page *page,
2272 extern void copy_user_huge_page(struct page *dst, struct page *src,
2294 static inline bool page_is_guard(struct page *page)  in page_is_guard()  argument
2301 	page_ext = lookup_page_ext(page);  in page_is_guard()
2307 static inline bool page_is_guard(struct page *page) { return false; }  in page_is_guard()  argument