699 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)  in maybe_pmd_mkwrite()  argument
702 		pmd = pmd_mkwrite(pmd);  in maybe_pmd_mkwrite()
703 	return pmd;  in maybe_pmd_mkwrite()
716 					unsigned long address, pmd_t *pmd,  in __do_huge_pmd_anonymous_page()  argument
748 	ptl = pmd_lock(mm, pmd);  in __do_huge_pmd_anonymous_page()
749 	if (unlikely(!pmd_none(*pmd))) {  in __do_huge_pmd_anonymous_page()
776 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in __do_huge_pmd_anonymous_page()
777 		set_pmd_at(mm, haddr, pmd, entry);  in __do_huge_pmd_anonymous_page()
794 		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,  in set_huge_zero_page()  argument
798 	if (!pmd_none(*pmd))  in set_huge_zero_page()
802 	pgtable_trans_huge_deposit(mm, pmd, pgtable);  in set_huge_zero_page()
803 	set_pmd_at(mm, haddr, pmd, entry);  in set_huge_zero_page()
809 			       unsigned long address, pmd_t *pmd,  in do_huge_pmd_anonymous_page()  argument
838 		ptl = pmd_lock(mm, pmd);  in do_huge_pmd_anonymous_page()
841 		if (pmd_none(*pmd)) {  in do_huge_pmd_anonymous_page()
849 						   haddr, pmd,  in do_huge_pmd_anonymous_page()
868 	return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp,  in do_huge_pmd_anonymous_page()
873 		pmd_t *pmd, unsigned long pfn, pgprot_t prot, bool write)  in insert_pfn_pmd()  argument
879 	ptl = pmd_lock(mm, pmd);  in insert_pfn_pmd()
880 	if (pmd_none(*pmd)) {  in insert_pfn_pmd()
886 		set_pmd_at(mm, addr, pmd, entry);  in insert_pfn_pmd()
887 		update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
893 			pmd_t *pmd, unsigned long pfn, bool write)  in vmf_insert_pfn_pmd()  argument
911 	insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write);  in vmf_insert_pfn_pmd()
921 	pmd_t pmd;  in copy_huge_pmd()  local
935 	pmd = *src_pmd;  in copy_huge_pmd()
936 	if (unlikely(!pmd_trans_huge(pmd))) {  in copy_huge_pmd()
945 	if (is_huge_zero_pmd(pmd)) {  in copy_huge_pmd()
959 	if (unlikely(pmd_trans_splitting(pmd))) {  in copy_huge_pmd()
968 	src_page = pmd_page(pmd);  in copy_huge_pmd()
975 	pmd = pmd_mkold(pmd_wrprotect(pmd));  in copy_huge_pmd()
977 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
991 			   pmd_t *pmd, pmd_t orig_pmd,  in huge_pmd_set_accessed()  argument
998 	ptl = pmd_lock(mm, pmd);  in huge_pmd_set_accessed()
999 	if (unlikely(!pmd_same(*pmd, orig_pmd)))  in huge_pmd_set_accessed()
1004 	if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty))  in huge_pmd_set_accessed()
1005 		update_mmu_cache_pmd(vma, address, pmd);  in huge_pmd_set_accessed()
1045 					pmd_t *pmd, pmd_t orig_pmd,  in do_huge_pmd_wp_page_fallback()  argument
1098 	ptl = pmd_lock(mm, pmd);  in do_huge_pmd_wp_page_fallback()
1099 	if (unlikely(!pmd_same(*pmd, orig_pmd)))  in do_huge_pmd_wp_page_fallback()
1103 	pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in do_huge_pmd_wp_page_fallback()
1106 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in do_huge_pmd_wp_page_fallback()
1126 	pmd_populate(mm, pmd, pgtable);  in do_huge_pmd_wp_page_fallback()
1152 			unsigned long address, pmd_t *pmd, pmd_t orig_pmd)  in do_huge_pmd_wp_page()  argument
1163 	ptl = pmd_lockptr(mm, pmd);  in do_huge_pmd_wp_page()
1169 	if (unlikely(!pmd_same(*pmd, orig_pmd)))  in do_huge_pmd_wp_page()
1178 		if (pmdp_set_access_flags(vma, haddr, pmd, entry,  1))  in do_huge_pmd_wp_page()
1179 			update_mmu_cache_pmd(vma, address, pmd);  in do_huge_pmd_wp_page()
1195 			split_huge_page_pmd(vma, address, pmd);  in do_huge_pmd_wp_page()
1199 					pmd, orig_pmd, page, haddr);  in do_huge_pmd_wp_page()
1216 			split_huge_page_pmd(vma, address, pmd);  in do_huge_pmd_wp_page()
1237 	if (unlikely(!pmd_same(*pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1246 		pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in do_huge_pmd_wp_page()
1250 		set_pmd_at(mm, haddr, pmd, entry);  in do_huge_pmd_wp_page()
1251 		update_mmu_cache_pmd(vma, address, pmd);  in do_huge_pmd_wp_page()
1274 				   pmd_t *pmd,  in follow_trans_huge_pmd()  argument
1280 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_trans_huge_pmd()
1282 	if (flags & FOLL_WRITE && !pmd_write(*pmd))  in follow_trans_huge_pmd()
1286 	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))  in follow_trans_huge_pmd()
1290 	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))  in follow_trans_huge_pmd()
1293 	page = pmd_page(*pmd);  in follow_trans_huge_pmd()
1305 		_pmd = pmd_mkyoung(pmd_mkdirty(*pmd));  in follow_trans_huge_pmd()
1307 					  pmd, _pmd,  1))  in follow_trans_huge_pmd()
1308 			update_mmu_cache_pmd(vma, addr, pmd);  in follow_trans_huge_pmd()
1329 				unsigned long addr, pmd_t pmd, pmd_t *pmdp)  in do_huge_pmd_numa_page()  argument
1346 	if (unlikely(!pmd_same(pmd, *pmdp)))  in do_huge_pmd_numa_page()
1361 	page = pmd_page(pmd);  in do_huge_pmd_numa_page()
1405 	if (unlikely(!pmd_same(pmd, *pmdp))) {  in do_huge_pmd_numa_page()
1425 				pmdp, pmd, addr, page, target_nid);  in do_huge_pmd_numa_page()
1435 	was_writable = pmd_write(pmd);  in do_huge_pmd_numa_page()
1436 	pmd = pmd_modify(pmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1437 	pmd = pmd_mkyoung(pmd);  in do_huge_pmd_numa_page()
1439 		pmd = pmd_mkwrite(pmd);  in do_huge_pmd_numa_page()
1440 	set_pmd_at(mm, haddr, pmdp, pmd);  in do_huge_pmd_numa_page()
1457 		 pmd_t *pmd, unsigned long addr)  in zap_huge_pmd()  argument
1462 	if (__pmd_trans_huge_lock(pmd, vma, &ptl) != 1)  in zap_huge_pmd()
1470 	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,  in zap_huge_pmd()
1472 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in zap_huge_pmd()
1478 		pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));  in zap_huge_pmd()
1488 		pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));  in zap_huge_pmd()
1503 	pmd_t pmd;  in move_huge_pmd()  local
1531 		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);  in move_huge_pmd()
1539 		set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));  in move_huge_pmd()
1554 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in change_huge_pmd()  argument
1561 	if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {  in change_huge_pmd()
1563 		bool preserve_write = prot_numa && pmd_write(*pmd);  in change_huge_pmd()
1571 		if (prot_numa && is_huge_zero_pmd(*pmd)) {  in change_huge_pmd()
1576 		if (!prot_numa || !pmd_protnone(*pmd)) {  in change_huge_pmd()
1577 			entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd);  in change_huge_pmd()
1582 			set_pmd_at(mm, addr, pmd, entry);  in change_huge_pmd()
1598 int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,  in __pmd_trans_huge_lock()  argument
1601 	*ptl = pmd_lock(vma->vm_mm, pmd);  in __pmd_trans_huge_lock()
1602 	if (likely(pmd_trans_huge(*pmd))) {  in __pmd_trans_huge_lock()
1603 		if (unlikely(pmd_trans_splitting(*pmd))) {  in __pmd_trans_huge_lock()
1605 			wait_split_huge_page(vma->anon_vma, pmd);  in __pmd_trans_huge_lock()
1633 	pmd_t *pmd;  in page_check_address_pmd()  local
1644 	pmd = pmd_offset(pud, address);  in page_check_address_pmd()
1646 	*ptl = pmd_lock(mm, pmd);  in page_check_address_pmd()
1647 	if (!pmd_present(*pmd))  in page_check_address_pmd()
1649 	if (pmd_page(*pmd) != page)  in page_check_address_pmd()
1659 	    pmd_trans_splitting(*pmd))  in page_check_address_pmd()
1661 	if (pmd_trans_huge(*pmd)) {  in page_check_address_pmd()
1663 			  !pmd_trans_splitting(*pmd));  in page_check_address_pmd()
1664 		return pmd;  in page_check_address_pmd()
1677 	pmd_t *pmd;  in __split_huge_page_splitting()  local
1684 	pmd = page_check_address_pmd(page, mm, address,  in __split_huge_page_splitting()
1686 	if (pmd) {  in __split_huge_page_splitting()
1694 		pmdp_splitting_flush(vma, address, pmd);  in __split_huge_page_splitting()
1829 	pmd_t *pmd, _pmd;  in __split_huge_page_map()  local
1834 	pmd = page_check_address_pmd(page, mm, address,  in __split_huge_page_map()
1836 	if (pmd) {  in __split_huge_page_map()
1837 		pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_page_map()
1839 		if (pmd_write(*pmd))  in __split_huge_page_map()
1853 			if (!pmd_write(*pmd))  in __split_huge_page_map()
1855 			if (!pmd_young(*pmd))  in __split_huge_page_map()
1890 		pmdp_invalidate(vma, address, pmd);  in __split_huge_page_map()
1891 		pmd_populate(mm, pmd, pgtable);  in __split_huge_page_map()
2509 	pmd_t *pmd, _pmd;  in collapse_huge_page()  local
2554 	pmd = mm_find_pmd(mm, address);  in collapse_huge_page()
2555 	if (!pmd)  in collapse_huge_page()
2560 	pte = pte_offset_map(pmd, address);  in collapse_huge_page()
2561 	pte_ptl = pte_lockptr(mm, pmd);  in collapse_huge_page()
2566 	pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */  in collapse_huge_page()
2573 	_pmd = pmdp_collapse_flush(vma, address, pmd);  in collapse_huge_page()
2584 		BUG_ON(!pmd_none(*pmd));  in collapse_huge_page()
2590 		pmd_populate(mm, pmd, pmd_pgtable(_pmd));  in collapse_huge_page()
2618 	BUG_ON(!pmd_none(*pmd));  in collapse_huge_page()
2622 	pgtable_trans_huge_deposit(mm, pmd, pgtable);  in collapse_huge_page()
2623 	set_pmd_at(mm, address, pmd, _pmd);  in collapse_huge_page()
2624 	update_mmu_cache_pmd(vma, address, pmd);  in collapse_huge_page()
2644 	pmd_t *pmd;  in khugepaged_scan_pmd()  local
2655 	pmd = mm_find_pmd(mm, address);  in khugepaged_scan_pmd()
2656 	if (!pmd)  in khugepaged_scan_pmd()
2660 	pte = pte_offset_map_lock(mm, pmd, address, &ptl);  in khugepaged_scan_pmd()
2932 		unsigned long haddr, pmd_t *pmd)  in __split_huge_zero_page_pmd()  argument
2939 	pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in __split_huge_zero_page_pmd()
2942 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_zero_page_pmd()
2955 	pmd_populate(mm, pmd, pgtable);  in __split_huge_zero_page_pmd()
2960 		pmd_t *pmd)  in __split_huge_page_pmd()  argument
2975 	ptl = pmd_lock(mm, pmd);  in __split_huge_page_pmd()
2976 	if (unlikely(!pmd_trans_huge(*pmd)))  in __split_huge_page_pmd()
2979 		pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in __split_huge_page_pmd()
2982 	} else if (is_huge_zero_pmd(*pmd)) {  in __split_huge_page_pmd()
2983 		__split_huge_zero_page_pmd(vma, haddr, pmd);  in __split_huge_page_pmd()
2985 		page = pmd_page(*pmd);  in __split_huge_page_pmd()
3004 	if (unlikely(pmd_trans_huge(*pmd)))  in __split_huge_page_pmd()
3009 		pmd_t *pmd)  in split_huge_page_pmd_mm()  argument
3015 	split_huge_page_pmd(vma, address, pmd);  in split_huge_page_pmd_mm()
3023 	pmd_t *pmd;  in split_huge_page_address()  local
3035 	pmd = pmd_offset(pud, address);  in split_huge_page_address()
3036 	if (!pmd_present(*pmd))  in split_huge_page_address()
3042 	split_huge_page_pmd_mm(mm, address, pmd);  in split_huge_page_address()