392 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,  in free_pte_range()  argument
395 	pgtable_t token = pmd_pgtable(*pmd);  in free_pte_range()
396 	pmd_clear(pmd);  in free_pte_range()
405 	pmd_t *pmd;  in free_pmd_range()  local
410 	pmd = pmd_offset(pud, addr);  in free_pmd_range()
413 		if (pmd_none_or_clear_bad(pmd))  in free_pmd_range()
415 		free_pte_range(tlb, pmd, addr);  in free_pmd_range()
416 	} while (pmd++, addr = next, addr != end);  in free_pmd_range()
429 	pmd = pmd_offset(pud, start);  in free_pmd_range()
431 	pmd_free_tlb(tlb, pmd, start);  in free_pmd_range()
565 		pmd_t *pmd, unsigned long address)  in __pte_alloc()  argument
588 	ptl = pmd_lock(mm, pmd);  in __pte_alloc()
590 	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */  in __pte_alloc()
592 		pmd_populate(mm, pmd, new);  in __pte_alloc()
594 	} else if (unlikely(pmd_trans_splitting(*pmd)))  in __pte_alloc()
600 		wait_split_huge_page(vma->anon_vma, pmd);  in __pte_alloc()
604 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)  in __pte_alloc_kernel()  argument
613 	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */  in __pte_alloc_kernel()
614 		pmd_populate_kernel(&init_mm, pmd, new);  in __pte_alloc_kernel()
617 		VM_BUG_ON(pmd_trans_splitting(*pmd));  in __pte_alloc_kernel()
652 	pmd_t *pmd = pmd_offset(pud, addr);  in print_bad_pte()  local
685 		(long long)pte_val(pte), (long long)pmd_val(*pmd));  in print_bad_pte()
802 				pmd_t pmd)  in vm_normal_page_pmd()  argument
804 	unsigned long pfn = pmd_pfn(pmd);  in vm_normal_page_pmd()
1114 				struct vm_area_struct *vma, pmd_t *pmd,  in zap_pte_range()  argument
1128 	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);  in zap_pte_range()
1230 	pmd_t *pmd;  in zap_pmd_range()  local
1233 	pmd = pmd_offset(pud, addr);  in zap_pmd_range()
1236 		if (pmd_trans_huge(*pmd)) {  in zap_pmd_range()
1247 				split_huge_page_pmd(vma, addr, pmd);  in zap_pmd_range()
1248 			} else if (zap_huge_pmd(tlb, vma, pmd, addr))  in zap_pmd_range()
1259 		if (pmd_none_or_trans_huge_or_clear_bad(pmd))  in zap_pmd_range()
1261 		next = zap_pte_range(tlb, vma, pmd, addr, next, details);  in zap_pmd_range()
1264 	} while (pmd++, addr = next, addr != end);  in zap_pmd_range()
1465 		pmd_t * pmd = pmd_alloc(mm, pud, addr);  in __get_locked_pte()  local
1466 		if (pmd) {  in __get_locked_pte()
1467 			VM_BUG_ON(pmd_trans_huge(*pmd));  in __get_locked_pte()
1468 			return pte_alloc_map_lock(mm, pmd, addr, ptl);  in __get_locked_pte()
1662 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,  in remap_pte_range()  argument
1669 	pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);  in remap_pte_range()
1687 	pmd_t *pmd;  in remap_pmd_range()  local
1691 	pmd = pmd_alloc(mm, pud, addr);  in remap_pmd_range()
1692 	if (!pmd)  in remap_pmd_range()
1694 	VM_BUG_ON(pmd_trans_huge(*pmd));  in remap_pmd_range()
1697 		if (remap_pte_range(mm, pmd, addr, next,  in remap_pmd_range()
1700 	} while (pmd++, addr = next, addr != end);  in remap_pmd_range()
1839 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,  in apply_to_pte_range()  argument
1849 		pte_alloc_kernel(pmd, addr) :  in apply_to_pte_range()
1850 		pte_alloc_map_lock(mm, pmd, addr, &ptl);  in apply_to_pte_range()
1854 	BUG_ON(pmd_huge(*pmd));  in apply_to_pte_range()
1858 	token = pmd_pgtable(*pmd);  in apply_to_pte_range()
1877 	pmd_t *pmd;  in apply_to_pmd_range()  local
1883 	pmd = pmd_alloc(mm, pud, addr);  in apply_to_pmd_range()
1884 	if (!pmd)  in apply_to_pmd_range()
1888 		err = apply_to_pte_range(mm, pmd, addr, next, fn, data);  in apply_to_pmd_range()
1891 	} while (pmd++, addr = next, addr != end);  in apply_to_pmd_range()
1948 static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,  in pte_unmap_same()  argument
1954 		spinlock_t *ptl = pte_lockptr(mm, pmd);  in pte_unmap_same()
2101 			unsigned long address, pte_t *page_table, pmd_t *pmd,  in wp_page_copy()  argument
2136 	page_table = pte_offset_map_lock(mm, pmd, address, &ptl);  in wp_page_copy()
2232 			pmd_t *pmd)  in wp_pfn_shared()  argument
2247 		page_table = pte_offset_map_lock(mm, pmd, address, &ptl);  in wp_pfn_shared()
2263 			  pmd_t *pmd, spinlock_t *ptl, pte_t orig_pte,  in wp_page_shared()  argument
2292 		page_table = pte_offset_map_lock(mm, pmd, address,  in wp_page_shared()
2326 		unsigned long address, pte_t *page_table, pmd_t *pmd,  in do_wp_page()  argument
2344 					     orig_pte, pmd);  in do_wp_page()
2347 		return wp_page_copy(mm, vma, address, page_table, pmd,  in do_wp_page()
2360 			page_table = pte_offset_map_lock(mm, pmd, address,  in do_wp_page()
2384 		return wp_page_shared(mm, vma, address, page_table, pmd,  in do_wp_page()
2394 	return wp_page_copy(mm, vma, address, page_table, pmd,  in do_wp_page()
2487 		unsigned long address, pte_t *page_table, pmd_t *pmd,  in do_swap_page()  argument
2499 	if (!pte_unmap_same(mm, pmd, page_table, orig_pte))  in do_swap_page()
2505 			migration_entry_wait(mm, pmd, address);  in do_swap_page()
2524 			page_table = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_swap_page()
2579 	page_table = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_swap_page()
2638 		ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte);  in do_swap_page()
2704 		unsigned long address, pte_t *page_table, pmd_t *pmd,  in do_anonymous_page()  argument
2726 		page_table = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_anonymous_page()
2759 	page_table = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_anonymous_page()
2980 		unsigned long address, pmd_t *pmd,  in do_read_fault()  argument
2994 		pte = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_read_fault()
3005 	pte = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_read_fault()
3020 		unsigned long address, pmd_t *pmd,  in do_cow_fault()  argument
3049 	pte = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_cow_fault()
3086 		unsigned long address, pmd_t *pmd,  in do_shared_fault()  argument
3114 	pte = pte_offset_map_lock(mm, pmd, address, &ptl);  in do_shared_fault()
3155 		unsigned long address, pte_t *page_table, pmd_t *pmd,  in do_fault()  argument
3166 		return do_read_fault(mm, vma, address, pmd, pgoff, flags,  in do_fault()
3169 		return do_cow_fault(mm, vma, address, pmd, pgoff, flags,  in do_fault()
3171 	return do_shared_fault(mm, vma, address, pmd, pgoff, flags, orig_pte);  in do_fault()
3190 		   unsigned long addr, pte_t pte, pte_t *ptep, pmd_t *pmd)  in do_numa_page()  argument
3213 	ptl = pte_lockptr(mm, pmd);  in do_numa_page()
3276 			unsigned long address, pmd_t *pmd, unsigned int flags)  in create_huge_pmd()  argument
3279 		return do_huge_pmd_anonymous_page(mm, vma, address, pmd, flags);  in create_huge_pmd()
3281 		return vma->vm_ops->pmd_fault(vma, address, pmd, flags);  in create_huge_pmd()
3286 			unsigned long address, pmd_t *pmd, pmd_t orig_pmd,  in wp_huge_pmd()  argument
3290 		return do_huge_pmd_wp_page(mm, vma, address, pmd, orig_pmd);  in wp_huge_pmd()
3292 		return vma->vm_ops->pmd_fault(vma, address, pmd, flags);  in wp_huge_pmd()
3314 		     pte_t *pte, pmd_t *pmd, unsigned int flags)  in handle_pte_fault()  argument
3333 							 pte, pmd, flags);  in handle_pte_fault()
3335 				return do_fault(mm, vma, address, pte, pmd,  in handle_pte_fault()
3339 					pte, pmd, flags, entry);  in handle_pte_fault()
3343 		return do_numa_page(mm, vma, address, entry, pte, pmd);  in handle_pte_fault()
3345 	ptl = pte_lockptr(mm, pmd);  in handle_pte_fault()
3352 					pte, pmd, ptl, entry);  in handle_pte_fault()
3384 	pmd_t *pmd;  in __handle_mm_fault()  local
3394 	pmd = pmd_alloc(mm, pud, address);  in __handle_mm_fault()
3395 	if (!pmd)  in __handle_mm_fault()
3397 	if (pmd_none(*pmd) && transparent_hugepage_enabled(vma)) {  in __handle_mm_fault()
3398 		int ret = create_huge_pmd(mm, vma, address, pmd, flags);  in __handle_mm_fault()
3402 		pmd_t orig_pmd = *pmd;  in __handle_mm_fault()
3419 							     orig_pmd, pmd);  in __handle_mm_fault()
3422 				ret = wp_huge_pmd(mm, vma, address, pmd,  in __handle_mm_fault()
3427 				huge_pmd_set_accessed(mm, vma, address, pmd,  in __handle_mm_fault()
3439 	if (unlikely(pmd_none(*pmd)) &&  in __handle_mm_fault()
3440 	    unlikely(__pte_alloc(mm, vma, pmd, address)))  in __handle_mm_fault()
3453 	if (unlikely(pmd_trans_unstable(pmd)))  in __handle_mm_fault()
3461 	pte = pte_offset_map(pmd, address);  in __handle_mm_fault()
3463 	return handle_pte_fault(mm, vma, address, pte, pmd, flags);  in __handle_mm_fault()
3570 	pmd_t *pmd;  in __follow_pte()  local
3581 	pmd = pmd_offset(pud, address);  in __follow_pte()
3582 	VM_BUG_ON(pmd_trans_huge(*pmd));  in __follow_pte()
3583 	if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))  in __follow_pte()
3587 	if (pmd_huge(*pmd))  in __follow_pte()
3590 	ptep = pte_offset_map_lock(mm, pmd, address, ptlp);  in __follow_pte()