419 __visible pgdval_t xen_pgd_val(pgd_t pgd)  in xen_pgd_val()  argument
421 	return pte_mfn_to_pfn(pgd.pgd);  in xen_pgd_val()
433 __visible pgd_t xen_make_pgd(pgdval_t pgd)  in xen_make_pgd()  argument
435 	pgd = pte_pfn_to_mfn(pgd);  in xen_make_pgd()
436 	return native_make_pgd(pgd);  in xen_make_pgd()
521 static pgd_t *xen_get_user_pgd(pgd_t *pgd)  in xen_get_user_pgd()  argument
523 	pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);  in xen_get_user_pgd()
524 	unsigned offset = pgd - pgd_page;  in xen_get_user_pgd()
610 static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,  in __xen_pgd_walk()  argument
653 		if (!pgd_val(pgd[pgdidx]))  in __xen_pgd_walk()
656 		pud = pud_offset(&pgd[pgdidx], 0);  in __xen_pgd_walk()
696 	flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);  in __xen_pgd_walk()
706 	return __xen_pgd_walk(mm, mm->pgd, func, limit);  in xen_pgd_walk()
802 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)  in __xen_pgd_pin()  argument
804 	trace_xen_mmu_pgd_pin(mm, pgd);  in __xen_pgd_pin()
808 	if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {  in __xen_pgd_pin()
819 		pgd_t *user_pgd = xen_get_user_pgd(pgd);  in __xen_pgd_pin()
821 		xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));  in __xen_pgd_pin()
832 	xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),  in __xen_pgd_pin()
835 	xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));  in __xen_pgd_pin()
842 	__xen_pgd_pin(mm, mm->pgd);  in xen_pgd_pin()
929 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)  in __xen_pgd_unpin()  argument
931 	trace_xen_mmu_pgd_unpin(mm, pgd);  in __xen_pgd_unpin()
935 	xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));  in __xen_pgd_unpin()
939 		pgd_t *user_pgd = xen_get_user_pgd(pgd);  in __xen_pgd_unpin()
951 	xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),  in __xen_pgd_unpin()
955 	__xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);  in __xen_pgd_unpin()
962 	__xen_pgd_unpin(mm, mm->pgd);  in xen_pgd_unpin()
1016 	if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))  in drop_other_mm_ref()
1036 			    && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))  in xen_drop_mm_ref()
1050 		if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))  in xen_drop_mm_ref()
1089 	if (xen_page_pinned(mm->pgd))  in xen_exit_mmap()
1160 	pgd_t *pgd = pgd_offset_k(va);  in xen_cleanmfnmap()  local
1161 	pud_t *pud_page = pud_offset(pgd, 0);  in xen_cleanmfnmap()
1169 	set_pgd(pgd, __pgd(0));  in xen_cleanmfnmap()
1512 	pgd_t *pgd = mm->pgd;  in xen_pgd_alloc()  local
1515 	BUG_ON(PagePinned(virt_to_page(pgd)));  in xen_pgd_alloc()
1519 		struct page *page = virt_to_page(pgd);  in xen_pgd_alloc()
1537 		BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));  in xen_pgd_alloc()
1544 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)  in xen_pgd_free()  argument
1547 	pgd_t *user_pgd = xen_get_user_pgd(pgd);  in xen_pgd_free()
1674 	bool pinned = PagePinned(virt_to_page(mm->pgd));  in xen_alloc_ptpage()
1918 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)  in xen_setup_kernel_pagetable()  argument
1957 	l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);  in xen_setup_kernel_pagetable()
1960 	addr[0] = (unsigned long)pgd;  in xen_setup_kernel_pagetable()
1994 		pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));  in xen_setup_kernel_pagetable()
2046 	pgd_t pgd;  in xen_early_virt_to_phys()  local
2052 	pgd = native_make_pgd(xen_read_phys_ulong(pa + pgd_index(vaddr) *  in xen_early_virt_to_phys()
2053 						       sizeof(pgd)));  in xen_early_virt_to_phys()
2054 	if (!pgd_present(pgd))  in xen_early_virt_to_phys()
2057 	pa = pgd_val(pgd) & PTE_PFN_MASK;  in xen_early_virt_to_phys()
2095 	pgd_t *pgd;  in xen_relocate_p2m()  local
2124 	pgd = __va(read_cr3());  in xen_relocate_p2m()
2165 		set_pgd(pgd + 2 + idx_pud, __pgd(_PAGE_TABLE | pud_phys));  in xen_relocate_p2m()
2182 		set_pgd(pgd + 1, __pgd(0));  in xen_relocate_p2m()
2264 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)  in xen_setup_kernel_pagetable()  argument
2268 	kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);  in xen_setup_kernel_pagetable()
2282 	copy_page(initial_page_table, pgd);  in xen_setup_kernel_pagetable()
2290 	pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));  in xen_setup_kernel_pagetable()
2751 	a.gpa = __pa(mm->pgd);  in xen_hvm_exit_mmap()