p4d 1047 arch/arm64/mm/mmu.c int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) p4d 1095 arch/powerpc/mm/book3s64/radix_pgtable.c int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) p4d 217 arch/riscv/mm/fault.c p4d_t *p4d, *p4d_k; p4d 242 arch/riscv/mm/fault.c p4d = p4d_offset(pgd, addr); p4d 247 arch/riscv/mm/fault.c pud = pud_offset(p4d, addr); p4d 84 arch/s390/include/asm/page.h typedef struct { unsigned long p4d; } p4d_t; p4d 93 arch/s390/include/asm/page.h #define p4d_val(x) ((x).p4d) p4d 60 arch/s390/include/asm/pgalloc.h static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) p4d 63 arch/s390/include/asm/pgalloc.h crst_table_free(mm, (unsigned long *) p4d); p4d 102 arch/s390/include/asm/pgalloc.h static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) p4d 104 arch/s390/include/asm/pgalloc.h pgd_val(*pgd) = _REGION1_ENTRY | __pa(p4d); p4d 107 arch/s390/include/asm/pgalloc.h static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) p4d 109 arch/s390/include/asm/pgalloc.h p4d_val(*p4d) = _REGION2_ENTRY | __pa(pud); p4d 632 arch/s390/include/asm/pgtable.h static inline int p4d_folded(p4d_t p4d) p4d 634 arch/s390/include/asm/pgtable.h return (p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2; p4d 637 arch/s390/include/asm/pgtable.h static inline int p4d_present(p4d_t p4d) p4d 639 arch/s390/include/asm/pgtable.h if (p4d_folded(p4d)) p4d 641 arch/s390/include/asm/pgtable.h return (p4d_val(p4d) & _REGION_ENTRY_ORIGIN) != 0UL; p4d 644 arch/s390/include/asm/pgtable.h static inline int p4d_none(p4d_t p4d) p4d 646 arch/s390/include/asm/pgtable.h if (p4d_folded(p4d)) p4d 648 arch/s390/include/asm/pgtable.h return p4d_val(p4d) == _REGION2_ENTRY_EMPTY; p4d 651 arch/s390/include/asm/pgtable.h static inline unsigned long p4d_pfn(p4d_t p4d) p4d 656 arch/s390/include/asm/pgtable.h return (p4d_val(p4d) & origin_mask) >> PAGE_SHIFT; p4d 722 arch/s390/include/asm/pgtable.h static inline int p4d_bad(p4d_t p4d) p4d 724 arch/s390/include/asm/pgtable.h unsigned long type = p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK; p4d 730 arch/s390/include/asm/pgtable.h return (p4d_val(p4d) & ~_REGION_ENTRY_BITS) != 0; p4d 896 arch/s390/include/asm/pgtable.h static inline void p4d_clear(p4d_t *p4d) p4d 898 arch/s390/include/asm/pgtable.h if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) p4d 899 arch/s390/include/asm/pgtable.h p4d_val(*p4d) = _REGION2_ENTRY_EMPTY; p4d 1257 arch/s390/include/asm/pgtable.h static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) p4d 1259 arch/s390/include/asm/pgtable.h if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R2) p4d 1260 arch/s390/include/asm/pgtable.h return (pud_t *) p4d_deref(*p4d) + pud_index(address); p4d 1261 arch/s390/include/asm/pgtable.h return (pud_t *) p4d; p4d 1293 arch/s390/include/asm/pgtable.h #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d)) p4d 106 arch/s390/include/asm/tlb.h static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, p4d 115 arch/s390/include/asm/tlb.h tlb_remove_table(tlb, p4d); p4d 182 arch/s390/mm/dump_pagetables.c p4d_t *p4d, unsigned long addr) p4d 189 arch/s390/mm/dump_pagetables.c if ((p4d_val(*p4d) & PAGE_MASK) == __pa(kasan_early_shadow_pud)) { p4d 195 arch/s390/mm/dump_pagetables.c pud = pud_offset(p4d, addr); p4d 215 arch/s390/mm/dump_pagetables.c p4d_t *p4d; p4d 225 arch/s390/mm/dump_pagetables.c p4d = p4d_offset(pgd, addr); p4d 226 arch/s390/mm/dump_pagetables.c for (i = 0; i < PTRS_PER_P4D && addr < max_addr; i++, p4d++) { p4d 228 arch/s390/mm/dump_pagetables.c if (!p4d_none(*p4d)) p4d 229 arch/s390/mm/dump_pagetables.c walk_pud_level(m, st, p4d, addr); p4d 546 arch/s390/mm/gmap.c p4d_t *p4d; p4d 584 arch/s390/mm/gmap.c p4d = p4d_offset(pgd, vmaddr); p4d 585 arch/s390/mm/gmap.c VM_BUG_ON(p4d_none(*p4d)); p4d 586 arch/s390/mm/gmap.c pud = pud_offset(p4d, vmaddr); p4d 120 arch/s390/mm/page-states.c static void mark_kernel_pud(p4d_t *p4d, unsigned long addr, unsigned long end) p4d 127 arch/s390/mm/page-states.c pud = pud_offset(p4d, addr); p4d 145 arch/s390/mm/page-states.c p4d_t *p4d; p4d 148 arch/s390/mm/page-states.c p4d = p4d_offset(pgd, addr); p4d 151 arch/s390/mm/page-states.c if (p4d_none(*p4d)) p4d 153 arch/s390/mm/page-states.c if (!p4d_folded(*p4d)) { p4d 154 arch/s390/mm/page-states.c page = virt_to_page(p4d_val(*p4d)); p4d 158 arch/s390/mm/page-states.c mark_kernel_pud(p4d, addr, next); p4d 159 arch/s390/mm/page-states.c } while (p4d++, addr = next, addr != end); p4d 232 arch/s390/mm/pageattr.c static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end, p4d 239 arch/s390/mm/pageattr.c pudp = pud_offset(p4d, addr); p4d 342 arch/s390/mm/pageattr.c p4d_t *p4d; p4d 350 arch/s390/mm/pageattr.c p4d = p4d_offset(pgd, address); p4d 351 arch/s390/mm/pageattr.c pud = pud_offset(p4d, address); p4d 417 arch/s390/mm/pgtable.c p4d_t *p4d; p4d 422 arch/s390/mm/pgtable.c p4d = p4d_alloc(mm, pgd, addr); p4d 423 arch/s390/mm/pgtable.c if (!p4d) p4d 425 arch/s390/mm/pgtable.c pud = pud_alloc(mm, p4d, addr); p4d 360 arch/x86/entry/vsyscall/vsyscall_64.c p4d_t *p4d; p4d 366 arch/x86/entry/vsyscall/vsyscall_64.c p4d = p4d_offset(pgd, VSYSCALL_ADDR); p4d 368 arch/x86/entry/vsyscall/vsyscall_64.c set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER)); p4d 370 arch/x86/entry/vsyscall/vsyscall_64.c pud = pud_offset(p4d, VSYSCALL_ADDR); p4d 153 arch/x86/include/asm/kexec.h p4d_t *p4d; p4d 518 arch/x86/include/asm/paravirt.h static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) p4d 520 arch/x86/include/asm/paravirt.h p4dval_t val = native_p4d_val(p4d); p4d 534 arch/x86/include/asm/paravirt.h static inline p4dval_t p4d_val(p4d_t p4d) p4d 536 arch/x86/include/asm/paravirt.h return PVOP_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d); p4d 138 arch/x86/include/asm/pgalloc.h static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) p4d 141 arch/x86/include/asm/pgalloc.h set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); p4d 144 arch/x86/include/asm/pgalloc.h static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) p4d 147 arch/x86/include/asm/pgalloc.h set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud))); p4d 174 arch/x86/include/asm/pgalloc.h static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) p4d 178 arch/x86/include/asm/pgalloc.h paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); p4d 179 arch/x86/include/asm/pgalloc.h set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); p4d 182 arch/x86/include/asm/pgalloc.h static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) p4d 186 arch/x86/include/asm/pgalloc.h paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); p4d 187 arch/x86/include/asm/pgalloc.h set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); p4d 199 arch/x86/include/asm/pgalloc.h static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) p4d 204 arch/x86/include/asm/pgalloc.h BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); p4d 205 arch/x86/include/asm/pgalloc.h free_page((unsigned long)p4d); p4d 208 arch/x86/include/asm/pgalloc.h extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); p4d 210 arch/x86/include/asm/pgalloc.h static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, p4d 214 arch/x86/include/asm/pgalloc.h ___p4d_free_tlb(tlb, p4d); p4d 104 arch/x86/include/asm/pgtable-3level.h pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd); p4d 235 arch/x86/include/asm/pgtable-3level.h pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0)); p4d 77 arch/x86/include/asm/pgtable.h # define set_p4d(p4dp, p4d) native_set_p4d(p4dp, p4d) p4d 81 arch/x86/include/asm/pgtable.h #define p4d_clear(p4d) native_p4d_clear(p4d) p4d 232 arch/x86/include/asm/pgtable.h static inline unsigned long p4d_pfn(p4d_t p4d) p4d 234 arch/x86/include/asm/pgtable.h return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT; p4d 242 arch/x86/include/asm/pgtable.h static inline int p4d_large(p4d_t p4d) p4d 904 arch/x86/include/asm/pgtable.h static inline int p4d_none(p4d_t p4d) p4d 906 arch/x86/include/asm/pgtable.h return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0; p4d 909 arch/x86/include/asm/pgtable.h static inline int p4d_present(p4d_t p4d) p4d 911 arch/x86/include/asm/pgtable.h return p4d_flags(p4d) & _PAGE_PRESENT; p4d 914 arch/x86/include/asm/pgtable.h static inline unsigned long p4d_page_vaddr(p4d_t p4d) p4d 916 arch/x86/include/asm/pgtable.h return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d)); p4d 923 arch/x86/include/asm/pgtable.h #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d)) p4d 926 arch/x86/include/asm/pgtable.h static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) p4d 928 arch/x86/include/asm/pgtable.h return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); p4d 931 arch/x86/include/asm/pgtable.h static inline int p4d_bad(p4d_t p4d) p4d 938 arch/x86/include/asm/pgtable.h return (p4d_flags(p4d) & ~ignore_flags) != 0; p4d 136 arch/x86/include/asm/pgtable_64.h static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) p4d 141 arch/x86/include/asm/pgtable_64.h WRITE_ONCE(*p4dp, p4d); p4d 145 arch/x86/include/asm/pgtable_64.h pgd = native_make_pgd(native_p4d_val(p4d)); p4d 150 arch/x86/include/asm/pgtable_64.h static inline void native_p4d_clear(p4d_t *p4d) p4d 152 arch/x86/include/asm/pgtable_64.h native_set_p4d(p4d, native_make_p4d(0)); p4d 308 arch/x86/include/asm/pgtable_types.h typedef struct { p4dval_t p4d; } p4d_t; p4d 315 arch/x86/include/asm/pgtable_types.h static inline p4dval_t native_p4d_val(p4d_t p4d) p4d 317 arch/x86/include/asm/pgtable_types.h return p4d.p4d; p4d 327 arch/x86/include/asm/pgtable_types.h static inline p4dval_t native_p4d_val(p4d_t p4d) p4d 329 arch/x86/include/asm/pgtable_types.h return native_pgd_val(p4d.pgd); p4d 350 arch/x86/include/asm/pgtable_types.h return (pud_t) { .p4d.pgd = native_make_pgd(val) }; p4d 355 arch/x86/include/asm/pgtable_types.h return native_pgd_val(pud.p4d.pgd); p4d 376 arch/x86/include/asm/pgtable_types.h return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; p4d 381 arch/x86/include/asm/pgtable_types.h return native_pgd_val(pmd.pud.p4d.pgd); p4d 385 arch/x86/include/asm/pgtable_types.h static inline p4dval_t p4d_pfn_mask(p4d_t p4d) p4d 391 arch/x86/include/asm/pgtable_types.h static inline p4dval_t p4d_flags_mask(p4d_t p4d) p4d 393 arch/x86/include/asm/pgtable_types.h return ~p4d_pfn_mask(p4d); p4d 396 arch/x86/include/asm/pgtable_types.h static inline p4dval_t p4d_flags(p4d_t p4d) p4d 398 arch/x86/include/asm/pgtable_types.h return native_p4d_val(p4d) & p4d_flags_mask(p4d); p4d 342 arch/x86/include/asm/xen/page.h #define pud_val_ma(v) ((v).p4d.pgd.pgd) p4d 351 arch/x86/include/asm/xen/page.h #define p4d_val_ma(x) ((x).p4d) p4d 117 arch/x86/kernel/espfix_64.c p4d_t *p4d; p4d 121 arch/x86/kernel/espfix_64.c p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR); p4d 122 arch/x86/kernel/espfix_64.c p4d_populate(&init_mm, p4d, espfix_pud_page); p4d 120 arch/x86/kernel/head64.c p4dval_t *p4d; p4d 161 arch/x86/kernel/head64.c p4d = fixup_pointer(&level4_kernel_pgt, physaddr); p4d 162 arch/x86/kernel/head64.c p4d[511] += load_delta; p4d 187 arch/x86/kernel/head64.c p4d = fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], p4d 191 arch/x86/kernel/head64.c pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; p4d 192 arch/x86/kernel/head64.c pgd[i + 1] = (pgdval_t)p4d + pgtable_flags; p4d 195 arch/x86/kernel/head64.c p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; p4d 196 arch/x86/kernel/head64.c p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; p4d 304 arch/x86/kernel/head64.c p4dval_t p4d, *p4d_p; p4d 336 arch/x86/kernel/head64.c p4d = *p4d_p; p4d 338 arch/x86/kernel/head64.c if (p4d) p4d 339 arch/x86/kernel/head64.c pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base); p4d 133 arch/x86/kernel/ldt.c p4d_t *p4d; p4d 139 arch/x86/kernel/ldt.c p4d = p4d_offset(pgd, va); p4d 140 arch/x86/kernel/ldt.c if (p4d_none(*p4d)) p4d 143 arch/x86/kernel/ldt.c pud = pud_offset(p4d, va); p4d 95 arch/x86/kernel/machine_kexec_32.c p4d_t *p4d; p4d 103 arch/x86/kernel/machine_kexec_32.c p4d = p4d_offset(pgd, vaddr); p4d 104 arch/x86/kernel/machine_kexec_32.c pud = pud_offset(p4d, vaddr); p4d 114 arch/x86/kernel/machine_kexec_64.c free_page((unsigned long)image->arch.p4d); p4d 115 arch/x86/kernel/machine_kexec_64.c image->arch.p4d = NULL; p4d 129 arch/x86/kernel/machine_kexec_64.c p4d_t *p4d; p4d 138 arch/x86/kernel/machine_kexec_64.c p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL); p4d 139 arch/x86/kernel/machine_kexec_64.c if (!p4d) p4d 141 arch/x86/kernel/machine_kexec_64.c image->arch.p4d = p4d; p4d 142 arch/x86/kernel/machine_kexec_64.c set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE)); p4d 144 arch/x86/kernel/machine_kexec_64.c p4d = p4d_offset(pgd, vaddr); p4d 145 arch/x86/kernel/machine_kexec_64.c if (!p4d_present(*p4d)) { p4d 150 arch/x86/kernel/machine_kexec_64.c set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE)); p4d 152 arch/x86/kernel/machine_kexec_64.c pud = pud_offset(p4d, vaddr); p4d 108 arch/x86/kernel/tboot.c p4d_t *p4d; p4d 114 arch/x86/kernel/tboot.c p4d = p4d_alloc(&tboot_mm, pgd, vaddr); p4d 115 arch/x86/kernel/tboot.c if (!p4d) p4d 117 arch/x86/kernel/tboot.c pud = pud_alloc(&tboot_mm, p4d, vaddr); p4d 169 arch/x86/kernel/vm86_32.c p4d_t *p4d; p4d 179 arch/x86/kernel/vm86_32.c p4d = p4d_offset(pgd, 0xA0000); p4d 180 arch/x86/kernel/vm86_32.c if (p4d_none_or_clear_bad(p4d)) p4d 182 arch/x86/kernel/vm86_32.c pud = pud_offset(p4d, 0xA0000); p4d 153 arch/x86/mm/fault.c p4d_t *p4d, *p4d_k; p4d 168 arch/x86/mm/fault.c p4d = p4d_offset(pgd, address); p4d 173 arch/x86/mm/fault.c pud = pud_offset(p4d, address); p4d 295 arch/x86/mm/fault.c p4d_t *p4d; p4d 308 arch/x86/mm/fault.c p4d = p4d_offset(pgd, address); p4d 309 arch/x86/mm/fault.c pud = pud_offset(p4d, address); p4d 356 arch/x86/mm/fault.c p4d_t *p4d, *p4d_k; p4d 385 arch/x86/mm/fault.c p4d = p4d_offset(pgd, address); p4d 390 arch/x86/mm/fault.c if (p4d_none(*p4d) && !pgtable_l5_enabled()) { p4d 391 arch/x86/mm/fault.c set_p4d(p4d, *p4d_k); p4d 394 arch/x86/mm/fault.c BUG_ON(p4d_pfn(*p4d) != p4d_pfn(*p4d_k)); p4d 399 arch/x86/mm/fault.c pud = pud_offset(p4d, address); p4d 450 arch/x86/mm/fault.c p4d_t *p4d; p4d 463 arch/x86/mm/fault.c p4d = p4d_offset(pgd, address); p4d 464 arch/x86/mm/fault.c if (bad_address(p4d)) p4d 467 arch/x86/mm/fault.c pr_cont("P4D %lx ", p4d_val(*p4d)); p4d 468 arch/x86/mm/fault.c if (!p4d_present(*p4d) || p4d_large(*p4d)) p4d 471 arch/x86/mm/fault.c pud = pud_offset(p4d, address); p4d 1124 arch/x86/mm/fault.c p4d_t *p4d; p4d 1147 arch/x86/mm/fault.c p4d = p4d_offset(pgd, address); p4d 1148 arch/x86/mm/fault.c if (!p4d_present(*p4d)) p4d 1151 arch/x86/mm/fault.c if (p4d_large(*p4d)) p4d 1152 arch/x86/mm/fault.c return spurious_kernel_fault_check(error_code, (pte_t *) p4d); p4d 1154 arch/x86/mm/fault.c pud = pud_offset(p4d, address); p4d 67 arch/x86/mm/ident_map.c p4d_t *p4d = p4d_page + p4d_index(addr); p4d 74 arch/x86/mm/ident_map.c if (p4d_present(*p4d)) { p4d 75 arch/x86/mm/ident_map.c pud = pud_offset(p4d, 0); p4d 83 arch/x86/mm/ident_map.c set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); p4d 106 arch/x86/mm/ident_map.c p4d_t *p4d; p4d 113 arch/x86/mm/ident_map.c p4d = p4d_offset(pgd, 0); p4d 114 arch/x86/mm/ident_map.c result = ident_p4d_init(info, p4d, addr, next); p4d 120 arch/x86/mm/ident_map.c p4d = (p4d_t *)info->alloc_pgt_page(info->context); p4d 121 arch/x86/mm/ident_map.c if (!p4d) p4d 123 arch/x86/mm/ident_map.c result = ident_p4d_init(info, p4d, addr, next); p4d 127 arch/x86/mm/ident_map.c set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); p4d 133 arch/x86/mm/ident_map.c pud_t *pud = pud_offset(p4d, 0); p4d 69 arch/x86/mm/init_32.c p4d_t *p4d; p4d 78 arch/x86/mm/init_32.c p4d = p4d_offset(pgd, 0); p4d 79 arch/x86/mm/init_32.c pud = pud_offset(p4d, 0); p4d 85 arch/x86/mm/init_32.c p4d = p4d_offset(pgd, 0); p4d 86 arch/x86/mm/init_32.c pud = pud_offset(p4d, 0); p4d 396 arch/x86/mm/init_32.c p4d_t *p4d = p4d_offset(pgd, vaddr); p4d 397 arch/x86/mm/init_32.c pud_t *pud = pud_offset(p4d, vaddr); p4d 418 arch/x86/mm/init_32.c p4d_t *p4d; p4d 427 arch/x86/mm/init_32.c p4d = p4d_offset(pgd, vaddr); p4d 428 arch/x86/mm/init_32.c pud = pud_offset(p4d, vaddr); p4d 475 arch/x86/mm/init_32.c p4d_t *p4d; p4d 495 arch/x86/mm/init_32.c p4d = p4d_offset(pgd, va); p4d 496 arch/x86/mm/init_32.c pud = pud_offset(p4d, va); p4d 72 arch/x86/mm/init_64.c DEFINE_POPULATE(p4d_populate, p4d, pud, init) p4d 73 arch/x86/mm/init_64.c DEFINE_POPULATE(pgd_populate, pgd, p4d, init) p4d 87 arch/x86/mm/init_64.c DEFINE_ENTRY(p4d, p4d, init) p4d 186 arch/x86/mm/init_64.c p4d_t *p4d; p4d 190 arch/x86/mm/init_64.c p4d = p4d_offset(pgd, addr); p4d 195 arch/x86/mm/init_64.c if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) p4d 196 arch/x86/mm/init_64.c BUG_ON(p4d_page_vaddr(*p4d) p4d 199 arch/x86/mm/init_64.c if (p4d_none(*p4d)) p4d 200 arch/x86/mm/init_64.c set_p4d(p4d, *p4d_ref); p4d 246 arch/x86/mm/init_64.c p4d_t *p4d = (p4d_t *)spp_getpage(); p4d 247 arch/x86/mm/init_64.c pgd_populate(&init_mm, pgd, p4d); p4d 248 arch/x86/mm/init_64.c if (p4d != p4d_offset(pgd, 0)) p4d 250 arch/x86/mm/init_64.c p4d, p4d_offset(pgd, 0)); p4d 255 arch/x86/mm/init_64.c static pud_t *fill_pud(p4d_t *p4d, unsigned long vaddr) p4d 257 arch/x86/mm/init_64.c if (p4d_none(*p4d)) { p4d 259 arch/x86/mm/init_64.c p4d_populate(&init_mm, p4d, pud); p4d 260 arch/x86/mm/init_64.c if (pud != pud_offset(p4d, 0)) p4d 262 arch/x86/mm/init_64.c pud, pud_offset(p4d, 0)); p4d 264 arch/x86/mm/init_64.c return pud_offset(p4d, vaddr); p4d 306 arch/x86/mm/init_64.c p4d_t *p4d = p4d_page + p4d_index(vaddr); p4d 307 arch/x86/mm/init_64.c pud_t *pud = fill_pud(p4d, vaddr); p4d 340 arch/x86/mm/init_64.c p4d_t *p4d; p4d 344 arch/x86/mm/init_64.c p4d = fill_p4d(pgd, vaddr); p4d 345 arch/x86/mm/init_64.c pud = fill_pud(p4d, vaddr); p4d 364 arch/x86/mm/init_64.c p4d_t *p4d; p4d 375 arch/x86/mm/init_64.c p4d = (p4d_t *) spp_getpage(); p4d 376 arch/x86/mm/init_64.c set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE | p4d 379 arch/x86/mm/init_64.c p4d = p4d_offset(pgd, (unsigned long)__va(phys)); p4d 380 arch/x86/mm/init_64.c if (p4d_none(*p4d)) { p4d 382 arch/x86/mm/init_64.c set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE | p4d 385 arch/x86/mm/init_64.c pud = pud_offset(p4d, (unsigned long)__va(phys)); p4d 685 arch/x86/mm/init_64.c p4d_t *p4d = p4d_page + p4d_index(vaddr); p4d 698 arch/x86/mm/init_64.c set_p4d_init(p4d, __p4d(0), init); p4d 702 arch/x86/mm/init_64.c if (!p4d_none(*p4d)) { p4d 703 arch/x86/mm/init_64.c pud = pud_offset(p4d, 0); p4d 714 arch/x86/mm/init_64.c p4d_populate_init(&init_mm, p4d, pud, init); p4d 737 arch/x86/mm/init_64.c p4d_t *p4d; p4d 742 arch/x86/mm/init_64.c p4d = (p4d_t *)pgd_page_vaddr(*pgd); p4d 743 arch/x86/mm/init_64.c paddr_last = phys_p4d_init(p4d, __pa(vaddr), p4d 750 arch/x86/mm/init_64.c p4d = alloc_low_page(); p4d 751 arch/x86/mm/init_64.c paddr_last = phys_p4d_init(p4d, __pa(vaddr), __pa(vaddr_end), p4d 756 arch/x86/mm/init_64.c pgd_populate_init(&init_mm, pgd, p4d, init); p4d 759 arch/x86/mm/init_64.c (pud_t *) p4d, init); p4d 939 arch/x86/mm/init_64.c static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) p4d 951 arch/x86/mm/init_64.c free_pagetable(p4d_page(*p4d), 0); p4d 953 arch/x86/mm/init_64.c p4d_clear(p4d); p4d 1145 arch/x86/mm/init_64.c p4d_t *p4d; p4d 1147 arch/x86/mm/init_64.c p4d = p4d_start + p4d_index(addr); p4d 1148 arch/x86/mm/init_64.c for (; addr < end; addr = next, p4d++) { p4d 1151 arch/x86/mm/init_64.c if (!p4d_present(*p4d)) p4d 1154 arch/x86/mm/init_64.c BUILD_BUG_ON(p4d_large(*p4d)); p4d 1156 arch/x86/mm/init_64.c pud_base = pud_offset(p4d, 0); p4d 1164 arch/x86/mm/init_64.c free_pud_table(pud_base, p4d); p4d 1179 arch/x86/mm/init_64.c p4d_t *p4d; p4d 1188 arch/x86/mm/init_64.c p4d = p4d_offset(pgd, 0); p4d 1189 arch/x86/mm/init_64.c remove_p4d_table(p4d, addr, next, altmap, direct); p4d 1345 arch/x86/mm/init_64.c p4d_t *p4d; p4d 1357 arch/x86/mm/init_64.c p4d = p4d_offset(pgd, addr); p4d 1358 arch/x86/mm/init_64.c if (p4d_none(*p4d)) p4d 1361 arch/x86/mm/init_64.c pud = pud_offset(p4d, addr); p4d 1455 arch/x86/mm/init_64.c p4d_t *p4d; p4d 1466 arch/x86/mm/init_64.c p4d = vmemmap_p4d_populate(pgd, addr, node); p4d 1467 arch/x86/mm/init_64.c if (!p4d) p4d 1470 arch/x86/mm/init_64.c pud = vmemmap_pud_populate(p4d, addr, node); p4d 1542 arch/x86/mm/init_64.c p4d_t *p4d; p4d 1558 arch/x86/mm/init_64.c p4d = p4d_offset(pgd, addr); p4d 1559 arch/x86/mm/init_64.c if (p4d_none(*p4d)) { p4d 1563 arch/x86/mm/init_64.c get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO); p4d 1565 arch/x86/mm/init_64.c pud = pud_offset(p4d, addr); p4d 806 arch/x86/mm/ioremap.c p4d_t *p4d = p4d_offset(pgd, addr); p4d 807 arch/x86/mm/ioremap.c pud_t *pud = pud_offset(p4d, addr); p4d 106 arch/x86/mm/kasan_init_64.c static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, p4d 112 arch/x86/mm/kasan_init_64.c if (p4d_none(*p4d)) { p4d 115 arch/x86/mm/kasan_init_64.c p4d_populate(&init_mm, p4d, p); p4d 118 arch/x86/mm/kasan_init_64.c pud = pud_offset(p4d, addr); p4d 130 arch/x86/mm/kasan_init_64.c p4d_t *p4d; p4d 138 arch/x86/mm/kasan_init_64.c p4d = p4d_offset(pgd, addr); p4d 141 arch/x86/mm/kasan_init_64.c kasan_populate_p4d(p4d, addr, next, nid); p4d 142 arch/x86/mm/kasan_init_64.c } while (p4d++, addr = next, addr != end); p4d 197 arch/x86/mm/kasan_init_64.c unsigned long p4d; p4d 202 arch/x86/mm/kasan_init_64.c p4d = pgd_val(*pgd) & PTE_PFN_MASK; p4d 203 arch/x86/mm/kasan_init_64.c p4d += __START_KERNEL_map - phys_base; p4d 204 arch/x86/mm/kasan_init_64.c return (p4d_t *)p4d + p4d_index(addr); p4d 212 arch/x86/mm/kasan_init_64.c p4d_t *p4d, p4d_entry; p4d 221 arch/x86/mm/kasan_init_64.c p4d = early_p4d_offset(pgd, addr); p4d 225 arch/x86/mm/kasan_init_64.c if (!p4d_none(*p4d)) p4d 230 arch/x86/mm/kasan_init_64.c set_p4d(p4d, p4d_entry); p4d 231 arch/x86/mm/kasan_init_64.c } while (p4d++, addr = next, addr != end && p4d_none(*p4d)); p4d 154 arch/x86/mm/kaslr.c p4d_t *p4d_page_tramp, *p4d, *p4d_tramp; p4d 171 arch/x86/mm/kaslr.c p4d = p4d_offset(pgd, vaddr); p4d 172 arch/x86/mm/kaslr.c pud = pud_offset(p4d, vaddr); p4d 108 arch/x86/mm/mem_encrypt_identity.c p4d_t *p4d; p4d 114 arch/x86/mm/mem_encrypt_identity.c p4d = ppd->pgtable_area; p4d 115 arch/x86/mm/mem_encrypt_identity.c memset(p4d, 0, sizeof(*p4d) * PTRS_PER_P4D); p4d 116 arch/x86/mm/mem_encrypt_identity.c ppd->pgtable_area += sizeof(*p4d) * PTRS_PER_P4D; p4d 117 arch/x86/mm/mem_encrypt_identity.c set_pgd(pgd, __pgd(PGD_FLAGS | __pa(p4d))); p4d 120 arch/x86/mm/mem_encrypt_identity.c p4d = p4d_offset(pgd, ppd->vaddr); p4d 121 arch/x86/mm/mem_encrypt_identity.c if (p4d_none(*p4d)) { p4d 125 arch/x86/mm/mem_encrypt_identity.c set_p4d(p4d, __p4d(P4D_FLAGS | __pa(pud))); p4d 128 arch/x86/mm/mem_encrypt_identity.c pud = pud_offset(p4d, ppd->vaddr); p4d 570 arch/x86/mm/pageattr.c p4d_t *p4d; p4d 579 arch/x86/mm/pageattr.c p4d = p4d_offset(pgd, address); p4d 580 arch/x86/mm/pageattr.c if (p4d_none(*p4d)) p4d 584 arch/x86/mm/pageattr.c if (p4d_large(*p4d) || !p4d_present(*p4d)) p4d 585 arch/x86/mm/pageattr.c return (pte_t *)p4d; p4d 587 arch/x86/mm/pageattr.c pud = pud_offset(p4d, address); p4d 639 arch/x86/mm/pageattr.c p4d_t *p4d; p4d 646 arch/x86/mm/pageattr.c p4d = p4d_offset(pgd, address); p4d 647 arch/x86/mm/pageattr.c if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d)) p4d 650 arch/x86/mm/pageattr.c pud = pud_offset(p4d, address); p4d 715 arch/x86/mm/pageattr.c p4d_t *p4d; p4d 720 arch/x86/mm/pageattr.c p4d = p4d_offset(pgd, address); p4d 721 arch/x86/mm/pageattr.c pud = pud_offset(p4d, address); p4d 1158 arch/x86/mm/pageattr.c static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end) p4d 1160 arch/x86/mm/pageattr.c pud_t *pud = pud_offset(p4d, start); p4d 1314 arch/x86/mm/pageattr.c static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d, p4d 1336 arch/x86/mm/pageattr.c pud = pud_offset(p4d, start); p4d 1357 arch/x86/mm/pageattr.c pud = pud_offset(p4d, start); p4d 1377 arch/x86/mm/pageattr.c pud = pud_offset(p4d, start); p4d 1400 arch/x86/mm/pageattr.c p4d_t *p4d; p4d 1407 arch/x86/mm/pageattr.c p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL); p4d 1408 arch/x86/mm/pageattr.c if (!p4d) p4d 1411 arch/x86/mm/pageattr.c set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE)); p4d 1417 arch/x86/mm/pageattr.c p4d = p4d_offset(pgd_entry, addr); p4d 1418 arch/x86/mm/pageattr.c if (p4d_none(*p4d)) { p4d 1423 arch/x86/mm/pageattr.c set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE)); p4d 1429 arch/x86/mm/pageattr.c ret = populate_pud(cpa, addr, p4d, pgprot); p4d 1436 arch/x86/mm/pageattr.c unmap_pud_range(p4d, addr, p4d 77 arch/x86/mm/pgtable.c void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d) p4d 79 arch/x86/mm/pgtable.c paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT); p4d 80 arch/x86/mm/pgtable.c paravirt_tlb_remove_table(tlb, virt_to_page(p4d)); p4d 291 arch/x86/mm/pgtable.c p4d_t *p4d; p4d 298 arch/x86/mm/pgtable.c p4d = p4d_offset(pgd, 0); p4d 299 arch/x86/mm/pgtable.c pud = pud_offset(p4d, 0); p4d 662 arch/x86/mm/pgtable.c int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) p4d 672 arch/x86/mm/pgtable.c int p4d_clear_huge(p4d_t *p4d) p4d 783 arch/x86/mm/pgtable.c int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) p4d 31 arch/x86/mm/pgtable_32.c p4d_t *p4d; p4d 41 arch/x86/mm/pgtable_32.c p4d = p4d_offset(pgd, vaddr); p4d 42 arch/x86/mm/pgtable_32.c if (p4d_none(*p4d)) { p4d 46 arch/x86/mm/pgtable_32.c pud = pud_offset(p4d, vaddr); p4d 203 arch/x86/mm/pti.c p4d_t *p4d; p4d 206 arch/x86/mm/pti.c p4d = pti_user_pagetable_walk_p4d(address); p4d 207 arch/x86/mm/pti.c if (!p4d) p4d 210 arch/x86/mm/pti.c BUILD_BUG_ON(p4d_large(*p4d) != 0); p4d 211 arch/x86/mm/pti.c if (p4d_none(*p4d)) { p4d 216 arch/x86/mm/pti.c set_p4d(p4d, __p4d(_KERNPG_TABLE | __pa(new_pud_page))); p4d 219 arch/x86/mm/pti.c pud = pud_offset(p4d, address); p4d 317 arch/x86/mm/pti.c p4d_t *p4d; p4d 327 arch/x86/mm/pti.c p4d = p4d_offset(pgd, addr); p4d 328 arch/x86/mm/pti.c if (WARN_ON(p4d_none(*p4d))) p4d 331 arch/x86/mm/pti.c pud = pud_offset(p4d, addr); p4d 181 arch/x86/mm/tlb.c p4d_t *p4d = p4d_offset(pgd, sp); p4d 183 arch/x86/mm/tlb.c if (unlikely(p4d_none(*p4d))) { p4d 187 arch/x86/mm/tlb.c set_p4d(p4d, *p4d_ref); p4d 79 arch/x86/platform/efi/efi_64.c p4d_t *p4d, *p4d_k, *p4d_efi; p4d 111 arch/x86/platform/efi/efi_64.c p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd); p4d 112 arch/x86/platform/efi/efi_64.c if (!p4d) { p4d 119 arch/x86/platform/efi/efi_64.c p4d_efi = p4d + p4d_index(addr_p4d); p4d 158 arch/x86/platform/efi/efi_64.c p4d_t *p4d; p4d 176 arch/x86/platform/efi/efi_64.c p4d = p4d_offset(pgd, p4d 179 arch/x86/platform/efi/efi_64.c if (!p4d_present(*p4d)) p4d 182 arch/x86/platform/efi/efi_64.c pud = (pud_t *)p4d_page_vaddr(*p4d); p4d 186 arch/x86/platform/efi/efi_64.c p4d = (p4d_t *)pgd_page_vaddr(*pgd); p4d 187 arch/x86/platform/efi/efi_64.c p4d_free(&init_mm, p4d); p4d 210 arch/x86/platform/efi/efi_64.c p4d_t *p4d; p4d 223 arch/x86/platform/efi/efi_64.c p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END); p4d 224 arch/x86/platform/efi/efi_64.c if (!p4d) { p4d 229 arch/x86/platform/efi/efi_64.c pud = pud_alloc(&init_mm, p4d, EFI_VA_END); p4d 214 arch/x86/power/hibernate.c p4d_t *p4d; p4d 228 arch/x86/power/hibernate.c p4d = p4d_offset(pgd, relocated_restore_code); p4d 229 arch/x86/power/hibernate.c if (p4d_large(*p4d)) { p4d 230 arch/x86/power/hibernate.c set_p4d(p4d, __p4d(p4d_val(*p4d) & ~_PAGE_NX)); p4d 233 arch/x86/power/hibernate.c pud = pud_offset(p4d, relocated_restore_code); p4d 32 arch/x86/power/hibernate_32.c p4d_t *p4d; p4d 42 arch/x86/power/hibernate_32.c p4d = p4d_offset(pgd, 0); p4d 43 arch/x86/power/hibernate_32.c pud = pud_offset(p4d, 0); p4d 47 arch/x86/power/hibernate_32.c p4d = p4d_offset(pgd, 0); p4d 48 arch/x86/power/hibernate_32.c pud = pud_offset(p4d, 0); p4d 32 arch/x86/power/hibernate_64.c p4d_t *p4d = NULL; p4d 55 arch/x86/power/hibernate_64.c p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); p4d 56 arch/x86/power/hibernate_64.c if (!p4d) p4d 72 arch/x86/power/hibernate_64.c if (p4d) { p4d 74 arch/x86/power/hibernate_64.c pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot)); p4d 76 arch/x86/power/hibernate_64.c set_p4d(p4d + p4d_index(restore_jump_address), new_p4d); p4d 560 arch/x86/xen/mmu_pv.c __visible p4dval_t xen_p4d_val(p4d_t p4d) p4d 562 arch/x86/xen/mmu_pv.c return pte_mfn_to_pfn(p4d.p4d); p4d 566 arch/x86/xen/mmu_pv.c __visible p4d_t xen_make_p4d(p4dval_t p4d) p4d 568 arch/x86/xen/mmu_pv.c p4d = pte_pfn_to_mfn(p4d); p4d 570 arch/x86/xen/mmu_pv.c return native_make_p4d(p4d); p4d 612 arch/x86/xen/mmu_pv.c static int xen_p4d_walk(struct mm_struct *mm, p4d_t *p4d, p4d 620 arch/x86/xen/mmu_pv.c if (p4d_none(*p4d)) p4d 623 arch/x86/xen/mmu_pv.c pud = pud_offset(p4d, 0); p4d 668 arch/x86/xen/mmu_pv.c p4d_t *p4d; p4d 676 arch/x86/xen/mmu_pv.c p4d = p4d_offset(&pgd[i], 0); p4d 677 arch/x86/xen/mmu_pv.c flush |= xen_p4d_walk(mm, p4d, func, i == nr - 1, limit); p4d 1182 arch/x86/xen/mmu_pv.c static void __init xen_cleanmfnmap_p4d(p4d_t *p4d, bool unpin) p4d 1188 arch/x86/xen/mmu_pv.c if (p4d_large(*p4d)) { p4d 1189 arch/x86/xen/mmu_pv.c pa = p4d_val(*p4d) & PHYSICAL_PAGE_MASK; p4d 1194 arch/x86/xen/mmu_pv.c pud_tbl = pud_offset(p4d, 0); p4d 1200 arch/x86/xen/mmu_pv.c set_p4d(p4d, __p4d(0)); p4d 1211 arch/x86/xen/mmu_pv.c p4d_t *p4d; p4d 1217 arch/x86/xen/mmu_pv.c p4d = p4d_offset(pgd, 0); p4d 1218 arch/x86/xen/mmu_pv.c if (!p4d_none(*p4d)) p4d 1219 arch/x86/xen/mmu_pv.c xen_cleanmfnmap_p4d(p4d, unpin); p4d 283 fs/userfaultfd.c p4d_t *p4d; p4d 294 fs/userfaultfd.c p4d = p4d_offset(pgd, address); p4d 295 fs/userfaultfd.c if (!p4d_present(*p4d)) p4d 297 fs/userfaultfd.c pud = pud_offset(p4d, address); p4d 16 include/asm-generic/5level-fixup.h #define pud_alloc(mm, p4d, address) \ p4d 17 include/asm-generic/5level-fixup.h ((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address)) ? \ p4d 18 include/asm-generic/5level-fixup.h NULL : pud_offset(p4d, address)) p4d 24 include/asm-generic/5level-fixup.h static inline int p4d_none(p4d_t p4d) p4d 29 include/asm-generic/5level-fixup.h static inline int p4d_bad(p4d_t p4d) p4d 34 include/asm-generic/5level-fixup.h static inline int p4d_present(p4d_t p4d) p4d 40 include/asm-generic/5level-fixup.h #define p4d_ERROR(p4d) do { } while (0) p4d 41 include/asm-generic/5level-fixup.h #define p4d_clear(p4d) pgd_clear(p4d) p4d 42 include/asm-generic/5level-fixup.h #define p4d_val(p4d) pgd_val(p4d) p4d 43 include/asm-generic/5level-fixup.h #define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud) p4d 44 include/asm-generic/5level-fixup.h #define p4d_populate_safe(mm, p4d, pud) pgd_populate(mm, p4d, pud) p4d 45 include/asm-generic/5level-fixup.h #define p4d_page(p4d) pgd_page(p4d) p4d 46 include/asm-generic/5level-fixup.h #define p4d_page_vaddr(p4d) pgd_page_vaddr(p4d) p4d 49 include/asm-generic/5level-fixup.h #define set_p4d(p4dp, p4d) set_pgd(p4dp, p4d) p4d 26 include/asm-generic/pgtable-nop4d.h #define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd)) p4d 28 include/asm-generic/pgtable-nop4d.h #define pgd_populate(mm, pgd, p4d) do { } while (0) p4d 29 include/asm-generic/pgtable-nop4d.h #define pgd_populate_safe(mm, pgd, p4d) do { } while (0) p4d 19 include/asm-generic/pgtable-nopud.h typedef struct { p4d_t p4d; } pud_t; p4d 31 include/asm-generic/pgtable-nopud.h static inline int p4d_none(p4d_t p4d) { return 0; } p4d 32 include/asm-generic/pgtable-nopud.h static inline int p4d_bad(p4d_t p4d) { return 0; } p4d 33 include/asm-generic/pgtable-nopud.h static inline int p4d_present(p4d_t p4d) { return 1; } p4d 34 include/asm-generic/pgtable-nopud.h static inline void p4d_clear(p4d_t *p4d) { } p4d 35 include/asm-generic/pgtable-nopud.h #define pud_ERROR(pud) (p4d_ERROR((pud).p4d)) p4d 37 include/asm-generic/pgtable-nopud.h #define p4d_populate(mm, p4d, pud) do { } while (0) p4d 38 include/asm-generic/pgtable-nopud.h #define p4d_populate_safe(mm, p4d, pud) do { } while (0) p4d 45 include/asm-generic/pgtable-nopud.h static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) p4d 47 include/asm-generic/pgtable-nopud.h return (pud_t *)p4d; p4d 50 include/asm-generic/pgtable-nopud.h #define pud_val(x) (p4d_val((x).p4d)) p4d 53 include/asm-generic/pgtable-nopud.h #define p4d_page(p4d) (pud_page((pud_t){ p4d })) p4d 54 include/asm-generic/pgtable-nopud.h #define p4d_page_vaddr(p4d) (pud_page_vaddr((pud_t){ p4d })) p4d 368 include/asm-generic/pgtable.h #define p4d_access_permitted(p4d, write) \ p4d 369 include/asm-generic/pgtable.h (p4d_present(p4d) && (!(write) || p4d_write(p4d))) p4d 429 include/asm-generic/pgtable.h #define set_p4d_safe(p4dp, p4d) \ p4d 431 include/asm-generic/pgtable.h WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ p4d 432 include/asm-generic/pgtable.h set_p4d(p4dp, p4d); \ p4d 576 include/asm-generic/pgtable.h static inline int p4d_none_or_clear_bad(p4d_t *p4d) p4d 578 include/asm-generic/pgtable.h if (p4d_none(*p4d)) p4d 580 include/asm-generic/pgtable.h if (unlikely(p4d_bad(*p4d))) { p4d 581 include/asm-generic/pgtable.h p4d_clear_bad(p4d); p4d 1042 include/asm-generic/pgtable.h int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); p4d 1043 include/asm-generic/pgtable.h int p4d_clear_huge(p4d_t *p4d); p4d 1045 include/asm-generic/pgtable.h static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) p4d 1049 include/asm-generic/pgtable.h static inline int p4d_clear_huge(p4d_t *p4d) p4d 1059 include/asm-generic/pgtable.h int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); p4d 1063 include/asm-generic/pgtable.h static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) p4d 1075 include/asm-generic/pgtable.h static inline int p4d_clear_huge(p4d_t *p4d) p4d 1087 include/asm-generic/pgtable.h static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) p4d 1771 include/linux/mm.h static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, p4d 1780 include/linux/mm.h int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address); p4d 1874 include/linux/mm.h static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, p4d 1877 include/linux/mm.h return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? p4d 1878 include/linux/mm.h NULL : pud_offset(p4d, address); p4d 2777 include/linux/mm.h pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node); p4d 146 lib/ioremap.c static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, p4d 152 lib/ioremap.c pud = pud_alloc(&init_mm, p4d, addr); p4d 167 lib/ioremap.c static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, p4d 183 lib/ioremap.c if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) p4d 186 lib/ioremap.c return p4d_set_huge(p4d, phys_addr, prot); p4d 192 lib/ioremap.c p4d_t *p4d; p4d 195 lib/ioremap.c p4d = p4d_alloc(&init_mm, pgd, addr); p4d 196 lib/ioremap.c if (!p4d) p4d 201 lib/ioremap.c if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) p4d 204 lib/ioremap.c if (ioremap_pud_range(p4d, addr, next, phys_addr, prot)) p4d 206 lib/ioremap.c } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); p4d 477 mm/gup.c p4d_t *p4d; p4d 480 mm/gup.c p4d = p4d_offset(pgdp, address); p4d 481 mm/gup.c if (p4d_none(*p4d)) p4d 483 mm/gup.c BUILD_BUG_ON(p4d_huge(*p4d)); p4d 484 mm/gup.c if (unlikely(p4d_bad(*p4d))) p4d 487 mm/gup.c if (is_hugepd(__hugepd(p4d_val(*p4d)))) { p4d 489 mm/gup.c __hugepd(p4d_val(*p4d)), flags, p4d 495 mm/gup.c return follow_pud_mask(vma, address, p4d, flags, ctx); p4d 574 mm/gup.c p4d_t *p4d; p4d 589 mm/gup.c p4d = p4d_offset(pgd, address); p4d 590 mm/gup.c if (p4d_none(*p4d)) p4d 592 mm/gup.c pud = pud_offset(p4d, address); p4d 2230 mm/gup.c static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end, p4d 2236 mm/gup.c pudp = pud_offset(&p4d, addr); p4d 2266 mm/gup.c p4d_t p4d = READ_ONCE(*p4dp); p4d 2269 mm/gup.c if (p4d_none(p4d)) p4d 2271 mm/gup.c BUILD_BUG_ON(p4d_huge(p4d)); p4d 2272 mm/gup.c if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) { p4d 2273 mm/gup.c if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr, p4d 2276 mm/gup.c } else if (!gup_pud_range(p4d, addr, next, flags, pages, nr)) p4d 2348 mm/huge_memory.c p4d_t *p4d; p4d 2356 mm/huge_memory.c p4d = p4d_offset(pgd, address); p4d 2357 mm/huge_memory.c if (!p4d_present(*p4d)) p4d 2360 mm/huge_memory.c pud = pud_offset(p4d, address); p4d 4943 mm/hugetlb.c p4d_t *p4d = p4d_offset(pgd, *addr); p4d 4944 mm/hugetlb.c pud_t *pud = pud_offset(p4d, *addr); p4d 4980 mm/hugetlb.c p4d_t *p4d; p4d 4985 mm/hugetlb.c p4d = p4d_alloc(mm, pgd, addr); p4d 4986 mm/hugetlb.c if (!p4d) p4d 4988 mm/hugetlb.c pud = pud_alloc(mm, p4d, addr); p4d 5018 mm/hugetlb.c p4d_t *p4d; p4d 5025 mm/hugetlb.c p4d = p4d_offset(pgd, addr); p4d 5026 mm/hugetlb.c if (!p4d_present(*p4d)) p4d 5029 mm/hugetlb.c pud = pud_offset(p4d, addr); p4d 642 mm/kasan/common.c p4d_t *p4d; p4d 649 mm/kasan/common.c p4d = p4d_offset(pgd, addr); p4d 650 mm/kasan/common.c if (p4d_none(*p4d)) p4d 652 mm/kasan/common.c pud = pud_offset(p4d, addr); p4d 50 mm/kasan/init.c static inline bool kasan_pud_table(p4d_t p4d) p4d 52 mm/kasan/init.c return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); p4d 55 mm/kasan/init.c static inline bool kasan_pud_table(p4d_t p4d) p4d 146 mm/kasan/init.c static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, p4d 149 mm/kasan/init.c pud_t *pud = pud_offset(p4d, addr); p4d 186 mm/kasan/init.c p4d_t *p4d = p4d_offset(pgd, addr); p4d 195 mm/kasan/init.c p4d_populate(&init_mm, p4d, p4d 197 mm/kasan/init.c pud = pud_offset(p4d, addr); p4d 206 mm/kasan/init.c if (p4d_none(*p4d)) { p4d 210 mm/kasan/init.c p = pud_alloc(&init_mm, p4d, addr); p4d 214 mm/kasan/init.c p4d_populate(&init_mm, p4d, p4d 218 mm/kasan/init.c zero_pud_populate(p4d, addr, next); p4d 219 mm/kasan/init.c } while (p4d++, addr = next, addr != end); p4d 242 mm/kasan/init.c p4d_t *p4d; p4d 267 mm/kasan/init.c p4d = p4d_offset(pgd, addr); p4d 268 mm/kasan/init.c p4d_populate(&init_mm, p4d, p4d 270 mm/kasan/init.c pud = pud_offset(p4d, addr); p4d 327 mm/kasan/init.c static void kasan_free_pud(pud_t *pud_start, p4d_t *p4d) p4d 338 mm/kasan/init.c pud_free(&init_mm, (pud_t *)page_to_virt(p4d_page(*p4d))); p4d 339 mm/kasan/init.c p4d_clear(p4d); p4d 344 mm/kasan/init.c p4d_t *p4d; p4d 348 mm/kasan/init.c p4d = p4d_start + i; p4d 349 mm/kasan/init.c if (!p4d_none(*p4d)) p4d 427 mm/kasan/init.c static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr, p4d 432 mm/kasan/init.c for (; addr < end; addr = next, p4d++) { p4d 437 mm/kasan/init.c if (!p4d_present(*p4d)) p4d 440 mm/kasan/init.c if (kasan_pud_table(*p4d)) { p4d 443 mm/kasan/init.c p4d_clear(p4d); p4d 446 mm/kasan/init.c pud = pud_offset(p4d, addr); p4d 448 mm/kasan/init.c kasan_free_pud(pud_offset(p4d, 0), p4d); p4d 466 mm/kasan/init.c p4d_t *p4d; p4d 481 mm/kasan/init.c p4d = p4d_offset(pgd, addr); p4d 482 mm/kasan/init.c kasan_remove_p4d_table(p4d, addr, next); p4d 269 mm/memory-failure.c p4d_t *p4d; p4d 277 mm/memory-failure.c p4d = p4d_offset(pgd, address); p4d 278 mm/memory-failure.c if (!p4d_present(*p4d)) p4d 280 mm/memory-failure.c pud = pud_offset(p4d, address); p4d 238 mm/memory.c static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, p4d 247 mm/memory.c pud = pud_offset(p4d, addr); p4d 266 mm/memory.c pud = pud_offset(p4d, start); p4d 267 mm/memory.c p4d_clear(p4d); p4d 276 mm/memory.c p4d_t *p4d; p4d 281 mm/memory.c p4d = p4d_offset(pgd, addr); p4d 284 mm/memory.c if (p4d_none_or_clear_bad(p4d)) p4d 286 mm/memory.c free_pud_range(tlb, p4d, addr, next, floor, ceiling); p4d 287 mm/memory.c } while (p4d++, addr = next, addr != end); p4d 300 mm/memory.c p4d = p4d_offset(pgd, start); p4d 302 mm/memory.c p4d_free_tlb(tlb, p4d, start); p4d 485 mm/memory.c p4d_t *p4d = p4d_offset(pgd, addr); p4d 486 mm/memory.c pud_t *pud = pud_offset(p4d, addr); p4d 1175 mm/memory.c struct vm_area_struct *vma, p4d_t *p4d, p4d 1182 mm/memory.c pud = pud_offset(p4d, addr); p4d 1208 mm/memory.c p4d_t *p4d; p4d 1211 mm/memory.c p4d = p4d_offset(pgd, addr); p4d 1214 mm/memory.c if (p4d_none_or_clear_bad(p4d)) p4d 1216 mm/memory.c next = zap_pud_range(tlb, vma, p4d, addr, next, details); p4d 1217 mm/memory.c } while (p4d++, addr = next, addr != end); p4d 1396 mm/memory.c p4d_t *p4d; p4d 1401 mm/memory.c p4d = p4d_alloc(mm, pgd, addr); p4d 1402 mm/memory.c if (!p4d) p4d 1404 mm/memory.c pud = pud_alloc(mm, p4d, addr); p4d 1840 mm/memory.c static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, p4d 1849 mm/memory.c pud = pud_alloc(mm, p4d, addr); p4d 1866 mm/memory.c p4d_t *p4d; p4d 1871 mm/memory.c p4d = p4d_alloc(mm, pgd, addr); p4d 1872 mm/memory.c if (!p4d) p4d 1876 mm/memory.c err = remap_pud_range(mm, p4d, addr, next, p4d 1880 mm/memory.c } while (p4d++, addr = next, addr != end); p4d 2057 mm/memory.c static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d, p4d 2065 mm/memory.c pud = pud_alloc(mm, p4d, addr); p4d 2081 mm/memory.c p4d_t *p4d; p4d 2085 mm/memory.c p4d = p4d_alloc(mm, pgd, addr); p4d 2086 mm/memory.c if (!p4d) p4d 2090 mm/memory.c err = apply_to_pud_range(mm, p4d, addr, next, fn, data); p4d 2093 mm/memory.c } while (p4d++, addr = next, addr != end); p4d 3922 mm/memory.c p4d_t *p4d; p4d 3926 mm/memory.c p4d = p4d_alloc(mm, pgd, address); p4d 3927 mm/memory.c if (!p4d) p4d 3930 mm/memory.c vmf.pud = pud_alloc(mm, p4d, address); p4d 4072 mm/memory.c int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) p4d 4082 mm/memory.c if (!p4d_present(*p4d)) { p4d 4084 mm/memory.c p4d_populate(mm, p4d, new); p4d 4088 mm/memory.c if (!pgd_present(*p4d)) { p4d 4090 mm/memory.c pgd_populate(mm, p4d, new); p4d 4137 mm/memory.c p4d_t *p4d; p4d 4146 mm/memory.c p4d = p4d_offset(pgd, address); p4d 4147 mm/memory.c if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d))) p4d 4150 mm/memory.c pud = pud_offset(p4d, address); p4d 262 mm/mprotect.c p4d_t *p4d, unsigned long addr, unsigned long end, p4d 269 mm/mprotect.c pud = pud_offset(p4d, addr); p4d 285 mm/mprotect.c p4d_t *p4d; p4d 289 mm/mprotect.c p4d = p4d_offset(pgd, addr); p4d 292 mm/mprotect.c if (p4d_none_or_clear_bad(p4d)) p4d 294 mm/mprotect.c pages += change_pud_range(vma, p4d, addr, next, newprot, p4d 296 mm/mprotect.c } while (p4d++, addr = next, addr != end); p4d 36 mm/mremap.c p4d_t *p4d; p4d 44 mm/mremap.c p4d = p4d_offset(pgd, addr); p4d 45 mm/mremap.c if (p4d_none_or_clear_bad(p4d)) p4d 48 mm/mremap.c pud = pud_offset(p4d, addr); p4d 63 mm/mremap.c p4d_t *p4d; p4d 68 mm/mremap.c p4d = p4d_alloc(mm, pgd, addr); p4d 69 mm/mremap.c if (!p4d) p4d 71 mm/mremap.c pud = pud_alloc(mm, p4d, addr); p4d 143 mm/page_vma_mapped.c p4d_t *p4d; p4d 170 mm/page_vma_mapped.c p4d = p4d_offset(pgd, pvmw->address); p4d 171 mm/page_vma_mapped.c if (!p4d_present(*p4d)) p4d 173 mm/page_vma_mapped.c pud = pud_offset(p4d, pvmw->address); p4d 75 mm/pagewalk.c static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, p4d 83 mm/pagewalk.c pud = pud_offset(p4d, addr); p4d 123 mm/pagewalk.c p4d_t *p4d; p4d 128 mm/pagewalk.c p4d = p4d_offset(pgd, addr); p4d 131 mm/pagewalk.c if (p4d_none_or_clear_bad(p4d)) { p4d 139 mm/pagewalk.c err = walk_pud_range(p4d, addr, next, walk); p4d 142 mm/pagewalk.c } while (p4d++, addr = next, addr != end); p4d 27 mm/pgtable-generic.c void p4d_clear_bad(p4d_t *p4d) p4d 29 mm/pgtable-generic.c p4d_ERROR(*p4d); p4d 30 mm/pgtable-generic.c p4d_clear(p4d); p4d 714 mm/rmap.c p4d_t *p4d; p4d 723 mm/rmap.c p4d = p4d_offset(pgd, address); p4d 724 mm/rmap.c if (!p4d_present(*p4d)) p4d 727 mm/rmap.c pud = pud_offset(p4d, address); p4d 180 mm/sparse-vmemmap.c pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) p4d 182 mm/sparse-vmemmap.c pud_t *pud = pud_offset(p4d, addr); p4d 194 mm/sparse-vmemmap.c p4d_t *p4d = p4d_offset(pgd, addr); p4d 195 mm/sparse-vmemmap.c if (p4d_none(*p4d)) { p4d 199 mm/sparse-vmemmap.c p4d_populate(&init_mm, p4d, p); p4d 201 mm/sparse-vmemmap.c return p4d; p4d 221 mm/sparse-vmemmap.c p4d_t *p4d; p4d 230 mm/sparse-vmemmap.c p4d = vmemmap_p4d_populate(pgd, addr, node); p4d 231 mm/sparse-vmemmap.c if (!p4d) p4d 233 mm/sparse-vmemmap.c pud = vmemmap_pud_populate(p4d, addr, node); p4d 2000 mm/swapfile.c static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, p4d 2009 mm/swapfile.c pud = pud_offset(p4d, addr); p4d 2027 mm/swapfile.c p4d_t *p4d; p4d 2031 mm/swapfile.c p4d = p4d_offset(pgd, addr); p4d 2034 mm/swapfile.c if (p4d_none_or_clear_bad(p4d)) p4d 2036 mm/swapfile.c ret = unuse_pud_range(vma, p4d, addr, next, type, p4d 2040 mm/swapfile.c } while (p4d++, addr = next, addr != end); p4d 150 mm/userfaultfd.c p4d_t *p4d; p4d 154 mm/userfaultfd.c p4d = p4d_alloc(mm, pgd, address); p4d 155 mm/userfaultfd.c if (!p4d) p4d 157 mm/userfaultfd.c pud = pud_alloc(mm, p4d, address); p4d 91 mm/vmalloc.c static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end) p4d 96 mm/vmalloc.c pud = pud_offset(p4d, addr); p4d 109 mm/vmalloc.c p4d_t *p4d; p4d 112 mm/vmalloc.c p4d = p4d_offset(pgd, addr); p4d 115 mm/vmalloc.c if (p4d_clear_huge(p4d)) p4d 117 mm/vmalloc.c if (p4d_none_or_clear_bad(p4d)) p4d 119 mm/vmalloc.c vunmap_pud_range(p4d, addr, next); p4d 120 mm/vmalloc.c } while (p4d++, addr = next, addr != end); p4d 181 mm/vmalloc.c static int vmap_pud_range(p4d_t *p4d, unsigned long addr, p4d 187 mm/vmalloc.c pud = pud_alloc(&init_mm, p4d, addr); p4d 201 mm/vmalloc.c p4d_t *p4d; p4d 204 mm/vmalloc.c p4d = p4d_alloc(&init_mm, pgd, addr); p4d 205 mm/vmalloc.c if (!p4d) p4d 209 mm/vmalloc.c if (vmap_pud_range(p4d, addr, next, prot, pages, nr)) p4d 211 mm/vmalloc.c } while (p4d++, addr = next, addr != end); p4d 275 mm/vmalloc.c p4d_t *p4d; p4d 288 mm/vmalloc.c p4d = p4d_offset(pgd, addr); p4d 289 mm/vmalloc.c if (p4d_none(*p4d)) p4d 291 mm/vmalloc.c pud = pud_offset(p4d, addr);