60 			   unsigned long addr, unsigned long end)  in ident_pmd_init()  argument
62 	addr &= PMD_MASK;  in ident_pmd_init()
63 	for (; addr < end; addr += PMD_SIZE) {  in ident_pmd_init()
64 		pmd_t *pmd = pmd_page + pmd_index(addr);  in ident_pmd_init()
67 			set_pmd(pmd, __pmd(addr | pmd_flag));  in ident_pmd_init()
71 			  unsigned long addr, unsigned long end)  in ident_pud_init()  argument
75 	for (; addr < end; addr = next) {  in ident_pud_init()
76 		pud_t *pud = pud_page + pud_index(addr);  in ident_pud_init()
79 		next = (addr & PUD_MASK) + PUD_SIZE;  in ident_pud_init()
85 			ident_pmd_init(info->pmd_flag, pmd, addr, next);  in ident_pud_init()
91 		ident_pmd_init(info->pmd_flag, pmd, addr, next);  in ident_pud_init()
99 			      unsigned long addr, unsigned long end)  in kernel_ident_mapping_init()  argument
105 	for (; addr < end; addr = next) {  in kernel_ident_mapping_init()
106 		pgd_t *pgd = pgd_page + pgd_index(addr) + off;  in kernel_ident_mapping_init()
109 		next = (addr & PGDIR_MASK) + PGDIR_SIZE;  in kernel_ident_mapping_init()
115 			result = ident_pud_init(info, pud, addr, next);  in kernel_ident_mapping_init()
124 		result = ident_pud_init(info, pud, addr, next);  in kernel_ident_mapping_init()
402 phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end,  in phys_pte_init()  argument
409 	pte_t *pte = pte_page + pte_index(addr);  in phys_pte_init()
411 	for (i = pte_index(addr); i < PTRS_PER_PTE; i++, addr = next, pte++) {  in phys_pte_init()
412 		next = (addr & PAGE_MASK) + PAGE_SIZE;  in phys_pte_init()
413 		if (addr >= end) {  in phys_pte_init()
415 			    !e820_any_mapped(addr & PAGE_MASK, next, E820_RAM) &&  in phys_pte_init()
416 			    !e820_any_mapped(addr & PAGE_MASK, next, E820_RESERVED_KERN))  in phys_pte_init()
435 			       pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte);  in phys_pte_init()
437 		set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, prot));  in phys_pte_init()
438 		last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE;  in phys_pte_init()
522 phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,  in phys_pud_init()  argument
527 	int i = pud_index(addr);  in phys_pud_init()
529 	for (; i < PTRS_PER_PUD; i++, addr = next) {  in phys_pud_init()
530 		pud_t *pud = pud_page + pud_index(addr);  in phys_pud_init()
534 		next = (addr & PUD_MASK) + PUD_SIZE;  in phys_pud_init()
535 		if (addr >= end) {  in phys_pud_init()
537 			    !e820_any_mapped(addr & PUD_MASK, next, E820_RAM) &&  in phys_pud_init()
538 			    !e820_any_mapped(addr & PUD_MASK, next, E820_RESERVED_KERN))  in phys_pud_init()
546 				last_map_addr = phys_pmd_init(pmd, addr, end,  in phys_pud_init()
576 				pfn_pte((addr & PUD_MASK) >> PAGE_SHIFT,  in phys_pud_init()
584 		last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask,  in phys_pud_init()
605 	unsigned long addr;  in kernel_physical_mapping_init()  local
609 	addr = start;  in kernel_physical_mapping_init()
635 		sync_global_pgds(addr, end - 1, 0);  in kernel_physical_mapping_init()
791 remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,  in remove_pte_table()  argument
799 	pte = pte_start + pte_index(addr);  in remove_pte_table()
800 	for (; addr < end; addr = next, pte++) {  in remove_pte_table()
801 		next = (addr + PAGE_SIZE) & PAGE_MASK;  in remove_pte_table()
813 		phys_addr = pte_val(*pte) + (addr & PAGE_MASK);  in remove_pte_table()
817 		if (IS_ALIGNED(addr, PAGE_SIZE) &&  in remove_pte_table()
827 			pte_clear(&init_mm, addr, pte);  in remove_pte_table()
843 			memset((void *)addr, PAGE_INUSE, next - addr);  in remove_pte_table()
850 				pte_clear(&init_mm, addr, pte);  in remove_pte_table()
863 remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,  in remove_pmd_table()  argument
871 	pmd = pmd_start + pmd_index(addr);  in remove_pmd_table()
872 	for (; addr < end; addr = next, pmd++) {  in remove_pmd_table()
873 		next = pmd_addr_end(addr, end);  in remove_pmd_table()
879 			if (IS_ALIGNED(addr, PMD_SIZE) &&  in remove_pmd_table()
891 				memset((void *)addr, PAGE_INUSE, next - addr);  in remove_pmd_table()
909 		remove_pte_table(pte_base, addr, next, direct);  in remove_pmd_table()
919 remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,  in remove_pud_table()  argument
927 	pud = pud_start + pud_index(addr);  in remove_pud_table()
928 	for (; addr < end; addr = next, pud++) {  in remove_pud_table()
929 		next = pud_addr_end(addr, end);  in remove_pud_table()
935 			if (IS_ALIGNED(addr, PUD_SIZE) &&  in remove_pud_table()
947 				memset((void *)addr, PAGE_INUSE, next - addr);  in remove_pud_table()
965 		remove_pmd_table(pmd_base, addr, next, direct);  in remove_pud_table()
978 	unsigned long addr;  in remove_pagetable()  local
983 	for (addr = start; addr < end; addr = next) {  in remove_pagetable()
984 		next = pgd_addr_end(addr, end);  in remove_pagetable()
986 		pgd = pgd_offset_k(addr);  in remove_pagetable()
991 		remove_pud_table(pud, addr, next, direct);  in remove_pagetable()
1159 int kern_addr_valid(unsigned long addr)  in kern_addr_valid()  argument
1161 	unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;  in kern_addr_valid()
1170 	pgd = pgd_offset_k(addr);  in kern_addr_valid()
1174 	pud = pud_offset(pgd, addr);  in kern_addr_valid()
1181 	pmd = pmd_offset(pud, addr);  in kern_addr_valid()
1188 	pte = pte_offset_kernel(pmd, addr);  in kern_addr_valid()
1241 	unsigned long addr;  in vmemmap_populate_hugepages()  local
1247 	for (addr = start; addr < end; addr = next) {  in vmemmap_populate_hugepages()
1248 		next = pmd_addr_end(addr, end);  in vmemmap_populate_hugepages()
1250 		pgd = vmemmap_pgd_populate(addr, node);  in vmemmap_populate_hugepages()
1254 		pud = vmemmap_pud_populate(pgd, addr, node);  in vmemmap_populate_hugepages()
1258 		pmd = pmd_offset(pud, addr);  in vmemmap_populate_hugepages()
1275 					addr_start = addr;  in vmemmap_populate_hugepages()
1280 				addr_end = addr + PMD_SIZE;  in vmemmap_populate_hugepages()
1285 			vmemmap_verify((pte_t *)pmd, node, addr, next);  in vmemmap_populate_hugepages()
1289 		if (vmemmap_populate_basepages(addr, next, node))  in vmemmap_populate_hugepages()
1312 	unsigned long addr = (unsigned long)start_page;  in register_page_bootmem_memmap()  local
1321 	for (; addr < end; addr = next) {  in register_page_bootmem_memmap()
1324 		pgd = pgd_offset_k(addr);  in register_page_bootmem_memmap()
1326 			next = (addr + PAGE_SIZE) & PAGE_MASK;  in register_page_bootmem_memmap()
1331 		pud = pud_offset(pgd, addr);  in register_page_bootmem_memmap()
1333 			next = (addr + PAGE_SIZE) & PAGE_MASK;  in register_page_bootmem_memmap()
1339 			next = (addr + PAGE_SIZE) & PAGE_MASK;  in register_page_bootmem_memmap()
1340 			pmd = pmd_offset(pud, addr);  in register_page_bootmem_memmap()
1346 			pte = pte_offset_kernel(pmd, addr);  in register_page_bootmem_memmap()
1352 			next = pmd_addr_end(addr, end);  in register_page_bootmem_memmap()
1354 			pmd = pmd_offset(pud, addr);  in register_page_bootmem_memmap()