1417 static unsigned long __ref kernel_map_hugepud(unsigned long vstart,  in kernel_map_hugepud()  argument
1422 	u64 pte_val = vstart;  in kernel_map_hugepud()
1425 	if ((vstart & mask16gb) ||  in kernel_map_hugepud()
1426 	    (vend - vstart <= mask16gb)) {  in kernel_map_hugepud()
1430 		return vstart + PUD_SIZE;  in kernel_map_hugepud()
1436 	vend = vstart + mask16gb + 1UL;  in kernel_map_hugepud()
1437 	while (vstart < vend) {  in kernel_map_hugepud()
1441 		vstart += PUD_SIZE;  in kernel_map_hugepud()
1444 	return vstart;  in kernel_map_hugepud()
1447 static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend,  in kernel_can_map_hugepud()  argument
1450 	if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE)  in kernel_can_map_hugepud()
1456 static unsigned long __ref kernel_map_hugepmd(unsigned long vstart,  in kernel_map_hugepmd()  argument
1462 	u64 pte_val = vstart;  in kernel_map_hugepmd()
1465 	if ((vstart & mask256mb) ||  in kernel_map_hugepmd()
1466 	    (vend - vstart <= mask256mb)) {  in kernel_map_hugepmd()
1470 		return vstart + PMD_SIZE;  in kernel_map_hugepmd()
1473 	if ((vstart & mask2gb) ||  in kernel_map_hugepmd()
1474 	    (vend - vstart <= mask2gb)) {  in kernel_map_hugepmd()
1477 		vend = vstart + mask256mb + 1UL;  in kernel_map_hugepmd()
1481 		vend = vstart + mask2gb + 1UL;  in kernel_map_hugepmd()
1484 	while (vstart < vend) {  in kernel_map_hugepmd()
1488 		vstart += PMD_SIZE;  in kernel_map_hugepmd()
1492 	return vstart;  in kernel_map_hugepmd()
1495 static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend,  in kernel_can_map_hugepmd()  argument
1498 	if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE)  in kernel_can_map_hugepmd()
1508 	unsigned long vstart = PAGE_OFFSET + pstart;  in kernel_map_range()  local
1512 	if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {  in kernel_map_range()
1514 			    vstart, vend);  in kernel_map_range()
1518 	while (vstart < vend) {  in kernel_map_range()
1519 		unsigned long this_end, paddr = __pa(vstart);  in kernel_map_range()
1520 		pgd_t *pgd = pgd_offset_k(vstart);  in kernel_map_range()
1532 		pud = pud_offset(pgd, vstart);  in kernel_map_range()
1536 			if (kernel_can_map_hugepud(vstart, vend, use_huge)) {  in kernel_map_range()
1537 				vstart = kernel_map_hugepud(vstart, vend, pud);  in kernel_map_range()
1545 		pmd = pmd_offset(pud, vstart);  in kernel_map_range()
1549 			if (kernel_can_map_hugepmd(vstart, vend, use_huge)) {  in kernel_map_range()
1550 				vstart = kernel_map_hugepmd(vstart, vend, pmd);  in kernel_map_range()
1558 		pte = pte_offset_kernel(pmd, vstart);  in kernel_map_range()
1559 		this_end = (vstart + PMD_SIZE) & PMD_MASK;  in kernel_map_range()
1563 		while (vstart < this_end) {  in kernel_map_range()
1566 			vstart += PAGE_SIZE;  in kernel_map_range()
2338 int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,  in vmemmap_populate()  argument
2352 	vstart = vstart & PMD_MASK;  in vmemmap_populate()
2354 	for (; vstart < vend; vstart += PMD_SIZE) {  in vmemmap_populate()
2355 		pgd_t *pgd = pgd_offset_k(vstart);  in vmemmap_populate()
2368 		pud = pud_offset(pgd, vstart);  in vmemmap_populate()
2377 		pmd = pmd_offset(pud, vstart);  in vmemmap_populate()