87 static inline void pgd_list_add(pgd_t *pgd)  in pgd_list_add()  argument
89 	struct page *page = virt_to_page(pgd);  in pgd_list_add()
94 static inline void pgd_list_del(pgd_t *pgd)  in pgd_list_del()  argument
96 	struct page *page = virt_to_page(pgd);  in pgd_list_del()
105 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)  in pgd_set_mm()  argument
107 	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));  in pgd_set_mm()
108 	virt_to_page(pgd)->index = (pgoff_t)mm;  in pgd_set_mm()
116 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)  in pgd_ctor()  argument
124 		clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,  in pgd_ctor()
131 		pgd_set_mm(pgd, mm);  in pgd_ctor()
132 		pgd_list_add(pgd);  in pgd_ctor()
136 static void pgd_dtor(pgd_t *pgd)  in pgd_dtor()  argument
142 	pgd_list_del(pgd);  in pgd_dtor()
244 		pgd_t pgd = pgdp[i];  in pgd_mop_up_pmds()  local
246 		if (pgd_val(pgd) != 0) {  in pgd_mop_up_pmds()
247 			pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);  in pgd_mop_up_pmds()
251 			paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);  in pgd_mop_up_pmds()
258 static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])  in pgd_prepopulate_pmd()  argument
266 	pud = pud_offset(pgd, 0);  in pgd_prepopulate_pmd()
335 static inline void _pgd_free(pgd_t *pgd)  in _pgd_free()  argument
338 		free_page((unsigned long)pgd);  in _pgd_free()
340 		kmem_cache_free(pgd_cache, pgd);  in _pgd_free()
348 static inline void _pgd_free(pgd_t *pgd)  in _pgd_free()  argument
350 	free_page((unsigned long)pgd);  in _pgd_free()
356 	pgd_t *pgd;  in pgd_alloc()  local
359 	pgd = _pgd_alloc();  in pgd_alloc()
361 	if (pgd == NULL)  in pgd_alloc()
364 	mm->pgd = pgd;  in pgd_alloc()
379 	pgd_ctor(mm, pgd);  in pgd_alloc()
380 	pgd_prepopulate_pmd(mm, pgd, pmds);  in pgd_alloc()
384 	return pgd;  in pgd_alloc()
389 	_pgd_free(pgd);  in pgd_alloc()
394 void pgd_free(struct mm_struct *mm, pgd_t *pgd)  in pgd_free()  argument
396 	pgd_mop_up_pmds(mm, pgd);  in pgd_free()
397 	pgd_dtor(pgd);  in pgd_free()
398 	paravirt_pgd_free(mm, pgd);  in pgd_free()
399 	_pgd_free(pgd);  in pgd_free()