85 #define PT64_LEVEL_SHIFT(level) \  argument
86 		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
88 #define PT64_INDEX(address, level)\  argument
89 	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
94 #define PT32_LEVEL_SHIFT(level) \  argument
95 		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
97 #define PT32_LVL_OFFSET_MASK(level) \  argument
98 	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
101 #define PT32_INDEX(address, level)\  argument
102 	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
108 #define PT64_LVL_ADDR_MASK(level) \  argument
109 	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
111 #define PT64_LVL_OFFSET_MASK(level) \  argument
112 	(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
118 #define PT32_LVL_ADDR_MASK(level) \  argument
119 	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
138 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)  argument
152 	int level;  member
319 static int is_last_spte(u64 pte, int level)  in is_last_spte()  argument
321 	if (level == PT_PAGE_TABLE_LEVEL)  in is_last_spte()
759 	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));  in kvm_mmu_page_get_gfn()
776 					      int level)  in lpage_info_slot()  argument
780 	idx = gfn_to_index(gfn, slot->base_gfn, level);  in lpage_info_slot()
781 	return &slot->arch.lpage_info[level - 2][idx];  in lpage_info_slot()
817 				int level)  in has_wrprotected_page()  argument
824 		linfo = lpage_info_slot(gfn, slot, level);  in has_wrprotected_page()
870 	int host_level, level, max_level;  in mapping_level()  local
879 	for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)  in mapping_level()
880 		if (has_wrprotected_page(vcpu->kvm, large_gfn, level))  in mapping_level()
883 	return level - 1;  in mapping_level()
1010 static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,  in __gfn_to_rmap()  argument
1015 	idx = gfn_to_index(gfn, slot->base_gfn, level);  in __gfn_to_rmap()
1016 	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];  in __gfn_to_rmap()
1022 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)  in gfn_to_rmap()  argument
1027 	return __gfn_to_rmap(gfn, level, slot);  in gfn_to_rmap()
1045 	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);  in rmap_add()
1057 	rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);  in rmap_remove()
1132 		WARN_ON(page_header(__pa(sptep))->role.level ==  in __drop_large_spte()
1341 			   struct kvm_memory_slot *slot, gfn_t gfn, int level,  in kvm_unmap_rmapp()  argument
1351 			     sptep, *sptep, gfn, level);  in kvm_unmap_rmapp()
1361 			     struct kvm_memory_slot *slot, gfn_t gfn, int level,  in kvm_set_pte_rmapp()  argument
1377 			     sptep, *sptep, gfn, level);  in kvm_set_pte_rmapp()
1412 					       int level,  in kvm_handle_hva_range()  argument
1467 					 gfn_t gfn, int level,  in kvm_handle_hva()  argument
1489 			 struct kvm_memory_slot *slot, gfn_t gfn, int level,  in kvm_age_rmapp()  argument
1508 	trace_kvm_age_page(gfn, level, slot, young);  in kvm_age_rmapp()
1514 			      int level, unsigned long data)  in kvm_test_age_rmapp()  argument
1550 	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);  in rmap_recycle()
1552 	kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, gfn, sp->role.level, 0);  in rmap_recycle()
1882 		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);  in kvm_sync_pages()
1917 		if (sp->role.level == PT_PAGE_TABLE_LEVEL) {  in mmu_pages_next()
1922 		parents->parent[sp->role.level-2] = sp;  in mmu_pages_next()
1923 		parents->idx[sp->role.level-1] = pvec->page[n].idx;  in mmu_pages_next()
1932 	unsigned int level = 0;  in mmu_pages_clear_parents()  local
1935 		unsigned int idx = parents->idx[level];  in mmu_pages_clear_parents()
1937 		sp = parents->parent[level];  in mmu_pages_clear_parents()
1944 		level++;  in mmu_pages_clear_parents()
1945 	} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);  in mmu_pages_clear_parents()
1952 	parents->parent[parent->role.level-1] = NULL;  in kvm_mmu_pages_init()
2013 					     unsigned level,  in kvm_mmu_get_page()  argument
2024 	role.level = level;  in kvm_mmu_get_page()
2031 		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));  in kvm_mmu_get_page()
2032 		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;  in kvm_mmu_get_page()
2070 		if (level > PT_PAGE_TABLE_LEVEL && need_sync)  in kvm_mmu_get_page()
2086 	iterator->level = vcpu->arch.mmu.shadow_root_level;  in shadow_walk_init()
2088 	if (iterator->level == PT64_ROOT_LEVEL &&  in shadow_walk_init()
2091 		--iterator->level;  in shadow_walk_init()
2093 	if (iterator->level == PT32E_ROOT_LEVEL) {  in shadow_walk_init()
2097 		--iterator->level;  in shadow_walk_init()
2099 			iterator->level = 0;  in shadow_walk_init()
2105 	if (iterator->level < PT_PAGE_TABLE_LEVEL)  in shadow_walk_okay()
2108 	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);  in shadow_walk_okay()
2116 	if (is_last_spte(spte, iterator->level)) {  in __shadow_walk_next()
2117 		iterator->level = 0;  in __shadow_walk_next()
2122 	--iterator->level;  in __shadow_walk_next()
2176 		if (is_last_spte(pte, sp->role.level)) {  in mmu_page_zap_pte()
2224 	if (parent->role.level == PT_PAGE_TABLE_LEVEL)  in mmu_zap_unsync_children()
2487 		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);  in kvm_unsync_pages()
2502 		if (s->role.level != PT_PAGE_TABLE_LEVEL)  in mmu_need_write_protect()
2514 		    unsigned pte_access, int level,  in set_spte()  argument
2536 	if (level > PT_PAGE_TABLE_LEVEL)  in set_spte()
2557 		if (level > PT_PAGE_TABLE_LEVEL &&  in set_spte()
2558 		    has_wrprotected_page(vcpu->kvm, gfn, level))  in set_spte()
2595 			 int level, gfn_t gfn, pfn_t pfn, bool speculative,  in mmu_set_spte()  argument
2609 		if (level > PT_PAGE_TABLE_LEVEL &&  in mmu_set_spte()
2626 	if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,  in mmu_set_spte()
2686 			     sp->role.level, gfn, page_to_pfn(pages[i]),  in direct_pte_prefetch_many()
2729 	if (sp->role.level > PT_PAGE_TABLE_LEVEL)  in direct_pte_prefetch()
2736 			int map_writable, int level, gfn_t gfn, pfn_t pfn,  in __direct_map()  argument
2748 		if (iterator.level == level) {  in __direct_map()
2750 				     write, &emulate, level, gfn, pfn,  in __direct_map()
2761 			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);  in __direct_map()
2764 					      iterator.level - 1,  in __direct_map()
2810 	int level = *levelp;  in transparent_hugepage_adjust()  local
2819 	    level == PT_PAGE_TABLE_LEVEL &&  in transparent_hugepage_adjust()
2832 		*levelp = level = PT_DIRECTORY_LEVEL;  in transparent_hugepage_adjust()
2833 		mask = KVM_PAGES_PER_HPAGE(level) - 1;  in transparent_hugepage_adjust()
2923 static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,  in fast_page_fault()  argument
2939 		if (!is_shadow_present_pte(spte) || iterator.level < level)  in fast_page_fault()
2952 	if (!is_last_spte(spte, sp->role.level))  in fast_page_fault()
2983 	if (sp->role.level > PT_PAGE_TABLE_LEVEL)  in fast_page_fault()
3008 	int level;  in nonpaging_map()  local
3016 		level = mapping_level(vcpu, gfn);  in nonpaging_map()
3022 		if (level > PT_DIRECTORY_LEVEL)  in nonpaging_map()
3023 			level = PT_DIRECTORY_LEVEL;  in nonpaging_map()
3025 		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);  in nonpaging_map()
3027 		level = PT_PAGE_TABLE_LEVEL;  in nonpaging_map()
3029 	if (fast_page_fault(vcpu, v, level, error_code))  in nonpaging_map()
3046 		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);  in nonpaging_map()
3047 	r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,  in nonpaging_map()
3460 	int level;  in tdp_page_fault()  local
3482 		level = mapping_level(vcpu, gfn);  in tdp_page_fault()
3483 		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);  in tdp_page_fault()
3485 		level = PT_PAGE_TABLE_LEVEL;  in tdp_page_fault()
3487 	if (fast_page_fault(vcpu, gpa, level, error_code))  in tdp_page_fault()
3504 		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);  in tdp_page_fault()
3506 			 level, gfn, pfn, prefault);  in tdp_page_fault()
3565 static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)  in is_last_gpte()  argument
3569 	index = level - 1;  in is_last_gpte()
3762 	unsigned level, root_level = mmu->root_level;  in update_last_pte_bitmap()  local
3769 	for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {  in update_last_pte_bitmap()
3770 		if (level <= PT_PDPE_LEVEL  in update_last_pte_bitmap()
3772 			map |= 1 << (ps_set_index | (level - 1));  in update_last_pte_bitmap()
3779 					 int level)  in paging64_init_context_common()  argument
3782 	context->root_level = level;  in paging64_init_context_common()
3794 	context->shadow_root_level = level;  in paging64_init_context_common()
4019 	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {  in mmu_pte_write_new_pte()
4099 	if (sp->role.level == PT_PAGE_TABLE_LEVEL)  in detect_write_flooding()
4137 	int level;  in get_written_sptes()  local
4140 	level = sp->role.level;  in get_written_sptes()
4149 		if (level == PT32_ROOT_LEVEL) {  in get_written_sptes()
4851 		sptes[iterator.level-1] = spte;  in kvm_mmu_get_spte_hierarchy()