201 					     struct guest_walker *walker,  in FNAME()
214 	for (level = walker->max_level; level >= walker->level; --level) {  in FNAME()
215 		pte = orig_pte = walker->ptes[level - 1];  in FNAME()
216 		table_gfn = walker->table_gfn[level - 1];  in FNAME()
217 		ptep_user = walker->ptep_user[level - 1];  in FNAME()
223 		if (level == walker->level && write_fault &&  in FNAME()
244 		if (unlikely(!walker->pte_writable[level - 1]))  in FNAME()
252 		walker->ptes[level - 1] = pte;  in FNAME()
260 static int FNAME(walk_addr_generic)(struct guest_walker *walker,  in FNAME()
280 	walker->level = mmu->root_level;  in FNAME()
284 	if (walker->level == PT32E_ROOT_LEVEL) {  in FNAME()
286 		trace_kvm_mmu_paging_element(pte, walker->level);  in FNAME()
289 		--walker->level;  in FNAME()
292 	walker->max_level = walker->level;  in FNAME()
297 	++walker->level;  in FNAME()
304 		--walker->level;  in FNAME()
306 		index = PT_INDEX(addr, walker->level);  in FNAME()
311 		walker->table_gfn[walker->level - 1] = table_gfn;  in FNAME()
312 		walker->pte_gpa[walker->level - 1] = pte_gpa;  in FNAME()
316 					      &walker->fault);  in FNAME()
334 					    &walker->pte_writable[walker->level - 1]);  in FNAME()
341 		walker->ptep_user[walker->level - 1] = ptep_user;  in FNAME()
343 		trace_kvm_mmu_paging_element(pte, walker->level);  in FNAME()
348 		if (unlikely(is_rsvd_bits_set(mmu, pte, walker->level))) {  in FNAME()
356 		walker->ptes[walker->level - 1] = pte;  in FNAME()
357 	} while (!is_last_gpte(mmu, walker->level, pte));  in FNAME()
364 	gfn = gpte_to_gfn_lvl(pte, walker->level);  in FNAME()
365 	gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT;  in FNAME()
367 	if (PTTYPE == 32 && walker->level == PT_DIRECTORY_LEVEL && is_cpuid_PSE36())  in FNAME()
370 	real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), access, &walker->fault);  in FNAME()
374 	walker->gfn = real_gpa >> PAGE_SHIFT;  in FNAME()
388 		ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker, write_fault);  in FNAME()
395 	walker->pt_access = pt_access;  in FNAME()
396 	walker->pte_access = pte_access;  in FNAME()
407 	walker->fault.vector = PF_VECTOR;  in FNAME()
408 	walker->fault.error_code_valid = true;  in FNAME()
409 	walker->fault.error_code = errcode;  in FNAME()
429 	walker->fault.address = addr;  in FNAME()
430 	walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;  in FNAME()
432 	trace_kvm_mmu_walker_error(walker->fault.error_code);  in FNAME()
436 static int FNAME(walk_addr)(struct guest_walker *walker,  in FNAME()
439 	return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.mmu, addr,  in FNAME()
444 static int FNAME(walk_addr_nested)(struct guest_walker *walker,  in FNAME()
448 	return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.nested_mmu,  in FNAME()
657 			      struct guest_walker *walker, int user_fault,  in FNAME()
661 	gfn_t mask = ~(KVM_PAGES_PER_HPAGE(walker->level) - 1);  in FNAME()
664 	if (!(walker->pte_access & ACC_WRITE_MASK ||  in FNAME()
668 	for (level = walker->level; level <= walker->max_level; level++) {  in FNAME()
669 		gfn_t gfn = walker->gfn ^ walker->table_gfn[level - 1];  in FNAME()
697 	struct guest_walker walker;  in FNAME()  local
727 	r = FNAME(walk_addr)(&walker, vcpu, addr, error_code);  in FNAME()
735 			inject_page_fault(vcpu, &walker.fault);  in FNAME()
743 	      &walker, user_fault, &vcpu->arch.write_fault_to_shadow_pgtable);  in FNAME()
745 	if (walker.level >= PT_DIRECTORY_LEVEL && !is_self_change_mapping) {  in FNAME()
746 		level = mapping_level(vcpu, walker.gfn, &force_pt_level);  in FNAME()
748 			level = min(walker.level, level);  in FNAME()
749 			walker.gfn = walker.gfn & ~(KVM_PAGES_PER_HPAGE(level) - 1);  in FNAME()
757 	if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault,  in FNAME()
762 				walker.gfn, pfn, walker.pte_access, &r))  in FNAME()
769 	if (write_fault && !(walker.pte_access & ACC_WRITE_MASK) &&  in FNAME()
772 		walker.pte_access |= ACC_WRITE_MASK;  in FNAME()
773 		walker.pte_access &= ~ACC_USER_MASK;  in FNAME()
782 			walker.pte_access &= ~ACC_EXEC_MASK;  in FNAME()
792 		transparent_hugepage_adjust(vcpu, &walker.gfn, &pfn, &level);  in FNAME()
793 	r = FNAME(fetch)(vcpu, addr, &walker, write_fault,  in FNAME()
877 	struct guest_walker walker;  in FNAME()  local
881 	r = FNAME(walk_addr)(&walker, vcpu, vaddr, access);  in FNAME()
884 		gpa = gfn_to_gpa(walker.gfn);  in FNAME()
887 		*exception = walker.fault;  in FNAME()
897 	struct guest_walker walker;  in FNAME()  local
901 	r = FNAME(walk_addr_nested)(&walker, vcpu, vaddr, access);  in FNAME()
904 		gpa = gfn_to_gpa(walker.gfn);  in FNAME()
907 		*exception = walker.fault;  in FNAME()