100 static bool __read_mostly nested = 0;  variable
101 module_param(nested, bool, S_IRUGO);
541 	struct nested_vmx nested;  member
784 	return to_vmx(vcpu)->nested.current_vmcs12;  in get_vmcs12()
2271 		to_vmx(vcpu)->nested.vmcs01_tsc_offset :  in vmx_read_l1_tsc()
2310 		to_vmx(vcpu)->nested.vmcs01_tsc_offset = offset;  in vmx_write_tsc_offset()
2330 		to_vmx(vcpu)->nested.vmcs01_tsc_offset += adjustment;  in vmx_adjust_tsc_offset()
2355 	return nested && guest_cpuid_has_vmx(vcpu);  in nested_vmx_allowed()
2387 		vmx->nested.nested_vmx_pinbased_ctls_low,  in nested_vmx_setup_ctls_msrs()
2388 		vmx->nested.nested_vmx_pinbased_ctls_high);  in nested_vmx_setup_ctls_msrs()
2389 	vmx->nested.nested_vmx_pinbased_ctls_low |=  in nested_vmx_setup_ctls_msrs()
2391 	vmx->nested.nested_vmx_pinbased_ctls_high &=  in nested_vmx_setup_ctls_msrs()
2395 	vmx->nested.nested_vmx_pinbased_ctls_high |=  in nested_vmx_setup_ctls_msrs()
2399 		vmx->nested.nested_vmx_pinbased_ctls_high |=  in nested_vmx_setup_ctls_msrs()
2404 		vmx->nested.nested_vmx_exit_ctls_low,  in nested_vmx_setup_ctls_msrs()
2405 		vmx->nested.nested_vmx_exit_ctls_high);  in nested_vmx_setup_ctls_msrs()
2406 	vmx->nested.nested_vmx_exit_ctls_low =  in nested_vmx_setup_ctls_msrs()
2409 	vmx->nested.nested_vmx_exit_ctls_high &=  in nested_vmx_setup_ctls_msrs()
2414 	vmx->nested.nested_vmx_exit_ctls_high |=  in nested_vmx_setup_ctls_msrs()
2420 		vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;  in nested_vmx_setup_ctls_msrs()
2423 	vmx->nested.nested_vmx_true_exit_ctls_low =  in nested_vmx_setup_ctls_msrs()
2424 		vmx->nested.nested_vmx_exit_ctls_low &  in nested_vmx_setup_ctls_msrs()
2429 		vmx->nested.nested_vmx_entry_ctls_low,  in nested_vmx_setup_ctls_msrs()
2430 		vmx->nested.nested_vmx_entry_ctls_high);  in nested_vmx_setup_ctls_msrs()
2431 	vmx->nested.nested_vmx_entry_ctls_low =  in nested_vmx_setup_ctls_msrs()
2433 	vmx->nested.nested_vmx_entry_ctls_high &=  in nested_vmx_setup_ctls_msrs()
2438 	vmx->nested.nested_vmx_entry_ctls_high |=  in nested_vmx_setup_ctls_msrs()
2441 		vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;  in nested_vmx_setup_ctls_msrs()
2444 	vmx->nested.nested_vmx_true_entry_ctls_low =  in nested_vmx_setup_ctls_msrs()
2445 		vmx->nested.nested_vmx_entry_ctls_low &  in nested_vmx_setup_ctls_msrs()
2450 		vmx->nested.nested_vmx_procbased_ctls_low,  in nested_vmx_setup_ctls_msrs()
2451 		vmx->nested.nested_vmx_procbased_ctls_high);  in nested_vmx_setup_ctls_msrs()
2452 	vmx->nested.nested_vmx_procbased_ctls_low =  in nested_vmx_setup_ctls_msrs()
2454 	vmx->nested.nested_vmx_procbased_ctls_high &=  in nested_vmx_setup_ctls_msrs()
2474 	vmx->nested.nested_vmx_procbased_ctls_high |=  in nested_vmx_setup_ctls_msrs()
2479 	vmx->nested.nested_vmx_true_procbased_ctls_low =  in nested_vmx_setup_ctls_msrs()
2480 		vmx->nested.nested_vmx_procbased_ctls_low &  in nested_vmx_setup_ctls_msrs()
2485 		vmx->nested.nested_vmx_secondary_ctls_low,  in nested_vmx_setup_ctls_msrs()
2486 		vmx->nested.nested_vmx_secondary_ctls_high);  in nested_vmx_setup_ctls_msrs()
2487 	vmx->nested.nested_vmx_secondary_ctls_low = 0;  in nested_vmx_setup_ctls_msrs()
2488 	vmx->nested.nested_vmx_secondary_ctls_high &=  in nested_vmx_setup_ctls_msrs()
2499 		vmx->nested.nested_vmx_secondary_ctls_high |=  in nested_vmx_setup_ctls_msrs()
2501 		vmx->nested.nested_vmx_ept_caps = VMX_EPT_PAGE_WALK_4_BIT |  in nested_vmx_setup_ctls_msrs()
2504 		vmx->nested.nested_vmx_ept_caps &= vmx_capability.ept;  in nested_vmx_setup_ctls_msrs()
2510 		vmx->nested.nested_vmx_ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT;  in nested_vmx_setup_ctls_msrs()
2512 		vmx->nested.nested_vmx_ept_caps = 0;  in nested_vmx_setup_ctls_msrs()
2515 		vmx->nested.nested_vmx_secondary_ctls_high |=  in nested_vmx_setup_ctls_msrs()
2520 		vmx->nested.nested_vmx_misc_low,  in nested_vmx_setup_ctls_msrs()
2521 		vmx->nested.nested_vmx_misc_high);  in nested_vmx_setup_ctls_msrs()
2522 	vmx->nested.nested_vmx_misc_low &= VMX_MISC_SAVE_EFER_LMA;  in nested_vmx_setup_ctls_msrs()
2523 	vmx->nested.nested_vmx_misc_low |=  in nested_vmx_setup_ctls_msrs()
2526 	vmx->nested.nested_vmx_misc_high = 0;  in nested_vmx_setup_ctls_msrs()
2562 			vmx->nested.nested_vmx_pinbased_ctls_low,  in vmx_get_vmx_msr()
2563 			vmx->nested.nested_vmx_pinbased_ctls_high);  in vmx_get_vmx_msr()
2567 			vmx->nested.nested_vmx_true_procbased_ctls_low,  in vmx_get_vmx_msr()
2568 			vmx->nested.nested_vmx_procbased_ctls_high);  in vmx_get_vmx_msr()
2572 			vmx->nested.nested_vmx_procbased_ctls_low,  in vmx_get_vmx_msr()
2573 			vmx->nested.nested_vmx_procbased_ctls_high);  in vmx_get_vmx_msr()
2577 			vmx->nested.nested_vmx_true_exit_ctls_low,  in vmx_get_vmx_msr()
2578 			vmx->nested.nested_vmx_exit_ctls_high);  in vmx_get_vmx_msr()
2582 			vmx->nested.nested_vmx_exit_ctls_low,  in vmx_get_vmx_msr()
2583 			vmx->nested.nested_vmx_exit_ctls_high);  in vmx_get_vmx_msr()
2587 			vmx->nested.nested_vmx_true_entry_ctls_low,  in vmx_get_vmx_msr()
2588 			vmx->nested.nested_vmx_entry_ctls_high);  in vmx_get_vmx_msr()
2592 			vmx->nested.nested_vmx_entry_ctls_low,  in vmx_get_vmx_msr()
2593 			vmx->nested.nested_vmx_entry_ctls_high);  in vmx_get_vmx_msr()
2597 			vmx->nested.nested_vmx_misc_low,  in vmx_get_vmx_msr()
2598 			vmx->nested.nested_vmx_misc_high);  in vmx_get_vmx_msr()
2624 			vmx->nested.nested_vmx_secondary_ctls_low,  in vmx_get_vmx_msr()
2625 			vmx->nested.nested_vmx_secondary_ctls_high);  in vmx_get_vmx_msr()
2629 		*pdata = vmx->nested.nested_vmx_ept_caps;  in vmx_get_vmx_msr()
2688 		data = to_vmx(vcpu)->nested.msr_ia32_feature_control;  in vmx_get_msr()
2781 		    (to_vmx(vcpu)->nested.msr_ia32_feature_control &  in vmx_set_msr()
2784 		vmx->nested.msr_ia32_feature_control = data;  in vmx_set_msr()
3663 	if (to_vmx(vcpu)->nested.vmxon &&  in vmx_set_cr4()
4373 	if (vmx->nested.pi_desc &&  in vmx_complete_nested_posted_interrupt()
4374 	    vmx->nested.pi_pending) {  in vmx_complete_nested_posted_interrupt()
4375 		vmx->nested.pi_pending = false;  in vmx_complete_nested_posted_interrupt()
4376 		if (!pi_test_and_clear_on(vmx->nested.pi_desc))  in vmx_complete_nested_posted_interrupt()
4380 			(unsigned long *)vmx->nested.pi_desc->pir, 256);  in vmx_complete_nested_posted_interrupt()
4385 		vapic_page = kmap(vmx->nested.virtual_apic_page);  in vmx_complete_nested_posted_interrupt()
4390 		__kvm_apic_update_irr(vmx->nested.pi_desc->pir, vapic_page);  in vmx_complete_nested_posted_interrupt()
4391 		kunmap(vmx->nested.virtual_apic_page);  in vmx_complete_nested_posted_interrupt()
4421 	    vector == vmx->nested.posted_intr_nv) {  in vmx_deliver_nested_posted_interrupt()
4428 		vmx->nested.pi_pending = true;  in vmx_deliver_nested_posted_interrupt()
4957 	if (to_vmx(vcpu)->nested.nested_run_pending)  in vmx_nmi_allowed()
4970 	return (!to_vmx(vcpu)->nested.nested_run_pending &&  in vmx_interrupt_allowed()
5238 	if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &  in nested_cr0_valid()
5271 		if (to_vmx(vcpu)->nested.vmxon &&  in handle_set_cr0()
6063 	if (nested) {  in hardware_setup()
6092 	if (nested)  in hardware_setup()
6221 	if (nested)  in hardware_setup()
6249 	if (nested)  in hardware_unsetup()
6305 	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)  in nested_get_current_vmcs02()
6306 		if (item->vmptr == vmx->nested.current_vmptr) {  in nested_get_current_vmcs02()
6307 			list_move(&item->list, &vmx->nested.vmcs02_pool);  in nested_get_current_vmcs02()
6311 	if (vmx->nested.vmcs02_num >= max(VMCS02_POOL_SIZE, 1)) {  in nested_get_current_vmcs02()
6313 		item = list_entry(vmx->nested.vmcs02_pool.prev,  in nested_get_current_vmcs02()
6315 		item->vmptr = vmx->nested.current_vmptr;  in nested_get_current_vmcs02()
6316 		list_move(&item->list, &vmx->nested.vmcs02_pool);  in nested_get_current_vmcs02()
6330 	item->vmptr = vmx->nested.current_vmptr;  in nested_get_current_vmcs02()
6331 	list_add(&(item->list), &(vmx->nested.vmcs02_pool));  in nested_get_current_vmcs02()
6332 	vmx->nested.vmcs02_num++;  in nested_get_current_vmcs02()
6340 	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)  in nested_free_vmcs02()
6345 			vmx->nested.vmcs02_num--;  in nested_free_vmcs02()
6360 	list_for_each_entry_safe(item, n, &vmx->nested.vmcs02_pool, list) {  in nested_free_all_saved_vmcss()
6371 		vmx->nested.vmcs02_num--;  in nested_free_all_saved_vmcss()
6398 	if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {  in nested_vmx_failValid()
6427 		container_of(timer, struct vcpu_vmx, nested.preemption_timer);  in vmx_preemption_timer_fn()
6429 	vmx->nested.preemption_timer_expired = true;  in vmx_preemption_timer_fn()
6543 		vmx->nested.vmxon_ptr = vmptr;  in nested_vmx_check_vmptr()
6553 		if (vmptr == vmx->nested.vmxon_ptr) {  in nested_vmx_check_vmptr()
6568 		if (vmptr == vmx->nested.vmxon_ptr) {  in nested_vmx_check_vmptr()
6626 	if (vmx->nested.vmxon) {  in handle_vmon()
6632 	if ((vmx->nested.msr_ia32_feature_control & VMXON_NEEDED_FEATURES)  in handle_vmon()
6646 		vmx->nested.current_shadow_vmcs = shadow_vmcs;  in handle_vmon()
6649 	INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool));  in handle_vmon()
6650 	vmx->nested.vmcs02_num = 0;  in handle_vmon()
6652 	hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,  in handle_vmon()
6654 	vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;  in handle_vmon()
6656 	vmx->nested.vmxon = true;  in handle_vmon()
6673 	if (!vmx->nested.vmxon) {  in nested_vmx_check_permission()
6696 	if (vmx->nested.current_vmptr == -1ull)  in nested_release_vmcs12()
6700 	if (WARN_ON(vmx->nested.current_vmcs12 == NULL))  in nested_release_vmcs12()
6707 		vmx->nested.sync_shadow_vmcs = false;  in nested_release_vmcs12()
6713 	vmx->nested.posted_intr_nv = -1;  in nested_release_vmcs12()
6714 	kunmap(vmx->nested.current_vmcs12_page);  in nested_release_vmcs12()
6715 	nested_release_page(vmx->nested.current_vmcs12_page);  in nested_release_vmcs12()
6716 	vmx->nested.current_vmptr = -1ull;  in nested_release_vmcs12()
6717 	vmx->nested.current_vmcs12 = NULL;  in nested_release_vmcs12()
6726 	if (!vmx->nested.vmxon)  in free_nested()
6729 	vmx->nested.vmxon = false;  in free_nested()
6732 		free_vmcs(vmx->nested.current_shadow_vmcs);  in free_nested()
6734 	if (vmx->nested.apic_access_page) {  in free_nested()
6735 		nested_release_page(vmx->nested.apic_access_page);  in free_nested()
6736 		vmx->nested.apic_access_page = NULL;  in free_nested()
6738 	if (vmx->nested.virtual_apic_page) {  in free_nested()
6739 		nested_release_page(vmx->nested.virtual_apic_page);  in free_nested()
6740 		vmx->nested.virtual_apic_page = NULL;  in free_nested()
6742 	if (vmx->nested.pi_desc_page) {  in free_nested()
6743 		kunmap(vmx->nested.pi_desc_page);  in free_nested()
6744 		nested_release_page(vmx->nested.pi_desc_page);  in free_nested()
6745 		vmx->nested.pi_desc_page = NULL;  in free_nested()
6746 		vmx->nested.pi_desc = NULL;  in free_nested()
6777 	if (vmptr == vmx->nested.current_vmptr)  in handle_vmclear()
6908 	struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs;  in copy_shadow_to_vmcs12()
6957 	struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs;  in copy_vmcs12_to_shadow()
6997 	if (vmx->nested.current_vmptr == -1ull) {  in nested_vmx_check_vmcs12()
7114 	if (vmx->nested.current_vmptr != vmptr) {  in handle_vmptrld()
7134 		vmx->nested.current_vmptr = vmptr;  in handle_vmptrld()
7135 		vmx->nested.current_vmcs12 = new_vmcs12;  in handle_vmptrld()
7136 		vmx->nested.current_vmcs12_page = page;  in handle_vmptrld()
7142 				     __pa(vmx->nested.current_shadow_vmcs));  in handle_vmptrld()
7143 			vmx->nested.sync_shadow_vmcs = true;  in handle_vmptrld()
7168 				 (void *)&to_vmx(vcpu)->nested.current_vmptr,  in handle_vmptrst()
7190 	if (!(vmx->nested.nested_vmx_secondary_ctls_high &  in handle_invept()
7192 	    !(vmx->nested.nested_vmx_ept_caps & VMX_EPT_INVEPT_BIT)) {  in handle_invept()
7208 	types = (vmx->nested.nested_vmx_ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;  in handle_invept()
7511 	if (vmx->nested.nested_run_pending)  in nested_vmx_exit_handled()
7875 	    !nested_cpu_has2(vmx->nested.current_vmcs12,  in vmx_set_apic_access_page_addr()
8185 	if (vmx->nested.sync_shadow_vmcs) {  in vmx_vcpu_run()
8187 		vmx->nested.sync_shadow_vmcs = false;  in vmx_vcpu_run()
8354 	if (vmx->nested.nested_run_pending)  in vmx_vcpu_run()
8357 	vmx->nested.nested_run_pending = 0;  in vmx_vcpu_run()
8453 	if (nested)  in vmx_create_vcpu()
8456 	vmx->nested.posted_intr_nv = -1;  in vmx_create_vcpu()
8457 	vmx->nested.current_vmptr = -1ull;  in vmx_create_vcpu()
8458 	vmx->nested.current_vmcs12 = NULL;  in vmx_create_vcpu()
8560 		if (nested && !vmx->rdtscp_enabled)  in vmx_cpuid_update()
8561 			vmx->nested.nested_vmx_secondary_ctls_high &=  in vmx_cpuid_update()
8588 	if (func == 1 && nested)  in vmx_set_supported_cpuid()
8618 			to_vmx(vcpu)->nested.nested_vmx_ept_caps &  in nested_ept_init_mmu_context()
8676 		if (vmx->nested.apic_access_page) /* shouldn't happen */  in nested_get_vmcs12_pages()
8677 			nested_release_page(vmx->nested.apic_access_page);  in nested_get_vmcs12_pages()
8678 		vmx->nested.apic_access_page =  in nested_get_vmcs12_pages()
8687 		if (vmx->nested.virtual_apic_page) /* shouldn't happen */  in nested_get_vmcs12_pages()
8688 			nested_release_page(vmx->nested.virtual_apic_page);  in nested_get_vmcs12_pages()
8689 		vmx->nested.virtual_apic_page =  in nested_get_vmcs12_pages()
8702 		if (!vmx->nested.virtual_apic_page)  in nested_get_vmcs12_pages()
8711 		if (vmx->nested.pi_desc_page) { /* shouldn't happen */  in nested_get_vmcs12_pages()
8712 			kunmap(vmx->nested.pi_desc_page);  in nested_get_vmcs12_pages()
8713 			nested_release_page(vmx->nested.pi_desc_page);  in nested_get_vmcs12_pages()
8715 		vmx->nested.pi_desc_page =  in nested_get_vmcs12_pages()
8717 		if (!vmx->nested.pi_desc_page)  in nested_get_vmcs12_pages()
8720 		vmx->nested.pi_desc =  in nested_get_vmcs12_pages()
8721 			(struct pi_desc *)kmap(vmx->nested.pi_desc_page);  in nested_get_vmcs12_pages()
8722 		if (!vmx->nested.pi_desc) {  in nested_get_vmcs12_pages()
8723 			nested_release_page_clean(vmx->nested.pi_desc_page);  in nested_get_vmcs12_pages()
8726 		vmx->nested.pi_desc =  in nested_get_vmcs12_pages()
8727 			(struct pi_desc *)((void *)vmx->nested.pi_desc +  in nested_get_vmcs12_pages()
8746 		vmx_preemption_timer_fn(&vmx->nested.preemption_timer);  in vmx_start_preemption_timer()
8753 	hrtimer_start(&vmx->nested.preemption_timer,  in vmx_start_preemption_timer()
9115 		vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);  in prepare_vmcs02()
9145 		vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;  in prepare_vmcs02()
9146 		vmx->nested.pi_pending = false;  in prepare_vmcs02()
9149 			page_to_phys(vmx->nested.pi_desc_page) +  in prepare_vmcs02()
9157 	vmx->nested.preemption_timer_expired = false;  in prepare_vmcs02()
9206 			if (!vmx->nested.apic_access_page)  in prepare_vmcs02()
9211 				  page_to_phys(vmx->nested.apic_access_page));  in prepare_vmcs02()
9261 				page_to_phys(vmx->nested.virtual_apic_page));  in prepare_vmcs02()
9317 			vmx->nested.vmcs01_tsc_offset + vmcs12->tsc_offset);  in prepare_vmcs02()
9319 		vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);  in prepare_vmcs02()
9447 				vmx->nested.nested_vmx_true_procbased_ctls_low,  in nested_vmx_run()
9448 				vmx->nested.nested_vmx_procbased_ctls_high) ||  in nested_vmx_run()
9450 				vmx->nested.nested_vmx_secondary_ctls_low,  in nested_vmx_run()
9451 				vmx->nested.nested_vmx_secondary_ctls_high) ||  in nested_vmx_run()
9453 				vmx->nested.nested_vmx_pinbased_ctls_low,  in nested_vmx_run()
9454 				vmx->nested.nested_vmx_pinbased_ctls_high) ||  in nested_vmx_run()
9456 				vmx->nested.nested_vmx_true_exit_ctls_low,  in nested_vmx_run()
9457 				vmx->nested.nested_vmx_exit_ctls_high) ||  in nested_vmx_run()
9459 				vmx->nested.nested_vmx_true_entry_ctls_low,  in nested_vmx_run()
9460 				vmx->nested.nested_vmx_entry_ctls_high))  in nested_vmx_run()
9535 	vmx->nested.vmcs01_tsc_offset = vmcs_read64(TSC_OFFSET);  in nested_vmx_run()
9538 		vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);  in nested_vmx_run()
9567 	vmx->nested.nested_run_pending = 1;  in nested_vmx_run()
9662 	    vmx->nested.preemption_timer_expired) {  in vmx_check_nested_events()
9663 		if (vmx->nested.nested_run_pending)  in vmx_check_nested_events()
9670 		if (vmx->nested.nested_run_pending ||  in vmx_check_nested_events()
9687 		if (vmx->nested.nested_run_pending)  in vmx_check_nested_events()
9699 		hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);  in vmx_get_preemption_timer_value()
9784 		hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);  in prepare_vmcs12()
10019 	WARN_ON_ONCE(vmx->nested.nested_run_pending);  in nested_vmx_vmexit()
10052 		nested_free_vmcs02(vmx, vmx->nested.current_vmptr);  in nested_vmx_vmexit()
10057 	vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);  in nested_vmx_vmexit()
10063 	if (vmx->nested.apic_access_page) {  in nested_vmx_vmexit()
10064 		nested_release_page(vmx->nested.apic_access_page);  in nested_vmx_vmexit()
10065 		vmx->nested.apic_access_page = NULL;  in nested_vmx_vmexit()
10067 	if (vmx->nested.virtual_apic_page) {  in nested_vmx_vmexit()
10068 		nested_release_page(vmx->nested.virtual_apic_page);  in nested_vmx_vmexit()
10069 		vmx->nested.virtual_apic_page = NULL;  in nested_vmx_vmexit()
10071 	if (vmx->nested.pi_desc_page) {  in nested_vmx_vmexit()
10072 		kunmap(vmx->nested.pi_desc_page);  in nested_vmx_vmexit()
10073 		nested_release_page(vmx->nested.pi_desc_page);  in nested_vmx_vmexit()
10074 		vmx->nested.pi_desc_page = NULL;  in nested_vmx_vmexit()
10075 		vmx->nested.pi_desc = NULL;  in nested_vmx_vmexit()
10095 		vmx->nested.sync_shadow_vmcs = true;  in nested_vmx_vmexit()
10127 		to_vmx(vcpu)->nested.sync_shadow_vmcs = true;  in nested_vmx_entry_failure()