Lines Matching refs:vmcs12
203 struct __packed vmcs12 { struct
395 struct vmcs12 *current_vmcs12;
566 #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
782 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu) in get_vmcs12()
1121 static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit) in nested_cpu_has() argument
1123 return vmcs12->cpu_based_vm_exec_control & bit; in nested_cpu_has()
1126 static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit) in nested_cpu_has2() argument
1128 return (vmcs12->cpu_based_vm_exec_control & in nested_cpu_has2()
1130 (vmcs12->secondary_vm_exec_control & bit); in nested_cpu_has2()
1133 static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12) in nested_cpu_has_virtual_nmis() argument
1135 return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS; in nested_cpu_has_virtual_nmis()
1138 static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12) in nested_cpu_has_preemption_timer() argument
1140 return vmcs12->pin_based_vm_exec_control & in nested_cpu_has_preemption_timer()
1144 static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12) in nested_cpu_has_ept() argument
1146 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT); in nested_cpu_has_ept()
1149 static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12) in nested_cpu_has_xsaves() argument
1151 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES) && in nested_cpu_has_xsaves()
1155 static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12) in nested_cpu_has_virt_x2apic_mode() argument
1157 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); in nested_cpu_has_virt_x2apic_mode()
1160 static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12) in nested_cpu_has_apic_reg_virt() argument
1162 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT); in nested_cpu_has_apic_reg_virt()
1165 static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12) in nested_cpu_has_vid() argument
1167 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); in nested_cpu_has_vid()
1170 static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12) in nested_cpu_has_posted_intr() argument
1172 return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR; in nested_cpu_has_posted_intr()
1185 struct vmcs12 *vmcs12,
2003 static inline unsigned long nested_read_cr0(struct vmcs12 *fields) in nested_read_cr0()
2008 static inline unsigned long nested_read_cr4(struct vmcs12 *fields) in nested_read_cr4()
2033 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in vmx_fpu_deactivate() local
2034 vmcs12->guest_cr0 = (vmcs12->guest_cr0 & ~X86_CR0_TS) | in vmx_fpu_deactivate()
2036 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); in vmx_fpu_deactivate()
2116 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in nested_vmx_check_exception() local
2118 if (!(vmcs12->exception_bitmap & (1u << nr))) in nested_vmx_check_exception()
2309 struct vmcs12 *vmcs12; in vmx_write_tsc_offset() local
2312 vmcs12 = get_vmcs12(vcpu); in vmx_write_tsc_offset()
2314 (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ? in vmx_write_tsc_offset()
2315 vmcs12->tsc_offset : 0)); in vmx_write_tsc_offset()
5236 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in nested_cr0_valid() local
5240 nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) in nested_cr0_valid()
5249 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in handle_set_cr0() local
5260 val = (val & ~vmcs12->cr0_guest_host_mask) | in handle_set_cr0()
5261 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); in handle_set_cr0()
5281 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in handle_set_cr4() local
5285 val = (val & ~vmcs12->cr4_guest_host_mask) | in handle_set_cr4()
5286 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); in handle_set_cr4()
6768 struct vmcs12 *vmcs12; in handle_vmclear() local
6792 vmcs12 = kmap(page); in handle_vmclear()
6793 vmcs12->launch_state = 0; in handle_vmclear()
7115 struct vmcs12 *new_vmcs12; in handle_vmptrld()
7331 struct vmcs12 *vmcs12) in nested_vmx_exit_handled_io() argument
7339 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) in nested_vmx_exit_handled_io()
7340 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); in nested_vmx_exit_handled_io()
7352 bitmap = vmcs12->io_bitmap_a; in nested_vmx_exit_handled_io()
7354 bitmap = vmcs12->io_bitmap_b; in nested_vmx_exit_handled_io()
7380 struct vmcs12 *vmcs12, u32 exit_reason) in nested_vmx_exit_handled_msr() argument
7385 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) in nested_vmx_exit_handled_msr()
7393 bitmap = vmcs12->msr_bitmap; in nested_vmx_exit_handled_msr()
7417 struct vmcs12 *vmcs12) in nested_vmx_exit_handled_cr() argument
7428 if (vmcs12->cr0_guest_host_mask & in nested_vmx_exit_handled_cr()
7429 (val ^ vmcs12->cr0_read_shadow)) in nested_vmx_exit_handled_cr()
7433 if ((vmcs12->cr3_target_count >= 1 && in nested_vmx_exit_handled_cr()
7434 vmcs12->cr3_target_value0 == val) || in nested_vmx_exit_handled_cr()
7435 (vmcs12->cr3_target_count >= 2 && in nested_vmx_exit_handled_cr()
7436 vmcs12->cr3_target_value1 == val) || in nested_vmx_exit_handled_cr()
7437 (vmcs12->cr3_target_count >= 3 && in nested_vmx_exit_handled_cr()
7438 vmcs12->cr3_target_value2 == val) || in nested_vmx_exit_handled_cr()
7439 (vmcs12->cr3_target_count >= 4 && in nested_vmx_exit_handled_cr()
7440 vmcs12->cr3_target_value3 == val)) in nested_vmx_exit_handled_cr()
7442 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) in nested_vmx_exit_handled_cr()
7446 if (vmcs12->cr4_guest_host_mask & in nested_vmx_exit_handled_cr()
7447 (vmcs12->cr4_read_shadow ^ val)) in nested_vmx_exit_handled_cr()
7451 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) in nested_vmx_exit_handled_cr()
7457 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && in nested_vmx_exit_handled_cr()
7458 (vmcs12->cr0_read_shadow & X86_CR0_TS)) in nested_vmx_exit_handled_cr()
7464 if (vmcs12->cpu_based_vm_exec_control & in nested_vmx_exit_handled_cr()
7469 if (vmcs12->cpu_based_vm_exec_control & in nested_vmx_exit_handled_cr()
7480 if (vmcs12->cr0_guest_host_mask & 0xe & in nested_vmx_exit_handled_cr()
7481 (val ^ vmcs12->cr0_read_shadow)) in nested_vmx_exit_handled_cr()
7483 if ((vmcs12->cr0_guest_host_mask & 0x1) && in nested_vmx_exit_handled_cr()
7484 !(vmcs12->cr0_read_shadow & 0x1) && in nested_vmx_exit_handled_cr()
7501 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in nested_vmx_exit_handled() local
7527 !(vmcs12->guest_cr0 & X86_CR0_TS)) in nested_vmx_exit_handled()
7529 return vmcs12->exception_bitmap & in nested_vmx_exit_handled()
7536 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING); in nested_vmx_exit_handled()
7538 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING); in nested_vmx_exit_handled()
7546 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); in nested_vmx_exit_handled()
7550 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); in nested_vmx_exit_handled()
7552 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); in nested_vmx_exit_handled()
7554 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); in nested_vmx_exit_handled()
7567 return nested_vmx_exit_handled_cr(vcpu, vmcs12); in nested_vmx_exit_handled()
7569 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); in nested_vmx_exit_handled()
7571 return nested_vmx_exit_handled_io(vcpu, vmcs12); in nested_vmx_exit_handled()
7574 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); in nested_vmx_exit_handled()
7578 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); in nested_vmx_exit_handled()
7580 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); in nested_vmx_exit_handled()
7582 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || in nested_vmx_exit_handled()
7583 nested_cpu_has2(vmcs12, in nested_vmx_exit_handled()
7588 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); in nested_vmx_exit_handled()
7590 return nested_cpu_has2(vmcs12, in nested_vmx_exit_handled()
7613 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); in nested_vmx_exit_handled()
7623 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); in nested_vmx_exit_handled()
7814 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in update_cr8_intercept() local
7817 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) in update_cr8_intercept()
8595 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in nested_ept_inject_page_fault() local
8603 vmcs12->guest_physical_address = fault->address; in nested_ept_inject_page_fault()
8632 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, in nested_vmx_is_page_fault_vmexit() argument
8637 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; in nested_vmx_is_page_fault_vmexit()
8639 (error_code & vmcs12->page_fault_error_code_mask) != in nested_vmx_is_page_fault_vmexit()
8640 vmcs12->page_fault_error_code_match; in nested_vmx_is_page_fault_vmexit()
8647 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in vmx_inject_page_fault_nested() local
8651 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code)) in vmx_inject_page_fault_nested()
8660 struct vmcs12 *vmcs12) in nested_get_vmcs12_pages() argument
8665 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { in nested_get_vmcs12_pages()
8666 if (!PAGE_ALIGNED(vmcs12->apic_access_addr) || in nested_get_vmcs12_pages()
8667 vmcs12->apic_access_addr >> maxphyaddr) in nested_get_vmcs12_pages()
8679 nested_get_page(vcpu, vmcs12->apic_access_addr); in nested_get_vmcs12_pages()
8682 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { in nested_get_vmcs12_pages()
8683 if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr) || in nested_get_vmcs12_pages()
8684 vmcs12->virtual_apic_page_addr >> maxphyaddr) in nested_get_vmcs12_pages()
8690 nested_get_page(vcpu, vmcs12->virtual_apic_page_addr); in nested_get_vmcs12_pages()
8706 if (nested_cpu_has_posted_intr(vmcs12)) { in nested_get_vmcs12_pages()
8707 if (!IS_ALIGNED(vmcs12->posted_intr_desc_addr, 64) || in nested_get_vmcs12_pages()
8708 vmcs12->posted_intr_desc_addr >> maxphyaddr) in nested_get_vmcs12_pages()
8716 nested_get_page(vcpu, vmcs12->posted_intr_desc_addr); in nested_get_vmcs12_pages()
8728 (unsigned long)(vmcs12->posted_intr_desc_addr & in nested_get_vmcs12_pages()
8758 struct vmcs12 *vmcs12) in nested_vmx_check_msr_bitmap_controls() argument
8763 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) in nested_vmx_check_msr_bitmap_controls()
8772 if (!PAGE_ALIGNED(vmcs12->msr_bitmap) || in nested_vmx_check_msr_bitmap_controls()
8784 struct vmcs12 *vmcs12) in nested_vmx_merge_msr_bitmap() argument
8790 if (!nested_cpu_has_virt_x2apic_mode(vmcs12)) in nested_vmx_merge_msr_bitmap()
8793 page = nested_get_page(vcpu, vmcs12->msr_bitmap); in nested_vmx_merge_msr_bitmap()
8805 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { in nested_vmx_merge_msr_bitmap()
8806 if (nested_cpu_has_apic_reg_virt(vmcs12)) in nested_vmx_merge_msr_bitmap()
8817 if (nested_cpu_has_vid(vmcs12)) { in nested_vmx_merge_msr_bitmap()
8863 struct vmcs12 *vmcs12) in nested_vmx_check_apicv_controls() argument
8865 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && in nested_vmx_check_apicv_controls()
8866 !nested_cpu_has_apic_reg_virt(vmcs12) && in nested_vmx_check_apicv_controls()
8867 !nested_cpu_has_vid(vmcs12) && in nested_vmx_check_apicv_controls()
8868 !nested_cpu_has_posted_intr(vmcs12)) in nested_vmx_check_apicv_controls()
8875 if (nested_cpu_has_virt_x2apic_mode(vmcs12) && in nested_vmx_check_apicv_controls()
8876 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) in nested_vmx_check_apicv_controls()
8883 if (nested_cpu_has_vid(vmcs12) && in nested_vmx_check_apicv_controls()
8892 if (nested_cpu_has_posted_intr(vmcs12) && in nested_vmx_check_apicv_controls()
8893 (!nested_cpu_has_vid(vmcs12) || in nested_vmx_check_apicv_controls()
8895 vmcs12->posted_intr_nv & 0xff00)) in nested_vmx_check_apicv_controls()
8899 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) in nested_vmx_check_apicv_controls()
8931 struct vmcs12 *vmcs12) in nested_vmx_check_msr_switch_controls() argument
8933 if (vmcs12->vm_exit_msr_load_count == 0 && in nested_vmx_check_msr_switch_controls()
8934 vmcs12->vm_exit_msr_store_count == 0 && in nested_vmx_check_msr_switch_controls()
8935 vmcs12->vm_entry_msr_load_count == 0) in nested_vmx_check_msr_switch_controls()
9068 static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) in prepare_vmcs02() argument
9073 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); in prepare_vmcs02()
9074 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); in prepare_vmcs02()
9075 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); in prepare_vmcs02()
9076 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); in prepare_vmcs02()
9077 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); in prepare_vmcs02()
9078 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); in prepare_vmcs02()
9079 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); in prepare_vmcs02()
9080 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); in prepare_vmcs02()
9081 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); in prepare_vmcs02()
9082 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); in prepare_vmcs02()
9083 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); in prepare_vmcs02()
9084 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); in prepare_vmcs02()
9085 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); in prepare_vmcs02()
9086 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); in prepare_vmcs02()
9087 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); in prepare_vmcs02()
9088 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); in prepare_vmcs02()
9089 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); in prepare_vmcs02()
9090 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); in prepare_vmcs02()
9091 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); in prepare_vmcs02()
9092 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); in prepare_vmcs02()
9093 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); in prepare_vmcs02()
9094 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); in prepare_vmcs02()
9095 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); in prepare_vmcs02()
9096 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); in prepare_vmcs02()
9097 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); in prepare_vmcs02()
9098 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); in prepare_vmcs02()
9099 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); in prepare_vmcs02()
9100 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); in prepare_vmcs02()
9101 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); in prepare_vmcs02()
9102 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); in prepare_vmcs02()
9103 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); in prepare_vmcs02()
9104 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); in prepare_vmcs02()
9105 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); in prepare_vmcs02()
9106 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); in prepare_vmcs02()
9107 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); in prepare_vmcs02()
9108 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); in prepare_vmcs02()
9110 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { in prepare_vmcs02()
9111 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); in prepare_vmcs02()
9112 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); in prepare_vmcs02()
9118 vmcs12->vm_entry_intr_info_field); in prepare_vmcs02()
9120 vmcs12->vm_entry_exception_error_code); in prepare_vmcs02()
9122 vmcs12->vm_entry_instruction_len); in prepare_vmcs02()
9124 vmcs12->guest_interruptibility_info); in prepare_vmcs02()
9125 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); in prepare_vmcs02()
9126 vmx_set_rflags(vcpu, vmcs12->guest_rflags); in prepare_vmcs02()
9128 vmcs12->guest_pending_dbg_exceptions); in prepare_vmcs02()
9129 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); in prepare_vmcs02()
9130 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); in prepare_vmcs02()
9132 if (nested_cpu_has_xsaves(vmcs12)) in prepare_vmcs02()
9133 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); in prepare_vmcs02()
9136 exec_control = vmcs12->pin_based_vm_exec_control; in prepare_vmcs02()
9140 if (nested_cpu_has_posted_intr(vmcs12)) { in prepare_vmcs02()
9145 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; in prepare_vmcs02()
9150 (unsigned long)(vmcs12->posted_intr_desc_addr & in prepare_vmcs02()
9158 if (nested_cpu_has_preemption_timer(vmcs12)) in prepare_vmcs02()
9182 enable_ept ? vmcs12->page_fault_error_code_mask : 0); in prepare_vmcs02()
9184 enable_ept ? vmcs12->page_fault_error_code_match : 0); in prepare_vmcs02()
9195 if (nested_cpu_has(vmcs12, in prepare_vmcs02()
9197 exec_control |= vmcs12->secondary_vm_exec_control; in prepare_vmcs02()
9212 } else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) && in prepare_vmcs02()
9221 vmcs12->eoi_exit_bitmap0); in prepare_vmcs02()
9223 vmcs12->eoi_exit_bitmap1); in prepare_vmcs02()
9225 vmcs12->eoi_exit_bitmap2); in prepare_vmcs02()
9227 vmcs12->eoi_exit_bitmap3); in prepare_vmcs02()
9229 vmcs12->guest_intr_status); in prepare_vmcs02()
9257 exec_control |= vmcs12->cpu_based_vm_exec_control; in prepare_vmcs02()
9262 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); in prepare_vmcs02()
9267 nested_vmx_merge_msr_bitmap(vcpu, vmcs12); in prepare_vmcs02()
9286 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; in prepare_vmcs02()
9299 (vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER & in prepare_vmcs02()
9303 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) { in prepare_vmcs02()
9304 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); in prepare_vmcs02()
9305 vcpu->arch.pat = vmcs12->guest_ia32_pat; in prepare_vmcs02()
9312 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) in prepare_vmcs02()
9313 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); in prepare_vmcs02()
9315 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) in prepare_vmcs02()
9317 vmx->nested.vmcs01_tsc_offset + vmcs12->tsc_offset); in prepare_vmcs02()
9331 if (nested_cpu_has_ept(vmcs12)) { in prepare_vmcs02()
9336 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER) in prepare_vmcs02()
9337 vcpu->arch.efer = vmcs12->guest_ia32_efer; in prepare_vmcs02()
9338 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) in prepare_vmcs02()
9353 vmx_set_cr0(vcpu, vmcs12->guest_cr0); in prepare_vmcs02()
9354 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); in prepare_vmcs02()
9356 vmx_set_cr4(vcpu, vmcs12->guest_cr4); in prepare_vmcs02()
9357 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); in prepare_vmcs02()
9360 kvm_set_cr3(vcpu, vmcs12->guest_cr3); in prepare_vmcs02()
9370 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); in prepare_vmcs02()
9371 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); in prepare_vmcs02()
9372 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); in prepare_vmcs02()
9373 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); in prepare_vmcs02()
9376 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp); in prepare_vmcs02()
9377 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip); in prepare_vmcs02()
9386 struct vmcs12 *vmcs12; in nested_vmx_run() local
9398 vmcs12 = get_vmcs12(vcpu); in nested_vmx_run()
9413 if (vmcs12->launch_state == launch) { in nested_vmx_run()
9420 if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && in nested_vmx_run()
9421 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) { in nested_vmx_run()
9426 if (!nested_get_vmcs12_pages(vcpu, vmcs12)) { in nested_vmx_run()
9431 if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12)) { in nested_vmx_run()
9436 if (nested_vmx_check_apicv_controls(vcpu, vmcs12)) { in nested_vmx_run()
9441 if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12)) { in nested_vmx_run()
9446 if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, in nested_vmx_run()
9449 !vmx_control_verify(vmcs12->secondary_vm_exec_control, in nested_vmx_run()
9452 !vmx_control_verify(vmcs12->pin_based_vm_exec_control, in nested_vmx_run()
9455 !vmx_control_verify(vmcs12->vm_exit_controls, in nested_vmx_run()
9458 !vmx_control_verify(vmcs12->vm_entry_controls, in nested_vmx_run()
9466 if (((vmcs12->host_cr0 & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON) || in nested_vmx_run()
9467 ((vmcs12->host_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) { in nested_vmx_run()
9473 if (!nested_cr0_valid(vcpu, vmcs12->guest_cr0) || in nested_vmx_run()
9474 ((vmcs12->guest_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) { in nested_vmx_run()
9475 nested_vmx_entry_failure(vcpu, vmcs12, in nested_vmx_run()
9479 if (vmcs12->vmcs_link_pointer != -1ull) { in nested_vmx_run()
9480 nested_vmx_entry_failure(vcpu, vmcs12, in nested_vmx_run()
9494 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER) { in nested_vmx_run()
9495 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; in nested_vmx_run()
9496 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) || in nested_vmx_run()
9497 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) || in nested_vmx_run()
9498 ((vmcs12->guest_cr0 & X86_CR0_PG) && in nested_vmx_run()
9499 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) { in nested_vmx_run()
9500 nested_vmx_entry_failure(vcpu, vmcs12, in nested_vmx_run()
9512 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { in nested_vmx_run()
9513 ia32e = (vmcs12->vm_exit_controls & in nested_vmx_run()
9515 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) || in nested_vmx_run()
9516 ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) || in nested_vmx_run()
9517 ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) { in nested_vmx_run()
9518 nested_vmx_entry_failure(vcpu, vmcs12, in nested_vmx_run()
9537 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) in nested_vmx_run()
9549 prepare_vmcs02(vcpu, vmcs12); in nested_vmx_run()
9552 vmcs12->vm_entry_msr_load_addr, in nested_vmx_run()
9553 vmcs12->vm_entry_msr_load_count); in nested_vmx_run()
9557 nested_vmx_entry_failure(vcpu, vmcs12, in nested_vmx_run()
9562 vmcs12->launch_state = 1; in nested_vmx_run()
9564 if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) in nested_vmx_run()
9596 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) in vmcs12_guest_cr0() argument
9600 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | in vmcs12_guest_cr0()
9601 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | in vmcs12_guest_cr0()
9606 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) in vmcs12_guest_cr4() argument
9610 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | in vmcs12_guest_cr4()
9611 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | in vmcs12_guest_cr4()
9616 struct vmcs12 *vmcs12) in vmcs12_save_pending_event() argument
9626 vmcs12->vm_exit_instruction_len = in vmcs12_save_pending_event()
9634 vmcs12->idt_vectoring_error_code = in vmcs12_save_pending_event()
9638 vmcs12->idt_vectoring_info_field = idt_vectoring; in vmcs12_save_pending_event()
9640 vmcs12->idt_vectoring_info_field = in vmcs12_save_pending_event()
9648 vmcs12->vm_entry_instruction_len = in vmcs12_save_pending_event()
9653 vmcs12->idt_vectoring_info_field = idt_vectoring; in vmcs12_save_pending_event()
9721 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, in prepare_vmcs12() argument
9726 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); in prepare_vmcs12()
9727 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); in prepare_vmcs12()
9729 vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); in prepare_vmcs12()
9730 vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP); in prepare_vmcs12()
9731 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); in prepare_vmcs12()
9733 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); in prepare_vmcs12()
9734 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); in prepare_vmcs12()
9735 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); in prepare_vmcs12()
9736 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); in prepare_vmcs12()
9737 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); in prepare_vmcs12()
9738 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); in prepare_vmcs12()
9739 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); in prepare_vmcs12()
9740 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); in prepare_vmcs12()
9741 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); in prepare_vmcs12()
9742 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); in prepare_vmcs12()
9743 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); in prepare_vmcs12()
9744 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); in prepare_vmcs12()
9745 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); in prepare_vmcs12()
9746 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); in prepare_vmcs12()
9747 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); in prepare_vmcs12()
9748 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); in prepare_vmcs12()
9749 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); in prepare_vmcs12()
9750 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); in prepare_vmcs12()
9751 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); in prepare_vmcs12()
9752 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); in prepare_vmcs12()
9753 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); in prepare_vmcs12()
9754 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); in prepare_vmcs12()
9755 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); in prepare_vmcs12()
9756 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); in prepare_vmcs12()
9757 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); in prepare_vmcs12()
9758 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); in prepare_vmcs12()
9759 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); in prepare_vmcs12()
9760 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); in prepare_vmcs12()
9761 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); in prepare_vmcs12()
9762 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); in prepare_vmcs12()
9763 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); in prepare_vmcs12()
9764 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); in prepare_vmcs12()
9765 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); in prepare_vmcs12()
9766 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); in prepare_vmcs12()
9767 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); in prepare_vmcs12()
9768 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); in prepare_vmcs12()
9770 vmcs12->guest_interruptibility_info = in prepare_vmcs12()
9772 vmcs12->guest_pending_dbg_exceptions = in prepare_vmcs12()
9775 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; in prepare_vmcs12()
9777 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; in prepare_vmcs12()
9779 if (nested_cpu_has_preemption_timer(vmcs12)) { in prepare_vmcs12()
9780 if (vmcs12->vm_exit_controls & in prepare_vmcs12()
9782 vmcs12->vmx_preemption_timer_value = in prepare_vmcs12()
9796 vmcs12->guest_cr3 = vmcs_read64(GUEST_CR3); in prepare_vmcs12()
9797 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); in prepare_vmcs12()
9798 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); in prepare_vmcs12()
9799 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); in prepare_vmcs12()
9800 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); in prepare_vmcs12()
9803 if (nested_cpu_has_vid(vmcs12)) in prepare_vmcs12()
9804 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); in prepare_vmcs12()
9806 vmcs12->vm_entry_controls = in prepare_vmcs12()
9807 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | in prepare_vmcs12()
9810 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) { in prepare_vmcs12()
9811 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); in prepare_vmcs12()
9812 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); in prepare_vmcs12()
9817 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) in prepare_vmcs12()
9818 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT); in prepare_vmcs12()
9819 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) in prepare_vmcs12()
9820 vmcs12->guest_ia32_efer = vcpu->arch.efer; in prepare_vmcs12()
9821 vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS); in prepare_vmcs12()
9822 vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP); in prepare_vmcs12()
9823 vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP); in prepare_vmcs12()
9825 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); in prepare_vmcs12()
9826 if (nested_cpu_has_xsaves(vmcs12)) in prepare_vmcs12()
9827 vmcs12->xss_exit_bitmap = vmcs_read64(XSS_EXIT_BITMAP); in prepare_vmcs12()
9831 vmcs12->vm_exit_reason = exit_reason; in prepare_vmcs12()
9832 vmcs12->exit_qualification = exit_qualification; in prepare_vmcs12()
9834 vmcs12->vm_exit_intr_info = exit_intr_info; in prepare_vmcs12()
9835 if ((vmcs12->vm_exit_intr_info & in prepare_vmcs12()
9838 vmcs12->vm_exit_intr_error_code = in prepare_vmcs12()
9840 vmcs12->idt_vectoring_info_field = 0; in prepare_vmcs12()
9841 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); in prepare_vmcs12()
9842 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); in prepare_vmcs12()
9844 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { in prepare_vmcs12()
9847 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; in prepare_vmcs12()
9853 vmcs12_save_pending_event(vcpu, vmcs12); in prepare_vmcs12()
9875 struct vmcs12 *vmcs12) in load_vmcs12_host_state() argument
9879 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) in load_vmcs12_host_state()
9880 vcpu->arch.efer = vmcs12->host_ia32_efer; in load_vmcs12_host_state()
9881 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) in load_vmcs12_host_state()
9887 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp); in load_vmcs12_host_state()
9888 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip); in load_vmcs12_host_state()
9896 vmx_set_cr0(vcpu, vmcs12->host_cr0); in load_vmcs12_host_state()
9911 kvm_set_cr4(vcpu, vmcs12->host_cr4); in load_vmcs12_host_state()
9915 kvm_set_cr3(vcpu, vmcs12->host_cr3); in load_vmcs12_host_state()
9931 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); in load_vmcs12_host_state()
9932 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); in load_vmcs12_host_state()
9933 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); in load_vmcs12_host_state()
9934 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); in load_vmcs12_host_state()
9935 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); in load_vmcs12_host_state()
9938 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) in load_vmcs12_host_state()
9941 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { in load_vmcs12_host_state()
9942 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); in load_vmcs12_host_state()
9943 vcpu->arch.pat = vmcs12->host_ia32_pat; in load_vmcs12_host_state()
9945 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) in load_vmcs12_host_state()
9947 vmcs12->host_ia32_perf_global_ctrl); in load_vmcs12_host_state()
9954 .selector = vmcs12->host_cs_selector, in load_vmcs12_host_state()
9960 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) in load_vmcs12_host_state()
9974 seg.selector = vmcs12->host_ds_selector; in load_vmcs12_host_state()
9976 seg.selector = vmcs12->host_es_selector; in load_vmcs12_host_state()
9978 seg.selector = vmcs12->host_ss_selector; in load_vmcs12_host_state()
9980 seg.selector = vmcs12->host_fs_selector; in load_vmcs12_host_state()
9981 seg.base = vmcs12->host_fs_base; in load_vmcs12_host_state()
9983 seg.selector = vmcs12->host_gs_selector; in load_vmcs12_host_state()
9984 seg.base = vmcs12->host_gs_base; in load_vmcs12_host_state()
9987 .base = vmcs12->host_tr_base, in load_vmcs12_host_state()
9989 .selector = vmcs12->host_tr_selector, in load_vmcs12_host_state()
10001 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, in load_vmcs12_host_state()
10002 vmcs12->vm_exit_msr_load_count)) in load_vmcs12_host_state()
10016 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); in nested_vmx_vmexit() local
10022 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info, in nested_vmx_vmexit()
10025 if (nested_vmx_store_msr(vcpu, vmcs12->vm_exit_msr_store_addr, in nested_vmx_vmexit()
10026 vmcs12->vm_exit_msr_store_count)) in nested_vmx_vmexit()
10035 vmcs12->vm_exit_intr_info = irq | in nested_vmx_vmexit()
10039 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, in nested_vmx_vmexit()
10040 vmcs12->exit_qualification, in nested_vmx_vmexit()
10041 vmcs12->idt_vectoring_info_field, in nested_vmx_vmexit()
10042 vmcs12->vm_exit_intr_info, in nested_vmx_vmexit()
10043 vmcs12->vm_exit_intr_error_code, in nested_vmx_vmexit()
10054 load_vmcs12_host_state(vcpu, vmcs12); in nested_vmx_vmexit()
10119 struct vmcs12 *vmcs12, in nested_vmx_entry_failure() argument
10122 load_vmcs12_host_state(vcpu, vmcs12); in nested_vmx_entry_failure()
10123 vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY; in nested_vmx_entry_failure()
10124 vmcs12->exit_qualification = qualification; in nested_vmx_entry_failure()