/* * Copyright (c) 2018-2021 Maxime Villard, m00nbsd.net * All rights reserved. * * This code is part of the NVMM hypervisor. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _NVMM_X86_H_ #define _NVMM_X86_H_ #if defined(__NetBSD__) #include #elif defined(__DragonFly__) #include #endif /* -------------------------------------------------------------------------- */ #ifndef ASM_NVMM struct nvmm_x86_exit_memory { int prot; gpaddr_t gpa; uint8_t inst_len; uint8_t inst_bytes[15]; }; struct nvmm_x86_exit_io { bool in; uint16_t port; int8_t seg; uint8_t address_size; uint8_t operand_size; bool rep; bool str; uint64_t npc; }; struct nvmm_x86_exit_rdmsr { uint32_t msr; uint64_t npc; }; struct nvmm_x86_exit_wrmsr { uint32_t msr; uint64_t val; uint64_t npc; }; struct nvmm_x86_exit_insn { uint64_t npc; }; struct nvmm_x86_exit_invalid { uint64_t hwcode; }; /* Generic. */ #define NVMM_VCPU_EXIT_NONE 0x0000000000000000ULL #define NVMM_VCPU_EXIT_INVALID 0xFFFFFFFFFFFFFFFFULL /* x86: operations. */ #define NVMM_VCPU_EXIT_MEMORY 0x0000000000000001ULL #define NVMM_VCPU_EXIT_IO 0x0000000000000002ULL /* x86: changes in VCPU state. */ #define NVMM_VCPU_EXIT_SHUTDOWN 0x0000000000001000ULL #define NVMM_VCPU_EXIT_INT_READY 0x0000000000001001ULL #define NVMM_VCPU_EXIT_NMI_READY 0x0000000000001002ULL #define NVMM_VCPU_EXIT_HALTED 0x0000000000001003ULL #define NVMM_VCPU_EXIT_TPR_CHANGED 0x0000000000001004ULL /* x86: instructions. */ #define NVMM_VCPU_EXIT_RDMSR 0x0000000000002000ULL #define NVMM_VCPU_EXIT_WRMSR 0x0000000000002001ULL #define NVMM_VCPU_EXIT_MONITOR 0x0000000000002002ULL #define NVMM_VCPU_EXIT_MWAIT 0x0000000000002003ULL #define NVMM_VCPU_EXIT_CPUID 0x0000000000002004ULL struct nvmm_x86_exit { uint64_t reason; union { struct nvmm_x86_exit_memory mem; struct nvmm_x86_exit_io io; struct nvmm_x86_exit_rdmsr rdmsr; struct nvmm_x86_exit_wrmsr wrmsr; struct nvmm_x86_exit_insn insn; struct nvmm_x86_exit_invalid inv; } u; struct { uint64_t rflags; uint64_t cr8; uint64_t int_shadow:1; uint64_t int_window_exiting:1; uint64_t nmi_window_exiting:1; uint64_t evt_pending:1; uint64_t rsvd:60; } exitstate; }; #define nvmm_vcpu_exit nvmm_x86_exit #define NVMM_VCPU_EVENT_EXCP 0 #define NVMM_VCPU_EVENT_INTR 1 struct nvmm_x86_event { u_int type; uint8_t vector; union { struct { uint64_t error; } excp; } u; }; #define nvmm_vcpu_event nvmm_x86_event struct nvmm_cap_md { uint64_t mach_conf_support; uint64_t vcpu_conf_support; #define NVMM_CAP_ARCH_VCPU_CONF_CPUID __BIT(0) #define NVMM_CAP_ARCH_VCPU_CONF_TPR __BIT(1) uint64_t xcr0_mask; uint32_t mxcsr_mask; uint32_t conf_cpuid_maxops; uint64_t rsvd[6]; }; #endif /* ASM_NVMM */ /* -------------------------------------------------------------------------- */ /* * State indexes. We use X64 as naming convention, not to confuse with X86 * which originally implied 32bit. */ /* Segments. */ #define NVMM_X64_SEG_ES 0 #define NVMM_X64_SEG_CS 1 #define NVMM_X64_SEG_SS 2 #define NVMM_X64_SEG_DS 3 #define NVMM_X64_SEG_FS 4 #define NVMM_X64_SEG_GS 5 #define NVMM_X64_SEG_GDT 6 #define NVMM_X64_SEG_IDT 7 #define NVMM_X64_SEG_LDT 8 #define NVMM_X64_SEG_TR 9 #define NVMM_X64_NSEG 10 /* General Purpose Registers. */ #define NVMM_X64_GPR_RAX 0 #define NVMM_X64_GPR_RCX 1 #define NVMM_X64_GPR_RDX 2 #define NVMM_X64_GPR_RBX 3 #define NVMM_X64_GPR_RSP 4 #define NVMM_X64_GPR_RBP 5 #define NVMM_X64_GPR_RSI 6 #define NVMM_X64_GPR_RDI 7 #define NVMM_X64_GPR_R8 8 #define NVMM_X64_GPR_R9 9 #define NVMM_X64_GPR_R10 10 #define NVMM_X64_GPR_R11 11 #define NVMM_X64_GPR_R12 12 #define NVMM_X64_GPR_R13 13 #define NVMM_X64_GPR_R14 14 #define NVMM_X64_GPR_R15 15 #define NVMM_X64_GPR_RIP 16 #define NVMM_X64_GPR_RFLAGS 17 #define NVMM_X64_NGPR 18 /* Control Registers. */ #define NVMM_X64_CR_CR0 0 #define NVMM_X64_CR_CR2 1 #define NVMM_X64_CR_CR3 2 #define NVMM_X64_CR_CR4 3 #define NVMM_X64_CR_CR8 4 #define NVMM_X64_CR_XCR0 5 #define NVMM_X64_NCR 6 /* Debug Registers. */ #define NVMM_X64_DR_DR0 0 #define NVMM_X64_DR_DR1 1 #define NVMM_X64_DR_DR2 2 #define NVMM_X64_DR_DR3 3 #define NVMM_X64_DR_DR6 4 #define NVMM_X64_DR_DR7 5 #define NVMM_X64_NDR 6 /* MSRs. */ #define NVMM_X64_MSR_EFER 0 #define NVMM_X64_MSR_STAR 1 #define NVMM_X64_MSR_LSTAR 2 #define NVMM_X64_MSR_CSTAR 3 #define NVMM_X64_MSR_SFMASK 4 #define NVMM_X64_MSR_KERNELGSBASE 5 #define NVMM_X64_MSR_SYSENTER_CS 6 #define NVMM_X64_MSR_SYSENTER_ESP 7 #define NVMM_X64_MSR_SYSENTER_EIP 8 #define NVMM_X64_MSR_PAT 9 #define NVMM_X64_MSR_TSC 10 #define NVMM_X64_NMSR 11 #ifndef ASM_NVMM #include #include #if defined(__DragonFly__) #ifdef __x86_64__ #undef __BIT #define __BIT(__n) __BIT64(__n) #undef __BITS #define __BITS(__m, __n) __BITS64(__m, __n) #endif /* __x86_64__ */ #endif /* Segment state. */ struct nvmm_x64_state_seg { uint16_t selector; struct { /* hidden */ uint16_t type:4; uint16_t s:1; uint16_t dpl:2; uint16_t p:1; uint16_t avl:1; uint16_t l:1; uint16_t def:1; uint16_t g:1; uint16_t rsvd:4; } attrib; uint32_t limit; /* hidden */ uint64_t base; /* hidden */ }; /* Interrupt state. */ struct nvmm_x64_state_intr { uint64_t int_shadow:1; uint64_t int_window_exiting:1; uint64_t nmi_window_exiting:1; uint64_t evt_pending:1; uint64_t rsvd:60; }; /* FPU state structures. */ union nvmm_x64_state_fpu_addr { uint64_t fa_64; struct { uint32_t fa_off; uint16_t fa_seg; uint16_t fa_opcode; } fa_32; }; CTASSERT(sizeof(union nvmm_x64_state_fpu_addr) == 8); struct nvmm_x64_state_fpu_mmreg { uint64_t mm_significand; uint16_t mm_exp_sign; uint8_t mm_rsvd[6]; }; CTASSERT(sizeof(struct nvmm_x64_state_fpu_mmreg) == 16); struct nvmm_x64_state_fpu_xmmreg { uint8_t xmm_bytes[16]; }; CTASSERT(sizeof(struct nvmm_x64_state_fpu_xmmreg) == 16); /* FPU state (x87 + SSE). */ struct nvmm_x64_state_fpu { uint16_t fx_cw; /* Control Word */ uint16_t fx_sw; /* Status Word */ uint8_t fx_tw; /* Tag Word */ uint8_t fx_zero; uint16_t fx_opcode; union nvmm_x64_state_fpu_addr fx_ip; /* Instruction Pointer */ union nvmm_x64_state_fpu_addr fx_dp; /* Data pointer */ uint32_t fx_mxcsr; uint32_t fx_mxcsr_mask; struct nvmm_x64_state_fpu_mmreg fx_87_ac[8]; /* x87 registers */ struct nvmm_x64_state_fpu_xmmreg fx_xmm[16]; /* XMM registers */ uint8_t fx_rsvd[96]; } __aligned(16); CTASSERT(sizeof(struct nvmm_x64_state_fpu) == 512); /* Flags. */ #define NVMM_X64_STATE_SEGS 0x01 #define NVMM_X64_STATE_GPRS 0x02 #define NVMM_X64_STATE_CRS 0x04 #define NVMM_X64_STATE_DRS 0x08 #define NVMM_X64_STATE_MSRS 0x10 #define NVMM_X64_STATE_INTR 0x20 #define NVMM_X64_STATE_FPU 0x40 #define NVMM_X64_STATE_ALL \ (NVMM_X64_STATE_SEGS | NVMM_X64_STATE_GPRS | NVMM_X64_STATE_CRS | \ NVMM_X64_STATE_DRS | NVMM_X64_STATE_MSRS | NVMM_X64_STATE_INTR | \ NVMM_X64_STATE_FPU) struct nvmm_x64_state { struct nvmm_x64_state_seg segs[NVMM_X64_NSEG]; uint64_t gprs[NVMM_X64_NGPR]; uint64_t crs[NVMM_X64_NCR]; uint64_t drs[NVMM_X64_NDR]; uint64_t msrs[NVMM_X64_NMSR]; struct nvmm_x64_state_intr intr; struct nvmm_x64_state_fpu fpu; }; #define nvmm_vcpu_state nvmm_x64_state /* -------------------------------------------------------------------------- */ #define NVMM_VCPU_CONF_CPUID NVMM_VCPU_CONF_MD_BEGIN #define NVMM_VCPU_CONF_TPR (NVMM_VCPU_CONF_MD_BEGIN + 1) struct nvmm_vcpu_conf_cpuid { /* The options. */ uint32_t mask:1; uint32_t exit:1; uint32_t rsvd:30; /* The leaf. */ uint32_t leaf; /* The params. */ union { struct { struct { uint32_t eax; uint32_t ebx; uint32_t ecx; uint32_t edx; } set; struct { uint32_t eax; uint32_t ebx; uint32_t ecx; uint32_t edx; } del; } mask; } u; }; struct nvmm_vcpu_conf_tpr { uint32_t exit_changed:1; uint32_t rsvd:31; }; /* -------------------------------------------------------------------------- */ /* * CPUID defines. */ /* Fn0000_0001:EBX */ #define CPUID_0_01_EBX_BRAND_INDEX __BITS(7,0) #define CPUID_0_01_EBX_CLFLUSH_SIZE __BITS(15,8) #define CPUID_0_01_EBX_HTT_CORES __BITS(23,16) #define CPUID_0_01_EBX_LOCAL_APIC_ID __BITS(31,24) /* Fn0000_0001:ECX */ #define CPUID_0_01_ECX_SSE3 __BIT(0) #define CPUID_0_01_ECX_PCLMULQDQ __BIT(1) #define CPUID_0_01_ECX_DTES64 __BIT(2) #define CPUID_0_01_ECX_MONITOR __BIT(3) #define CPUID_0_01_ECX_DS_CPL __BIT(4) #define CPUID_0_01_ECX_VMX __BIT(5) #define CPUID_0_01_ECX_SMX __BIT(6) #define CPUID_0_01_ECX_EIST __BIT(7) #define CPUID_0_01_ECX_TM2 __BIT(8) #define CPUID_0_01_ECX_SSSE3 __BIT(9) #define CPUID_0_01_ECX_CNXTID __BIT(10) #define CPUID_0_01_ECX_SDBG __BIT(11) #define CPUID_0_01_ECX_FMA __BIT(12) #define CPUID_0_01_ECX_CX16 __BIT(13) #define CPUID_0_01_ECX_XTPR __BIT(14) #define CPUID_0_01_ECX_PDCM __BIT(15) #define CPUID_0_01_ECX_PCID __BIT(17) #define CPUID_0_01_ECX_DCA __BIT(18) #define CPUID_0_01_ECX_SSE41 __BIT(19) #define CPUID_0_01_ECX_SSE42 __BIT(20) #define CPUID_0_01_ECX_X2APIC __BIT(21) #define CPUID_0_01_ECX_MOVBE __BIT(22) #define CPUID_0_01_ECX_POPCNT __BIT(23) #define CPUID_0_01_ECX_TSC_DEADLINE __BIT(24) #define CPUID_0_01_ECX_AESNI __BIT(25) #define CPUID_0_01_ECX_XSAVE __BIT(26) #define CPUID_0_01_ECX_OSXSAVE __BIT(27) #define CPUID_0_01_ECX_AVX __BIT(28) #define CPUID_0_01_ECX_F16C __BIT(29) #define CPUID_0_01_ECX_RDRAND __BIT(30) #define CPUID_0_01_ECX_RAZ __BIT(31) /* Fn0000_0001:EDX */ #define CPUID_0_01_EDX_FPU __BIT(0) #define CPUID_0_01_EDX_VME __BIT(1) #define CPUID_0_01_EDX_DE __BIT(2) #define CPUID_0_01_EDX_PSE __BIT(3) #define CPUID_0_01_EDX_TSC __BIT(4) #define CPUID_0_01_EDX_MSR __BIT(5) #define CPUID_0_01_EDX_PAE __BIT(6) #define CPUID_0_01_EDX_MCE __BIT(7) #define CPUID_0_01_EDX_CX8 __BIT(8) #define CPUID_0_01_EDX_APIC __BIT(9) #define CPUID_0_01_EDX_SEP __BIT(11) #define CPUID_0_01_EDX_MTRR __BIT(12) #define CPUID_0_01_EDX_PGE __BIT(13) #define CPUID_0_01_EDX_MCA __BIT(14) #define CPUID_0_01_EDX_CMOV __BIT(15) #define CPUID_0_01_EDX_PAT __BIT(16) #define CPUID_0_01_EDX_PSE36 __BIT(17) #define CPUID_0_01_EDX_PSN __BIT(18) #define CPUID_0_01_EDX_CLFSH __BIT(19) #define CPUID_0_01_EDX_DS __BIT(21) #define CPUID_0_01_EDX_ACPI __BIT(22) #define CPUID_0_01_EDX_MMX __BIT(23) #define CPUID_0_01_EDX_FXSR __BIT(24) #define CPUID_0_01_EDX_SSE __BIT(25) #define CPUID_0_01_EDX_SSE2 __BIT(26) #define CPUID_0_01_EDX_SS __BIT(27) #define CPUID_0_01_EDX_HTT __BIT(28) #define CPUID_0_01_EDX_TM __BIT(29) #define CPUID_0_01_EDX_PBE __BIT(31) /* Fn0000_0004:EAX (Intel Deterministic Cache Parameter Leaf) */ #define CPUID_0_04_EAX_CACHETYPE __BITS(4, 0) #define CPUID_0_04_EAX_CACHETYPE_NULL 0 #define CPUID_0_04_EAX_CACHETYPE_DATA 1 #define CPUID_0_04_EAX_CACHETYPE_INSN 2 #define CPUID_0_04_EAX_CACHETYPE_UNIFIED 3 #define CPUID_0_04_EAX_CACHELEVEL __BITS(7, 5) #define CPUID_0_04_EAX_SELFINITCL __BIT(8) #define CPUID_0_04_EAX_FULLASSOC __BIT(9) #define CPUID_0_04_EAX_SHARING __BITS(25, 14) #define CPUID_0_04_EAX_CORE_P_PKG __BITS(31, 26) /* [ECX=0] Fn0000_0007:EBX (Structured Extended Features) */ #define CPUID_0_07_EBX_FSGSBASE __BIT(0) #define CPUID_0_07_EBX_TSC_ADJUST __BIT(1) #define CPUID_0_07_EBX_SGX __BIT(2) #define CPUID_0_07_EBX_BMI1 __BIT(3) #define CPUID_0_07_EBX_HLE __BIT(4) #define CPUID_0_07_EBX_AVX2 __BIT(5) #define CPUID_0_07_EBX_FDPEXONLY __BIT(6) #define CPUID_0_07_EBX_SMEP __BIT(7) #define CPUID_0_07_EBX_BMI2 __BIT(8) #define CPUID_0_07_EBX_ERMS __BIT(9) #define CPUID_0_07_EBX_INVPCID __BIT(10) #define CPUID_0_07_EBX_RTM __BIT(11) #define CPUID_0_07_EBX_QM __BIT(12) #define CPUID_0_07_EBX_FPUCSDS __BIT(13) #define CPUID_0_07_EBX_MPX __BIT(14) #define CPUID_0_07_EBX_PQE __BIT(15) #define CPUID_0_07_EBX_AVX512F __BIT(16) #define CPUID_0_07_EBX_AVX512DQ __BIT(17) #define CPUID_0_07_EBX_RDSEED __BIT(18) #define CPUID_0_07_EBX_ADX __BIT(19) #define CPUID_0_07_EBX_SMAP __BIT(20) #define CPUID_0_07_EBX_AVX512_IFMA __BIT(21) #define CPUID_0_07_EBX_CLFLUSHOPT __BIT(23) #define CPUID_0_07_EBX_CLWB __BIT(24) #define CPUID_0_07_EBX_PT __BIT(25) #define CPUID_0_07_EBX_AVX512PF __BIT(26) #define CPUID_0_07_EBX_AVX512ER __BIT(27) #define CPUID_0_07_EBX_AVX512CD __BIT(28) #define CPUID_0_07_EBX_SHA __BIT(29) #define CPUID_0_07_EBX_AVX512BW __BIT(30) #define CPUID_0_07_EBX_AVX512VL __BIT(31) /* [ECX=0] Fn0000_0007:ECX (Structured Extended Features) */ #define CPUID_0_07_ECX_PREFETCHWT1 __BIT(0) #define CPUID_0_07_ECX_AVX512_VBMI __BIT(1) #define CPUID_0_07_ECX_UMIP __BIT(2) #define CPUID_0_07_ECX_PKU __BIT(3) #define CPUID_0_07_ECX_OSPKE __BIT(4) #define CPUID_0_07_ECX_WAITPKG __BIT(5) #define CPUID_0_07_ECX_AVX512_VBMI2 __BIT(6) #define CPUID_0_07_ECX_CET_SS __BIT(7) #define CPUID_0_07_ECX_GFNI __BIT(8) #define CPUID_0_07_ECX_VAES __BIT(9) #define CPUID_0_07_ECX_VPCLMULQDQ __BIT(10) #define CPUID_0_07_ECX_AVX512_VNNI __BIT(11) #define CPUID_0_07_ECX_AVX512_BITALG __BIT(12) #define CPUID_0_07_ECX_AVX512_VPOPCNTDQ __BIT(14) #define CPUID_0_07_ECX_LA57 __BIT(16) #define CPUID_0_07_ECX_MAWAU __BITS(21, 17) #define CPUID_0_07_ECX_RDPID __BIT(22) #define CPUID_0_07_ECX_KL __BIT(23) #define CPUID_0_07_ECX_CLDEMOTE __BIT(25) #define CPUID_0_07_ECX_MOVDIRI __BIT(27) #define CPUID_0_07_ECX_MOVDIR64B __BIT(28) #define CPUID_0_07_ECX_SGXLC __BIT(30) #define CPUID_0_07_ECX_PKS __BIT(31) /* [ECX=0] Fn0000_0007:EDX (Structured Extended Features) */ #define CPUID_0_07_EDX_AVX512_4VNNIW __BIT(2) #define CPUID_0_07_EDX_AVX512_4FMAPS __BIT(3) #define CPUID_0_07_EDX_FSREP_MOV __BIT(4) #define CPUID_0_07_EDX_AVX512_VP2INTERSECT __BIT(8) #define CPUID_0_07_EDX_SRBDS_CTRL __BIT(9) #define CPUID_0_07_EDX_MD_CLEAR __BIT(10) #define CPUID_0_07_EDX_TSX_FORCE_ABORT __BIT(13) #define CPUID_0_07_EDX_SERIALIZE __BIT(14) #define CPUID_0_07_EDX_HYBRID __BIT(15) #define CPUID_0_07_EDX_TSXLDTRK __BIT(16) #define CPUID_0_07_EDX_CET_IBT __BIT(20) #define CPUID_0_07_EDX_IBRS __BIT(26) #define CPUID_0_07_EDX_STIBP __BIT(27) #define CPUID_0_07_EDX_L1D_FLUSH __BIT(28) #define CPUID_0_07_EDX_ARCH_CAP __BIT(29) #define CPUID_0_07_EDX_CORE_CAP __BIT(30) #define CPUID_0_07_EDX_SSBD __BIT(31) /* Fn0000_000B:EAX (Extended Topology Enumeration) */ #define CPUID_0_0B_EAX_SHIFTNUM __BITS(4, 0) /* Fn0000_000B:ECX (Extended Topology Enumeration) */ #define CPUID_0_0B_ECX_LVLNUM __BITS(7, 0) #define CPUID_0_0B_ECX_LVLTYPE __BITS(15, 8) #define CPUID_0_0B_ECX_LVLTYPE_INVAL 0 #define CPUID_0_0B_ECX_LVLTYPE_SMT 1 #define CPUID_0_0B_ECX_LVLTYPE_CORE 2 /* [ECX=1] Fn0000_000D:EAX (Processor Extended State Enumeration) */ #define CPUID_0_0D_ECX1_EAX_XSAVEOPT __BIT(0) #define CPUID_0_0D_ECX1_EAX_XSAVEC __BIT(1) #define CPUID_0_0D_ECX1_EAX_XGETBV __BIT(2) #define CPUID_0_0D_ECX1_EAX_XSAVES __BIT(3) /* Fn8000_0001:ECX */ #define CPUID_8_01_ECX_LAHF __BIT(0) #define CPUID_8_01_ECX_CMPLEGACY __BIT(1) #define CPUID_8_01_ECX_SVM __BIT(2) #define CPUID_8_01_ECX_EAPIC __BIT(3) #define CPUID_8_01_ECX_ALTMOVCR8 __BIT(4) #define CPUID_8_01_ECX_ABM __BIT(5) #define CPUID_8_01_ECX_SSE4A __BIT(6) #define CPUID_8_01_ECX_MISALIGNSSE __BIT(7) #define CPUID_8_01_ECX_3DNOWPF __BIT(8) #define CPUID_8_01_ECX_OSVW __BIT(9) #define CPUID_8_01_ECX_IBS __BIT(10) #define CPUID_8_01_ECX_XOP __BIT(11) #define CPUID_8_01_ECX_SKINIT __BIT(12) #define CPUID_8_01_ECX_WDT __BIT(13) #define CPUID_8_01_ECX_LWP __BIT(15) #define CPUID_8_01_ECX_FMA4 __BIT(16) #define CPUID_8_01_ECX_TCE __BIT(17) #define CPUID_8_01_ECX_NODEID __BIT(19) #define CPUID_8_01_ECX_TBM __BIT(21) #define CPUID_8_01_ECX_TOPOEXT __BIT(22) #define CPUID_8_01_ECX_PCEC __BIT(23) #define CPUID_8_01_ECX_PCENB __BIT(24) #define CPUID_8_01_ECX_DBE __BIT(26) #define CPUID_8_01_ECX_PERFTSC __BIT(27) #define CPUID_8_01_ECX_PERFEXTLLC __BIT(28) #define CPUID_8_01_ECX_MWAITX __BIT(29) /* Fn8000_0001:EDX */ #define CPUID_8_01_EDX_FPU __BIT(0) #define CPUID_8_01_EDX_VME __BIT(1) #define CPUID_8_01_EDX_DE __BIT(2) #define CPUID_8_01_EDX_PSE __BIT(3) #define CPUID_8_01_EDX_TSC __BIT(4) #define CPUID_8_01_EDX_MSR __BIT(5) #define CPUID_8_01_EDX_PAE __BIT(6) #define CPUID_8_01_EDX_MCE __BIT(7) #define CPUID_8_01_EDX_CX8 __BIT(8) #define CPUID_8_01_EDX_APIC __BIT(9) #define CPUID_8_01_EDX_SYSCALL __BIT(11) #define CPUID_8_01_EDX_MTRR __BIT(12) #define CPUID_8_01_EDX_PGE __BIT(13) #define CPUID_8_01_EDX_MCA __BIT(14) #define CPUID_8_01_EDX_CMOV __BIT(15) #define CPUID_8_01_EDX_PAT __BIT(16) #define CPUID_8_01_EDX_PSE36 __BIT(17) #define CPUID_8_01_EDX_XD __BIT(20) #define CPUID_8_01_EDX_MMXEXT __BIT(22) #define CPUID_8_01_EDX_MMX __BIT(23) #define CPUID_8_01_EDX_FXSR __BIT(24) #define CPUID_8_01_EDX_FFXSR __BIT(25) #define CPUID_8_01_EDX_PAGE1GB __BIT(26) #define CPUID_8_01_EDX_RDTSCP __BIT(27) #define CPUID_8_01_EDX_LM __BIT(29) #define CPUID_8_01_EDX_3DNOWEXT __BIT(30) #define CPUID_8_01_EDX_3DNOW __BIT(31) /* Fn8000_0007:EDX (Advanced Power Management) */ #define CPUID_8_07_EDX_TS __BIT(0) #define CPUID_8_07_EDX_FID __BIT(1) #define CPUID_8_07_EDX_VID __BIT(2) #define CPUID_8_07_EDX_TTP __BIT(3) #define CPUID_8_07_EDX_TM __BIT(4) #define CPUID_8_07_EDX_100MHzSteps __BIT(6) #define CPUID_8_07_EDX_HwPstate __BIT(7) #define CPUID_8_07_EDX_TscInvariant __BIT(8) #define CPUID_8_07_EDX_CPB __BIT(9) #define CPUID_8_07_EDX_EffFreqRO __BIT(10) #define CPUID_8_07_EDX_ProcFeedbackIntf __BIT(11) #define CPUID_8_07_EDX_ProcPowerReport __BIT(12) /* Fn8000_0008:EBX */ #define CPUID_8_08_EBX_CLZERO __BIT(0) #define CPUID_8_08_EBX_InstRetCntMsr __BIT(1) #define CPUID_8_08_EBX_RstrFpErrPtrs __BIT(2) #define CPUID_8_08_EBX_INVLPGB __BIT(3) #define CPUID_8_08_EBX_RDPRU __BIT(4) #define CPUID_8_08_EBX_MCOMMIT __BIT(8) #define CPUID_8_08_EBX_WBNOINVD __BIT(9) #define CPUID_8_08_EBX_IBPB __BIT(12) #define CPUID_8_08_EBX_INT_WBINVD __BIT(13) #define CPUID_8_08_EBX_IBRS __BIT(14) #define CPUID_8_08_EBX_STIBP __BIT(15) #define CPUID_8_08_EBX_IBRS_ALWAYSON __BIT(16) #define CPUID_8_08_EBX_STIBP_ALWAYSON __BIT(17) #define CPUID_8_08_EBX_PREFER_IBRS __BIT(18) #define CPUID_8_08_EBX_EferLmsleUnsupp __BIT(20) #define CPUID_8_08_EBX_INVLPGBnestedPg __BIT(21) #define CPUID_8_08_EBX_SSBD __BIT(24) #define CPUID_8_08_EBX_VIRT_SSBD __BIT(25) #define CPUID_8_08_EBX_SSB_NO __BIT(26) /* Fn8000_0008:ECX */ #define CPUID_8_08_ECX_NC __BITS(7,0) #define CPUID_8_08_ECX_ApicIdSize __BITS(15,12) #define CPUID_8_08_ECX_PerfTscSize __BITS(17,16) /* Fn8000_000A:EAX (SVM features) */ #define CPUID_8_0A_EAX_SvmRev __BITS(7,0) /* Fn8000_000A:EDX (SVM features) */ #define CPUID_8_0A_EDX_NP __BIT(0) #define CPUID_8_0A_EDX_LbrVirt __BIT(1) #define CPUID_8_0A_EDX_SVML __BIT(2) #define CPUID_8_0A_EDX_NRIPS __BIT(3) #define CPUID_8_0A_EDX_TscRateMsr __BIT(4) #define CPUID_8_0A_EDX_VmcbClean __BIT(5) #define CPUID_8_0A_EDX_FlushByASID __BIT(6) #define CPUID_8_0A_EDX_DecodeAssists __BIT(7) #define CPUID_8_0A_EDX_PauseFilter __BIT(10) #define CPUID_8_0A_EDX_PFThreshold __BIT(12) #define CPUID_8_0A_EDX_AVIC __BIT(13) #define CPUID_8_0A_EDX_VMSAVEvirt __BIT(15) #define CPUID_8_0A_EDX_VGIF __BIT(16) #define CPUID_8_0A_EDX_GMET __BIT(17) #define CPUID_8_0A_EDX_SSSCheck __BIT(19) #define CPUID_8_0A_EDX_SpecCtrl __BIT(20) #define CPUID_8_0A_EDX_TlbiCtl __BIT(24) /* -------------------------------------------------------------------------- */ /* * Register defines. We mainly rely on the already-existing OS definitions. */ #if defined(__DragonFly__) #define XCR0_X87 CPU_XFEATURE_X87 /* 0x00000001 */ #define XCR0_SSE CPU_XFEATURE_SSE /* 0x00000002 */ #define MSR_MISC_ENABLE MSR_IA32_MISC_ENABLE /* 0x1a0 */ #define MSR_CR_PAT MSR_PAT /* 0x277 */ #define MSR_SFMASK MSR_SF_MASK /* 0xc0000084 */ #define MSR_KERNELGSBASE MSR_KGSBASE /* 0xc0000102 */ #define MSR_NB_CFG MSR_AMD_NB_CFG /* 0xc001001f */ #define MSR_IC_CFG MSR_AMD_IC_CFG /* 0xc0011021 */ #define MSR_DE_CFG MSR_AMD_DE_CFG /* 0xc0011029 */ #define MSR_UCODE_AMD_PATCHLEVEL MSR_AMD_PATCH_LEVEL /* 0x0000008b */ /* MSR_IA32_ARCH_CAPABILITIES (0x10a) */ #define IA32_ARCH_RDCL_NO IA32_ARCH_CAP_RDCL_NO #define IA32_ARCH_IBRS_ALL IA32_ARCH_CAP_IBRS_ALL #define IA32_ARCH_RSBA IA32_ARCH_CAP_RSBA #define IA32_ARCH_SKIP_L1DFL_VMENTRY IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY #define IA32_ARCH_SSB_NO IA32_ARCH_CAP_SSB_NO #define IA32_ARCH_MDS_NO IA32_ARCH_CAP_MDS_NO #define IA32_ARCH_IF_PSCHANGE_MC_NO IA32_ARCH_CAP_IF_PSCHANGE_MC_NO #define IA32_ARCH_TSX_CTRL IA32_ARCH_CAP_TSX_CTRL #define IA32_ARCH_TAA_NO IA32_ARCH_CAP_TAA_NO /* MSR_IA32_FLUSH_CMD (0x10b) */ #define IA32_FLUSH_CMD_L1D_FLUSH IA32_FLUSH_CMD_L1D #endif /* __DragonFly__ */ /* -------------------------------------------------------------------------- */ #ifdef _KERNEL #define NVMM_X86_MACH_NCONF 0 #define NVMM_X86_VCPU_NCONF 2 struct nvmm_x86_cpuid_mask { uint32_t eax; uint32_t ebx; uint32_t ecx; uint32_t edx; }; /* FPU area + XSAVE header. */ struct nvmm_x86_xsave { struct nvmm_x64_state_fpu fpu; uint64_t xstate_bv; uint64_t xcomp_bv; uint8_t rsvd0[8]; uint8_t rsvd[40]; }; CTASSERT(sizeof(struct nvmm_x86_xsave) == 512 + 64); extern const struct nvmm_x64_state nvmm_x86_reset_state; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_00000001; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_00000007; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000001; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000007; extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000008; bool nvmm_x86_pat_validate(uint64_t); uint32_t nvmm_x86_xsave_size(uint64_t); /* -------------------------------------------------------------------------- */ /* * ASM defines. We mainly rely on the already-existing OS definitions. */ #if defined(__NetBSD__) #include #include #elif defined(__DragonFly__) #include #include #endif /* CPUID. */ typedef struct { uint32_t eax, ebx, ecx, edx; } cpuid_desc_t; #if defined(__NetBSD__) #define x86_get_cpuid(l, d) x86_cpuid(l, (uint32_t *)d) #define x86_get_cpuid2(l, c, d) x86_cpuid2(l, c, (uint32_t *)d) #elif defined(__DragonFly__) #define x86_get_cpuid(l, d) do_cpuid(l, (uint32_t *)d) #define x86_get_cpuid2(l, c, d) cpuid_count(l, c, (uint32_t *)d) #endif /* Control registers. */ #if defined(__NetBSD__) #define x86_get_cr0() rcr0() #define x86_get_cr2() rcr2() #define x86_get_cr3() rcr3() #define x86_get_cr4() rcr4() #define x86_set_cr0(v) lcr0(v) #define x86_set_cr2(v) lcr2(v) #define x86_set_cr4(v) lcr4(v) #elif defined(__DragonFly__) #define x86_get_cr0() rcr0() #define x86_get_cr2() rcr2() #define x86_get_cr3() rcr3() #define x86_get_cr4() rcr4() #define x86_set_cr0(v) load_cr0(v) #define x86_set_cr2(v) load_cr2(v) #define x86_set_cr4(v) load_cr4(v) #endif /* Debug registers. */ #if defined(__NetBSD__) #include static inline void x86_curthread_save_dbregs(uint64_t *drs __unused) { x86_dbregs_save(curlwp); } static inline void x86_curthread_restore_dbregs(uint64_t *drs __unused) { x86_dbregs_restore(curlwp); } #define x86_get_dr0() rdr0() #define x86_get_dr1() rdr1() #define x86_get_dr2() rdr2() #define x86_get_dr3() rdr3() #define x86_get_dr6() rdr6() #define x86_get_dr7() rdr7() #define x86_set_dr0(v) ldr0(v) #define x86_set_dr1(v) ldr1(v) #define x86_set_dr2(v) ldr2(v) #define x86_set_dr3(v) ldr3(v) #define x86_set_dr6(v) ldr6(v) #define x86_set_dr7(v) ldr7(v) #elif defined(__DragonFly__) #include /* struct lwp */ static inline void x86_curthread_save_dbregs(uint64_t *drs) { struct pcb *pcb = curthread->td_lwp->lwp_thread->td_pcb; if (__predict_true(!(pcb->pcb_flags & PCB_DBREGS))) return; drs[NVMM_X64_DR_DR0] = rdr0(); drs[NVMM_X64_DR_DR1] = rdr1(); drs[NVMM_X64_DR_DR2] = rdr2(); drs[NVMM_X64_DR_DR3] = rdr3(); drs[NVMM_X64_DR_DR6] = rdr6(); drs[NVMM_X64_DR_DR7] = rdr7(); } static inline void x86_curthread_restore_dbregs(uint64_t *drs) { struct pcb *pcb = curthread->td_lwp->lwp_thread->td_pcb; if (__predict_true(!(pcb->pcb_flags & PCB_DBREGS))) return; load_dr0(drs[NVMM_X64_DR_DR0]); load_dr1(drs[NVMM_X64_DR_DR1]); load_dr2(drs[NVMM_X64_DR_DR2]); load_dr3(drs[NVMM_X64_DR_DR3]); load_dr6(drs[NVMM_X64_DR_DR6]); load_dr7(drs[NVMM_X64_DR_DR7]); } #define x86_get_dr0() rdr0() #define x86_get_dr1() rdr1() #define x86_get_dr2() rdr2() #define x86_get_dr3() rdr3() #define x86_get_dr6() rdr6() #define x86_get_dr7() rdr7() #define x86_set_dr0(v) load_dr0(v) #define x86_set_dr1(v) load_dr1(v) #define x86_set_dr2(v) load_dr2(v) #define x86_set_dr3(v) load_dr3(v) #define x86_set_dr6(v) load_dr6(v) #define x86_set_dr7(v) load_dr7(v) #endif /* FPU. */ #if defined(__NetBSD__) #define x86_curthread_save_fpu() fpu_kern_enter() #define x86_curthread_restore_fpu() fpu_kern_leave() #define x86_save_fpu(a, m) fpu_area_save(a, m, true) #define x86_restore_fpu(a, m) fpu_area_restore(a, m, true) #elif defined(__DragonFly__) #define x86_curthread_save_fpu() /* TODO */ #define x86_curthread_restore_fpu() /* TODO */ #define x86_save_fpu(a, m) \ ({ \ fpusave((union savefpu *)(a), m); \ load_cr0(rcr0() | CR0_TS); \ }) #define x86_restore_fpu(a, m) \ ({ \ __asm volatile("clts" ::: "memory"); \ fpurstor((union savefpu *)(a), m); \ }) #endif /* XCRs. */ static inline uint64_t x86_get_xcr(uint32_t xcr) { uint32_t low, high; __asm volatile ( "xgetbv" : "=a" (low), "=d" (high) : "c" (xcr) ); return (low | ((uint64_t)high << 32)); } static inline void x86_set_xcr(uint32_t xcr, uint64_t val) { uint32_t low, high; low = val; high = val >> 32; __asm volatile ( "xsetbv" : : "a" (low), "d" (high), "c" (xcr) : "memory" ); } #if defined(__DragonFly__) #define x86_xsave_features npx_xcr0_mask #define x86_fpu_mxcsr_mask npx_mxcsr_mask #endif #endif /* _KERNEL */ #endif /* ASM_NVMM */ #endif /* _NVMM_X86_H_ */