diff options
| -rw-r--r-- | arch/x86/include/asm/calling.h | 18 | ||||
| -rw-r--r-- | arch/x86/include/asm/cpufeatures.h | 3 | ||||
| -rw-r--r-- | arch/x86/kernel/entry_64.S | 31 |
3 files changed, 49 insertions, 3 deletions
diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h index cb4c73bfeb48..129e29721835 100644 --- a/arch/x86/include/asm/calling.h +++ b/arch/x86/include/asm/calling.h @@ -47,6 +47,7 @@ For 32-bit we have the following conventions - kernel is built with */ #include <asm/dwarf2.h> +#include <asm/cpufeatures.h> #ifdef CONFIG_X86_64 @@ -195,6 +196,23 @@ For 32-bit we have the following conventions - kernel is built with .byte 0xf1 .endm +/* + * Mitigate Spectre v1 for conditional swapgs code paths. + * + * FENCE_SWAPGS_USER_ENTRY is used in the user entry swapgs code path, to + * prevent a speculative swapgs when coming from kernel space. + * + * FENCE_SWAPGS_KERNEL_ENTRY is used in the kernel entry non-swapgs code path, + * to prevent the swapgs from getting speculatively skipped when coming from + * user space. + */ +.macro FENCE_SWAPGS_USER_ENTRY + ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_USER +.endm +.macro FENCE_SWAPGS_KERNEL_ENTRY + ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_KERNEL +.endm + #else /* CONFIG_X86_64 */ /* diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 58051817664f..4e79d301d208 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -183,7 +183,8 @@ #define X86_FEATURE_DTHERM ( 7*32+ 7) /* Digital Thermal Sensor */ #define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */ #define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ - +#define X86_FEATURE_FENCE_SWAPGS_USER ( 7*32+10) /* "" LFENCE in user entry SWAPGS path */ +#define X86_FEATURE_FENCE_SWAPGS_KERNEL ( 7*32+11) /* "" LFENCE in kernel entry SWAPGS path */ #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ #define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 1a0c3643ff81..b4274d9a4e2e 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -265,14 +265,19 @@ ENDPROC(native_usergs_sysret64) testl $3, CS-RBP(%rsi) je 1f SWAPGS + FENCE_SWAPGS_USER_ENTRY SWITCH_KERNEL_CR3 + jmpq 2f +1: + FENCE_SWAPGS_KERNEL_ENTRY +2: /* * irq_count is used to check if a CPU is already on an interrupt stack * or not. While this is essentially redundant with preempt_count it is * a little cheaper to use a separate counter in the PDA (short of * moving irq_enter into assembly, which would be too much work) */ -1: incl PER_CPU_VAR(irq_count) + incl PER_CPU_VAR(irq_count) cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp CFI_DEF_CFA_REGISTER rsi @@ -337,6 +342,13 @@ ENTRY(save_paranoid) movq %rax, %cr3 2: #endif + /* + * The above doesn't do an unconditional CR3 write, even in the PTI + * case. So do an lfence to prevent GS speculation, regardless of + * whether PTI is enabled. + */ + FENCE_SWAPGS_KERNEL_ENTRY + ret CFI_ENDPROC END(save_paranoid) @@ -1452,8 +1464,19 @@ ENTRY(error_entry) * from user mode due to an IRET fault. */ SWAPGS + FENCE_SWAPGS_USER_ENTRY .Lerror_entry_from_usermode_after_swapgs: + /* + * We need to tell lockdep that IRQs are off. We can't do this until + * we fix gsbase, and we should do it before enter_from_user_mode + * (which can take locks). + */ + TRACE_IRQS_OFF + ret + +.Lerror_entry_done_lfence: + FENCE_SWAPGS_KERNEL_ENTRY .Lerror_entry_done: TRACE_IRQS_OFF ret @@ -1472,7 +1495,7 @@ ENTRY(error_entry) cmpq %rax,RIP+8(%rsp) je .Lbstep_iret cmpq $gs_change,RIP+8(%rsp) - jne .Lerror_entry_done + jne .Lerror_entry_done_lfence /* * hack: gs_change can fail with user gsbase. If this happens, fix up @@ -1480,6 +1503,7 @@ ENTRY(error_entry) * gs_change's error handler with kernel gsbase. */ SWAPGS + FENCE_SWAPGS_USER_ENTRY jmp .Lerror_entry_done .Lbstep_iret: @@ -1493,6 +1517,7 @@ ENTRY(error_entry) * Switch to kernel gsbase: */ SWAPGS + FENCE_SWAPGS_USER_ENTRY /* * Pretend that the exception came from user mode: set up pt_regs @@ -1601,6 +1626,7 @@ ENTRY(nmi) * to switch CR3 here. */ cld + FENCE_SWAPGS_USER_ENTRY movq %rsp, %rdx movq PER_CPU_VAR(kernel_stack), %rsp addq $KERNEL_STACK_OFFSET, %rsp @@ -1646,6 +1672,7 @@ ENTRY(nmi) movq %rax, %cr3 2: #endif + FENCE_SWAPGS_KERNEL_ENTRY call do_nmi #ifdef CONFIG_PAGE_TABLE_ISOLATION |