diff options
Diffstat (limited to 'arch/x86/entry/entry_64.S')
-rw-r--r-- | arch/x86/entry/entry_64.S | 21 |
1 files changed, 19 insertions, 2 deletions
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 8252d9dc48eb..592f62fc49f0 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -420,6 +420,7 @@ END(irq_entries_start) * tracking that we're in kernel mode. */ SWAPGS + FENCE_SWAPGS_USER_ENTRY SWITCH_KERNEL_CR3 /* @@ -433,8 +434,10 @@ END(irq_entries_start) TRACE_IRQS_OFF CALL_enter_from_user_mode - + jmpq 2f 1: + FENCE_SWAPGS_KERNEL_ENTRY +2: /* * Save previous stack pointer, optionally switch to interrupt stack. * irq_count is used to check if a CPU is already on an interrupt stack @@ -1004,6 +1007,13 @@ ENTRY(paranoid_entry) movq %rax, %cr3 2: #endif + /* + * The above doesn't do an unconditional CR3 write, even in the PTI + * case. So do an lfence to prevent GS speculation, regardless of + * whether PTI is enabled. + */ + FENCE_SWAPGS_KERNEL_ENTRY + ret END(paranoid_entry) @@ -1065,6 +1075,7 @@ ENTRY(error_entry) * from user mode due to an IRET fault. */ SWAPGS + FENCE_SWAPGS_USER_ENTRY .Lerror_entry_from_usermode_after_swapgs: /* @@ -1076,6 +1087,8 @@ ENTRY(error_entry) CALL_enter_from_user_mode ret +.Lerror_entry_done_lfence: + FENCE_SWAPGS_KERNEL_ENTRY .Lerror_entry_done: TRACE_IRQS_OFF ret @@ -1094,7 +1107,7 @@ ENTRY(error_entry) cmpq %rax, RIP+8(%rsp) je .Lbstep_iret cmpq $.Lgs_change, RIP+8(%rsp) - jne .Lerror_entry_done + jne .Lerror_entry_done_lfence /* * hack: .Lgs_change can fail with user gsbase. If this happens, fix up @@ -1102,6 +1115,7 @@ ENTRY(error_entry) * .Lgs_change's error handler with kernel gsbase. */ SWAPGS + FENCE_SWAPGS_USER_ENTRY jmp .Lerror_entry_done .Lbstep_iret: @@ -1115,6 +1129,7 @@ ENTRY(error_entry) * Switch to kernel gsbase: */ SWAPGS + FENCE_SWAPGS_USER_ENTRY /* * Pretend that the exception came from user mode: set up pt_regs @@ -1211,6 +1226,7 @@ ENTRY(nmi) * to switch CR3 here. */ cld + FENCE_SWAPGS_USER_ENTRY movq %rsp, %rdx movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp pushq 5*8(%rdx) /* pt_regs->ss */ @@ -1499,6 +1515,7 @@ end_repeat_nmi: movq %rax, %cr3 2: #endif + FENCE_SWAPGS_KERNEL_ENTRY /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ call do_nmi |