224 lines
8.7 KiB
Diff
224 lines
8.7 KiB
Diff
From e7505a450c34e89009ba48c459c08397ee3fc227 Mon Sep 17 00:00:00 2001
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
|
Date: Mon, 18 Feb 2019 23:04:01 +0100
|
|
Subject: [PATCH 12/30] x86/speculation/mds: Conditionally clear CPU buffers on
|
|
idle entry
|
|
|
|
commit 07f07f55a29cb705e221eda7894dd67ab81ef343 upstream
|
|
|
|
Add a static key which controls the invocation of the CPU buffer clear
|
|
mechanism on idle entry. This is independent of other MDS mitigations
|
|
because the idle entry invocation to mitigate the potential leakage due to
|
|
store buffer repartitioning is only necessary on SMT systems.
|
|
|
|
Add the actual invocations to the different halt/mwait variants which
|
|
covers all usage sites. mwaitx is not patched as it's not available on
|
|
Intel CPUs.
|
|
|
|
The buffer clear is only invoked before entering the C-State to prevent
|
|
that stale data from the idling CPU is spilled to the Hyper-Thread sibling
|
|
after the Store buffer got repartitioned and all entries are available to
|
|
the non idle sibling.
|
|
|
|
When coming out of idle the store buffer is partitioned again so each
|
|
sibling has half of it available. Now CPU which returned from idle could be
|
|
speculatively exposed to contents of the sibling, but the buffers are
|
|
flushed either on exit to user space or on VMENTER.
|
|
|
|
When later on conditional buffer clearing is implemented on top of this,
|
|
then there is no action required either because before returning to user
|
|
space the context switch will set the condition flag which causes a flush
|
|
on the return to user path.
|
|
|
|
Note, that the buffer clearing on idle is only sensible on CPUs which are
|
|
solely affected by MSBDS and not any other variant of MDS because the other
|
|
MDS variants cannot be mitigated when SMT is enabled, so the buffer
|
|
clearing on idle would be a window dressing exercise.
|
|
|
|
This intentionally does not handle the case in the acpi/processor_idle
|
|
driver which uses the legacy IO port interface for C-State transitions for
|
|
two reasons:
|
|
|
|
- The acpi/processor_idle driver was replaced by the intel_idle driver
|
|
almost a decade ago. Anything Nehalem upwards supports it and defaults
|
|
to that new driver.
|
|
|
|
- The legacy IO port interface is likely to be used on older and therefore
|
|
unaffected CPUs or on systems which do not receive microcode updates
|
|
anymore, so there is no point in adding that.
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
Reviewed-by: Borislav Petkov <bp@suse.de>
|
|
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
|
|
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
|
|
Reviewed-by: Jon Masters <jcm@redhat.com>
|
|
Tested-by: Jon Masters <jcm@redhat.com>
|
|
---
|
|
Documentation/x86/mds.rst | 42 ++++++++++++++++++++++++++++
|
|
arch/x86/include/asm/irqflags.h | 4 +++
|
|
arch/x86/include/asm/mwait.h | 7 +++++
|
|
arch/x86/include/asm/nospec-branch.h | 12 ++++++++
|
|
arch/x86/kernel/cpu/bugs.c | 3 ++
|
|
5 files changed, 68 insertions(+)
|
|
|
|
diff --git a/Documentation/x86/mds.rst b/Documentation/x86/mds.rst
|
|
index 54d935bf283b..87ce8ac9f36e 100644
|
|
--- a/Documentation/x86/mds.rst
|
|
+++ b/Documentation/x86/mds.rst
|
|
@@ -149,3 +149,45 @@ Mitigation points
|
|
This takes the paranoid exit path only when the INT1 breakpoint is in
|
|
kernel space. #DB on a user space address takes the regular exit path,
|
|
so no extra mitigation required.
|
|
+
|
|
+
|
|
+2. C-State transition
|
|
+^^^^^^^^^^^^^^^^^^^^^
|
|
+
|
|
+ When a CPU goes idle and enters a C-State the CPU buffers need to be
|
|
+ cleared on affected CPUs when SMT is active. This addresses the
|
|
+ repartitioning of the store buffer when one of the Hyper-Threads enters
|
|
+ a C-State.
|
|
+
|
|
+ When SMT is inactive, i.e. either the CPU does not support it or all
|
|
+ sibling threads are offline CPU buffer clearing is not required.
|
|
+
|
|
+ The idle clearing is enabled on CPUs which are only affected by MSBDS
|
|
+ and not by any other MDS variant. The other MDS variants cannot be
|
|
+ protected against cross Hyper-Thread attacks because the Fill Buffer and
|
|
+ the Load Ports are shared. So on CPUs affected by other variants, the
|
|
+ idle clearing would be a window dressing exercise and is therefore not
|
|
+ activated.
|
|
+
|
|
+ The invocation is controlled by the static key mds_idle_clear which is
|
|
+ switched depending on the chosen mitigation mode and the SMT state of
|
|
+ the system.
|
|
+
|
|
+ The buffer clear is only invoked before entering the C-State to prevent
|
|
+ that stale data from the idling CPU from spilling to the Hyper-Thread
|
|
+ sibling after the store buffer got repartitioned and all entries are
|
|
+ available to the non idle sibling.
|
|
+
|
|
+ When coming out of idle the store buffer is partitioned again so each
|
|
+ sibling has half of it available. The back from idle CPU could be then
|
|
+ speculatively exposed to contents of the sibling. The buffers are
|
|
+ flushed either on exit to user space or on VMENTER so malicious code
|
|
+ in user space or the guest cannot speculatively access them.
|
|
+
|
|
+ The mitigation is hooked into all variants of halt()/mwait(), but does
|
|
+ not cover the legacy ACPI IO-Port mechanism because the ACPI idle driver
|
|
+ has been superseded by the intel_idle driver around 2010 and is
|
|
+ preferred on all affected CPUs which are expected to gain the MD_CLEAR
|
|
+ functionality in microcode. Aside of that the IO-Port mechanism is a
|
|
+ legacy interface which is only used on older systems which are either
|
|
+ not affected or do not receive microcode updates anymore.
|
|
diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
|
|
index 15450a675031..c99c66b41e53 100644
|
|
--- a/arch/x86/include/asm/irqflags.h
|
|
+++ b/arch/x86/include/asm/irqflags.h
|
|
@@ -6,6 +6,8 @@
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
+#include <asm/nospec-branch.h>
|
|
+
|
|
/* Provide __cpuidle; we can't safely include <linux/cpu.h> */
|
|
#define __cpuidle __attribute__((__section__(".cpuidle.text")))
|
|
|
|
@@ -54,11 +56,13 @@ static inline void native_irq_enable(void)
|
|
|
|
static inline __cpuidle void native_safe_halt(void)
|
|
{
|
|
+ mds_idle_clear_cpu_buffers();
|
|
asm volatile("sti; hlt": : :"memory");
|
|
}
|
|
|
|
static inline __cpuidle void native_halt(void)
|
|
{
|
|
+ mds_idle_clear_cpu_buffers();
|
|
asm volatile("hlt": : :"memory");
|
|
}
|
|
|
|
diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
|
|
index 39a2fb29378a..eb0f80ce8524 100644
|
|
--- a/arch/x86/include/asm/mwait.h
|
|
+++ b/arch/x86/include/asm/mwait.h
|
|
@@ -6,6 +6,7 @@
|
|
#include <linux/sched/idle.h>
|
|
|
|
#include <asm/cpufeature.h>
|
|
+#include <asm/nospec-branch.h>
|
|
|
|
#define MWAIT_SUBSTATE_MASK 0xf
|
|
#define MWAIT_CSTATE_MASK 0xf
|
|
@@ -40,6 +41,8 @@ static inline void __monitorx(const void *eax, unsigned long ecx,
|
|
|
|
static inline void __mwait(unsigned long eax, unsigned long ecx)
|
|
{
|
|
+ mds_idle_clear_cpu_buffers();
|
|
+
|
|
/* "mwait %eax, %ecx;" */
|
|
asm volatile(".byte 0x0f, 0x01, 0xc9;"
|
|
:: "a" (eax), "c" (ecx));
|
|
@@ -74,6 +77,8 @@ static inline void __mwait(unsigned long eax, unsigned long ecx)
|
|
static inline void __mwaitx(unsigned long eax, unsigned long ebx,
|
|
unsigned long ecx)
|
|
{
|
|
+ /* No MDS buffer clear as this is AMD/HYGON only */
|
|
+
|
|
/* "mwaitx %eax, %ebx, %ecx;" */
|
|
asm volatile(".byte 0x0f, 0x01, 0xfb;"
|
|
:: "a" (eax), "b" (ebx), "c" (ecx));
|
|
@@ -81,6 +86,8 @@ static inline void __mwaitx(unsigned long eax, unsigned long ebx,
|
|
|
|
static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
|
|
{
|
|
+ mds_idle_clear_cpu_buffers();
|
|
+
|
|
trace_hardirqs_on();
|
|
/* "mwait %eax, %ecx;" */
|
|
asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
|
|
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
|
|
index 912d509d34fc..599c273f5d00 100644
|
|
--- a/arch/x86/include/asm/nospec-branch.h
|
|
+++ b/arch/x86/include/asm/nospec-branch.h
|
|
@@ -318,6 +318,7 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
|
|
DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
|
|
|
|
DECLARE_STATIC_KEY_FALSE(mds_user_clear);
|
|
+DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
|
|
|
|
#include <asm/segment.h>
|
|
|
|
@@ -355,6 +356,17 @@ static inline void mds_user_clear_cpu_buffers(void)
|
|
mds_clear_cpu_buffers();
|
|
}
|
|
|
|
+/**
|
|
+ * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
|
|
+ *
|
|
+ * Clear CPU buffers if the corresponding static key is enabled
|
|
+ */
|
|
+static inline void mds_idle_clear_cpu_buffers(void)
|
|
+{
|
|
+ if (static_branch_likely(&mds_idle_clear))
|
|
+ mds_clear_cpu_buffers();
|
|
+}
|
|
+
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
/*
|
|
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
|
|
index c01468ccefc1..428fe6590360 100644
|
|
--- a/arch/x86/kernel/cpu/bugs.c
|
|
+++ b/arch/x86/kernel/cpu/bugs.c
|
|
@@ -64,6 +64,9 @@ DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
|
|
/* Control MDS CPU buffer clear before returning to user space */
|
|
DEFINE_STATIC_KEY_FALSE(mds_user_clear);
|
|
EXPORT_SYMBOL_GPL(mds_user_clear);
|
|
+/* Control MDS CPU buffer clear before idling (halt, mwait) */
|
|
+DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
|
|
+EXPORT_SYMBOL_GPL(mds_idle_clear);
|
|
|
|
void __init check_bugs(void)
|
|
{
|