167 lines
5.6 KiB
Diff
167 lines
5.6 KiB
Diff
Subject: rcu: Make ksoftirqd do RCU quiescent states
|
|
From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
|
|
Date: Wed, 5 Oct 2011 11:45:18 -0700
|
|
|
|
Implementing RCU-bh in terms of RCU-preempt makes the system vulnerable
|
|
to network-based denial-of-service attacks. This patch therefore
|
|
makes __do_softirq() invoke rcu_bh_qs(), but only when __do_softirq()
|
|
is running in ksoftirqd context. A wrapper layer in interposed so that
|
|
other calls to __do_softirq() avoid invoking rcu_bh_qs(). The underlying
|
|
function __do_softirq_common() does the actual work.
|
|
|
|
The reason that rcu_bh_qs() is bad in these non-ksoftirqd contexts is
|
|
that there might be a local_bh_enable() inside an RCU-preempt read-side
|
|
critical section. This local_bh_enable() can invoke __do_softirq()
|
|
directly, so if __do_softirq() were to invoke rcu_bh_qs() (which just
|
|
calls rcu_preempt_qs() in the PREEMPT_RT_FULL case), there would be
|
|
an illegal RCU-preempt quiescent state in the middle of an RCU-preempt
|
|
read-side critical section. Therefore, quiescent states can only happen
|
|
in cases where __do_softirq() is invoked directly from ksoftirqd.
|
|
|
|
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
|
|
Link: http://lkml.kernel.org/r/20111005184518.GA21601@linux.vnet.ibm.com
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
---
|
|
include/linux/rcupdate.h | 6 ------
|
|
kernel/rcutree.c | 7 ++++++-
|
|
kernel/rcutree.h | 1 +
|
|
kernel/rcutree_plugin.h | 2 +-
|
|
kernel/softirq.c | 20 +++++++++++++-------
|
|
5 files changed, 21 insertions(+), 15 deletions(-)
|
|
|
|
Index: linux-3.2/include/linux/rcupdate.h
|
|
===================================================================
|
|
--- linux-3.2.orig/include/linux/rcupdate.h
|
|
+++ linux-3.2/include/linux/rcupdate.h
|
|
@@ -185,13 +185,7 @@ static inline int rcu_preempt_depth(void
|
|
|
|
/* Internal to kernel */
|
|
extern void rcu_sched_qs(int cpu);
|
|
-
|
|
-#ifndef CONFIG_PREEMPT_RT_FULL
|
|
extern void rcu_bh_qs(int cpu);
|
|
-#else
|
|
-static inline void rcu_bh_qs(int cpu) { }
|
|
-#endif
|
|
-
|
|
extern void rcu_check_callbacks(int cpu, int user);
|
|
struct notifier_block;
|
|
|
|
Index: linux-3.2/kernel/rcutree.c
|
|
===================================================================
|
|
--- linux-3.2.orig/kernel/rcutree.c
|
|
+++ linux-3.2/kernel/rcutree.c
|
|
@@ -170,7 +170,12 @@ void rcu_sched_qs(int cpu)
|
|
rdp->passed_quiesce = 1;
|
|
}
|
|
|
|
-#ifndef CONFIG_PREEMPT_RT_FULL
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
+void rcu_bh_qs(int cpu)
|
|
+{
|
|
+ rcu_preempt_qs(cpu);
|
|
+}
|
|
+#else
|
|
void rcu_bh_qs(int cpu)
|
|
{
|
|
struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
|
|
Index: linux-3.2/kernel/rcutree.h
|
|
===================================================================
|
|
--- linux-3.2.orig/kernel/rcutree.h
|
|
+++ linux-3.2/kernel/rcutree.h
|
|
@@ -430,6 +430,7 @@ DECLARE_PER_CPU(char, rcu_cpu_has_work);
|
|
/* Forward declarations for rcutree_plugin.h */
|
|
static void rcu_bootup_announce(void);
|
|
long rcu_batches_completed(void);
|
|
+static void rcu_preempt_qs(int cpu);
|
|
static void rcu_preempt_note_context_switch(int cpu);
|
|
static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
Index: linux-3.2/kernel/rcutree_plugin.h
|
|
===================================================================
|
|
--- linux-3.2.orig/kernel/rcutree_plugin.h
|
|
+++ linux-3.2/kernel/rcutree_plugin.h
|
|
@@ -1933,7 +1933,7 @@ EXPORT_SYMBOL_GPL(synchronize_sched_expe
|
|
|
|
#endif /* #else #ifndef CONFIG_SMP */
|
|
|
|
-#if !defined(CONFIG_RCU_FAST_NO_HZ)
|
|
+#if 1 /* !defined(CONFIG_RCU_FAST_NO_HZ) */
|
|
|
|
/*
|
|
* Check to see if any future RCU-related work will need to be done
|
|
Index: linux-3.2/kernel/softirq.c
|
|
===================================================================
|
|
--- linux-3.2.orig/kernel/softirq.c
|
|
+++ linux-3.2/kernel/softirq.c
|
|
@@ -139,7 +139,7 @@ static void wakeup_softirqd(void)
|
|
wake_up_process(tsk);
|
|
}
|
|
|
|
-static void handle_pending_softirqs(u32 pending, int cpu)
|
|
+static void handle_pending_softirqs(u32 pending, int cpu, int need_rcu_bh_qs)
|
|
{
|
|
struct softirq_action *h = softirq_vec;
|
|
unsigned int prev_count = preempt_count();
|
|
@@ -162,7 +162,8 @@ static void handle_pending_softirqs(u32
|
|
prev_count, (unsigned int) preempt_count());
|
|
preempt_count() = prev_count;
|
|
}
|
|
- rcu_bh_qs(cpu);
|
|
+ if (need_rcu_bh_qs)
|
|
+ rcu_bh_qs(cpu);
|
|
}
|
|
local_irq_disable();
|
|
}
|
|
@@ -314,7 +315,7 @@ restart:
|
|
/* Reset the pending bitmask before enabling irqs */
|
|
set_softirq_pending(0);
|
|
|
|
- handle_pending_softirqs(pending, cpu);
|
|
+ handle_pending_softirqs(pending, cpu, 1);
|
|
|
|
pending = local_softirq_pending();
|
|
if (pending && --max_restart)
|
|
@@ -384,7 +385,12 @@ static inline void ksoftirqd_clr_sched_p
|
|
static DEFINE_LOCAL_IRQ_LOCK(local_softirq_lock);
|
|
static DEFINE_PER_CPU(struct task_struct *, local_softirq_runner);
|
|
|
|
-static void __do_softirq(void);
|
|
+static void __do_softirq_common(int need_rcu_bh_qs);
|
|
+
|
|
+void __do_softirq(void)
|
|
+{
|
|
+ __do_softirq_common(0);
|
|
+}
|
|
|
|
void __init softirq_early_init(void)
|
|
{
|
|
@@ -455,7 +461,7 @@ EXPORT_SYMBOL(in_serving_softirq);
|
|
* Called with bh and local interrupts disabled. For full RT cpu must
|
|
* be pinned.
|
|
*/
|
|
-static void __do_softirq(void)
|
|
+static void __do_softirq_common(int need_rcu_bh_qs)
|
|
{
|
|
u32 pending = local_softirq_pending();
|
|
int cpu = smp_processor_id();
|
|
@@ -469,7 +475,7 @@ static void __do_softirq(void)
|
|
|
|
lockdep_softirq_enter();
|
|
|
|
- handle_pending_softirqs(pending, cpu);
|
|
+ handle_pending_softirqs(pending, cpu, need_rcu_bh_qs);
|
|
|
|
pending = local_softirq_pending();
|
|
if (pending)
|
|
@@ -508,7 +514,7 @@ static int __thread_do_softirq(int cpu)
|
|
* schedule!
|
|
*/
|
|
if (local_softirq_pending())
|
|
- __do_softirq();
|
|
+ __do_softirq_common(cpu >= 0);
|
|
local_unlock(local_softirq_lock);
|
|
unpin_current_cpu();
|
|
preempt_disable();
|