debian-packaging
/
linux
8
0
Fork 0
Code Releases Activity
linux/debian/patches-rt/rcu-merge-rcu-bh-into-rcu-p...

335 lines
9.4 KiB
Diff
Raw Blame History

Subject: rcu: Merge RCU-bh into RCU-preempt
Date: Wed, 5 Oct 2011 11:59:38 -0700
From: Thomas Gleixner <tglx@linutronix.de>
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.1-rt3.tar.xz
The Linux kernel has long RCU-bh read-side critical sections that
intolerably increase scheduling latency under mainline's RCU-bh rules,
which include RCU-bh read-side critical sections being non-preemptible.
This patch therefore arranges for RCU-bh to be implemented in terms of
RCU-preempt for CONFIG_PREEMPT_RT_FULL=y.
This has the downside of defeating the purpose of RCU-bh, namely,
handling the case where the system is subjected to a network-based
denial-of-service attack that keeps at least one CPU doing full-time
softirq processing. This issue will be fixed by a later commit.
The current commit will need some work to make it appropriate for
mainline use, for example, it needs to be extended to cover Tiny RCU.
[ paulmck: Added a useful changelog ]
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Link: http://lkml.kernel.org/r/20111005185938.GA20403@linux.vnet.ibm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
include/linux/rcupdate.h | 19 +++++++++++++++++++
include/linux/rcutree.h | 8 ++++++++
kernel/rcu/rcu.h | 11 +++++++++--
kernel/rcu/rcutorture.c | 7 +++++++
kernel/rcu/tree.c | 26 ++++++++++++++++++++++++++
kernel/rcu/tree.h | 2 ++
kernel/rcu/update.c | 2 ++
7 files changed, 73 insertions(+), 2 deletions(-)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -56,7 +56,11 @@ void call_rcu(struct rcu_head *head, rcu
#define call_rcu call_rcu_sched
#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
+#ifdef CONFIG_PREEMPT_RT_FULL
+#define call_rcu_bh call_rcu
+#else
void call_rcu_bh(struct rcu_head *head, rcu_callback_t func);
+#endif
void call_rcu_sched(struct rcu_head *head, rcu_callback_t func);
void synchronize_sched(void);
void rcu_barrier_tasks(void);
@@ -263,7 +267,14 @@ extern struct lockdep_map rcu_sched_lock
extern struct lockdep_map rcu_callback_map;
int debug_lockdep_rcu_enabled(void);
int rcu_read_lock_held(void);
+#ifdef CONFIG_PREEMPT_RT_FULL
+static inline int rcu_read_lock_bh_held(void)
+{
+ return rcu_read_lock_held();
+}
+#else
int rcu_read_lock_bh_held(void);
+#endif
int rcu_read_lock_sched_held(void);
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -663,10 +674,14 @@ static inline void rcu_read_unlock(void)
static inline void rcu_read_lock_bh(void)
{
local_bh_disable();
+#ifdef CONFIG_PREEMPT_RT_FULL
+ rcu_read_lock();
+#else
__acquire(RCU_BH);
rcu_lock_acquire(&rcu_bh_lock_map);
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_lock_bh() used illegally while idle");
+#endif
}
/*
@@ -676,10 +691,14 @@ static inline void rcu_read_lock_bh(void
*/
static inline void rcu_read_unlock_bh(void)
{
+#ifdef CONFIG_PREEMPT_RT_FULL
+ rcu_read_unlock();
+#else
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_unlock_bh() used illegally while idle");
rcu_lock_release(&rcu_bh_lock_map);
__release(RCU_BH);
+#endif
local_bh_enable();
}
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -44,7 +44,11 @@ static inline void rcu_virt_note_context
rcu_note_context_switch(false);
}
+#ifdef CONFIG_PREEMPT_RT_FULL
+# define synchronize_rcu_bh synchronize_rcu
+#else
void synchronize_rcu_bh(void);
+#endif
void synchronize_sched_expedited(void);
void synchronize_rcu_expedited(void);
@@ -72,7 +76,11 @@ static inline void synchronize_rcu_bh_ex
}
void rcu_barrier(void);
+#ifdef CONFIG_PREEMPT_RT_FULL
+# define rcu_barrier_bh rcu_barrier
+#else
void rcu_barrier_bh(void);
+#endif
void rcu_barrier_sched(void);
bool rcu_eqs_special_set(int cpu);
unsigned long get_state_synchronize_rcu(void);
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -528,7 +528,6 @@ static inline void show_rcu_gp_kthreads(
static inline int rcu_get_gp_kthreads_prio(void) { return 0; }
#else /* #ifdef CONFIG_TINY_RCU */
unsigned long rcu_get_gp_seq(void);
-unsigned long rcu_bh_get_gp_seq(void);
unsigned long rcu_sched_get_gp_seq(void);
unsigned long rcu_exp_batches_completed(void);
unsigned long rcu_exp_batches_completed_sched(void);
@@ -536,10 +535,18 @@ unsigned long srcu_batches_completed(str
void show_rcu_gp_kthreads(void);
int rcu_get_gp_kthreads_prio(void);
void rcu_force_quiescent_state(void);
-void rcu_bh_force_quiescent_state(void);
void rcu_sched_force_quiescent_state(void);
extern struct workqueue_struct *rcu_gp_wq;
extern struct workqueue_struct *rcu_par_gp_wq;
+
+#ifdef CONFIG_PREEMPT_RT_FULL
+#define rcu_bh_get_gp_seq rcu_get_gp_seq
+#define rcu_bh_force_quiescent_state rcu_force_quiescent_state
+#else
+unsigned long rcu_bh_get_gp_seq(void);
+void rcu_bh_force_quiescent_state(void);
+#endif
+
#endif /* #else #ifdef CONFIG_TINY_RCU */
#ifdef CONFIG_RCU_NOCB_CPU
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -434,6 +434,7 @@ static struct rcu_torture_ops rcu_ops =
.name = "rcu"
};
+#ifndef CONFIG_PREEMPT_RT_FULL
/*
* Definitions for rcu_bh torture testing.
*/
@@ -475,6 +476,12 @@ static struct rcu_torture_ops rcu_bh_ops
.name = "rcu_bh"
};
+#else
+static struct rcu_torture_ops rcu_bh_ops = {
+ .ttype = INVALID_RCU_FLAVOR,
+};
+#endif
+
/*
* Don't even think about trying any of these in real life!!!
* The names includes "busted", and they really means it!
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -244,6 +244,7 @@ void rcu_sched_qs(void)
this_cpu_ptr(&rcu_sched_data), true);
}
+#ifndef CONFIG_PREEMPT_RT_FULL
void rcu_bh_qs(void)
{
RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
@@ -254,6 +255,11 @@ void rcu_bh_qs(void)
__this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
}
}
+#else
+void rcu_bh_qs(void)
+{
+}
+#endif
/*
* Steal a bit from the bottom of ->dynticks for idle entry/exit
@@ -568,6 +574,7 @@ unsigned long rcu_sched_get_gp_seq(void)
}
EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
+#ifndef CONFIG_PREEMPT_RT_FULL
/*
* Return the number of RCU-bh GPs completed thus far for debug & stats.
*/
@@ -576,6 +583,7 @@ unsigned long rcu_bh_get_gp_seq(void)
return READ_ONCE(rcu_bh_state.gp_seq);
}
EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);
+#endif
/*
* Return the number of RCU expedited batches completed thus far for
@@ -599,6 +607,7 @@ unsigned long rcu_exp_batches_completed_
}
EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched);
+#ifndef CONFIG_PREEMPT_RT_FULL
/*
* Force a quiescent state.
*/
@@ -617,6 +626,13 @@ void rcu_bh_force_quiescent_state(void)
}
EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
+#else
+void rcu_force_quiescent_state(void)
+{
+}
+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
+#endif
+
/*
* Force a quiescent state for RCU-sched.
*/
@@ -674,9 +690,11 @@ void rcutorture_get_gp_data(enum rcutort
case RCU_FLAVOR:
rsp = rcu_state_p;
break;
+#ifndef CONFIG_PREEMPT_RT_FULL
case RCU_BH_FLAVOR:
rsp = &rcu_bh_state;
break;
+#endif
case RCU_SCHED_FLAVOR:
rsp = &rcu_sched_state;
break;
@@ -3040,6 +3058,7 @@ void call_rcu_sched(struct rcu_head *hea
}
EXPORT_SYMBOL_GPL(call_rcu_sched);
+#ifndef CONFIG_PREEMPT_RT_FULL
/**
* call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
* @head: structure to be used for queueing the RCU updates.
@@ -3067,6 +3086,7 @@ void call_rcu_bh(struct rcu_head *head,
__call_rcu(head, func, &rcu_bh_state, -1, 0);
}
EXPORT_SYMBOL_GPL(call_rcu_bh);
+#endif
/*
* Queue an RCU callback for lazy invocation after a grace period.
@@ -3152,6 +3172,7 @@ void synchronize_sched(void)
}
EXPORT_SYMBOL_GPL(synchronize_sched);
+#ifndef CONFIG_PREEMPT_RT_FULL
/**
* synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
*
@@ -3178,6 +3199,7 @@ void synchronize_rcu_bh(void)
wait_rcu_gp(call_rcu_bh);
}
EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
+#endif
/**
* get_state_synchronize_rcu - Snapshot current RCU state
@@ -3485,6 +3507,7 @@ static void _rcu_barrier(struct rcu_stat
mutex_unlock(&rsp->barrier_mutex);
}
+#ifndef CONFIG_PREEMPT_RT_FULL
/**
* rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
*/
@@ -3493,6 +3516,7 @@ void rcu_barrier_bh(void)
_rcu_barrier(&rcu_bh_state);
}
EXPORT_SYMBOL_GPL(rcu_barrier_bh);
+#endif
/**
* rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
@@ -4140,7 +4164,9 @@ void __init rcu_init(void)
rcu_bootup_announce();
rcu_init_geometry();
+#ifndef CONFIG_PREEMPT_RT_FULL
rcu_init_one(&rcu_bh_state);
+#endif
rcu_init_one(&rcu_sched_state);
if (dump_tree)
rcu_dump_rcu_node_tree(&rcu_sched_state);
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -413,7 +413,9 @@ extern struct list_head rcu_struct_flavo
*/
extern struct rcu_state rcu_sched_state;
+#ifndef CONFIG_PREEMPT_RT_FULL
extern struct rcu_state rcu_bh_state;
+#endif
#ifdef CONFIG_PREEMPT_RCU
extern struct rcu_state rcu_preempt_state;
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -286,6 +286,7 @@ int rcu_read_lock_held(void)
}
EXPORT_SYMBOL_GPL(rcu_read_lock_held);
+#ifndef CONFIG_PREEMPT_RT_FULL
/**
* rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
*
@@ -312,6 +313,7 @@ int rcu_read_lock_bh_held(void)
return in_softirq() || irqs_disabled();
}
EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
+#endif
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
View Git Blame Copy Permalink