119 lines
3.6 KiB
Diff
119 lines
3.6 KiB
Diff
From c3c52cd587d395b046d02e9abbc0569b66f78ade Mon Sep 17 00:00:00 2001
|
|
Message-Id: <c3c52cd587d395b046d02e9abbc0569b66f78ade.1601675152.git.zanussi@kernel.org>
|
|
In-Reply-To: <5b5a156f9808b1acf1205606e03da117214549ea.1601675151.git.zanussi@kernel.org>
|
|
References: <5b5a156f9808b1acf1205606e03da117214549ea.1601675151.git.zanussi@kernel.org>
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
|
Date: Mon, 6 Jun 2011 12:20:33 +0200
|
|
Subject: [PATCH 107/333] sched: Move mmdrop to RCU on RT
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.148-rt64.tar.xz
|
|
|
|
Takes sleeping locks and calls into the memory allocator, so nothing
|
|
we want to do in task switch and oder atomic contexts.
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
---
|
|
include/linux/mm_types.h | 4 ++++
|
|
include/linux/sched/mm.h | 11 +++++++++++
|
|
kernel/fork.c | 13 +++++++++++++
|
|
kernel/sched/core.c | 7 ++++++-
|
|
4 files changed, 34 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
|
|
index 3a9a996af229..202b736ccbfa 100644
|
|
--- a/include/linux/mm_types.h
|
|
+++ b/include/linux/mm_types.h
|
|
@@ -12,6 +12,7 @@
|
|
#include <linux/completion.h>
|
|
#include <linux/cpumask.h>
|
|
#include <linux/uprobes.h>
|
|
+#include <linux/rcupdate.h>
|
|
#include <linux/page-flags-layout.h>
|
|
#include <linux/workqueue.h>
|
|
|
|
@@ -487,6 +488,9 @@ struct mm_struct {
|
|
bool tlb_flush_batched;
|
|
#endif
|
|
struct uprobes_state uprobes_state;
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+ struct rcu_head delayed_drop;
|
|
+#endif
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
atomic_long_t hugetlb_usage;
|
|
#endif
|
|
diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
|
|
index 766bbe813861..51828764f932 100644
|
|
--- a/include/linux/sched/mm.h
|
|
+++ b/include/linux/sched/mm.h
|
|
@@ -49,6 +49,17 @@ static inline void mmdrop(struct mm_struct *mm)
|
|
__mmdrop(mm);
|
|
}
|
|
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+extern void __mmdrop_delayed(struct rcu_head *rhp);
|
|
+static inline void mmdrop_delayed(struct mm_struct *mm)
|
|
+{
|
|
+ if (atomic_dec_and_test(&mm->mm_count))
|
|
+ call_rcu(&mm->delayed_drop, __mmdrop_delayed);
|
|
+}
|
|
+#else
|
|
+# define mmdrop_delayed(mm) mmdrop(mm)
|
|
+#endif
|
|
+
|
|
void mmdrop(struct mm_struct *mm);
|
|
|
|
/*
|
|
diff --git a/kernel/fork.c b/kernel/fork.c
|
|
index 309f4a20d4ac..d4ec53c72577 100644
|
|
--- a/kernel/fork.c
|
|
+++ b/kernel/fork.c
|
|
@@ -642,6 +642,19 @@ void __mmdrop(struct mm_struct *mm)
|
|
}
|
|
EXPORT_SYMBOL_GPL(__mmdrop);
|
|
|
|
+#ifdef CONFIG_PREEMPT_RT_BASE
|
|
+/*
|
|
+ * RCU callback for delayed mm drop. Not strictly rcu, but we don't
|
|
+ * want another facility to make this work.
|
|
+ */
|
|
+void __mmdrop_delayed(struct rcu_head *rhp)
|
|
+{
|
|
+ struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop);
|
|
+
|
|
+ __mmdrop(mm);
|
|
+}
|
|
+#endif
|
|
+
|
|
static void mmdrop_async_fn(struct work_struct *work)
|
|
{
|
|
struct mm_struct *mm;
|
|
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
|
|
index 87c04b6a78d2..ec979c93a488 100644
|
|
--- a/kernel/sched/core.c
|
|
+++ b/kernel/sched/core.c
|
|
@@ -2731,9 +2731,13 @@ static struct rq *finish_task_switch(struct task_struct *prev)
|
|
* provided by mmdrop(),
|
|
* - a sync_core for SYNC_CORE.
|
|
*/
|
|
+ /*
|
|
+ * We use mmdrop_delayed() here so we don't have to do the
|
|
+ * full __mmdrop() when we are the last user.
|
|
+ */
|
|
if (mm) {
|
|
membarrier_mm_sync_core_before_usermode(mm);
|
|
- mmdrop(mm);
|
|
+ mmdrop_delayed(mm);
|
|
}
|
|
if (unlikely(prev_state == TASK_DEAD)) {
|
|
if (prev->sched_class->task_dead)
|
|
@@ -5604,6 +5608,7 @@ void sched_setnuma(struct task_struct *p, int nid)
|
|
#endif /* CONFIG_NUMA_BALANCING */
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
+
|
|
/*
|
|
* Ensure that the idle task is using init_mm right before its CPU goes
|
|
* offline.
|
|
--
|
|
2.17.1
|
|
|