390 lines
14 KiB
Diff
390 lines
14 KiB
Diff
From 3d3bf4ea98185b3fec5fe75e16e576a957375a80 Mon Sep 17 00:00:00 2001
|
|
From: Peter Zijlstra <a.p.zijlstra@chello.nl>
|
|
Date: Thu, 11 Aug 2011 15:31:31 +0200
|
|
Subject: [034/254] mm: pagefault_disabled()
|
|
|
|
Wrap the test for pagefault_disabled() into a helper, this allows us
|
|
to remove the need for current->pagefault_disabled on !-rt kernels.
|
|
|
|
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
|
|
Link: http://lkml.kernel.org/n/tip-3yy517m8zsi9fpsf14xfaqkw@git.kernel.org
|
|
---
|
|
arch/alpha/mm/fault.c | 2 +-
|
|
arch/arm/mm/fault.c | 2 +-
|
|
arch/avr32/mm/fault.c | 3 +--
|
|
arch/cris/mm/fault.c | 2 +-
|
|
arch/frv/mm/fault.c | 2 +-
|
|
arch/ia64/mm/fault.c | 2 +-
|
|
arch/m32r/mm/fault.c | 2 +-
|
|
arch/m68k/mm/fault.c | 2 +-
|
|
arch/microblaze/mm/fault.c | 2 +-
|
|
arch/mips/mm/fault.c | 2 +-
|
|
arch/mn10300/mm/fault.c | 2 +-
|
|
arch/parisc/mm/fault.c | 2 +-
|
|
arch/powerpc/mm/fault.c | 2 +-
|
|
arch/s390/mm/fault.c | 6 +++---
|
|
arch/score/mm/fault.c | 2 +-
|
|
arch/sh/mm/fault_32.c | 2 +-
|
|
arch/sparc/mm/fault_32.c | 2 +-
|
|
arch/sparc/mm/fault_64.c | 2 +-
|
|
arch/tile/mm/fault.c | 2 +-
|
|
arch/um/kernel/trap.c | 2 +-
|
|
arch/x86/mm/fault.c | 2 +-
|
|
arch/xtensa/mm/fault.c | 2 +-
|
|
include/linux/sched.h | 14 ++++++++++++++
|
|
kernel/fork.c | 2 ++
|
|
24 files changed, 40 insertions(+), 25 deletions(-)
|
|
|
|
diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c
|
|
index 0ddf6c0..ab6b9d13 100644
|
|
--- a/arch/alpha/mm/fault.c
|
|
+++ b/arch/alpha/mm/fault.c
|
|
@@ -106,7 +106,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
|
|
|
|
/* If we're in an interrupt context, or have no user context,
|
|
we must not take the fault. */
|
|
- if (!mm || in_atomic() || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
#ifdef CONFIG_ALPHA_LARGE_VMALLOC
|
|
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
|
|
index 40bd40f..17a9f4a 100644
|
|
--- a/arch/arm/mm/fault.c
|
|
+++ b/arch/arm/mm/fault.c
|
|
@@ -279,7 +279,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
/*
|
|
diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c
|
|
index 623a027..155ad8d 100644
|
|
--- a/arch/avr32/mm/fault.c
|
|
+++ b/arch/avr32/mm/fault.c
|
|
@@ -81,8 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs)
|
|
* If we're in an interrupt or have no user context, we must
|
|
* not take the fault...
|
|
*/
|
|
- if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM) ||
|
|
- current->pagefault_disabled)
|
|
+ if (!mm || regs->sr & SYSREG_BIT(GM) || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
local_irq_enable();
|
|
diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c
|
|
index 1c1630e..886be8e 100644
|
|
--- a/arch/cris/mm/fault.c
|
|
+++ b/arch/cris/mm/fault.c
|
|
@@ -112,7 +112,7 @@ do_page_fault(unsigned long address, struct pt_regs *regs,
|
|
* user context, we must not take the fault.
|
|
*/
|
|
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c
|
|
index 6372088..e87972c 100644
|
|
--- a/arch/frv/mm/fault.c
|
|
+++ b/arch/frv/mm/fault.c
|
|
@@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c
|
|
index 721dbb9..8ca850e 100644
|
|
--- a/arch/ia64/mm/fault.c
|
|
+++ b/arch/ia64/mm/fault.c
|
|
@@ -88,7 +88,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
|
|
/*
|
|
* If we're in an interrupt or have no user context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
#ifdef CONFIG_VIRTUAL_MEM_MAP
|
|
diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c
|
|
index 1eec8af..6945056 100644
|
|
--- a/arch/m32r/mm/fault.c
|
|
+++ b/arch/m32r/mm/fault.c
|
|
@@ -114,7 +114,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code,
|
|
* If we're in an interrupt or have no user context or are running in an
|
|
* atomic region then we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto bad_area_nosemaphore;
|
|
|
|
/* When running in the kernel we expect faults to occur only to
|
|
diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c
|
|
index 7a4cc68..46b8cce 100644
|
|
--- a/arch/m68k/mm/fault.c
|
|
+++ b/arch/m68k/mm/fault.c
|
|
@@ -84,7 +84,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c
|
|
index 621ad49..a438434 100644
|
|
--- a/arch/microblaze/mm/fault.c
|
|
+++ b/arch/microblaze/mm/fault.c
|
|
@@ -106,7 +106,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
|
|
if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11)
|
|
is_write = 0;
|
|
|
|
- if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) {
|
|
+ if (unlikely(!mm || pagefault_disabled())) {
|
|
if (kernel_mode(regs))
|
|
goto bad_area_nosemaphore;
|
|
|
|
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
|
|
index 3468e69..39a3180 100644
|
|
--- a/arch/mips/mm/fault.c
|
|
+++ b/arch/mips/mm/fault.c
|
|
@@ -89,7 +89,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, unsigned long writ
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto bad_area_nosemaphore;
|
|
|
|
retry:
|
|
diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c
|
|
index ff5acf5..5d9e10f 100644
|
|
--- a/arch/mn10300/mm/fault.c
|
|
+++ b/arch/mn10300/mm/fault.c
|
|
@@ -167,7 +167,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code,
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c
|
|
index 09ecc8a..df22f39 100644
|
|
--- a/arch/parisc/mm/fault.c
|
|
+++ b/arch/parisc/mm/fault.c
|
|
@@ -176,7 +176,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
|
|
unsigned long acc_type;
|
|
int fault;
|
|
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
|
|
index 6642dff..7bd8f27 100644
|
|
--- a/arch/powerpc/mm/fault.c
|
|
+++ b/arch/powerpc/mm/fault.c
|
|
@@ -261,7 +261,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
|
|
if (!arch_irq_disabled_regs(regs))
|
|
local_irq_enable();
|
|
|
|
- if (in_atomic() || mm == NULL || current->pagefault_disabled) {
|
|
+ if (!mm || pagefault_disabled()) {
|
|
if (!user_mode(regs))
|
|
return SIGSEGV;
|
|
/* in_atomic() in user mode is really bad,
|
|
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
|
|
index f7d796f..e2994cb 100644
|
|
--- a/arch/s390/mm/fault.c
|
|
+++ b/arch/s390/mm/fault.c
|
|
@@ -283,8 +283,8 @@ static inline int do_exception(struct pt_regs *regs, int access)
|
|
* user context.
|
|
*/
|
|
fault = VM_FAULT_BADCONTEXT;
|
|
- if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm ||
|
|
- tsk->pagefault_disabled))
|
|
+ if (unlikely(!user_space_fault(trans_exc_code) ||
|
|
+ !mm || pagefault_disabled()))
|
|
goto out;
|
|
|
|
address = trans_exc_code & __FAIL_ADDR_MASK;
|
|
@@ -417,7 +417,7 @@ void __kprobes do_asce_exception(struct pt_regs *regs)
|
|
|
|
trans_exc_code = regs->int_parm_long;
|
|
if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm ||
|
|
- current->pagefault_disabled))
|
|
+ pagefault_disabled()))
|
|
goto no_context;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c
|
|
index 4c12824..59fccbe 100644
|
|
--- a/arch/score/mm/fault.c
|
|
+++ b/arch/score/mm/fault.c
|
|
@@ -72,7 +72,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto bad_area_nosemaphore;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/sh/mm/fault_32.c b/arch/sh/mm/fault_32.c
|
|
index 8c3ae42..1aca948 100644
|
|
--- a/arch/sh/mm/fault_32.c
|
|
+++ b/arch/sh/mm/fault_32.c
|
|
@@ -166,7 +166,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
|
|
* If we're in an interrupt, have no user context or are running
|
|
* in an atomic region then we must not take the fault:
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
|
|
index ffaa99e..77b37e0 100644
|
|
--- a/arch/sparc/mm/fault_32.c
|
|
+++ b/arch/sparc/mm/fault_32.c
|
|
@@ -248,7 +248,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto no_context;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
|
|
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
|
|
index fbd5e11..ea4e14b 100644
|
|
--- a/arch/sparc/mm/fault_64.c
|
|
+++ b/arch/sparc/mm/fault_64.c
|
|
@@ -323,7 +323,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_enabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto intr_or_no_mm;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
|
|
diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c
|
|
index f683aad..3e85178 100644
|
|
--- a/arch/tile/mm/fault.c
|
|
+++ b/arch/tile/mm/fault.c
|
|
@@ -355,7 +355,7 @@ static int handle_page_fault(struct pt_regs *regs,
|
|
* If we're in an interrupt, have no user context or are running in an
|
|
* atomic region then we must not take the fault.
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled) {
|
|
+ if (!mm || pagefault_disabled()) {
|
|
vma = NULL; /* happy compiler */
|
|
goto bad_area_nosemaphore;
|
|
}
|
|
diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
|
|
index a283400..7878069 100644
|
|
--- a/arch/um/kernel/trap.c
|
|
+++ b/arch/um/kernel/trap.c
|
|
@@ -37,7 +37,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
|
|
* If the fault was during atomic operation, don't take the fault, just
|
|
* fail.
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled)
|
|
+ if (!mm || pagefault_disabled())
|
|
goto out_nosemaphore;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
|
|
index bb891f2..9d57357 100644
|
|
--- a/arch/x86/mm/fault.c
|
|
+++ b/arch/x86/mm/fault.c
|
|
@@ -1094,7 +1094,7 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
|
|
* If we're in an interrupt, have no user context or are running
|
|
* in an atomic region then we must not take the fault:
|
|
*/
|
|
- if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) {
|
|
+ if (unlikely(!mm || pagefault_disabled())) {
|
|
bad_area_nosemaphore(regs, error_code, address);
|
|
return;
|
|
}
|
|
diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c
|
|
index cc58b7c..93d33ee 100644
|
|
--- a/arch/xtensa/mm/fault.c
|
|
+++ b/arch/xtensa/mm/fault.c
|
|
@@ -56,7 +56,7 @@ void do_page_fault(struct pt_regs *regs)
|
|
/* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
- if (in_atomic() || !mm || current->pagefault_disabled) {
|
|
+ if (!mm || pagefault_disabled()) {
|
|
bad_page_fault(regs, address, SIGSEGV);
|
|
return;
|
|
}
|
|
diff --git a/include/linux/sched.h b/include/linux/sched.h
|
|
index f0ee37f..f14b965 100644
|
|
--- a/include/linux/sched.h
|
|
+++ b/include/linux/sched.h
|
|
@@ -90,6 +90,7 @@ struct sched_param {
|
|
#include <linux/latencytop.h>
|
|
#include <linux/cred.h>
|
|
#include <linux/llist.h>
|
|
+#include <linux/hardirq.h>
|
|
|
|
#include <asm/processor.h>
|
|
|
|
@@ -1478,7 +1479,9 @@ struct task_struct {
|
|
/* mutex deadlock detection */
|
|
struct mutex_waiter *blocked_on;
|
|
#endif
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
int pagefault_disabled;
|
|
+#endif
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
unsigned int irq_events;
|
|
unsigned long hardirq_enable_ip;
|
|
@@ -1628,6 +1631,17 @@ struct task_struct {
|
|
/* Future-safe accessor for struct task_struct's cpus_allowed. */
|
|
#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
|
|
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
+static inline bool cur_pf_disabled(void) { return current->pagefault_disabled; }
|
|
+#else
|
|
+static inline bool cur_pf_disabled(void) { return false; }
|
|
+#endif
|
|
+
|
|
+static inline bool pagefault_disabled(void)
|
|
+{
|
|
+ return in_atomic() || cur_pf_disabled();
|
|
+}
|
|
+
|
|
/*
|
|
* Priority of a process goes from 0..MAX_PRIO-1, valid RT
|
|
* priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH
|
|
diff --git a/kernel/fork.c b/kernel/fork.c
|
|
index fdf0795..4c9def3 100644
|
|
--- a/kernel/fork.c
|
|
+++ b/kernel/fork.c
|
|
@@ -1265,7 +1265,9 @@ static struct task_struct *copy_process(unsigned long clone_flags,
|
|
p->hardirq_context = 0;
|
|
p->softirq_context = 0;
|
|
#endif
|
|
+#ifdef CONFIG_PREEMPT_RT_FULL
|
|
p->pagefault_disabled = 0;
|
|
+#endif
|
|
#ifdef CONFIG_LOCKDEP
|
|
p->lockdep_depth = 0; /* no locks held yet */
|
|
p->curr_chain_key = 0;
|