Subject: mm: pagefault_disabled() From: Peter Zijlstra Date: Thu Aug 11 15:31:31 CEST 2011 Wrap the test for pagefault_disabled() into a helper, this allows us to remove the need for current->pagefault_disabled on !-rt kernels. Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/n/tip-3yy517m8zsi9fpsf14xfaqkw@git.kernel.org --- arch/alpha/mm/fault.c | 2 +- arch/arm/mm/fault.c | 2 +- arch/avr32/mm/fault.c | 3 +-- arch/cris/mm/fault.c | 2 +- arch/frv/mm/fault.c | 2 +- arch/ia64/mm/fault.c | 2 +- arch/m32r/mm/fault.c | 2 +- arch/m68k/mm/fault.c | 2 +- arch/microblaze/mm/fault.c | 2 +- arch/mips/mm/fault.c | 2 +- arch/mn10300/mm/fault.c | 2 +- arch/parisc/mm/fault.c | 2 +- arch/powerpc/mm/fault.c | 2 +- arch/s390/mm/fault.c | 8 ++++---- arch/score/mm/fault.c | 2 +- arch/sh/mm/fault_32.c | 2 +- arch/sparc/mm/fault_32.c | 2 +- arch/sparc/mm/fault_64.c | 2 +- arch/tile/mm/fault.c | 2 +- arch/um/kernel/trap.c | 2 +- arch/x86/mm/fault.c | 2 +- arch/xtensa/mm/fault.c | 2 +- include/linux/sched.h | 14 ++++++++++++++ kernel/fork.c | 2 ++ 24 files changed, 41 insertions(+), 26 deletions(-) Index: linux-3.2/arch/alpha/mm/fault.c =================================================================== --- linux-3.2.orig/arch/alpha/mm/fault.c +++ linux-3.2/arch/alpha/mm/fault.c @@ -107,7 +107,7 @@ do_page_fault(unsigned long address, uns /* If we're in an interrupt context, or have no user context, we must not take the fault. */ - if (!mm || in_atomic() || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_ALPHA_LARGE_VMALLOC Index: linux-3.2/arch/arm/mm/fault.c =================================================================== --- linux-3.2.orig/arch/arm/mm/fault.c +++ linux-3.2/arch/arm/mm/fault.c @@ -294,7 +294,7 @@ do_page_fault(unsigned long addr, unsign * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; /* Index: linux-3.2/arch/avr32/mm/fault.c =================================================================== --- linux-3.2.orig/arch/avr32/mm/fault.c +++ linux-3.2/arch/avr32/mm/fault.c @@ -81,8 +81,7 @@ asmlinkage void do_page_fault(unsigned l * If we're in an interrupt or have no user context, we must * not take the fault... */ - if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM) || - current->pagefault_disabled) + if (!mm || regs->sr & SYSREG_BIT(GM) || pagefault_disabled()) goto no_context; local_irq_enable(); Index: linux-3.2/arch/cris/mm/fault.c =================================================================== --- linux-3.2.orig/arch/cris/mm/fault.c +++ linux-3.2/arch/cris/mm/fault.c @@ -111,7 +111,7 @@ do_page_fault(unsigned long address, str * user context, we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-3.2/arch/frv/mm/fault.c =================================================================== --- linux-3.2.orig/arch/frv/mm/fault.c +++ linux-3.2/arch/frv/mm/fault.c @@ -79,7 +79,7 @@ asmlinkage void do_page_fault(int datamm * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-3.2/arch/ia64/mm/fault.c =================================================================== --- linux-3.2.orig/arch/ia64/mm/fault.c +++ linux-3.2/arch/ia64/mm/fault.c @@ -89,7 +89,7 @@ ia64_do_page_fault (unsigned long addres /* * If we're in an interrupt or have no user context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_VIRTUAL_MEM_MAP Index: linux-3.2/arch/m32r/mm/fault.c =================================================================== --- linux-3.2.orig/arch/m32r/mm/fault.c +++ linux-3.2/arch/m32r/mm/fault.c @@ -115,7 +115,7 @@ asmlinkage void do_page_fault(struct pt_ * If we're in an interrupt or have no user context or are running in an * atomic region then we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; /* When running in the kernel we expect faults to occur only to Index: linux-3.2/arch/m68k/mm/fault.c =================================================================== --- linux-3.2.orig/arch/m68k/mm/fault.c +++ linux-3.2/arch/m68k/mm/fault.c @@ -85,7 +85,7 @@ int do_page_fault(struct pt_regs *regs, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-3.2/arch/microblaze/mm/fault.c =================================================================== --- linux-3.2.orig/arch/microblaze/mm/fault.c +++ linux-3.2/arch/microblaze/mm/fault.c @@ -107,7 +107,7 @@ void do_page_fault(struct pt_regs *regs, if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) is_write = 0; - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { if (kernel_mode(regs)) goto bad_area_nosemaphore; Index: linux-3.2/arch/mips/mm/fault.c =================================================================== --- linux-3.2.orig/arch/mips/mm/fault.c +++ linux-3.2/arch/mips/mm/fault.c @@ -88,7 +88,7 @@ asmlinkage void __kprobes do_page_fault( * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; down_read(&mm->mmap_sem); Index: linux-3.2/arch/mn10300/mm/fault.c =================================================================== --- linux-3.2.orig/arch/mn10300/mm/fault.c +++ linux-3.2/arch/mn10300/mm/fault.c @@ -168,7 +168,7 @@ asmlinkage void do_page_fault(struct pt_ * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-3.2/arch/parisc/mm/fault.c =================================================================== --- linux-3.2.orig/arch/parisc/mm/fault.c +++ linux-3.2/arch/parisc/mm/fault.c @@ -176,7 +176,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long acc_type; int fault; - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-3.2/arch/powerpc/mm/fault.c =================================================================== --- linux-3.2.orig/arch/powerpc/mm/fault.c +++ linux-3.2/arch/powerpc/mm/fault.c @@ -162,7 +162,7 @@ int __kprobes do_page_fault(struct pt_re } #endif - if (in_atomic() || mm == NULL || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { if (!user_mode(regs)) return SIGSEGV; /* in_atomic() in user mode is really bad, Index: linux-3.2/arch/s390/mm/fault.c =================================================================== --- linux-3.2.orig/arch/s390/mm/fault.c +++ linux-3.2/arch/s390/mm/fault.c @@ -295,8 +295,8 @@ static inline int do_exception(struct pt * user context. */ fault = VM_FAULT_BADCONTEXT; - if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || - tsk->pagefault_disabled)) + if (unlikely(!user_space_fault(trans_exc_code) || + !mm || pagefault_disabled())) goto out; address = trans_exc_code & __FAIL_ADDR_MASK; @@ -427,8 +427,8 @@ void __kprobes do_asce_exception(struct struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || - current->pagefault_disabled)) + if (unlikely(!user_space_fault(trans_exc_code) || + !mm || pagefault_disabled())) goto no_context; down_read(&mm->mmap_sem); Index: linux-3.2/arch/score/mm/fault.c =================================================================== --- linux-3.2.orig/arch/score/mm/fault.c +++ linux-3.2/arch/score/mm/fault.c @@ -72,7 +72,7 @@ asmlinkage void do_page_fault(struct pt_ * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; down_read(&mm->mmap_sem); Index: linux-3.2/arch/sh/mm/fault_32.c =================================================================== --- linux-3.2.orig/arch/sh/mm/fault_32.c +++ linux-3.2/arch/sh/mm/fault_32.c @@ -166,7 +166,7 @@ asmlinkage void __kprobes do_page_fault( * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-3.2/arch/sparc/mm/fault_32.c =================================================================== --- linux-3.2.orig/arch/sparc/mm/fault_32.c +++ linux-3.2/arch/sparc/mm/fault_32.c @@ -247,7 +247,7 @@ asmlinkage void do_sparc_fault(struct pt * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); Index: linux-3.2/arch/sparc/mm/fault_64.c =================================================================== --- linux-3.2.orig/arch/sparc/mm/fault_64.c +++ linux-3.2/arch/sparc/mm/fault_64.c @@ -322,7 +322,7 @@ asmlinkage void __kprobes do_sparc64_fau * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_enabled) + if (!mm || pagefault_disabled()) goto intr_or_no_mm; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); Index: linux-3.2/arch/tile/mm/fault.c =================================================================== --- linux-3.2.orig/arch/tile/mm/fault.c +++ linux-3.2/arch/tile/mm/fault.c @@ -346,7 +346,7 @@ static int handle_page_fault(struct pt_r * If we're in an interrupt, have no user context or are running in an * atomic region then we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } Index: linux-3.2/arch/um/kernel/trap.c =================================================================== --- linux-3.2.orig/arch/um/kernel/trap.c +++ linux-3.2/arch/um/kernel/trap.c @@ -37,7 +37,7 @@ int handle_page_fault(unsigned long addr * If the fault was during atomic operation, don't take the fault, just * fail. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto out_nosemaphore; down_read(&mm->mmap_sem); Index: linux-3.2/arch/x86/mm/fault.c =================================================================== --- linux-3.2.orig/arch/x86/mm/fault.c +++ linux-3.2/arch/x86/mm/fault.c @@ -1084,7 +1084,7 @@ do_page_fault(struct pt_regs *regs, unsi * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { bad_area_nosemaphore(regs, error_code, address); return; } Index: linux-3.2/arch/xtensa/mm/fault.c =================================================================== --- linux-3.2.orig/arch/xtensa/mm/fault.c +++ linux-3.2/arch/xtensa/mm/fault.c @@ -57,7 +57,7 @@ void do_page_fault(struct pt_regs *regs) /* If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { bad_page_fault(regs, address, SIGSEGV); return; } Index: linux-3.2/include/linux/sched.h =================================================================== --- linux-3.2.orig/include/linux/sched.h +++ linux-3.2/include/linux/sched.h @@ -91,6 +91,7 @@ struct sched_param { #include #include #include +#include #include @@ -1435,7 +1436,9 @@ struct task_struct { /* mutex deadlock detection */ struct mutex_waiter *blocked_on; #endif +#ifdef CONFIG_PREEMPT_RT_FULL int pagefault_disabled; +#endif #ifdef CONFIG_TRACE_IRQFLAGS unsigned int irq_events; unsigned long hardirq_enable_ip; @@ -1584,6 +1587,17 @@ struct task_struct { /* Future-safe accessor for struct task_struct's cpus_allowed. */ #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) +#ifdef CONFIG_PREEMPT_RT_FULL +static inline bool cur_pf_disabled(void) { return current->pagefault_disabled; } +#else +static inline bool cur_pf_disabled(void) { return false; } +#endif + +static inline bool pagefault_disabled(void) +{ + return in_atomic() || cur_pf_disabled(); +} + /* * Priority of a process goes from 0..MAX_PRIO-1, valid RT * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH Index: linux-3.2/kernel/fork.c =================================================================== --- linux-3.2.orig/kernel/fork.c +++ linux-3.2/kernel/fork.c @@ -1193,7 +1193,9 @@ static struct task_struct *copy_process( p->hardirq_context = 0; p->softirq_context = 0; #endif +#ifdef CONFIG_PREEMPT_RT_FULL p->pagefault_disabled = 0; +#endif #ifdef CONFIG_LOCKDEP p->lockdep_depth = 0; /* no locks held yet */ p->curr_chain_key = 0;