[x86] mmap: Add an exception to the stack gap for Hotspot JVM compatibility

Closes: #865303

debian/changelog

@@ -23,6 +23,9 @@ linux (4.14.2-1) UNRELEASED; urgency=medium
   * [rt] Add new signing subkey for Steven Rostedt
   * [rt] Update to 4.14.1-rt3
   * Set ABI to 1
+  * mmap: Remember the MAP_FIXED flag as VM_FIXED
+  * [x86] mmap: Add an exception to the stack gap for Hotspot JVM compatibility
+    (Closes: #865303)
 
   [ Salvatore Bonaccorso ]
   * mm, thp: Do not make page table dirty unconditionally in touch_p[mu]d()

debian/patches/bugfix/x86/mmap-add-an-exception-to-the-stack-gap-for-hotspot-jvm.patch

@@ -0,0 +1,45 @@
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Thu, 30 Nov 2017 00:29:18 +0000
+Subject: mmap: Add an exception to the stack gap for Hotspot JVM compatibility
+Bug-Debian: https://bugs.debian.org/865303
+
+The Hotspot JVM can easily exhaust the default stack, and has a
+SIGSEGV handler to cope with this by switching to a new stack segment.
+
+However, on i386 it creates a single writable and executable page just
+under the stack limit as a workaround for a bug in Exec Shield.  That
+together with the enlarged stack gap causes the SIGSEGV handler to be
+triggered when the stack pointer is further away from the stack limit,
+and it doesn't recognise this as being a stack overflow.
+
+This specifically affects programs that use JNI.  Hotspot doesn't
+normally run Java code on the initial thread.
+
+Reduce the effective stack guard gap on x86 if the previous vma is
+a single page allocated as MAP_FIXED.
+
+References: https://bugs.debian.org/865303
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+---
+ mm/mmap.c | 10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+--- a/mm/mmap.c
++++ b/mm/mmap.c
+@@ -2328,6 +2328,16 @@ int expand_downwards(struct vm_area_struct *vma,
+ 	/* Check that both stack segments have the same anon_vma? */
+ 	if (prev && !(prev->vm_flags & VM_GROWSDOWN) &&
+ 			(prev->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) {
++		/*
++		 * bwh: Reduce the stack guard gap if this looks like
++		 * Hotspot JVM craziness - see Debian bug #865303
++		 */
++		if (IS_ENABLED(CONFIG_X86) && (prev->vm_flags & VM_FIXED) &&
++		    prev->vm_end - prev->vm_start == PAGE_SIZE) {
++			if (address - prev->vm_end <
++			    min(stack_guard_gap, 4UL << PAGE_SHIFT))
++				return -ENOMEM;
++		} else
+ 		if (address - prev->vm_end < stack_guard_gap)
+ 			return -ENOMEM;
+ 	}

debian/patches/bugfix/x86/mmap-remember-the-map_fixed-flag-as-vm_fixed.patch

@@ -0,0 +1,36 @@
+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Wed, 5 Jul 2017 13:32:43 +0100
+Subject: mmap: Remember the MAP_FIXED flag as VM_FIXED
+
+Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
+---
+ include/linux/mm.h   | 1 +
+ include/linux/mman.h | 3 ++-
+ 2 files changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/include/linux/mm.h b/include/linux/mm.h
+index 43edf659453b..1f84cc52389e 100644
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -190,6 +190,7 @@ extern unsigned int kobjsize(const void *objp);
+ #define VM_ACCOUNT	0x00100000	/* Is a VM accounted object */
+ #define VM_NORESERVE	0x00200000	/* should the VM suppress accounting */
+ #define VM_HUGETLB	0x00400000	/* Huge TLB Page VM */
++#define VM_FIXED	0x00800000	/* Allocated at fixed address */
+ #define VM_ARCH_1	0x01000000	/* Architecture-specific flag */
+ #define VM_WIPEONFORK	0x02000000	/* Wipe VMA contents in child. */
+ #define VM_DONTDUMP	0x04000000	/* Do not include in the core dump */
+diff --git a/include/linux/mman.h b/include/linux/mman.h
+index 7c87b6652244..f22c15d5e24c 100644
+--- a/include/linux/mman.h
++++ b/include/linux/mman.h
+@@ -87,7 +87,8 @@ calc_vm_flag_bits(unsigned long flags)
+ {
+ 	return _calc_vm_trans(flags, MAP_GROWSDOWN,  VM_GROWSDOWN ) |
+ 	       _calc_vm_trans(flags, MAP_DENYWRITE,  VM_DENYWRITE ) |
+-	       _calc_vm_trans(flags, MAP_LOCKED,     VM_LOCKED    );
++	       _calc_vm_trans(flags, MAP_LOCKED,     VM_LOCKED    ) |
++	       _calc_vm_trans(flags, MAP_FIXED,      VM_FIXED     );
+ }
+ 
+ unsigned long vm_commit_limit(void);

debian/patches/series

@@ -63,6 +63,8 @@ debian/revert-gpu-host1x-add-iommu-support.patch
 bugfix/x86/perf-tools-fix-unwind-build-on-i386.patch
 bugfix/sh/sh-boot-do-not-use-hyphen-in-exported-variable-name.patch
 bugfix/arm/arm-dts-exynos-add-dwc3-susphy-quirk.patch
+bugfix/x86/mmap-remember-the-map_fixed-flag-as-vm_fixed.patch
+bugfix/x86/mmap-add-an-exception-to-the-stack-gap-for-hotspot-jvm.patch
 
 # Arch features
 features/mips/MIPS-increase-MAX-PHYSMEM-BITS-on-Loongson-3-only.patch