diff --git a/debian/changelog b/debian/changelog
index c283a4a4a..9962fa738 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -14,6 +14,12 @@ linux (3.16.3-3) UNRELEASED; urgency=low
   * [x86] drm/i915: read HEAD register back in init_ring_common() to enforce
     ordering (Closes: #763583)
 
+   [ Aurelien Jarno ]
+   * [arm64] Change RTC_DRV_PL031 and RTC_DRV_XGENE from modules to built-ins
+     as the kernel isn't able to initialize the system clock from a hardware
+     clock whose driver is a module, and thre is no initramfs mechanism to do
+     that.
+
  -- Ian Campbell <ijc@debian.org>  Sun, 21 Sep 2014 12:09:24 +0100
 
 linux (3.16.3-2) unstable; urgency=medium
diff --git a/debian/config/arm64/config b/debian/config/arm64/config
index a64c5776e..669c8bc87 100644
--- a/debian/config/arm64/config
+++ b/debian/config/arm64/config
@@ -88,8 +88,8 @@ CONFIG_POWER_RESET_XGENE=y
 ##
 ## file: drivers/rtc/Kconfig
 ##
-CONFIG_RTC_DRV_PL031=m
-CONFIG_RTC_DRV_XGENE=m
+CONFIG_RTC_DRV_PL031=y
+CONFIG_RTC_DRV_XGENE=y
 
 ##
 ## file: drivers/tty/serial/Kconfig