diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 86cf3a8ae8..f44543de2c 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -75,6 +75,18 @@ def get_ui(config):
         sys.exit("FATAL: Invalid user interface '%s' specified.\n"
                  "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
 
+def gather_extra_cache_data():
+    extra = []
+    interfaces = ['depexp', 'goggle', 'ncurses', 'hob', 'knotty']
+    for i in interfaces:
+        try:
+            ui = __import__("bb.ui." + i, fromlist = [i])
+            if hasattr(ui, "extraCaches"):
+                extra = extra + ui.extraCaches
+            del ui
+        except:
+            pass
+    return extra
 
 # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
 warnlog = logging.getLogger("BitBake.Warnings")
@@ -237,6 +249,12 @@ def main():
 
     try:
         configuration.setServerRegIdleCallback(server.getServerIdleCB())
+
+        if configParams.server_only:
+            configuration.extra_caches = gather_extra_cache_data()
+        else:
+            configuration.extra_caches = getattr(ui_module, "extraCaches", [])
+
         cooker = bb.cooker.BBCooker(configuration)
 
         server.addcooker(cooker)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2ae3e9d239..2edfa9faeb 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -94,28 +94,8 @@ class BBCooker:
         self.configuration = configuration
 
         self.caches_array = []
-        # Currently, only Image Creator hob ui needs extra cache.
-        # So, we save Extra Cache class name and container file
-        # information into a extraCaches field in hob UI.  
-        # TODO: In future, bin/bitbake should pass information into cooker,
-        # instead of getting information from configuration.ui. Also, some
-        # UI start up issues need to be addressed at the same time.
-        caches_name_array = ['bb.cache:CoreRecipeInfo']
-        if configuration.ui:
-            try:
-                module = __import__('bb.ui', fromlist=[configuration.ui])
-                name_array = (getattr(module, configuration.ui)).extraCaches
-                for recipeInfoName in name_array:
-                    caches_name_array.append(recipeInfoName)
-            except ImportError as exc:
-                # bb.ui.XXX is not defined and imported. It's an error!
-                logger.critical("Unable to import '%s' interface from bb.ui: %s" % (configuration.ui, exc))
-                sys.exit("FATAL: Failed to import '%s' interface." % configuration.ui)
-            except AttributeError:
-                # This is not an error. If the field is not defined in the ui,
-                # this interface might need no extra cache fields, so
-                # just skip this error!
-                logger.debug(2, "UI '%s' does not require extra cache!" % (configuration.ui))
+
+        caches_name_array = ['bb.cache:CoreRecipeInfo'] + configuration.extra_caches
 
         # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
         # This is the entry point, no further check needed!
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 11063b4af2..cbaa0bed47 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -119,7 +119,6 @@ class CookerConfiguration(object):
         self.cmd = None
         self.abort = True
         self.force = False
-        self.ui = None
         self.profile = False
         self.nosetscene = False
         self.invalidate_stamp = False