bitbake: Hob: log for Hob and allow users to show logs after successful build

If users build images in Hob, record logs and allow users to retrieve the lo
after successful build.

The logs are generated if and only if:
      - users do "just bake"
      - users do "build image" after "build packages"
      - users do "build packages" only

[YOCTO #1991]

(Bitbake rev: 291289787bb042b99f0599babc2d67c220aadb87)

Signed-off-by: Shane Wang <shane.wang@intel.com>
Signed-off-by: Liming An <limingx.l.an@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/ui/crumbs/builder.py

@@ -27,6 +27,7 @@ import os
 import subprocess
 import shlex
 import re
+import logging
 from bb.ui.crumbs.template import TemplateMgr
 from bb.ui.crumbs.imageconfigurationpage import ImageConfigurationPage
 from bb.ui.crumbs.recipeselectionpage import RecipeSelectionPage
@@ -395,6 +396,11 @@ class Builder(gtk.Window):
 
         self.template = None
 
+        # logger
+        self.logger = logging.getLogger("BitBake")
+        self.consolelog = None
+        self.current_logfile = None
+
         # configuration and parameters
         self.configuration = Configuration()
         self.parameters = Parameters()
@@ -433,6 +439,7 @@ class Builder(gtk.Window):
         self.handler.build.connect("build-aborted",      self.handler_build_aborted_cb)
         self.handler.build.connect("task-started",       self.handler_task_started_cb)
         self.handler.build.connect("log-error",          self.handler_build_failure_cb)
+        self.handler.build.connect("log",                self.handler_build_log_cb)
         self.handler.build.connect("no-provider",        self.handler_no_provider_cb)
         self.handler.connect("generating-data",          self.handler_generating_data_cb)
         self.handler.connect("data-generated",           self.handler_data_generated_cb)
@@ -501,25 +508,34 @@ class Builder(gtk.Window):
         self.set_user_config()
         self.handler.generate_recipes()
 
-    def generate_packages_async(self):
+    def generate_packages_async(self, log = False):
         self.switch_page(self.PACKAGE_GENERATING)
+        if log:
+            self.current_logfile = self.handler.get_logfile()
+            self.do_log(self.current_logfile)
         # Build packages
         _, all_recipes = self.recipe_model.get_selected_recipes()
         self.set_user_config()
         self.handler.reset_build()
         self.handler.generate_packages(all_recipes, self.configuration.default_task)
 
-    def fast_generate_image_async(self):
+    def fast_generate_image_async(self, log = False):
         self.switch_page(self.FAST_IMAGE_GENERATING)
+        if log:
+            self.current_logfile = self.handler.get_logfile()
+            self.do_log(self.current_logfile)
         # Build packages
         _, all_recipes = self.recipe_model.get_selected_recipes()
         self.set_user_config()
         self.handler.reset_build()
         self.handler.generate_packages(all_recipes, self.configuration.default_task)
 
-    def generate_image_async(self):
+    def generate_image_async(self, cont = False):
         self.switch_page(self.IMAGE_GENERATING)
         self.handler.reset_build()
+        if not cont:
+            self.current_logfile = self.handler.get_logfile()
+            self.do_log(self.current_logfile)
         # Build image
         self.set_user_config()
         toolchain_packages = []
@@ -627,14 +643,14 @@ class Builder(gtk.Window):
             pass
 
         elif next_step == self.PACKAGE_SELECTION:
-            pass
+            self.package_details_page.show_page(self.current_logfile)
 
         elif next_step == self.PACKAGE_GENERATING or next_step == self.FAST_IMAGE_GENERATING:
             # both PACKAGE_GENEATING and FAST_IMAGE_GENERATING share the same page
             self.build_details_page.show_page(next_step)
 
         elif next_step == self.PACKAGE_GENERATED:
-            pass
+            self.package_details_page.show_page(self.current_logfile)
 
         elif next_step == self.IMAGE_GENERATING:
             # after packages are generated, selected_packages need to
@@ -736,7 +752,7 @@ class Builder(gtk.Window):
 
             self.rcppkglist_populated()
             if self.current_step == self.FAST_IMAGE_GENERATING:
-                self.generate_image_async()
+                self.generate_image_async(True)
 
     def show_error_dialog(self, msg):
         lbl = "<b>Error</b>\n"
@@ -963,6 +979,10 @@ class Builder(gtk.Window):
     def handler_build_failure_cb(self, running_build):
         self.build_details_page.show_issues()
 
+    def handler_build_log_cb(self, running_build, func, obj):
+        if hasattr(self.logger, func):
+            getattr(self.logger, func)(obj)
+
     def destroy_window_cb(self, widget, event):
         if not self.sensitive:
             return True
@@ -992,7 +1012,7 @@ class Builder(gtk.Window):
             dialog.run()
             dialog.destroy()
             return
-        self.generate_packages_async()
+        self.generate_packages_async(True)
 
     def build_image(self):
         selected_packages = self.package_model.get_selected_packages()
@@ -1005,7 +1025,7 @@ class Builder(gtk.Window):
             dialog.run()
             dialog.destroy()
             return
-        self.generate_image_async()
+        self.generate_image_async(True)
 
     def just_bake(self):
         selected_image = self.recipe_model.get_selected_image()
@@ -1022,7 +1042,7 @@ class Builder(gtk.Window):
             dialog.destroy()
             return
 
-        self.fast_generate_image_async()
+        self.fast_generate_image_async(True)
 
     def show_binb_dialog(self, binb):
         markup = "<b>Brought in by:</b>\n%s" % binb
@@ -1241,7 +1261,7 @@ class Builder(gtk.Window):
             response = dialog.run()
             dialog.destroy()
             if response == gtk.RESPONSE_YES:
-                self.generate_packages_async()
+                self.generate_packages_async(True)
             else:
                 self.switch_page(self.PACKAGE_SELECTION)
         else:
@@ -1289,3 +1309,15 @@ class Builder(gtk.Window):
             self.cancel_build_sync()
         elif response == gtk.RESPONSE_YES:
             self.cancel_build_sync(True)
+
+    def do_log(self, consolelogfile = None):
+        if consolelogfile:
+            if self.consolelog:
+                self.logger.removeHandler(self.consolelog)
+                self.consolelog = None
+            self.consolelog = logging.FileHandler(consolelogfile)
+            bb.msg.addDefaultlogFilter(self.consolelog)
+            format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+            self.consolelog.setFormatter(format)
+
+            self.logger.addHandler(self.consolelog)

bitbake/lib/bb/ui/crumbs/hobeventhandler.py

@@ -390,6 +390,9 @@ class HobHandler(gobject.GObject):
     def reset_build(self):
         self.build.reset()
 
+    def get_logfile(self):
+        return self.server.runCommand(["getVariable", "BB_CONSOLELOG"])
+
     def _remove_redundant(self, string):
         ret = []
         for i in string.split():

bitbake/lib/bb/ui/crumbs/imagedetailspage.py

@@ -33,7 +33,7 @@ from bb.ui.crumbs.hig import CrumbsDialog
 class ImageDetailsPage (HobPage):
 
     class DetailBox (gtk.EventBox):
-        def __init__(self, widget = None, varlist = None, vallist = None, icon = None, button = None, color = HobColors.LIGHT_GRAY):
+        def __init__(self, widget = None, varlist = None, vallist = None, icon = None, button = None, button2=None, color = HobColors.LIGHT_GRAY):
             gtk.EventBox.__init__(self)
 
             # set color
@@ -72,7 +72,11 @@ class ImageDetailsPage (HobPage):
                     self.table.attach(self.line_widgets[varlist[index]], colid, 20, row, row + 1)
             # pack the button on the right
             if button:
-                self.hbox.pack_end(button, expand=False, fill=False)
+                self.bbox = gtk.VBox()
+                self.bbox.pack_start(button, expand=True, fill=True)
+                if button2:
+                    self.bbox.pack_start(button2, expand=True, fill=True)
+                self.hbox.pack_end(self.bbox, expand=False, fill=False)
 
         def update_line_widgets(self, variable, value):
             if len(self.line_widgets) == 0:
@@ -164,8 +168,10 @@ class ImageDetailsPage (HobPage):
             base_image = self.builder.recipe_model.get_selected_image()
             layers = self.builder.configuration.layers
             pkg_num = "%s" % len(self.builder.package_model.get_selected_packages())
+            log_file = self.builder.current_logfile
         else:
             pkg_num = "N/A"
+            log_file = None
 
         # remove
         for button_id, button in self.button_ids.items():
@@ -243,11 +249,17 @@ class ImageDetailsPage (HobPage):
         view_files_button = HobAltButton("View files")
         view_files_button.connect("clicked", self.view_files_clicked_cb, image_addr)
         view_files_button.set_tooltip_text("Open the directory containing the image files")
-        self.image_detail = self.DetailBox(varlist=varlist, vallist=vallist, button=view_files_button)
+        view_log_button = None
+        if log_file:
+            view_log_button = HobAltButton("View log")
+            view_log_button.connect("clicked", self.view_log_clicked_cb, log_file)
+            view_log_button.set_tooltip_text("Open the building log files")
+        self.image_detail = self.DetailBox(varlist=varlist, vallist=vallist, button=view_files_button, button2=view_log_button)
         self.box_group_area.pack_start(self.image_detail, expand=False, fill=True)
 
         # The default kernel box for the qemu images
         self.sel_kernel = ""
+        self.kernel_detail = None
         if 'qemu' in image_name:
             self.sel_kernel = self.get_kernel_file_name()
 
@@ -315,12 +327,16 @@ class ImageDetailsPage (HobPage):
             self.box_group_area.pack_start(self.details_bottom_buttons, expand=False, fill=False)
 
         self.show_all()
-        if not is_runnable:
+        if self.kernel_detail and (not is_runnable):
             self.kernel_detail.hide()
 
     def view_files_clicked_cb(self, button, image_addr):
         subprocess.call("xdg-open /%s" % image_addr, shell=True)
 
+    def view_log_clicked_cb(self, button, log_file):
+        if log_file:
+            os.system("xdg-open /%s" % log_file)
+
     def refresh_package_detail_box(self, image_size):
         self.package_detail.update_line_widgets("Total image size: ", image_size)
 

bitbake/lib/bb/ui/crumbs/packageselectionpage.py

@@ -143,8 +143,8 @@ class PackageSelectionPage (HobPage):
         # add all into the dialog
         self.box_group_area.pack_start(self.ins, expand=True, fill=True)
 
-        button_box = gtk.HBox(False, 6)
+        self.button_box = gtk.HBox(False, 6)
-        self.box_group_area.pack_start(button_box, expand=False, fill=False)
+        self.box_group_area.pack_start(self.button_box, expand=False, fill=False)
 
         self.build_image_button = HobButton('Build image')
         self.build_image_button.set_size_request(205, 49)
@@ -152,11 +152,11 @@ class PackageSelectionPage (HobPage):
         self.build_image_button.set_flags(gtk.CAN_DEFAULT)
         self.build_image_button.grab_default()
         self.build_image_button.connect("clicked", self.build_image_clicked_cb)
-        button_box.pack_end(self.build_image_button, expand=False, fill=False)
+        self.button_box.pack_end(self.build_image_button, expand=False, fill=False)
 
         self.back_button = HobAltButton("<< Back to image configuration")
         self.back_button.connect("clicked", self.back_button_clicked_cb)
-        button_box.pack_start(self.back_button, expand=False, fill=False)
+        self.button_box.pack_start(self.back_button, expand=False, fill=False)
 
     def button_click_cb(self, widget, event):
         path, col = widget.table_tree.get_cursor()
@@ -166,6 +166,25 @@ class PackageSelectionPage (HobPage):
             if binb:
                 self.builder.show_binb_dialog(binb)
 
+    def view_log_clicked_cb(self, button, log_file):
+        if log_file:
+            os.system("xdg-open /%s" % log_file)
+
+    def show_page(self, log_file):
+        children = self.button_box.get_children() or []
+        for child in children:
+            self.button_box.remove(child)
+        # re-packed the buttons as request, add the 'view log' button if build success
+        self.button_box.pack_start(self.back_button, expand=False, fill=False)
+        self.button_box.pack_end(self.build_image_button, expand=False, fill=False)
+        if log_file:
+            view_log_button = HobAltButton("View log")
+            view_log_button.connect("clicked", self.view_log_clicked_cb, log_file)
+            view_log_button.set_tooltip_text("Open the building log files")
+            self.button_box.pack_end(view_log_button, expand=False, fill=False)
+
+        self.show_all()
+
     def build_image_clicked_cb(self, button):
         self.builder.build_image()
 

bitbake/lib/bb/ui/crumbs/runningbuild.py

@@ -88,6 +88,9 @@ class RunningBuild (gobject.GObject):
           'no-provider'     :  (gobject.SIGNAL_RUN_LAST,
                                 gobject.TYPE_NONE,
                                (gobject.TYPE_PYOBJECT,)),
+          'log'             :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
+                               (gobject.TYPE_STRING, gobject.TYPE_PYOBJECT,)),
           }
     pids_to_task = {}
     tasks_to_iter = {}
@@ -126,6 +129,8 @@ class RunningBuild (gobject.GObject):
             parent = self.tasks_to_iter[(package, task)]
 
         if(isinstance(event, logging.LogRecord)):
+            if event.taskpid == 0 or event.levelno > logging.INFO:
+                self.emit("log", "handle", event)
             # FIXME: this is a hack! More info in Yocto #1433
             # http://bugzilla.pokylinux.org/show_bug.cgi?id=1433, temporarily
             # mask the error message as it's not informative for the user.
@@ -211,6 +216,7 @@ class RunningBuild (gobject.GObject):
             self.tasks_to_iter[(package, task)] = i
 
         elif isinstance(event, bb.build.TaskBase):
+            self.emit("log", "info", event._message)
             current = self.tasks_to_iter[(package, task)]
             parent = self.tasks_to_iter[(package, None)]
 
@@ -296,6 +302,7 @@ class RunningBuild (gobject.GObject):
             self.buildaborted = True
 
         elif isinstance(event, bb.command.CommandFailed):
+            self.emit("log", "error", "Command execution failed: %s" % (event.error))
             if event.error.startswith("Exited with"):
                 # If the command fails with an exit code we're done, emit the
                 # generic signal for the UI to notify the user
@@ -323,7 +330,24 @@ class RunningBuild (gobject.GObject):
         elif isinstance(event, bb.event.ParseCompleted) and pbar:
             pbar.hide()
         #using runqueue events as many as possible to update the progress bar
+        elif isinstance(event, bb.runqueue.runQueueTaskFailed):
+            self.emit("log", "error", "Task %s (%s) failed with exit code '%s'" % (event.taskid, event.taskstring, event.exitcode))
+        elif isinstance(event, bb.runqueue.sceneQueueTaskFailed):
+            self.emit("log", "warn", "Setscene task %s (%s) failed with exit code '%s' - real task will be run instead" \
+                                     % (event.taskid, event.taskstring, event.exitcode))
         elif isinstance(event, (bb.runqueue.runQueueTaskStarted, bb.runqueue.sceneQueueTaskStarted)):
+            if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+                self.emit("log", "info", "Running setscene task %d of %d (%s)" % \
+                                         (event.stats.completed + event.stats.active + event.stats.failed + 1,
+                                          event.stats.total, event.taskstring))
+            else:
+                if event.noexec:
+                    tasktype = 'noexec task'
+                else:
+                    tasktype = 'task'
+                self.emit("log", "info", "Running %s %s of %s (ID: %s, %s)" % \
+                                         (tasktype, event.stats.completed + event.stats.active + event.stats.failed + 1,
+                                          event.stats.total, event.taskid, event.taskstring))
             message = {}
             message["eventname"] = bb.event.getName(event)
             num_of_completed = event.stats.completed + event.stats.failed
@@ -332,6 +356,10 @@ class RunningBuild (gobject.GObject):
             message["title"] = ""
             message["task"] = event.taskstring
             self.emit("task-started", message)
+        elif isinstance(event, bb.event.MultipleProviders):
+            self.emit("log", "info", "multiple providers are available for %s%s (%s)" \
+                                     % (event._is_runtime and "runtime " or "", event._item, ", ".join(event._candidates)))
+            self.emit("log", "info", "consider defining a PREFERRED_PROVIDER entry to match %s" % (event._item))
         elif isinstance(event, bb.event.NoProvider):
             msg = ""
             if event._runtime:
@@ -346,6 +374,19 @@ class RunningBuild (gobject.GObject):
                 for reason in event._reasons:
                     msg += ("%s\n" % reason)
             self.emit("no-provider", msg)
+            self.emit("log", msg)
+        else:
+            if not isinstance(event, (bb.event.BuildBase,
+                                      bb.event.StampUpdate,
+                                      bb.event.ConfigParsed,
+                                      bb.event.RecipeParsed,
+                                      bb.event.RecipePreFinalise,
+                                      bb.runqueue.runQueueEvent,
+                                      bb.runqueue.runQueueExitWait,
+                                      bb.event.OperationStarted,
+                                      bb.event.OperationCompleted,
+                                      bb.event.OperationProgress)):
+                self.emit("log", "error", "Unknown event: %s" % (event.error if hasattr(event, 'error') else 'error'))
 
         return