bitbake: Automatically start local PR service.

[YOCTO #1126]
A local PR service will be started and stopped automatically along
with the bitbake invocation/ternimation.

This local PR service will be started only and if only when the
PRSERV_HOST is set to 'localhost' and PRSERV_PORT is set to '0'.

When started, the sqlite3 database is stored at
"${PERSISTEN_DIR}/prserv.sqlite3" or "${CACHE}/prserv.sqlite3".

(Bitbake rev: 9d8f45407c67ed0d3c4f820cf646de3c385067c7)

Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-prserv

@@ -10,7 +10,7 @@ import prserv.serv
 
 __version__="1.0.0"
 
-PRHOST_DEFAULT=''
+PRHOST_DEFAULT='0.0.0.0'
 PRPORT_DEFAULT=8585
 
 def main():
@@ -18,8 +18,8 @@ def main():
         version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__),
         usage = "%prog < --start | --stop > [options]")
 
-    parser.add_option("-f", "--file", help="database filename(default: prserv.db)", action="store",
+    parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store",
-                      dest="dbfile", type="string", default="prserv.db")
+                      dest="dbfile", type="string", default="prserv.sqlite3")
     parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store",
                       dest="logfile", type="string", default="prserv.log")
     parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
@@ -37,8 +37,7 @@ def main():
     prserv.init_logger(os.path.abspath(options.logfile),options.loglevel)
 
     if options.start:
-        ret=prserv.serv.start_daemon(dbfile=options.dbfile, interface=(options.host, options.port),
+        ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile))
-                      logfile=os.path.abspath(options.logfile))
     elif options.stop:
         ret=prserv.serv.stop_daemon(options.host, options.port)
     else:

bitbake/lib/bb/cooker.py

@@ -36,6 +36,7 @@ from functools import wraps
 from collections import defaultdict
 import bb, bb.exceptions, bb.command
 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
+import prserv.serv
 
 logger      = logging.getLogger("BitBake")
 collectlog  = logging.getLogger("BitBake.Collection")
@@ -1311,9 +1312,11 @@ class BBCooker:
         # Empty the environment. The environment will be populated as
         # necessary from the data store.
         #bb.utils.empty_environment()
+        prserv.serv.auto_start(self.configuration.data)
         return
 
     def post_serve(self):
+        prserv.serv.auto_shutdown(self.configuration.data)
         bb.event.fire(CookerExit(), self.configuration.event_data)
 
     def shutdown(self):

bitbake/lib/prserv/db.py

@@ -8,6 +8,8 @@ try:
 except ImportError:
     from pysqlite2 import dbapi2 as sqlite3
 
+logger = logging.getLogger("BitBake.PRserv")
+
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
     raise Exception("sqlite3 version 3.3.0 or later is required.")
@@ -55,7 +57,7 @@ class PRTable():
                            (version,pkgarch, checksum,version, pkgarch))
                 self.conn.commit()
             except sqlite3.IntegrityError as exc:
-                logging.error(str(exc))
+                logger.error(str(exc))
 
             data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
                                (version, pkgarch, checksum))
@@ -83,7 +85,7 @@ class PRTable():
                                (version, pkgarch, checksum, version, pkgarch))
                 self.conn.commit()
             except sqlite3.IntegrityError as exc:
-                logging.error(str(exc))
+                logger.error(str(exc))
                 self.conn.rollback()
 
             data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
@@ -115,7 +117,7 @@ class PRTable():
                            (version, pkgarch, checksum, value))
                 self.conn.commit()
             except sqlite3.IntegrityError as exc:
-                logging.error(str(exc))
+                logger.error(str(exc))
 
             data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
                            (version, pkgarch, checksum))
@@ -140,7 +142,7 @@ class PRTable():
                                (value,version,pkgarch,checksum,value))
                 self.conn.commit()
             except sqlite3.IntegrityError as exc:
-                logging.error(str(exc))
+                logger.error(str(exc))
 
         data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
                             (version,pkgarch,checksum,value))
@@ -241,5 +243,5 @@ class PRData(object):
     def __delitem__(self, tblname):
         if tblname in self._tables:
             del self._tables[tblname]
-        logging.info("drop table %s" % (tblname))
+        logger.info("drop table %s" % (tblname))
         self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 

bitbake/lib/prserv/serv.py

@@ -1,5 +1,5 @@
 import os,sys,logging
-import signal,time, atexit
+import signal, time, atexit, threading
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 import xmlrpclib,sqlite3
 
@@ -7,6 +7,8 @@ import bb.server.xmlrpc
 import prserv
 import prserv.db
 
+logger = logging.getLogger("BitBake.PRserv")
+
 if sys.hexversion < 0x020600F0:
     print("Sorry, python 2.6 or later is required.")
     sys.exit(1)
@@ -22,9 +24,9 @@ class Handler(SimpleXMLRPCRequestHandler):
         return value
 
 PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
+singleton = None
 
 class PRServer(SimpleXMLRPCServer):
-    pidfile="/tmp/PRServer.pid"
     def __init__(self, dbfile, logfile, interface, daemon=True):
         ''' constructor '''
         SimpleXMLRPCServer.__init__(self, interface,
@@ -33,10 +35,11 @@ class PRServer(SimpleXMLRPCServer):
         self.dbfile=dbfile
         self.daemon=daemon
         self.logfile=logfile
+        self.working_thread=None
         self.host, self.port = self.socket.getsockname()
         self.db=prserv.db.PRData(dbfile)
         self.table=self.db["PRMAIN"]
-        self.pidfile=PIDPREFIX % interface
+        self.pidfile=PIDPREFIX % (self.host, self.port)
 
         self.register_function(self.getPR, "getPR")
         self.register_function(self.quit, "quit")
@@ -44,12 +47,12 @@ class PRServer(SimpleXMLRPCServer):
         self.register_function(self.export, "export")
         self.register_function(self.importone, "importone")
         self.register_introspection_functions()
-        
+
     def export(self, version=None, pkgarch=None, checksum=None, colinfo=True):
         try:
             return self.table.export(version, pkgarch, checksum, colinfo)
         except sqlite3.Error as exc:
-            logging.error(str(exc))
+            logger.error(str(exc))
             return None
 
     def importone(self, version, pkgarch, checksum, value):
@@ -58,45 +61,47 @@ class PRServer(SimpleXMLRPCServer):
     def ping(self):
         return not self.quit
 
+    def getinfo(self):
+        return (self.host, self.port)
+
     def getPR(self, version, pkgarch, checksum):
         try:
             return self.table.getValue(version, pkgarch, checksum)
         except prserv.NotFoundError:
-            logging.error("can not find value for (%s, %s)",version, checksum)
+            logger.error("can not find value for (%s, %s)",version, checksum)
             return None
         except sqlite3.Error as exc:
-            logging.error(str(exc))
+            logger.error(str(exc))
             return None
 
     def quit(self):
         self.quit=True
         return
 
-    def _serve_forever(self):
+    def work_forever(self,):
         self.quit = False
         self.timeout = 0.5
+        logger.info("PRServer: started! DBfile: %s, IP: %s, PORT: %s, PID: %s" %
+                     (self.dbfile, self.host, self.port, str(os.getpid())))
+
         while not self.quit:
             self.handle_request()
 
-        logging.info("PRServer: stopping...")
+        logger.info("PRServer: stopping...")
         self.server_close()
         return
 
     def start(self):
         if self.daemon is True:
-            logging.info("PRServer: try to start daemon...")
+            logger.info("PRServer: try to start daemon...")
             self.daemonize()
         else:
             atexit.register(self.delpid)
             pid = str(os.getpid()) 
             pf = file(self.pidfile, 'w+')
             pf.write("%s\n" % pid)
-            pf.write("%s\n" % self.host)
-            pf.write("%s\n" % self.port)
             pf.close()
-            logging.info("PRServer: start success! DBfile: %s, IP: %s, PORT: %d" % 
+            self.work_forever()
-                     (self.dbfile, self.host, self.port))
-            self._serve_forever()
 
     def delpid(self):
         os.remove(self.pidfile)
@@ -144,17 +149,40 @@ class PRServer(SimpleXMLRPCServer):
         pf.write("%s\n" % pid)
         pf.close()
 
-        logging.info("PRServer: starting daemon success! DBfile: %s, IP: %s, PORT: %s, PID: %s" % 
+        self.work_forever()
-                     (self.dbfile, self.host, self.port, pid))
+        sys.exit(0)
 
-        self._serve_forever()
+class PRServSingleton():
-        exit(0)
+    def __init__(self, dbfile, logfile, interface):
+        self.dbfile = dbfile
+        self.logfile = logfile
+        self.interface = interface
+        self.host = None
+        self.port = None
+        self.event = threading.Event()
+
+    def _work(self):
+        self.prserv = PRServer(self.dbfile, self.logfile, self.interface, False)
+        self.host, self.port = self.prserv.getinfo()
+        self.event.set()
+        self.prserv.work_forever()
+        del self.prserv.db
+
+    def start(self):
+        self.working_thread = threading.Thread(target=self._work)
+        self.working_thread.start()
+
+    def getinfo(self):
+        self.event.wait()
+        return (self.host, self.port)
 
 class PRServerConnection():
     def __init__(self, host, port):
-        self.connection = bb.server.xmlrpc._create_server(host, port)
+        if is_local_special(host, port):
+            host, port = singleton.getinfo()
         self.host = host
         self.port = port
+        self.connection = bb.server.xmlrpc._create_server(self.host, self.port)
 
     def terminate(self):
         # Don't wait for server indefinitely
@@ -173,13 +201,14 @@ class PRServerConnection():
 
     def export(self,version=None, pkgarch=None, checksum=None, colinfo=True):
         return self.connection.export(version, pkgarch, checksum, colinfo)
-    
+
     def importone(self, version, pkgarch, checksum, value):
         return self.connection.importone(version, pkgarch, checksum, value)
 
-def start_daemon(dbfile, logfile, interface):
+def start_daemon(dbfile, host, port, logfile):
+    pidfile = PIDPREFIX % (host, port)
     try:
-        pf = file(PRServer.pidfile,'r')
+        pf = file(pidfile,'r')
         pid = int(pf.readline().strip())
         pf.close()
     except IOError:
@@ -187,10 +216,10 @@ def start_daemon(dbfile, logfile, interface):
 
     if pid:
         sys.stderr.write("pidfile %s already exist. Daemon already running?\n"
-                            % PRServer.pidfile)
+                            % pidfile)
         return 1
 
-    server = PRServer(os.path.abspath(dbfile), os.path.abspath(logfile), interface)
+    server = PRServer(os.path.abspath(dbfile), os.path.abspath(logfile), (host,port))
     server.start()
     return 0
 
@@ -206,25 +235,70 @@ def stop_daemon(host, port):
     if not pid:
         sys.stderr.write("pidfile %s does not exist. Daemon not running?\n"
                         % pidfile)
-        return 1
 
-    PRServerConnection(host, port).terminate()
+    try:
+        PRServerConnection(host, port).terminate()
+    except:
+        logger.critical("Stop PRService %s:%d failed" % (host,port))
     time.sleep(0.5)
 
     try:
-        while 1:
+        if pid:
+            if os.path.exists(pidfile):
+                os.remove(pidfile)
             os.kill(pid,signal.SIGTERM)
             time.sleep(0.1)
     except OSError as e:
         err = str(e)
-        if err.find("No such process") > 0:
+        if err.find("No such process") <= 0:
-            if os.path.exists(PRServer.pidfile):
+            raise e
-                os.remove(PRServer.pidfile)
-        else:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
 
     return 0
 
+def is_local_special(host, port):
+    if host.strip().upper() == 'localhost'.upper() and (not port):
+        return True
+    else:
+        return False
+
+def auto_start(d):
+    global singleton
+    if d.getVar('USE_PR_SERV', True) == '0':
+        return True
+
+    if is_local_special(d.getVar('PRSERV_HOST', True), int(d.getVar('PRSERV_PORT', True))) and not singleton:
+        import bb.utils
+        cachedir = (d.getVar("PERSISTENT_DIR", True) or d.getVar("CACHE", True))
+        if not cachedir:
+            logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
+            sys.exit(1)
+        bb.utils.mkdirhier(cachedir)
+        dbfile = os.path.join(cachedir, "prserv.sqlite3")
+        logfile = os.path.join(cachedir, "prserv.log")
+        singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), ("localhost",0))
+        singleton.start()
+    if singleton:
+        host, port = singleton.getinfo()
+    else:
+        host = d.getVar('PRSERV_HOST', True)
+        port = int(d.getVar('PRSERV_PORT', True))
+
+    try:
+        return PRServerConnection(host,port).ping()
+    except Exception:
+        logger.critical("PRservice %s:%d not available" % (host, port))
+    return False
+
+def auto_shutdown(d=None):
+    global singleton
+    if singleton:
+        host, port = singleton.getinfo()
+        try:
+            PRServerConnection(host, port).terminate()
+        except:
+            logger.critical("Stop PRService %s:%d failed" % (host,port))
+        singleton = None
+
 def ping(host, port):
-    print PRServerConnection(host,port).ping()
+    conn=PRServerConnection(host, port)
-    return 0
+    return conn.ping()