bitbake: bitbake: cooker,xmlrpc,servers: implement CookerFeatures
Implementing feature set selection that allows a client to enable specific features in the server at connection time. Only enabling of features is supported, as there is no way to safely remove data loaded into the cooker. Once enabled, a feature will remain enabled for the life of the cooker. Client-server connection now supports specifying the feature set required by the client. This is implemented in the Process server using a managed proxy list, so the server cooker will now load dynamically needed features based on what client connects to it. In the XMLRPC server the feature set is requested by using a parameter for registerUIHandler function. This allows observer-only clients to also specify features for the server. The server code configuration now is completly separated from the client code. All hardcoding of client knowledge is removed from the server. The extra_caches is removed as the client can now specify the caches it needs using the feature. The UI modules now need to specify the desired featureSet. HOB is modified to conform to the featureSet specification. The only feature available is CookerFeatures.HOB_EXTRA_CACHES which forces loading the bb.cache_extra:HobRecipeInfo class. (Bitbake rev: 98e594837aab89ea042cfa9f3740d20a661b14e2) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
parent
87e86d4fd3
commit
ba83eb315d
|
@ -75,18 +75,6 @@ def get_ui(config):
|
||||||
sys.exit("FATAL: Invalid user interface '%s' specified.\n"
|
sys.exit("FATAL: Invalid user interface '%s' specified.\n"
|
||||||
"Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
|
"Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
|
||||||
|
|
||||||
def gather_extra_cache_data():
|
|
||||||
extra = []
|
|
||||||
interfaces = ['depexp', 'goggle', 'ncurses', 'hob', 'knotty']
|
|
||||||
for i in interfaces:
|
|
||||||
try:
|
|
||||||
ui = __import__("bb.ui." + i, fromlist = [i])
|
|
||||||
if hasattr(ui, "extraCaches"):
|
|
||||||
extra = extra + ui.extraCaches
|
|
||||||
del ui
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
return extra
|
|
||||||
|
|
||||||
# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
|
# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
|
||||||
warnlog = logging.getLogger("BitBake.Warnings")
|
warnlog = logging.getLogger("BitBake.Warnings")
|
||||||
|
@ -302,25 +290,22 @@ def main():
|
||||||
# Clear away any spurious environment variables while we stoke up the cooker
|
# Clear away any spurious environment variables while we stoke up the cooker
|
||||||
cleanedvars = bb.utils.clean_environment()
|
cleanedvars = bb.utils.clean_environment()
|
||||||
|
|
||||||
# Collect all the caches we need
|
|
||||||
if configParams.server_only:
|
|
||||||
configuration.extra_caches = gather_extra_cache_data()
|
|
||||||
else:
|
|
||||||
configuration.extra_caches = getattr(ui_module, "extraCaches", [])
|
|
||||||
|
|
||||||
if not configParams.remote_server:
|
if not configParams.remote_server:
|
||||||
# we start a server with a given configuration
|
# we start a server with a given configuration
|
||||||
server = start_server(servermodule, configParams, configuration)
|
server = start_server(servermodule, configParams, configuration)
|
||||||
else:
|
else:
|
||||||
# we start a stub server that is actually a XMLRPClient to
|
# we start a stub server that is actually a XMLRPClient that connects to a real server
|
||||||
server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
|
server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
|
||||||
server.saveConnectionDetails(configParams.remote_server)
|
server.saveConnectionDetails(configParams.remote_server)
|
||||||
|
|
||||||
logger.removeHandler(handler)
|
logger.removeHandler(handler)
|
||||||
|
|
||||||
if not configParams.server_only:
|
if not configParams.server_only:
|
||||||
|
# Collect the feature set for the UI
|
||||||
|
featureset = getattr(ui_module, "featureSet", [])
|
||||||
|
|
||||||
# Setup a connection to the server (cooker)
|
# Setup a connection to the server (cooker)
|
||||||
server_connection = server.establishConnection()
|
server_connection = server.establishConnection(featureset)
|
||||||
|
|
||||||
# Restore the environment in case the UI needs it
|
# Restore the environment in case the UI needs it
|
||||||
for k in cleanedvars:
|
for k in cleanedvars:
|
||||||
|
|
|
@ -79,6 +79,29 @@ class SkippedPackage:
|
||||||
elif reason:
|
elif reason:
|
||||||
self.skipreason = reason
|
self.skipreason = reason
|
||||||
|
|
||||||
|
|
||||||
|
class CookerFeatures(object):
|
||||||
|
_feature_list = [HOB_EXTRA_CACHES] = range(1)
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._features=set()
|
||||||
|
|
||||||
|
def setFeature(self, f):
|
||||||
|
# validate we got a request for a feature we support
|
||||||
|
if f not in CookerFeatures._feature_list:
|
||||||
|
return
|
||||||
|
self._features.add(f)
|
||||||
|
|
||||||
|
def __contains__(self, f):
|
||||||
|
return f in self._features
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
return self._features.__iter__()
|
||||||
|
|
||||||
|
def next(self):
|
||||||
|
return self._features.next()
|
||||||
|
|
||||||
|
|
||||||
#============================================================================#
|
#============================================================================#
|
||||||
# BBCooker
|
# BBCooker
|
||||||
#============================================================================#
|
#============================================================================#
|
||||||
|
@ -90,6 +113,7 @@ class BBCooker:
|
||||||
def __init__(self, configuration):
|
def __init__(self, configuration):
|
||||||
self.recipecache = None
|
self.recipecache = None
|
||||||
self.skiplist = {}
|
self.skiplist = {}
|
||||||
|
self.featureset = CookerFeatures()
|
||||||
|
|
||||||
self.configuration = configuration
|
self.configuration = configuration
|
||||||
|
|
||||||
|
@ -122,7 +146,13 @@ class BBCooker:
|
||||||
self.state = state.initial
|
self.state = state.initial
|
||||||
|
|
||||||
self.caches_array = []
|
self.caches_array = []
|
||||||
caches_name_array = ['bb.cache:CoreRecipeInfo'] + self.configuration.extra_caches
|
|
||||||
|
all_extra_cache_names = []
|
||||||
|
# We hardcode all known cache types in a single place, here.
|
||||||
|
if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
|
||||||
|
all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
|
||||||
|
|
||||||
|
caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
|
||||||
|
|
||||||
# At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
|
# At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
|
||||||
# This is the entry point, no further check needed!
|
# This is the entry point, no further check needed!
|
||||||
|
|
|
@ -127,7 +127,6 @@ class CookerConfiguration(object):
|
||||||
self.dump_signatures = False
|
self.dump_signatures = False
|
||||||
self.dry_run = False
|
self.dry_run = False
|
||||||
self.tracking = False
|
self.tracking = False
|
||||||
self.extra_caches = []
|
|
||||||
|
|
||||||
self.env = {}
|
self.env = {}
|
||||||
|
|
||||||
|
|
|
@ -89,7 +89,7 @@ class BitBakeBaseServer(object):
|
||||||
def detach(self):
|
def detach(self):
|
||||||
return
|
return
|
||||||
|
|
||||||
def establishConnection(self):
|
def establishConnection(self, featureset):
|
||||||
raise "Must redefine the %s.establishConnection()" % self.__class__.__name__
|
raise "Must redefine the %s.establishConnection()" % self.__class__.__name__
|
||||||
|
|
||||||
def endSession(self):
|
def endSession(self):
|
||||||
|
|
|
@ -31,7 +31,7 @@ import sys
|
||||||
import time
|
import time
|
||||||
import select
|
import select
|
||||||
from Queue import Empty
|
from Queue import Empty
|
||||||
from multiprocessing import Event, Process, util, Queue, Pipe, queues
|
from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
|
||||||
|
|
||||||
from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
|
from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
|
||||||
|
|
||||||
|
@ -78,12 +78,13 @@ class ProcessServer(Process, BaseImplServer):
|
||||||
profile_filename = "profile.log"
|
profile_filename = "profile.log"
|
||||||
profile_processed_filename = "profile.log.processed"
|
profile_processed_filename = "profile.log.processed"
|
||||||
|
|
||||||
def __init__(self, command_channel, event_queue):
|
def __init__(self, command_channel, event_queue, featurelist):
|
||||||
BaseImplServer.__init__(self)
|
BaseImplServer.__init__(self)
|
||||||
Process.__init__(self)
|
Process.__init__(self, args=(featurelist))
|
||||||
self.command_channel = command_channel
|
self.command_channel = command_channel
|
||||||
self.event_queue = event_queue
|
self.event_queue = event_queue
|
||||||
self.event = EventAdapter(event_queue)
|
self.event = EventAdapter(event_queue)
|
||||||
|
self.featurelist = featurelist
|
||||||
self.quit = False
|
self.quit = False
|
||||||
|
|
||||||
self.keep_running = Event()
|
self.keep_running = Event()
|
||||||
|
@ -94,6 +95,14 @@ class ProcessServer(Process, BaseImplServer):
|
||||||
for event in bb.event.ui_queue:
|
for event in bb.event.ui_queue:
|
||||||
self.event_queue.put(event)
|
self.event_queue.put(event)
|
||||||
self.event_handle.value = bb.event.register_UIHhandler(self)
|
self.event_handle.value = bb.event.register_UIHhandler(self)
|
||||||
|
|
||||||
|
# process any feature changes based on what UI requested
|
||||||
|
original_featureset = list(self.cooker.featureset)
|
||||||
|
while len(self.featurelist)> 0:
|
||||||
|
self.cooker.featureset.setFeature(self.featurelist.pop())
|
||||||
|
if (original_featureset != list(self.cooker.featureset)):
|
||||||
|
self.cooker.reset()
|
||||||
|
|
||||||
bb.cooker.server_main(self.cooker, self.main)
|
bb.cooker.server_main(self.cooker, self.main)
|
||||||
|
|
||||||
def main(self):
|
def main(self):
|
||||||
|
@ -198,13 +207,17 @@ class BitBakeServer(BitBakeBaseServer):
|
||||||
#
|
#
|
||||||
self.ui_channel, self.server_channel = Pipe()
|
self.ui_channel, self.server_channel = Pipe()
|
||||||
self.event_queue = ProcessEventQueue(0)
|
self.event_queue = ProcessEventQueue(0)
|
||||||
self.serverImpl = ProcessServer(self.server_channel, self.event_queue)
|
manager = Manager()
|
||||||
|
self.featurelist = manager.list()
|
||||||
|
self.serverImpl = ProcessServer(self.server_channel, self.event_queue, self.featurelist)
|
||||||
|
|
||||||
def detach(self):
|
def detach(self):
|
||||||
self.serverImpl.start()
|
self.serverImpl.start()
|
||||||
return
|
return
|
||||||
|
|
||||||
def establishConnection(self):
|
def establishConnection(self, featureset):
|
||||||
|
for f in featureset:
|
||||||
|
self.featurelist.append(f)
|
||||||
self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
|
self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
|
||||||
signal.signal(signal.SIGTERM, lambda i, s: self.connection.terminate())
|
signal.signal(signal.SIGTERM, lambda i, s: self.connection.terminate())
|
||||||
return self.connection
|
return self.connection
|
||||||
|
|
|
@ -89,12 +89,23 @@ class BitBakeServerCommands():
|
||||||
self.server = server
|
self.server = server
|
||||||
self.has_client = False
|
self.has_client = False
|
||||||
|
|
||||||
def registerEventHandler(self, host, port):
|
def registerEventHandler(self, host, port, featureset = []):
|
||||||
"""
|
"""
|
||||||
Register a remote UI Event Handler
|
Register a remote UI Event Handler
|
||||||
"""
|
"""
|
||||||
s, t = _create_server(host, port)
|
s, t = _create_server(host, port)
|
||||||
|
|
||||||
|
# we don't allow connections if the cooker is running
|
||||||
|
if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
|
||||||
|
return None
|
||||||
|
|
||||||
|
original_featureset = list(self.cooker.featureset)
|
||||||
|
for f in featureset:
|
||||||
|
self.cooker.featureset.setFeature(f)
|
||||||
|
|
||||||
|
if (original_featureset != list(self.cooker.featureset)):
|
||||||
|
self.cooker.reset()
|
||||||
|
|
||||||
self.event_handle = bb.event.register_UIHhandler(s)
|
self.event_handle = bb.event.register_UIHhandler(s)
|
||||||
return self.event_handle
|
return self.event_handle
|
||||||
|
|
||||||
|
@ -263,11 +274,12 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
|
||||||
self.connection_token = token
|
self.connection_token = token
|
||||||
|
|
||||||
class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
|
class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
|
||||||
def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False):
|
def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = []):
|
||||||
self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
|
self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
|
||||||
self.clientinfo = clientinfo
|
self.clientinfo = clientinfo
|
||||||
self.serverImpl = serverImpl
|
self.serverImpl = serverImpl
|
||||||
self.observer_only = observer_only
|
self.observer_only = observer_only
|
||||||
|
self.featureset = featureset
|
||||||
|
|
||||||
def connect(self):
|
def connect(self):
|
||||||
if not self.observer_only:
|
if not self.observer_only:
|
||||||
|
@ -277,7 +289,8 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
|
||||||
if token is None:
|
if token is None:
|
||||||
return None
|
return None
|
||||||
self.transport.set_connection_token(token)
|
self.transport.set_connection_token(token)
|
||||||
self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
|
|
||||||
|
self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo, self.featureset)
|
||||||
for event in bb.event.ui_queue:
|
for event in bb.event.ui_queue:
|
||||||
self.events.queue_event(event)
|
self.events.queue_event(event)
|
||||||
return self
|
return self
|
||||||
|
@ -301,14 +314,15 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
|
||||||
|
|
||||||
class BitBakeServer(BitBakeBaseServer):
|
class BitBakeServer(BitBakeBaseServer):
|
||||||
def initServer(self, interface = ("localhost", 0)):
|
def initServer(self, interface = ("localhost", 0)):
|
||||||
|
self.interface = interface
|
||||||
self.serverImpl = XMLRPCServer(interface)
|
self.serverImpl = XMLRPCServer(interface)
|
||||||
|
|
||||||
def detach(self):
|
def detach(self):
|
||||||
daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
|
daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
|
||||||
del self.cooker
|
del self.cooker
|
||||||
|
|
||||||
def establishConnection(self):
|
def establishConnection(self, featureset):
|
||||||
self.connection = BitBakeXMLRPCServerConnection(self.serverImpl)
|
self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
|
||||||
return self.connection.connect()
|
return self.connection.connect()
|
||||||
|
|
||||||
def set_connection_token(self, token):
|
def set_connection_token(self, token):
|
||||||
|
@ -318,12 +332,13 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
|
||||||
|
|
||||||
def __init__(self, observer_only = False):
|
def __init__(self, observer_only = False):
|
||||||
self.observer_only = observer_only
|
self.observer_only = observer_only
|
||||||
|
# if we need extra caches, just tell the server to load them all
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def saveConnectionDetails(self, remote):
|
def saveConnectionDetails(self, remote):
|
||||||
self.remote = remote
|
self.remote = remote
|
||||||
|
|
||||||
def establishConnection(self):
|
def establishConnection(self, featureset):
|
||||||
# The format of "remote" must be "server:port"
|
# The format of "remote" must be "server:port"
|
||||||
try:
|
try:
|
||||||
[host, port] = self.remote.split(":")
|
[host, port] = self.remote.split(":")
|
||||||
|
@ -340,7 +355,7 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
|
||||||
except:
|
except:
|
||||||
return None
|
return None
|
||||||
self.serverImpl = XMLRPCProxyServer(host, port)
|
self.serverImpl = XMLRPCProxyServer(host, port)
|
||||||
self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only)
|
self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
|
||||||
return self.connection.connect()
|
return self.connection.connect()
|
||||||
|
|
||||||
def endSession(self):
|
def endSession(self):
|
||||||
|
|
|
@ -46,7 +46,7 @@ from bb.ui.crumbs.hoblistmodel import RecipeListModel, PackageListModel
|
||||||
from bb.ui.crumbs.hobeventhandler import HobHandler
|
from bb.ui.crumbs.hobeventhandler import HobHandler
|
||||||
from bb.ui.crumbs.builder import Builder
|
from bb.ui.crumbs.builder import Builder
|
||||||
|
|
||||||
extraCaches = ['bb.cache_extra:HobRecipeInfo']
|
featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES]
|
||||||
|
|
||||||
def event_handle_idle_func(eventHandler, hobHandler):
|
def event_handle_idle_func(eventHandler, hobHandler):
|
||||||
# Consume as many messages as we can in the time available to us
|
# Consume as many messages as we can in the time available to us
|
||||||
|
|
|
@ -472,6 +472,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
|
||||||
event.taskid, event.taskstring, event.exitcode)
|
event.taskid, event.taskstring, event.exitcode)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if isinstance(event, bb.event.DepTreeGenerated):
|
||||||
|
continue
|
||||||
|
|
||||||
# ignore
|
# ignore
|
||||||
if isinstance(event, (bb.event.BuildBase,
|
if isinstance(event, (bb.event.BuildBase,
|
||||||
bb.event.StampUpdate,
|
bb.event.StampUpdate,
|
||||||
|
|
|
@ -28,7 +28,7 @@ import socket, threading, pickle
|
||||||
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
|
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
|
||||||
|
|
||||||
class BBUIEventQueue:
|
class BBUIEventQueue:
|
||||||
def __init__(self, BBServer, clientinfo=("localhost, 0")):
|
def __init__(self, BBServer, clientinfo=("localhost, 0"), featureset=[]):
|
||||||
|
|
||||||
self.eventQueue = []
|
self.eventQueue = []
|
||||||
self.eventQueueLock = threading.Lock()
|
self.eventQueueLock = threading.Lock()
|
||||||
|
@ -44,7 +44,10 @@ class BBUIEventQueue:
|
||||||
server.register_function( self.send_event, "event.sendpickle" )
|
server.register_function( self.send_event, "event.sendpickle" )
|
||||||
server.socket.settimeout(1)
|
server.socket.settimeout(1)
|
||||||
|
|
||||||
self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port)
|
self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port, featureset)
|
||||||
|
|
||||||
|
if (self.EventHandle == None):
|
||||||
|
bb.fatal("Could not register UI event handler")
|
||||||
|
|
||||||
self.server = server
|
self.server = server
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue