persist_data: cache connection and use cursor
Store database connection to persistent database in fetcher. (Bitbake rev: 8a6876752b90efd81d92f0947bfc9527d8260969) Signed-off-by: Bernhard Reutner-Fischer <rep.dot.nop@gmail.com> Signed-off-by: Chris Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
This commit is contained in:
parent
30216c65e5
commit
726802b980
|
@ -146,13 +146,14 @@ def uri_replace(uri, uri_find, uri_replace, d):
|
||||||
methods = []
|
methods = []
|
||||||
urldata_cache = {}
|
urldata_cache = {}
|
||||||
saved_headrevs = {}
|
saved_headrevs = {}
|
||||||
|
persistent_database_connection = {}
|
||||||
|
|
||||||
def fetcher_init(d):
|
def fetcher_init(d):
|
||||||
"""
|
"""
|
||||||
Called to initialize the fetchers once the configuration data is known.
|
Called to initialize the fetchers once the configuration data is known.
|
||||||
Calls before this must not hit the cache.
|
Calls before this must not hit the cache.
|
||||||
"""
|
"""
|
||||||
pd = persist_data.PersistData(d)
|
pd = persist_data.PersistData(d, persistent_database_connection)
|
||||||
# When to drop SCM head revisions controlled by user policy
|
# When to drop SCM head revisions controlled by user policy
|
||||||
srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, 1) or "clear"
|
srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, 1) or "clear"
|
||||||
if srcrev_policy == "cache":
|
if srcrev_policy == "cache":
|
||||||
|
@ -181,7 +182,7 @@ def fetcher_compare_revisons(d):
|
||||||
return true/false on whether they've changed.
|
return true/false on whether they've changed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
pd = persist_data.PersistData(d)
|
pd = persist_data.PersistData(d, persistent_database_connection)
|
||||||
data = pd.getKeyValues("BB_URI_HEADREVS")
|
data = pd.getKeyValues("BB_URI_HEADREVS")
|
||||||
data2 = bb.fetch.saved_headrevs
|
data2 = bb.fetch.saved_headrevs
|
||||||
|
|
||||||
|
@ -673,7 +674,7 @@ class Fetch(object):
|
||||||
if not hasattr(self, "_latest_revision"):
|
if not hasattr(self, "_latest_revision"):
|
||||||
raise ParameterError
|
raise ParameterError
|
||||||
|
|
||||||
pd = persist_data.PersistData(d)
|
pd = persist_data.PersistData(d, persistent_database_connection)
|
||||||
key = self.generate_revision_key(url, ud, d)
|
key = self.generate_revision_key(url, ud, d)
|
||||||
rev = pd.getValue("BB_URI_HEADREVS", key)
|
rev = pd.getValue("BB_URI_HEADREVS", key)
|
||||||
if rev != None:
|
if rev != None:
|
||||||
|
@ -690,7 +691,7 @@ class Fetch(object):
|
||||||
if hasattr(self, "_sortable_revision"):
|
if hasattr(self, "_sortable_revision"):
|
||||||
return self._sortable_revision(url, ud, d)
|
return self._sortable_revision(url, ud, d)
|
||||||
|
|
||||||
pd = persist_data.PersistData(d)
|
pd = persist_data.PersistData(d, persistent_database_connection)
|
||||||
key = self.generate_revision_key(url, ud, d)
|
key = self.generate_revision_key(url, ud, d)
|
||||||
|
|
||||||
latest_rev = self._build_revision(url, ud, d)
|
latest_rev = self._build_revision(url, ud, d)
|
||||||
|
|
|
@ -43,7 +43,10 @@ class PersistData:
|
||||||
|
|
||||||
Why sqlite? It handles all the locking issues for us.
|
Why sqlite? It handles all the locking issues for us.
|
||||||
"""
|
"""
|
||||||
def __init__(self, d):
|
def __init__(self, d, persistent_database_connection):
|
||||||
|
if "connection" in persistent_database_connection:
|
||||||
|
self.cursor = persistent_database_connection["connection"].cursor()
|
||||||
|
return
|
||||||
self.cachedir = bb.data.getVar("PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True)
|
self.cachedir = bb.data.getVar("PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True)
|
||||||
if self.cachedir in [None, '']:
|
if self.cachedir in [None, '']:
|
||||||
bb.msg.fatal(bb.msg.domain.PersistData, "Please set the 'PERSISTENT_DIR' or 'CACHE' variable.")
|
bb.msg.fatal(bb.msg.domain.PersistData, "Please set the 'PERSISTENT_DIR' or 'CACHE' variable.")
|
||||||
|
@ -55,27 +58,29 @@ class PersistData:
|
||||||
self.cachefile = os.path.join(self.cachedir, "bb_persist_data.sqlite3")
|
self.cachefile = os.path.join(self.cachedir, "bb_persist_data.sqlite3")
|
||||||
bb.msg.debug(1, bb.msg.domain.PersistData, "Using '%s' as the persistent data cache" % self.cachefile)
|
bb.msg.debug(1, bb.msg.domain.PersistData, "Using '%s' as the persistent data cache" % self.cachefile)
|
||||||
|
|
||||||
self.connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None)
|
connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None)
|
||||||
|
persistent_database_connection["connection"] = connection
|
||||||
|
self.cursor = persistent_database_connection["connection"].cursor()
|
||||||
|
|
||||||
def addDomain(self, domain):
|
def addDomain(self, domain):
|
||||||
"""
|
"""
|
||||||
Should be called before any domain is used
|
Should be called before any domain is used
|
||||||
Creates it if it doesn't exist.
|
Creates it if it doesn't exist.
|
||||||
"""
|
"""
|
||||||
self.connection.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain)
|
self.cursor.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain)
|
||||||
|
|
||||||
def delDomain(self, domain):
|
def delDomain(self, domain):
|
||||||
"""
|
"""
|
||||||
Removes a domain and all the data it contains
|
Removes a domain and all the data it contains
|
||||||
"""
|
"""
|
||||||
self.connection.execute("DROP TABLE IF EXISTS %s;" % domain)
|
self.cursor.execute("DROP TABLE IF EXISTS %s;" % domain)
|
||||||
|
|
||||||
def getKeyValues(self, domain):
|
def getKeyValues(self, domain):
|
||||||
"""
|
"""
|
||||||
Return a list of key + value pairs for a domain
|
Return a list of key + value pairs for a domain
|
||||||
"""
|
"""
|
||||||
ret = {}
|
ret = {}
|
||||||
data = self.connection.execute("SELECT key, value from %s;" % domain)
|
data = self.cursor.execute("SELECT key, value from %s;" % domain)
|
||||||
for row in data:
|
for row in data:
|
||||||
ret[str(row[0])] = str(row[1])
|
ret[str(row[0])] = str(row[1])
|
||||||
|
|
||||||
|
@ -85,7 +90,7 @@ class PersistData:
|
||||||
"""
|
"""
|
||||||
Return the value of a key for a domain
|
Return the value of a key for a domain
|
||||||
"""
|
"""
|
||||||
data = self.connection.execute("SELECT * from %s where key=?;" % domain, [key])
|
data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key])
|
||||||
for row in data:
|
for row in data:
|
||||||
return row[1]
|
return row[1]
|
||||||
|
|
||||||
|
@ -93,7 +98,7 @@ class PersistData:
|
||||||
"""
|
"""
|
||||||
Sets the value of a key for a domain
|
Sets the value of a key for a domain
|
||||||
"""
|
"""
|
||||||
data = self.connection.execute("SELECT * from %s where key=?;" % domain, [key])
|
data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key])
|
||||||
rows = 0
|
rows = 0
|
||||||
for row in data:
|
for row in data:
|
||||||
rows = rows + 1
|
rows = rows + 1
|
||||||
|
@ -111,7 +116,7 @@ class PersistData:
|
||||||
def _execute(self, *query):
|
def _execute(self, *query):
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
self.connection.execute(*query)
|
self.cursor.execute(*query)
|
||||||
return
|
return
|
||||||
except sqlite3.OperationalError as e:
|
except sqlite3.OperationalError as e:
|
||||||
if 'database is locked' in str(e):
|
if 'database is locked' in str(e):
|
||||||
|
|
Loading…
Reference in New Issue