sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

Add a new task checklicense and fix some bugs in distro_check.py 

distro_check.py: Create a new function called create_log_file to reduce a lot of repeat code in distrodata.bbclass.
                 We needn't to create log file in function save_distro_check_result, because the log file has been generated in check_eventhandler.
                 Another bug is that we maybe access the /tmp/Meego-1.0 before we create this file.
                 Add a judge statement to decide whether we need to create this file firstly.
distrodata.bbclass: Add a new task checklicense to collect missing text license information.
                    This can help package-report system to know how many recipes are missing license text.

(From OE-Core rev: b41148cda9f0cc292b662a8473f26bc1ee0148f3)

Signed-off-by: Mei Lei <lei.mei@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Mei Lei 2011-05-16 19:06:21 +08:00 committed by Richard Purdie
parent afe43ed090
commit 60ab6fc60c
2 changed files with 118 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
meta/classes/distrodata.bbclass
View File

@ -4,19 +4,9 @@ addhandler distro_eventhandler
python distro_eventhandler() { python distro_eventhandler() {
if bb.event.getName(e) == "BuildStarted": if bb.event.getName(e) == "BuildStarted":
"""initialize log files.""" import oe.distro_check as dc
logpath = bb.data.getVar('LOG_DIR', e.data, 1) logfile = dc.create_log_file(e.data, "distrodata.csv")
bb.utils.mkdirhier(logpath) lf = bb.utils.lockfile("%s.lock" % logfile)
logfile = os.path.join(logpath, "distrodata.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
if not os.path.exists(logfile):
slogfile = os.path.join(logpath, "distrodata.csv")
if os.path.exists(slogfile):
os.remove(slogfile)
os.system("touch %s" % logfile)
os.symlink(logfile, slogfile)
bb.data.setVar('LOG_FILE', logfile, e.data)
lf = bb.utils.lockfile(logfile + ".lock")
f = open(logfile, "a") f = open(logfile, "a")
f.write("Package,Description,Owner,License,ChkSum,Status,VerMatch,Version,Upsteam,Non-Update,Reason,Recipe Status\n") f.write("Package,Description,Owner,License,ChkSum,Status,VerMatch,Version,Upsteam,Non-Update,Reason,Recipe Status\n")
f.close() f.close()
@ -33,9 +23,9 @@ python do_distrodata_np() {
bb.note("Package Name: %s" % pn) bb.note("Package Name: %s" % pn)
import oe.distro_check as dist_check import oe.distro_check as dist_check
tmpdir = bb.data.getVar('TMPDIR', d, 1) tmpdir = bb.data.getVar('TMPDIR', d, True)
distro_check_dir = os.path.join(tmpdir, "distro_check") distro_check_dir = os.path.join(tmpdir, "distro_check")
datetime = bb.data.getVar('DATETIME', localdata, 1) datetime = bb.data.getVar('DATETIME', localdata, True)
dist_check.update_distro_data(distro_check_dir, datetime) dist_check.update_distro_data(distro_check_dir, datetime)
if pn.find("-native") != -1: if pn.find("-native") != -1:
@ -111,15 +101,15 @@ python do_distrodata_np() {
addtask distrodata addtask distrodata
do_distrodata[nostamp] = "1" do_distrodata[nostamp] = "1"
python do_distrodata() { python do_distrodata() {
logpath = bb.data.getVar('LOG_DIR', d, 1) logpath = bb.data.getVar('LOG_DIR', d, True)
bb.utils.mkdirhier(logpath) bb.utils.mkdirhier(logpath)
logfile = os.path.join(logpath, "distrodata.csv") logfile = os.path.join(logpath, "distrodata.csv")
import oe.distro_check as dist_check import oe.distro_check as dist_check
localdata = bb.data.createCopy(d) localdata = bb.data.createCopy(d)
tmpdir = bb.data.getVar('TMPDIR', d, 1) tmpdir = bb.data.getVar('TMPDIR', d, True)
distro_check_dir = os.path.join(tmpdir, "distro_check") distro_check_dir = os.path.join(tmpdir, "distro_check")
datetime = bb.data.getVar('DATETIME', localdata, 1) datetime = bb.data.getVar('DATETIME', localdata, True)
dist_check.update_distro_data(distro_check_dir, datetime) dist_check.update_distro_data(distro_check_dir, datetime)
pn = bb.data.getVar("PN", d, True) pn = bb.data.getVar("PN", d, True)
@ -189,7 +179,7 @@ python do_distrodata() {
# do the comparison # do the comparison
result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata) result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata)
lf = bb.utils.lockfile(logfile + ".lock") lf = bb.utils.lockfile("%s.lock" % logfile)
f = open(logfile, "a") f = open(logfile, "a")
f.write("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s," % \ f.write("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s," % \
(pname, pdesc, maintainer, plicense, pchksum, hasrstatus, vermatch, pcurver, pupver, noupdate, noupdate_reason, rstatus)) (pname, pdesc, maintainer, plicense, pchksum, hasrstatus, vermatch, pcurver, pupver, noupdate, noupdate_reason, rstatus))
@ -211,19 +201,10 @@ do_distrodataall() {
addhandler checkpkg_eventhandler addhandler checkpkg_eventhandler
python checkpkg_eventhandler() { python checkpkg_eventhandler() {
if bb.event.getName(e) == "BuildStarted": if bb.event.getName(e) == "BuildStarted":
"""initialize log files.""" import oe.distro_check as dc
logpath = bb.data.getVar('LOG_DIR', e.data, 1) logfile = dc.create_log_file(e.data, "checkpkg.csv")
bb.utils.mkdirhier(logpath)
logfile = os.path.join(logpath, "checkpkg.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
if not os.path.exists(logfile):
slogfile = os.path.join(logpath, "checkpkg.csv")
if os.path.exists(slogfile):
os.remove(slogfile)
os.system("touch %s" % logfile)
os.symlink(logfile, slogfile)
bb.data.setVar('LOG_FILE', logfile, e.data)
lf = bb.utils.lockfile(logfile + ".lock") lf = bb.utils.lockfile("%s.lock" % logfile)
f = open(logfile, "a") f = open(logfile, "a")
f.write("Package\tVersion\tUpver\tLicense\tSection\tHome\tRelease\tPriority\tDepends\tBugTracker\tPE\tDescription\tStatus\tTracking\tURI\tMAINTAINER\n") f.write("Package\tVersion\tUpver\tLicense\tSection\tHome\tRelease\tPriority\tDepends\tBugTracker\tPE\tDescription\tStatus\tTracking\tURI\tMAINTAINER\n")
f.close() f.close()
@ -304,7 +285,7 @@ python do_checkpkg() {
Clear internal url cache as it's a temporary check. Not doing so will have Clear internal url cache as it's a temporary check. Not doing so will have
bitbake check url multiple times when looping through a single url bitbake check url multiple times when looping through a single url
""" """
fn = bb.data.getVar('FILE', d, 1) fn = bb.data.getVar('FILE', d, True)
bb.fetch2.urldata_cache[fn] = {} bb.fetch2.urldata_cache[fn] = {}
""" """
@ -335,7 +316,7 @@ python do_checkpkg() {
Return new version if success, or else error in "Errxxxx" style Return new version if success, or else error in "Errxxxx" style
""" """
def check_new_dir(url, curver, d): def check_new_dir(url, curver, d):
pn = bb.data.getVar('PN', d, 1) pn = bb.data.getVar('PN', d, True)
f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-1-" % pn) f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-1-" % pn)
status = internal_fetch_wget(url, d, f) status = internal_fetch_wget(url, d, f)
fhtml = f.read() fhtml = f.read()
@ -394,7 +375,7 @@ python do_checkpkg() {
"""possible to have no version in pkg name, such as spectrum-fw""" """possible to have no version in pkg name, such as spectrum-fw"""
if not re.search("\d+", curname): if not re.search("\d+", curname):
return pcurver return pcurver
pn = bb.data.getVar('PN', d, 1) pn = bb.data.getVar('PN', d, True)
f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-2-" % pn) f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-2-" % pn)
status = internal_fetch_wget(url, d, f) status = internal_fetch_wget(url, d, f)
fhtml = f.read() fhtml = f.read()
@ -437,35 +418,35 @@ python do_checkpkg() {
f.close() f.close()
"""if host hasn't directory information, no need to save tmp file""" """if host hasn't directory information, no need to save tmp file"""
if status != "ErrHostNoDir" and re.match("Err", status): if status != "ErrHostNoDir" and re.match("Err", status):
logpath = bb.data.getVar('LOG_DIR', d, 1) logpath = bb.data.getVar('LOG_DIR', d, True)
os.system("cp %s %s/" % (f.name, logpath)) os.system("cp %s %s/" % (f.name, logpath))
os.unlink(f.name) os.unlink(f.name)
return status return status
"""first check whether a uri is provided""" """first check whether a uri is provided"""
src_uri = bb.data.getVar('SRC_URI', d, 1) src_uri = bb.data.getVar('SRC_URI', d, True)
if not src_uri: if not src_uri:
return return
"""initialize log files.""" """initialize log files."""
logpath = bb.data.getVar('LOG_DIR', d, 1) logpath = bb.data.getVar('LOG_DIR', d, True)
bb.utils.mkdirhier(logpath) bb.utils.mkdirhier(logpath)
logfile = os.path.join(logpath, "checkpkg.csv") logfile = os.path.join(logpath, "checkpkg.csv")
"""generate package information from .bb file""" """generate package information from .bb file"""
pname = bb.data.getVar('PN', d, 1) pname = bb.data.getVar('PN', d, True)
pdesc = bb.data.getVar('DESCRIPTION', d, 1) pdesc = bb.data.getVar('DESCRIPTION', d, True)
pgrp = bb.data.getVar('SECTION', d, 1) pgrp = bb.data.getVar('SECTION', d, True)
pversion = bb.data.getVar('PV', d, 1) pversion = bb.data.getVar('PV', d, True)
plicense = bb.data.getVar('LICENSE',d,1) plicense = bb.data.getVar('LICENSE', d, True)
psection = bb.data.getVar('SECTION',d,1) psection = bb.data.getVar('SECTION', d, True)
phome = bb.data.getVar('HOMEPAGE', d, 1) phome = bb.data.getVar('HOMEPAGE', d, True)
prelease = bb.data.getVar('PR',d,1) prelease = bb.data.getVar('PR', d, True)
ppriority = bb.data.getVar('PRIORITY',d,1) ppriority = bb.data.getVar('PRIORITY', d, True)
pdepends = bb.data.getVar('DEPENDS',d,1) pdepends = bb.data.getVar('DEPENDS', d, True)
pbugtracker = bb.data.getVar('BUGTRACKER',d,1) pbugtracker = bb.data.getVar('BUGTRACKER', d, True)
ppe = bb.data.getVar('PE',d,1) ppe = bb.data.getVar('PE', d, True)
psrcuri = bb.data.getVar('SRC_URI',d,1) psrcuri = bb.data.getVar('SRC_URI', d, True)
found = 0 found = 0
for uri in src_uri.split(): for uri in src_uri.split():
@ -483,9 +464,9 @@ python do_checkpkg() {
(type, host, path, user, pswd, parm) = bb.decodeurl(uri) (type, host, path, user, pswd, parm) = bb.decodeurl(uri)
if type in ['http', 'https', 'ftp']: if type in ['http', 'https', 'ftp']:
pcurver = bb.data.getVar('PV', d, 1) pcurver = bb.data.getVar('PV', d, True)
else: else:
pcurver = bb.data.getVar("SRCREV", d, 1) pcurver = bb.data.getVar("SRCREV", d, True)
if type in ['http', 'https', 'ftp']: if type in ['http', 'https', 'ftp']:
newver = pcurver newver = pcurver
@ -509,7 +490,7 @@ python do_checkpkg() {
newver = check_new_dir(alturi, dirver, d) newver = check_new_dir(alturi, dirver, d)
altpath = path altpath = path
if not re.match("Err", newver) and dirver != newver: if not re.match("Err", newver) and dirver != newver:
altpath = altpath.replace(dirver, newver, 1) altpath = altpath.replace(dirver, newver, True)
"""Now try to acquire all remote files in current directory""" """Now try to acquire all remote files in current directory"""
if not re.match("Err", newver): if not re.match("Err", newver):
@ -625,7 +606,7 @@ python do_checkpkg() {
pstatus += ":%s%s" % (host, path) pstatus += ":%s%s" % (host, path)
"""Read from manual distro tracking fields as alternative""" """Read from manual distro tracking fields as alternative"""
pmver = bb.data.getVar("RECIPE_LATEST_VERSION", d, 1) pmver = bb.data.getVar("RECIPE_LATEST_VERSION", d, True)
if not pmver: if not pmver:
pmver = "N/A" pmver = "N/A"
pmstatus = "ErrNoRecipeData" pmstatus = "ErrNoRecipeData"
@ -639,7 +620,7 @@ python do_checkpkg() {
psrcuri = psrcuri.split()[0] psrcuri = psrcuri.split()[0]
pdepends = "".join(pdepends.split("\t")) pdepends = "".join(pdepends.split("\t"))
pdesc = "".join(pdesc.split("\t")) pdesc = "".join(pdesc.split("\t"))
lf = bb.utils.lockfile(logfile + ".lock") lf = bb.utils.lockfile("%s.lock" % logfile)
f = open(logfile, "a") f = open(logfile, "a")
f.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % \ f.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % \
(pname,pversion,pupver,plicense,psection, phome,prelease, ppriority,pdepends,pbugtracker,ppe,pdesc,pstatus,pmver,psrcuri,maintainer)) (pname,pversion,pupver,plicense,psection, phome,prelease, ppriority,pdepends,pbugtracker,ppe,pdesc,pstatus,pmver,psrcuri,maintainer))
@ -654,25 +635,12 @@ do_checkpkgall() {
: :
} }
#addhandler check_eventhandler addhandler distro_check_eventhandler
python check_eventhandler() { python distro_check_eventhandler() {
if bb.event.getName(e) == "BuildStarted": if bb.event.getName(e) == "BuildStarted":
import oe.distro_check as dc
tmpdir = bb.data.getVar('TMPDIR', e.data, 1)
distro_check_dir = os.path.join(tmpdir, "distro_check")
datetime = bb.data.getVar('DATETIME', e.data, 1)
"""initialize log files.""" """initialize log files."""
logpath = bb.data.getVar('LOG_DIR', e.data, 1) import oe.distro_check as dc
bb.utils.mkdirhier(logpath) result_file = dc.create_log_file(e.data, "distrocheck.csv")
logfile = os.path.join(logpath, "distrocheck.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
if not os.path.exists(logfile):
slogfile = os.path.join(logpath, "distrocheck.csv")
if os.path.exists(slogfile):
os.remove(slogfile)
os.system("touch %s" % logfile)
os.symlink(logfile, slogfile)
bb.data.setVar('LOG_FILE', logfile, e.data)
return return
} }
@ -681,18 +649,23 @@ do_distro_check[nostamp] = "1"
python do_distro_check() { python do_distro_check() {
"""checks if the package is present in other public Linux distros""" """checks if the package is present in other public Linux distros"""
import oe.distro_check as dc import oe.distro_check as dc
import bb
import shutil
localdata = bb.data.createCopy(d) localdata = bb.data.createCopy(d)
bb.data.update_data(localdata) bb.data.update_data(localdata)
tmpdir = bb.data.getVar('TMPDIR', d, 1) tmpdir = bb.data.getVar('TMPDIR', d, True)
distro_check_dir = os.path.join(tmpdir, "distro_check") distro_check_dir = os.path.join(tmpdir, "distro_check")
datetime = bb.data.getVar('DATETIME', localdata, 1) logpath = bb.data.getVar('LOG_DIR', d, True)
bb.utils.mkdirhier(logpath)
result_file = os.path.join(logpath, "distrocheck.csv")
datetime = bb.data.getVar('DATETIME', localdata, True)
dc.update_distro_data(distro_check_dir, datetime) dc.update_distro_data(distro_check_dir, datetime)
# do the comparison # do the comparison
result = dc.compare_in_distro_packages_list(distro_check_dir, d) result = dc.compare_in_distro_packages_list(distro_check_dir, d)
# save the results # save the results
dc.save_distro_check_result(result, datetime, d) dc.save_distro_check_result(result, datetime, result_file, d)
} }
addtask distro_checkall after do_distro_check addtask distro_checkall after do_distro_check
@ -701,3 +674,55 @@ do_distro_checkall[nostamp] = "1"
do_distro_checkall() { do_distro_checkall() {
: :
} }
#
#Check Missing License Text.
#Use this task to generate the missing license text data for pkg-report system,
#then we can search those recipes which license text isn't exsit in common-licenses directory
#
addhandler checklicense_eventhandler
python checklicense_eventhandler() {
if bb.event.getName(e) == "BuildStarted":
"""initialize log files."""
import oe.distro_check as dc
logfile = dc.create_log_file(e.data, "missinglicense.csv")
lf = bb.utils.lockfile("%s.lock" % logfile)
f = open(logfile, "a")
f.write("Package\tLicense\tMissingLicense\n")
f.close()
bb.utils.unlockfile(lf)
return
}
addtask checklicense
do_checklicense[nostamp] = "1"
python do_checklicense() {
import os
import bb
import shutil
logpath = bb.data.getVar('LOG_DIR', d, True)
bb.utils.mkdirhier(logpath)
pn = bb.data.getVar('PN', d, True)
logfile = os.path.join(logpath, "missinglicense.csv")
generic_directory = bb.data.getVar('COMMON_LICENSE_DIR', d, True)
license_types = bb.data.getVar('LICENSE', d, True)
for license_type in ((license_types.replace('+', '').replace('|', '&')
.replace('(', '').replace(')', '').replace(';', '')
.replace(',', '').replace(" ", "").split("&"))):
if not os.path.isfile(os.path.join(generic_directory, license_type)):
lf = bb.utils.lockfile("%s.lock" % logfile)
f = open(logfile, "a")
f.write("%s\t%s\t%s\n" % \
(pn,license_types,license_type))
f.close()
bb.utils.unlockfile(lf)
return
}
addtask checklicenseall after do_checklicense
do_checklicenseall[recrdeptask] = "do_checklicense"
do_checklicenseall[nostamp] = "1"
do_checklicenseall() {
:
}

27
meta/lib/oe/distro_check.py
View File

@ -73,7 +73,8 @@ def clean_package_list(package_list):
def get_latest_released_meego_source_package_list(): def get_latest_released_meego_source_package_list():
"Returns list of all the name os packages in the latest meego distro" "Returns list of all the name os packages in the latest meego distro"
if not os.path.isfile("/tmp/Meego-1.0"):
os.mknod("/tmp/Meego-1.0")
f = open("/tmp/Meego-1.0", "r") f = open("/tmp/Meego-1.0", "r")
package_names = [] package_names = []
for line in f: for line in f:
@ -341,7 +342,22 @@ def compare_in_distro_packages_list(distro_check_dir, d):
bb.note("Matching: %s" % matching_distros) bb.note("Matching: %s" % matching_distros)
return matching_distros return matching_distros
def save_distro_check_result(result, datetime, d): def create_log_file(d, logname):
logpath = bb.data.getVar('LOG_DIR', d, True)
bb.utils.mkdirhier(logpath)
logfn, logsuffix = os.path.splitext(logname)
logfile = os.path.join(logpath, "%s.%s%s" % (logfn, bb.data.getVar('DATETIME', d, True), logsuffix))
if not os.path.exists(logfile):
slogfile = os.path.join(logpath, logname)
if os.path.exists(slogfile):
os.remove(slogfile)
os.system("touch %s" % logfile)
os.symlink(logfile, slogfile)
bb.data.setVar('LOG_FILE', logfile, d)
return logfile
def save_distro_check_result(result, datetime, result_file, d):
pn = bb.data.getVar('PN', d, True) pn = bb.data.getVar('PN', d, True)
logdir = bb.data.getVar('LOG_DIR', d, True) logdir = bb.data.getVar('LOG_DIR', d, True)
if not logdir: if not logdir:
@ -349,16 +365,9 @@ def save_distro_check_result(result, datetime, d):
return return
if not os.path.isdir(logdir): if not os.path.isdir(logdir):
os.makedirs(logdir) os.makedirs(logdir)
result_file = os.path.join(logdir, "distrocheck.%s.csv" % datetime)
line = pn line = pn
for i in result: for i in result:
line = line + "," + i line = line + "," + i
if not os.path.exists(result_file):
sresult_file = os.path.join(logdir, "distrocheck.csv")
if os.path.exists(sresult_file):
os.remove(sresult_file)
os.system("touch %s" % result_file)
os.symlink(result_file, sresult_file)
f = open(result_file, "a") f = open(result_file, "a")
import fcntl import fcntl
fcntl.lockf(f, fcntl.LOCK_EX) fcntl.lockf(f, fcntl.LOCK_EX)