sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

buildhistory: improve performance of image info collection 

Reduce the number of calls to the packaging tool, especially in the case
of rpm, using helper utilities to gather the required information more
efficiently where possible.

(From OE-Core rev: d0b8a98c5b46c305afd389fc862b3bf0c6f1eaab)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Paul Eggleton 2012-07-23 07:59:11 +01:00 committed by Richard Purdie
parent 98b0f956e7
commit 20ad566d60
5 changed files with 162 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
meta/classes/buildhistory.bbclass
View File

@ -285,48 +285,43 @@ buildhistory_get_image_installed() {
mkdir -p ${BUILDHISTORY_DIR_IMAGE}
# Get list of installed packages
list_installed_packages | sort > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt
INSTALLED_PKGS=`cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt`
pkgcache="${BUILDHISTORY_DIR_IMAGE}/installed-packages.tmp"
list_installed_packages file | sort > $pkgcache
# Produce installed package file and size lists and dependency graph
echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
cat $pkgcache | awk '{ print $1 }' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt
cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
# Produce dependency graph
# First, filter out characters that cause issues for dot
rootfs_list_installed_depends | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' > ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
# Change delimiter from pipe to -> and set style for recommend lines
sed -i -e 's:|: -> :' -e 's:\[REC\]:[style=dotted]:' -e 's:$:;:' ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
# Add header, sorted and de-duped contents and footer and then delete the temp file
echo -e "digraph depends {\n node [shape=plaintext]" > ${BUILDHISTORY_DIR_IMAGE}/depends.dot
for pkg in $INSTALLED_PKGS; do
pkgfile=`get_package_filename $pkg`
echo `basename $pkgfile` >> ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
cat ${BUILDHISTORY_DIR_IMAGE}/depends.tmp | sort | uniq >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
echo "}" >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
rm ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
# Produce installed package sizes list
echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
cat $pkgcache | while read pkg pkgfile
do
if [ -f $pkgfile ] ; then
pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'`
echo $pkgsize $pkg >> ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
fi
deps=`list_package_depends $pkg`
for dep in $deps ; do
echo "$pkg OPP $dep;" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g'
done
recs=`list_package_recommends $pkg`
for rec in $recs ; do
echo "$pkg OPP $rec [style=dotted];" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g'
done
done | sort | uniq >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
echo "}" >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
done
cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.txt
rm ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
# We're now done with the cache, delete it
rm $pkgcache
# Produce some cut-down graphs (for readability)
grep -v kernel_image ${BUILDHISTORY_DIR_IMAGE}/depends.dot | grep -v kernel_2 | grep -v kernel_3 > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot
grep -v libc6 ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot | grep -v libgcc > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot
grep -v update_ ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot
grep -v kernel_module ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate-nomodules.dot
# Workaround for broken shell function dependencies
if false ; then
get_package_filename
list_package_depends
list_package_recommends
fi
}
buildhistory_get_imageinfo() {

28
meta/classes/rootfs_deb.bbclass
View File

@ -96,26 +96,24 @@ list_installed_packages() {
if [ "$1" = "arch" ] ; then
# Here we want the PACKAGE_ARCH not the deb architecture
${DPKG_QUERY_COMMAND} -W -f='${Package} ${PackageArch}\n'
elif [ "$1" = "file" ] ; then
${DPKG_QUERY_COMMAND} -W -f='${Package} ${Package}_${Version}_${Architecture}.deb\n' | while read pkg pkgfile
do
fullpath=`find ${DEPLOY_DIR_DEB} -name "$pkgfile" || true`
if [ "$fullpath" = "" ] ; then
echo "$pkg $pkgfile"
else
echo "$pkg $fullpath"
fi
done
else
${DPKG_QUERY_COMMAND} -W -f='${Package}\n'
fi
}
get_package_filename() {
fullname=`find ${DEPLOY_DIR_DEB} -name "$1_*.deb" || true`
if [ "$fullname" = "" ] ; then
echo $name
else
echo $fullname
fi
}
list_package_depends() {
${DPKG_QUERY_COMMAND} -s $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
}
list_package_recommends() {
${DPKG_QUERY_COMMAND} -s $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
rootfs_list_installed_depends() {
# Cheat here a little bit by using the opkg query helper util
${DPKG_QUERY_COMMAND} -W -f='Package: ${Package}\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n' | opkg-query-helper.py
}
rootfs_install_packages() {

34
meta/classes/rootfs_ipk.bbclass
View File

@ -131,33 +131,23 @@ remove_packaging_data_files() {
list_installed_packages() {
if [ "$1" = "arch" ] ; then
opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py -a
elif [ "$1" = "file" ] ; then
opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py -f | while read pkg pkgfile
do
fullpath=`find ${DEPLOY_DIR_IPK} -name "$pkgfile" || true`
if [ "$fullpath" = "" ] ; then
echo "$pkg $pkgfile"
else
echo "$pkg $fullpath"
fi
done
else
opkg-cl ${IPKG_ARGS_POST} list_installed | awk '{ print $1 }'
fi
}
get_package_filename() {
set +x
info=`opkg-cl ${IPKG_ARGS_POST} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" || true`
name=`echo "${info}" | awk '/^Package/ {printf $2"_"}'`
name=$name`echo "${info}" | awk -F: '/^Version/ {printf $NF"_"}' | sed 's/^\s*//g'`
name=$name`echo "${info}" | awk '/^Archi/ {print $2".ipk"}'`
set -x
fullname=`find ${DEPLOY_DIR_IPK} -name "$name" || true`
if [ "$fullname" = "" ] ; then
echo $name
else
echo $fullname
fi
}
list_package_depends() {
opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
}
list_package_recommends() {
opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
rootfs_list_installed_depends() {
opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py
}
rootfs_install_packages() {

33
meta/classes/rootfs_rpm.bbclass
View File

@ -143,40 +143,15 @@ RPM_QUERY_CMD = '${RPM} --root $INSTALL_ROOTFS_RPM -D "_dbpath ${rpmlibdir}" \
list_installed_packages() {
if [ "$1" = "arch" ] ; then
${RPM_QUERY_CMD} -qa --qf "[%{NAME} %{ARCH}\n]"
elif [ "$1" = "file" ] ; then
${RPM_QUERY_CMD} -qa --qf "[%{NAME} %{PACKAGEORIGIN}\n]"
else
${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]"
fi
}
get_package_filename() {
resolve_package_rpm ${RPMCONF_TARGET_BASE}-base_archs.conf $1
}
list_package_depends() {
pkglist=`list_installed_packages`
# REQUIRE* lists "soft" requirements (which we know as recommends and RPM refers to
# as "suggests") so filter these out with the help of awk
for req in `${RPM_QUERY_CMD} -q --qf "[%{REQUIRENAME} %{REQUIREFLAGS}\n]" $1 | awk '{ if( and($2, 0x80000) == 0) print $1 }'`; do
if echo "$req" | grep -q "^rpmlib" ; then continue ; fi
realpkg=""
for dep in $pkglist; do
if [ "$dep" = "$req" ] ; then
realpkg="1"
echo $req
break
fi
done
if [ "$realdep" = "" ] ; then
${RPM_QUERY_CMD} -q --whatprovides $req --qf "%{NAME}\n"
fi
done
}
list_package_recommends() {
${RPM_QUERY_CMD} -q --suggests $1
rootfs_list_installed_depends() {
rpmresolve -d $INSTALL_ROOTFS_RPM/${rpmlibdir}
}
rootfs_install_packages() {

117
meta/recipes-devtools/rpm/rpmresolve/rpmresolve.c
View File

@ -204,10 +204,104 @@ int processPackages(rpmts *ts, int tscount, const char *packagelistfn, int ignor
return rc;
}
int lookupProvider(rpmts ts, const char *req, char **provider)
{
int rc = 0;
rpmmi provmi = rpmtsInitIterator(ts, RPMTAG_PROVIDENAME, req, 0);
if(provmi) {
Header h;
if ((h = rpmmiNext(provmi)) != NULL) {
HE_t he = (HE_t) memset(alloca(sizeof(*he)), 0, sizeof(*he));
he->tag = RPMTAG_NAME;
rc = (headerGet(h, he, 0) != 1);
if(rc==0)
*provider = strdup((char *)he->p.ptr);
}
(void)rpmmiFree(provmi);
}
else {
rc = -1;
}
return rc;
}
int printDepList(rpmts *ts, int tscount)
{
int rc = 0;
if( tscount > 1 )
printf(">1 database specified with dependency list, using first only\n");
/* Get list of names */
rpmdb db = rpmtsGetRdb(ts[0]);
ARGV_t names = NULL;
rc = rpmdbMireApply(db, RPMTAG_NAME,
RPMMIRE_STRCMP, NULL, &names);
int nnames = argvCount(names);
/* Get list of NVRAs */
ARGV_t keys = NULL;
rc = rpmdbMireApply(db, RPMTAG_NVRA,
RPMMIRE_STRCMP, NULL, &keys);
if (keys) {
int i, j;
HE_t he = (HE_t) memset(alloca(sizeof(*he)), 0, sizeof(*he));
int nkeys = argvCount(keys);
for(i=0; i<nkeys; i++) {
rpmmi mi = rpmtsInitIterator(ts[0], RPMTAG_NVRA, keys[i], 0);
Header h;
if ((h = rpmmiNext(mi)) != NULL) {
/* Get name of package */
he->tag = RPMTAG_NAME;
rc = (headerGet(h, he, 0) != 1);
char *name = strdup((char *)he->p.ptr);
/* Get its requires */
he->tag = RPMTAG_REQUIRENAME;
rc = (headerGet(h, he, 0) != 1);
ARGV_t reqs = (ARGV_t)he->p.ptr;
/* Get its requireflags */
he->tag = RPMTAG_REQUIREFLAGS;
rc = (headerGet(h, he, 0) != 1);
rpmuint32_t *reqflags = (rpmuint32_t *)he->p.ui32p;
for(j=0; j<he->c; j++) {
int k;
char *prov = NULL;
for(k=0; k<nnames; k++) {
if(strcmp(names[k], reqs[j]) == 0) {
prov = names[k];
break;
}
}
if(prov) {
if((int)reqflags[j] & 0x80000)
printf("%s|%s [REC]\n", name, prov);
else
printf("%s|%s\n", name, prov);
}
else {
rc = lookupProvider(ts[0], reqs[j], &prov);
if(rc==0 && prov) {
if((int)reqflags[j] & 0x80000)
printf("%s|%s [REC]\n", name, prov);
else
printf("%s|%s\n", name, prov);
free(prov);
}
}
}
free(name);
}
(void)rpmmiFree(mi);
}
}
return rc;
}
void usage()
{
fprintf(stderr, "OpenEmbedded rpm resolver utility\n");
fprintf(stderr, "syntax: rpmresolve [-i] <dblistfile> <packagelistfile>\n");
fprintf(stderr, "syntax: rpmresolve [-i] [-d] <dblistfile> <packagelistfile>\n");
}
int main(int argc, char **argv)
@ -218,13 +312,17 @@ int main(int argc, char **argv)
int i;
int c;
int ignoremissing = 0;
int deplistmode = 0;
opterr = 0;
while ((c = getopt (argc, argv, "i")) != -1) {
while ((c = getopt (argc, argv, "id")) != -1) {
switch (c) {
case 'i':
ignoremissing = 1;
break;
case 'd':
deplistmode = 1;
break;
case '?':
if(isprint(optopt))
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
@ -258,12 +356,17 @@ int main(int argc, char **argv)
return 1;
}
if( argc - optind < 2 ) {
fprintf(stderr, "Please specify package list file\n");
return 1;
if(deplistmode) {
rc = printDepList(ts, tscount);
}
else {
if( argc - optind < 2 ) {
fprintf(stderr, "Please specify package list file\n");
return 1;
}
const char *pkglistfn = argv[optind+1];
rc = processPackages(ts, tscount, pkglistfn, ignoremissing);
}
const char *pkglistfn = argv[optind+1];
rc = processPackages(ts, tscount, pkglistfn, ignoremissing);
for(i=0; i<tscount; i++)
(void) rpmtsCloseDB(ts[i]);