Rework installation of dev, dbg, doc, and locale packages

Use a similar mechanism that was previously used to install locales at
rootfs generation time to install other "complementary" packages (e.g.
*-dev packages) - i.e. install all of the explicitly requested packages
and their dependencies, then get a list of the packages that were
installed, and use that list to install the complementary packages. This
has been implemented by using a list of globs which should make it
easier to extend in future.

The previous locale package installation code assumed that the locale
packages did not have any dependencies that were not already installed;
now that we are installing non-locale packages this is no longer
correct. In practice only the rpm backend actually made use of this
assumption, so it needed to be changed to call into the existing package
backend code to do the complementary package installation rather than
calling rpm directly.

This fixes the doc-pkgs IMAGE_FEATURES feature to work correctly, and
also ensures that all dev/dbg packages get installed for
dev-pkgs/dbg-pkgs respectively even if the dependency chains between
those packages was not ensuring that already.

The code has also been adapted to work correctly with the new
SDK-from-image functionality. To that end, an SDKIMAGE_FEATURES variable
has been added to allow specifying what extra image features should go
into the SDK (extra, because by virtue of installing all of the packages
in the image into the target part of the SDK, we already include all of
IMAGE_FEATURES) with a default value of "dev-pkgs dbg-pkgs".

Fixes [YOCTO #2614].

(From OE-Core rev: 72d1048a8381fa4a8c4c0d082047536727b4be47)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

meta/classes/image.bbclass

@@ -6,7 +6,8 @@ inherit imagetest-${IMAGETEST}
 inherit populate_sdk_base
 
 TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}"
-TOOLCHAIN_TARGET_TASK_ATTEMPTONLY += "${PACKAGE_INSTALL_ATTEMPTONLY} ${PACKAGE_GROUP_dev-pkgs} ${PACKAGE_GROUP_dbg-pkgs}"
+TOOLCHAIN_TARGET_TASK_ATTEMPTONLY += "${PACKAGE_INSTALL_ATTEMPTONLY}"
+POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_install_complementary populate_sdk; "
 
 inherit gzipnative
 
@@ -38,25 +39,23 @@ def normal_groups(d):
     features = set(oe.data.typed_value('IMAGE_FEATURES', d))
     return features.difference(extras)
 
-def normal_pkgs_to_install(d):
+# Wildcards specifying complementary packages to install for every package that has been explicitly
-    import oe.packagedata
+# installed into the rootfs
+def complementary_globs(featurevar, d):
+    globs = []
+    features = set((d.getVar(featurevar, True) or '').split())
+    for feature in features:
+        if feature == 'dev-pkgs':
+            globs.append('*-dev')
+        elif feature == 'doc-pkgs':
+            globs.append('*-doc')
+        elif feature == 'dbg-pkgs':
+            globs.append('*-dbg')
+    return ' '.join(globs)
 
-    to_install = oe.data.typed_value('IMAGE_INSTALL', d)
+IMAGE_INSTALL_COMPLEMENTARY = '${@complementary_globs("IMAGE_FEATURES", d)}'
-    features = normal_groups(d)
+SDKIMAGE_FEATURES ??= "dev-pkgs dbg-pkgs"
-    required = list(oe.packagegroup.required_packages(features, d))
+SDKIMAGE_INSTALL_COMPLEMENTARY = '${@complementary_globs("SDKIMAGE_FEATURES", d)}'
-    optional = list(oe.packagegroup.optional_packages(features, d))
-    all_packages = to_install + required + optional
-
-    recipes = filter(None, [oe.packagedata.recipename(pkg, d) for pkg in all_packages])
-
-    return all_packages + recipes
-
-PACKAGE_GROUP_dbg-pkgs = "${@' '.join('%s-dbg' % pkg for pkg in normal_pkgs_to_install(d))}"
-PACKAGE_GROUP_dbg-pkgs[optional] = "1"
-PACKAGE_GROUP_dev-pkgs = "${@' '.join('%s-dev' % pkg for pkg in normal_pkgs_to_install(d))}"
-PACKAGE_GROUP_dev-pkgs[optional] = "1"
-PACKAGE_GROUP_doc-pkgs = "${@' '.join('%s-doc' % pkg for pkg in normal_pkgs_to_install(d))}"
-PACKAGE_GROUP_doc-pkgs[optional] = "1"
 
 # "export IMAGE_BASENAME" not supported at this time
 IMAGE_INSTALL ?= ""
@@ -306,32 +305,44 @@ get_split_linguas() {
     done | sort | uniq
 }
 
-rootfs_install_all_locales() {
+rootfs_install_complementary() {
-    # Generate list of installed packages for which additional locale packages might be available
+    # Install complementary packages based upon the list of currently installed packages
-    INSTALLED_PACKAGES=`list_installed_packages | egrep -v -- "(-locale-|^locale-base-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"`
+    # e.g. locales, *-dev, *-dbg, etc. This will only attempt to install these packages,
+    # if they don't exist then no error will occur.
+    # Note: every backend needs to call this function explicitly after the normal
+    # package installation
 
-    # Generate a list of locale packages that exist
+    # Get list of installed packages
-    SPLIT_LINGUAS=`get_split_linguas`
+    list_installed_packages arch > ${WORKDIR}/installed_pkgs.txt
-    PACKAGES_TO_INSTALL=""
+
-    for lang in $SPLIT_LINGUAS; do
+    # Apply the globs to all the packages currently installed
-        for pkg in $INSTALLED_PACKAGES; do
+    if [ "$1" = "populate_sdk" ] ; then
-            existing_pkg=`rootfs_check_package_exists $pkg-locale-$lang`
+        GLOBS="${SDKIMAGE_INSTALL_COMPLEMENTARY}"
-            if [ "$existing_pkg" != "" ]; then
+    else
-                PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $existing_pkg"
+        GLOBS="${IMAGE_INSTALL_COMPLEMENTARY}"
-            fi
+        # Add locales
+        SPLIT_LINGUAS=`get_split_linguas`
+        PACKAGES_TO_INSTALL=""
+        for lang in $SPLIT_LINGUAS ; do
+            GLOBS="$GLOBS *-locale-$lang"
         done
-    done
+    fi
 
-    # Install the packages, if any
+    if [ "$GLOBS" != "" ] ; then
-    if [ "$PACKAGES_TO_INSTALL" != "" ]; then
+        # Use the magic script to do all the work for us :)
-        rootfs_install_packages $PACKAGES_TO_INSTALL
+        oe-pkgdata-util glob ${TMPDIR}/pkgdata ${TARGET_VENDOR}-${TARGET_OS} ${WORKDIR}/installed_pkgs.txt "$GLOBS" > ${WORKDIR}/complementary_pkgs.txt
+
+        # Install the packages, if any
+        sed -i '/^$/d' ${WORKDIR}/complementary_pkgs.txt
+        if [ -s ${WORKDIR}/complementary_pkgs.txt ]; then
+            echo "Installing complementary packages"
+            rootfs_install_packages ${WORKDIR}/complementary_pkgs.txt
+        fi
     fi
 
     # Workaround for broken shell function dependencies
     if false ; then
         get_split_linguas
-        list_installed_packages
-        rootfs_check_package_exists
     fi
 }
 

meta/classes/package_rpm.bbclass

@@ -248,6 +248,7 @@ process_pkg_list_rpm() {
 # INSTALL_PACKAGES_LINGUAS_RPM - additional packages for uclibc
 # INSTALL_PROVIDENAME_RPM - content for provide name
 # INSTALL_TASK_RPM - task name
+# INSTALL_COMPLEMENTARY_RPM - 1 to enable complementary package install mode
 
 package_install_internal_rpm () {
 
@@ -261,31 +262,35 @@ package_install_internal_rpm () {
 	local providename="${INSTALL_PROVIDENAME_RPM}"
 	local task="${INSTALL_TASK_RPM}"
 
-	# Setup base system configuration
+	if [ "${INSTALL_COMPLEMENTARY_RPM}" != "1" ] ; then
-	mkdir -p ${target_rootfs}/etc/rpm/
+		# Setup base system configuration
-	echo "${platform}${TARGET_VENDOR}-${TARGET_OS}" > ${target_rootfs}/etc/rpm/platform
+		mkdir -p ${target_rootfs}/etc/rpm/
-	if [ ! -z "$platform_extra" ]; then
+		echo "${platform}${TARGET_VENDOR}-${TARGET_OS}" > ${target_rootfs}/etc/rpm/platform
-		for pt in $platform_extra ; do
+		if [ ! -z "$platform_extra" ]; then
-			case $pt in
+			for pt in $platform_extra ; do
-				noarch | any | all)
+				case $pt in
-					os="`echo ${TARGET_OS} | sed "s,-.*,,"`.*"
+					noarch | any | all)
-					;;
+						os="`echo ${TARGET_OS} | sed "s,-.*,,"`.*"
-				*)
+						;;
-					os="${TARGET_OS}"
+					*)
-					;;
+						os="${TARGET_OS}"
-			esac
+						;;
-			echo "$pt-.*-$os" >> ${target_rootfs}/etc/rpm/platform
+				esac
-		done
+				echo "$pt-.*-$os" >> ${target_rootfs}/etc/rpm/platform
-	fi
+			done
+		fi
 
-	# Tell RPM that the "/" directory exist and is available
+		# Tell RPM that the "/" directory exist and is available
-	mkdir -p ${target_rootfs}/etc/rpm/sysinfo
+		mkdir -p ${target_rootfs}/etc/rpm/sysinfo
-	echo "/" >${target_rootfs}/etc/rpm/sysinfo/Dirnames
+		echo "/" >${target_rootfs}/etc/rpm/sysinfo/Dirnames
-	if [ ! -z "$providename" ]; then
+		if [ ! -z "$providename" ]; then
-		cat /dev/null > ${target_rootfs}/etc/rpm/sysinfo/Providename
+			cat /dev/null > ${target_rootfs}/etc/rpm/sysinfo/Providename
-		for provide in $providename ; do
+			for provide in $providename ; do
-			echo $provide >> ${target_rootfs}/etc/rpm/sysinfo/Providename
+				echo $provide >> ${target_rootfs}/etc/rpm/sysinfo/Providename
-		done
+			done
+		fi
+	else
+		mv ${target_rootfs}/install/total_solution.manifest ${target_rootfs}/install/original_solution.manifest
 	fi
 
 	# Setup manifest of packages to install...
@@ -480,13 +485,22 @@ mutex_set_max 163840
 # ================ Replication
 EOF
 
-	# RPM is special. It can't handle dependencies and preinstall scripts correctly. Its
+	if [ "${INSTALL_COMPLEMENTARY_RPM}" = "1" ] ; then
-	# probably a feature. The only way to convince rpm to actually run the preinstall scripts 
+		# Only install packages not already installed (dependency calculation will
-	# for base-passwd and shadow first before installing packages that depend on these packages 
+		# almost certainly have added some that have been)
-	# is to do two image installs, installing one set of packages, then the other.
+		sort ${target_rootfs}/install/original_solution.manifest > ${target_rootfs}/install/original_solution_sorted.manifest
-	if [ "${INC_RPM_IMAGE_GEN}" = "1" -a -f "$pre_btmanifest" ]; then
+		sort ${target_rootfs}/install/total_solution.manifest > ${target_rootfs}/install/total_solution_sorted.manifest
-		echo "Skipping pre install due to exisitng image"
+		comm -2 -3 ${target_rootfs}/install/total_solution_sorted.manifest \
+			${target_rootfs}/install/original_solution_sorted.manifest | awk '{print $1}' > \
+			${target_rootfs}/install/diff.manifest
+		mv ${target_rootfs}/install/diff.manifest ${target_rootfs}/install/total_solution.manifest
+	elif [ "${INC_RPM_IMAGE_GEN}" = "1" -a -f "$pre_btmanifest" ]; then
+		echo "Skipping pre install due to existing image"
 	else
+		# RPM is special. It can't handle dependencies and preinstall scripts correctly. Its
+		# probably a feature. The only way to convince rpm to actually run the preinstall scripts
+		# for base-passwd and shadow first before installing packages that depend on these packages
+		# is to do two image installs, installing one set of packages, then the other.
 		rm -f ${target_rootfs}/install/initial_install.manifest
 		echo "Installing base dependencies first (base-passwd, base-files and shadow) since rpm is special"
 		grep /base-passwd-[0-9] ${target_rootfs}/install/total_solution.manifest >> ${target_rootfs}/install/initial_install.manifest || true

meta/classes/populate_sdk_deb.bbclass

@@ -36,6 +36,8 @@ populate_sdk_deb () {
 
 	package_install_internal_deb
 
+	${POPULATE_SDK_POST_TARGET_COMMAND}
+
 	populate_sdk_post_deb ${INSTALL_ROOTFS_DEB}
 
 	populate_sdk_log_check populate_sdk

meta/classes/populate_sdk_ipk.bbclass

@@ -29,6 +29,8 @@ populate_sdk_ipk() {
 
 	package_install_internal_ipk
 
+	${POPULATE_SDK_POST_TARGET_COMMAND}
+
 	#install host
 	export INSTALL_ROOTFS_IPK="${SDK_OUTPUT}"
 	export INSTALL_CONF_IPK="${IPKGCONF_SDK}"

meta/classes/populate_sdk_rpm.bbclass

@@ -37,6 +37,7 @@ populate_sdk_rpm () {
 	export INSTALL_PACKAGES_LINGUAS_RPM=""
 	export INSTALL_PROVIDENAME_RPM="/bin/sh /bin/bash /usr/bin/env /usr/bin/perl pkgconfig pkgconfig(pkg-config)"
 	export INSTALL_TASK_RPM="populate_sdk-target"
+	export INSTALL_COMPLEMENTARY_RPM=""
 
 	# Setup base system configuration
 	mkdir -p ${INSTALL_ROOTFS_RPM}/etc/rpm/
@@ -74,6 +75,7 @@ EOF
 	export INSTALL_PLATFORM_EXTRA_RPM
 
 	package_install_internal_rpm
+	${POPULATE_SDK_POST_TARGET_COMMAND}
 	populate_sdk_post_rpm ${INSTALL_ROOTFS_RPM}
 
 	## install nativesdk ##
@@ -86,6 +88,7 @@ EOF
 	export INSTALL_PACKAGES_LINGUAS_RPM=""
 	export INSTALL_PROVIDENAME_RPM="/bin/sh /bin/bash /usr/bin/env /usr/bin/perl pkgconfig libGL.so()(64bit) libGL.so"
 	export INSTALL_TASK_RPM="populate_sdk_rpm-nativesdk"
+	export INSTALL_COMPLEMENTARY_RPM=""
 
 	# List must be prefered to least preferred order
 	INSTALL_PLATFORM_EXTRA_RPM=""

meta/classes/rootfs_deb.bbclass

@@ -10,7 +10,7 @@ do_rootfs[recrdeptask] += "do_package_write_deb"
 
 do_rootfs[lockfiles] += "${WORKDIR}/deb.lock"
 
-DEB_POSTPROCESS_COMMANDS = "rootfs_install_all_locales; "
+DEB_POSTPROCESS_COMMANDS = ""
 
 opkglibdir = "${localstatedir}/lib/opkg"
 
@@ -42,6 +42,8 @@ fakeroot rootfs_deb_do_rootfs () {
 	package_install_internal_deb
 	${DEB_POSTPROCESS_COMMANDS}
 
+	rootfs_install_complementary
+
 	export D=${IMAGE_ROOTFS}
 	export OFFLINE_ROOT=${IMAGE_ROOTFS}
 	export IPKG_OFFLINE_ROOT=${IMAGE_ROOTFS}
@@ -87,10 +89,16 @@ remove_packaging_data_files() {
 	rm -rf ${IMAGE_ROOTFS}/usr/dpkg/
 }
 
-DPKG_QUERY_COMMAND = "${STAGING_BINDIR_NATIVE}/dpkg --admindir=${IMAGE_ROOTFS}/var/lib/dpkg"
+# This will of course only work after rootfs_deb_do_rootfs has been called
+DPKG_QUERY_COMMAND = "${STAGING_BINDIR_NATIVE}/dpkg-query --admindir=$INSTALL_ROOTFS_DEB/var/lib/dpkg"
 
 list_installed_packages() {
-	${DPKG_QUERY_COMMAND} -l | grep ^ii | awk '{ print $2 }'
+	if [ "$1" = "arch" ] ; then
+		# Here we want the PACKAGE_ARCH not the deb architecture
+		${DPKG_QUERY_COMMAND} -W -f='${Package} ${PackageArch}\n'
+	else
+		${DPKG_QUERY_COMMAND} -W -f='${Package}\n'
+	fi
 }
 
 get_package_filename() {
@@ -110,16 +118,9 @@ list_package_recommends() {
 	${DPKG_QUERY_COMMAND} -s $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
 }
 
-rootfs_check_package_exists() {
-	if [ `apt-cache policy $1 | wc -l` -gt 4 ]; then
-		echo $1
-	fi
-}
-
 rootfs_install_packages() {
-	${STAGING_BINDIR_NATIVE}/apt-get install $@ --force-yes --allow-unauthenticated
+	${STAGING_BINDIR_NATIVE}/apt-get install `cat $1` --force-yes --allow-unauthenticated
 
-	for pkg in $@ ; do
+	# Mark all packages installed
-		deb_package_setflag installed $pkg
+	sed -i -e "s/Status: install ok unpacked/Status: install ok installed/;" $INSTALL_ROOTFS_DEB/var/lib/dpkg/status
-	done
 }

meta/classes/rootfs_ipk.bbclass

@@ -15,10 +15,12 @@ do_rootfs[recrdeptask] += "do_package_write_ipk"
 do_rootfs[lockfiles] += "${WORKDIR}/ipk.lock"
 
 IPKG_ARGS = "-f ${IPKGCONF_TARGET} -o ${IMAGE_ROOTFS} --force-overwrite"
+# The _POST version also works when constructing the matching SDK
+IPKG_ARGS_POST = "-f ${IPKGCONF_TARGET} -o $INSTALL_ROOTFS_IPK --force-overwrite"
 
 OPKG_PREPROCESS_COMMANDS = "package_update_index_ipk; package_generate_ipkg_conf"
 
-OPKG_POSTPROCESS_COMMANDS = "ipk_insert_feed_uris; rootfs_install_all_locales; "
+OPKG_POSTPROCESS_COMMANDS = "ipk_insert_feed_uris; "
 
 opkglibdir = "${localstatedir}/lib/opkg"
 
@@ -74,6 +76,8 @@ fakeroot rootfs_ipk_do_rootfs () {
 	#mkdir -p ${IMAGE_ROOTFS}/etc/opkg/
 	#grep "^arch" ${IPKGCONF_TARGET} >${IMAGE_ROOTFS}/etc/opkg/arch.conf
 
+	rootfs_install_complementary
+
 	${OPKG_POSTPROCESS_COMMANDS}
 	${ROOTFS_POSTINSTALL_COMMAND}
 	
@@ -125,12 +129,16 @@ remove_packaging_data_files() {
 }
 
 list_installed_packages() {
-	grep ^Package: ${IMAGE_ROOTFS}${opkglibdir}/status | sed "s/^Package: //"
+	if [ "$1" = "arch" ] ; then
+		opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py -a
+	else
+		opkg-cl ${IPKG_ARGS_POST} list_installed | awk '{ print $1 }'
+	fi
 }
 
 get_package_filename() {
 	set +x
-	info=`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" || true`
+	info=`opkg-cl ${IPKG_ARGS_POST} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" || true`
 	name=`echo "${info}" | awk '/^Package/ {printf $2"_"}'`
 	name=$name`echo "${info}" | awk -F: '/^Version/ {printf $NF"_"}' | sed 's/^\s*//g'`
 	name=$name`echo "${info}" | awk '/^Archi/ {print $2".ipk"}'`
@@ -145,21 +153,15 @@ get_package_filename() {
 }
 
 list_package_depends() {
-	opkg-cl ${IPKG_ARGS} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
+	opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
 }
 
 list_package_recommends() {
-	opkg-cl ${IPKG_ARGS} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
+	opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
-}
-
-rootfs_check_package_exists() {
-	if [ `opkg-cl ${IPKG_ARGS} info $1 | wc -l` -gt 2 ]; then
-		echo $1
-	fi
 }
 
 rootfs_install_packages() {
-	opkg-cl ${IPKG_ARGS} install $PACKAGES_TO_INSTALL
+	opkg-cl ${IPKG_ARGS_POST} install `cat $1`
 }
 
 ipk_insert_feed_uris () {

meta/classes/rootfs_rpm.bbclass

@@ -22,7 +22,7 @@ do_rootfs[depends] += "opkg-native:do_populate_sysroot"
 do_rootfs[recrdeptask] += "do_package_write_rpm"
 
 RPM_PREPROCESS_COMMANDS = "package_update_index_rpm; package_generate_rpm_conf; "
-RPM_POSTPROCESS_COMMANDS = "rootfs_install_all_locales; "
+RPM_POSTPROCESS_COMMANDS = ""
 
 # 
 # Allow distributions to alter when [postponed] package install scripts are run
@@ -56,6 +56,7 @@ fakeroot rootfs_rpm_do_rootfs () {
 	export INSTALL_PACKAGES_LINGUAS_RPM="${LINGUAS_INSTALL}"
 	export INSTALL_PROVIDENAME_RPM=""
 	export INSTALL_TASK_RPM="rootfs_rpm_do_rootfs"
+	export INSTALL_COMPLEMENTARY_RPM=""
 
 	# Setup base system configuration
 	mkdir -p ${INSTALL_ROOTFS_RPM}/etc/rpm/
@@ -69,6 +70,8 @@ fakeroot rootfs_rpm_do_rootfs () {
 
 	package_install_internal_rpm
 
+	rootfs_install_complementary
+
 	export D=${IMAGE_ROOTFS}
 	export OFFLINE_ROOT=${IMAGE_ROOTFS}
 	export IPKG_OFFLINE_ROOT=${IMAGE_ROOTFS}
@@ -134,11 +137,15 @@ remove_packaging_data_files() {
 	rm -rf ${IMAGE_ROOTFS}${opkglibdir}
 }
 
-RPM_QUERY_CMD = '${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \
+RPM_QUERY_CMD = '${RPM} --root $INSTALL_ROOTFS_RPM -D "_dbpath ${rpmlibdir}" \
 		-D "__dbi_txn create nofsync private"'
 
 list_installed_packages() {
-	${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]"
+	if [ "$1" = "arch" ] ; then
+		${RPM_QUERY_CMD} -qa --qf "[%{NAME} %{ARCH}\n]"
+	else
+		${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]"
+	fi
 }
 
 get_package_filename() {
@@ -172,21 +179,15 @@ list_package_recommends() {
 	${RPM_QUERY_CMD} -q --suggests $1
 }
 
-rootfs_check_package_exists() {
-	resolve_package_rpm ${RPMCONF_TARGET_BASE}-base_archs.conf $1
-}
-
 rootfs_install_packages() {
-    # The pkg to be installed here is not controlled by the
+	# Note - we expect the variables not set here to already have been set
-    # package_install_internal_rpm, so it may have already been
+	export INSTALL_PACKAGES_RPM=""
-    # installed(e.g, installed in the first time when generate the
+	export INSTALL_PACKAGES_ATTEMPTONLY_RPM="`cat $1`"
-    # rootfs), use '--replacepkgs' to always install them
+	export INSTALL_PROVIDENAME_RPM=""
-	for pkg in $@; do
+	export INSTALL_TASK_RPM="rootfs_install_packages"
-		${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \
+	export INSTALL_COMPLEMENTARY_RPM="1"
-			-D "__dbi_txn create nofsync private" \
+
-			--noscripts --notriggers --noparentdirs --nolinktos \
+	package_install_internal_rpm
-			--replacepkgs -Uhv $pkg || true
-	done
 }
 
 python () {

scripts/oe-pkgdata-util

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+
+# OpenEmbedded pkgdata utility
+#
+# Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Copyright 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#
+# Currently only has one function - mapping of packages to their dev/dbg/doc/locale etc. 
+# counterparts ("glob" command). Could be extended in future to perform other useful querying
+# functions on the pkgdata though.
+#
+
+import sys
+import os
+import os.path
+import fnmatch
+import re
+
+def usage():
+    print("syntax: pkgdata-util glob [-d] <pkgdatadir> <vendor-os> <pkglist> \"<globs>\"");
+
+
+
+def glob(args):
+    if len(args) < 4:
+        usage()
+        sys.exit(1)
+
+    pkgdata_dir = args[0]
+    target_suffix = args[1]
+    pkglist_file = args[2]
+    globs = args[3].split()
+
+    if target_suffix.startswith("-"):
+        target_suffix = target_suffix[1:]
+
+    skipregex = re.compile("-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-")
+
+    mappedpkgs = set()
+    with open(pkglist_file, 'r') as f:
+        for line in f:
+            fields = line.rstrip().split()
+            if len(fields) < 2:
+                continue
+            pkg = fields[0]
+            arch = fields[1]
+            multimach_target_sys = "%s-%s" % (arch, target_suffix)
+
+            # Skip packages for which there is no point applying globs
+            if skipregex.search(pkg):
+                if debug:
+                    print("%s -> !!" % pkg)
+                continue
+
+            # Skip packages that already match the globs, so if e.g. a dev package
+            # is already installed and thus in the list, we don't process it any further
+            # Most of these will be caught by skipregex already, but just in case...
+            already = False
+            for g in globs:
+                if fnmatch.fnmatchcase(pkg, g):
+                    already = True
+                    break
+            if already:
+                if debug:
+                    print("%s -> !" % pkg)
+                continue
+
+            # Define some functions
+            def revpkgdata(pkgn):
+                return os.path.join(pkgdata_dir, multimach_target_sys, "runtime-reverse", pkgn)
+            def fwdpkgdata(pkgn):
+                return os.path.join(pkgdata_dir, multimach_target_sys, "runtime", pkgn)
+            def readpn(pkgdata_file):
+                pn = ""
+                with open(pkgdata_file, 'r') as f:
+                    for line in f:
+                        if line.startswith("PN:"):
+                            pn = line.split(': ')[1].rstrip()
+                return pn
+            def readrenamed(pkgdata_file):
+                renamed = ""
+                pn = os.path.basename(pkgdata_file)
+                with open(pkgdata_file, 'r') as f:
+                    for line in f:
+                        if line.startswith("PKG_%s:" % pn):
+                            renamed = line.split(': ')[1].rstrip()
+                return renamed
+
+            # Main processing loop
+            for g in globs:
+                mappedpkg = ""
+                # First just try substitution (i.e. packagename -> packagename-dev)
+                newpkg = g.replace("*", pkg)
+                revlink = revpkgdata(newpkg)
+                if os.path.exists(revlink):
+                    mappedpkg = os.path.basename(os.readlink(revlink))
+                    fwdfile = fwdpkgdata(mappedpkg)
+                    if os.path.exists(fwdfile):
+                        mappedpkg = readrenamed(fwdfile)
+                else:
+                    # That didn't work, so now get the PN, substitute that, then map in the other direction
+                    revlink = revpkgdata(pkg)
+                    if os.path.exists(revlink):
+                        pn = readpn(revlink)
+                        newpkg = g.replace("*", pn)
+                        fwdfile = fwdpkgdata(newpkg)
+                        if os.path.exists(fwdfile):
+                            mappedpkg = readrenamed(fwdfile)
+                    else:
+                        # Package doesn't even exist...
+                        if debug:
+                            print "%s is not a valid package!" % (pkg)
+                        break
+
+                if mappedpkg:
+                    if debug:
+                        print "%s (%s) -> %s" % (pkg, g, mappedpkg)
+                    mappedpkgs.add(mappedpkg)
+                else:
+                    if debug:
+                        print "%s (%s) -> ?" % (pkg, g)
+
+    if debug:
+        print "------"
+
+    print("\n".join(mappedpkgs))
+
+
+
+# Too lazy to use getopt
+debug = False
+noopt = False
+args = []
+for arg in sys.argv[1:]:
+    if arg == "--":
+        noopt = True
+    else:
+        if not noopt:
+            if arg == "-d":
+                debug = True
+                continue
+        args.append(arg)
+
+if len(args) < 1:
+    usage()
+    sys.exit(1)
+
+if args[0] == "glob":
+    glob(args[1:])
+else:
+    usage()
+    sys.exit(1)

scripts/opkg-query-helper.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+# OpenEmbedded opkg query helper utility
+#
+# Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Copyright 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#
+
+
+import sys
+import fileinput
+import re
+
+archmode = False
+filemode = False
+
+args = []
+for arg in sys.argv[1:]:
+    if arg == '-a':
+        archmode = True
+    elif arg == '-f':
+        filemode = True
+    else:
+        args.append(arg)
+
+# Regex for removing version specs after dependency items
+verregex = re.compile(' \([=<>]* [^ )]*\)')
+
+pkg = ""
+ver = ""
+for line in fileinput.input(args):
+    line = line.rstrip()
+    if ': ' in line:
+        if line.startswith("Package:"):
+            pkg = line.split(": ")[1]
+            ver = ""
+        else:
+            if archmode:
+                if line.startswith("Architecture:"):
+                    arch = line.split(": ")[1]
+                    print("%s %s" % (pkg,arch))
+            elif filemode:
+                if line.startswith("Version:"):
+                    ver = line.split(": ")[1]
+                elif line.startswith("Architecture:"):
+                    arch = line.split(": ")[1]
+                    print("%s %s_%s_%s.ipk" % (pkg,pkg,ver,arch))
+            else:
+                if line.startswith("Depends:"):
+                    depval = line.split(": ")[1]
+                    deps = depval.split(", ")
+                    for dep in deps:
+                        dep = verregex.sub('', dep)
+                        print("%s|%s" % (pkg,dep))
+                elif line.startswith("Recommends:"):
+                    recval = line.split(": ")[1]
+                    recs = recval.split(", ")
+                    for rec in recs:
+                        rec = verregex.sub('', rec)
+                        print("%s|%s [REC]" % (pkg, rec))
+