sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity
generic-poky/scripts/sstate-cache-management.sh

419 lines
13 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# Copyright (c) 2012 Wind River Systems, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Global vars
cache_dir=
confirm=
fsym=
total_deleted=0
verbose=
debug=0
usage () {
cat << EOF
Welcome to sstate cache management utilities.
sstate-cache-management.sh <OPTION>
Options:
-h, --help
Display this help and exit.
--cache-dir=<sstate cache dir>
Specify sstate cache directory, will use the environment
variable SSTATE_CACHE_DIR if it is not specified.
--extra-layer=<layer1>,<layer2>...<layern>
Specify the layer which will be used for searching the archs,
it will search the meta and meta-* layers in the top dir by
default, and will search meta, meta-*, <layer1>, <layer2>,
...<layern> when specified. Use "," as the separator.
This is useless for --stamps-dir.
-d, --remove-duplicated
Remove the duplicated sstate cache files of one package, only
the newest one will be kept.
Conflicts with --stamps-dir.
--stamps-dir=<dir1>,<dir2>...<dirn>
Specify the build directory's stamps directories, the sstate
cache file which IS USED by these build diretories will be KEPT,
other sstate cache files in cache-dir will be removed. Use ","
as the separator. For example:
--stamps-dir=build1/tmp/stamps,build2/tmp/stamps
Conflicts with --remove-duplicated.
-L, --follow-symlink
Rmove both the symbol link and the destination file, default: no.
-y, --yes
Automatic yes to prompts; assume "yes" as answer to all prompts
and run non-interactively.
-v, --verbose
explain what is being done
-d, --debug
show debug info, repeat for more debug info
EOF
}
if [ $# -lt 1 ]; then
usage
exit 0
fi
# Echo no files to remove
no_files () {
echo No files to remove
}
# Echo nothing to do
do_nothing () {
echo Nothing to do
}
# Read the input "y"
read_confirm () {
echo -n "$total_deleted files will be removed! "
if [ "$confirm" != "y" ]; then
echo -n "Do you want to continue (y/n)? "
while read confirm; do
[ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \
-o "$confirm" = "N" ] && break
echo -n "Invalid input \"$confirm\", please input 'y' or 'n': "
done
else
echo
fi
}
# Print error information and exit.
echo_error () {
echo "ERROR: $1" >&2
exit 1
}
# Generate the remove list:
#
# * Add .done/.siginfo to the remove list
# * Add destination of symlink to the remove list
#
# $1: output file, others: sstate cache file (.tgz)
gen_rmlist (){
local rmlist_file="$1"
shift
local files="$@"
for i in $files; do
echo $i >> $rmlist_file
# Add the ".siginfo"
if [ -e $i.siginfo ]; then
echo $i.siginfo >> $rmlist_file
fi
# Add the destination of symlink
if [ -L "$i" ]; then
if [ "$fsym" = "y" ]; then
dest="`readlink -e $i`"
if [ -n "$dest" ]; then
echo $dest >> $rmlist_file
# Remove the .siginfo when .tgz is removed
if [ -f "$dest.siginfo" ]; then
echo $dest.siginfo >> $rmlist_file
fi
fi
fi
# Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
base_fn="${i##/*/}"
t_fn="$base_fn.done"
s_fn="$base_fn.siginfo.done"
for d in $t_fn $s_fn; do
if [ -f $cache_dir/$d ]; then
echo $cache_dir/$d >> $rmlist_file
fi
done
fi
done
}
# Remove the duplicated cache files for the pkg, keep the newest one
remove_duplicated () {
local topdir
local oe_core_dir
local tunedirs
local all_archs
local all_machines
local ava_archs
local arch
local file_names
local sstate_list
local fn_tmp
local list_suffix=`mktemp` || exit 1
# Find out the archs in all the layers
echo -n "Figuring out the archs in the layers ... "
oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
topdir=$(dirname $oe_core_dir)
tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include'`"
[ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
# Add the qemu and native archs
# Use the "_" to substitute "-", e.g., x86-64 to x86_64
# Sort to remove the duplicated ones
all_archs=$(echo $all_archs $all_machines $(uname -m) \
| sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
echo "Done"
sstate_suffixes="deploy-rpm deploy-ipk deploy-deb deploy package populate-lic populate-sysroot"
# Save all the sstate files in a file
sstate_list=`mktemp` || exit 1
find $cache_dir -name 'sstate-*.tgz' >$sstate_list
echo -n "Figuring out the archs in the sstate cache dir ... "
for arch in $all_archs; do
grep -q "\-$arch-" $sstate_list
[ $? -eq 0 ] && ava_archs="$ava_archs $arch"
done
echo "Done"
echo "The following archs have been found in the cache dir:"
echo $ava_archs
echo ""
# Save the file list which needs to be removed
local remove_listdir=`mktemp -d` || exit 1
for suffix in $sstate_suffixes; do
# Save the file list to a file, some suffix's file may not exist
grep "sstate-.*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null
local deleted=0
echo -n "Figuring out the sstate-xxx_$suffix.tgz ... "
# There are at list 6 dashes (-) after arch, use this to avoid the
# greedy match of sed.
file_names=`for arch in $ava_archs; do
sed -ne 's#.*/\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*#\1#p' $list_suffix
done | sort -u`
fn_tmp=`mktemp` || exit 1
rm_list="$remove_listdir/sstate-xxx_$suffix"
for fn in $file_names; do
[ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz"
for arch in $ava_archs; do
grep -h "/$fn-$arch-" $list_suffix >$fn_tmp
if [ -s $fn_tmp ] ; then
[ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix:" && cat $fn_tmp
# Use the modification time
to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
[ $debug -gt 2 ] && echo "Considering to delete: $to_del"
# The sstate file which is downloaded from the SSTATE_MIRROR is
# put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
# it, so filter it out from the remove list if it should not be
# removed.
to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
[ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
for k in $to_keep; do
if [ -L "$k" ]; then
# The symlink's destination
k_dest="`readlink -e $k`"
# Maybe it is the one in cache_dir
k_maybe="$cache_dir/${k##/*/}"
# Remove it from the remove list if they are the same.
if [ "$k_dest" = "$k_maybe" ]; then
to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`"
fi
fi
done
rm -f $fn_tmp
[ $debug -gt 2 ] && echo "Decided to delete: $to_del"
gen_rmlist $rm_list "$to_del"
fi
done
done
[ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l`
[ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list
echo "($deleted files will be removed)"
let total_deleted=$total_deleted+$deleted
done
rm -f $list_suffix
rm -f $sstate_list
if [ $total_deleted -gt 0 ]; then
read_confirm
if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
for list in `ls $remove_listdir/`; do
echo -n "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
# Remove them one by one to avoid the argument list too long error
for i in `cat $remove_listdir/$list`; do
rm -f $verbose $i
done
echo "Done"
done
echo "$total_deleted files have been removed!"
else
do_nothing
fi
else
no_files
fi
[ -d $remove_listdir ] && rm -fr $remove_listdir
}
# Remove the sstate file by stamps dir, the file not used by the stamps dir
# will be removed.
rm_by_stamps (){
local cache_list=`mktemp` || exit 1
local keep_list=`mktemp` || exit 1
local rm_list=`mktemp` || exit 1
local suffixes
local sums
local all_sums
suffixes="populate_sysroot populate_lic package_write_ipk \
package_write_rpm package_write_deb package deploy"
# Figure out all the md5sums in the stamps dir.
echo -n "Figuring out all the md5sums in stamps dir ... "
for i in $suffixes; do
# There is no "\.sigdata" but "_setcene" when it is mirrored
# from the SSTATE_MIRRORS, use them to figure out the sum.
sums=`find $stamps -maxdepth 2 -name "*.do_$i.*" \
-o -name "*.do_${i}_setscene.*" | \
sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \
sed -e 's#\..*##' | sort -u`
all_sums="$all_sums $sums"
done
echo "Done"
# Save all the state file list to a file
find $cache_dir -name 'sstate-*.tgz' | sort -u -o $cache_list
echo -n "Figuring out the files which will be removed ... "
for i in $all_sums; do
grep ".*-${i}_.*" $cache_list >>$keep_list
done
echo "Done"
if [ -s $keep_list ]; then
sort -u $keep_list -o $keep_list
to_del=`comm -1 -3 $keep_list $cache_list`
gen_rmlist $rm_list "$to_del"
let total_deleted=(`cat $rm_list | wc -w`)
if [ $total_deleted -gt 0 ]; then
[ $debug -gt 0 ] && cat $rm_list
read_confirm
if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
echo "Removing sstate cache files ... ($total_deleted files)"
# Remove them one by one to avoid the argument list too long error
for i in `cat $rm_list`; do
rm -f $verbose $i
done
echo "$total_deleted files have been removed"
else
do_nothing
fi
else
no_files
fi
else
echo_error "All files in cache dir will be removed! Abort!"
fi
rm -f $cache_list
rm -f $keep_list
rm -f $rm_list
}
# Parse arguments
while [ -n "$1" ]; do
case $1 in
--cache-dir=*)
cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e`
[ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir"
shift
;;
--remove-duplicated|-d)
rm_duplicated="y"
shift
;;
--yes|-y)
confirm="y"
shift
;;
--follow-symlink|-L)
fsym="y"
shift
;;
--extra-layer=*)
extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
[ -n "$extra_layers" ] || echo_error "Invalid extra layer $i"
for i in $extra_layers; do
l=`readlink -e $i`
if [ -d "$l" ]; then
layers="$layers $l"
else
echo_error "Can't find layer $i"
fi
done
shift
;;
--stamps-dir=*)
stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'`
[ -n "$stamps" ] || echo_error "Invalid stamps dir $i"
for i in $stamps; do
[ -d "$i" ] || echo_error "Invalid stamps dir $i"
done
shift
;;
--verbose|-v)
verbose="-v"
shift
;;
--debug)
debug=`expr $debug + 1`
echo "Debug level $debug"
shift
;;
--help|-h)
usage
exit 0
;;
*)
echo "Invalid arguments $*"
echo_error "Try 'sstate-cache-management.sh -h' for more information."
;;
esac
done
# sstate cache directory, use environment variable SSTATE_CACHE_DIR
# if it was not specified, otherwise, error.
[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR
[ -n "$cache_dir" ] || echo_error "No cache dir found!"
[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\""
[ -n "$rm_duplicated" -a -n "$stamps" ] && \
echo_error "Can not use both --remove-duplicated and --stamps-dir"
[ "$rm_duplicated" = "y" ] && remove_duplicated
[ -n "$stamps" ] && rm_by_stamps
[ -z "$rm_duplicated" -a -z "$stamps" ] && \
echo "What do you want to do?"
View Git Blame Copy Permalink