From fd1d661b391dfad1edaac937a17c1165f635031a Mon Sep 17 00:00:00 2001 From: Jiajun Xu Date: Thu, 8 Jul 2010 02:06:58 +0800 Subject: [PATCH] test: add automation framework and sanitytest Automation test is disabled by default. User need set TESTCLASS to qemu in conf/local.conf and run bitbake command "bitbake poky-image-xxx" or "bitbake poky-image-xxx -c qemuimagetest" to trigger it. Currently only the sanity test with two testcases are added. To run the test, user need prepare a testing environment: 1) "expect" should be installed on system 2) NOPASSWD should be set for user to run bitbake Signed-off-by Jiajun Xu --- build/conf/local.conf.sample | 8 ++ meta/classes/image.bbclass | 3 + meta/classes/imagetest-dummy.bbclass | 1 + meta/classes/imagetest-qemu.bbclass | 137 ++++++++++++++++++ meta/conf/layer.conf | 7 + scripts/qemuimage-testlib | 202 +++++++++++++++++++++++++++ scripts/qemuimage-tests/sanity/boot | 31 ++++ scripts/qemuimage-tests/sanity/ssh | 40 ++++++ 8 files changed, 429 insertions(+) create mode 100644 meta/classes/imagetest-dummy.bbclass create mode 100644 meta/classes/imagetest-qemu.bbclass create mode 100644 scripts/qemuimage-testlib create mode 100755 scripts/qemuimage-tests/sanity/boot create mode 100755 scripts/qemuimage-tests/sanity/ssh diff --git a/build/conf/local.conf.sample b/build/conf/local.conf.sample index fc013bcd10..e58c476a7d 100644 --- a/build/conf/local.conf.sample +++ b/build/conf/local.conf.sample @@ -153,3 +153,11 @@ ENABLE_BINARY_LOCALE_GENERATION = "1" # Poky can try and fetch packaged-staging packages from a http, https or ftp # mirror. Set this variable to the root of a pstage directory on a server. #PSTAGE_MIRROR ?= "http://someserver.tld/share/pstage" + +# Set IMAGETEST to qemu if you want to build testcases and start +# testing in qemu after do_rootfs. +#IMAGETEST="qemu" + +# By default testing will run the sanitytest suite. If you want to run other tests +# (e.g. bat), list them here +#TEST_SCEN="sanitytest bat" diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass index 4a5b83e30b..7318ba9a28 100644 --- a/meta/classes/image.bbclass +++ b/meta/classes/image.bbclass @@ -1,5 +1,8 @@ inherit rootfs_${IMAGE_PKGTYPE} +IMAGETEST ?= "dummy" +inherit imagetest-${IMAGETEST} + LICENSE = "MIT" PACKAGES = "" RDEPENDS += "${IMAGE_INSTALL}" diff --git a/meta/classes/imagetest-dummy.bbclass b/meta/classes/imagetest-dummy.bbclass new file mode 100644 index 0000000000..bcacae604f --- /dev/null +++ b/meta/classes/imagetest-dummy.bbclass @@ -0,0 +1 @@ +# dummy testclass file diff --git a/meta/classes/imagetest-qemu.bbclass b/meta/classes/imagetest-qemu.bbclass new file mode 100644 index 0000000000..742b759208 --- /dev/null +++ b/meta/classes/imagetest-qemu.bbclass @@ -0,0 +1,137 @@ +addtask qemuimagetest before do_build +# after do_rootfs +do_qemuimagetest[nostamp] = "1" +do_qemuimagetest[depends] += "qemu-native:do_populate_sysroot" + +# Test related variables +# By default, TEST_DIR is created under WORKDIR +TEST_DIR ?= "${WORKDIR}/qemuimagetest" +TEST_LOG ?= "${LOG_DIR}/qemuimagetests" +TEST_RESULT ?= "${TEST_DIR}/result" +TEST_LIST ?= "${TEST_DIR}/list" +TEST_TMP ?= "${TEST_DIR}/tmp" +TEST_SCEN ?= "sanity" + +python do_qemuimagetest() { + import sys + import re + import os + + """ + Test Controller for Poky Testing. + """ + + casestr = re.compile(r'(?P\w+\b)\s*(?P\w+$)') + resultstr = re.compile(r'\s*(?P\w+)\s*(?P\d+)\s*(?P\d+)\s*(?P\d+)') + + """funtion to run each case under scenario""" + def runtest(scen, case, fulltestpath): + resultpath = bb.data.getVar('TEST_RESULT', d, 1) + machine = bb.data.getVar('MACHINE', d, 1) + pname = bb.data.getVar('PN', d, 1) + + testpath = bb.data.getVar('TEST_DIR', d, 1) + + """initialize log file for testcase""" + logpath = bb.data.getVar('TEST_LOG', d, 1) + bb.utils.mkdirhier("%s/%s" % (logpath, scen)) + caselog = os.path.join(logpath, "%s/log_%s.%s" % (scen, case, bb.data.getVar('DATETIME', d, 1))) + os.system("touch %s" % caselog) + + + """export TEST_TMP, TEST_RESULT, DEPLOY_DIR and QEMUARCH""" + os.environ["PATH"] = bb.data.getVar("PATH", d, True) + os.environ["TEST_TMP"] = testpath + os.environ["TEST_RESULT"] = resultpath + os.environ["DEPLOY_DIR"] = bb.data.getVar("DEPLOY_DIR", d, True) + os.environ["QEMUARCH"] = machine + os.environ["QEMUTARGET"] = pname + os.environ["DISPLAY"] = bb.data.getVar("DISPLAY", d, True) + + """run Test Case""" + bb.note("Run %s test in scenario %s" % (case, scen)) + os.system("%s | tee -a %s" % (fulltestpath, caselog)) + + """Generate testcase list in runtime""" + def generate_list(testfile, testlist): + list = [] + if len(testlist) == 0: + raise bb.build.FuncFailed("No testcase defined in TEST_SCEN") + + """remove old testcase list file""" + if os.path.exists(testfile): + os.remove(testfile) + os.system("touch %s" % testfile) + + """check testcase folder and add case to TEST_LIST""" + for item in testlist.split(" "): + found = False + for dir in bb.data.getVar("QEMUIMAGETESTS", d, True).split(): + casepath = os.path.join(dir, item) + if not os.path.isdir(casepath): + continue + files = os.listdir(casepath) + for casefile in files: + fulltestcase = "%s/%s" % (casepath, casefile) + if os.path.isfile(fulltestcase): + list.append((item, casefile, fulltestcase)) + found = True + if not found: + raise bb.build.FuncFailed("Testcase folder not found for test %s" % item) + return list + + """check testcase folder and create test log folder""" + testpath = bb.data.getVar('TEST_DIR', d, 1) + bb.utils.mkdirhier(testpath) + + logpath = bb.data.getVar('TEST_LOG', d, 1) + bb.utils.mkdirhier(logpath) + + tmppath = bb.data.getVar('TEST_TMP', d, 1) + bb.utils.mkdirhier(tmppath) + + """initialize result file""" + resultpath = bb.data.getVar('TEST_RESULT', d, 1) + bb.utils.mkdirhier(resultpath) + resultfile = os.path.join(resultpath, "testresult.%s" % bb.data.getVar('DATETIME', d, 1)) + sresultfile = os.path.join(resultpath, "testresult.log") + + machine = bb.data.getVar('MACHINE', d, 1) + + if os.path.exists(sresultfile): + os.remove(sresultfile) + os.system("touch %s" % resultfile) + os.symlink(resultfile, sresultfile) + f = open(sresultfile, "a") + f.write("\tTest Result for %s\n" % machine) + f.write("\tTestcase\tPASS\tFAIL\tNORESULT\n") + f.close() + + """generate pre-defined testcase list""" + testfile = bb.data.getVar('TEST_LIST', d, 1) + testlist = bb.data.getVar('TEST_SCEN', d, 1) + fulllist = generate_list(testfile, testlist) + + """Begin testing""" + for test in fulllist: + (scen, case, fullpath) = test + runtest(scen, case, fullpath) + + """Print Test Result""" + ret = 0 + f = open(sresultfile, "r") + for line in f: + m = resultstr.match(line) + if m: + if m.group('fail') == "1": + ret = 1 + elif m.group('noresult') == "1": + ret = 2 + print line, + else: + print line, + f.close() + + if ret != 0: + raise bb.build.FuncFailed("Some testcases fail, pls. check test result and test log!!!") +} diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf index 7b3d539245..a780245c29 100644 --- a/meta/conf/layer.conf +++ b/meta/conf/layer.conf @@ -8,5 +8,12 @@ BBFILE_COLLECTIONS += "normal" BBFILE_PATTERN_normal := "^${LAYERDIR}/" BBFILE_PRIORITY_normal = "5" +# Add scripts to PATH +PATH := "${PATH}:${LAYERDIR}/scripts" + +# Set path to qemu image tests included in this layer + +QEMUIMAGETESTS := "${OEROOT}/scripts/qemuimage-tests" + require conf/distro/include/poky-default-revisions.inc diff --git a/scripts/qemuimage-testlib b/scripts/qemuimage-testlib new file mode 100644 index 0000000000..4e27f8bc6f --- /dev/null +++ b/scripts/qemuimage-testlib @@ -0,0 +1,202 @@ +#!/bin/bash +# Common function for test +# Expect should be installed for SSH Testing +# To execute `runqemu`, NOPASSWD needs to be set in /etc/sudoers for user +# For example, for user "builder", /etc/sudoers can be like following: +# ######### +# #Members of the admin group may gain root privileges +# %builder ALL=(ALL) NOPASSWD: NOPASSWD: ALL +# ######### +# +# Author: Jiajun Xu +# +# This file is licensed under the GNU General Public License, +# Version 2. +# + +TYPE="ext3" + +# common function for information print +Test_Error() +{ + echo -e "\tTest_Error: $*" +} + +Test_Info() +{ + echo -e "\tTest_Info: $*" +} + +# function to run command in $ip_addr via ssh +Test_SSH() +{ + local ip_addr=$1 + shift + local command=$@ + local tmpfile=`mktemp` + local timeout=60 + local ret=0 + local exp_cmd=`cat << EOF +eval spawn ssh -o UserKnownHostsFile=$tmpfile root@$ip_addr "$command" +set timeout $time_out +expect { + "*assword:" { send "\r"; exp_continue} + "*(yes/no)?" { send "yes\r"; exp_continue } + eof { exit [ lindex [wait] 3 ] } +} +EOF` + expect -c "$exp_cmd" + ret=$? + rm -rf $tmpfile + return $ret +} + +# function to check if ssh is up in $ip_addr +Test_SSH_UP() +{ + local ip_addr=$1 + local timeout=$2 + local interval=0 + + while [ ${interval} -lt ${timeout} ] + do + Test_SSH ${ip_addr} "hostname" + if [ $? -ne 0 ]; then + interval=`expr $interval + 10` + sleep 10 + else + Test_Info "We can ssh on ${ip_addr} now" + return 0 + fi + + done + + Test_Info "We can not ssh on ${ip_addr} in ${timeout}" + return 1 +} + +# function to record test result in $TEST_RESULT/testresult.log +Test_Print_Result() +{ + local PASS=0 + local FAIL=0 + local NORESULT=0 + if [ $2 -eq 0 ]; then + PASS=1 + elif [ $2 -eq 1 ]; then + FAIL=1 + else + NORESULT=1 + fi + + echo -e "\t$1\t\t$PASS\t$FAIL\t$NORESULT" >> $TEST_RESULT/testresult.log +} + +# function to kill qemu +Test_Kill_Qemu() +{ + sudo pkill -9 qemu + return $? +} + +# function to check if there is any qemu process +Test_Check_Qemu_UP() +{ + local count=`ps -ef | grep -c qemu` + if [ ${count} -lt 2 ]; then + Test_Info "There is no Qemu process" + return 1 + else + Test_Info "There is at least Qemu process running" + return 0 + fi +} + +# function to check if network is up +Test_Check_IP_UP() +{ + ping -c1 $1 + if [ $? -ne 0 ]; then + Test_Info "IP $1 is not up" + return 1 + else + Test_Info "IP $1 is up" + return 0 + fi +} + +# function to check if qemu and its network +Test_Create_Qemu() +{ + local ret=1 + local up_time=0 + local ip_addr=$1 + local timeout=$2 + local kernel="" + local rootfs="" + + which poky-qemu + if [ $? -eq 0 ]; then + RUNQEMU=`which poky-qemu` + else + Test_Error "Can not find poky-qemu in \$PATH, return fail" + exit 1 + fi + + if [ "$QEMUARCH" = "qemux86" ]; then + KERNEL=${DEPLOY_DIR}/images/bzImage-${QEMUARCH}.bin + elif [ "$QEMUARCH" = "qemuarm" -o "$QEMUARCH" = "spitz" -o "$QEMUARCH" = "borzoi" -o "$QEMUARCH" = "akita" -o "$QEMUARCH" = "nokia800" ]; then + KERNEL=${DEPLOY_DIR}/images/zImage-${QEMUARCH}.bin + fi + + ROOTFS_IMAGE=`readlink -f ${DEPLOY_DIR}/images/${QEMUTARGET}-${QEMUARCH}.ext3` + TEST_ROOTFS_IMAGE="${TEST_TMP}/${QEMUTARGET}-${QEMUARCH}-test.ext3" + + CP=`which cp` + if [ -e "$TEST_ROOTFS_IMAGE" ]; then + rm -rf $TEST_ROOTFS_IMAGE + fi + $CP $ROOTFS_IMAGE $TEST_ROOTFS_IMAGE + + export MACHINE=$QEMUARCH + # Create Qemu in localhost VNC Port 1 + + xterm -display ${DISPLAY} -e "${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE}" & + + sleep 5 + + while [ ${up_time} -lt ${timeout} ] + do + Test_Check_Qemu_UP + if [ $? -ne 0 ]; then + Test_Info "Wait for qemu up..." + up_time=`expr $up_time + 5` + sleep 5 + else + Test_Info "Begin to check if qemu network is up" + break + fi + done + + while [ ${up_time} -lt ${timeout} ] + do + Test_Check_IP_UP ${ip_addr} + if [ $? -eq 0 ]; then + Test_Info "Qemu Network is up, ping with ${ip_addr} is OK" + ret=0 + break + else + Test_Info "Wait for Qemu Network up" + up_time=`expr $up_time + 5` + sleep 5 + fi + done + + if [ $ret -eq 0 ]; then + Test_Info "Qemu and its network is up" + return $ret + else + Test_Info "Qemu or its network is not up in ${timeout}" + return $ret + fi +} diff --git a/scripts/qemuimage-tests/sanity/boot b/scripts/qemuimage-tests/sanity/boot new file mode 100755 index 0000000000..b4c0094f25 --- /dev/null +++ b/scripts/qemuimage-tests/sanity/boot @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Boot Test Case for Sanity Test +# The case boot up the Qemu target with `runqemu qemux86`. +# Then check if qemu and qemu network is up. +# +# Author: Jiajun Xu +# +# This file is licensed under the GNU General Public License, +# Version 2. +# + +. $OEROOT/scripts/qemuimage-testlib + +TIMEOUT=120 +QEMU_IPADDR="192.168.7.2" + +# Start qemu and check its network +Test_Create_Qemu ${QEMU_IPADDR} ${TIMEOUT} + +if [ $? -eq 0 ]; then + Test_Info "Boot Test PASS" + Test_Kill_Qemu + Test_Print_Result "Boot" 0 + exit 0 +else + Test_Info "Boot Test FAIL" + Test_Kill_Qemu + Test_Print_Result "Boot" 1 + exit 1 +fi diff --git a/scripts/qemuimage-tests/sanity/ssh b/scripts/qemuimage-tests/sanity/ssh new file mode 100755 index 0000000000..7dd895951c --- /dev/null +++ b/scripts/qemuimage-tests/sanity/ssh @@ -0,0 +1,40 @@ +#!/bin/bash +# SSH Test Case for Sanity Test +# The case boot up the Qemu target with `runqemu qemux86`. +# Then check if ssh service in qemu is up. +# +# Author: Jiajun Xu +# +# This file is licensed under the GNU General Public License, +# Version 2. +# + +. $OEROOT/scripts/qemuimage-testlib + +TIMEOUT=360 +QEMU_IPADDR="192.168.7.2" +RET=1 + +# Start qemu and check its network +Test_Create_Qemu ${QEMU_IPADDR} ${TIMEOUT} + +# If qemu network is up, check ssh service in qemu +if [ $? -eq 0 ]; then + Test_Info "Begin to Test SSH Service in Qemu" + Test_SSH_UP ${QEMU_IPADDR} ${TIMEOUT} + RET=$? +else + RET=1 +fi + +if [ ${RET} -eq 0 ]; then + Test_Info "SSH Test PASS" + Test_Kill_Qemu + Test_Print_Result "SSH" 0 + exit 0 +else + Test_Info "SSH Test FAIL" + Test_Kill_Qemu + Test_Print_Result "SSH" 1 + exit 1 +fi