X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=vbuild-nightly.sh;h=56167d69d1039e5fef61abed611f25fc432206fa;hb=76ecb79f22ff98519312286274740240d8fe80bc;hp=436d037cebc62c7253ec3aa50fee0783a37fe65c;hpb=ab9c56bfe383075f93102b47dd0c3f32346f723a;p=build.git diff --git a/vbuild-nightly.sh b/vbuild-nightly.sh index 436d037c..56167d69 100755 --- a/vbuild-nightly.sh +++ b/vbuild-nightly.sh @@ -10,15 +10,21 @@ DEFAULT_PLDISTRO=planetlab DEFAULT_PERSONALITY=linux32 DEFAULT_BASE="@DATE@--@PLDISTRO@-@FCDISTRO@-@PERSONALITY@" DEFAULT_SVNPATH="http://svn.planet-lab.org/svn/build/trunk" +# TESTSVNPATH to be computed from the -tags.mk file - no default anymore +DEFAULT_TESTCONFIG="default" +DEFAULT_IFNAME=eth0 # web publishing results DEFAULT_WEBPATH="/build/@PLDISTRO@/" +# default gpg path used in signing yum repo +DEFAULT_GPGPATH="/etc/planetlab" +# default email to use in gpg secring +DEFAULT_GPGUID="root@$( /bin/hostname )" + # for the test part TESTBUILDURL="http://build.one-lab.org/" -TESTBOX=onelab-test.inria.fr -TESTBOXSSH=root@onelab-test.inria.fr -TESTSVNPATH="http://svn.planet-lab.org/svn/tests/trunk/system/" +TESTBOXSSH=root@testbox.one-lab.org #################### # assuming vserver runs in UTC DATE=$(date +'%Y.%m.%d') @@ -94,11 +100,12 @@ EOF function failure() { set -x WEBLOG=${WEBPATH}/${BASE}.log.txt + mkdir -p ${WEBPATH} cp $LOG ${WEBLOG} summary $LOG >> ${WEBLOG} - (echo -n "============================== $COMMAND: failure at " ; date ; tail -c 20k $WEBLOG) > ${WEBLOG}.ko + (echo -n "============================== $COMMAND: failure at " ; date ; tail -c 30k $WEBLOG) > ${WEBLOG}.ko if [ -n "$MAILTO" ] ; then - tail -c 20k ${WEBLOG} | mail -s "Failures for build ${BASE}" $MAILTO + tail -c 30k ${WEBLOG} | mail -s "Failures for build ${BASE}" $MAILTO fi exit 1 } @@ -106,9 +113,15 @@ function failure() { function success () { set -x WEBLOG=${WEBPATH}/${BASE}.log.txt + mkdir -p ${WEBPATH} cp $LOG ${WEBLOG} summary $LOG >> ${WEBLOG} - touch ${WEBLOG}.ok + if [ -n "$DO_TEST" ] ; then + echo "Successfully built and tested - see testlogs for details" > ${WEBLOG}.pass + rm -f ${WEBLOG}.ok + else + echo "Successfully built"> ${WEBLOG}.ok + fi if [ -n "$MAILTO" ] ; then (echo "$PLDISTRO ($BASE) build for $FCDISTRO completed on $(date)" ) | mail -s "Successful build for ${BASE}" $MAILTO fi @@ -138,11 +151,14 @@ function build () { MAKEVARS=("PLDISTROTAGS=${PLDISTROTAGS}" "${MAKEVARS[@]}") MAKEVARS=("NIGHTLY_BASE=${BASE}" "${MAKEVARS[@]}") MAKEVARS=("NIGHTLY_PERSONALITY=${PERSONALITY}" "${MAKEVARS[@]}") + MAKEVARS=("build-SVNPATH=${SVNPATH}" "${MAKEVARS[@]}") # stage1 make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true # versions make -C /build $DRY_RUN "${MAKEVARS[@]}" versions + # store testsvnpath + make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true testsvnpath # actual stuff make -C /build $DRY_RUN "${MAKEVARS[@]}" $MAKETARGETS @@ -157,26 +173,57 @@ function runtest () { echo -n "============================== Starting $COMMAND:runtest on $(date)" - ### the URL to the myplc package - rpm=$( (cd /vservers/$BASE/build/RPMS/i386 ; ls myplc-[0-9]*.rpm) ) - if [ ${#rpm[@]} != 1 ] ; then - echo "$COMMAND: Cannot locate rpm for testing" + # where to find TESTSVNPATH + stamp=/vservers/$BASE/build/testsvnpath + if [ ! -f $stamp ] ; then + echo "$COMMAND: Cannot figure TESTSVNPATH from missing $stamp" + failure + exit 1 + fi + TESTSVNPATH=$(cat $stamp) + # use only this pat of the tests right now + TESTSVNPATH=${TESTSVNPATH}/system + + ### the URL to the RPMS/ location + url="" + for a in i386 x86_64; do + archdir=/vservers/$BASE/build/RPMS/$a + if [ -d $archdir ] ; then + url=$(echo $archdir | sed -e "s,/vservers/$BASE/build,${TESTBUILDURL}${PLDISTRO}/${BASE},") + break + fi + done + + if [ -z "$url" ] ; then + echo "$COMMAND: Cannot locate arch URL for testing" failure exit 1 fi - url=${TESTBUILDURL}${PLDISTRO}/${BASE}/RPMS/i386/${rpm} - # compute test directory name on test box - testdir=test-${BASE} + # test directory name on test box + testdir=${BASE} # clean it - ssh ${TESTBOXSSH} rm -rf ${testdir} + ssh -n ${TESTBOXSSH} rm -rf ${testdir} # check it out - ssh ${TESTBOXSSH} svn co ${TESTSVNPATH} ${testdir} + ssh -n ${TESTBOXSSH} svn co ${TESTSVNPATH} ${testdir} # invoke test on testbox - pass url and build url - so the tests can use vtest-init-vserver.sh - ssh 2>&1 ${TESTBOXSSH} python -u ${testdir}/runtest --build ${SVNPATH} --url ${url} --all - # still missing - need to populate /var/www/html/install-rpms on the myplc - - if [ "$?" != 0 ] ; then + configs="" + for config in ${TESTCONFIG} ; do + configs="$configs --config $config" + done + + # need to proceed despite of set -e + success=true + ssh 2>&1 -n ${TESTBOXSSH} ${testdir}/runtest --build ${SVNPATH} --url ${url} $configs --all || success= + + # gather logs in the vserver + mkdir -p /vservers/$BASE/build/testlogs + ssh 2>&1 -n ${TESTBOXSSH} tar -C ${testdir}/logs -cf - . | tar -C /vservers/$BASE/build/testlogs -xf - || true + # push them to the build web + rsync --archive --delete /vservers/$BASE/build/testlogs/ $WEBPATH/$BASE/testlogs/ + chmod -R a+r $WEBPATH/$BASE/testlogs/ + + if [ -z "$success" ] ; then failure fi @@ -209,7 +256,6 @@ function usage () { echo "Usage: $COMMAND [option] make-targets" echo "This is $REVISION" echo "Supported options" - echo " -n dry-run : -n passed to make - vserver gets created though - no mail sent" echo " -f fcdistro - defaults to $DEFAULT_FCDISTRO" echo " -d pldistro - defaults to $DEFAULT_PLDISTRO" echo " -p personality - defaults to $DEFAULT_PERSONALITY" @@ -218,14 +264,20 @@ function usage () { echo " -t pldistrotags - defaults to \${PLDISTRO}-tags.mk" echo " -r tagsrelease - a release number that refers to PLDISTROTAGS - defaults to HEAD" echo " -s svnpath - where to fetch the build module" - echo " -o : overwrite - re-run in base directory, do not create vserver" - echo " -m mailto" - echo " -a makevar=value - space in values are not supported" + echo " -c testconfig - defaults to $DEFAULT_TESTCONFIG" echo " -w webpath - defaults to $DEFAULT_WEBPATH" + echo " -y create (and sign) yum repo in webpath" + echo " -g path to gpg secring used to sign rpms. Defaults to $DEFAULT_GPGPATH" + echo " -u gpg email used in secring. Defaults to $DEFAULT_GPGUID" + echo " -m mailto - no default" + echo " -O : overwrite - re-run in base directory, do not re-create vserver" echo " -B : run build only" echo " -T : run test only" + echo " -n dry-run : -n passed to make - vserver gets created though - no mail sent" echo " -v : be verbose" echo " -7 : uses weekday-@FCDISTRO@ as base" + echo " -a makevar=value - space in values are not supported" + echo " -i ifname - defaults to $DEFAULT_IFNAME - used to determine local IP" exit 1 } @@ -233,18 +285,13 @@ function main () { set -e - # preserve arguments for passing them again later - declare -a argv - for arg in "$@"; do argv=(${argv[@]} "$arg") ; done - # parse arguments MAKEVARS=() DRY_RUN= DO_BUILD=true DO_TEST=true - while getopts "nf:d:b:p:t:r:s:om:a:w:BTvh7" opt ; do + while getopts "f:d:p:b:t:r:s:x:c:w:g:u:m:OBTnyv7a:i:" opt ; do case $opt in - n) DRY_RUN="-n" ;; f) FCDISTRO=$OPTARG ;; d) PLDISTRO=$OPTARG ;; p) PERSONALITY=$OPTARG ;; @@ -252,19 +299,29 @@ function main () { t) PLDISTROTAGS=$OPTARG ;; r) TAGSRELEASE=$OPTARG ;; s) SVNPATH=$OPTARG ;; - o) OVERWRITEMODE=true ;; - m) MAILTO=$OPTARG ;; - a) MAKEVARS=(${MAKEVARS[@]} "$OPTARG") ;; + c) TESTCONFIG="$TESTCONFIG $OPTARG" ;; w) WEBPATH=$OPTARG ;; + y) MKYUMREPO=$OPTARG;; + g) GPGPATH=$OPTARG;; + u) GPGUID=$OPTARG;; + m) MAILTO=$OPTARG ;; + O) OVERWRITEMODE=true ;; B) DO_TEST= ;; T) DO_BUILD= ; OVERWRITEMODE=true ;; + n) DRY_RUN="-n" ;; v) set -x ;; 7) BASE="$(date +%a|tr A-Z a-z)-@FCDISTRO@" ;; + a) MAKEVARS=(${MAKEVARS[@]} "$OPTARG") ;; + i) IFNAME=$OPTARG ;; h|*) usage ;; esac done - shift $(($OPTIND - 1)) + # preserve options for passing them again later, together with expanded base + declare -a options + toshift=$(($OPTIND - 1)) + arg=1; while [ $arg -le $toshift ] ; do options=(${options[@]} "$1") ; shift; arg=$(($arg+1)) ; done + MAKETARGETS="$@" # set defaults @@ -274,7 +331,11 @@ function main () { [ -z "$PLDISTROTAGS" ] && PLDISTROTAGS="${PLDISTRO}-tags.mk" [ -z "$BASE" ] && BASE="$DEFAULT_BASE" [ -z "$WEBPATH" ] && WEBPATH="$DEFAULT_WEBPATH" + [ -z "$GPGPATH" ] && GPGPATH="$DEFAULT_GPGPATH" + [ -z "$GPGUID" ] && GPGUID="$DEFAULT_GPGUID" + [ -z "$IFNAME" ] && IFNAME="$DEFAULT_IFNAME" [ -z "$SVNPATH" ] && SVNPATH="$DEFAULT_SVNPATH" + [ -z "$TESTCONFIG" ] && TESTCONFIG="$DEFAULT_TESTCONFIG" [ -n "$DRY_RUN" ] && MAILTO="" @@ -309,6 +370,8 @@ function main () { set -x echo "XXXXXXXXXX $COMMAND: using existing vserver $BASE" $(date) show_env + # start in case e.g. we just rebooted + vserver ${BASE} start || : # update build vserver ${BASE} exec svn update /build else @@ -338,7 +401,7 @@ function main () { svn export $SVNPATH $tmpdir # Create vserver cd $tmpdir - ./vbuild-init-vserver.sh -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${BASE} + ./vbuild-init-vserver.sh -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} -i ${IFNAME} ${BASE} # cleanup cd - rm -rf $tmpdir @@ -365,14 +428,88 @@ function main () { # invoke this command in the vserver for building (-T) vserver ${BASE} exec chmod +x /build/$COMMAND - vserver ${BASE} exec /build/$COMMAND "${argv[@]}" -b "${BASE}" + vserver ${BASE} exec /build/$COMMAND "${options[@]}" -b "${BASE}" $MAKETARGETS fi # publish to the web so runtest can find them rm -rf $WEBPATH/$BASE ; mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS} rsync --archive --delete --verbose /vservers/$BASE/build/RPMS/ $WEBPATH/$BASE/RPMS/ rsync --archive --delete --verbose /vservers/$BASE/build/SRPMS/ $WEBPATH/$BASE/SRPMS/ - + # publish myplc-release + rsync --verbose /vservers/$BASE/build/myplc-release $WEBPATH/$BASE + + # create yum repo and sign packages. + if [ -n $MKYUMREPO ] ; then + echo "Signing signing node packages" + + ### availability of repo indexing tools + # old one - might be needed for old-style nodes + type -p yum-arch > /dev/null && have_yum_arch="true" + # new one + type -p createrepo > /dev/null && have_createrepo="true" + + repository=$WEBPATH/$BASE/RPMS/ + # the rpms that need signing + new_rpms= + # and the corresponding stamps + new_stamps= + # is there a need to refresh yum metadata + need_yum_arch= + need_createrepo= + + # right after installation, no package is present + # but we still need to create index + [ -n "$have_yum_arch" -a ! -f $repository/headers/header.info ] && need_yum_arch=true + [ -n "$have_createrepo" -a ! -f $repository/repodata/repomd.xml ] && need_createrepo=true + + for package in $(find $repository/ -name '*.rpm') ; do + stamp=$repository/signed-stamps/$(basename $package).signed + # If package is newer than signature stamp + if [ $package -nt $stamp ] ; then + new_rpms="$new_rpms $package" + new_stamps="$new_stamps $stamp" + fi + # Or than yum-arch headers + [ -n "$have_yum_arch" ] && [ $package -nt $repository/headers/header.info ] && need_yum +_arch=true + # Or than createrepo database + [ -n "$have_createrepo" ] && [ $package -nt $repository/repodata/repomd.xml ] && need_c +reaterepo=true + done + + if [ -n "$new_rpms" ] ; then + # Create a stamp once the package gets signed + mkdir $repository/signed-stamps 2> /dev/null + + # Sign RPMS. setsid detaches rpm from the terminal, + # allowing the (hopefully blank) GPG password to be + # entered from stdin instead of /dev/tty. + echo | setsid rpm \ + --define "_signature gpg" \ + --define "_gpg_path $GPGPATH" \ + --define "_gpg_name $GPGUID" \ + --resign $new_rpms && touch $new_stamps + fi + + # Update repository index / yum metadata. + + if [ -n "$need_yum_arch" ] ; then + # yum-arch sometimes leaves behind + # .oldheaders and .olddata directories accidentally. + rm -rf $repository/{.oldheaders,.olddata} + yum-arch $repository + fi + + if [ -n "$need_createrepo" ] ; then + if [ -f $repository/yumgroups.xml ] ; then + createrepo --quiet -g yumgroups.xml $repository + else + createrepo --quiet $repository + fi + fi + + fi + if [ -n "$DO_TEST" ] ; then runtest fi