X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=vbuild-nightly.sh;h=56167d69d1039e5fef61abed611f25fc432206fa;hb=af1010c8f8f2fa738340da096a67ec629e369bc2;hp=4846169cd9f3a339d510ffca5beed18d405d8ccc;hpb=3718978612537f60964d4c737a1b13317ad0623b;p=build.git diff --git a/vbuild-nightly.sh b/vbuild-nightly.sh index 4846169c..56167d69 100755 --- a/vbuild-nightly.sh +++ b/vbuild-nightly.sh @@ -10,13 +10,18 @@ DEFAULT_PLDISTRO=planetlab DEFAULT_PERSONALITY=linux32 DEFAULT_BASE="@DATE@--@PLDISTRO@-@FCDISTRO@-@PERSONALITY@" DEFAULT_SVNPATH="http://svn.planet-lab.org/svn/build/trunk" -DEFAULT_TESTSVNPATH="http://svn.planet-lab.org/svn/tests/trunk/system/" +# TESTSVNPATH to be computed from the -tags.mk file - no default anymore DEFAULT_TESTCONFIG="default" DEFAULT_IFNAME=eth0 # web publishing results DEFAULT_WEBPATH="/build/@PLDISTRO@/" +# default gpg path used in signing yum repo +DEFAULT_GPGPATH="/etc/planetlab" +# default email to use in gpg secring +DEFAULT_GPGUID="root@$( /bin/hostname )" + # for the test part TESTBUILDURL="http://build.one-lab.org/" TESTBOXSSH=root@testbox.one-lab.org @@ -146,11 +151,14 @@ function build () { MAKEVARS=("PLDISTROTAGS=${PLDISTROTAGS}" "${MAKEVARS[@]}") MAKEVARS=("NIGHTLY_BASE=${BASE}" "${MAKEVARS[@]}") MAKEVARS=("NIGHTLY_PERSONALITY=${PERSONALITY}" "${MAKEVARS[@]}") + MAKEVARS=("build-SVNPATH=${SVNPATH}" "${MAKEVARS[@]}") # stage1 make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true # versions make -C /build $DRY_RUN "${MAKEVARS[@]}" versions + # store testsvnpath + make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true testsvnpath # actual stuff make -C /build $DRY_RUN "${MAKEVARS[@]}" $MAKETARGETS @@ -165,38 +173,52 @@ function runtest () { echo -n "============================== Starting $COMMAND:runtest on $(date)" - ### the URL to the myplc package - rpm=$( find /vservers/$BASE/build/RPMS -name 'myplc-[0-9]*' ) - if [ ${#rpm[@]} != 1 ] ; then - echo "$COMMAND: Cannot locate rpm for testing" + # where to find TESTSVNPATH + stamp=/vservers/$BASE/build/testsvnpath + if [ ! -f $stamp ] ; then + echo "$COMMAND: Cannot figure TESTSVNPATH from missing $stamp" + failure + exit 1 + fi + TESTSVNPATH=$(cat $stamp) + # use only this pat of the tests right now + TESTSVNPATH=${TESTSVNPATH}/system + + ### the URL to the RPMS/ location + url="" + for a in i386 x86_64; do + archdir=/vservers/$BASE/build/RPMS/$a + if [ -d $archdir ] ; then + url=$(echo $archdir | sed -e "s,/vservers/$BASE/build,${TESTBUILDURL}${PLDISTRO}/${BASE},") + break + fi + done + + if [ -z "$url" ] ; then + echo "$COMMAND: Cannot locate arch URL for testing" failure exit 1 fi - url=$(echo $rpm | sed -e "s,/vservers/$BASE/build,${TESTBUILDURL}${PLDISTRO}/${BASE},") - # compute test directory name on test box - testdir=chroot-${BASE} - # use another name if any config contains vserver - echo $TESTCONFIG | grep vserver &> /dev/null && testdir=vserver-${BASE} + # test directory name on test box + testdir=${BASE} # clean it - ssh ${TESTBOXSSH} rm -rf ${testdir} + ssh -n ${TESTBOXSSH} rm -rf ${testdir} # check it out - ssh ${TESTBOXSSH} svn co ${TESTSVNPATH} ${testdir} + ssh -n ${TESTBOXSSH} svn co ${TESTSVNPATH} ${testdir} # invoke test on testbox - pass url and build url - so the tests can use vtest-init-vserver.sh configs="" for config in ${TESTCONFIG} ; do configs="$configs --config $config" done - [ "$PERSONALITY" == linux64 ] && personality="--personality linux64" - # need to proceed despite of set -e success=true - ssh 2>&1 ${TESTBOXSSH} ${testdir}/runtest --build ${SVNPATH} --url ${url} $configs $personality --all || success= + ssh 2>&1 -n ${TESTBOXSSH} ${testdir}/runtest --build ${SVNPATH} --url ${url} $configs --all || success= # gather logs in the vserver mkdir -p /vservers/$BASE/build/testlogs - ssh 2>&1 ${TESTBOXSSH} tar -C ${testdir}/logs -cf - . | tar -C /vservers/$BASE/build/testlogs -xf - || true + ssh 2>&1 -n ${TESTBOXSSH} tar -C ${testdir}/logs -cf - . | tar -C /vservers/$BASE/build/testlogs -xf - || true # push them to the build web rsync --archive --delete /vservers/$BASE/build/testlogs/ $WEBPATH/$BASE/testlogs/ chmod -R a+r $WEBPATH/$BASE/testlogs/ @@ -242,9 +264,11 @@ function usage () { echo " -t pldistrotags - defaults to \${PLDISTRO}-tags.mk" echo " -r tagsrelease - a release number that refers to PLDISTROTAGS - defaults to HEAD" echo " -s svnpath - where to fetch the build module" - echo " -x testsvnpath - defaults to $DEFAULT_TESTSVNPATH" echo " -c testconfig - defaults to $DEFAULT_TESTCONFIG" echo " -w webpath - defaults to $DEFAULT_WEBPATH" + echo " -y create (and sign) yum repo in webpath" + echo " -g path to gpg secring used to sign rpms. Defaults to $DEFAULT_GPGPATH" + echo " -u gpg email used in secring. Defaults to $DEFAULT_GPGUID" echo " -m mailto - no default" echo " -O : overwrite - re-run in base directory, do not re-create vserver" echo " -B : run build only" @@ -266,7 +290,7 @@ function main () { DRY_RUN= DO_BUILD=true DO_TEST=true - while getopts "f:d:p:b:t:r:s:x:c:w:m:OBTnv7a:i:" opt ; do + while getopts "f:d:p:b:t:r:s:x:c:w:g:u:m:OBTnyv7a:i:" opt ; do case $opt in f) FCDISTRO=$OPTARG ;; d) PLDISTRO=$OPTARG ;; @@ -275,9 +299,11 @@ function main () { t) PLDISTROTAGS=$OPTARG ;; r) TAGSRELEASE=$OPTARG ;; s) SVNPATH=$OPTARG ;; - x) TESTSVNPATH=$OPTARG ;; c) TESTCONFIG="$TESTCONFIG $OPTARG" ;; w) WEBPATH=$OPTARG ;; + y) MKYUMREPO=$OPTARG;; + g) GPGPATH=$OPTARG;; + u) GPGUID=$OPTARG;; m) MAILTO=$OPTARG ;; O) OVERWRITEMODE=true ;; B) DO_TEST= ;; @@ -305,9 +331,10 @@ function main () { [ -z "$PLDISTROTAGS" ] && PLDISTROTAGS="${PLDISTRO}-tags.mk" [ -z "$BASE" ] && BASE="$DEFAULT_BASE" [ -z "$WEBPATH" ] && WEBPATH="$DEFAULT_WEBPATH" + [ -z "$GPGPATH" ] && GPGPATH="$DEFAULT_GPGPATH" + [ -z "$GPGUID" ] && GPGUID="$DEFAULT_GPGUID" [ -z "$IFNAME" ] && IFNAME="$DEFAULT_IFNAME" [ -z "$SVNPATH" ] && SVNPATH="$DEFAULT_SVNPATH" - [ -z "$TESTSVNPATH" ] && TESTSVNPATH="$DEFAULT_TESTSVNPATH" [ -z "$TESTCONFIG" ] && TESTCONFIG="$DEFAULT_TESTCONFIG" [ -n "$DRY_RUN" ] && MAILTO="" @@ -343,6 +370,8 @@ function main () { set -x echo "XXXXXXXXXX $COMMAND: using existing vserver $BASE" $(date) show_env + # start in case e.g. we just rebooted + vserver ${BASE} start || : # update build vserver ${BASE} exec svn update /build else @@ -406,7 +435,81 @@ function main () { rm -rf $WEBPATH/$BASE ; mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS} rsync --archive --delete --verbose /vservers/$BASE/build/RPMS/ $WEBPATH/$BASE/RPMS/ rsync --archive --delete --verbose /vservers/$BASE/build/SRPMS/ $WEBPATH/$BASE/SRPMS/ - + # publish myplc-release + rsync --verbose /vservers/$BASE/build/myplc-release $WEBPATH/$BASE + + # create yum repo and sign packages. + if [ -n $MKYUMREPO ] ; then + echo "Signing signing node packages" + + ### availability of repo indexing tools + # old one - might be needed for old-style nodes + type -p yum-arch > /dev/null && have_yum_arch="true" + # new one + type -p createrepo > /dev/null && have_createrepo="true" + + repository=$WEBPATH/$BASE/RPMS/ + # the rpms that need signing + new_rpms= + # and the corresponding stamps + new_stamps= + # is there a need to refresh yum metadata + need_yum_arch= + need_createrepo= + + # right after installation, no package is present + # but we still need to create index + [ -n "$have_yum_arch" -a ! -f $repository/headers/header.info ] && need_yum_arch=true + [ -n "$have_createrepo" -a ! -f $repository/repodata/repomd.xml ] && need_createrepo=true + + for package in $(find $repository/ -name '*.rpm') ; do + stamp=$repository/signed-stamps/$(basename $package).signed + # If package is newer than signature stamp + if [ $package -nt $stamp ] ; then + new_rpms="$new_rpms $package" + new_stamps="$new_stamps $stamp" + fi + # Or than yum-arch headers + [ -n "$have_yum_arch" ] && [ $package -nt $repository/headers/header.info ] && need_yum +_arch=true + # Or than createrepo database + [ -n "$have_createrepo" ] && [ $package -nt $repository/repodata/repomd.xml ] && need_c +reaterepo=true + done + + if [ -n "$new_rpms" ] ; then + # Create a stamp once the package gets signed + mkdir $repository/signed-stamps 2> /dev/null + + # Sign RPMS. setsid detaches rpm from the terminal, + # allowing the (hopefully blank) GPG password to be + # entered from stdin instead of /dev/tty. + echo | setsid rpm \ + --define "_signature gpg" \ + --define "_gpg_path $GPGPATH" \ + --define "_gpg_name $GPGUID" \ + --resign $new_rpms && touch $new_stamps + fi + + # Update repository index / yum metadata. + + if [ -n "$need_yum_arch" ] ; then + # yum-arch sometimes leaves behind + # .oldheaders and .olddata directories accidentally. + rm -rf $repository/{.oldheaders,.olddata} + yum-arch $repository + fi + + if [ -n "$need_createrepo" ] ; then + if [ -f $repository/yumgroups.xml ] ; then + createrepo --quiet -g yumgroups.xml $repository + else + createrepo --quiet $repository + fi + fi + + fi + if [ -n "$DO_TEST" ] ; then runtest fi