From: parmentelat Date: Sun, 2 Dec 2018 16:15:44 +0000 (+0100) Subject: use dnf instead of yum, and other untabifications X-Git-Tag: py3ok~55 X-Git-Url: http://git.onelab.eu/?p=build.git;a=commitdiff_plain;h=f32374281fa212ac418b4679551f3d33a0007ba3 use dnf instead of yum, and other untabifications --- diff --git a/lbuild-initvm.sh b/lbuild-initvm.sh index ccbb7b16..5a4cc19e 100755 --- a/lbuild-initvm.sh +++ b/lbuild-initvm.sh @@ -45,7 +45,7 @@ VIF_GUEST=eth0 ########## FEDORA_MIRROR_BASE="http://mirror.onelab.eu/fedora/" FEDORA_MIRROR_KEYS="http://mirror.onelab.eu/keys/" -FEDORA_PREINSTALLED="yum initscripts passwd rsyslog vim-minimal dhclient chkconfig rootfiles policycoreutils openssh-server openssh-clients" +FEDORA_PREINSTALLED="dnf dnf-yum passwd rsyslog vim-minimal dhclient chkconfig rootfiles policycoreutils openssh-server openssh-clients" DEBIAN_PREINSTALLED="openssh-server openssh-client" ########## networking utilities @@ -77,12 +77,12 @@ EOF } ############################## -# return yum or debootstrap +# return dnf or debootstrap function package_method () { fcdistro=$1; shift case $fcdistro in f[0-9]*|centos[0-9]*|sl[0-9]*) - echo yum ;; + echo dnf ;; wheezy|jessie|precise|trusty|utopic|vivid|wily|xenial) echo debootstrap ;; *) @@ -95,7 +95,7 @@ function canonical_arch () { personality=$1; shift fcdistro=$1; shift case $(package_method $fcdistro) in - yum) + dnf) case $personality in *32) echo i386 ;; *64) echo x86_64 ;; *) echo Unknown-arch-1 ;; esac ;; debootstrap) case $personality in *32) echo i386 ;; *64) echo amd64 ;; *) echo Unknown-arch-2 ;; esac ;; @@ -132,7 +132,7 @@ function fedora_install() { fedora_download $cache || { echo "Failed to download 'fedora base'"; return 1; } else echo "Updating cache $cache/rootfs ..." - if ! yum --installroot $cache/rootfs --releasever ${fedora_release} -y --nogpgcheck update ; then + if ! dnf --installroot $cache/rootfs --releasever ${fedora_release} -y --nogpgcheck update ; then echo "Failed to update 'fedora base', continuing with last known good cache" else echo "Update finished" @@ -178,21 +178,22 @@ function fedora_download() { done MIRROR_URL=$FEDORA_MIRROR_BASE/releases/${fedora_release}/Everything/$arch/os - RELEASE_URL1="$MIRROR_URL/Packages/fedora-release-${fedora_release}-1.noarch.rpm" - # with fedora18 the rpms are scattered by first name + # since fedora18 the rpms are scattered by first name # first try the second version of fedora-release first - RELEASE_URL2="$MIRROR_URL/Packages/f/fedora-release-${fedora_release}-2.noarch.rpm" - RELEASE_URL3="$MIRROR_URL/Packages/f/fedora-release-${fedora_release}-1.noarch.rpm" + RELEASE_URLS="" + for subindex in 3 2 1; do + RELEASE_URLS="$RELEASE_URLS $MIRROR_URL/Packages/f/fedora-release-${fedora_release}-1.noarch.rpm" + done RELEASE_TARGET=$INSTALL_ROOT/fedora-release-${fedora_release}.noarch.rpm found="" - for attempt in $RELEASE_URL1 $RELEASE_URL2 $RELEASE_URL3; do - if curl -f $attempt -o $RELEASE_TARGET ; then - echo "Retrieved $attempt" + for attempt in $RELEASE_URLS; do + if curl --silent --fail $attempt -o $RELEASE_TARGET; then + echo "Successfully Retrieved $attempt" found=true break else - echo "Failed attempt $attempt" + echo "Failed (not to worry about) with attempt $attempt" fi done [ -n "$found" ] || { echo "Could not retrieve fedora-release rpm - exiting" ; exit 1; } @@ -211,9 +212,9 @@ function fedora_download() { # So ideally if we want to be able to build f12 images from f18 we need an rpm that has # this patch undone, like we have in place on our f14 boxes (our f14 boxes need a f18-like rpm) - YUM="yum --installroot=$INSTALL_ROOT --releasever=${fedora_release} --nogpgcheck -y" - echo "$YUM install $FEDORA_PREINSTALLED" - $YUM install $FEDORA_PREINSTALLED || { echo "Failed to download rootfs, aborting." ; return 1; } + DNF="dnf --installroot=$INSTALL_ROOT --releasever=${fedora_release} --nogpgcheck -y" + echo "$DNF install $FEDORA_PREINSTALLED" + $DNF install $FEDORA_PREINSTALLED || { echo "Failed to download rootfs, aborting." ; return 1; } mv "$INSTALL_ROOT" "$cache/rootfs" echo "Download complete." @@ -466,11 +467,11 @@ function setup_lxc() { pkg_method=$(package_method $fcdistro) case $pkg_method in - yum) + dnf) if [ -z "$IMAGE" ]; then fedora_install $lxc || { echo "failed to install fedora root image"; exit 1 ; } # this appears to be safer; observed in Jan. 2016 on a f23 host and a f14 cached image - # we were getting this message when attempting the first chroot yum install + # we were getting this message when attempting the first chroot dnf install # rpmdb: Program version 4.8 doesn't match environment version 5.3 chroot $(lxcroot $lxc) $personality rm -rf /var/lib/rpm/__db.00{0,1,2,3,4,5,6,7,8,9} chroot $(lxcroot $lxc) $personality rpm --rebuilddb @@ -653,7 +654,7 @@ function devel_or_test_tools () { groups=$(pl_getGroups -a $lxc_arch $fcdistro $pldistro $pkgsfile) case "$pkg_method" in - yum) + dnf) # --allowerasing required starting with fedora24 # has_dnf="" @@ -663,9 +664,9 @@ function devel_or_test_tools () { pkg_installer="dnf -y install --allowerasing" grp_installer="dnf -y groupinstall --allowerasing" else - echo "container has only yum" - pkg_installer="yum -y install" - grp_installer="yum -y groupinstall" + echo "container has only dnf" + pkg_installer="dnf -y install" + grp_installer="dnf -y groupinstall" fi [ -n "$packages" ] && chroot ${lxc_root} $personality $pkg_installer $packages for group_plus in $groups; do diff --git a/lbuild-nightly.sh b/lbuild-nightly.sh index ace4b474..715d3684 100755 --- a/lbuild-nightly.sh +++ b/lbuild-nightly.sh @@ -162,27 +162,27 @@ function failure() { set -x # early stage ? - let's not create /build/@PLDISTRO@ if [ -z "$WEBLOG" ] ; then - WEBHOST=localhost - WEBPATH=/tmp - WEBBASE=/tmp/lbuild-early-$(date +%Y-%m-%d) - WEBLOG=/tmp/lbuild-early-$(date +%Y-%m-%d).log.txt + WEBHOST=localhost + WEBPATH=/tmp + WEBBASE=/tmp/lbuild-early-$(date +%Y-%m-%d) + WEBLOG=/tmp/lbuild-early-$(date +%Y-%m-%d).log.txt fi webpublish mkdir -p $WEBBASE ||: webpublish_cp_local_to_remote $LOG $WEBLOG ||: summary $LOG | webpublish_append_stdin_to_file $WEBLOG ||: (echo -n "============================== $COMMAND: failure at " ; date ; \ - webpublish tail --lines=1000 $WEBLOG) | \ - webpublish_cp_stdin_to_file $WEBBASE.ko ||: + webpublish tail --lines=1000 $WEBLOG) | \ + webpublish_cp_stdin_to_file $WEBBASE.ko ||: if [ -n "$MAILDEST" ] ; then - ( \ - echo "Subject: KO ${BASE} ${MAIL_SUBJECT}" ; \ - echo "To: $MAILDEST" ; \ - echo "see build results at $WEBBASE_URL" ; \ - echo "including full build log at $WEBBASE_URL/log.txt" ; \ - echo "and complete test logs at $WEBBASE_URL/testlogs" ; \ - echo "........................................" ; \ - webpublish tail --lines=1000 $WEBLOG ) | \ - sendmail $MAILDEST + ( \ + echo "Subject: KO ${BASE} ${MAIL_SUBJECT}" ; \ + echo "To: $MAILDEST" ; \ + echo "see build results at $WEBBASE_URL" ; \ + echo "including full build log at $WEBBASE_URL/log.txt" ; \ + echo "and complete test logs at $WEBBASE_URL/testlogs" ; \ + echo "........................................" ; \ + webpublish tail --lines=1000 $WEBLOG ) | \ + sendmail $MAILDEST fi exit 1 } @@ -191,47 +191,47 @@ function success () { set -x # early stage ? - let's not create /build/@PLDISTRO@ if [ -z "$WEBLOG" ] ; then - WEBHOST=localhost - WEBPATH=/tmp - WEBLOG=/tmp/lbuild-early-$(date +%Y-%m-%d).log.txt + WEBHOST=localhost + WEBPATH=/tmp + WEBLOG=/tmp/lbuild-early-$(date +%Y-%m-%d).log.txt fi webpublish mkdir -p $WEBBASE webpublish_cp_local_to_remote $LOG $WEBLOG summary $LOG | webpublish_append_stdin_to_file $WEBLOG if [ -n "$DO_TEST" ] ; then - short_message="PASS" - ext="pass" - if [ -n "$IGNORED" ] ; then short_message="PASS/WARN"; ext="warn"; fi - ( \ - echo "Successfully built and tested" ; \ - echo "see build results at $WEBBASE_URL" ; \ - echo "including full build log at $WEBBASE_URL/log.txt" ; \ - echo "and complete test logs at $WEBBASE_URL/testlogs" ; \ - [ -n "$IGNORED" ] && echo "WARNING: some tests steps failed but were ignored - see trace file" ; \ - ) | webpublish_cp_stdin_to_file $WEBBASE.$ext - webpublish rm -f $WEBBASE.pkg-ok $WEBBASE.ko + short_message="PASS" + ext="pass" + if [ -n "$IGNORED" ] ; then short_message="PASS/WARN"; ext="warn"; fi + ( \ + echo "Successfully built and tested" ; \ + echo "see build results at $WEBBASE_URL" ; \ + echo "including full build log at $WEBBASE_URL/log.txt" ; \ + echo "and complete test logs at $WEBBASE_URL/testlogs" ; \ + [ -n "$IGNORED" ] && echo "WARNING: some tests steps failed but were ignored - see trace file" ; \ + ) | webpublish_cp_stdin_to_file $WEBBASE.$ext + webpublish rm -f $WEBBASE.pkg-ok $WEBBASE.ko else - short_message="PKGOK" - ( \ - echo "Successful package-only build, no test requested" ; \ - echo "see build results at $WEBBASE_URL" ; \ - echo "including full build log at $WEBBASE_URL/log.txt" ; \ - ) | webpublish_cp_stdin_to_file $WEBBASE.pkg-ok - webpublish rm -f $WEBBASE.ko + short_message="PKGOK" + ( \ + echo "Successful package-only build, no test requested" ; \ + echo "see build results at $WEBBASE_URL" ; \ + echo "including full build log at $WEBBASE_URL/log.txt" ; \ + ) | webpublish_cp_stdin_to_file $WEBBASE.pkg-ok + webpublish rm -f $WEBBASE.ko fi BUILD_END=$(date +'%H:%M') BUILD_END_S=$(date +'%s') if [ -n "$MAILDEST" ] ; then - ( \ - echo "Subject: $short_message ${BASE} ${MAIL_SUBJECT}" ; \ - echo "To: $MAILDEST" ; \ - echo "$PLDISTRO ($BASE) build for $FCDISTRO completed on $(date)" ; \ - echo "see build results at $WEBBASE_URL" ; \ - echo "including full build log at $WEBBASE_URL/log.txt" ; \ + ( \ + echo "Subject: $short_message ${BASE} ${MAIL_SUBJECT}" ; \ + echo "To: $MAILDEST" ; \ + echo "$PLDISTRO ($BASE) build for $FCDISTRO completed on $(date)" ; \ + echo "see build results at $WEBBASE_URL" ; \ + echo "including full build log at $WEBBASE_URL/log.txt" ; \ [ -n "$DO_TEST" ] && echo "and complete test logs at $WEBBASE_URL/testlogs" ; \ - [ -n "$IGNORED" ] && echo "WARNING: some tests steps failed but were ignored - see trace file" ; \ - echo "BUILD TIME: begin $BUILD_BEG -- end $BUILD_END -- duration $(pretty_duration $(($BUILD_END_S-$BUILD_BEG_S)))" ; \ - ) | sendmail $MAILDEST + [ -n "$IGNORED" ] && echo "WARNING: some tests steps failed but were ignored - see trace file" ; \ + echo "BUILD TIME: begin $BUILD_BEG -- end $BUILD_END -- duration $(pretty_duration $(($BUILD_END_S-$BUILD_BEG_S)))" ; \ + ) | sendmail $MAILDEST fi # XXX For some reason, we haven't been getting this email for successful builds. If this sleep # doesn't fix the problem, I'll remove it -- Sapan. @@ -301,19 +301,19 @@ function run_log () { # f12 now has everything in i686; try i386 first as older fedoras have both url="" for a in i386 i686 x86_64; do - archdir=$(rootdir $BASE)/build/RPMS/$a - if [ -d $archdir ] ; then - # where was that installed - url=$(echo $archdir | sed -e "s,$(rootdir $BASE)/build,${WEBPATH}/${BASE},") - url=$(echo $url | sed -e "s,${WEBROOT},${TESTBUILDURL},") - break - fi + archdir=$(rootdir $BASE)/build/RPMS/$a + if [ -d $archdir ] ; then + # where was that installed + url=$(echo $archdir | sed -e "s,$(rootdir $BASE)/build,${WEBPATH}/${BASE},") + url=$(echo $url | sed -e "s,${WEBROOT},${TESTBUILDURL},") + break + fi done if [ -z "$url" ] ; then - echo "$COMMAND: Cannot locate arch URL for testing" - failure - exit 1 + echo "$COMMAND: Cannot locate arch URL for testing" + failure + exit 1 fi testmaster_ssh="root@${TESTMASTER}" @@ -345,9 +345,9 @@ function run_log () { # interpret retcod of TestMain.py; 2 means there were ignored steps that failed echo "retcod from run_log" $retcod case $retcod in - 0) success=true; IGNORED="" ;; - 2) success=true; IGNORED=true ;; - *) success=""; IGNORED="" ;; + 0) success=true; IGNORED="" ;; + 2) success=true; IGNORED=true ;; + *) success=""; IGNORED="" ;; esac # gather logs in the build vm @@ -360,8 +360,8 @@ function run_log () { echo "============================== END $COMMAND:run_log on $(date)" if [ -z "$success" ] ; then - echo "Tests have failed - bailing out" - failure + echo "Tests have failed - bailing out" + failure fi } @@ -408,7 +408,7 @@ function sign_node_packages () { # Update repository index / yum metadata. if [ -n "$need_createrepo" ] ; then - echo "Indexing node packages after signing" + echo "Indexing node packages after signing" if [ -f $repository/yumgroups.xml ] ; then createrepo --quiet -g yumgroups.xml $repository else @@ -432,15 +432,15 @@ function show_env () { echo TESTBUILDURL="$TESTBUILDURL" echo WEBHOST="$WEBHOST" if in_root_context ; then - echo PLDISTROTAGS="$PLDISTROTAGS" + echo PLDISTROTAGS="$PLDISTROTAGS" else - if [ -f /build/$PLDISTROTAGS ] ; then - echo "XXXXXXXXXXXXXXXXXXXX Contents of tags definition file /build/$PLDISTROTAGS" - cat /build/$PLDISTROTAGS - echo "XXXXXXXXXXXXXXXXXXXX end tags definition" - else - echo "XXXXXXXXXXXXXXXXXXXX Cannot find tags definition file /build/$PLDISTROTAGS, assuming remote pldistro" - fi + if [ -f /build/$PLDISTROTAGS ] ; then + echo "XXXXXXXXXXXXXXXXXXXX Contents of tags definition file /build/$PLDISTROTAGS" + cat /build/$PLDISTROTAGS + echo "XXXXXXXXXXXXXXXXXXXX end tags definition" + else + echo "XXXXXXXXXXXXXXXXXXXX Cannot find tags definition file /build/$PLDISTROTAGS, assuming remote pldistro" + fi fi set -x } @@ -450,16 +450,16 @@ function setupssh () { sshkey=$1; shift if [ -f ${sshkey} ] ; then - SSHDIR=$(rootdir ${base})/root/.ssh - mkdir -p ${SSHDIR} - cp $sshkey ${SSHDIR}/thekey - (echo "host *"; \ - echo " IdentityFile ~/.ssh/thekey"; \ - echo " StrictHostKeyChecking no" ) > ${SSHDIR}/config - chmod 700 ${SSHDIR} - chmod 400 ${SSHDIR}/* + SSHDIR=$(rootdir ${base})/root/.ssh + mkdir -p ${SSHDIR} + cp $sshkey ${SSHDIR}/thekey + (echo "host *"; \ + echo " IdentityFile ~/.ssh/thekey"; \ + echo " StrictHostKeyChecking no" ) > ${SSHDIR}/config + chmod 700 ${SSHDIR} + chmod 400 ${SSHDIR}/* else - echo "WARNING : could not find provided ssh key $sshkey - ignored" + echo "WARNING : could not find provided ssh key $sshkey - ignored" fi } @@ -525,48 +525,48 @@ function main () { fi eval set -- "$OPTS" while true; do - case $1 in - -f) FCDISTRO=$2; shift 2 ;; - -d) PLDISTRO=$2; shift 2 ;; - -p) PERSONALITY=$2; shift 2 ;; - -m) MAILDEST=$2; shift 2 ;; - -s) BUILD_SCM_URL=$2; shift 2 ;; - -t) PLDISTROTAGS=$2; shift 2 ;; - -b) BASE=$2; shift 2 ;; - -o) OVERBASE=$2; shift 2 ;; - -c) TESTCONFIG="$TESTCONFIG $2"; shift 2 ;; - ########## passing stuff to run_log - # -y foo -> run_log -y foo - -y) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --rspec-style $2"; shift 2 ;; - # -e foo -> run_log -e foo - -e) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --exclude $2"; shift 2 ;; - -i) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --ignore $2"; shift 2 ;; - # -X -> run_log --lxc - -X) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --lxc"; shift;; - # -S -> run_log --vs - -S) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --vs"; shift;; - # more general form to pass args to run_log - # -x foo -> run_log foo - -x) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS $2"; shift 2;; - ########## - -w) WEBPATH=$2; shift 2 ;; - -W) TESTBUILDURL=$2; shift 2 ;; - -r) WEBROOT=$2; shift 2 ;; - -M) TESTMASTER=$2; shift 2 ;; + case $1 in + -f) FCDISTRO=$2; shift 2 ;; + -d) PLDISTRO=$2; shift 2 ;; + -p) PERSONALITY=$2; shift 2 ;; + -m) MAILDEST=$2; shift 2 ;; + -s) BUILD_SCM_URL=$2; shift 2 ;; + -t) PLDISTROTAGS=$2; shift 2 ;; + -b) BASE=$2; shift 2 ;; + -o) OVERBASE=$2; shift 2 ;; + -c) TESTCONFIG="$TESTCONFIG $2"; shift 2 ;; + ########## passing stuff to run_log + # -y foo -> run_log -y foo + -y) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --rspec-style $2"; shift 2 ;; + # -e foo -> run_log -e foo + -e) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --exclude $2"; shift 2 ;; + -i) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --ignore $2"; shift 2 ;; + # -X -> run_log --lxc + -X) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --lxc"; shift;; + # -S -> run_log --vs + -S) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --vs"; shift;; + # more general form to pass args to run_log + # -x foo -> run_log foo + -x) RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS $2"; shift 2;; + ########## + -w) WEBPATH=$2; shift 2 ;; + -W) TESTBUILDURL=$2; shift 2 ;; + -r) WEBROOT=$2; shift 2 ;; + -M) TESTMASTER=$2; shift 2 ;; -Y) SIGNYUMREPO=true; shift ;; -g) GPGPATH=$2; shift 2 ;; -u) GPGUID=$2; shift 2 ;; - -K) SSH_KEY=$2; shift 2 ;; - -S) PUBLISH_SRPMS="" ; shift ;; - -B) DO_TEST= ; shift ;; - -T) DO_BUILD= ; shift;; - -n) DRY_RUN="-n" ; shift ;; - -v) set -x ; VERBOSE="-v" ; shift ;; - -7) BASE="$(date +%a|tr A-Z a-z)-@FCDISTRO@" ; shift ;; - -P) PREINSTALLED="-P $2"; shift 2;; - -h) usage ; shift ;; + -K) SSH_KEY=$2; shift 2 ;; + -S) PUBLISH_SRPMS="" ; shift ;; + -B) DO_TEST= ; shift ;; + -T) DO_BUILD= ; shift;; + -n) DRY_RUN="-n" ; shift ;; + -v) set -x ; VERBOSE="-v" ; shift ;; + -7) BASE="$(date +%a|tr A-Z a-z)-@FCDISTRO@" ; shift ;; + -P) PREINSTALLED="-P $2"; shift 2;; + -h) usage ; shift ;; --) shift; break ;; - esac + esac done # preserve options for passing them again later, together with expanded base @@ -574,13 +574,13 @@ function main () { # allow var=value stuff; for target in "$@" ; do - # check if contains '=' - target1=$(echo $target | sed -e s,=,,) - if [ "$target" = "$target1" ] ; then - MAKETARGETS=(${MAKETARGETS[@]} "$target") - else - MAKEVARS=(${MAKEVARS[@]} "$target") - fi + # check if contains '=' + target1=$(echo $target | sed -e s,=,,) + if [ "$target" = "$target1" ] ; then + MAKETARGETS=(${MAKETARGETS[@]} "$target") + else + MAKEVARS=(${MAKEVARS[@]} "$target") + fi done # set defaults @@ -603,30 +603,30 @@ function main () { # elaborate the extra args to be passed to run_log for config in ${TESTCONFIG} ; do - RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --config $config" + RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --config $config" done if [ -n "$OVERBASE" ] ; then - sedargs="-e s,@DATE@,${DATE},g" - BASE=$(echo ${OVERBASE} | sed $sedargs) + sedargs="-e s,@DATE@,${DATE},g" + BASE=$(echo ${OVERBASE} | sed $sedargs) else - sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" - BASE=$(echo ${BASE} | sed $sedargs) + sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" + BASE=$(echo ${BASE} | sed $sedargs) fi ### elaborate mail subject if [ -n "$DO_BUILD" -a -n "$DO_TEST" ] ; then - MAIL_SUBJECT="full" + MAIL_SUBJECT="full" elif [ -n "$DO_BUILD" ] ; then - MAIL_SUBJECT="pkg-only" + MAIL_SUBJECT="pkg-only" elif [ -n "$DO_TEST" ] ; then - MAIL_SUBJECT="test-only" + MAIL_SUBJECT="test-only" fi if [ -n "$OVERBASE" ] ; then - MAIL_SUBJECT="${MAIL_SUBJECT} rerun" + MAIL_SUBJECT="${MAIL_SUBJECT} rerun" else - MAIL_SUBJECT="${MAIL_SUBJECT} fresh" + MAIL_SUBJECT="${MAIL_SUBJECT} fresh" fi short_hostname=$(hostname | cut -d. -f1) MAIL_SUBJECT="on ${short_hostname} - ${MAIL_SUBJECT}" @@ -638,169 +638,169 @@ function main () { if ! in_root_context ; then # in the vm - echo "==================== Within LXC BEG $(date)" - build - echo "==================== Within LXC END $(date)" + echo "==================== Within LXC BEG $(date)" + build + echo "==================== Within LXC END $(date)" else - trap failure ERR INT + trap failure ERR INT # we run in the root context : # (*) create or check for the vm to use # (*) copy this command in the vm # (*) invoke it - if [ -n "$OVERBASE" ] ; then + if [ -n "$OVERBASE" ] ; then ### Re-use a vm (finish an unfinished build..) - if [ ! -d $(rootdir ${BASE}) ] ; then - echo $COMMAND : cannot find vm $BASE - exit 1 - fi - # manage LOG - beware it might be a symlink so nuke it first - LOG=$(logfile ${BASE}) - rm -f $LOG - exec > $LOG 2>&1 - set -x - echo "XXXXXXXXXX $COMMAND: using existing vm $BASE" $(date) - # start in case e.g. we just rebooted - virsh -c lxc:/// start ${BASE} || : - # retrieve environment from the previous run - FCDISTRO=$(run_in_build_guest $BASE /build/getdistroname.sh) - BUILD_SCM_URL=$(run_in_build_guest $BASE make --no-print-directory -C /build stage1=skip +build-GITPATH) - # for efficiency, crop everything in one make run - tmp=/tmp/${BASE}-env.sh - run_in_build_guest $BASE make --no-print-directory -C /build stage1=skip \ - ++PLDISTRO ++PLDISTROTAGS ++PERSONALITY ++MAILDEST ++WEBPATH ++TESTBUILDURL ++WEBROOT > $tmp - . $tmp - rm -f $tmp - # update build - [ -n "$SSH_KEY" ] && setupssh ${BASE} ${SSH_KEY} - run_in_build_guest $BASE "(cd /build; git pull; make tests-clean)" - # make sure we refresh the tests place in case it has changed - rm -f /build/MODULES/tests - options=(${options[@]} -d $PLDISTRO -t $PLDISTROTAGS -s $BUILD_SCM_URL) - [ -n "$PERSONALITY" ] && options=(${options[@]} -p $PERSONALITY) - [ -n "$MAILDEST" ] && options=(${options[@]} -m $MAILDEST) - [ -n "$WEBPATH" ] && options=(${options[@]} -w $WEBPATH) - [ -n "$TESTBUILDURL" ] && options=(${options[@]} -W $TESTBUILDURL) - [ -n "$WEBROOT" ] && options=(${options[@]} -r $WEBROOT) - show_env - else - # create vm: check it does not exist yet - i= - while [ -d $(rootdir ${BASE})${i} ] ; do - # we name subsequent builds -n so the logs and builds get sorted properly - [ -z ${i} ] && BASE=${BASE}-n - i=$((${i}+1)) - if [ $i -gt 100 ] ; then - echo "$COMMAND: Failed to create build vm $(rootdir ${BASE})${i}" - exit 1 - fi - done - BASE=${BASE}${i} - # need update - # manage LOG - beware it might be a symlink so nuke it first - LOG=$(logfile ${BASE}) - rm -f $LOG - exec > $LOG 2>&1 - set -x - echo "XXXXXXXXXX $COMMAND: creating vm $BASE" $(date) - show_env - - ### extract the whole build - much simpler - tmpdir=/tmp/$COMMAND-$$ - GIT_REPO=$(echo $BUILD_SCM_URL | cut -d@ -f1) - GIT_TAG=$(echo $BUILD_SCM_URL | cut -s -d@ -f2) - GIT_TAG=${GIT_TAG:-master} - mkdir -p $tmpdir + if [ ! -d $(rootdir ${BASE}) ] ; then + echo $COMMAND : cannot find vm $BASE + exit 1 + fi + # manage LOG - beware it might be a symlink so nuke it first + LOG=$(logfile ${BASE}) + rm -f $LOG + exec > $LOG 2>&1 + set -x + echo "XXXXXXXXXX $COMMAND: using existing vm $BASE" $(date) + # start in case e.g. we just rebooted + virsh -c lxc:/// start ${BASE} || : + # retrieve environment from the previous run + FCDISTRO=$(run_in_build_guest $BASE /build/getdistroname.sh) + BUILD_SCM_URL=$(run_in_build_guest $BASE make --no-print-directory -C /build stage1=skip +build-GITPATH) + # for efficiency, crop everything in one make run + tmp=/tmp/${BASE}-env.sh + run_in_build_guest $BASE make --no-print-directory -C /build stage1=skip \ + ++PLDISTRO ++PLDISTROTAGS ++PERSONALITY ++MAILDEST ++WEBPATH ++TESTBUILDURL ++WEBROOT > $tmp + . $tmp + rm -f $tmp + # update build + [ -n "$SSH_KEY" ] && setupssh ${BASE} ${SSH_KEY} + run_in_build_guest $BASE "(cd /build; git pull; make tests-clean)" + # make sure we refresh the tests place in case it has changed + rm -f /build/MODULES/tests + options=(${options[@]} -d $PLDISTRO -t $PLDISTROTAGS -s $BUILD_SCM_URL) + [ -n "$PERSONALITY" ] && options=(${options[@]} -p $PERSONALITY) + [ -n "$MAILDEST" ] && options=(${options[@]} -m $MAILDEST) + [ -n "$WEBPATH" ] && options=(${options[@]} -w $WEBPATH) + [ -n "$TESTBUILDURL" ] && options=(${options[@]} -W $TESTBUILDURL) + [ -n "$WEBROOT" ] && options=(${options[@]} -r $WEBROOT) + show_env + else + # create vm: check it does not exist yet + i= + while [ -d $(rootdir ${BASE})${i} ] ; do + # we name subsequent builds -n so the logs and builds get sorted properly + [ -z ${i} ] && BASE=${BASE}-n + i=$((${i}+1)) + if [ $i -gt 100 ] ; then + echo "$COMMAND: Failed to create build vm $(rootdir ${BASE})${i}" + exit 1 + fi + done + BASE=${BASE}${i} + # need update + # manage LOG - beware it might be a symlink so nuke it first + LOG=$(logfile ${BASE}) + rm -f $LOG + exec > $LOG 2>&1 + set -x + echo "XXXXXXXXXX $COMMAND: creating vm $BASE" $(date) + show_env + + ### extract the whole build - much simpler + tmpdir=/tmp/$COMMAND-$$ + GIT_REPO=$(echo $BUILD_SCM_URL | cut -d@ -f1) + GIT_TAG=$(echo $BUILD_SCM_URL | cut -s -d@ -f2) + GIT_TAG=${GIT_TAG:-master} + mkdir -p $tmpdir ( git archive --remote=$GIT_REPO $GIT_TAG | tar -C $tmpdir -xf -) || \ - ( echo "==================== git archive FAILED, trying git clone instead" ; \ - git clone $GIT_REPO $tmpdir && cd $tmpdir && git checkout $GIT_TAG && rm -rf .git) + ( echo "==================== git archive FAILED, trying git clone instead" ; \ + git clone $GIT_REPO $tmpdir && cd $tmpdir && git checkout $GIT_TAG && rm -rf .git) # Create lxc vm - cd $tmpdir - ./lbuild-initvm.sh $VERBOSE -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${PREINSTALLED} ${BASE} - # cleanup - cd - - rm -rf $tmpdir - # Extract build again - in the vm - [ -n "$SSH_KEY" ] && setupssh ${BASE} ${SSH_KEY} - run_in_build_guest $BASE "(git clone $GIT_REPO /build; cd /build; git checkout $GIT_TAG)" - fi - echo "XXXXXXXXXX $COMMAND: preparation of vm $BASE done" $(date) - - # The log inside the vm contains everything - LOG2=$(rootdir ${BASE})/log.txt - (echo "==================== BEG LXC Transcript of vm creation" ; \ - cat $LOG ; \ - echo "==================== END LXC Transcript of vm creation" ; \ - echo "xxxxxxxxxx Messing with logs, symlinking $LOG2 to $LOG" ) >> $LOG2 - ### not too nice : nuke the former log, symlink it to the new one - rm $LOG; ln -s $LOG2 $LOG - LOG=$LOG2 - # redirect log again - exec >> $LOG 2>&1 - - sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" - WEBPATH=$(echo ${WEBPATH} | sed $sedargs) - webpublish mkdir -p ${WEBPATH} + cd $tmpdir + ./lbuild-initvm.sh $VERBOSE -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${PREINSTALLED} ${BASE} + # cleanup + cd - + rm -rf $tmpdir + # Extract build again - in the vm + [ -n "$SSH_KEY" ] && setupssh ${BASE} ${SSH_KEY} + run_in_build_guest $BASE "(git clone $GIT_REPO /build; cd /build; git checkout $GIT_TAG)" + fi + echo "XXXXXXXXXX $COMMAND: preparation of vm $BASE done" $(date) + + # The log inside the vm contains everything + LOG2=$(rootdir ${BASE})/log.txt + (echo "==================== BEG LXC Transcript of vm creation" ; \ + cat $LOG ; \ + echo "==================== END LXC Transcript of vm creation" ; \ + echo "xxxxxxxxxx Messing with logs, symlinking $LOG2 to $LOG" ) >> $LOG2 + ### not too nice : nuke the former log, symlink it to the new one + rm $LOG; ln -s $LOG2 $LOG + LOG=$LOG2 + # redirect log again + exec >> $LOG 2>&1 + + sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" + WEBPATH=$(echo ${WEBPATH} | sed $sedargs) + webpublish mkdir -p ${WEBPATH} # where to store the log for web access - WEBBASE=${WEBPATH}/${BASE} - WEBLOG=${WEBPATH}/${BASE}/log.txt + WEBBASE=${WEBPATH}/${BASE} + WEBLOG=${WEBPATH}/${BASE}/log.txt # compute the log URL - inserted in the mail messages for convenience - WEBBASE_URL=$(echo $WEBBASE | sed -e "s,//,/,g" -e "s,${WEBROOT},${TESTBUILDURL},") - - if [ -n "$DO_BUILD" ] ; then - - # invoke this command into the build directory of the vm - cp $COMMANDPATH $(rootdir ${BASE})/build/ - - # invoke this command in the vm for building (-T) - run_in_build_guest $BASE chmod +x /build/$COMMAND - run_in_build_guest $BASE /build/$COMMAND "${options[@]}" -b "${BASE}" "${MAKEVARS[@]}" "${MAKETARGETS[@]}" - fi - - # publish to the web so run_log can find them - set +e - trap - ERR INT - webpublish rm -rf $WEBPATH/$BASE - # guess if we've been doing any debian-related build - if [ ! -f $(rootdir $BASE)/etc/debian_version ] ; then - webpublish mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS} - webpublish_rsync_dir $WEBPATH/$BASE/RPMS/ $(rootdir $BASE)/build/RPMS/ - [[ -n "$PUBLISH_SRPMS" ]] && webpublish_rsync_dir $WEBPATH/$BASE/SRPMS/ $(rootdir $BASE)/build/SRPMS/ - else - # run scanpackages so we can use apt-get on this - # (not needed on fedora b/c this is done by the regular build already) - run_in_build_guest $BASE "(cd /build ; dpkg-scanpackages DEBIAN/ | gzip -9c > Packages.gz)" - webpublish mkdir -p $WEBPATH/$BASE/DEBIAN - webpublish_rsync_files $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb - webpublish_rsync_files $WEBPATH/$BASE/ $(rootdir $BASE)/build/Packages.gz - fi - # publish myplc-release if this exists - release=$(rootdir $BASE)/build/myplc-release - [ -f $release ] && webpublish_rsync_files $WEBPATH/$BASE $release - set -e - trap failure ERR INT + WEBBASE_URL=$(echo $WEBBASE | sed -e "s,//,/,g" -e "s,${WEBROOT},${TESTBUILDURL},") + + if [ -n "$DO_BUILD" ] ; then + + # invoke this command into the build directory of the vm + cp $COMMANDPATH $(rootdir ${BASE})/build/ + + # invoke this command in the vm for building (-T) + run_in_build_guest $BASE chmod +x /build/$COMMAND + run_in_build_guest $BASE /build/$COMMAND "${options[@]}" -b "${BASE}" "${MAKEVARS[@]}" "${MAKETARGETS[@]}" + fi + + # publish to the web so run_log can find them + set +e + trap - ERR INT + webpublish rm -rf $WEBPATH/$BASE + # guess if we've been doing any debian-related build + if [ ! -f $(rootdir $BASE)/etc/debian_version ] ; then + webpublish mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS} + webpublish_rsync_dir $WEBPATH/$BASE/RPMS/ $(rootdir $BASE)/build/RPMS/ + [[ -n "$PUBLISH_SRPMS" ]] && webpublish_rsync_dir $WEBPATH/$BASE/SRPMS/ $(rootdir $BASE)/build/SRPMS/ + else + # run scanpackages so we can use apt-get on this + # (not needed on fedora b/c this is done by the regular build already) + run_in_build_guest $BASE "(cd /build ; dpkg-scanpackages DEBIAN/ | gzip -9c > Packages.gz)" + webpublish mkdir -p $WEBPATH/$BASE/DEBIAN + webpublish_rsync_files $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb + webpublish_rsync_files $WEBPATH/$BASE/ $(rootdir $BASE)/build/Packages.gz + fi + # publish myplc-release if this exists + release=$(rootdir $BASE)/build/myplc-release + [ -f $release ] && webpublish_rsync_files $WEBPATH/$BASE $release + set -e + trap failure ERR INT # create yum repo and sign packages. - if [ -n "$SIGNYUMREPO" ] ; then - # this script does not yet support signing on a remote (webhost) repo - sign_here=$(hostname) ; sign_web=$(webpublish hostname) - if [ "$hostname" = "$sign_here" ] ; then - sign_node_packages - else - echo "$COMMAND does not support signing on a remote yum repo" - echo "you might want to turn off the -y option, or run this on the web server box itself" - exit 1 - fi - fi - - if [ -n "$DO_TEST" ] ; then - run_log - fi - - success + if [ -n "$SIGNYUMREPO" ] ; then + # this script does not yet support signing on a remote (webhost) repo + sign_here=$(hostname) ; sign_web=$(webpublish hostname) + if [ "$hostname" = "$sign_here" ] ; then + sign_node_packages + else + echo "$COMMAND does not support signing on a remote yum repo" + echo "you might want to turn off the -y option, or run this on the web server box itself" + exit 1 + fi + fi + + if [ -n "$DO_TEST" ] ; then + run_log + fi + + success echo "==================== MAIN END $(date)" fi