X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=lbuild-nightly.sh;h=1d115e9c1f7a5e14404fb717eb65e059c739e90d;hb=38239bbbb43ddbeaf8e64f89826a47ddc6aa5b50;hp=715d3684c4464572b7f10237c2358abf2aa1e68c;hpb=f32374281fa212ac418b4679551f3d33a0007ba3;p=build.git

diff --git a/lbuild-nightly.sh b/lbuild-nightly.sh
index 715d3684..1d115e9c 100755
--- a/lbuild-nightly.sh
+++ b/lbuild-nightly.sh
@@ -11,7 +11,7 @@ COMMAND=$(basename $0)
 export PATH=$PATH:/bin:/sbin
 
 # default values, tunable with command-line options
-DEFAULT_FCDISTRO=f29
+DEFAULT_FCDISTRO=f31
 DEFAULT_PLDISTRO=lxc
 DEFAULT_PERSONALITY=linux64
 DEFAULT_MAILDEST="build at onelab.eu"
@@ -72,27 +72,28 @@ function guest_ipv4() {
 function summary () {
     from=$1; shift
     echo "******************** BEG SUMMARY"
-    python - $from <<EOF
-#!/usr/bin/env python
+    python3 - $from <<EOF
+#!/usr/bin/env python3
 # read a full log and tries to extract the interesting stuff
 
-import sys,re
-m_show_line=re.compile(".* (BEG|END) (RPM|LXC).*|.*'boot'.*|\* .*| \* .*|.*is not installed.*|.*PROPFIND.*|.* (BEG|END).*:run_log.*|.* Within LXC (BEG|END) .*|.* MAIN (BEG|END).*")
-m_installing_any=re.compile('\r  (Installing:[^\]]*]) ')
-m_installing_err=re.compile('\r  (Installing:[^\]]*])(..+)')
-m_installing_end=re.compile('Installed:.*')
-m_installing_doc1=re.compile("(.*)install-info: No such file or directory for /usr/share/info/\S+(.*)")
-m_installing_doc2=re.compile("(.*)grep: /usr/share/info/dir: No such file or directory(.*)")
+import sys, re
+m_show_line = re.compile(
+".* (BEG|END) (RPM|LXC).*|.*'boot'.*|\* .*| \* .*|.*is not installed.*|.*PROPFIND.*|.* (BEG|END).*:run_log.*|.* Within LXC (BEG|END) .*|.* MAIN (BEG|END).*")
+m_installing_any = re.compile('\r  (Installing:[^\]]*]) ')
+m_installing_err = re.compile('\r  (Installing:[^\]]*])(..+)')
+m_installing_end = re.compile('Installed:.*')
+m_installing_doc1 = re.compile("(.*)install-info: No such file or directory for /usr/share/info/\S+(.*)")
+m_installing_doc2 = re.compile("(.*)grep: /usr/share/info/dir: No such file or directory(.*)")
 
 def summary (filename):
 
     try:
-        if filename=="-":
-            filename="stdin"
-            f=sys.stdin
+        if filename == "-":
+            filename = "stdin"
+            f = sys.stdin
         else:
-            f=open(filename)
-        echo=False
+            f = open(filename)
+        echo = False
         for line in f.xreadlines():
             # first off : discard warnings related to doc
             if m_installing_doc1.match(line):
@@ -103,12 +104,12 @@ def summary (filename):
                 line=begin+end
             # unconditionnally show these lines
             if m_show_line.match(line):
-                print '>>>',line,
+                print('>>>', line, end="")
             # an 'installing' line with messages afterwards : needs to be echoed
             elif m_installing_err.match(line):
                 (installing,error)=m_installing_err.match(line).groups()
-                print '>>>',installing
-                print '>>>',error
+                print('>>>',installing)
+                print('>>>',error)
                 echo=True
             # closing an 'installing' section
             elif m_installing_end.match(line):
@@ -117,14 +118,14 @@ def summary (filename):
             elif m_installing_any.match(line):
                 if echo:
                     installing=m_installing_any.match(line).group(1)
-                    print '>>>',installing
+                    print('>>>',installing)
                 echo=False
             # print lines when echo is true
             else:
-                if echo: print '>>>',line,
+                if echo: print('>>>',line, end="")
         f.close()
     except:
-        print 'Failed to analyze',filename
+        print('Failed to analyze',filename)
 
 for arg in sys.argv[1:]:
     summary(arg)
@@ -134,17 +135,24 @@ EOF
 
 ### we might build on a box other than the actual web server
 # utilities for handling the pushed material (rpms, logfiles, ...)
-function webpublish_misses_dir () { ssh root@${WEBHOST}  "bash -c \"test \! -d $1\"" ; }
-function webpublish () { ssh root@${WEBHOST} "$@" ; }
-function webpublish_cp_local_to_remote () { scp $1 root@${WEBHOST}:$2 ; }
-function webpublish_cp_stdin_to_file () { ssh root@${WEBHOST} cat \> $1; }
-function webpublish_append_stdin_to_file () { ssh root@${WEBHOST} cat \>\> $1; }
-# provide remote dir as first argument, so any number of local files can be passed next
-function webpublish_rsync_dir () { rsync --archive --delete $VERBOSE $2 root@${WEBHOST}:$1 ; }
-function webpublish_rsync_files () {
-    remote="$1"; shift
-    rsync --archive $VERBOSE "$@" root@${WEBHOST}:"$remote" ;
+function webpublish_misses_dir () {
+    ssh root@${WEBHOST}  "bash -c \"test \! -d $1\""
 }
+function webpublish () {
+    ssh root@${WEBHOST} "$@"
+}
+function webpublish_cp_stdin_to_file () {
+    ssh root@${WEBHOST} cat \> $1 \; chmod g+r,o+r $1
+}
+function webpublish_append_stdin_to_file () {
+    ssh root@${WEBHOST} cat \>\> $1 \; chmod g+r,o+r $1
+}
+# provide remote dir as first argument,
+# so any number of local files can be passed next
+function webpublish_rsync () {
+    local remote="$1"; shift
+    rsync --archive --delete $VERBOSE "$@" root@${WEBHOST}:"$remote"
+ }
 
 function pretty_duration () {
     total_seconds=$1; shift
@@ -168,7 +176,7 @@ function failure() {
         WEBLOG=/tmp/lbuild-early-$(date +%Y-%m-%d).log.txt
     fi
     webpublish mkdir -p $WEBBASE ||:
-    webpublish_cp_local_to_remote $LOG $WEBLOG ||:
+    webpublish_rsync $WEBLOG $LOG  ||:
     summary $LOG | webpublish_append_stdin_to_file $WEBLOG ||:
     (echo -n "============================== $COMMAND: failure at " ; date ; \
         webpublish tail --lines=1000 $WEBLOG) | \
@@ -196,7 +204,7 @@ function success () {
         WEBLOG=/tmp/lbuild-early-$(date +%Y-%m-%d).log.txt
     fi
     webpublish mkdir -p $WEBBASE
-    webpublish_cp_local_to_remote $LOG $WEBLOG
+    webpublish_rsync $WEBLOG $LOG
     summary $LOG | webpublish_append_stdin_to_file $WEBLOG
     if [ -n "$DO_TEST" ] ; then
         short_message="PASS"
@@ -248,7 +256,7 @@ function in_root_context () {
 # convenient for simple commands
 function run_in_build_guest () {
     buildname=$1; shift
-    ssh -o "StrictHostKeyChecking no" root@$(guest_ipv4 $buildname) "$@"
+    ssh -o StrictHostKeyChecking=no root@$(guest_ipv4 $buildname) "$@"
 }
 
 # run in the vm - do not manage success/failure, will be done from the root ctx
@@ -355,7 +363,7 @@ function run_log () {
     rsync --verbose --archive ${testmaster_ssh}:$BASE/logs/ $(rootdir $BASE)/build/testlogs
     # push them to the build web
     chmod -R a+r $(rootdir $BASE)/build/testlogs/
-    webpublish_rsync_dir $WEBPATH/$BASE/testlogs/ $(rootdir $BASE)/build/testlogs/
+    webpublish_rsync $WEBPATH/$BASE/testlogs/ $(rootdir $BASE)/build/testlogs/
 
     echo  "============================== END $COMMAND:run_log on $(date)"
 
@@ -763,23 +771,26 @@ function main () {
         # publish to the web so run_log can find them
         set +e
         trap - ERR INT
-        webpublish rm -rf $WEBPATH/$BASE
+#        webpublish rm -rf $WEBPATH/$BASE
         # guess if we've been doing any debian-related build
         if [ ! -f $(rootdir $BASE)/etc/debian_version  ] ; then
             webpublish mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS}
-            webpublish_rsync_dir $WEBPATH/$BASE/RPMS/ $(rootdir $BASE)/build/RPMS/
-            [[ -n "$PUBLISH_SRPMS" ]] && webpublish_rsync_dir $WEBPATH/$BASE/SRPMS/ $(rootdir $BASE)/build/SRPMS/
+            # after moving to f29, we see this dir created as 700
+            # as remote umask is 077
+            webpublish chmod 755 $WEBPATH/$BASE
+            webpublish_rsync $WEBPATH/$BASE/RPMS/ $(rootdir $BASE)/build/RPMS/
+            [[ -n "$PUBLISH_SRPMS" ]] && webpublish_rsync $WEBPATH/$BASE/SRPMS/ $(rootdir $BASE)/build/SRPMS/
         else
             # run scanpackages so we can use apt-get on this
             # (not needed on fedora b/c this is done by the regular build already)
             run_in_build_guest $BASE "(cd /build ; dpkg-scanpackages DEBIAN/ | gzip -9c > Packages.gz)"
             webpublish mkdir -p $WEBPATH/$BASE/DEBIAN
-            webpublish_rsync_files $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb
-            webpublish_rsync_files $WEBPATH/$BASE/ $(rootdir $BASE)/build/Packages.gz
+            webpublish_rsync $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb
+            webpublish_rsync $WEBPATH/$BASE/ $(rootdir $BASE)/build/Packages.gz
         fi
         # publish myplc-release if this exists
         release=$(rootdir $BASE)/build/myplc-release
-        [ -f $release ] && webpublish_rsync_files $WEBPATH/$BASE $release
+        [ -f $release ] && webpublish_rsync $WEBPATH/$BASE $release
         set -e
         trap failure ERR INT