review how to extract tests

diff --git a/Makefile b/Makefile
index 17473fd..3d3c2eb 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -286,6 +286,7 @@ else
 $(1)-GITPATH := $(strip $($(1)-GITPATH))
 $(1).gitrepo := $(firstword $(subst @, ,$($(1)-GITPATH)))
 $(1).gittag := $(word 2,$(subst @, ,$($(1)-GITPATH)))
+$(1).gittag := $(if $($(1).gittag),$($(1).gittag),master)
 endif
 endef
 
@@ -351,9 +352,8 @@ MODULES/$(1):
        cd MODULES && \
        $(if $($(1)-SVNPATH),\
          svn export $($(1)-SVNPATH) $(1),\
-         git clone $($(1).gitrepo) $(1); \
-         $(if $($(1).gittag), cd $(1); git checkout "$($(1).gittag)"; cd -; ,) \
-         rm -rf $(1)/.git )
+         mkdir $(1) ; (git archive --remote=$($(1).gitrepo) $($(1).gittag) | tar -C $(1) -xf - ) \
+          || { rm -rf $(1); false; } )
        @(echo -n "XXXXXXXXXXXXXXX -- END MODULE $(module) : $@ " ; date)
 
 $(1)-module: MODULES/$(1)
@@ -362,6 +362,13 @@ endef
 
 $(foreach module,$(ALL.modules),$(eval $(call target_extract_module,$(module))))
 
+### the tests area
+# use this makefile to extract tests rather than extracting manually in vbuild-nightly
+$(eval $(call stage1_module_vars,tests))
+$(eval $(call target_extract_module,tests))
+tests-clean:
+       rm -rf MODULES/tests
+.PHONY: tests-clean
 
 ###
 # Base rpmbuild in the current directory
@@ -684,7 +691,7 @@ myplc-release:
        echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx build info" >> $@
        $(MAKE) --no-print-directory version-build >> $@
        echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx svn info" >> $@
-       $(MAKE) --no-print-directory version-svns >> $@
+       $(MAKE) --no-print-directory version-tags >> $@
        echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx rpm info" >> $@
        $(MAKE) --no-print-directory version-rpms >> $@
        @echo $(BASE) > .base
@@ -766,12 +773,6 @@ info: info-packages info-modules info-branches
 
 .PHONY: info info-packages info-modules info-branches module-tools
 
-####################
-tests_gitpath:
-       @$(if $(tests-GITPATH), echo $(tests-GITPATH) > $@, \
-       echo "http://git.onelab.eu/tests.git" > $@)
-
-
 ####################
 help:
        @echo "********** Run make in two stages:"

diff --git a/coblitz-latest-tags.mk b/coblitz-latest-tags.mk
index 5a66084..8640cf7 100644 (file)
--- a/coblitz-latest-tags.mk
+++ b/coblitz-latest-tags.mk
@@ -45,4 +45,4 @@ pyopenssl-SVNPATH             := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9
 PLCRT-SVNPATH                  := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-11
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                  := http://git.onelab.eu/tests.git@tests-4.3-6
+tests-GITPATH                  := git://git.onelab.eu/tests.git@tests-4.3-6

diff --git a/coblitz-stable-tags.mk b/coblitz-stable-tags.mk
index 4431116..4b4e2bd 100644 (file)
--- a/coblitz-stable-tags.mk
+++ b/coblitz-stable-tags.mk
@@ -45,4 +45,4 @@ pyopenssl-SVNPATH             := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9
 PLCRT-SVNPATH                  := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-11
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                  := http://git.onelab.eu/tests.git@tests-4.3-6
+tests-GITPATH                  := git://git.onelab.eu/tests.git@tests-4.3-6

diff --git a/onelab-k27-tags.mk b/onelab-k27-tags.mk
index ab28d2a..f7cbc42 100644 (file)
--- a/onelab-k27-tags.mk
+++ b/onelab-k27-tags.mk
@@ -55,7 +55,7 @@ MyPLC-SVNPATH                 := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-5.0-4
 DistributedRateLimiting-SVNPATH        := http://svn.planet-lab.org/svn/DistributedRateLimiting/tags/DistributedRateLimiting-0.1-1
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                   := http://git.onelab.eu/tests.git@tests-5.0-5
+tests-GITPATH                   := git://git.onelab.eu/tests.git@tests-5.0-5
 
 ### temporary
 # nozomi not needed anymore for 2.6.27

diff --git a/onelab-tags.mk b/onelab-tags.mk
index 41ee9ec..eb18013 100644 (file)
--- a/onelab-tags.mk
+++ b/onelab-tags.mk
@@ -58,4 +58,4 @@ MyPLC-SVNPATH                 := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-5.0-4
 DistributedRateLimiting-SVNPATH        := http://svn.planet-lab.org/svn/DistributedRateLimiting/tags/DistributedRateLimiting-0.1-1
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                   := http://git.onelab.eu/tests.git@tests-5.0-5
+tests-GITPATH                   := git://git.onelab.eu/tests.git@tests-5.0-5

diff --git a/planetbridge-tags.mk b/planetbridge-tags.mk
index 047bdc6..155eaa9 100644 (file)
--- a/planetbridge-tags.mk
+++ b/planetbridge-tags.mk
@@ -45,4 +45,4 @@ pyopenssl-SVNPATH               := http://svn.planet-lab.org/svn/pyopenssl/tags/
 PLCRT-SVNPATH                  := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-10
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                   := http://git.onelab.eu/tests.git@tests-4.3-6
+tests-GITPATH                   := git://git.onelab.eu/tests.git@tests-4.3-6

diff --git a/planetlab-k27-tags.mk b/planetlab-k27-tags.mk
index 3d46d9d..2ff921c 100644 (file)
--- a/planetlab-k27-tags.mk
+++ b/planetlab-k27-tags.mk
@@ -54,4 +54,4 @@ MyPLC-SVNPATH                 := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-5.0-4
 DistributedRateLimiting-SVNPATH        := http://svn.planet-lab.org/svn/DistributedRateLimiting/tags/DistributedRateLimiting-0.1-1
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                  := http://git.onelab.eu/tests.git
+tests-GITPATH                  := git://git.onelab.eu/tests.git@master

diff --git a/planetlab-k32-tags.mk b/planetlab-k32-tags.mk
index 509bee3..4c323c1 100644 (file)
--- a/planetlab-k32-tags.mk
+++ b/planetlab-k32-tags.mk
@@ -55,4 +55,4 @@ MyPLC-SVNPATH                 := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-5.0-4
 DistributedRateLimiting-SVNPATH        := http://svn.planet-lab.org/svn/DistributedRateLimiting/tags/DistributedRateLimiting-0.1-1
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                  := http://git.onelab.eu/tests.git@tests-5.0-5
+tests-GITPATH                  := git://git.onelab.eu/tests.git@tests-5.0-5

diff --git a/planetlab-tags.mk b/planetlab-tags.mk
index 029d4d4..ee3b6e9 100644 (file)
--- a/planetlab-tags.mk
+++ b/planetlab-tags.mk
@@ -52,4 +52,4 @@ MyPLC-SVNPATH                 := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-5.0-4
 DistributedRateLimiting-SVNPATH                        := http://svn.planet-lab.org/svn/DistributedRateLimiting/tags/DistributedRateLimiting-0.1-1
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                  := http://git.onelab.eu/tests.git
+tests-GITPATH                  := git://git.onelab.eu/tests.git@master

diff --git a/trellis-tags.mk b/trellis-tags.mk
index 0337546..b1ed283 100644 (file)
--- a/trellis-tags.mk
+++ b/trellis-tags.mk
@@ -50,4 +50,4 @@ pyopenssl-SVNPATH               := http://svn.planet-lab.org/svn/pyopenssl/tags/
 PLCRT-SVNPATH                   := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-10
 
 # locating the right test directory - see make tests_gitpath
-tests-GITPATH                   := http://git.onelab.eu/tests.git@tests-4.3-6
+tests-GITPATH                   := git://git.onelab.eu/tests.git@tests-4.3-6

diff --git a/vbuild-nightly.sh b/vbuild-nightly.sh
index 76b42fb..8e03468 100755 (executable)
--- a/vbuild-nightly.sh
+++ b/vbuild-nightly.sh
@@ -212,8 +212,6 @@ function build () {
 
     # stage1
     make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true 
-    # store tests_gitpath
-    make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true tests_gitpath
     # versions
     make -C /build $DRY_RUN "${MAKEVARS[@]}" versions
     # actual stuff
@@ -230,15 +228,6 @@ function run_log () {
 
     echo "============================== BEG $COMMAND:run_log on $(date)"
 
-    # where to find TESTS_GITPATH
-    stamp=/vservers/$BASE/build/tests_gitpath
-    if [ ! -f $stamp ] ; then
-       echo "$COMMAND: Cannot figure TESTS_GITPATH from missing $stamp"
-       failure
-       exit 1
-    fi
-    TESTS_GITPATH=$(cat $stamp)
-
     ### the URL to the RPMS/<arch> location
     # f12 now has everything in i686; try i386 first as older fedoras have both
     url=""
@@ -262,15 +251,15 @@ function run_log () {
 
     # test directory name on test box
     testdir=${BASE}
+
     # clean it
     ssh -n ${testmaster_ssh} rm -rf ${testdir} ${testdir}.git
 
-    # check it out - just the 'system' subdir is enough
-    gitrepo=$(echo $TESTS_GITPATH | cut -d@ -f1)
-    gittag=$(echo $TESTS_GITPATH | cut -s -d@ -f2)
-    ssh -n ${testmaster_ssh} git clone ${gitrepo} ${testdir}.git 
-    [ -n "$gittag" ] && ssh -n ${testmaster_ssh} "cd ${testdir}.git ; git checkout ${gittag}"
-    ssh -n ${testmaster_ssh} "mv ${testdir}.git/system ${testdir} ; rm -rf ${testdir}.git"
+    # check it out in the build
+    vserver $BASE exec make -C /build tests-module
+    
+    # push it onto the testmaster - just the 'system' subdir is enough
+    rsync --verbose --archive /vservers/$BASE/build/MODULES/tests/system/ ${testmaster_ssh}:${BASE}
 
     # invoke test on testbox - pass url and build url - so the tests can use vtest-init-vserver.sh
     configs=""
@@ -283,9 +272,9 @@ function run_log () {
     success=true
     ssh 2>&1 -n ${testmaster_ssh} ${testdir}/run_log --build ${build_SVNPATH} --url ${url} $configs $test_env $VERBOSE --all || success=
 
-    # gather logs in the vserver
+    # gather logs in the build vserver
     mkdir -p /vservers/$BASE/build/testlogs
-    ssh 2>&1 -n ${testmaster_ssh} tar -C ${testdir}/logs -cf - . | tar -C /vservers/$BASE/build/testlogs -xf - || :
+    rsync --verbose --archive ${testmaster_ssh}:$BASE/logs/ /vservers/$BASE/build/testlogs
     # push them to the build web
     chmod -R a+r /vservers/$BASE/build/testlogs/
     webpublish_rsync_dir /vservers/$BASE/build/testlogs/ $WEBPATH/$BASE/testlogs/
@@ -568,7 +557,7 @@ function main () {
            [ -n "$SSH_KEY" ] && setupssh ${BASE} ${SSH_KEY}
            vserver ${BASE} exec svn update /build
            # make sure we refresh the tests place in case it has changed
-           rm -f /build/tests_gitpath
+           rm -f /build/MODULES/tests
            # get environment from the first run 
            FCDISTRO=$(vserver ${BASE} exec /build/getdistroname.sh)
            # retrieve all in one run