X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=Makefile;h=78533ca6a5e8631994f847dd9d3e23d7a6364ab6;hb=refs%2Fheads%2Fcoblitz;hp=efa08fdc20db88fdd692c4f5390c1265f5be5efd;hpb=ffd1700898d7f024da09fc841a9cc747e7d66c16;p=build.git

diff --git a/Makefile b/Makefile
index efa08fdc..78533ca6 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,6 @@
 #
 # Thierry Parmentelat - INRIA Sophia Antipolis 
 #
-### $Id$
-### $URL$
-# 
 ####################
 # invocation:
 #
@@ -31,47 +28,62 @@
 # 
 # (*) packages are named upon the RPM name; they are mostly lowercase
 #     Add a package to ALL if you want it built as part of the default set.
-# (*) modules are named after the subversion tree; as of this writing their names 
-#     are mostly mixed case like MyPLC or VserverReference
+# (*) modules are named after the subversion or git tree; we would wish to keep
+#     these names lowercase as far as possible
 # (*) rpms are named in the spec files. A package typically defines several rpms;
 #     rpms are used for defining DEPEND-DEVEL-RPMS. See also package.rpmnames
 # 
+# in simple cases, one package uses one module (in which case using the same name sounds right)
+# but others might need several modules (e.g. image-creation packages like bootstrapfs need 
+# bootstrapfs and build); in this case the FIRST ONE is used for locating the specfile 
+#
 #################### packages
-# basics: how to build a package - you need/may define the following variables
+# basics: how to build a package - you need to define the following variables
 # 
 # (*) package-MODULES
-#     a package needs one or several modules to build. 
-#     to this end, define 
+#     a package needs one or several modules to build; the first one is used for 
+#     some special purposes, like locating the specfile
 # (*) package-SPEC
 #     the package's specfile; this is relative to the FIRST module in package-MODULES
-#     see 'codebase' below
 #
 # Optional:
 #
-# (*) package-SPECVARS
-#     space-separated list of spec variable definitions, where you can reference make variable that relate to 
-#     packages defined BEFORE the current one (note: you should use = - as opposed to := - to define these)
-#     e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release) 
-#     would let you use the %release from the kernel's package when rpmbuild'ing mydriver - see automatic below
 # (*) package-DEPEND-PACKAGES
-#     a set of *packages* that this package depends on
+#     a set of (obviously local) *packages* that this package depends on, e.g.
+#     bootstrapfs-DEPEND-PACKAGES += kernel
+#     this impacts the order of the build
+# (*) package-DEVEL-RPMS
+#     a set of stock rpms that this package needs at build-time
+#     this can also be set in config.<distro>/devel.pkgs or config.planetlab/devel.pkgs as appropriate
+# (*) package-EXCLUDE-DEVEL-RPMS
+#     a set of stock *rpms* that the build will rpm-uninstall before building <package>
+#     this is intended to denote stock rpms, and the build will attempt to yum-install them
+#     back after the package is rebuilt
+#     This feature is not used at the moment and kept only just in case
 # (*) package-DEPEND-DEVEL-RPMS
-#     a set of *rpms* that the build will rpm-install before building <package>
+#     a set of local *rpms* that the build will rpm-install before building <package>
 #     the build will attempt to uninstall those once the package is built, this is not fatal though
 #     this is intended to denote local rpms, i.e. ones that are results of our own build
-#     stock rpms should be mentioned in config.planetlab/devel.pkgs
+#     stock rpms should be mentioned in DEVEL-RPMS or in devel.pkgs as described above
 # (*) package-DEPEND-FILES
 #     a set of files that the package depends on - and that make needs to know about
 #     if this contains RPMS/yumgroups.xml, then the toplevel RPMS's index 
 #     is refreshed with createrepo prior to running rpmbuild
-# (*) package-EXCLUDE-DEVEL-RPMS
-#     a set of *rpms* that the build will rpm-uninstall before building <package>
-#     this is intended to denote stock rpms, and the build will attempt to yum-install them
-#     back after the package is rebuilt
+# (*) package-SPECVARS
+#     space-separated list of spec variable definitions, where you can reference make variables that 
+#     belong to packages defined BEFORE the current one 
+#     note: you should use = to define these (as opposed to :=)
+#     e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release) 
+#     would let you use the %release from the kernel's package when rpmbuild'ing mydriver 
+#     see automatic below
 # (*) package-RPMFLAGS: Miscellaneous RPM flags
-# (*) package-RPMBUILD: If not rpmbuild - mostly used for sudo'ing rpmbuild
-# (*) package-BUILD-FROM-SRPM: set this to any non-empty value, if your package is able to produce 
-#     a source rpms by running 'make srpm'
+#     this is passed to rpmbuild, as well as to spec2make for "exporting" various rpm variable
+#     to make; beware that some features, like --with=debug or --define 'foo bar' are not
+#     well handled by spec2make as of this writing, which can cause issues.
+#     hopefully this will be fixed...
+# (*) package-BUILD-FROM-SRPM: set this to any non-empty value, 
+#     if your package is able to produce a source rpm. 
+#     In this case the build will first invoke 'make srpm', and then rebuild binaries from that 
 # (*) package-RPMDATE: set this to any non-empty value to get the rpm package's release field hold the current date
 #     this is useful for container packages, like e.g. bootstrapfs or vserver, that contains much more than the
 #     correspondng module
@@ -80,18 +92,18 @@
 # Required information about the various modules (set this in e.g. planetlab-tags.mk)
 #
 # (*) module-SVNPATH
+#     for svn modules
 #     the complete path where this module lies; 
 #     you can specify the trunk or a given tag with this variable
 # 
-# OR if the module is managed under cvs (will be obsoleted)
-# 
-# (*) module-CVSROOT
-# (*) module-TAG
+# (*) module-GITPATH
+#     for git modules
+#     which by analogy with svn revision scheme, is expected to be <url>@<tag>
 #
 #################### automatic variables
 #
 # the build defines some make variables that are extracted from spec files
-# see for example
+# to inspect those, see for example
 # (*)  $ make ulogd-pkginfo
 #        to see the list f variables attached to a given package
 # (*)  $ make kernel-devel-rpminfo
@@ -100,7 +112,7 @@
 ####################
 
 # exported to spec files as plrelease
-PLANETLAB_RELEASE = 4.3
+PLANETLAB_RELEASE = 5.0
 
 #
 # Default values
@@ -116,9 +128,6 @@ RPM-INSTALL-DEVEL := rpm --force -Uvh
 RPM-UNINSTALL-DEVEL := rpm -e
 YUM-INSTALL-DEVEL := yum -y install
 
-# see also below
-REMOTE-PLDISTROS="wextoolbox"
-
 #################### Makefile
 # Default target
 all:
@@ -127,17 +136,16 @@ all:
 ### default values
 PLDISTRO := planetlab
 RPMBUILD := rpmbuild
-export CVS_RSH := ssh
 
-########## savedpldistro.mk holds PLDISTRO - it is generated at stage1 (see below)
+########## envfrompreviousrun.mk holds PLDISTRO and others - it is generated at stage1 (see below)
 ifeq "$(stage1)" ""
-include savedpldistro.mk
+include envfrompreviousrun.mk
 endif
 
 # when re-running the nightly build after failure, we need to gather the former values
 # do this by running make stage1=skip +PLDISTRO
 ifeq "$(stage1)" "skip"
-include savedpldistro.mk
+include envfrompreviousrun.mk
 endif
 
 #################### include onelab.mk
@@ -146,48 +154,62 @@ PLDISTROCONTENTS := $(PLDISTRO).mk
 include $(PLDISTROCONTENTS)
 
 #################### include <pldistro>-tags.mk
-# describes where to fetch components, and the related tags if using cvs
+# describes where to fetch components
+ifeq "$(PLDISTROTAGS)" ""
 PLDISTROTAGS := $(PLDISTRO)-tags.mk
+endif
 include $(PLDISTROTAGS)
 
 # this used to be set in the -tags.mk files, but that turned out to require
 # error-prone duplicate changes 
 # so now the nightly build script sets this to what it is currently using
-# we set a default in case we run the build manually:
-# if the local directory was svn checked out, then use the corresponding URL
-svn-info-url-line := $(shell svn info 2> /dev/null | grep URL:)
-default-build-SVNPATH := $(lastword $(svn-info-url-line))
-# otherwise, use this hard-coded default
-ifeq "$(default-build-SVNPATH)" ""
-default-build-SVNPATH := http://svn.planet-lab.org/svn/build/trunk
+# in case we run this manually, i.e. if build-GITPATH is not set
+ifeq "$(build-GITPATH)" ""
+build-GITPATH := $(shell git config remote.origin.url)
 endif
-# use default if necessary
-build-SVNPATH ?= $(default-build-SVNPATH)
 
-####################
+#################### pldistros that are defined remotely
+# fetching with svn
+define remote_pldistro_fetch_svn
+$(1).config.SVNPATH := $(shell grep -v "^#" config.$(1).svnpath)
+config.$(1): config.$(1).svnpath
+	@echo "Fetching (svn) details for remote pldistro $(1)"
+	svn export $$($(1).config.SVNPATH) config.$(1)
+endef
+
+FOO=@
+# fetching with git
+define remote_pldistro_fetch_git
+$(1).config.GITPATH := $(shell grep -v "^#" config.$(1).gitpath)
+$(1).config.gitrepo := $$(firstword $$(subst @, ,$$($(1).config.GITPATH)))
+$(1).config.gittag := $$(word 2,$$(subst @, ,$$($(1).config.GITPATH)))
+$(1).config.gittag := $$(if $$($(1).config.gittag),$$($(1).config.gittag),master)
+config.$(1): GITPATH=$(shell grep -v "^#" config.$(1).gitpath)
+config.$(1): config.$(1).gitpath
+	@echo "Fetching (git) details for remote pldistro $(1)"
+	mkdir -p config.$(1)
+	git $(GITTAG) archive --remote=$$($(1).config.gitrepo) $$($(1).config.gittag) | tar -C config.$(1) -xf -
+endef
+
+
+# put it together
 define remote_pldistro
 $(1).mk: config.$(1)/$(1).mk
-	@echo 'creating $(1) from config subdir'
+	@echo 'creating $(1) from config.$(1)'
 	cp config.$(1)/$(1).mk $(1).mk
 
 $(2).mk: config.$(1)/$(2).mk
-	@echo 'creating $(1) tags from config subdir'
+	@echo 'creating $(2) tags from config.$(1)'
 	cp config.$(1)/$(2).mk $(2).mk
 
 config.$(1)/$(1).mk: config.$(1)
 config.$(1)/$(2).mk: config.$(1)
 
-config.$(1): config.$(1).svnpath
-	@echo "Fetching details for pldistro $(1)"
-	svn export $(shell grep -v "^#" config.$(1).svnpath) config.$(1)
-
 DISTCLEANS += $(1).mk $(2).mk config.$(1)
-
+$(eval $(call remote_pldistro_fetch_$(3),$(1)))
 endef
 
-# somehow this does not work, handle manually instead
-#$(foreach distro, $(REMOTE-PLDISTROS), $(eval $(call remote_pldistro,$(distro),$(distro)-tags)))
-$(eval $(call remote_pldistro,wextoolbox,wextoolbox-tags))
+$(eval $(call remote_pldistro,wextoolbox,wextoolbox-tags,git))
 
 ########## stage1 and stage1iter
 # extract specs and compute .mk files by running 
@@ -235,7 +257,7 @@ else
 include $(ALLMKS)
 #all : tarballs
 #all : sources
-#all : codebases
+#all : modules
 #all : rpms
 #all : srpms
 # mention $(ALL) here rather than rpms 
@@ -260,46 +282,55 @@ repo: RPMS/yumgroups.xml
 
 ####################
 # notes: 
-# * to make configuration easier, we always use the first module's
-# definitions (CVSROOT,TAG, or SVNPATH) to extract the spec file
-# * for the same reason, in case cvs is used, the first module name is added to 
-# $(package)-SPEC - otherwise the cvs modules have to define spec as 
-# <module>/<module>.spec while svn modules just define it as <module>.spec
+# * we always use the first module's location (SVNPATH/GITPATH) to extract the spec file
+# * no matter what SCM is used, SPEC should hold a relative path from the module's root
 #
 define stage1_package_vars
-$(1).spec := $(notdir $($(1)-SPEC))
-$(1).specpath := SPECS/$(1).spec
 $(1).module := $(firstword $($(1)-MODULES))
+$(1).specpath := SPECS/$(notdir $($(1)-SPEC))
+$(1).moduledir := MODULES/$$($(1).module)
+$(1).codespec := MODULES/$$($(1).module)/$($(1)-SPEC)
+$(1).spec2makeflags := $(if $($(1)-WHITELIST-RPMS),--whitelist-rpms $($(1)-WHITELIST-RPMS),)
 endef
 
 $(foreach package, $(ALL), $(eval $(call stage1_package_vars,$(package))))
 
 # compute all modules
-ALL-MODULES :=
-$(foreach package,$(ALL), $(eval ALL-MODULES+=$($(package)-MODULES)))
-ALL-MODULES:=$(sort $(ALL-MODULES))
+ALL.modules := build tests
+$(foreach package,$(ALL), $(eval ALL.modules+=$($(package)-MODULES)))
+ALL.modules:=$(sort $(ALL.modules))
 
-# extract revision from -SVNPATH
+# extract revision from -SVNPATH or tag from -GITPATH
 define stage1_module_vars
+ifneq "$($(1)-SVNPATH)" ""
 $(1)-SVNPATH := $(strip $($(1)-SVNPATH))
-$(1).svnpath := $(firstword $(subst @, ,$($(1)-SVNPATH)))
-$(1).svnrev := $(word 2,$(subst @, @,$($(1)-SVNPATH)))
+$(1)-SCMPATH := $(strip $($(1)-SVNPATH))
+else
+$(1)-GITPATH := $$(strip $$($(1)-GITPATH))
+$(1)-SCMPATH := $$(strip $$($(1)-GITPATH))
+$(1).gitrepo := $$(firstword $$(subst @, ,$$($(1)-GITPATH)))
+$(1).gittag := $$(word 2,$$(subst @, ,$$($(1)-GITPATH)))
+$(1).gittag := $$(if $$($(1).gittag),$$($(1).gittag),master)
+endif
 endef
 
-$(foreach module,$(ALL-MODULES), $(eval $(call stage1_module_vars,$(module))))
+$(foreach module,$(ALL.modules), $(eval $(call stage1_module_vars,$(module))))
 
 #
 # for each package, compute whether we need to set date 
 # the heuristic is that we mention the date as part of the rpm release flag if
 # (*) the package has requested it by setting package-RPMDATE (container packages should do that)
 # (*) or SVNPATH contains 'trunk' or 'branches' 
+# (*) or GITPATH has no '@' (trunk) 
+# (*) or GITPATH contains a '@', and the gittag part has no '-' (this is to tell a tag from a branch..)
 # 
 define package_hasdate
 $(1).has-date = $(if $($(1)-RPMDATE),yes, \
                   $(if $($($(1).module)-SVNPATH), \
 		     $(if $(findstring /trunk,$($($(1).module)-SVNPATH)),yes, \
 			$(if $(findstring /branches,$($($(1).module)-SVNPATH)),yes,)), \
-		     $(if $(findstring HEAD,$($($(1).module)-TAG)),yes,)))
+		     $(if $(not $($($(1).module).gittag)), yes,\
+			$(if $(findstring -,$($($(1).module).gittag)),,yes))))
 endef
 
 $(foreach package, $(ALL), $(eval $(call package_hasdate,$(package))))
@@ -313,26 +344,55 @@ header.spec:
 	echo "%define distroname $(DISTRONAME)" >> $@
 	echo "%define pldistro $(PLDISTRO)" >> $@
 	echo "%define plrelease $(PLANETLAB_RELEASE)" >> $@
+	echo "# use MD5 and gzip for binary and source files" >> $@
+	echo "%global _binary_filedigest_algorithm 1" >> $@
+	echo "%global _source_filedigest_algorithm 1" >> $@
+	echo "%global _source_payload       w9.gzdio" >> $@
+	echo "%global _binary_payload       w9.gzdio" >> $@
 
-### extract spec file from scm
+### make up spec file - extract module first
 define target_spec
-$($(1).specpath): header.spec
+$($(1).specpath): header.spec $($(1).codespec)
 	mkdir -p SPECS
 	cat header.spec > $($(1).specpath)
+	echo "%define SCMURL $($($(1).module)-SCMPATH)" >> $($(1).specpath)
 	$(if $($(1).has-date),echo "%define date $(shell date +%Y.%m.%d)" >> $($(1).specpath),)
 	$(if $($(1)-SPECVARS), \
 	  $(foreach line,$($(1)-SPECVARS), \
 	    echo "%define" $(word 1,$(subst =, ,$(line))) "$(word 2,$(subst =, ,$(line)))" >> $($(1).specpath) ;))
 	echo "# included from $($(1)-SPEC)" >> $($(1).specpath)
-	$(if $($($(1).module)-SVNPATH),\
-          svn cat $($($(1).module).svnpath)/$($(1)-SPEC)$($($(1).module).svnrev) >> $($(1).specpath) || rm $($(1).specpath),\
-          cvs -d $($($(1).module)-CVSROOT) checkout \
-	      -r $($($(1).module)-TAG) \
-	      -p $($(1).module)/$($(1)-SPEC) >> $($(1).specpath))
+	cat $($(1).codespec) >> $($(1).specpath)
+
+$($(1).codespec): $($(1).moduledir)
+
 endef
 
 $(foreach package,$(ALL),$(eval $(call target_spec,$(package))))
 
+### module extraction
+define target_extract_module
+MODULES/$(1):
+	@(echo -n "XXXXXXXXXXXXXXX -- BEG MODULE $(1) : $@ " ; date)
+	mkdir -p MODULES
+	cd MODULES && \
+	$(if $($(1)-SVNPATH),\
+	  svn export $($(1)-SVNPATH) $(1),\
+	  mkdir $(1) ; (git archive --remote=$($(1).gitrepo) $($(1).gittag) | tar -C $(1) -xf - ) \
+	   || { rm -rf $(1); false; } )
+	@(echo -n "XXXXXXXXXXXXXXX -- END MODULE $(1) : $@ " ; date)
+
+$(1)-module: MODULES/$(1)
+.PHONY: $(1)-module
+endef
+
+$(foreach module,$(ALL.modules),$(eval $(call target_extract_module,$(module))))
+
+### the tests area
+# use this makefile to extract tests rather than extracting manually in vbuild-nightly
+tests-clean:
+	rm -rf MODULES/tests
+.PHONY: tests-clean
+
 ###
 # Base rpmbuild in the current directory
 # issues on fedora 8 : see the following posts
@@ -358,44 +418,50 @@ endif
 
 ### this utility allows to extract various info from a spec file
 ### and to define them in makefiles
+# use the C code where it works as it's the original one, use the python code otherwise
 spec2make: spec2make.c
-	$(CC) -g -Wall $< -o $@ -lrpm -lrpmbuild
+	$(CC) -g -Wall $< -o $@ -lrpm -lrpmbuild -lrpmio -lpopt || ln -s spec2make.py $@
 
 ### run spec2make on the spec file and include the result
 # usage: spec2make package
+# with old rpms (f8&c5) and too recent specfiles (the kernel), we need a patch to spec2make
+# so when <package-NEEDSPECK2MAKEHACK is set, we run spec2make with the --hack flag
 define target_mk
 MAKE/$(1).mk: $($(1).specpath) spec2make .rpmmacros
 	mkdir -p MAKE
-	./spec2make $($(1)-RPMFLAGS) $($(1).specpath) $(1) > MAKE/$(1).mk || { rm MAKE/$(1).mk; exit 1; }
+	./spec2make $($(1).spec2makeflags) $($(1)-RPMFLAGS) $($(1).specpath) $(1) > MAKE/$(1).mk || { rm MAKE/$(1).mk; exit 1; }
 endef
 
 $(foreach package,$(ALL),$(eval $(call target_mk,$(package))))
 
-# stores PLDISTRO in a file
+# stores env variables in a file
 # this is done at stage1. later run wont get confused
-savedpldistro.mk:
-	echo "PLDISTRO:=$(PLDISTRO)" > $@
-	echo "PLDISTROTAGS:=$(PLDISTROTAGS)" >> $@
-	echo "build-SVNPATH:=$(build-SVNPATH)" >> $@
-	echo "PERSONALITY:=$(PERSONALITY)" >> $@
-	echo "MAILTO:=$(MAILTO)" >> $@
-	echo "BASE:=$(BASE)" >> $@
-	echo "WEBPATH:=$(WEBPATH)" >> $@
-	echo "TESTBUILDURL:=$(TESTBUILDURL)" >> $@
-	echo "WEBROOT:=$(WEBROOT)" >> $@
-	echo "alias m=\"make PLDISTRO=$(PLDISTRO) PLDISTROTAGS=$(PLDISTROTAGS))\"" > aliases
-
-savedpldistro: savedpldistro.mk
-.PHONY: savedpldistro
+SAVED_VARS=PLDISTRO PLDISTROTAGS build-GITPATH PERSONALITY MAILTO BASE WEBPATH TESTBUILDURL WEBROOT
+# also remember variable settings in alias, like sfa-GITPATH=git://git.f-lab.fr/sfa.git@generic
+# but don't save stage1
+ASSIGNS=$(foreach chunk,$(MAKEFLAGS),$(if $(findstring =,$(chunk)),$(if $(findstring stage1,$(chunk)),,$(chunk)),))
+envfrompreviousrun.mk:
+	@echo "# do not edit" > $@
+	@$(foreach var,$(SAVED_VARS),echo "$(var):=$($(var))" >> $@ ;)
+	@$(foreach chunk,$(ASSIGNS),echo "override $(chunk)" | sed -e s,=,:=, >> $@;)
+	@echo "# do not edit" > aliases
+	@echo -n "alias m=\"make " >> aliases
+	@$(foreach var,$(SAVED_VARS),echo -n " $(var)=$($(var))" >> aliases ;)
+	@echo -n $(ASSIGNS) >> aliases
+	@echo "\"" >> aliases
+	@echo "alias m1=\"m stage1=true\"" >> aliases
+
+envfrompreviousrun: envfrompreviousrun.mk
+.PHONY: envfrompreviousrun
 
 # always refresh this
-all: savedpldistro
+all: envfrompreviousrun
 
 #################### regular make
 
 define stage2_variables
 ### devel dependencies
-$(1).rpmbuild = $(if $($(1)-RPMBUILD),$($(1)-RPMBUILD),$(RPMBUILD)) $($(1)-RPMFLAGS)
+$(1).rpmbuild = $(RPMBUILD) $($(1)-RPMFLAGS)
 $(1).all-devel-rpm-paths := $(foreach rpm,$($(1)-DEPEND-DEVEL-RPMS),$($(rpm).rpm-path))
 $(1).depend-devel-packages := $(sort $(foreach rpm,$($(1)-DEPEND-DEVEL-RPMS),$($(rpm).package)))
 ALL-DEVEL-RPMS += $($(1)-DEPEND-DEVEL-RPMS)
@@ -428,8 +494,8 @@ URLS/%:
 	touch $@
 
 ### the directory SOURCES/<package>-<version> is made 
-# with a copy -rl from CODEBASES/<package>
-# the former is $(package.source) and the latter is $(package.codebase)
+# with a (set of) copy -rl from MODULES/<module>
+# the former is $(package.source) 
 ALLSOURCES:=$(foreach package, $(ALL), $($(package).source))
 # so that make does not use the rule below directly for creating the tarball files
 .SECONDARY: $(ALLSOURCES)
@@ -438,42 +504,23 @@ sources: $(ALLSOURCES)
 	@echo $(words $(ALLSOURCES)) versioned source trees OK
 .PHONY: sources
 
-define target_link_codebase_sources
-$($(1).source): $($(1).codebase) ; mkdir -p SOURCES ; cp -rl $($(1).codebase) $($(1).source)
+# argument is a package
+# do things differently if multiple modules are mentioned (sigh..)
+define target_copy_link_modules_sources
+$($(1).source): $(foreach module,$($(1)-MODULES),MODULES/$(module))
+	$(if $(word 2,$($(1)-MODULES)),\
+	 mkdir -p $($(1).source) && $(foreach module,$($(1)-MODULES), cp -rl MODULES/$(module) $($(1).source)/$(module);) ,\
+	 mkdir -p SOURCES && cp -rl MODULES/$($(1)-MODULES) $($(1).source))
 endef
 
-$(foreach package,$(ALL),$(eval $(call target_link_codebase_sources,$(package))))
-
-### codebase extraction
-ALLCODEBASES:=$(foreach package, $(ALL), $($(package).codebase))
-# so that make does not use the rule below directly for creating the tarball files
-.SECONDARY: $(ALLCODEBASES)
+$(foreach package,$(ALL),$(eval $(call target_copy_link_modules_sources,$(package))))
 
-codebases : $(ALLCODEBASES)
-	@echo $(words $(ALLCODEBASES)) codebase OK
-.PHONY: codebases
-
-### extract codebase 
-# usage: extract_single_module package 
-define extract_single_module
-	mkdir -p CODEBASES
-	$(if $($($(1).module)-SVNPATH), cd CODEBASES && svn export $($($(1).module)-SVNPATH) $(1), cd CODEBASES && cvs -d $($($(1).module)-CVSROOT) export -r $($($(1).module)-TAG) -d $(1) $($(1).module))
-endef
-
-# usage: extract_multi_module package 
-define extract_multi_module
-	mkdir -p CODEBASES/$(1) && cd CODEBASES/$(1) && (\
-	$(foreach m,$($(1)-MODULES), $(if $($(m)-SVNPATH), svn export $($(m)-SVNPATH) $(m);, cvs -d $($(m)-CVSROOT) export -r $($(m)-TAG) $(m);)))
-endef
-
-CODEBASES/%: package=$(notdir $@)
-CODEBASES/%: multi_module=$(word 2,$($(package)-MODULES))
-CODEBASES/%: 
-	@(echo -n "XXXXXXXXXXXXXXX -- BEG CODEBASE $(package) : $@ " ; date)
-	$(if $(multi_module),\
-	  $(call extract_multi_module,$(package)),\
-	  $(call extract_single_module,$(package)))
-	@(echo -n "XXXXXXXXXXXXXXX -- END CODEBASE $(package) : $@ " ; date)
+### code extraction
+ALLMODULES:=$(foreach module, $(ALL.modules), MODULES/$(module))
+.SECONDARY: $(ALLMODULES)
+modules: $(ALLMODULES)
+	@echo $(words $(ALLMODULES)) modules OK "(fetched from scm)"
+.PHONY: modules
 
 ### source rpms
 ALLSRPMS:=$(foreach package,$(ALL),$($(package).srpm))
@@ -481,14 +528,20 @@ srpms: $(ALLSRPMS)
 	@echo $(words $(ALLSRPMS)) source rpms OK
 .PHONY: srpms
 
+
+# install the DEVEL-RPMS rpms if defined
+define handle_stock_devel_rpms_pre 
+	$(if $($(1)-DEVEL-RPMS), echo "Installing for $(1)-DEVEL-RPMS" ; $(YUM-INSTALL-DEVEL) $($(1)-DEVEL-RPMS))
+endef
+
 ### these macro handles the DEPEND-DEVEL-RPMS and EXCLUDE-DEVEL-RPMS tags for a hiven package
 # before building : rpm-install DEPEND-DEVEL-RPMS and rpm-uninstall EXCLUDE
-define handle_devel_rpms_pre 
+define handle_local_devel_rpms_pre 
 	$(if $($(1).all-devel-rpm-paths), echo "Installing for $(1)-DEPEND-DEVEL-RPMS" ; $(RPM-INSTALL-DEVEL) $($(1).all-devel-rpm-paths)) 
 	$(if $($(1)-EXCLUDE-DEVEL-RPMS), echo "Uninstalling for $(1)-EXCLUDE-DEVEL-RPMS" ; $(RPM-UNINSTALL-DEVEL) $($(1)-EXCLUDE-DEVEL-RPMS))
 endef
 
-define handle_devel_rpms_post
+define handle_local_devel_rpms_post
 	-$(if $($(1)-DEPEND-DEVEL-RPMS), echo "Unstalling for $(1)-DEPEND-DEVEL-RPMS" ; $(RPM-UNINSTALL-DEVEL) $($(1)-DEPEND-DEVEL-RPMS))
 	$(if $($(1)-EXCLUDE-DEVEL-RPMS), "Reinstalling for $(1)-EXCLUDE-DEVEL-RPMS" ; $(YUM-INSTALL-DEVEL) $($(1)-EXCLUDE-DEVEL-RPMS) )
 endef
@@ -499,19 +552,21 @@ ifeq "$($(1)-BUILD-FROM-SRPM)" ""
 $($(1).srpm): $($(1).specpath) .rpmmacros $($(1).tarballs) 
 	mkdir -p BUILD SRPMS tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using SOURCES) " ; date)
-	$(call handle_devel_rpms_pre,$(1))
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
 	$($(1).rpmbuild) -bs $($(1).specpath)
-	$(call handle_devel_rpms_post,$(1))
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date)
 else
-$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).codebase)
+$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).source)
 	mkdir -p BUILD SRPMS tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using make srpm) " ; date)
-	$(call handle_devel_rpms_pre,$(1))
-	make -C $($(1).codebase) srpm SPECFILE=$(HOME)/$($(1).specpath) EXPECTED_SRPM=$(notdir $($(1).srpm)) && \
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
+	make -C $($(1).source) srpm SPECFILE=$(HOME)/$($(1).specpath) EXPECTED_SRPM=$(notdir $($(1).srpm)) && \
            rm -f SRPMS/$(notdir $($(1).srpm)) && \
-           ln $($(1).codebase)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm)) 
-	$(call handle_devel_rpms_post,$(1))
+           ln $($(1).source)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm)) 
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date)
 endif
 endef
@@ -537,18 +592,20 @@ $($(1).rpms): $($(1).srpm)
 	mkdir -p RPMS tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG RPM $(1) " ; date)
 	$(if $(findstring RPMS/yumgroups.xml,$($(1)-DEPEND-FILES)), $(createrepo) , )
-	$(call handle_devel_rpms_pre,$(1))
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
 	$($(1).rpmbuild) --rebuild $(RPM-USE-TMP-DIRS) $($(1).srpm)
-	$(call handle_devel_rpms_post,$(1))
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END RPM $(1) " ; date)
 # for manual use only - in case we need to investigate the results of an rpmbuild
 $(1)-compile: $($(1).srpm)
 	mkdir -p COMPILE tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG compile $(1) " ; date)
 	$(if $(findstring RPMS/yumgroups.xml,$($(1)-DEPEND-FILES)), $(createrepo) , )
-	$(call handle_devel_rpms_pre,$(1))
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
 	$($(1).rpmbuild) --recompile $(RPM-USE-TMP-DIRS) $($(1).srpm)
-	$(call handle_devel_rpms_post,$(1))
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END compile $(1) " ; date)
 .PHONY: $(1)-compile
 endef
@@ -566,10 +623,8 @@ $(1)-mk: $($(1)-MK)
 .PHONY: $(1)-mk
 $(1)-tarball: $($(1).tarballs)
 .PHONY: $(1)-tarball
-$(1)-codebase: $($(1).codebase)
 .PHONY: $(1)-source
 $(1)-source: $($(1).source)
-.PHONY: $(1)-codebase
 $(1)-rpms: $($(1).rpms)
 .PHONY: $(1)-rpms
 $(1)-srpm: $($(1).srpm)
@@ -606,10 +661,10 @@ $(foreach package,$(ALL),$(eval $(call target_depends,$(package))))
 ### clean target
 # usage: target_clean package
 define target_clean
-$(1)-clean-codebase:
-	rm -rf $($(1).codebase)
-.PHONY: $(1)-clean-codebase
-CLEANS += $(1)-clean-codebase
+$(1)-clean-modules:
+	$(foreach module,$($(1)-MODULES),rm -rf MODULES/$(module);)
+.PHONY: $(1)-clean-modules
+CLEANS += $(1)-clean-modules
 $(1)-clean-source:
 	rm -rf $($(1).source)
 .PHONY: $(1)-clean-source
@@ -630,7 +685,7 @@ $(1)-clean-srpm:
 .PHONY: $(1)-clean-srpm
 CLEANS += $(1)-clean-srpm
 $(1)-codeclean: $(1)-clean-source $(1)-clean-tarball $(1)-clean-build $(1)-clean-rpms $(1)-clean-srpm
-$(1)-clean: $(1)-clean-codebase $(1)-codeclean
+$(1)-clean: $(1)-clean-modules $(1)-codeclean
 .PHONY: $(1)-codeclean $(1)-clean 
 $(1)-clean-spec:
 	rm -rf $($(1).specpath)
@@ -657,9 +712,9 @@ clean-help:
 
 ### brute force clean
 distclean1:
-	rm -rf savedpldistro.mk .rpmmacros spec2make header.spec SPECS MAKE $(DISTCLEANS)
+	rm -rf envfrompreviousrun.mk .rpmmacros spec2make header.spec SPECS MAKE $(DISTCLEANS)
 distclean2:
-	rm -rf CODEBASES SOURCES BUILD BUILDROOT RPMS SRPMS tmp
+	rm -rf MODULES SOURCES BUILD BUILDROOT RPMS SRPMS tmp
 distclean: distclean1 distclean2
 .PHONY: distclean1 distclean2 distclean
 
@@ -677,7 +732,7 @@ myplc-release:
 	echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx build info" >> $@
 	$(MAKE) --no-print-directory version-build >> $@
 	echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx svn info" >> $@
-	$(MAKE) --no-print-directory version-svns >> $@
+	$(MAKE) --no-print-directory version-tags >> $@
 	echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx rpm info" >> $@
 	$(MAKE) --no-print-directory version-rpms >> $@
 	@echo $(BASE) > .base
@@ -689,7 +744,6 @@ version-build:
 	@echo    "Build build-base: $(BASE)"
 	@echo    "Build planetlab-distro: $(PLDISTRO)"
 	@echo    "Build planetlab-tags: $(PLDISTROTAGS)"
-	@echo -n 'Build planetlab-tagsid: ' ; fgrep '$$''Id' $(PLDISTROTAGS)
 	@echo    "Build target-arch: $(HOSTARCH)"
 	@echo    "Build target-distro: $(DISTRO)"
 	@echo    "Build target-distroname: $(DISTRONAME)"
@@ -699,16 +753,16 @@ version-build:
 #################### 
 # for a given module
 VFORMAT="%30s := %s\n"
-define svn_version_target
-$(1)-version-svn:
+define print_tag_target
+$(1)-version-tag:
 	@$(if $($(1)-SVNPATH),\
 	   printf $(VFORMAT) $(1)-SVNPATH "$($(1)-SVNPATH)",\
-	   printf $(VFORMAT) $(1)-CVSROOT "$($(1)-CVSROOT)" ; printf $(VFORMAT) $(1)-TAG "$($(1)-TAG)")
+	   printf $(VFORMAT) $(1)-GITPATH "$($(1)-GITPATH)" )
 endef
 
-$(foreach module,$(ALL-MODULES), $(eval $(call svn_version_target,$(module))))
+$(foreach module,$(ALL.modules), $(eval $(call print_tag_target,$(module))))
 
-version-svns: $(foreach module, $(ALL-MODULES), $(module)-version-svn)
+version-tags: $(foreach module, $(ALL.modules), $(module)-version-tag)
 
 RFORMAT="%20s :: version=%s release=%s\n"
 define rpm_version_target
@@ -719,11 +773,11 @@ endef
 
 $(foreach package,$(sort $(ALL)), $(eval $(call rpm_version_target,$(package))))
 
-versions: myplc-release version-build version-svns version-rpms
-.PHONY: versions version-build version-rpms version-svns
+versions: myplc-release version-build version-tags version-rpms
+.PHONY: versions version-build version-rpms version-tags
 
 #################### package info
-PKGKEYS := tarballs source codebase srpm rpms rpmnames rpm-release rpm-name rpm-version rpm-subversion
+PKGKEYS := tarballs source srpm rpms rpmnames rpm-release rpm-name rpm-version rpm-subversion
 %-pkginfo: package=$(subst -pkginfo,,$@)
 %-pkginfo: 
 	@$(foreach key,$(PKGKEYS),echo "$(package).$(key)=$($(package).$(key))";)
@@ -734,33 +788,34 @@ RPMKEYS := rpm-path package
 	@$(foreach key,$(RPMKEYS),echo "$(rpm).$(key)=$($(rpm).$(key))";)
 
 #################### various lists - designed to run with stage1=true
-packages:
+info-packages:
 	@$(foreach package,$(ALL), echo package=$(package) ref_module=$($(package).module) modules=$($(package)-MODULES) rpmnames=$($(package).rpmnames); )
 
-modules:
-	@$(foreach module,$(ALL-MODULES), echo module=$(module) svnpath=$($(module)-SVNPATH); )
+info-modules:
+	@$(foreach module,$(ALL.modules), echo module=$(module) \
+	   $(if $($(module)-SVNPATH),svnpath=$($(module)-SVNPATH),gitpath=$($(module)-GITPATH)) ; )
 
-branches:
-	@$(foreach module,$(ALL-MODULES), \
-	  $(if $($(module)-SVNBRANCH),echo module=$(module) branch=$($(module)-SVNBRANCH);))
+info-branches:
+	@$(foreach module,$(ALL.modules), \
+	  $(if $($(module)-BRANCH),echo module=$(module) branch=$($(module)-BRANCH);))
 
 module-tools:
-	@$(foreach module,$(ALL-MODULES), \
- 	 $(if $($(module)-SVNPATH), \
-	  $(if $($(module)-SVNBRANCH), \
-	     echo $(module):$($(module)-SVNBRANCH); , \
+	@$(foreach module,$(ALL.modules), \
+	 $(if $($(module)-GITPATH), \
+	  $(if $($(module)-BRANCH), \
+	     echo $(module):$($(module)-BRANCH); , \
+	     echo $(module); ), \
+	  $(if $($(module)-BRANCH), \
+	     echo $(module):$($(module)-BRANCH); , \
 	     echo $(module); )))
 
-info: packages modules branches 
-
-.PHONY: info packages modules branches module-tools
-
-####################
-tests_svnpath:
-	@$(if $(tests-SVNPATH), echo $(tests-SVNPATH) > $@, \
-	echo "http://svn.planet-lab.org/svn/tests/trunk" > $@)
+info: info-packages info-modules info-branches 
 
+.PHONY: info info-packages info-modules info-branches module-tools
 
+show-multi-modules:
+	@$(foreach package,$(ALL),$(if $(subst x1,,x$(words $($(package)-MODULES))), echo p=$(package) mods=$($(package)-MODULES) ;))
+.PHONY: show-multi-modules
 ####################
 help:
 	@echo "********** Run make in two stages:"
@@ -777,8 +832,6 @@ help:
 	@echo "  equivalent to 'make util-vserver-rpms'"
 	@echo ""
 	@echo "********** Or, vertically - step-by-step for a given package"
-	@echo 'make util-vserver-codebase'
-	@echo "  performs codebase extraction in CODEBASES/util-vserver"
 	@echo 'make util-vserver-source'
 	@echo "  creates source link in SOURCES/util-vserver-<version>"
 	@echo 'make util-vserver-tarball'
@@ -789,7 +842,6 @@ help:
 	@echo "  build rpm(s) in RPMS/"
 	@echo ""
 	@echo "********** Or, horizontally, reach a step for all known packages"
-	@echo 'make codebases'
 	@echo 'make sources'
 	@echo 'make tarballs'
 	@echo 'make srpms'
@@ -811,11 +863,11 @@ help:
 	@echo "make iptables-distclean"
 	@echo "  deep clean for a given package"
 	@echo "make iptables-codeclean"
-	@echo "  run this if you've made a change in the CODEBASES area for iptables"
+	@echo "  run this if you've made a local/manual change in the MODULES area for iptables"
 	@echo ""
 	@echo "make util-vserver-clean"
 	@echo "  removes codebase, source, tarball, build, rpm and srpm for util-vserver"
-	@echo "make util-vserver-clean-codebase"
+	@echo "make util-vserver-clean-modules"
 	@echo "  and so on for source, tarball, build, rpm and srpm"
 	@echo ""
 	@echo "********** Info examples"