X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=Makefile;h=78533ca6a5e8631994f847dd9d3e23d7a6364ab6;hb=refs%2Fheads%2Flxc_devel;hp=718e00a157a8185205d7f4496522634a12ee0f28;hpb=835c01dd6cd87786234a90e52271f619e911de4a;p=build.git

diff --git a/Makefile b/Makefile
index 718e00a1..78533ca6 100644
--- a/Makefile
+++ b/Makefile
@@ -28,47 +28,62 @@
 # 
 # (*) packages are named upon the RPM name; they are mostly lowercase
 #     Add a package to ALL if you want it built as part of the default set.
-# (*) modules are named after the subversion tree; as of this writing their names 
-#     are mostly mixed case like MyPLC or VserverReference
-#     (this is something we'll fix while moving to git)
+# (*) modules are named after the subversion or git tree; we would wish to keep
+#     these names lowercase as far as possible
 # (*) rpms are named in the spec files. A package typically defines several rpms;
 #     rpms are used for defining DEPEND-DEVEL-RPMS. See also package.rpmnames
 # 
+# in simple cases, one package uses one module (in which case using the same name sounds right)
+# but others might need several modules (e.g. image-creation packages like bootstrapfs need 
+# bootstrapfs and build); in this case the FIRST ONE is used for locating the specfile 
+#
 #################### packages
-# basics: how to build a package - you need/may define the following variables
+# basics: how to build a package - you need to define the following variables
 # 
 # (*) package-MODULES
-#     a package needs one or several modules to build. 
-#     to this end, define 
+#     a package needs one or several modules to build; the first one is used for 
+#     some special purposes, like locating the specfile
 # (*) package-SPEC
 #     the package's specfile; this is relative to the FIRST module in package-MODULES
 #
 # Optional:
 #
-# (*) package-SPECVARS
-#     space-separated list of spec variable definitions, where you can reference make variable that relate to 
-#     packages defined BEFORE the current one (note: you should use = - as opposed to := - to define these)
-#     e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release) 
-#     would let you use the %release from the kernel's package when rpmbuild'ing mydriver - see automatic below
 # (*) package-DEPEND-PACKAGES
-#     a set of *packages* that this package depends on
+#     a set of (obviously local) *packages* that this package depends on, e.g.
+#     bootstrapfs-DEPEND-PACKAGES += kernel
+#     this impacts the order of the build
+# (*) package-DEVEL-RPMS
+#     a set of stock rpms that this package needs at build-time
+#     this can also be set in config.<distro>/devel.pkgs or config.planetlab/devel.pkgs as appropriate
+# (*) package-EXCLUDE-DEVEL-RPMS
+#     a set of stock *rpms* that the build will rpm-uninstall before building <package>
+#     this is intended to denote stock rpms, and the build will attempt to yum-install them
+#     back after the package is rebuilt
+#     This feature is not used at the moment and kept only just in case
 # (*) package-DEPEND-DEVEL-RPMS
-#     a set of *rpms* that the build will rpm-install before building <package>
+#     a set of local *rpms* that the build will rpm-install before building <package>
 #     the build will attempt to uninstall those once the package is built, this is not fatal though
 #     this is intended to denote local rpms, i.e. ones that are results of our own build
-#     stock rpms should be mentioned in config.planetlab/devel.pkgs
+#     stock rpms should be mentioned in DEVEL-RPMS or in devel.pkgs as described above
 # (*) package-DEPEND-FILES
 #     a set of files that the package depends on - and that make needs to know about
 #     if this contains RPMS/yumgroups.xml, then the toplevel RPMS's index 
 #     is refreshed with createrepo prior to running rpmbuild
-# (*) package-EXCLUDE-DEVEL-RPMS
-#     a set of *rpms* that the build will rpm-uninstall before building <package>
-#     this is intended to denote stock rpms, and the build will attempt to yum-install them
-#     back after the package is rebuilt
+# (*) package-SPECVARS
+#     space-separated list of spec variable definitions, where you can reference make variables that 
+#     belong to packages defined BEFORE the current one 
+#     note: you should use = to define these (as opposed to :=)
+#     e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release) 
+#     would let you use the %release from the kernel's package when rpmbuild'ing mydriver 
+#     see automatic below
 # (*) package-RPMFLAGS: Miscellaneous RPM flags
-# (*) package-RPMBUILD: If not rpmbuild - mostly used for sudo'ing rpmbuild
-# (*) package-BUILD-FROM-SRPM: set this to any non-empty value, if your package is able to produce 
-#     a source rpms by running 'make srpm'
+#     this is passed to rpmbuild, as well as to spec2make for "exporting" various rpm variable
+#     to make; beware that some features, like --with=debug or --define 'foo bar' are not
+#     well handled by spec2make as of this writing, which can cause issues.
+#     hopefully this will be fixed...
+# (*) package-BUILD-FROM-SRPM: set this to any non-empty value, 
+#     if your package is able to produce a source rpm. 
+#     In this case the build will first invoke 'make srpm', and then rebuild binaries from that 
 # (*) package-RPMDATE: set this to any non-empty value to get the rpm package's release field hold the current date
 #     this is useful for container packages, like e.g. bootstrapfs or vserver, that contains much more than the
 #     correspondng module
@@ -88,7 +103,7 @@
 #################### automatic variables
 #
 # the build defines some make variables that are extracted from spec files
-# see for example
+# to inspect those, see for example
 # (*)  $ make ulogd-pkginfo
 #        to see the list f variables attached to a given package
 # (*)  $ make kernel-devel-rpminfo
@@ -122,15 +137,15 @@ all:
 PLDISTRO := planetlab
 RPMBUILD := rpmbuild
 
-########## savedpldistro.mk holds PLDISTRO - it is generated at stage1 (see below)
+########## envfrompreviousrun.mk holds PLDISTRO and others - it is generated at stage1 (see below)
 ifeq "$(stage1)" ""
-include savedpldistro.mk
+include envfrompreviousrun.mk
 endif
 
 # when re-running the nightly build after failure, we need to gather the former values
 # do this by running make stage1=skip +PLDISTRO
 ifeq "$(stage1)" "skip"
-include savedpldistro.mk
+include envfrompreviousrun.mk
 endif
 
 #################### include onelab.mk
@@ -148,14 +163,9 @@ include $(PLDISTROTAGS)
 # this used to be set in the -tags.mk files, but that turned out to require
 # error-prone duplicate changes 
 # so now the nightly build script sets this to what it is currently using
-# in case we run this manually, i.e. if neither build-SVNPATH nor build-GITPATH is set
-ifeq "$(build-SVNPATH)$(build-GITPATH)" ""
-build-GITPATH-local := $(shell git config remote.origin.url)
-ifneq "$(build-GITPATH-local)" ""
-build-GITPATH := $(build-GITPATH-local)
-else
-build-SVNPATH := $(lastword $(shell svn info 2> /dev/null | grep URL:))
-endif
+# in case we run this manually, i.e. if build-GITPATH is not set
+ifeq "$(build-GITPATH)" ""
+build-GITPATH := $(shell git config remote.origin.url)
 endif
 
 #################### pldistros that are defined remotely
@@ -167,16 +177,17 @@ config.$(1): config.$(1).svnpath
 	svn export $$($(1).config.SVNPATH) config.$(1)
 endef
 
+FOO=@
 # fetching with git
 define remote_pldistro_fetch_git
 $(1).config.GITPATH := $(shell grep -v "^#" config.$(1).gitpath)
-$(1).config.gitrepo := $(firstword $(subst @, ,$$($(1).config.GITPATH)))
-$(1).config.gittag := $(word 2,$(subst @, ,$$($(1).config.GITPATH)))
-$(1).config.gittag := $(if $$($(1).config.gittag),$$($(1).config.gittag),master)
+$(1).config.gitrepo := $$(firstword $$(subst @, ,$$($(1).config.GITPATH)))
+$(1).config.gittag := $$(word 2,$$(subst @, ,$$($(1).config.GITPATH)))
+$(1).config.gittag := $$(if $$($(1).config.gittag),$$($(1).config.gittag),master)
 config.$(1): GITPATH=$(shell grep -v "^#" config.$(1).gitpath)
 config.$(1): config.$(1).gitpath
 	@echo "Fetching (git) details for remote pldistro $(1)"
-	mkdir config.$(1)
+	mkdir -p config.$(1)
 	git $(GITTAG) archive --remote=$$($(1).config.gitrepo) $$($(1).config.gittag) | tar -C config.$(1) -xf -
 endef
 
@@ -198,9 +209,6 @@ DISTCLEANS += $(1).mk $(2).mk config.$(1)
 $(eval $(call remote_pldistro_fetch_$(3),$(1)))
 endef
 
-# somehow this does not work, handle manually instead
-#REMOTE-PLDISTROS="wextoolbox"
-#$(foreach distro, $(REMOTE-PLDISTROS), $(eval $(call remote_pldistro,$(distro),$(distro)-tags)))
 $(eval $(call remote_pldistro,wextoolbox,wextoolbox-tags,git))
 
 ########## stage1 and stage1iter
@@ -282,6 +290,7 @@ $(1).module := $(firstword $($(1)-MODULES))
 $(1).specpath := SPECS/$(notdir $($(1)-SPEC))
 $(1).moduledir := MODULES/$$($(1).module)
 $(1).codespec := MODULES/$$($(1).module)/$($(1)-SPEC)
+$(1).spec2makeflags := $(if $($(1)-WHITELIST-RPMS),--whitelist-rpms $($(1)-WHITELIST-RPMS),)
 endef
 
 $(foreach package, $(ALL), $(eval $(call stage1_package_vars,$(package))))
@@ -295,11 +304,13 @@ ALL.modules:=$(sort $(ALL.modules))
 define stage1_module_vars
 ifneq "$($(1)-SVNPATH)" ""
 $(1)-SVNPATH := $(strip $($(1)-SVNPATH))
+$(1)-SCMPATH := $(strip $($(1)-SVNPATH))
 else
-$(1)-GITPATH := $(strip $($(1)-GITPATH))
-$(1).gitrepo := $(firstword $(subst @, ,$($(1)-GITPATH)))
-$(1).gittag := $(word 2,$(subst @, ,$($(1)-GITPATH)))
-$(1).gittag := $(if $($(1).gittag),$($(1).gittag),master)
+$(1)-GITPATH := $$(strip $$($(1)-GITPATH))
+$(1)-SCMPATH := $$(strip $$($(1)-GITPATH))
+$(1).gitrepo := $$(firstword $$(subst @, ,$$($(1)-GITPATH)))
+$(1).gittag := $$(word 2,$$(subst @, ,$$($(1)-GITPATH)))
+$(1).gittag := $$(if $$($(1).gittag),$$($(1).gittag),master)
 endif
 endef
 
@@ -344,6 +355,7 @@ define target_spec
 $($(1).specpath): header.spec $($(1).codespec)
 	mkdir -p SPECS
 	cat header.spec > $($(1).specpath)
+	echo "%define SCMURL $($($(1).module)-SCMPATH)" >> $($(1).specpath)
 	$(if $($(1).has-date),echo "%define date $(shell date +%Y.%m.%d)" >> $($(1).specpath),)
 	$(if $($(1)-SPECVARS), \
 	  $(foreach line,$($(1)-SPECVARS), \
@@ -406,42 +418,50 @@ endif
 
 ### this utility allows to extract various info from a spec file
 ### and to define them in makefiles
+# use the C code where it works as it's the original one, use the python code otherwise
 spec2make: spec2make.c
-	$(CC) -g -Wall $< -o $@ -lrpm -lrpmbuild
+	$(CC) -g -Wall $< -o $@ -lrpm -lrpmbuild -lrpmio -lpopt || ln -s spec2make.py $@
 
 ### run spec2make on the spec file and include the result
 # usage: spec2make package
+# with old rpms (f8&c5) and too recent specfiles (the kernel), we need a patch to spec2make
+# so when <package-NEEDSPECK2MAKEHACK is set, we run spec2make with the --hack flag
 define target_mk
 MAKE/$(1).mk: $($(1).specpath) spec2make .rpmmacros
 	mkdir -p MAKE
-	./spec2make $($(1)-RPMFLAGS) $($(1).specpath) $(1) > MAKE/$(1).mk || { rm MAKE/$(1).mk; exit 1; }
+	./spec2make $($(1).spec2makeflags) $($(1)-RPMFLAGS) $($(1).specpath) $(1) > MAKE/$(1).mk || { rm MAKE/$(1).mk; exit 1; }
 endef
 
 $(foreach package,$(ALL),$(eval $(call target_mk,$(package))))
 
-# stores PLDISTRO in a file
+# stores env variables in a file
 # this is done at stage1. later run wont get confused
-SAVED_VARS=PLDISTRO PLDISTROTAGS build-SVNPATH PERSONALITY MAILTO BASE WEBPATH TESTBUILDURL WEBROOT
-savedpldistro.mk:
+SAVED_VARS=PLDISTRO PLDISTROTAGS build-GITPATH PERSONALITY MAILTO BASE WEBPATH TESTBUILDURL WEBROOT
+# also remember variable settings in alias, like sfa-GITPATH=git://git.f-lab.fr/sfa.git@generic
+# but don't save stage1
+ASSIGNS=$(foreach chunk,$(MAKEFLAGS),$(if $(findstring =,$(chunk)),$(if $(findstring stage1,$(chunk)),,$(chunk)),))
+envfrompreviousrun.mk:
 	@echo "# do not edit" > $@
 	@$(foreach var,$(SAVED_VARS),echo "$(var):=$($(var))" >> $@ ;)
+	@$(foreach chunk,$(ASSIGNS),echo "override $(chunk)" | sed -e s,=,:=, >> $@;)
 	@echo "# do not edit" > aliases
 	@echo -n "alias m=\"make " >> aliases
 	@$(foreach var,$(SAVED_VARS),echo -n " $(var)=$($(var))" >> aliases ;)
+	@echo -n $(ASSIGNS) >> aliases
 	@echo "\"" >> aliases
 	@echo "alias m1=\"m stage1=true\"" >> aliases
 
-savedpldistro: savedpldistro.mk
-.PHONY: savedpldistro
+envfrompreviousrun: envfrompreviousrun.mk
+.PHONY: envfrompreviousrun
 
 # always refresh this
-all: savedpldistro
+all: envfrompreviousrun
 
 #################### regular make
 
 define stage2_variables
 ### devel dependencies
-$(1).rpmbuild = $(if $($(1)-RPMBUILD),$($(1)-RPMBUILD),$(RPMBUILD)) $($(1)-RPMFLAGS)
+$(1).rpmbuild = $(RPMBUILD) $($(1)-RPMFLAGS)
 $(1).all-devel-rpm-paths := $(foreach rpm,$($(1)-DEPEND-DEVEL-RPMS),$($(rpm).rpm-path))
 $(1).depend-devel-packages := $(sort $(foreach rpm,$($(1)-DEPEND-DEVEL-RPMS),$($(rpm).package)))
 ALL-DEVEL-RPMS += $($(1)-DEPEND-DEVEL-RPMS)
@@ -508,14 +528,20 @@ srpms: $(ALLSRPMS)
 	@echo $(words $(ALLSRPMS)) source rpms OK
 .PHONY: srpms
 
+
+# install the DEVEL-RPMS rpms if defined
+define handle_stock_devel_rpms_pre 
+	$(if $($(1)-DEVEL-RPMS), echo "Installing for $(1)-DEVEL-RPMS" ; $(YUM-INSTALL-DEVEL) $($(1)-DEVEL-RPMS))
+endef
+
 ### these macro handles the DEPEND-DEVEL-RPMS and EXCLUDE-DEVEL-RPMS tags for a hiven package
 # before building : rpm-install DEPEND-DEVEL-RPMS and rpm-uninstall EXCLUDE
-define handle_devel_rpms_pre 
+define handle_local_devel_rpms_pre 
 	$(if $($(1).all-devel-rpm-paths), echo "Installing for $(1)-DEPEND-DEVEL-RPMS" ; $(RPM-INSTALL-DEVEL) $($(1).all-devel-rpm-paths)) 
 	$(if $($(1)-EXCLUDE-DEVEL-RPMS), echo "Uninstalling for $(1)-EXCLUDE-DEVEL-RPMS" ; $(RPM-UNINSTALL-DEVEL) $($(1)-EXCLUDE-DEVEL-RPMS))
 endef
 
-define handle_devel_rpms_post
+define handle_local_devel_rpms_post
 	-$(if $($(1)-DEPEND-DEVEL-RPMS), echo "Unstalling for $(1)-DEPEND-DEVEL-RPMS" ; $(RPM-UNINSTALL-DEVEL) $($(1)-DEPEND-DEVEL-RPMS))
 	$(if $($(1)-EXCLUDE-DEVEL-RPMS), "Reinstalling for $(1)-EXCLUDE-DEVEL-RPMS" ; $(YUM-INSTALL-DEVEL) $($(1)-EXCLUDE-DEVEL-RPMS) )
 endef
@@ -526,19 +552,21 @@ ifeq "$($(1)-BUILD-FROM-SRPM)" ""
 $($(1).srpm): $($(1).specpath) .rpmmacros $($(1).tarballs) 
 	mkdir -p BUILD SRPMS tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using SOURCES) " ; date)
-	$(call handle_devel_rpms_pre,$(1))
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
 	$($(1).rpmbuild) -bs $($(1).specpath)
-	$(call handle_devel_rpms_post,$(1))
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date)
 else
 $($(1).srpm): $($(1).specpath) .rpmmacros $($(1).source)
 	mkdir -p BUILD SRPMS tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using make srpm) " ; date)
-	$(call handle_devel_rpms_pre,$(1))
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
 	make -C $($(1).source) srpm SPECFILE=$(HOME)/$($(1).specpath) EXPECTED_SRPM=$(notdir $($(1).srpm)) && \
            rm -f SRPMS/$(notdir $($(1).srpm)) && \
            ln $($(1).source)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm)) 
-	$(call handle_devel_rpms_post,$(1))
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date)
 endif
 endef
@@ -564,18 +592,20 @@ $($(1).rpms): $($(1).srpm)
 	mkdir -p RPMS tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG RPM $(1) " ; date)
 	$(if $(findstring RPMS/yumgroups.xml,$($(1)-DEPEND-FILES)), $(createrepo) , )
-	$(call handle_devel_rpms_pre,$(1))
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
 	$($(1).rpmbuild) --rebuild $(RPM-USE-TMP-DIRS) $($(1).srpm)
-	$(call handle_devel_rpms_post,$(1))
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END RPM $(1) " ; date)
 # for manual use only - in case we need to investigate the results of an rpmbuild
 $(1)-compile: $($(1).srpm)
 	mkdir -p COMPILE tmp
 	@(echo -n "XXXXXXXXXXXXXXX -- BEG compile $(1) " ; date)
 	$(if $(findstring RPMS/yumgroups.xml,$($(1)-DEPEND-FILES)), $(createrepo) , )
-	$(call handle_devel_rpms_pre,$(1))
+	$(call handle_stock_devel_rpms_pre,$(1))
+	$(call handle_local_devel_rpms_pre,$(1))
 	$($(1).rpmbuild) --recompile $(RPM-USE-TMP-DIRS) $($(1).srpm)
-	$(call handle_devel_rpms_post,$(1))
+	$(call handle_local_devel_rpms_post,$(1))
 	@(echo -n "XXXXXXXXXXXXXXX -- END compile $(1) " ; date)
 .PHONY: $(1)-compile
 endef
@@ -682,7 +712,7 @@ clean-help:
 
 ### brute force clean
 distclean1:
-	rm -rf savedpldistro.mk .rpmmacros spec2make header.spec SPECS MAKE $(DISTCLEANS)
+	rm -rf envfrompreviousrun.mk .rpmmacros spec2make header.spec SPECS MAKE $(DISTCLEANS)
 distclean2:
 	rm -rf MODULES SOURCES BUILD BUILDROOT RPMS SRPMS tmp
 distclean: distclean1 distclean2
@@ -771,10 +801,10 @@ info-branches:
 
 module-tools:
 	@$(foreach module,$(ALL.modules), \
- 	 $(if $($(module)-GITPATH), \
+	 $(if $($(module)-GITPATH), \
 	  $(if $($(module)-BRANCH), \
-	     echo git:$(module):$($(module)-BRANCH); , \
-	     echo git:$(module); ), \
+	     echo $(module):$($(module)-BRANCH); , \
+	     echo $(module); ), \
 	  $(if $($(module)-BRANCH), \
 	     echo $(module):$($(module)-BRANCH); , \
 	     echo $(module); )))
@@ -783,6 +813,9 @@ info: info-packages info-modules info-branches
 
 .PHONY: info info-packages info-modules info-branches module-tools
 
+show-multi-modules:
+	@$(foreach package,$(ALL),$(if $(subst x1,,x$(words $($(package)-MODULES))), echo p=$(package) mods=$($(package)-MODULES) ;))
+.PHONY: show-multi-modules
 ####################
 help:
 	@echo "********** Run make in two stages:"