From fa5268e4c1d7101ad3b3f202400a2bedf9a4ba03 Mon Sep 17 00:00:00 2001 From: Marc Fiuczynski Date: Fri, 6 Nov 2009 21:38:25 +0000 Subject: [PATCH] coblitz rc2 --- trunk/Makefile | 836 ++++++++++ trunk/README-build.txt | 49 + trunk/README-pkgsfiles.txt | 14 + trunk/README-pldistros.txt | 52 + trunk/README-specfiles.txt | 54 + trunk/build-conf-planetlab.py | 84 + trunk/build.common | 740 +++++++++ trunk/coblitz-tags.mk | 46 + trunk/coblitz.mk | 395 +++++ trunk/config.coblitz/bootstrapfs.pkgs | 126 ++ trunk/config.coblitz/bootstrapfs.post | 26 + trunk/config.coblitz/coblitz.mirrors | 4 + trunk/config.coblitz/devel.pkgs | 99 ++ trunk/config.onelab/bootstrapfs-umts.pkgs | 6 + trunk/config.onelab/bootstrapfs-wifi.pkgs | 20 + trunk/config.onelab/bootstrapfs.pkgs | 128 ++ trunk/config.onelab/onelab.mirrors | 3 + trunk/config.onelab/vserver-planetflow.pkgs | 1 + trunk/config.onelab/vserver.pkgs | 37 + trunk/config.planetbridge/bootstrapfs.pkgs | 123 ++ trunk/config.planetbridge/bootstrapfs.post | 29 + .../config.planetbridge/planetbridge.mirrors | 4 + trunk/config.planetbridge/vserver-ap.pks | 5 + trunk/config.planetlab/bootcd.pkgs | 78 + trunk/config.planetlab/bootstrapfs.pkgs | 123 ++ trunk/config.planetlab/bootstrapfs.post | 53 + trunk/config.planetlab/devel.pkgs | 99 ++ trunk/config.planetlab/planetlab.mirrors | 11 + .../config.planetlab/vserver-planetflow.pkgs | 4 + trunk/config.planetlab/vserver.pkgs | 35 + trunk/config.planetlab/vserver.post | 17 + trunk/config.planetlab/vtest.pkgs | 6 + trunk/config.wextoolbox.svnpath | 2 + trunk/getdistro.sh | 6 + trunk/getdistroname.sh | 6 + trunk/getkexcludes.sh | 16 + trunk/getpackages.sh | 28 + trunk/getrelease.sh | 6 + trunk/getrpmmacros.sh | 5 + .../centos5/yum.repos.d/building.repo.in | 17 + .../f10/yum.repos.d/building.repo.in | 15 + .../f11/yum.repos.d/building.repo.in | 17 + .../mirroring/f8/yum.repos.d/building.repo.in | 20 + trunk/mirroring/mirror.sh | 154 ++ trunk/module-branch | 1 + trunk/module-diff | 1 + trunk/module-list | 1 + trunk/module-log.py | 196 +++ trunk/module-sync | 1 + trunk/module-tag | 1 + trunk/module-tools.py | 1414 +++++++++++++++++ trunk/module-version | 1 + trunk/modules.list | 44 + trunk/modules.update | 13 + trunk/onelab-k27-tags.mk | 50 + trunk/onelab-tags.mk | 52 + trunk/onelab.mk | 461 ++++++ trunk/planetbridge-tags.mk | 46 + trunk/planetbridge.mk | 396 +++++ trunk/planetlab-k27-tags.mk | 55 + trunk/planetlab-tags.mk | 47 + trunk/planetlab.mk | 396 +++++ trunk/release-changelog | 1 + trunk/rpmbuild.sh | 14 + trunk/run-nightlies.py | 103 ++ trunk/spec2make.c | 217 +++ trunk/trellis-tags.mk | 51 + trunk/trellis.mk | 396 +++++ trunk/vbuild-fedora-mirror.sh | 248 +++ trunk/vbuild-init-vserver.sh | 472 ++++++ trunk/vbuild-nightly.sh | 643 ++++++++ trunk/vtest-init-vserver.sh | 1 + trunk/vtest-nightly.sh | 226 +++ trunk/yumgroups.sh | 17 + 74 files changed, 9164 insertions(+) create mode 100644 trunk/Makefile create mode 100644 trunk/README-build.txt create mode 100644 trunk/README-pkgsfiles.txt create mode 100644 trunk/README-pldistros.txt create mode 100644 trunk/README-specfiles.txt create mode 100755 trunk/build-conf-planetlab.py create mode 100644 trunk/build.common create mode 100644 trunk/coblitz-tags.mk create mode 100644 trunk/coblitz.mk create mode 100644 trunk/config.coblitz/bootstrapfs.pkgs create mode 100644 trunk/config.coblitz/bootstrapfs.post create mode 100644 trunk/config.coblitz/coblitz.mirrors create mode 100644 trunk/config.coblitz/devel.pkgs create mode 100644 trunk/config.onelab/bootstrapfs-umts.pkgs create mode 100644 trunk/config.onelab/bootstrapfs-wifi.pkgs create mode 100644 trunk/config.onelab/bootstrapfs.pkgs create mode 100644 trunk/config.onelab/onelab.mirrors create mode 120000 trunk/config.onelab/vserver-planetflow.pkgs create mode 100644 trunk/config.onelab/vserver.pkgs create mode 100644 trunk/config.planetbridge/bootstrapfs.pkgs create mode 100644 trunk/config.planetbridge/bootstrapfs.post create mode 100644 trunk/config.planetbridge/planetbridge.mirrors create mode 100644 trunk/config.planetbridge/vserver-ap.pks create mode 100644 trunk/config.planetlab/bootcd.pkgs create mode 100644 trunk/config.planetlab/bootstrapfs.pkgs create mode 100644 trunk/config.planetlab/bootstrapfs.post create mode 100644 trunk/config.planetlab/devel.pkgs create mode 100644 trunk/config.planetlab/planetlab.mirrors create mode 100644 trunk/config.planetlab/vserver-planetflow.pkgs create mode 100644 trunk/config.planetlab/vserver.pkgs create mode 100644 trunk/config.planetlab/vserver.post create mode 100644 trunk/config.planetlab/vtest.pkgs create mode 100644 trunk/config.wextoolbox.svnpath create mode 100755 trunk/getdistro.sh create mode 100755 trunk/getdistroname.sh create mode 100755 trunk/getkexcludes.sh create mode 100755 trunk/getpackages.sh create mode 100755 trunk/getrelease.sh create mode 100755 trunk/getrpmmacros.sh create mode 100644 trunk/mirroring/centos5/yum.repos.d/building.repo.in create mode 100644 trunk/mirroring/f10/yum.repos.d/building.repo.in create mode 100644 trunk/mirroring/f11/yum.repos.d/building.repo.in create mode 100644 trunk/mirroring/f8/yum.repos.d/building.repo.in create mode 100755 trunk/mirroring/mirror.sh create mode 120000 trunk/module-branch create mode 120000 trunk/module-diff create mode 120000 trunk/module-list create mode 100755 trunk/module-log.py create mode 120000 trunk/module-sync create mode 120000 trunk/module-tag create mode 100755 trunk/module-tools.py create mode 120000 trunk/module-version create mode 100644 trunk/modules.list create mode 100755 trunk/modules.update create mode 100644 trunk/onelab-k27-tags.mk create mode 100644 trunk/onelab-tags.mk create mode 100644 trunk/onelab.mk create mode 100644 trunk/planetbridge-tags.mk create mode 100644 trunk/planetbridge.mk create mode 100644 trunk/planetlab-k27-tags.mk create mode 100644 trunk/planetlab-tags.mk create mode 100644 trunk/planetlab.mk create mode 120000 trunk/release-changelog create mode 100644 trunk/rpmbuild.sh create mode 100755 trunk/run-nightlies.py create mode 100644 trunk/spec2make.c create mode 100644 trunk/trellis-tags.mk create mode 100644 trunk/trellis.mk create mode 100755 trunk/vbuild-fedora-mirror.sh create mode 100755 trunk/vbuild-init-vserver.sh create mode 100755 trunk/vbuild-nightly.sh create mode 120000 trunk/vtest-init-vserver.sh create mode 100644 trunk/vtest-nightly.sh create mode 100755 trunk/yumgroups.sh diff --git a/trunk/Makefile b/trunk/Makefile new file mode 100644 index 00000000..9c8ea08f --- /dev/null +++ b/trunk/Makefile @@ -0,0 +1,836 @@ +# +# Thierry Parmentelat - INRIA Sophia Antipolis +# +### $Id$ +# +#################### +# invocation: +# +# (*) make stage1=true +# this extracts all specfiles and computes .mk from specfiles +# you need to specify PLDISTRO here if relevant - see below +# (*) make help +# for more info on how to invoke this stuff +# +#################### (fedora) distributions +# +# (*) as of nov. 2007, myplc-devel is deprecated +# (*) instead, we create a fresh vserver that holds required tools (see e.g. planetlab-devel.lst) +# (*) the build uses the current fedora version as a target for the produced images +# (*) so you simply need to create a fedora 8 build image for building fedora-8 images +# +#################### (planetlab) distributions +# +# (*) see README-pldistros.txt +# (*) then you need to run +# make stage1=true PLDISTRO=onelab +# +#################### +# This build deals with 3 kinds of objects +# +# (*) packages are named upon the RPM name; they are mostly lowercase +# Add a package to ALL if you want it built as part of the default set. +# (*) modules are named after the subversion tree; as of this writing their names +# are mostly mixed case like MyPLC or VserverReference +# (*) rpms are named in the spec files. A package typically defines several rpms; +# rpms are used for defining DEPEND-DEVEL-RPMS. See also package.rpmnames +# +#################### packages +# basics: how to build a package - you need/may define the following variables +# +# (*) package-MODULES +# a package needs one or several modules to build. +# to this end, define +# (*) package-SPEC +# the package's specfile; this is relative to the FIRST module in package-MODULES +# see 'codebase' below +# +# Optional: +# +# (*) package-SPECVARS +# space-separated list of spec variable definitions, where you can reference make variable that relate to +# packages defined BEFORE the current one (note: you should use = - as opposed to := - to define these) +# e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release) +# would let you use the %release from the kernel's package when rpmbuild'ing mydriver - see automatic below +# (*) package-DEPEND-PACKAGES +# a set of *packages* that this package depends on +# (*) package-DEPEND-DEVEL-RPMS +# a set of *rpms* that the build will rpm-install before building +# the build will attempt to uninstall those once the package is built, this is not fatal though +# this is intended to denote local rpms, i.e. ones that are results of our own build +# stock rpms should be mentioned in config.planetlab/devel.pkgs +# (*) package-DEPEND-FILES +# a set of files that the package depends on - and that make needs to know about +# if this contains RPMS/yumgroups.xml, then the toplevel RPMS's index +# is refreshed with createrepo prior to running rpmbuild +# (*) package-EXCLUDE-DEVEL-RPMS +# a set of *rpms* that the build will rpm-uninstall before building +# this is intended to denote stock rpms, and the build will attempt to yum-install them +# back after the package is rebuilt +# (*) package-RPMFLAGS: Miscellaneous RPM flags +# (*) package-RPMBUILD: If not rpmbuild - mostly used for sudo'ing rpmbuild +# (*) package-BUILD-FROM-SRPM: set this to any non-empty value, if your package is able to produce +# a source rpms by running 'make srpm' +# (*) package-RPMDATE: set this to any non-empty value to get the rpm package's release field hold the current date +# this is useful for container packages, like e.g. bootstrapfs or vserver, that contains much more than the +# correspondng module +# +#################### modules +# Required information about the various modules (set this in e.g. planetlab-tags.mk) +# +# (*) module-SVNPATH +# the complete path where this module lies; +# you can specify the trunk or a given tag with this variable +# +# OR if the module is managed under cvs (will be obsoleted) +# +# (*) module-CVSROOT +# (*) module-TAG +# +#################### automatic variables +# +# the build defines some make variables that are extracted from spec files +# see for example +# (*) $ make ulogd-pkginfo +# to see the list f variables attached to a given package +# (*) $ make kernel-devel-rpminfo +# to see the list of variables attached to a given rpm +# +#################### + +# exported to spec files as plrelease +PLANETLAB_RELEASE = 4.3 + +# +# Default values +# +# minimal compat with macos, just so this does not complain +HOSTARCH := $(shell uname -i 2> /dev/null || uname -m 2> /dev/null) +DISTRO := $(shell ./getdistro.sh) +RELEASE := $(shell ./getrelease.sh) +DISTRONAME := $(shell ./getdistroname.sh) +RPM-INSTALL-DEVEL := rpm --force -Uvh +# uninstall -- cannot force rpm -e +# need to ignore result, kernel-headers cannot be uninstalled as glibc depends on it +RPM-UNINSTALL-DEVEL := rpm -e +YUM-INSTALL-DEVEL := yum -y install + +# see also below +REMOTE-PLDISTROS="wextoolbox" + +#################### Makefile +# Default target +all: +.PHONY:all + +### default values +PLDISTRO := planetlab +RPMBUILD := rpmbuild +export CVS_RSH := ssh + +########## savedpldistro.mk holds PLDISTRO - it is generated at stage1 (see below) +ifeq "$(stage1)" "" +include savedpldistro.mk +endif + +# when re-running the nightly build after failure, we need to gather the former values +# do this by running make stage1=skip +PLDISTRO +ifeq "$(stage1)" "skip" +include savedpldistro.mk +endif + +#################### include onelab.mk +# describes the set of components +PLDISTROCONTENTS := $(PLDISTRO).mk +include $(PLDISTROCONTENTS) + +#################### include -tags.mk +# describes where to fetch components, and the related tags if using cvs +PLDISTROTAGS := $(PLDISTRO)-tags.mk +include $(PLDISTROTAGS) + +# this used to be set in the -tags.mk files, but that turned out to require +# error-prone duplicate changes +# so now the nightly build script sets this to what it is currently using +# we set a default in case we run the build manually: +# if the local directory was svn checked out, then use the corresponding URL +svn-info-url-line := $(shell svn info 2> /dev/null | grep URL:) +default-build-SVNPATH := $(lastword $(svn-info-url-line)) +# otherwise, use this hard-coded default +ifeq "$(default-build-SVNPATH)" "" +default-build-SVNPATH := http://svn.planet-lab.org/svn/build/trunk +endif +# use default if necessary +build-SVNPATH ?= $(default-build-SVNPATH) + +#################### +define remote_pldistro +$(1).mk: config.$(1)/$(1).mk + @echo 'creating $(1) from config subdir' + cp config.$(1)/$(1).mk $(1).mk + +$(2).mk: config.$(1)/$(2).mk + @echo 'creating $(1) tags from config subdir' + cp config.$(1)/$(2).mk $(2).mk + +config.$(1)/$(1).mk: config.$(1) +config.$(1)/$(2).mk: config.$(1) + +config.$(1): config.$(1).svnpath + @echo "Fetching details for pldistro $(1)" + svn export $(shell grep -v "^#" config.$(1).svnpath) config.$(1) + +DISTCLEANS += $(1).mk $(2).mk config.$(1) + +endef + +# somehow this does not work, handle manually instead +#$(foreach distro, $(REMOTE-PLDISTROS), $(eval $(call remote_pldistro,$(distro),$(distro)-tags))) +$(eval $(call remote_pldistro,wextoolbox,wextoolbox-tags)) + +########## stage1 and stage1iter +# extract specs and compute .mk files by running +# make stage1=true +# entering stage1, we compute all the spec files +# then we use stage1iter to compute the .mk iteratively, +# ensuring that the n-1 first makefiles are loaded when creating the n-th one +# when stage1iter is set, it is supposed to be an index (starting at 1) in $(ALL) + +ALLMKS := $(foreach package, $(ALL), MAKE/$(package).mk) + +### stage1iter : need some arithmetic, see +# http://www.cmcrossroads.com/articles/ask-mr.-make/learning-gnu-make-functions-with-arithmetic.html +ifneq "$(stage1iter)" "" +# the first n packages +packages := $(wordlist 1,$(words $(stage1iter)),$(ALL)) +# the n-th package +package := $(word $(words $(packages)),$(packages)) +# the n-1 first packages +stage1iter_1 := $(wordlist 2,$(words $(stage1iter)),$(stage1iter)) +previous := $(wordlist 1,$(words $(stage1iter_1)),$(ALL)) +previousmks := $(foreach package,$(previous),MAKE/$(package).mk) +include $(previousmks) +all: verbose +verbose: + @echo "========== stage1iter : $(package)" +# @echo "stage1iter : included .mk files : $(previousmks)" +all: $($(package).specpath) +all: MAKE/$(package).mk +else +### stage1 +ifneq "$(stage1)" "" +all : verbose +verbose : + @echo "========== stage1" +all : spec2make +all : .rpmmacros +# specs and makes are done sequentially by stage1iter +all : stage1iter +stage1iter: + arg=""; for n in $(ALL) ; do arg="$$arg x"; $(MAKE) --no-print-directory stage1iter="$$arg"; done +### regular make +else +### once .mks are OK, you can run make normally +include $(ALLMKS) +#all : tarballs +#all : sources +#all : codebases +#all : rpms +#all : srpms +# mention $(ALL) here rather than rpms +# this is because the inter-package dependencies are expressed like +# util-vserver: util-python +all: rpms +all: repo +endif +endif + +### yumgroups.xml : compute from all known .pkgs files +RPMS/yumgroups.xml: + mkdir -p RPMS + ./yumgroups.sh $(PLDISTRO) > $@ + +createrepo = createrepo --quiet -g yumgroups.xml RPMS/ + +repo: RPMS/yumgroups.xml + $(createrepo) + +.PHONY: repo + +#################### +# notes: +# * to make configuration easier, we always use the first module's +# definitions (CVSROOT,TAG, or SVNPATH) to extract the spec file +# * for the same reason, in case cvs is used, the first module name is added to +# $(package)-SPEC - otherwise the cvs modules have to define spec as +# /.spec while svn modules just define it as .spec +# +define stage1_variables +$(1).spec = $(notdir $($(1)-SPEC)) +$(1).specpath = SPECS/$(1).spec +$(1).module = $(firstword $($(1)-MODULES)) +$(1)-SVNPATH = $(strip $($(1)-SVNPATH)) +endef + +$(foreach package, $(ALL), $(eval $(call stage1_variables,$(package)))) + +# +# for each package, compute whether we need to set date +# the heuristic is that we mention the date as part of the rpm release flag if +# (*) the package has requested it by setting package-RPMDATE (container packages should do that) +# (*) or SVNPATH contains 'trunk' or 'branches' +# +define package_hasdate +$(1).has-date = $(if $($(1)-RPMDATE),yes, \ + $(if $($($(1).module)-SVNPATH), \ + $(if $(findstring /trunk,$($($(1).module)-SVNPATH)),yes, \ + $(if $(findstring /branches,$($($(1).module)-SVNPATH)),yes,)), \ + $(if $(findstring HEAD,$($($(1).module)-TAG)),yes,))) +endef + +$(foreach package, $(ALL), $(eval $(call package_hasdate,$(package)))) + +### the common header for generated specfiles +# useful when trying new specfiles manually +header.spec: + (echo -n "# Generated by planetlab build from $($(1)-SPEC) on " ; date) > $@ + echo "%define distro $(DISTRO)" >> $@ + echo "%define distrorelease $(RELEASE)" >> $@ + echo "%define distroname $(DISTRONAME)" >> $@ + echo "%define pldistro $(PLDISTRO)" >> $@ + echo "%define plrelease $(PLANETLAB_RELEASE)" >> $@ + +### extract spec file from scm +define target_spec +$($(1).specpath): header.spec + mkdir -p SPECS + cat header.spec > $($(1).specpath) + $(if $($(1).has-date),echo "%define date $(shell date +%Y.%m.%d)" >> $($(1).specpath),) + $(if $($(1)-SPECVARS), \ + $(foreach line,$($(1)-SPECVARS), \ + echo "%define" $(word 1,$(subst =, ,$(line))) "$(word 2,$(subst =, ,$(line)))" >> $($(1).specpath) ;)) + echo "# included from $($(1)-SPEC)" >> $($(1).specpath) + $(if $($($(1).module)-SVNPATH),\ + svn cat $($($(1).module)-SVNPATH)/$($(1)-SPEC) >> $($(1).specpath) || rm $($(1).specpath),\ + cvs -d $($($(1).module)-CVSROOT) checkout \ + -r $($($(1).module)-TAG) \ + -p $($(1).module)/$($(1)-SPEC) >> $($(1).specpath)) +endef + +$(foreach package,$(ALL),$(eval $(call target_spec,$(package)))) + +### +# Base rpmbuild in the current directory +# issues on fedora 8 : see the following posts +# http://forums.fedoraforum.org/showthread.php?t=39625 - and more specifically post#6 +# https://www.redhat.com/archives/fedora-devel-list/2007-November/msg00171.html +REALROOT=/build +FAKEROOT=/longbuildroot +PWD=$(shell /bin/pwd) +ifeq "$(PWD)" "$(REALROOT)" +export HOME := $(FAKEROOT) +else +export HOME := $(PWD) +endif +.rpmmacros: +ifeq "$(shell pwd)" "/build" + rm -f $(FAKEROOT) ; ln -s $(REALROOT) $(FAKEROOT) +endif + rm -f $@ + echo "%_topdir $(HOME)" >> $@ + echo "%_tmppath $(HOME)/tmp" >> $@ + echo "%__spec_install_pre %{___build_pre}" >> $@ + ./getrpmmacros.sh >> $@ + +### this utility allows to extract various info from a spec file +### and to define them in makefiles +spec2make: spec2make.c + $(CC) -g -Wall $< -o $@ -lrpm -lrpmbuild + +### run spec2make on the spec file and include the result +# usage: spec2make package +define target_mk +MAKE/$(1).mk: $($(1).specpath) spec2make .rpmmacros + mkdir -p MAKE + ./spec2make $($(1)-RPMFLAGS) $($(1).specpath) $(1) > MAKE/$(1).mk || { rm MAKE/$(1).mk; exit 1; } +endef + +$(foreach package,$(ALL),$(eval $(call target_mk,$(package)))) + +# stores PLDISTRO in a file +# this is done at stage1. later run wont get confused +savedpldistro.mk: + echo "PLDISTRO:=$(PLDISTRO)" > $@ + echo "PLDISTROTAGS:=$(PLDISTROTAGS)" >> $@ + echo "build-SVNPATH:=$(build-SVNPATH)" >> $@ + echo "PERSONALITY:=$(PERSONALITY)" >> $@ + echo "MAILTO:=$(MAILTO)" >> $@ + echo "BASE:=$(BASE)" >> $@ + echo "WEBPATH:=$(WEBPATH)" >> $@ + echo "TESTBUILDURL:=$(TESTBUILDURL)" >> $@ + echo "WEBROOT:=$(WEBROOT)" >> $@ + +savedpldistro: savedpldistro.mk +.PHONY: savedpldistro + +# always refresh this +all: savedpldistro + +#################### regular make + +define stage2_variables +### devel dependencies +$(1).rpmbuild = $(if $($(1)-RPMBUILD),$($(1)-RPMBUILD),$(RPMBUILD)) $($(1)-RPMFLAGS) +$(1).all-devel-rpm-paths := $(foreach rpm,$($(1)-DEPEND-DEVEL-RPMS),$($(rpm).rpm-path)) +$(1).depend-devel-packages := $(sort $(foreach rpm,$($(1)-DEPEND-DEVEL-RPMS),$($(rpm).package))) +ALL-DEVEL-RPMS += $($(1)-DEPEND-DEVEL-RPMS) +endef + +$(foreach package,$(ALL),$(eval $(call stage2_variables,$(package)))) +ALL-DEVEL-RPMS := $(sort $(ALL-DEVEL-RPMS)) + + +### pack sources into tarballs +ALLTARBALLS:= $(foreach package, $(ALL), $($(package).tarballs)) +tarballs: $(ALLTARBALLS) + @echo $(words $(ALLTARBALLS)) source tarballs OK +.PHONY: tarballs + +SOURCES/%.tar.bz2: SOURCES/% + tar chpjf $@ -C SOURCES $* + +SOURCES/%.tar.gz: SOURCES/% + tar chpzf $@ -C SOURCES $* + +SOURCES/%.tgz: SOURCES/% + tar chpzf $@ -C SOURCES $* + +## +URLS/%: url=$(subst @colon@,:,$(subst @slash@,/,$(notdir $@))) +URLS/%: basename=$(notdir $(url)) +URLS/%: + echo curl $(url) -o SOURCES/$(basename) + touch $@ + +### the directory SOURCES/- is made +# with a copy -rl from CODEBASES/ +# the former is $(package.source) and the latter is $(package.codebase) +ALLSOURCES:=$(foreach package, $(ALL), $($(package).source)) +# so that make does not use the rule below directly for creating the tarball files +.SECONDARY: $(ALLSOURCES) + +sources: $(ALLSOURCES) + @echo $(words $(ALLSOURCES)) versioned source trees OK +.PHONY: sources + +define target_link_codebase_sources +$($(1).source): $($(1).codebase) ; mkdir -p SOURCES ; cp -rl $($(1).codebase) $($(1).source) +endef + +$(foreach package,$(ALL),$(eval $(call target_link_codebase_sources,$(package)))) + +### codebase extraction +ALLCODEBASES:=$(foreach package, $(ALL), $($(package).codebase)) +# so that make does not use the rule below directly for creating the tarball files +.SECONDARY: $(ALLCODEBASES) + +codebases : $(ALLCODEBASES) + @echo $(words $(ALLCODEBASES)) codebase OK +.PHONY: codebases + +### extract codebase +# usage: extract_single_module package +define extract_single_module + mkdir -p CODEBASES + $(if $($($(1).module)-SVNPATH), cd CODEBASES && svn export $($($(1).module)-SVNPATH) $(1), cd CODEBASES && cvs -d $($($(1).module)-CVSROOT) export -r $($($(1).module)-TAG) -d $(1) $($(1).module)) +endef + +# usage: extract_multi_module package +define extract_multi_module + mkdir -p CODEBASES/$(1) && cd CODEBASES/$(1) && (\ + $(foreach m,$($(1)-MODULES), $(if $($(m)-SVNPATH), svn export $($(m)-SVNPATH) $(m);, cvs -d $($(m)-CVSROOT) export -r $($(m)-TAG) $(m);))) +endef + +CODEBASES/%: package=$(notdir $@) +CODEBASES/%: multi_module=$(word 2,$($(package)-MODULES)) +CODEBASES/%: + @(echo -n "XXXXXXXXXXXXXXX -- BEG CODEBASE $(package) : $@ " ; date) + $(if $(multi_module),\ + $(call extract_multi_module,$(package)),\ + $(call extract_single_module,$(package))) + @(echo -n "XXXXXXXXXXXXXXX -- END CODEBASE $(package) : $@ " ; date) + +### source rpms +ALLSRPMS:=$(foreach package,$(ALL),$($(package).srpm)) +srpms: $(ALLSRPMS) + @echo $(words $(ALLSRPMS)) source rpms OK +.PHONY: srpms + +### these macro handles the DEPEND-DEVEL-RPMS and EXCLUDE-DEVEL-RPMS tags for a hiven package +# before building : rpm-install DEPEND-DEVEL-RPMS and rpm-uninstall EXCLUDE +define handle_devel_rpms_pre + $(if $($(1).all-devel-rpm-paths), echo "Installing for $(1)-DEPEND-DEVEL-RPMS" ; $(RPM-INSTALL-DEVEL) $($(1).all-devel-rpm-paths)) + $(if $($(1)-EXCLUDE-DEVEL-RPMS), echo "Uninstalling for $(1)-EXCLUDE-DEVEL-RPMS" ; $(RPM-UNINSTALL-DEVEL) $($(1)-EXCLUDE-DEVEL-RPMS)) +endef + +define handle_devel_rpms_post + -$(if $($(1)-DEPEND-DEVEL-RPMS), echo "Unstalling for $(1)-DEPEND-DEVEL-RPMS" ; $(RPM-UNINSTALL-DEVEL) $($(1)-DEPEND-DEVEL-RPMS)) + $(if $($(1)-EXCLUDE-DEVEL-RPMS), "Reinstalling for $(1)-EXCLUDE-DEVEL-RPMS" ; $(YUM-INSTALL-DEVEL) $($(1)-EXCLUDE-DEVEL-RPMS) ) +endef + +# usage: target_source_rpm package +define target_source_rpm +ifeq "$($(1)-BUILD-FROM-SRPM)" "" +$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).tarballs) + mkdir -p BUILD SRPMS tmp + @(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using SOURCES) " ; date) + $(call handle_devel_rpms_pre,$(1)) + $($(1).rpmbuild) -bs $($(1).specpath) + $(call handle_devel_rpms_post,$(1)) + @(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date) +else +$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).codebase) + mkdir -p BUILD SRPMS tmp + @(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using make srpm) " ; date) + $(call handle_devel_rpms_pre,$(1)) + make -C $($(1).codebase) srpm SPECFILE=$(HOME)/$($(1).specpath) && \ + rm -f SRPMS/$(notdir $($(1).srpm)) && \ + ln $($(1).codebase)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm)) + $(call handle_devel_rpms_post,$(1)) + @(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date) +endif +endef + +$(foreach package,$(ALL),$(eval $(call target_source_rpm,$(package)))) + +### binary rpms are made from source rpm +ALLRPMS:=$(foreach package,$(ALL),$($(package).rpms)) +# same as above, mention $(ALL) and not $(ALLRPMS) +rpms: $(ALLRPMS) + @echo $(words $(ALLRPMS)) binary rpms OK +.PHONY: rpms + +# use tmp dirs when building binary rpm so make remains idempotent +# otherwise SOURCES/ or SPEC gets touched again - which leads to rebuilding +RPM-USE-TMP-DIRS = --define "_sourcedir $(HOME)/tmp" --define "_specdir $(HOME)/tmp" +RPM-USE-COMPILE-DIRS = --define "_sourcedir $(HOME)/COMPILE" --define "_specdir $(HOME)/COMPILE" + +# usage: build_binary_rpm package +# xxx hacky - invoke createrepo if DEPEND-FILES mentions RPMS/yumgroups.xml +define target_binary_rpm +$($(1).rpms): $($(1).srpm) + mkdir -p RPMS tmp + @(echo -n "XXXXXXXXXXXXXXX -- BEG RPM $(1) " ; date) + $(if $(findstring RPMS/yumgroups.xml,$($(1)-DEPEND-FILES)), $(createrepo) , ) + $(call handle_devel_rpms_pre,$(1)) + $($(1).rpmbuild) --rebuild $(RPM-USE-TMP-DIRS) $($(1).srpm) + $(call handle_devel_rpms_post,$(1)) + @(echo -n "XXXXXXXXXXXXXXX -- END RPM $(1) " ; date) +# for manual use only - in case we need to investigate the results of an rpmbuild +$(1)-compile: $($(1).srpm) + mkdir -p COMPILE tmp + @(echo -n "XXXXXXXXXXXXXXX -- BEG compile $(1) " ; date) + $(if $(findstring RPMS/yumgroups.xml,$($(1)-DEPEND-FILES)), $(createrepo) , ) + $(call handle_devel_rpms_pre,$(1)) + $($(1).rpmbuild) --recompile $(RPM-USE-TMP-DIRS) $($(1).srpm) + $(call handle_devel_rpms_post,$(1)) + @(echo -n "XXXXXXXXXXXXXXX -- END compile $(1) " ; date) +.PHONY: $(1)-compile +endef + +$(foreach package,$(ALL),$(eval $(call target_binary_rpm,$(package)))) +### shorthand target +# e.g. make proper -> does propers rpms +# usage shorthand_target package +define target_shorthand +$(1): $($(1).rpms) +.PHONY: $(1) +$(1)-spec: $($(1)-SPEC) +.PHONY: $(1)-spec +$(1)-mk: $($(1)-MK) +.PHONY: $(1)-mk +$(1)-tarball: $($(1).tarballs) +.PHONY: $(1)-tarball +$(1)-codebase: $($(1).codebase) +.PHONY: $(1)-source +$(1)-source: $($(1).source) +.PHONY: $(1)-codebase +$(1)-rpms: $($(1).rpms) +.PHONY: $(1)-rpms +$(1)-srpm: $($(1).srpm) +.PHONY: $(1)-srpm +endef + +$(foreach package,$(ALL),$(eval $(call target_shorthand,$(package)))) + +### file dependencies +define package_depends_on_file +$(1):$(2) +$($(1).srpm):$(2) +endef + +define target_dependfiles +$(foreach file,$($(1)-DEPEND-FILES),$(eval $(call package_depends_on_file,$(1),$(file)))) +endef + +$(foreach package,$(ALL),$(eval $(call target_dependfiles,$(package)))) + +### package dependencies +define package_depends_on_package +$(1):$(2) +$(1):$($(2).rpms) +$($(1).srpm):$($(2).rpms) +endef + +define target_depends +$(foreach package,$($(1)-DEPEND-PACKAGES) $($(1).depend-devel-packages),$(eval $(call package_depends_on_package,$(1),$(package)))) +endef + +$(foreach package,$(ALL),$(eval $(call target_depends,$(package)))) + +### clean target +# usage: target_clean package +define target_clean +$(1)-clean-codebase: + rm -rf $($(1).codebase) +.PHONY: $(1)-clean-codebase +CLEANS += $(1)-clean-codebase +$(1)-clean-source: + rm -rf $($(1).source) +.PHONY: $(1)-clean-source +CLEANS += $(1)-clean-source +$(1)-clean-tarball: + rm -rf $($(1).tarballs) +.PHONY: $(1)-clean-tarball +CLEANS += $(1)-clean-tarball +$(1)-clean-build: + rm -rf BUILD/$(notdir $($(1).source)) +CLEANS += $(1)-clean-build +$(1)-clean-rpms: + rm -rf $($(1).rpms) +.PHONY: $(1)-clean-rpms +CLEANS += $(1)-clean-rpms +$(1)-clean-srpm: + rm -rf $($(1).srpm) +.PHONY: $(1)-clean-srpm +CLEANS += $(1)-clean-srpm +$(1)-codeclean: $(1)-clean-source $(1)-clean-tarball $(1)-clean-build $(1)-clean-rpms $(1)-clean-srpm +$(1)-clean: $(1)-clean-codebase $(1)-codeclean +.PHONY: $(1)-codeclean $(1)-clean +$(1)-clean-spec: + rm -rf $($(1).specpath) +.PHONY: $(1)-clean-spec +$(1)-clean-make: + rm -rf MAKE/$(1).mk +.PHONY: $(1)-clean-make +$(1)-distclean: $(1)-distclean1 $(1)-distclean2 +$(1)-distclean1: $(1)-clean-spec $(1)-clean-make +$(1)-distclean2: $(1)-clean +.PHONY: $(1)-distclean $(1)-distclean1 $(1)-distclean2 +endef + +$(foreach package,$(ALL),$(eval $(call target_clean,$(package)))) + +### clean precisely +clean: + $(MAKE) $(CLEANS) +.PHONY: clean + +clean-help: + @echo Available clean targets + @echo $(CLEANS) + +### brute force clean +distclean1: + rm -rf savedpldistro.mk .rpmmacros spec2make header.spec SPECS MAKE $(DISTCLEANS) +distclean2: + rm -rf CODEBASES SOURCES BUILD RPMS SRPMS tmp +distclean: distclean1 distclean2 +.PHONY: distclean1 distclean2 distclean + +develclean: + -$(RPM-UNINSTALL-DEVEL) $(ALL-DEVEL-RPMS) + +#################### +# gather build information for the 'About' page +# when run from crontab, INIT_CWD not properly set (says /root ..) +# so, the nightly build passes BASE here +# also store BASE in .base for any post-processing purposes +myplc-release: + @echo 'Creating myplc-release' + rm -f $@ + echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx build info" >> $@ + $(MAKE) --no-print-directory version-build >> $@ + echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx svn info" >> $@ + $(MAKE) --no-print-directory version-svns >> $@ + echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx rpm info" >> $@ + $(MAKE) --no-print-directory version-rpms >> $@ + @echo $(BASE) > .base + +version-build: + @echo -n 'Build build-date: ' ; date '+%Y.%m.%d' + @echo -n 'Build build-time: ' ; date '+%H:%M-%Z' + @echo -n 'Build build-hostname: ' ; hostname + @echo "Build build-base: $(BASE)" + @echo "Build planetlab-distro: $(PLDISTRO)" + @echo "Build planetlab-tags: $(PLDISTROTAGS)" + @echo -n 'Build planetlab-tagsid: ' ; fgrep '$$''Id' $(PLDISTROTAGS) + @echo "Build target-arch: $(HOSTARCH)" + @echo "Build target-distro: $(DISTRO)" + @echo "Build target-distroname: $(DISTRONAME)" + @echo "Build target-release: $(RELEASE)" + @echo "Build target-personality: $(PERSONALITY)" + +#################### +# for a given module +VFORMAT="%30s := %s\n" +define svn_version_target +$(1)-version-svn: + @$(if $($(1)-SVNPATH),\ + printf $(VFORMAT) $(1)-SVNPATH "$($(1)-SVNPATH)",\ + printf $(VFORMAT) $(1)-CVSROOT "$($(1)-CVSROOT)" ; printf $(VFORMAT) $(1)-TAG "$($(1)-TAG)") +endef + +# compute all modules +ALL-MODULES := +$(foreach package,$(ALL), $(eval ALL-MODULES+=$($(package)-MODULES))) +ALL-MODULES:=$(sort $(ALL-MODULES)) + +$(foreach module,$(ALL-MODULES), $(eval $(call svn_version_target,$(module)))) + +version-svns: $(foreach module, $(ALL-MODULES), $(module)-version-svn) + +RFORMAT="%20s :: version=%s release=%s\n" +define rpm_version_target +$(1)-version-rpm: + @printf $(RFORMAT) $($(1).rpm-name) $($(1).rpm-version) $($(1).rpm-release) +version-rpms: $(1)-version-rpm +endef + +$(foreach package,$(sort $(ALL)), $(eval $(call rpm_version_target,$(package)))) + +versions: myplc-release version-build version-svns version-rpms +.PHONY: versions version-build version-rpms version-svns + +#################### package info +PKGKEYS := tarballs source codebase srpm rpms rpmnames rpm-release rpm-name rpm-version rpm-subversion +%-pkginfo: package=$(subst -pkginfo,,$@) +%-pkginfo: + @$(foreach key,$(PKGKEYS),echo "$(package).$(key)=$($(package).$(key))";) +## rpm info +RPMKEYS := rpm-path package +%-rpminfo: rpm=$(subst -rpminfo,,$@) +%-rpminfo: + @$(foreach key,$(RPMKEYS),echo "$(rpm).$(key)=$($(rpm).$(key))";) + +#################### various lists - designed to run with stage1=true +packages: + @$(foreach package,$(ALL), echo package=$(package) ref_module=$($(package).module) modules=$($(package)-MODULES) rpmnames=$($(package).rpmnames); ) + +modules: + @$(foreach module,$(ALL-MODULES), echo module=$(module) svnpath=$($(module)-SVNPATH); ) + +branches: + @$(foreach module,$(ALL-MODULES), \ + $(if $($(module)-SVNBRANCH),echo module=$(module) branch=$($(module)-SVNBRANCH);)) + +module-tools: + @$(foreach module,$(ALL-MODULES), \ + $(if $($(module)-SVNPATH), \ + $(if $($(module)-SVNBRANCH), \ + echo $(module):$($(module)-SVNBRANCH); , \ + echo $(module); ))) + +info: packages modules branches + +.PHONY: info packages modules branches module-tools + +#################### +tests_svnpath: + @$(if $(tests-SVNPATH), echo $(tests-SVNPATH) > $@, \ + echo "http://svn.planet-lab.org/svn/tests/trunk" > $@) + + +#################### +help: + @echo "********** Run make in two stages:" + @echo "" + @echo "make stage1=true PLDISTRO=onelab" + @echo " -> extracts all spec files in SPECS/ and mk files in MAKE/" + @echo " as well as save PLDISTRO for subsequent runs" + @echo "" + @echo "********** Then you can use the following targets" + @echo 'make' + @echo " rebuilds everything" + @echo 'make util-vserver' + @echo " makes the RPMS related to util-vserver" + @echo " equivalent to 'make util-vserver-rpms'" + @echo "" + @echo "********** Or, vertically - step-by-step for a given package" + @echo 'make util-vserver-codebase' + @echo " performs codebase extraction in CODEBASES/util-vserver" + @echo 'make util-vserver-source' + @echo " creates source link in SOURCES/util-vserver-" + @echo 'make util-vserver-tarball' + @echo " creates source tarball in SOURCES/util-vserver-." + @echo 'make util-vserver-srpm' + @echo " build source rpm in SRPMS/" + @echo 'make util-vserver-rpms' + @echo " build rpm(s) in RPMS/" + @echo "" + @echo "********** Or, horizontally, reach a step for all known packages" + @echo 'make codebases' + @echo 'make sources' + @echo 'make tarballs' + @echo 'make srpms' + @echo 'make rpms' + @echo "" + @echo "********** Manual targets" + @echo "make package-compile" + @echo " The regular process uses rpmbuild --rebuild, that performs" + @echo " a compilation directory cleanup upon completion. If you need to investigate" + @echo " the intermediate compilation directory, use the -compile targets" + @echo "********** Cleaning examples" + @echo "make clean" + @echo " removes the files made by make" + @echo "make distclean" + @echo " brute-force cleaning, removes entire directories - requires a new stage1" + @echo "make develclean" + @echo " rpm-uninstalls all devel packages installed during build" + @echo "" + @echo "make iptables-distclean" + @echo " deep clean for a given package" + @echo "make iptables-codeclean" + @echo " run this if you've made a change in the CODEBASES area for iptables" + @echo "" + @echo "make util-vserver-clean" + @echo " removes codebase, source, tarball, build, rpm and srpm for util-vserver" + @echo "make util-vserver-clean-codebase" + @echo " and so on for source, tarball, build, rpm and srpm" + @echo "" + @echo "********** Info examples" + @echo "make ++ALL" + @echo " Displays the value of a given variable (here ALL)" + @echo " with only a single plus sign only the value is displayed" + @echo "make info" + @echo " is equivalent to make packages modules branches" + @echo " provides various info on these objects" + @echo "make ulogd-pkginfo" + @echo " Displays know attributes of a package" + @echo "make kernel-devel-rpminfo" + @echo " Displays know attributes of an rpm" + @echo "make stage1=true PLDISTROTAGS=planetlab-tags-4.2.mk packages modules branches module-tools" + @echo " Lists mentioned items - module-tools is used in modules.update" + @echo "" + @echo "********** Known pakages are" + @echo "$(ALL)" + +#################### convenience, for debugging only +# make +foo : prints the value of $(foo) +# make ++foo : idem but verbose, i.e. foo=$(foo) +++%: varname=$(subst +,,$@) +++%: + @echo "$(varname)=$($(varname))" ++%: varname=$(subst +,,$@) ++%: + @echo "$($(varname))" diff --git a/trunk/README-build.txt b/trunk/README-build.txt new file mode 100644 index 00000000..c738f35c --- /dev/null +++ b/trunk/README-build.txt @@ -0,0 +1,49 @@ +Notes on using this build + +The PlanetLab builds has the concept of a 'distribution' that allows +advanced user to customize the contents of the build. If you're not +an advanced user, please just use the regular 'planetlab' +distribution. + +If you want to take advantage of a custom distribution, see +README-pldistros.txt for further details. Please feel free to contact +devel@planet-lab.org, as all this is not precisely foolproof :) + +===== +* install a vserver-enabled kernel and util-vserver on your build box +* create a local fedora mirror - see vbuild-fedora-mirror.sh +* create a vserver - see vbuild-init-vserver.sh +* enter the vserver and svn export this build module into /build +* cd /build and run +# make stage1=true PLDISTRO= +# make help +# make + +=== automated builds +* the nightly build script wraps all this in a rather complicated way +it is not intended to be used as is without a few manual tweaks on your +build and test hosts +basically it +** recreates a fresh build vserver named after the arguments +** enters this to perform the actual build +** pushes the results to a web location so the tests can pull the +right myplc rpm +** invokes a runtest on a separate machine +** manages logs - that's the trickiest part, so the root context and +the vserver add their logs on the same file + +=== manual builds +* always run the stage1 make prior to anything else +* if you clean or patch a spec file you need to run the stage1 again +* see make help for how to do things incrementally + +* typically if you want to test a change without commiting first, you +need to: +** make package-codebase +** patch CODEBASE/module/ +** make package + +if further changes are needed, then do +** make package-clean-tarball package-clean-build package-clean-rpm +so the codebase remains intact, patch it again and run +** make package diff --git a/trunk/README-pkgsfiles.txt b/trunk/README-pkgsfiles.txt new file mode 100644 index 00000000..0ff20f2d --- /dev/null +++ b/trunk/README-pkgsfiles.txt @@ -0,0 +1,14 @@ +#lst files syntax: +# comments start with # + +# to define packages on all fcdistros +package: p1 p2 p3 p4 +# to add packages for say f9 +package+f9: p5 p6 +# to exclude packages +package-f9: p2 p3 + +# same for groups, except that you need to replace any white-space in +# the groupname with +++, like in +group: X+++Window+++System +group: GNOME+++Desktop+++Environment diff --git a/trunk/README-pldistros.txt b/trunk/README-pldistros.txt new file mode 100644 index 00000000..cdb3fee9 --- /dev/null +++ b/trunk/README-pldistros.txt @@ -0,0 +1,52 @@ +We've tried to isolate the distro-dependent configurations from the code + +Most of the .pgks files are optional to define a new distro: +missing files are searched in the planetlab distro + +========== build environment +./build/.mk + that defines the contents of the build -- see Makefile +./build/-tags.mk + that defines the svn locations of the various modules +./build/-install.mk + optional make file to define the install target + +========== kernel config +./Linux-2.6/configs/kernel-2.6.--.config + (subject to change location in the future) + +========== various system images +./build/config./devel.pkgs + set of packages required for building +./build/config./bootcd.pkgs + contents of the bootcd image +./build/config./bootstrapfs.pkgs + the standard contents of the node root + used to generate yumgroups.xml +./build/config./bootstrapfs.pkgs + specification for the contents of the core for the node + software + this results in a tarball (tar.bz2) +./build/config./bootstrapfs-*.pkgs + each of these files results in an extension tarball +./build/config./myplc.pkgs + contents of the myplc image +./build/config./vserver.pkgs + the contents of the standard vserver reference image +./build/config./vserver-*.pkgs + all *.pkgs files here - produce additional vserver images +./build/config./vtest.pkgs + used to create test vservers for myplc-native + +=== extensions +as of this writing extensions are managed as follows: +- at node installation, the tarball produced from bootstrapfs.pkgs is +downloaded and untared to produce the core node setup +- then we attempt to install an extension corresponding to each of the +nodegroups that the node is part of +- the first method is to try and download a tarball named after the +extension. such tarballs are produced by the build from a .pkgs file, +see above +- if that fails, then the extension install is attempted through a + yum groupinstall extension + diff --git a/trunk/README-specfiles.txt b/trunk/README-specfiles.txt new file mode 100644 index 00000000..68bae56a --- /dev/null +++ b/trunk/README-specfiles.txt @@ -0,0 +1,54 @@ +Copyright (c) 2003 The Trustees of Princeton University (Trustees). + +$Id$ + +Here are some general guidelines for writing spec files. + +* RPM does not allow you to use dashes in version or release + numbers. Use dots, or nothing. + +* Most versions of RPM poorly handle version and release numbers that + do not begin with a number. Start your version and release number with + at least one number. Append as many minor numbers as you want, but + leave alphabetic characters at the end of the string. + +* Always define BuildRoot based in %{_tmppath}. + +* In the %install step, always install files based in $RPM_BUILD_ROOT. + +* Don't be overly restrictive with Requires or BuildRequires tags. RPM + is already pretty smart about analyzing your package for necessary + dependencies. Usually, you only need to specify Requires tags to make + sure a set of packages get installed in the right order (if there is + one). Otherwise, it's likely your users will end up in awful + situations requiring manual bootstrap. + +Here are some specific guidelines for writing spec files for this setup. + +* Write your spec file as you would normally. It should be an + operational spec file on its own. + +* Explicitly %define %{name}, %{version}, %{release}, or don't use + them. You should do this anyway to support older versions of RPM. This + Makefile does not implicitly define these variables (based on the + Name:, Version:, and Release: tags) as RPM does. + +* Run cvsps(1) manually on your repository, and synchronize the PatchSet + numbers with any Patch: tags in your spec file that you would like + explicitly named. + +* This Makefile assumes that the build directory (as specified by the -n + option to %setup, or the RPM default %{name}-%{version}) is the + basename of the Source file (i.e. without .tar[.gz|.bz2]). + +* Add %{?date:.%{date}} to your %{release} tag. When executed, this + Makefile will define the variable to be the current date in YYYY.MM.DD + form. This will help automated nightly upgrades. Tagged releases will + not define this variable. + +* Unless you have a very good reason not to, use the following tags: + +Vendor: PlanetLab +Packager: PlanetLab Central +Distribution: PlanetLab 2.0 +URL: http://www.planet-lab.org diff --git a/trunk/build-conf-planetlab.py b/trunk/build-conf-planetlab.py new file mode 100755 index 00000000..f24f325d --- /dev/null +++ b/trunk/build-conf-planetlab.py @@ -0,0 +1,84 @@ +#!/usr/bin/python + +### +# Nightly build spec HOWTO +# +# * To add a 'build spec', define a dictionary as in the following examples, filling in the values you would like to override in the defaults. Any values you leave out +# will get picked up from the defaults at the bottom of this fiel. +# +# * A build spec may define multiple builds encapsulating various combinations of the available parameter options. To do so, +# set a parameter to a list, and the parent script will automatically turn it into the combinations it encloses. e.g., the following +# build spec defines 6 separate builds: +# +# my_build = { +# 'fcdistro':['centos5','f8','f10'], +# 'personality':['linux32','linux65'] +# +# * If your parameters have dependencies - e.g. you only want to build the linu64 personality on f10, then define the parameter as a lambda operating +# on the current build spec. e.g. in this case, it would be to the effect of lambda (build): if (build['fcdistro']=='f10') then ['linux32','linux64'] else ['linux32'] +# + + +marcs_trunk_build = { + 'tags':'planetlab-tags.mk', + 'fcdistro':['centos5','f8'], + 'personality':['linux32','linux64'], + 'test': 0, + 'release':'k22', +} + +sapans_k27_build = { + 'tags':'k27-tags.mk', + 'fcdistro':['centos5','f8'], + 'personality':'linux32', + 'test':1, + 'release':'k27' +} + +### +# +# DEFAULTS +# +# Any values that you leave out from the above specs will get filled in by the defaults specified below. +# You shouldn't need to modify these values to add new builds + +__personality_to_arch__={'linux32':'i386','linux64':'x86_64'} +__flag_to_test__={0:'-B', 1:''} + +def __check_out_build_script__(build): + import os + tmpname = os.popen('mktemp /tmp/'+build['build-script']+'.XXXXXX').read().rstrip('\n') + os.system("svn cat %s/%s > %s 2>/dev/null"%(build['svnpath'],build['build-script'],tmpname)) + return tmpname + +def __today__(): + import datetime + return datetime.datetime.now().strftime("%Y-%m-%d") + +__default_build__ = { + +### Simple parameters + 'tags':'planetlabs-tags.mk', + 'fcdistro':'centos5', + 'personality':'linux32', + 'test':0, + 'release':'k22', + 'path':'/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin', + 'sh':'/bin/bash', + 'mailto':'build@lists.planet-lab.org', + 'build-script':'vbuild-nightly.sh', + 'webpath':'/vservers/build.planet-lab.org/var/www/html/install-rpms/archive', + 'pldistro':'planetlab', + 'date': __today__(), + 'svnpath':'http://svn.planet-lab.org/svn/build/trunk', + 'personality':'linux32', + 'myplcversion':'4.3', + + +### Parameters with dependencies: define paramater mappings as lambdas here + + 'arch':lambda build: __personality_to_arch__[build['personality']], + 'runtests':lambda build: __flag_to_test__[build['test']], + 'vbuildnightly':lambda build: __check_out_build_script__(build) + +} diff --git a/trunk/build.common b/trunk/build.common new file mode 100644 index 00000000..4cd00d40 --- /dev/null +++ b/trunk/build.common @@ -0,0 +1,740 @@ +# -*-Shell-script-*- +# +# Common functions for build scripts used by various packages +# incorporated (e.g., build, myplc, myplc-devel, vserver-reference) +# +# Marc E. Fiuczynski +# Copyright (C) 2007 The Trustees of Princeton University +# +# $Id$ +# +# supported distros f8, f10, (f11) -- fc[46] probably does not work anymore +# centos5.3 + +# returns 'Fedora' or 'CentOS' for now +function pl_getDistro() { + if [ -f "/etc/redhat-release" ] ; then + distro=$(awk ' { print $1 } ' /etc/redhat-release) + else + echo "build.common.pl_getDistro-unknown" + exit 1 + fi + echo "$distro" + return 0 +} + +# returns something like 8, 10, or 5.3 +function pl_getRelease() { + if [ -f "/etc/redhat-release" ] ; then + release=$(awk ' { if ($1=="Fedora" && $2=="Core") print $4 ; if (($1=="Fedora" && $2!="Core")||$1=="CentOS") print $3 } ' /etc/redhat-release) + else + echo "build.common.pl_getRelease-unknown" + exit 1 + fi + echo "$release" + return 0 +} + +# returns stuff for vserver, i.e. something like 'f8' or 'centos5' +function pl_getReleaseName () { + distro=$1; shift + release=$1; shift + case $distro in + [Ff]edora*) + releasename=f$release + ;; + [Cc]entOS*) + old_IFS="$IFS" + IFS="." + set -- $release + IFS="$old_IFS" + releasename=centos$1 + ;; + *) + releasename="unknown-name-for-${pl_DISTRO}-please-edit-build.common" + echo 1>&2 "build.common: WARNING - releasename not set for distro=$distro" + return 1 + ;; + esac + echo "$releasename" + return 0 +} + +# on fedora 8 or above, we use libnl from the stock repos +# on centos5 we build it locally +function pl_getKexcludes () { + distroname=$1; shift + case $distroname in + f*) + echo 'kernel kernel-vserver kernel-devel kernel-headers kernel-debuginfo util-vserver* iptables iptables-ipv6 iproute drupal' ;; + centos5) + echo 'kernel kernel-vserver kernel-devel kernel-headers kernel-debuginfo util-vserver* iptables iproute drupal inotify-tools* libnl' ;; + *) + echo "pl_DISTRO_NAME=$pl_DISTRO_NAME not supported in getexcludes.sh" ; exit 1 ;; + esac +} + +# figure out which redhat distro we are using (fedora, centos, redhat) +pl_DISTRO=$(pl_getDistro) + +# select basearch of the host devel environment - protected for macos for local tests +pl_DISTRO_ARCH=$(uname -i 2>/dev/null|| echo unknownarch) + +# the release number (plain number) +pl_DISTRO_RELEASE=$(pl_getRelease) + +# the release name - for vserver build ... +pl_DISTRO_NAME=$(pl_getReleaseName $pl_DISTRO $pl_DISTRO_RELEASE) + +# the packages to exclude +pl_KEXCLUDES=$(pl_getKexcludes $pl_DISTRO_NAME) + +# get path to appropriate yumgroups.xml file +# Thierry: quick & dirty improvement +# this file is updated by the toplevel build, from the .pkgs files +pl_DISTRO_YUMGROUPS="../../../RPMS/yumgroups.xml" + +function pl_process_fedora_options () { + # Get options + shiftcount=0 + while getopts "l:r:a:h" opt ; do + case $opt in + l) + pl_DISTRO_URL=$OPTARG + let shiftcount=$shiftcount+2 + ;; + r) + pl_DISTRO_RELEASE=$OPTARG + let shiftcount=$shiftcount+2 + ;; + a) + pl_DISTRO_ARCH=$OPTARG + let shiftcount=$shiftcount+2 + ;; + h|*) + echo "Usage: $0 [OPTION]..." + echo " -l url distro mirror location (default: $pl_DISTRO_URL)" + echo " -r release distro release number (default: $pl_DISTRO_RELEASE)" + echo " -a arch distro architecture (default: $pl_DISTRO_ARCH)" + echo "where distro can be either fedora, centos, or redhat" + echo " -h This message" + exit 1 + ;; + esac + done + return $shiftcount +} + +######################################## handling a root image +function pl_root_rpm_macros () { + cat <.mirrors" + echo " -v Be verbose" + echo " -h This message" + echo " target selection (defaults based on current build vserver)" + echo " -r release Fedora release number (default: $releasever)" + echo " -a arch Fedora architecture (default: $basearch)" + exit 1 +} + +function pl_root_mkfedora () { + + echo "* Entering pl_root_mkfedora" "$@" + + if [ $UID -ne 0 ] ; then + echo "Error: You must run this script as root." + exit 1 + fi + +# Verbosity + verbose=0 + +# Release and architecture to install : defaults to current vserver's settings or previously parsed fedora_options + releasever=$pl_DISTRO_RELEASE + basearch=$pl_DISTRO_ARCH + +# Get options + while getopts "vh" opt ; do + case $opt in + v) verbose=1; set -x ;; + h|*) mkfedora_usage ;; + esac + done + + shift $(($OPTIND - 1)) + [[ "$#" -lt 3 ]] && mkfedora_usage + vroot=$1 ; shift + pldistro=$1 ; shift + pkgsfile="$@" + vroot=$(cd $vroot && pwd -P) + [ -d $vroot ] || mkfedora_usage + + + # parse pkgsfile and add to local vars + fcdistro=${pl_DISTRO_NAME} + pkgs_packages=$(pl_parsePkgs package $fcdistro $pldistro $pkgsfile) + pkgs_groups=$(pl_parsePkgs group $fcdistro $pldistro $pkgsfile) + # packages to exclude + pkgs_excludes=$(pl_parsePkgs exclude $fcdistro $pldistro $pkgsfile) + # what can get trashed to save space + pkgs_junk=$(pl_parsePkgs junk $fcdistro $pldistro $pkgsfile) + # but not this + pkgs_precious=$(pl_parsePkgs precious $fcdistro $pldistro $pkgsfile) + # formerly related to mkfedora -k : packages to take from our own build + # and thus need be excluded frem the stock repos + pkgs_kexcludes=$(pl_parsePkgs kexclude $fcdistro $pldistro $pkgsfile | sed -e s,@KEXCLUDE@,"$pl_KEXCLUDES",) + # get mirrors if not specified with -l + if [ -z "$mirrors" ] ; then + mirrorsfile=$(pl_locateDistroFile ../build/ $pldistro "$pldistro.mirrors") + mirrors=$(pl_parsePkgs mirror $fcdistro $pldistro $mirrorsfile) + fi + + kexclude_line="" + # add them manually as the output of pl_parsePkgs is line-separated + if [ -n "$pkgs_kexcludes" ] ; then + kexclude_line="exclude=" + for kexclude in $pkgs_kexcludes ; do + kexclude_line="$kexclude_line $kexclude" + done + fi + + echo "$0: candidate mirrors" + for mirror in $mirrors ; do + echo "* candidate mirror $mirror" + done + + # the repo part of the final yum.conf + yum_conf_repos=$vroot/xxxmkfedora-repos.confxxx + if ! yumconf_mirrors $yum_conf_repos ../build/ $fcdistro "$kexclude_line" $mirrors ; then + echo xxx -- error ; return 1 + fi + + public_gpg_key=$(yumconf_gpgkey $yum_conf_repos) + + # Do not tolerate errors + set -e + + ## make rpms ignore installing stuff to special fs entries like /proc + # Because of https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=52725 + # you have to use at least one language beside 'C' + # Prevent all locales from being installed in reference image + mkdir -p $vroot/etc/rpm + pl_root_rpm_macros > $vroot/etc/rpm/macros + + # Trick rpm and yum, who read the real root /etc/rpm/macros file + # rather than the one installed in the reference image, despite what + # you might expect the --root and --installroot options to mean. Both + # programs always read $HOME/.rpmmacros. + export HOME=$vroot/tmp + mkdir -p $vroot/tmp + pl_root_rpm_macros > $vroot/tmp/.rpmmacros + + function mkfedora_cleanup () { + umount -l $vroot/proc + umount -l $vroot/dev/shm + umount -l $vroot/dev/pts + } + + # Clean up before exiting if anything goes wrong + trap "mkfedora_cleanup" ERR INT + + # Mount in reference image + mount -t devpts none $vroot/dev/pts + mount -t tmpfs none $vroot/dev/shm + mkdir -p $vroot/proc + mount -t proc none $vroot/proc + + # Create a /var/lib dirs for yum & rpm + mkdir -p $vroot/var/lib/yum + mkdir -p $vroot/var/lib/rpm + mkdir -p $vroot/usr/share/info + + # Create a dummy /etc/fstab in reference image + mkdir -p $vroot/etc + cat >$vroot/etc/fstab < $vroot/etc/sysconfig/network + + # Initialize RPM database in reference image + mkdir -p $vroot/var/lib/rpm + rpm --root $vroot --initdb + rpm --root $vroot --import $public_gpg_key + + # Initialize yum in reference image + mkdir -p $vroot/var/cache/yum $vroot/var/log + +# yum.conf is for building only - store in different location than /etc/yum.conf + yum_conf=$vroot/etc/mkfedora-yum.conf + cat > $yum_conf <> $yum_conf + + # If we are being built as part of an automated RPM build, solve the + # bootstrap problem by including any just built packages in the yum + # configuration. This cooperates with the PlanetLab build system. + if [ -n "$RPM_BUILD_DIR" ] ; then + RPM_RPMS_DIR=$(cd $(dirname $RPM_BUILD_DIR)/RPMS && pwd -P) + # If run under sudo, allow user to delete the headers/ and + # repodata/ directories. + if [ -n "$SUDO_USER" ] ; then + chown -R $SUDO_USER $RPM_RPMS_DIR + fi + cat >> $yum_conf < /dev/null && yum_options="$yum_options --verbose" + yum_options="$yum_options -y" + yum_options="$yum_options -c $yum_conf" + yum_options="$yum_options --installroot=$vroot" + + exclude_arg="" + for exclude in $pkgs_excludes; do + exclude_arg="$exclude_arg --exclude $exclude" + done + + # glibc must be specified explicitly for the correct arch to be + # chosen. + echo "* Installing glibc" + # ignore yum's return code that is basically undefined + yum $yum_options $exclude_arg install glibc || : + + # Go, baby, go + if [ -n "$pkgs_packages" ] ; then + echo "* Installing optional packages" $pkgs_packages + # ignore yum's return code that is basically undefined + yum $yum_options $exclude_arg install $pkgs_packages || : + if ! rpm --root $vroot -q $pkgs_packages >/dev/null ; then + echo "* Warning: Missing packages" + rpm --root $vroot -q $pkgs_packages | grep "not installed" + fi + fi + + if [ -n "$pkgs_groups" ] ; then + ## call yum sequentially to get finer-grained info on dependencies + for group_plus in $pkgs_groups ; do + group=$(echo $group_plus | sed -e "s,+++, ,g") + echo "* Installing optional group $group" + # ignore yum's return code that is basically undefined + yum $yum_options $exclude_arg groupinstall "$group" || : + done + fi + + # formerly in bootcd/prep.sh : to optimize footprint + if [ -n "$pkgs_junk" ] ; then + echo "* Removing unnecessary junk" + pushd $vroot + # Save precious files + [ -n "$pkgs_precious" ] && tar --ignore-failed-read -cpf precious.tar $pkgs_precious + # Remove unnecessary junk + [ -n "$pkgs_junk" ] && rm -rf $pkgs_junk + # Restore precious files + [ -n "$pkgs_precious" ] && tar -xpf precious.tar && rm -f precious.tar + popd + fi + + # Clean yum cache + echo "* Cleaning up" + # ignore yum's return code that is basically undefined + yum $yum_options clean all || : + + # Clean RPM state + rm -f $vroot/var/lib/rpm/__db* + + # Set time zone to UTC + if [ -f $vroot/usr/share/zoneinfo/UTC -a -f $vroot/etc/localtime ] ; then + rm -f $vroot/etc/localtime + ln -s /usr/share/zoneinfo/UTC $vroot/etc/localtime + fi + + echo "Dumping current list of rpms in /etc/mkfedora-rpms.txt" + chroot $vroot rpm -aq | sort > $vroot/etc/mkfedora-rpms.txt + + # remove trap handler, as we are about to call it directly. + trap - ERR INT + + # Clean up + mkfedora_cleanup + + return 0 +} + +function pl_root_tune_image () { + root=$1; shift + + # Disable all services in reference image + chroot $root sh -c "/sbin/chkconfig --list | awk '{ print \$1 }' | xargs -i /sbin/chkconfig {} off" + + # FC2 minilogd starts up during shutdown and makes unmounting + # impossible. Just get rid of it. + rm -f $root/sbin/minilogd + ln -nsf /bin/true $root/sbin/minilogd + + # This tells the Boot Manager that it is okay to update + # /etc/resolv.conf and /etc/hosts whenever the network configuration + # changes. Users are free to delete this file. + touch $root/etc/AUTO_UPDATE_NET_FILES +} + +# Move specified directories out of a src tree into a dst tree, and +# then for each moved directory create a symlink in src to dst. +function pl_move_dirs() { + root=$1 + data=$2 + store=$3 + shift 3 + + mkdir -p $root/data + for datadir in "$@" ; do + mkdir -p ${data}${datadir} + if [ -d ${root}/${datadir} -a ! -h ${root}/${datadir} ] ; then + (cd ${root} && find ./${datadir} | cpio -p -d -u ../${data}/) + fi + rm -rf ${root}/${datadir} + mkdir -p $(dirname ${root}/${datadir}) + ln -nsf ${store}/${datadir} ${root}/${datadir} + done +} + +# Construct an image file from given some directory +# XXX in the future maybe use livecdtools? +function pl_make_image() { + root=$1 + image=$2 + extraspace=$3 + + # Leave about 100 MB free space and allow for about 20% inode overhead + bytes=$((($(du -sb $root | cut -f1) + $extraspace) * 120 / 100)) + bs=4096 + blocks=$(($bytes / $bs)) + dd bs=$bs count=$blocks if=/dev/zero of=$image + mkfs.ext3 -b $bs -j -F $image + + # Temporarily mount it + tmp=$(mktemp -d tmp.XXXXXX) + mount -o loop $image $tmp + trap "umount $tmp; rmdir $tmp" ERR INT + + # Move files to it + (cd $root && tar cpf - .) | (cd $tmp && tar xpf -) + + # Unmount it + umount $tmp + rmdir $tmp + trap - ERR INT +} + +# Fix permissions on tmp directories +function pl_fixtmp_permissions() { + root=$1 + chmod 1777 $root/tmp $root/usr/tmp $root/var/tmp +} + +function pl_fixdirs() { + root=$1 + datadirs=$2 + for datadir in datadirs ; do + if [ -h ${root}/${datadir} ] ; then + rm -f ${root}/${datadir} + mkdir -p ${root}/${datadir} + fi + done +} + +########## .pkgs format +# comments start with a # - this is needed only if you use a keyword in a comment + +function pl_getPkgsAttribute () { + keyword=$1; shift + file=$1; shift + # remove any initial white spaces from the result + grep -v '^#' $file | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,," -e "s,^[[:space:]][[:space:]]*,," +} + +# for a given keyword like 'package' : +# we support fcdistro-dependant format, for tokens (pkgname) without whitespace +# you can e.g. use +# package: pkg1 .. pkgn +# package+f8: pkg1 .. pkgn +# package-f8: pkg1 .. pkgn +# +# values can contain @arch@, @fcdistro@ or @pldistro@ that are replaced with the current values +# +# Usage: pl_parsePkgs keyword [-a arch] fcdistro pldistro pkgs-file[..s] +# the reason for the -a option is for when we build the build vserver itself; in this case +# pl_DISTRO_ARCH is the one we obtain from the root context, and that's wrong +# specify -sa arch AFTER keyword, so as to keep pl_getPackages and pl_getGroups simple +# +function pl_parsePkgs () { + + target_arch=$pl_DISTRO_ARCH + keyword=$1;shift + [ "$1" == "-a" ] && { shift; target_arch="$1"; shift; } + fcdistro=$1; shift + pldistro=$1; shift + # remaining arguments are paths to the pkgs files + + # grab regular descriptions + all=$(grep -v '^#' "$@" | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,,") + # grab additions + add=$(grep -v '^#' "$@" | grep --regexp="^${keyword}+${fcdistro}:" | sed -e "s,${keyword}+${fcdistro}:,,") + # grab exclusions + sub=$(grep -v '^#' "$@" | grep --regexp="^${keyword}-${fcdistro}:" | sed -e "s,${keyword}-${fcdistro}:,,") + + for rpm in $all $add; do + for exclude in $sub; do + [ "$rpm" = "$exclude" ] && continue 2 + done + echo "${rpm} " | sed -e "s,@arch@,${target_arch},g" -e "s,@fcdistro@,$fcdistro,g" -e "s,@pldistro@,$pldistro,g" + done + return 0 +} + +# usage: pl_getPackages [-a arch] fcdistro pldistro pkg-file[..s] +function pl_getPackages() { pl_parsePkgs package "$@" ; } +function pl_getGroups() { pl_parsePkgs group "$@" ; } + +# locates a pldistro-dependant file +# tries first in build//, then in build/planetlab/ +function pl_locateDistroFile () { + builddir=$1; shift + pldistro=$1; shift + pkgsfile=$1; shift + + pkgspath="" + # if config dir is missing but a .svnpath exists + if [[ -f "$builddir/config.${pldistro}.svnpath" && ! -d "$builddir/config.${pldistro}" ]] ; then + echo 1>&2 "Invoking make to extract remote config.${pldistro}" + make 1>&2 --silent --no-print-directory -C $builddir stage1=true config.${pldistro} + fi + # locate it + paths="$builddir/config.$pldistro/$pkgsfile $builddir/config.planetlab/$pkgsfile" + for path in $paths; do + if [ -f $path ] ; then + pkgspath=$path + break + fi + done + if [ -z "$pkgspath" ] ; then + echo 1>&2 "pl_locateDistroFile - in $(pwd) : cannot locate $pkgsfile in $builddir" + echo 1>&2 "candidates were $paths" + echo "not-found-by-pl_locateDistroFile" + return 1 + else + echo 1>&2 "pl_locateDistroFile: using $pkgspath" + echo $pkgspath + return 0 + fi +} + +function yumgroups_from_pkgs () { + builddir=$1; shift + pldistro=$1; shift + fcdistro=$1; shift + pkgsnames=$@ + + sedargs="-e s,@FCDISTRO@,$fcdistro,g" + + cat <<__header + + + +__header + + for pkgsname in $pkgsnames; do + pkgsfile=$(pl_locateDistroFile $builddir $pldistro $pkgsname) + packages=$(pl_getPackages $fcdistro $pldistro $pkgsfile) + + groupname=$(pl_getPkgsAttribute groupname $pkgsfile | sed $sedargs) + groupdesc=$(pl_getPkgsAttribute groupdesc $pkgsfile | sed $sedargs) + + if [ -z "$groupname" -o -z "$groupdesc" ] ; then + echo "Cannot find groupname: and groupdesc: in $pkgsfile -- skipped" 1>&2 + continue + fi + + cat << __group_header + + $(echo $groupname|tr A-Z a-z) + $groupname + $groupdesc + true + +__group_header + for package in $packages; do + echo "$package" + done + cat << __group_footer + + +__group_footer + done +cat <<__footer + +__footer +} + + +function build_fetch () { + curl --fail --silent --max-time 60 --output /dev/null "$1" +} + +# tries to compute a valid yum.conf for that pldistro from the template in mirroring/ +# returns 0 and writes on success +# returns 1 on failure, in which case is deleted +function yumconf_mirrors () { + dest_yumconf=$1; shift + builddir=$1; shift + fcdistro=$1; shift + kexclude_line="$1" ; shift + mirrors="$@" + + template=$builddir/mirroring/$fcdistro/yum.repos.d/building.repo.in + + if [ ! -f $template ] ; then + echo "yumconf_mirrors: cannot locate template $template" + rm -f $dest_yumconf + return 1 + fi + + for mirror in $mirrors; do + if yumconf_mirror $dest_yumconf $template "$kexclude_line" $mirror; then + return 0 + fi + done + rm -f $dest_yumconf + return 1 +} + +# computes a yum.conf from the template, and checks that all baseurl defined in there are valid repos +# returns 0 on success and 1 on failure +function yumconf_mirror () { + dest_yumconf=$1; shift + template=$1; shift + kexclude_line="$1" ; shift + mirror=$1; shift + + sed -e "s,@MIRRORURL@,$mirror,g" \ + -e "/baseurl=/i\\ +$kexclude_line" $template > $dest_yumconf + + # capture all lines defining baseurl + baseurl_defs=$(grep '^baseurl=' $dest_yumconf) + if [ -z "$baseurl_defs" ] ; then + return 1 + fi + + for baseurl_def in $baseurl_defs; do + baseurl=$(echo $baseurl_def | sed \ + -e s,baseurl=,, \ + -e 's,$basearch,'"$pl_DISTRO_ARCH",g) + repomd=$baseurl/repodata/repomd.xml + + echo "* Trying to fetch $repomd" + if ! build_fetch $repomd ; then + echo "* Failed to fetch $repomd" + return 1 + fi + done + echo "* Selected mirror $mirror" + return 0 +} + +# from a yum.conf as generated above, computes the (first) gpgkey url +function yumconf_gpgkey () { + dest_yumconf=$1; shift + + first_line=$(grep '^gpgkey=' $dest_yumconf | head -1) + values=$(echo $first_line | sed -e s,gpgkey=,,) + value=$(echo $values | awk '{print $1;}' | sed -e 's,$basearch,'"$pl_DISTRO_ARCH",g) + [ -n "$value" ] || return 1 + echo $value + return 0 +} + +# patches a yum conf to insert an exclude line in each declared repo +function yumconf_exclude () { + repo=$1; shift + kexclude_line="$1" ; shift + + sed -i -e "/baseurl=.*$/i\\ +$kexclude_line" $repo +} diff --git a/trunk/coblitz-tags.mk b/trunk/coblitz-tags.mk new file mode 100644 index 00000000..80f2fc10 --- /dev/null +++ b/trunk/coblitz-tags.mk @@ -0,0 +1,46 @@ +# $Id$ + +# build-SVNPATH is now set by vbuild-nightly.sh to avoid duplication + +linux-2.6-SVNBRANCH := 22 +linux-2.6-SVNPATH := http://svn.planet-lab.org/svn/linux-2.6/branches/coblitz +madwifi-SVNBRANCH := 0.9.4 +madwifi-SVNPATH := http://svn.planet-lab.org/svn/madwifi/tags/madwifi-0.9.4-3 +util-vserver-SVNBRANCH := scholz +util-vserver-SVNPATH := http://svn.planet-lab.org/svn/util-vserver/tags/util-vserver-0.30.215-6 +libnl-SVNPATH := http://svn.planet-lab.org/svn/libnl/tags/libnl-1.1-2 +util-vserver-pl-SVNPATH := http://svn.planet-lab.org/svn/util-vserver-pl/tags/util-vserver-pl-0.3-19 +NodeUpdate-SVNPATH := http://svn.planet-lab.org/svn/NodeUpdate/tags/NodeUpdate-0.5-5 +PingOfDeath-SVNPATH := http://svn.planet-lab.org/svn/PingOfDeath/tags/PingOfDeath-2.2-1 +NodeManager-SVNPATH := http://svn.planet-lab.org/svn/NodeManager/tags/NodeManager-1.8-20 +pyplnet-SVNPATH := http://svn.planet-lab.org/svn/pyplnet/tags/pyplnet-4.3-4 +pl_sshd-SVNPATH := http://svn.planet-lab.org/svn/pl_sshd/tags/pl_sshd-1.0-11 +CoDemux-SVNPATH := http://svn.planet-lab.org/svn/CoDemux/tags/CoDemux-0.1-13 +fprobe-ulog-SVNPATH := http://svn.planet-lab.org/svn/fprobe-ulog/tags/fprobe-ulog-1.1.3-1 +pf2slice-SVNPATH := http://svn.planet-lab.org/svn/pf2slice/tags/pf2slice-1.0-2 +Mom-SVNPATH := http://svn.planet-lab.org/svn/Mom/tags/Mom-2.3-1 +iptables-SVNBRANCH := 1.3.8 +iptables-SVNPATH := http://svn.planet-lab.org/svn/iptables/tags/iptables-1.3.8-9 +iproute2-SVNPATH := http://svn.planet-lab.org/svn/iproute2/tags/iproute2-2.6.16-2 +inotify-tools-SVNPATH := http://svn.planet-lab.org/svn/inotify-tools/tags/inotify-tools-3.13-2 +vsys-SVNBRANCH := 0.9 +vsys-SVNPATH := http://svn.planet-lab.org/svn/vsys/tags/vsys-0.9-3 +vsys-scripts-SVNPATH := http://svn.planet-lab.org/svn/vsys-scripts/tags/vsys-scripts-0.95-11 +PLCAPI-SVNPATH := http://svn.planet-lab.org/svn/PLCAPI/tags/PLCAPI-4.3-30 +drupal-SVNPATH := http://svn.planet-lab.org/svn/drupal/tags/drupal-4.7-13 +PLEWWW-SVNPATH := http://svn.planet-lab.org/svn/PLEWWW/tags/PLEWWW-4.3-33 +www-register-wizard-SVNPATH := http://svn.planet-lab.org/svn/www-register-wizard/tags/www-register-wizard-4.3-1 +Monitor-SVNPATH := http://svn.planet-lab.org/svn/Monitor/tags/Monitor-3.0-24 +nodeconfig-SVNPATH := http://svn.planet-lab.org/svn/nodeconfig/tags/nodeconfig-4.3-5 +BootManager-SVNPATH := http://svn.planet-lab.org/svn/BootManager/tags/BootManager-4.3-13 +pypcilib-SVNPATH := http://svn.planet-lab.org/svn/pypcilib/tags/pypcilib-0.2-8 +BootCD-SVNPATH := http://svn.planet-lab.org/svn/BootCD/tags/BootCD-4.2-15 +VserverReference-SVNPATH := http://svn.planet-lab.org/svn/VserverReference/tags/VserverReference-4.2-16 +BootstrapFS-SVNPATH := http://svn.planet-lab.org/svn/BootstrapFS/tags/BootstrapFS-1.0-10 +MyPLC-SVNPATH := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-4.3-30 +sfa-SVNPATH := http://svn.planet-lab.org/svn/sfa/tags/sfa-0.9-5 +pyopenssl-SVNPATH := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9-1 +PLCRT-SVNPATH := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-11 + +# locating the right test directory - see make tests_svnpath +tests-SVNPATH := http://svn.planet-lab.org/svn/tests/tags/tests-4.3-5 diff --git a/trunk/coblitz.mk b/trunk/coblitz.mk new file mode 100644 index 00000000..03d086fa --- /dev/null +++ b/trunk/coblitz.mk @@ -0,0 +1,395 @@ +# +# declare the packages to be built and their dependencies +# initial version from Mark Huang +# Mark Huang +# Copyright (C) 2003-2006 The Trustees of Princeton University +# rewritten by Thierry Parmentelat - INRIA Sophia Antipolis +# +# $Id: planetlab.mk 14230 2009-07-03 00:58:17Z thierry $ +# +# see doc in Makefile +# + +# +# kernel +# +# use a package name with srpm in it: +# so the source rpm is created by running make srpm in the codebase +# + +kernel-MODULES := linux-2.6 +kernel-SPEC := kernel-2.6.spec +kernel-BUILD-FROM-SRPM := yes +ifeq "$(HOSTARCH)" "i386" +kernel-RPMFLAGS:= --target i686 +else +kernel-RPMFLAGS:= --target $(HOSTARCH) +endif +KERNELS += kernel + +kernels: $(KERNELS) +kernels-clean: $(foreach package,$(KERNELS),$(package)-clean) + +ALL += $(KERNELS) +# this is to mark on which image a given rpm is supposed to go +IN_BOOTCD += $(KERNELS) +IN_VSERVER += $(KERNELS) +IN_BOOTSTRAPFS += $(KERNELS) +# turns out myplc installs kernel-vserver +IN_MYPLC += $(KERNELS) + +# +# madwifi +# +madwifi-MODULES := madwifi +madwifi-SPEC := madwifi.spec +madwifi-BUILD-FROM-SRPM := yes +madwifi-DEPEND-DEVEL-RPMS := kernel-devel +madwifi-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +#ALL += madwifi +#IN_BOOTSTRAPFS += madwifi + +# +# util-vserver +# +util-vserver-MODULES := util-vserver +util-vserver-SPEC := util-vserver.spec +util-vserver-RPMFLAGS:= --without dietlibc +ALL += util-vserver +IN_BOOTSTRAPFS += util-vserver + +# +# libnl - local import +# we need either 1.1 or at least 1.0.pre6 +# rebuild this on centos5 - see kexcludes in build.common +# +local_libnl=false +ifeq "$(DISTRONAME)" "centos5" +local_libnl=true +endif + +ifeq "$(local_libnl)" "true" +libnl-MODULES := libnl +libnl-SPEC := libnl.spec +libnl-BUILD-FROM-SRPM := yes +# this sounds like the thing to do, but in fact linux/if_vlan.h comes with kernel-headers +libnl-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += libnl +IN_BOOTSTRAPFS += libnl +endif + +# +# util-vserver-pl +# +util-vserver-pl-MODULES := util-vserver-pl +util-vserver-pl-SPEC := util-vserver-pl.spec +util-vserver-pl-DEPEND-DEVEL-RPMS := util-vserver-lib util-vserver-devel util-vserver-core +ifeq "$(local_libnl)" "true" +util-vserver-pl-DEPEND-DEVEL-RPMS += libnl libnl-devel +endif +ALL += util-vserver-pl +IN_BOOTSTRAPFS += util-vserver-pl + +# +# NodeUpdate +# +nodeupdate-MODULES := NodeUpdate +nodeupdate-SPEC := NodeUpdate.spec +ALL += nodeupdate +IN_BOOTSTRAPFS += nodeupdate + +# +# ipod +# +ipod-MODULES := PingOfDeath +ipod-SPEC := ipod.spec +ALL += ipod +IN_BOOTSTRAPFS += ipod + +# +# NodeManager +# +nodemanager-MODULES := NodeManager +nodemanager-SPEC := NodeManager.spec +ALL += nodemanager +IN_BOOTSTRAPFS += nodemanager + +# +# pl_sshd +# +sshd-MODULES := pl_sshd +sshd-SPEC := pl_sshd.spec +#ALL += sshd +#IN_BOOTSTRAPFS += sshd + +# +# codemux: Port 80 demux +# +codemux-MODULES := CoDemux +codemux-SPEC := codemux.spec +codemux-RPMBUILD := sudo bash ./rpmbuild.sh +ALL += codemux +IN_BOOTSTRAPFS += codemux + +# +# fprobe-ulog +# +fprobe-ulog-MODULES := fprobe-ulog +fprobe-ulog-SPEC := fprobe-ulog.spec +ALL += fprobe-ulog +IN_BOOTSTRAPFS += fprobe-ulog + +# +# pf2slice +# +pf2slice-MODULES := pf2slice +pf2slice-SPEC := pf2slice.spec +ALL += pf2slice + +# +# PlanetLab Mom: Cleans up your mess +# +mom-MODULES := Mom +mom-SPEC := pl_mom.spec +ALL += mom +IN_BOOTSTRAPFS += mom + +# +# iptables +# +iptables-MODULES := iptables +iptables-SPEC := iptables.spec +iptables-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += iptables +IN_BOOTSTRAPFS += iptables + +# +# iproute +# +iproute-MODULES := iproute2 +iproute-SPEC := iproute.spec +ALL += iproute +IN_BOOTSTRAPFS += iproute +IN_VSERVER += iproute +IN_BOOTCD += iproute + +# +# inotify-tools - local import +# rebuild this on centos5 (not found) - see kexcludes in build.common +# +local_inotify_tools=false +ifeq "$(DISTRONAME)" "centos5" +local_inotify_tools=true +endif + +ifeq "$(local_inotify_tools)" "true" +inotify-tools-MODULES := inotify-tools +inotify-tools-SPEC := inotify-tools.spec +inotify-tools-BUILD-FROM-SRPM := yes +IN_BOOTSTRAPFS += inotify-tools +ALL += inotify-tools +endif + +# +# vsys +# +vsys-MODULES := vsys +vsys-SPEC := vsys.spec +ifeq "$(local_inotify_tools)" "true" +vsys-DEPEND-DEVEL-RPMS := inotify-tools inotify-tools-devel +endif +IN_BOOTSTRAPFS += vsys +ALL += vsys + +# +# vsys-scripts +# +vsys-scripts-MODULES := vsys-scripts +vsys-scripts-SPEC := vsys-scripts.spec +IN_BOOTSTRAPFS += vsys-scripts +ALL += vsys-scripts + +# +# PLCAPI +# +PLCAPI-MODULES := PLCAPI +PLCAPI-SPEC := PLCAPI.spec +ALL += PLCAPI +IN_MYPLC += PLCAPI + +# +# drupal +# +drupal-MODULES := drupal +drupal-SPEC := drupal.spec +drupal-BUILD-FROM-SRPM := yes +ALL += drupal +IN_MYPLC += drupal + +# +# use the plewww module instead +# +plewww-MODULES := PLEWWW +plewww-SPEC := plewww.spec +ALL += plewww +IN_MYPLC += plewww + +# +# www-register-wizard +# +www-register-wizard-MODULES := www-register-wizard +www-register-wizard-SPEC := www-register-wizard.spec +ALL += www-register-wizard +IN_MYPLC += www-register-wizard + +# +# monitor +# +monitor-MODULES := Monitor +monitor-SPEC := Monitor.spec +ALL += monitor +IN_BOOTSTRAPFS += monitor + +# +# PLC RT +# +plcrt-MODULES := PLCRT +plcrt-SPEC := plcrt.spec +ALL += plcrt + +# +# zabbix +# +zabbix-MODULES := Monitor +zabbix-SPEC := zabbix.spec +zabbix-BUILD-FROM-SRPM := yes +ALL += zabbix + +# +# pyopenssl +# +pyopenssl-MODULES := pyopenssl +pyopenssl-SPEC := pyOpenSSL.spec +pyopenssl-BUILD-FROM-SRPM := yes +ALL += pyopenssl + + +# +# sfa - Slice Facility Architecture +# +# sfa-MODULES := sfa +# sfa-SPEC := sfa.spec +# ALL += sfa + +# +# nodeconfig +# +nodeconfig-MODULES := nodeconfig build +nodeconfig-SPEC := nodeconfig.spec +ALL += nodeconfig +IN_MYPLC += nodeconfig + +# +# bootmanager +# +bootmanager-MODULES := BootManager +bootmanager-SPEC := bootmanager.spec +ALL += bootmanager +IN_MYPLC += bootmanager + +# +# pypcilib : used in bootcd +# +pypcilib-MODULES := pypcilib +pypcilib-SPEC := pypcilib.spec +ALL += pypcilib +IN_BOOTCD += pypcilib + +# +# pyplnet +# +pyplnet-MODULES := pyplnet +pyplnet-SPEC := pyplnet.spec +ALL += pyplnet +IN_BOOTSTRAPFS += pyplnet +IN_MYPLC += pyplnet +IN_BOOTCD += pyplnet + +# +# bootcd +# +bootcd-MODULES := BootCD build +bootcd-SPEC := bootcd.spec +bootcd-RPMBUILD := sudo bash ./rpmbuild.sh +bootcd-DEPEND-PACKAGES := $(IN_BOOTCD) +bootcd-DEPEND-FILES := RPMS/yumgroups.xml +bootcd-RPMDATE := yes +ALL += bootcd +IN_MYPLC += bootcd + +# +# vserver : reference image for slices +# +vserver-MODULES := VserverReference build +vserver-SPEC := vserver-reference.spec +vserver-DEPEND-PACKAGES := $(IN_VSERVER) +vserver-DEPEND-FILES := RPMS/yumgroups.xml +vserver-RPMDATE := yes +ALL += vserver +IN_BOOTSTRAPFS += vserver + +# +# bootstrapfs +# +bootstrapfs-MODULES := BootstrapFS build +bootstrapfs-SPEC := bootstrapfs.spec +bootstrapfs-RPMBUILD := sudo bash ./rpmbuild.sh +bootstrapfs-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) +bootstrapfs-DEPEND-FILES := RPMS/yumgroups.xml +bootstrapfs-RPMDATE := yes +ALL += bootstrapfs +IN_MYPLC += bootstrapfs + +# +# noderepo +# +# all rpms resulting from packages marked as being in bootstrapfs and vserver +NODEREPO_RPMS = $(foreach package,$(IN_BOOTSTRAPFS) $(IN_VSERVER),$($(package).rpms)) +# replace space with +++ (specvars cannot deal with spaces) +SPACE=$(subst x, ,x) +NODEREPO_RPMS_3PLUS = $(subst $(SPACE),+++,$(NODEREPO_RPMS)) + +noderepo-MODULES := BootstrapFS +noderepo-SPEC := noderepo.spec +noderepo-RPMBUILD := sudo bash ./rpmbuild.sh +# package requires all regular packages +noderepo-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) $(IN_VSERVER) +noderepo-DEPEND-FILES := RPMS/yumgroups.xml +#export rpm list to the specfile +noderepo-SPECVARS = node_rpms_plus=$(NODEREPO_RPMS_3PLUS) +noderepo-RPMDATE := yes +ALL += noderepo +IN_MYPLC += noderepo + +# +# MyPLC : lightweight packaging, dependencies are yum-installed in a vserver +# +myplc-MODULES := MyPLC build +myplc-SPEC := myplc.spec +myplc-DEPEND-FILES := myplc-release RPMS/yumgroups.xml +ALL += myplc + +# myplc-docs only contains docs for PLCAPI and NMAPI, but +# we still need to pull MyPLC, as it is where the specfile lies, +# together with the utility script docbook2drupal.sh +myplc-docs-MODULES := MyPLC PLCAPI NodeManager Monitor +myplc-docs-SPEC := myplc-docs.spec +ALL += myplc-docs + +# using some other name than myplc-release, as this is a make target already +release-MODULES := MyPLC +release-SPEC := myplc-release.spec +release-RPMDATE := yes +ALL += release diff --git a/trunk/config.coblitz/bootstrapfs.pkgs b/trunk/config.coblitz/bootstrapfs.pkgs new file mode 100644 index 00000000..f4926c0e --- /dev/null +++ b/trunk/config.coblitz/bootstrapfs.pkgs @@ -0,0 +1,126 @@ +# Keeping the old name as it might be referenced elsewhere (node update..) +# groupname: Bootstrapfs +groupname: PlanetLab +groupdesc: PlanetLab Node Root +# packages to exclude from stock repositories +kexclude: @KEXCLUDE@ +# fedora-release or centos-release are pulled from deps +# define packages +package: kernel +package: udev +package: basesystem +package: filesystem +package: bash +package: coreutils +package: python +package: cpio +package: e2fsprogs +package: ed +package: file +package: glibc +package: hdparm +package: initscripts +package: iproute +package: iputils +package: kbd +package: passwd +package: procps +package: readline +package: rootfiles +package: rpm +package: setserial +package: setup +package: vim-minimal +package: shadow-utils +package: anacron +package: at +package: authconfig +package: bc +package: bind-utils +package: bzip2 +package: crontabs +package: dhclient +package: diffutils +package: ethtool +package: logrotate +package: libnl +package: lsof +package: mailcap +package: nano +package: nc +package: openssh-clients +package: parted +package: pciutils +package: psacct +package: quota +package: rsh +package: rsync +package: sendmail +package: sudo +package: telnet +package: traceroute +package: time +package: tmpwatch +package: tcpdump +package: openssh-server +package: wget +package: yum +package: curl +package: gzip +package: iptables +package: mkinitrd +package: ntp +package: perl +package: python +package: tar +package: expect +package: openvpn +package: vconfig +package: PyXML + +package: termcap libtermcap +package: vixie-cron + +# +# platform-dependent +# +package: rsyslog +package-centos5: rsyslog +package+centos5: sysklogd + +package: util-linux-ng +package-centos5: util-linux-ng +package+centos5: util-linux + +package-f10: termcap libtermcap +package-f10: vixie-cron +# +# planetlab +# +package: util-vserver +package: util-vserver-build +package: util-vserver-lib +package: util-vserver-core +package: util-vserver-sysv +package: util-vserver-legacy +package: util-vserver-pl +package: NodeManager +package: NodeUpdate +package: codemux +package: pl_sshd +package: ipod +package: vserver-@pldistro@-@fcdistro@-@arch@ +package: vserver-systemslices-@pldistro@-@fcdistro@-@arch@ +package: pl_mom +package: fprobe-ulog +package: vsys +package: vsys-scripts +package: monitor-client +package: monitor-runlevelagent + +# +# wireless support +# +#package: madwifi +#package: wireless-tools + diff --git a/trunk/config.coblitz/bootstrapfs.post b/trunk/config.coblitz/bootstrapfs.post new file mode 100644 index 00000000..91de1870 --- /dev/null +++ b/trunk/config.coblitz/bootstrapfs.post @@ -0,0 +1,26 @@ +#!/bin/bash + +vdir=$1 +if [ -z "${vdir}" ] ; then + echo "ERROR: $0" + echo "Provide the directory of the root filesystem to operate on" + exit +fi + +# NOTE: we're enabling util-vserver to allow it to help shutdown all slices +# before rebooting. This has been problematic in the past. +# Thierry : I'm enabling network since, for some reason, it ends up turned off on fedora9 +for service in network util-vserver; do + chroot ${vdir} /sbin/chkconfig $service on +done + +# Remove unneeded services +for service in vprocunhide vservers-default fprobe-ulog; do + chroot ${vdir} /sbin/chkconfig $service off +done + +# Disable splaying of cron. +echo > ${vdir}/etc/sysconfig/crontab + +# Add site_admin account +chroot ${vdir} /usr/sbin/useradd -p "" -u 502 -m site_admin diff --git a/trunk/config.coblitz/coblitz.mirrors b/trunk/config.coblitz/coblitz.mirrors new file mode 100644 index 00000000..65256607 --- /dev/null +++ b/trunk/config.coblitz/coblitz.mirrors @@ -0,0 +1,4 @@ +# define here the ordered set of mirrors that you'd like to use when running mkfedora +# used to be hard-wired in build.common / mkfedora +# see http://svn.planet-lab.org/wiki/VserverCentos for howto setup local mirroring +mirror: http://build.planet-lab.org/ diff --git a/trunk/config.coblitz/devel.pkgs b/trunk/config.coblitz/devel.pkgs new file mode 100644 index 00000000..7a97ffa6 --- /dev/null +++ b/trunk/config.coblitz/devel.pkgs @@ -0,0 +1,99 @@ +groupname: PlanetLabDevel +groupdesc: Building PlanetLab +# general utilities +package: sendmail sendmail-cf mailx +package: make install +package: glibc glibc-common +package: bzip2 gzip +package: cpio tar +package: coreutils +package: rpm rpm-build rpm-devel +package: redhat-rpm-config +package: curl curl-devel +package: subversion cvs +package: less +package: openssh +package: emacs +# undetermined +package: expect +package: gd +package: httpd mod_python mod_ssl +package: openssl +package: openssl-devel +package: zlib +package: zlib-devel +package: bison flex +package: libtool +package: metadata +package: mysql mysql-devel mysql-server +package: python python-devel +package: doxygen +package: vixie-cron +package: xmlsec1 xmlsec1-openssl +package: udev +package: expat-devel +package: db4-devel +package: ncurses-devel +package: readline-devel +package: dnsmasq +# for spec2make / rpmlib +package+f8: popt-devel +# kernel +package: gnupg +package: diffutils +package: vconfig +package: iptables +package: wget +package: beecrypt-devel +package: tetex-latex +package: gcc-c++ +# iptables +package: linuxdoc-tools +package: sudo +package: yum createrepo +# mysql +package: gperf +package: time +# bootmanager +package: sharutils +# bootcd +package: nasm +package: mkisofs +package: dosfstools +package: mtools +package: syslinux +# myplc +package: rsync +package: ghostscript +# PLCAPI +package: docbook-utils-pdf +package: postgresql postgresql-devel postgresql-python postgresql-server +# if we don't specify @arch@ for php-devel, we end up with the 2 variants (i386 and x86_64) +# in an undetermined order, and php-config --extension-dir might return the wrong place +package: php php-devel.@arch@ php-gd php-pgsql +package: PyXML +# used to reference SOAPpy as well +# for pypcilib +package: pciutils-devel +########## +# for vsys - same as for php-devel above +package: ocaml.@arch@ ocaml-docs +# use local inotify-tools on centos +package:inotify-tools-devel +package-centos5: inotify-tools-devel +########## +## for util-vserver-pl +# use local libnl on centos +package: libnl-devel +package-centos5: libnl-devel +# zabbix/monitor +package: net-snmp net-snmp-devel +package+centos5: krb5-devel.@arch@ e2fsprogs-devel.@arch@ libidn-devel.@arch@ +package: TurboGears +package: python-cherrypy +########## +# for sfa : rebuilding wsdl index at build-time +package: python-uuid pyOpenSSL m2crypto +########## +# for comon +package: libpng-devel freetype freetype-devel libart_lgpl-devel gd-devel db4 db4-devel diff --git a/trunk/config.onelab/bootstrapfs-umts.pkgs b/trunk/config.onelab/bootstrapfs-umts.pkgs new file mode 100644 index 00000000..477ea9e4 --- /dev/null +++ b/trunk/config.onelab/bootstrapfs-umts.pkgs @@ -0,0 +1,6 @@ +# $Id$ +groupname: extensionumts +groupdesc: planetlab node umts extensions +package: nozomi +package: comgt +package: wvdial diff --git a/trunk/config.onelab/bootstrapfs-wifi.pkgs b/trunk/config.onelab/bootstrapfs-wifi.pkgs new file mode 100644 index 00000000..1de15eb8 --- /dev/null +++ b/trunk/config.onelab/bootstrapfs-wifi.pkgs @@ -0,0 +1,20 @@ +# $Id$ +# +# first attempt for an extensible bootstrapfs contents +# +# the groupname should start with extension as the bootmanager +# computes it as extension +# the present file's name can be either +# (*) bootstrapfs-something.pkgs +# in which case the tarball will be elaborated at build-time +# (*) anything else otherwise +# +# at runtime, the choice between tarball-install or yum install +# depends on whether the tarball can be downloaded or not +# +groupname: extensionwifi +groupdesc: planetlab node wifi extensions +package: madwifi +package: wireless-tools +package: iwl3945-firmware iwl4965-firmware +package: wpa_supplicant diff --git a/trunk/config.onelab/bootstrapfs.pkgs b/trunk/config.onelab/bootstrapfs.pkgs new file mode 100644 index 00000000..7643ddcf --- /dev/null +++ b/trunk/config.onelab/bootstrapfs.pkgs @@ -0,0 +1,128 @@ +# Keeping the old name as it might be referenced elsewhere (node update..) +# groupname: Bootstrapfs +groupname: PlanetLab +groupdesc: PlanetLab Node Root +# packages to exclude from stock repositories +kexclude: @KEXCLUDE@ +# fedora-release or centos-release are pulled from deps +# define packages +package: kernel +package: udev +package: basesystem +package: filesystem +package: bash +package: coreutils +package: python +package: cpio +package: e2fsprogs +package: ed +package: file +package: glibc +package: hdparm +package: initscripts +package: iproute +package: iputils +package: kbd +package: passwd +package: procps +package: readline +package: rootfiles +package: rpm +package: setserial +package: setup +package: vim-minimal +package: shadow-utils +package: anacron +package: at +package: authconfig +package: bc +package: bind-utils +package: bzip2 +package: crontabs +package: dhclient +package: diffutils +package: ethtool +package: logrotate +package: libnl +package: lsof +package: mailcap +package: nano +package: nc +package: openssh-clients +package: parted +package: pciutils +package: psacct +package: quota +package: rsh +package: rsync +package: sendmail +package: sudo +package: telnet +package: traceroute +package: time +package: tmpwatch +package: tcpdump +package: openssh-server +package: wget +package: yum +package: curl +package: gzip +package: iptables +package: mkinitrd +package: ntp +package: perl +package: python +package: tar +package: expect +package: nfs-utils +package: openvpn +package: vconfig +package: PyXML + +package: termcap libtermcap +package: vixie-cron + +# +# platform-dependent +# +package: rsyslog +package-centos5: rsyslog +package+centos5: sysklogd + +package: util-linux-ng +package-centos5: util-linux-ng +package+centos5: util-linux + +package-f10: termcap libtermcap +package-f10: vixie-cron + +# for nagios +package: nrpe + +# +# planetlab +# +package: ipfw +package: madwifi +package: wireless-tools +package: util-vserver +package: util-vserver-build +package: util-vserver-lib +package: util-vserver-core +package: util-vserver-sysv +package: util-vserver-legacy +package: util-vserver-pl +#package: util-python +package: NodeManager +package: NodeUpdate +package: codemux +package: pl_sshd +package: ipod +package: vserver-@pldistro@-@fcdistro@-@arch@ +package: vserver-systemslices-@pldistro@-@fcdistro@-@arch@ +package: pl_mom +package: fprobe-ulog +package: vsys +package: vsys-scripts +package: monitor-client +package: monitor-runlevelagent diff --git a/trunk/config.onelab/onelab.mirrors b/trunk/config.onelab/onelab.mirrors new file mode 100644 index 00000000..ecf5bc65 --- /dev/null +++ b/trunk/config.onelab/onelab.mirrors @@ -0,0 +1,3 @@ +# use the local mirror only +# see http://svn.planet-lab.org/wiki/VserverCentos for howto setup localhost mirroring +mirror: http://mirror.onelab.eu/ diff --git a/trunk/config.onelab/vserver-planetflow.pkgs b/trunk/config.onelab/vserver-planetflow.pkgs new file mode 120000 index 00000000..a1d38c3f --- /dev/null +++ b/trunk/config.onelab/vserver-planetflow.pkgs @@ -0,0 +1 @@ +../config.planetlab/vserver-planetflow.pkgs \ No newline at end of file diff --git a/trunk/config.onelab/vserver.pkgs b/trunk/config.onelab/vserver.pkgs new file mode 100644 index 00000000..c683aba1 --- /dev/null +++ b/trunk/config.onelab/vserver.pkgs @@ -0,0 +1,37 @@ +groupname: PlanetLabSlice +groupdesc: PlanetLab Slice Reference Image +# packages to exclude from stock repositories +kexclude: @KEXCLUDE@ +# define packages +package: bash +package: coreutils +package: iputils +package: kernel-vserver +package: bzip2 +package: crontabs +package: diffutils +package: logrotate +package: openssh-clients +package: passwd +package: rsh +package: rsync +package: sudo +package: tcpdump +package: telnet +package: traceroute +package: time +package: wget +package: which +package: yum +package: curl +package: gzip +package: perl +package: python +package: python-devel +package: tar +package: findutils +package: filesystem +package: vixie-cron +package-f10: vixie-cron +package: planetlab-umts-tools-frontend +package: ipfw-slice diff --git a/trunk/config.planetbridge/bootstrapfs.pkgs b/trunk/config.planetbridge/bootstrapfs.pkgs new file mode 100644 index 00000000..591b45ed --- /dev/null +++ b/trunk/config.planetbridge/bootstrapfs.pkgs @@ -0,0 +1,123 @@ +# Keeping the old name as it might be referenced elsewhere (node update..) +# groupname: Bootstrapfs +groupname: PlanetLab +groupdesc: PlanetLab Node Root +# packages to exclude from stock repositories +kexclude: @KEXCLUDE@ +# fedora-release or centos-release are pulled from deps +# define packages +package: kernel +package: udev +package: basesystem +package: filesystem +package: bash +package: coreutils +package: python +package: cpio +package: e2fsprogs +package: ed +package: file +package: glibc +package: hdparm +package: initscripts +package: iproute +package: iputils +package: kbd +package: passwd +package: procps +package: readline +package: rootfiles +package: rpm +package: setserial +package: setup +package: vim-minimal +package: shadow-utils +package: anacron +package: at +package: authconfig +package: bc +package: bind-utils +package: bzip2 +package: crontabs +package: dhclient +package: diffutils +package: ethtool +package: logrotate +package: libnl +package: lsof +package: mailcap +package: nano +package: nc +package: openssh-clients +package: parted +package: pciutils +package: psacct +package: quota +package: rsh +package: rsync +package: sendmail +package: sudo +package: telnet +package: traceroute +package: time +package: tmpwatch +package: tcpdump +package: openssh-server +package: wget +package: yum +package: curl +package: gzip +package: iptables +package: mkinitrd +package: ntp +package: perl +package: python +package: tar +package: expect +package: nfs-utils +package: openvpn +package: vconfig +package: PyXML + +package: termcap libtermcap +package: vixie-cron + +# +# platform-dependent +# +package: rsyslog +package-centos5: rsyslog +package+centos5: sysklogd + +package: util-linux-ng +package-centos5: util-linux-ng +package+centos5: util-linux + +package-f10: termcap libtermcap +package-f10: vixie-cron +# +# planetlab +# +package: madwifi +package: wireless-tools +package: util-vserver +package: util-vserver-build +package: util-vserver-lib +package: util-vserver-core +package: util-vserver-sysv +package: util-vserver-legacy +package: util-vserver-pl +#package: util-python +package: NodeManager +package: NodeUpdate +package: codemux +package: pl_sshd +package: ipod +package: vserver-@pldistro@-@fcdistro@-@arch@ +package: vserver-systemslices-@pldistro@-@fcdistro@-@arch@ +package: pl_mom +package: fprobe-ulog +package: vsys +package: vsys-scripts +package: monitor-client +package: monitor-runlevelagent diff --git a/trunk/config.planetbridge/bootstrapfs.post b/trunk/config.planetbridge/bootstrapfs.post new file mode 100644 index 00000000..48fe6af5 --- /dev/null +++ b/trunk/config.planetbridge/bootstrapfs.post @@ -0,0 +1,29 @@ +#!/bin/bash + +vdir=$1 +if [ -z "${vdir}" ] ; then + echo "ERROR: $0" + echo "Provide the directory of the root filesystem to operate on" + exit +fi + +# Cleanup yum config entirely, waiting for the config files to populate this +rm -rf ${vdir}/etc/yum.conf ${vdir}/etc/yum.repos.d + +# NOTE: we're enabling util-vserver to allow it to help shutdown all slices +# before rebooting. This has been problematic in the past. +# Thierry : I'm enabling network since, for some reason, it ends up turned off on fedora9 +for service in network util-vserver; do + chroot ${vdir} /sbin/chkconfig $service on +done + +# Remove unneeded services +for service in vprocunhide vservers-default; do + chroot ${vdir} /sbin/chkconfig $service off +done + +# Disable splaying of cron. +echo > ${vdir}/etc/sysconfig/crontab + +# Add site_admin account +chroot ${vdir} /usr/sbin/useradd -p "" -u 502 -m site_admin diff --git a/trunk/config.planetbridge/planetbridge.mirrors b/trunk/config.planetbridge/planetbridge.mirrors new file mode 100644 index 00000000..65256607 --- /dev/null +++ b/trunk/config.planetbridge/planetbridge.mirrors @@ -0,0 +1,4 @@ +# define here the ordered set of mirrors that you'd like to use when running mkfedora +# used to be hard-wired in build.common / mkfedora +# see http://svn.planet-lab.org/wiki/VserverCentos for howto setup local mirroring +mirror: http://build.planet-lab.org/ diff --git a/trunk/config.planetbridge/vserver-ap.pks b/trunk/config.planetbridge/vserver-ap.pks new file mode 100644 index 00000000..f2e6d9a4 --- /dev/null +++ b/trunk/config.planetbridge/vserver-ap.pks @@ -0,0 +1,5 @@ +groupname: PlanetFlowReference +groupdesc: PlanetFlow Reference Image +# add in all rpms we need such that the ap vserver will have all of +# the necessary software to bring up wireless without ever needing +# to contact MyPLC diff --git a/trunk/config.planetlab/bootcd.pkgs b/trunk/config.planetlab/bootcd.pkgs new file mode 100644 index 00000000..b670023f --- /dev/null +++ b/trunk/config.planetlab/bootcd.pkgs @@ -0,0 +1,78 @@ +groupname: PlanetLabBootcd +groupdesc: PlanetLab BootCD +# packages to exclude from stock repositories +kexclude: @KEXCLUDE@ +# define packages +package: filesystem +package: udev +package: dhclient +package: bash +package: coreutils +package: iputils +package: kernel +package: bzip2 +package: diffutils +package: logrotate +package: passwd +package: rsync +package: sudo +package: tcpdump +package: telnet +package: traceroute +package: time +package: wget +package: yum +package: curl +package: gzip +package: python +package: tar +package: pciutils +package: kbd +package: authconfig +package: hdparm +package: lvm2 +package: kexec-tools +package: gnupg +package: nano +package: parted +package: pyparted +package: openssh-server +package: openssh-clients +package: ncftp +package: dosfstools +package: dos2unix +package: bind-utils +package: sharutils +package: vconfig +package: ntp +package: pypcilib +package: openvpn +#################### formerly in prep.sh +junk: lib/obsolete +junk: lib/tls +junk: usr/share/cracklib +junk: usr/share/emacs +junk: usr/share/gnupg +junk: usr/share/i18n +junk: usr/share/locale +junk: usr/share/terminfo +junk: usr/share/zoneinfo +junk: usr/sbin/build-locale-archive +junk: usr/sbin/dbconverter-2 +junk: usr/sbin/sasl* +junk: usr/sbin/tcpslice +junk: usr/lib/perl* +junk: usr/lib/locale +junk: usr/lib/sasl* +junk: usr/lib/gconv +junk: usr/lib/tls +# +precious: usr/share/i18n/locales/en_US +precious: usr/share/i18n/charmaps/UTF-8.gz +precious: usr/share/locale/en +precious: usr/share/terminfo/l/linux +precious: usr/share/terminfo/v/vt100 +precious: usr/share/terminfo/x/xterm +precious: usr/share/zoneinfo/UTC +precious+f8: usr/lib/locale/en_US.utf8 +precious+centos5: usr/lib/locale/en_US.utf8 diff --git a/trunk/config.planetlab/bootstrapfs.pkgs b/trunk/config.planetlab/bootstrapfs.pkgs new file mode 100644 index 00000000..591b45ed --- /dev/null +++ b/trunk/config.planetlab/bootstrapfs.pkgs @@ -0,0 +1,123 @@ +# Keeping the old name as it might be referenced elsewhere (node update..) +# groupname: Bootstrapfs +groupname: PlanetLab +groupdesc: PlanetLab Node Root +# packages to exclude from stock repositories +kexclude: @KEXCLUDE@ +# fedora-release or centos-release are pulled from deps +# define packages +package: kernel +package: udev +package: basesystem +package: filesystem +package: bash +package: coreutils +package: python +package: cpio +package: e2fsprogs +package: ed +package: file +package: glibc +package: hdparm +package: initscripts +package: iproute +package: iputils +package: kbd +package: passwd +package: procps +package: readline +package: rootfiles +package: rpm +package: setserial +package: setup +package: vim-minimal +package: shadow-utils +package: anacron +package: at +package: authconfig +package: bc +package: bind-utils +package: bzip2 +package: crontabs +package: dhclient +package: diffutils +package: ethtool +package: logrotate +package: libnl +package: lsof +package: mailcap +package: nano +package: nc +package: openssh-clients +package: parted +package: pciutils +package: psacct +package: quota +package: rsh +package: rsync +package: sendmail +package: sudo +package: telnet +package: traceroute +package: time +package: tmpwatch +package: tcpdump +package: openssh-server +package: wget +package: yum +package: curl +package: gzip +package: iptables +package: mkinitrd +package: ntp +package: perl +package: python +package: tar +package: expect +package: nfs-utils +package: openvpn +package: vconfig +package: PyXML + +package: termcap libtermcap +package: vixie-cron + +# +# platform-dependent +# +package: rsyslog +package-centos5: rsyslog +package+centos5: sysklogd + +package: util-linux-ng +package-centos5: util-linux-ng +package+centos5: util-linux + +package-f10: termcap libtermcap +package-f10: vixie-cron +# +# planetlab +# +package: madwifi +package: wireless-tools +package: util-vserver +package: util-vserver-build +package: util-vserver-lib +package: util-vserver-core +package: util-vserver-sysv +package: util-vserver-legacy +package: util-vserver-pl +#package: util-python +package: NodeManager +package: NodeUpdate +package: codemux +package: pl_sshd +package: ipod +package: vserver-@pldistro@-@fcdistro@-@arch@ +package: vserver-systemslices-@pldistro@-@fcdistro@-@arch@ +package: pl_mom +package: fprobe-ulog +package: vsys +package: vsys-scripts +package: monitor-client +package: monitor-runlevelagent diff --git a/trunk/config.planetlab/bootstrapfs.post b/trunk/config.planetlab/bootstrapfs.post new file mode 100644 index 00000000..0e722a88 --- /dev/null +++ b/trunk/config.planetlab/bootstrapfs.post @@ -0,0 +1,53 @@ +#!/bin/bash + +vdir=$1 +if [ -z "${vdir}" ] ; then + echo "ERROR: $0" + echo "Provide the directory of the root filesystem to operate on" + exit +fi + +# Cleanup yum config entirely, waiting for the config files to populate this +rm -rf ${vdir}/etc/yum.conf ${vdir}/etc/yum.repos.d + +# NOTE: we're enabling util-vserver to allow it to help shutdown all slices +# before rebooting. This has been problematic in the past. +# Thierry : I'm enabling network since, for some reason, it ends up turned off on fedora9 +for service in network util-vserver; do + chroot ${vdir} /sbin/chkconfig $service on +done + +# Remove unneeded services +for service in vprocunhide vservers-default; do + chroot ${vdir} /sbin/chkconfig $service off +done + +# Disable splaying of cron. +echo > ${vdir}/etc/sysconfig/crontab + +# Add site_admin account +chroot ${vdir} /usr/sbin/useradd -p "" -u 502 -m site_admin + +# NOTE: This is added to relieve one site's Cisco router configuration that +# fails to recognize the host once the arping is sent out. +# NOTE: this is pretty fragile, and fails on fedora 10 that as of today (oct. 20 2009) +# has initscripts-8.86.3-1.i386 which reads almost identical but with /sbin/arping instead +cat <<\EOF | patch -d ${vdir}/etc/sysconfig/network-scripts/ +--- ifup-eth 2008-07-08 13:19:49.000000000 -0400 ++++ ifup-eth-orig 2008-07-08 13:20:02.000000000 -0400 +@@ -263,10 +263,10 @@ + fi + + if ! LC_ALL=C ip addr ls ${REALDEVICE} | LC_ALL=C grep -q "${IPADDR}/${PREFIX}" ; then +- if ! arping -q -c 2 -w 3 -D -I ${REALDEVICE} ${IPADDR} ; then +- echo $"Error, some other host already uses address ${IPADDR}." +- exit 1 +- fi ++ #if ! arping -q -c 2 -w 3 -D -I ${REALDEVICE} ${IPADDR} ; then ++ # echo $"Error, some other host already uses address ${IPADDR}." ++ # exit 1 ++ #fi + if ! ip addr add ${IPADDR}/${PREFIX} \ + brd ${BROADCAST:-+} dev ${REALDEVICE} ${SCOPE} label ${DEVICE}; then + echo $"Error adding address ${IPADDR} for ${DEVICE}." +EOF diff --git a/trunk/config.planetlab/devel.pkgs b/trunk/config.planetlab/devel.pkgs new file mode 100644 index 00000000..62bbf99f --- /dev/null +++ b/trunk/config.planetlab/devel.pkgs @@ -0,0 +1,99 @@ +groupname: PlanetLabDevel +groupdesc: Building PlanetLab +# general utilities +package: sendmail sendmail-cf mailx +package: make install +package: glibc glibc-common +package: bzip2 gzip +package: cpio tar +package: coreutils +package: rpm rpm-build rpm-devel +package: redhat-rpm-config +package: curl curl-devel +package: subversion cvs +package: less +package: openssh +package: emacs +# undetermined +package: expect +package: gd +package: httpd mod_python mod_ssl +package: openssl +package: openssl-devel +package: zlib +package: zlib-devel +package: bison flex +package: libtool +package: metadata +package: mysql mysql-devel mysql-server +package: python python-devel +package: doxygen +package: vixie-cron +package: xmlsec1 xmlsec1-openssl +package: udev +package: expat-devel +package: db4-devel +package: ncurses-devel +package: readline-devel +package: dnsmasq +# for spec2make / rpmlib +package+f8: popt-devel +# kernel +package: gnupg +package: diffutils +package: vconfig +package: iptables +package: wget +package: beecrypt-devel +package: tetex-latex +package: gcc-c++ +# iptables +package: linuxdoc-tools +package: sudo +package: yum createrepo +# mysql +package: gperf +package: time +# bootmanager +package: sharutils +# bootcd +package: nasm +package: mkisofs +package: dosfstools +package: mtools +package: syslinux +# myplc +package: rsync +package: ghostscript +# PLCAPI +package: docbook-utils-pdf +package: postgresql postgresql-devel postgresql-python postgresql-server +# if we don't specify @arch@ for php-devel, we end up with the 2 variants (i386 and x86_64) +# in an undetermined order, and php-config --extension-dir might return the wrong place +package: php php-devel.@arch@ php-gd php-pgsql +package: PyXML +# used to reference SOAPpy as well +# for pypcilib +package: pciutils-devel +########## +# for vsys - same as for php-devel above +package: ocaml.@arch@ ocaml-docs +# use local inotify-tools on centos +package:inotify-tools-devel +package-centos5: inotify-tools-devel +########## +## for util-vserver-pl +# use local libnl on centos +package: libnl-devel +package-centos5: libnl-devel +# for util-vserver +package: ctags +# zabbix/monitor +package: net-snmp net-snmp-devel +package+centos5: krb5-devel.@arch@ e2fsprogs-devel.@arch@ libidn-devel.@arch@ +package: TurboGears +package: python-cherrypy +########## +# for sfa : rebuilding wsdl index at build-time +package: python-uuid pyOpenSSL m2crypto +package: libxml2-python libxslt-python diff --git a/trunk/config.planetlab/planetlab.mirrors b/trunk/config.planetlab/planetlab.mirrors new file mode 100644 index 00000000..07acaefc --- /dev/null +++ b/trunk/config.planetlab/planetlab.mirrors @@ -0,0 +1,11 @@ +# define here the ordered set of mirrors that you'd like to use when running mkfedora +# used to be hard-wired in build.common / mkfedora +# see http://svn.planet-lab.org/wiki/VserverCentos for howto setup local mirroring +mirror: http://localhost/mirror/ +mirror: http://build.planet-lab.org/ +#mirror: file:///data/fedora +#mirror: http://localhost/fedora +#mirror: http://coblitz.codeen.org/coblitz.planet-lab.org/pub/fedora +#mirror: ftp://mirror.cs.princeton.edu/pub/mirrors/fedora +#mirror: ftp://mirror.stanford.edu/pub/mirrors/fedora +#mirror: ftp://rpmfind.net/linux/fedora diff --git a/trunk/config.planetlab/vserver-planetflow.pkgs b/trunk/config.planetlab/vserver-planetflow.pkgs new file mode 100644 index 00000000..b6ceeddb --- /dev/null +++ b/trunk/config.planetlab/vserver-planetflow.pkgs @@ -0,0 +1,4 @@ +groupname: PlanetFlowReference +groupdesc: PlanetFlow Reference Image +package: rsync +package: pf2slice diff --git a/trunk/config.planetlab/vserver.pkgs b/trunk/config.planetlab/vserver.pkgs new file mode 100644 index 00000000..c114ef8e --- /dev/null +++ b/trunk/config.planetlab/vserver.pkgs @@ -0,0 +1,35 @@ +groupname: PlanetLabSlice +groupdesc: PlanetLab Slice Reference Image +# packages to exclude from stock repositories +kexclude: @KEXCLUDE@ +# define packages +package: bash +package: coreutils +package: iputils +package: kernel-vserver +package: bzip2 +package: crontabs +package: diffutils +package: logrotate +package: openssh-clients +package: passwd +package: rsh +package: rsync +package: sudo +package: tcpdump +package: telnet +package: traceroute +package: time +package: wget +package: which +package: yum +package: curl +package: gzip +package: perl +package: python +package: python-devel +package: tar +package: findutils +package: filesystem +package: vixie-cron +package-f10: vixie-cron diff --git a/trunk/config.planetlab/vserver.post b/trunk/config.planetlab/vserver.post new file mode 100644 index 00000000..fb304df5 --- /dev/null +++ b/trunk/config.planetlab/vserver.post @@ -0,0 +1,17 @@ +#!/bin/bash + +vdir=$1 + +if [ -z "${vdir}" ] ; then + echo "ERROR: $0" + echo "Provide the directory of the root filesystem to operate on" + exit +fi + +# reset root password to be empty +echo "Resetting root password in vserver-reference image" +sed -i -e "s/root:\*/root:/" ${vdir}/etc/passwd + +# modify the sudoers file to remove the 'requiretty' attribute +echo "Removing requiretty from default /etc/sudoers file" +sed -i -e 's,^\(Defaults.*requiretty\),#\1,' /etc/sudoers diff --git a/trunk/config.planetlab/vtest.pkgs b/trunk/config.planetlab/vtest.pkgs new file mode 100644 index 00000000..19ba1aab --- /dev/null +++ b/trunk/config.planetlab/vtest.pkgs @@ -0,0 +1,6 @@ +groupname: PlanetLabNative +groupdesc: Test Vserver for MyPlc Native +package: openssh-clients curl +package: emacs +package: bind-utils +package: file diff --git a/trunk/config.wextoolbox.svnpath b/trunk/config.wextoolbox.svnpath new file mode 100644 index 00000000..2cf5ff8e --- /dev/null +++ b/trunk/config.wextoolbox.svnpath @@ -0,0 +1,2 @@ +# make sure to invoke remote_pldistro appropriately in the Makefile +http://svn.onelab.eu/wextoolbox/trunk diff --git a/trunk/getdistro.sh b/trunk/getdistro.sh new file mode 100755 index 00000000..5cde6511 --- /dev/null +++ b/trunk/getdistro.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +. $(dirname $0)/build.common + +echo $pl_DISTRO + diff --git a/trunk/getdistroname.sh b/trunk/getdistroname.sh new file mode 100755 index 00000000..996cc1f2 --- /dev/null +++ b/trunk/getdistroname.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +. $(dirname $0)/build.common + +echo $pl_DISTRO_NAME + diff --git a/trunk/getkexcludes.sh b/trunk/getkexcludes.sh new file mode 100755 index 00000000..bcb025a1 --- /dev/null +++ b/trunk/getkexcludes.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +COMMAND=$(basename $0) +. $(dirname $0)/build.common + +function usage () { + echo "Usage: $COMMAND" + echo "outputs the list of packages to exclude from the stock repositories" + echo "this is set in build.common, and needs to fit the set of packages" + echo "that we override in the planetlab build" + exit 1 +} + +[[ -n "$@" ]] && usage + +echo "$pl_KEXCLUDES" diff --git a/trunk/getpackages.sh b/trunk/getpackages.sh new file mode 100755 index 00000000..7a385a31 --- /dev/null +++ b/trunk/getpackages.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +. build.common + +function usage () { + echo "Usage: $0 pldistro rhdistro lstfiles" + exit 1 +} + +#[ "$#" = 1 ] || usage +pldistro=$1; shift +rhdistro=$1; shift +lstfiles=$@ + +function pkgs_from_lst () { + builddir=$1; shift + pldistro=$1; shift + rhdistro=$1; shift + lstfiles=$@ + + for lstfile in $lstfiles; do + pkgsfile=$(pl_locateDistroFile $builddir $pldistro $lstfile 2> /dev/null) + packages=$(pl_getPackages $rhdistro $pldistro $pkgsfile) + echo $packages + done +} + +pkgs_from_lst $(dirname $0) $pldistro $rhdistro $lstfiles diff --git a/trunk/getrelease.sh b/trunk/getrelease.sh new file mode 100755 index 00000000..64ab5e64 --- /dev/null +++ b/trunk/getrelease.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +. $(dirname $0)/build.common + +echo $pl_DISTRO_RELEASE + diff --git a/trunk/getrpmmacros.sh b/trunk/getrpmmacros.sh new file mode 100755 index 00000000..e5f0fa73 --- /dev/null +++ b/trunk/getrpmmacros.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +. build.common + +pl_root_rpm_macros diff --git a/trunk/mirroring/centos5/yum.repos.d/building.repo.in b/trunk/mirroring/centos5/yum.repos.d/building.repo.in new file mode 100644 index 00000000..178d393f --- /dev/null +++ b/trunk/mirroring/centos5/yum.repos.d/building.repo.in @@ -0,0 +1,17 @@ +[base] +name=CentOS 5.4 - $basearch - Base +baseurl=@MIRRORURL@/centos/5.4/os/$basearch/ +gpgcheck=1 +gpgkey=http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-5 + +[updates] +name=CentOS 5.4 - $basearch - Released Updates +baseurl=@MIRRORURL@/centos/5.4/updates/$basearch/ +gpgcheck=1 +gpgkey=http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-5 + +[epel] +name=Extra Packages for Enterprise Linux 5 - $basearch +baseurl=@MIRRORURL@/epel/5/$basearch/ +gpgcheck=1 +gpgkey=http://download.fedora.redhat.com/pub/epel/RPM-GPG-KEY-EPEL diff --git a/trunk/mirroring/f10/yum.repos.d/building.repo.in b/trunk/mirroring/f10/yum.repos.d/building.repo.in new file mode 100644 index 00000000..8075ebe9 --- /dev/null +++ b/trunk/mirroring/f10/yum.repos.d/building.repo.in @@ -0,0 +1,15 @@ +[fedora] +name=Fedora 10 - $basearch +baseurl=@MIRRORURL@/fedora/releases/10/Everything/$basearch/os/ +enabled=1 +gpgcheck=0 +# where can we get this key ? +#gpgkey=http://redhat.download.fedoraproject.org/pub/fedora/linux/releases/8/Everything/$basearch/os/RPM-GPG-KEY-fedora + +[updates] +name=Fedora 10 - $basearch - Updates +baseurl=@MIRRORURL@/fedora/updates/10/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora + diff --git a/trunk/mirroring/f11/yum.repos.d/building.repo.in b/trunk/mirroring/f11/yum.repos.d/building.repo.in new file mode 100644 index 00000000..413e0f4e --- /dev/null +++ b/trunk/mirroring/f11/yum.repos.d/building.repo.in @@ -0,0 +1,17 @@ +[fedora] +name=Fedora 11 - $basearch +baseurl=@MIRRORURL@/fedora/releases/11/Everything/$basearch/os/ +enabled=1 +metadata_expire=7d +gpgcheck=0 +# where can we get this key ? +#gpgkey=http://redhat.download.fedoraproject.org/pub/fedora/linux/releases/8/Everything/$basearch/os/RPM-GPG-KEY-fedora + +[updates] +name=Fedora 11 - $basearch - Updates +baseurl=@MIRRORURL@/fedora/updates/11/$basearch/ +enabled=1 +metadata_expire=7d +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora + diff --git a/trunk/mirroring/f8/yum.repos.d/building.repo.in b/trunk/mirroring/f8/yum.repos.d/building.repo.in new file mode 100644 index 00000000..22527d31 --- /dev/null +++ b/trunk/mirroring/f8/yum.repos.d/building.repo.in @@ -0,0 +1,20 @@ +[fedora] +name=Fedora 8 - $basearch +baseurl=@MIRRORURL@/fedora/releases/8/Everything/$basearch/os/ +enabled=1 +gpgcheck=1 +gpgkey=http://build.planet-lab.org/fedora/releases/8/Everything/$basearch/os/RPM-GPG-KEY-fedora http://build.planet-lab.org/fedora/releases/8/Everything/$basearch/os/RPM-GPG-KEY + +[updates] +name=Fedora 8 - $basearch - Updates +baseurl=@MIRRORURL@/fedora/updates/8/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora + +[updates-newkey] +name=Fedora 8 - $basearch - Updates Newkey +baseurl=@MIRRORURL@/fedora/updates/8/$basearch.newkey/ +enabled=1 +gpgcheck=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-and-9 diff --git a/trunk/mirroring/mirror.sh b/trunk/mirroring/mirror.sh new file mode 100755 index 00000000..858da6c3 --- /dev/null +++ b/trunk/mirroring/mirror.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# $Id$ + +COMMAND=$(basename $0) +DIRNAME=$(dirname $0) + +default_url="http://localhost/mirror/" +default_distro="f8" +all_distros="f8 f10 f11 centos5" + +function check_distro () { + local distro=$1; shift + if [ ! -d $DIRNAME/$distro/yum.repos.d ] ; then + echo "Distro $distro not supported - skipped" + return 1 + fi + return 0 +} + +function do_repo () { + local distro=$1; shift + sedargs="-e s,@MIRRORURL@,$URL," + [ -n "$GPGOFF" ] && sedargs="$sedargs -e "'s,gpgcheck\W*=\W*1,gpgcheck=0,' + sed $sedargs $DIRNAME/$distro/yum.repos.d/building.repo.in +} + +function do_init () { + local distro=$1; shift + repo=/etc/vservers/.distributions/$distro/yum.repos.d/building.repo + dir=/etc/vservers/.distributions/$distro/yum.repos.d/ + if [ ! -d $dir ] ; then + [ -n "$VERBOSE" ] && echo Creating dir $dir + mkdir -p $dir + fi + [ -n "$VERBOSE" ] && echo "Creating $repo" + do_repo $distro > $repo +} + +function do_diff () { + local distro=$1; shift + repo=/etc/vservers/.distributions/$distro/yum.repos.d/building.repo + if [ ! -f $repo ] ; then + echo "Cannot find $repo" + else + would=/tmp/$COMMAND.$$ + do_repo $distro > $would + echo "==================== DIFF for $distro" '(current <-> would be)' + diff $repo $would + rm $would + fi +} + +function do_display () { + local distro=$1; shift + dir=/etc/vservers/.distributions/$distro/yum.repos.d/ + if [ -d $dir ] ; then + echo "====================" Contents of $dir + ls $dir/*.repo 2> /dev/null | xargs head --verbose --lines=1000 + else + echo "====================" $dir does not exist + fi +} + +function do_clean () { + local distro=$1; shift + repo=/etc/vservers/.distributions/$distro/yum.repos.d/building.repo + [ -n "$VERBOSE" ] && echo Removing $repo + rm $repo +} + +function do_superclean () { + local distro=$1; shift + dir=/etc/vservers/.distributions/$distro/yum.repos.d/ + [ -n "$VERBOSE" ] && echo Removing all repo files in $dir + rm $dir/*.repo +} + +function usage () { + echo "Usage $COMMAND [options] " + echo " a help to manage the yum.repos.d template in /etc/vservers/.distributions/" + echo "Available commands" + echo " display: shows content (default if is missing)" + echo " diff: shows diff between current and what init would do" + echo " init: creates /etc/vservers/.distributions//yum.repos.d/building.repo" + echo " clean: removes building.repo" + echo " superclean: removes yum.repos.d altogether" + echo "Options" + echo " -u URL to specify another location" + echo " default is to use mirror root at $default_url" + echo " -f : defaults to $default_distro" + echo " -a : runs on all distros $all_distros" + echo " -0 : turns off gpgcheck" + echo " -v : verbose" + echo "Examples" + echo " $COMMAND -a display " + echo " $COMMAND -a superclean" + echo " $COMMAND -a -u http://mirror.onelab.eu/ init" + echo " $COMMAND -a display" + exit 1 +} + +DISTROS="" +URL="" +VERBOSE="" +GPGOFF="" + +function main () { + + while getopts "u:f:a0v" opt; do + case $opt in + u) URL=$OPTARG ;; + f) DISTROS="$DISTROS $OPTARG" ;; + a) DISTROS="$DISTROS $all_distros" ;; + 0) GPGOFF=true ;; + v) VERBOSE=true ;; + *) usage ;; + esac + done + + shift $(($OPTIND - 1)) + + # no action = display + case "$#" in + 0) + action=display ;; + 1) + action=$1; shift + case $action in + disp*) action=display ;; + init*) action=init ;; + diff*) action=diff ;; + clea*) action=clean ;; + super*) action=superclean ;; + *) usage ;; + esac ;; + *) + usage ;; + esac + + [ -z "$URL" ] && URL=$default_url + [ -z "$DISTROS" ] && DISTROS="$default_distro" + + # remove trailing slash + URL=$(echo $URL | sed -e 's,/$,,') + + for distro in $DISTROS; do + [ -n "$VERBOSE" ] && echo ==================== Running $action for $distro + check_distro $distro && do_$action $distro + done + + exit 0 +} + +main "$@" diff --git a/trunk/module-branch b/trunk/module-branch new file mode 120000 index 00000000..323e9ad8 --- /dev/null +++ b/trunk/module-branch @@ -0,0 +1 @@ +module-tools.py \ No newline at end of file diff --git a/trunk/module-diff b/trunk/module-diff new file mode 120000 index 00000000..323e9ad8 --- /dev/null +++ b/trunk/module-diff @@ -0,0 +1 @@ +module-tools.py \ No newline at end of file diff --git a/trunk/module-list b/trunk/module-list new file mode 120000 index 00000000..323e9ad8 --- /dev/null +++ b/trunk/module-list @@ -0,0 +1 @@ +module-tools.py \ No newline at end of file diff --git a/trunk/module-log.py b/trunk/module-log.py new file mode 100755 index 00000000..2241460c --- /dev/null +++ b/trunk/module-log.py @@ -0,0 +1,196 @@ +#!/usr/bin/python +# -*- mode:python; var: python-guess-indent:false; python-indent:4; -*- + +import os +import sys +import re +from optparse import OptionParser + +modules_map = { + 'general' : [ 'build', 'tests', ], + 'server' : ['Monitor', 'MyPLC', 'PLCAPI', 'PLCRT', 'PLCWWW', 'PLEWWW', 'www-register-wizard', + 'PXEService', 'drupal', 'plcmdline',], + 'node': [ 'linux-2.6', 'util-vserver', 'util-vserver-pl', 'chopstix-L0', + 'BootCD', 'BootManager', 'BootstrapFS', 'VserverReference', + 'DistributedRateLimiting', 'Mom', 'PingOfDeath', + 'NodeManager', 'NodeManager-optin', 'NodeManager-topo', + 'NodeUpdate', 'CoDemux', + 'nodeconfig', 'pl_sshd', + 'libnl', 'pypcilib', 'pyplnet',], + 'wifi' : ['madwifi', 'PlanetBridge', 'hostapd',], + 'emulation': ['dummynet_image', 'ipfw', ], + 'netflow' : [ 'fprobe-ulog', 'pf2gui', 'pf2monitor', 'pf2slice', 'iproute2', 'iptables', 'silk',], + 'sfa' : [ 'sfa', 'xmlrspecs', 'pyopenssl', ], + 'vsys' : ['vsys', 'vsys-scripts', 'vsys-wrappers', 'inotify-tools'], + 'deprecated' : [ 'proper', 'libhttpd++', 'oombailout', 'ulogd', 'patchdep', 'pdelta', + 'sandbox', 'playground', 'infrastructure', 'util-python', 'vnetspec', + ], + } + +epoch='{2007-07-01}' + +class ModuleHistory: + + def __init__ (self, name, options): + self.name=name + self.options=options + self.user_commits=[] + self.user_revs={} + self.current_rev=None + self.current_user=None + + valid=re.compile('\Ar[0-9]+ \|') + tagging=re.compile('\ATagging|\ASetting tag') + + @staticmethod + def sort ( (u1,c1), (u2,c2) ): return c2-c1 + + def record(self,user,rev): + try: + self.user_revs[user].append(rev) + except: + self.user_revs[user] = [rev] + self.current_rev=rev + self.current_user=user + + def ignore(self): + if (not self.current_user) or (not self.current_rev): return + user_list=self.user_revs[self.current_user] + if len(user_list) >= 1 and user_list[-1] == self.current_rev: + user_list.pop() + + def scan (self): + cmd = "svn log -r %s:%s http://svn.planet-lab.org/svn/%s " % (self.options.fromv,self.options.tov,self.name) + if self.options.verbose: + print 'running',cmd + f = os.popen(cmd) + for line in f: + if not self.valid.match(line): + # mostly ignore commit body, except for ignoring the current commit if -i is set + if self.options.ignore_tags and self.tagging.match(line): + # roll back these changes + self.ignore() + continue + fields = line.split('|') + fields = [field.strip() for field in fields] + [rev,user,ctime,size] = fields[:4] + self.record(user,rev) + # translate into a list of tuples + user_commits = [ (user,len(revs)) for (user,revs) in self.user_revs.items() ] + user_commits.sort(self.sort) + self.user_commits=user_commits + + def show(self): + if len(self.user_commits) ==0: return + print '%s [%s-%s]'%(self.name,self.options.fromv,self.options.tov), + if self.options.ignore_tags: + print ' - Ignored tag commits' + else: + print '' + for (u,c) in self.user_commits: + print "\t",u,c + +class Aggregate: + + def __init__ (self,options): + # key=user, value=commits + self.options=options + self.user_commits_dict={} + self.user_commits=[] + + def merge (self, modulehistory): + for (u,c) in modulehistory.user_commits: + try: + self.user_commits_dict[u] += c + except: + self.user_commits_dict[u] = c + + def sort (self): + user_commits = [ (u,c) for (u,c) in self.user_commits_dict.items() ] + user_commits.sort(ModuleHistory.sort) + self.user_commits=user_commits + + def show(self): + print 'Overall', + if self.options.ignore_tags: + print ' - Ignored tag commits' + else: + print '' + for (u,c) in self.user_commits: + print "\t",u,c + +class Modules: + + def __init__ (self,map): + self.map=map + + def categories(self): + return self.map.keys() + + def all_modules(self,categories=None): + if not categories: categories=self.categories() + elif not isinstance(categories,list): categories=[categories] + result=[] + for category in categories: + result += self.map[category] + return result + + def locate (self,keywords): + result=[] + for kw in keywords: + if self.map.has_key(kw): + result += self.map[kw] + else: + result += [kw] + return result + + def list(self,scope): + for (cat,mod_list) in self.map.items(): + for mod in mod_list: + if mod in scope: + print cat,mod + +def main (): + usage="%prog [module_or_category ...]" + parser = OptionParser(usage=usage) + parser.add_option("-f", "--from", action = "store", dest='fromv', + default = epoch, help = "The revision to start from, default %s"%epoch) + parser.add_option("-t", "--to", action = "store", dest='tov', + default = 'HEAD', help = "The revision to end with, default HEAD") + parser.add_option("-n","--no-aggregate", action='store_false',dest='aggregate',default=True, + help='Do not aggregate over modules') + parser.add_option("-v","--verbose", action='store_true',dest='verbose',default=False, + help='Run in verbose/debug mode') + parser.add_option("-i","--ignore-tags",action='store_true',dest='ignore_tags', + help='ignore commits related to tagging') + parser.add_option("-l","--list",action='store_true',dest='list_modules', + help='list available modules and categories') + # pass this to the invoked shell if any + (options, args) = parser.parse_args() + + map=Modules(modules_map) + if not args: + modules=map.all_modules() + else: + modules=map.locate(args) + + if options.list_modules: + map.list(modules) + return + + if len(modules) <=1: + options.aggregate=False + + aggregate = Aggregate(options) + for module in modules: + history=ModuleHistory(module,options) + history.scan() + history.show() + aggregate.merge(history) + + if options.aggregate: + aggregate.sort() + aggregate.show() + +if __name__ == '__main__': + main() diff --git a/trunk/module-sync b/trunk/module-sync new file mode 120000 index 00000000..323e9ad8 --- /dev/null +++ b/trunk/module-sync @@ -0,0 +1 @@ +module-tools.py \ No newline at end of file diff --git a/trunk/module-tag b/trunk/module-tag new file mode 120000 index 00000000..323e9ad8 --- /dev/null +++ b/trunk/module-tag @@ -0,0 +1 @@ +module-tools.py \ No newline at end of file diff --git a/trunk/module-tools.py b/trunk/module-tools.py new file mode 100755 index 00000000..c35cd136 --- /dev/null +++ b/trunk/module-tools.py @@ -0,0 +1,1414 @@ +#!/usr/bin/python -u + +subversion_id = "$Id$" + +import sys, os, os.path +import re +import time +from glob import glob +from optparse import OptionParser + +# e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase +def prompt (question,default=True,other_choices=[],allow_outside=False): + if not isinstance (other_choices,list): + other_choices = [ other_choices ] + chars = [ c for (c,rest) in other_choices ] + + choices = [] + if 'y' not in chars: + if default is True: choices.append('[y]') + else : choices.append('y') + if 'n' not in chars: + if default is False: choices.append('[n]') + else : choices.append('n') + + for (char,choice) in other_choices: + if default == char: + choices.append("["+char+"]"+choice) + else: + choices.append("<"+char+">"+choice) + try: + answer=raw_input(question + " " + "/".join(choices) + " ? ") + if not answer: + return default + answer=answer[0].lower() + if answer == 'y': + if 'y' in chars: return 'y' + else: return True + elif answer == 'n': + if 'n' in chars: return 'n' + else: return False + elif other_choices: + for (char,choice) in other_choices: + if answer == char: + return char + if allow_outside: + return answer + return prompt(question,default,other_choices) + except: + raise + +def default_editor(): + try: + editor = os.environ['EDITOR'] + except: + editor = "emacs" + return editor + +### fold long lines +fold_length=132 + +def print_fold (line): + while len(line) >= fold_length: + print line[:fold_length],'\\' + line=line[fold_length:] + print line + +class Command: + def __init__ (self,command,options): + self.command=command + self.options=options + self.tmp="/tmp/command-%d"%os.getpid() + + def run (self): + if self.options.dry_run: + print 'dry_run',self.command + return 0 + if self.options.verbose and self.options.mode not in Main.silent_modes: + print '+',self.command + sys.stdout.flush() + return os.system(self.command) + + def run_silent (self): + if self.options.dry_run: + print 'dry_run',self.command + return 0 + if self.options.verbose: + print '+',self.command,' .. ', + sys.stdout.flush() + retcod=os.system(self.command + " &> " + self.tmp) + if retcod != 0: + print "FAILED ! -- out+err below (command was %s)"%self.command + os.system("cat " + self.tmp) + print "FAILED ! -- end of quoted output" + elif self.options.verbose: + print "OK" + os.unlink(self.tmp) + return retcod + + def run_fatal(self): + if self.run_silent() !=0: + raise Exception,"Command %s failed"%self.command + + # returns stdout, like bash's $(mycommand) + def output_of (self,with_stderr=False): + if self.options.dry_run: + print 'dry_run',self.command + return 'dry_run output' + tmp="/tmp/status-%d"%os.getpid() + if self.options.debug: + print '+',self.command,' .. ', + sys.stdout.flush() + command=self.command + if with_stderr: + command += " &> " + else: + command += " > " + command += tmp + os.system(command) + result=file(tmp).read() + os.unlink(tmp) + if self.options.debug: + print 'Done', + return result + +class Svnpath: + def __init__(self,path,options): + self.path=path + self.options=options + + def url_exists (self): + return os.system("svn list %s &> /dev/null"%self.path) == 0 + + def dir_needs_revert (self): + command="svn status %s"%self.path + return len(Command(command,self.options).output_of(True)) != 0 + # turns out it's the same implem. + def file_needs_commit (self): + command="svn status %s"%self.path + return len(Command(command,self.options).output_of(True)) != 0 + +# support for tagged module is minimal, and is for the Build class only +class Module: + + svn_magic_line="--This line, and those below, will be ignored--" + setting_tag_format = "Setting tag %s" + + redirectors=[ # ('module_name_varname','name'), + ('module_version_varname','version'), + ('module_taglevel_varname','taglevel'), ] + + # where to store user's config + config_storage="CONFIG" + # + config={} + + import commands + configKeys=[ ('svnpath',"Enter your toplevel svnpath", + "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")), + ("build", "Enter the name of your build module","build"), + ('username',"Enter your firstname and lastname for changelogs",""), + ("email","Enter your email address for changelogs",""), + ] + + @staticmethod + def prompt_config (): + for (key,message,default) in Module.configKeys: + Module.config[key]="" + while not Module.config[key]: + Module.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default + + + # for parsing module spec name:branch + matcher_branch_spec=re.compile("\A(?P[\w\.-]+):(?P[\w\.-]+)\Z") + # special form for tagged module - for Build + matcher_tag_spec=re.compile("\A(?P[\w-]+)@(?P[\w\.-]+)\Z") + # parsing specfiles + matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") + + def __init__ (self,module_spec,options): + # parse module spec + attempt=Module.matcher_branch_spec.match(module_spec) + if attempt: + self.name=attempt.group('name') + self.branch=attempt.group('branch') + else: + attempt=Module.matcher_tag_spec.match(module_spec) + if attempt: + self.name=attempt.group('name') + self.tagname=attempt.group('tagname') + else: + self.name=module_spec + + self.options=options + self.module_dir="%s/%s"%(options.workdir,self.name) + + def friendly_name (self): + if hasattr(self,'branch'): + return "%s:%s"%(self.name,self.branch) + elif hasattr(self,'tagname'): + return "%s@%s"%(self.name,self.tagname) + else: + return self.name + + def edge_dir (self): + if hasattr(self,'branch'): + return "%s/branches/%s"%(self.module_dir,self.branch) + elif hasattr(self,'tagname'): + return "%s/tags/%s"%(self.module_dir,self.tagname) + else: + return "%s/trunk"%(self.module_dir) + + def tags_dir (self): + return "%s/tags"%(self.module_dir) + + def run (self,command): + return Command(command,self.options).run() + def run_fatal (self,command): + return Command(command,self.options).run_fatal() + def run_prompt (self,message,command): + if not self.options.verbose: + while True: + choice=prompt(message,True,('s','how')) + if choice is True: + self.run(command) + return + elif choice is False: + return + else: + print 'About to run:',command + else: + question=message+" - want to run " + command + if prompt(question,True): + self.run(command) + + #################### + # store and restitute html fragments + @staticmethod + def html_href (url,text): return '%s'%(url,text) + @staticmethod + def html_anchor (url,text): return '%s'%(url,text) + # there must be some smarter means to do that - dirty for now + @staticmethod + def html_quote (text): + return text.replace('&','&').replace('<','<').replace('>','>') + + # only the fake error module has multiple titles + def html_store_title (self, title): + if not hasattr(self,'titles'): self.titles=[] + self.titles.append(title) + def html_store_raw (self, html): + if not hasattr(self,'body'): self.body='' + self.body += html + def html_store_pre (self, text): + if not hasattr(self,'body'): self.body='' + self.body += '
' + self.html_quote(text) + '
' + + def html_print (self, txt): + if not self.options.www: + print txt + else: + if not hasattr(self,'in_list') or not self.in_list: + self.html_store_raw('
    ') + self.in_list=True + self.html_store_raw('
  • '+txt+'
    • ') + def html_print_end (self): + if self.options.www: + self.html_store_raw ('
') + + @staticmethod + def html_dump_header(title): + nowdate=time.strftime("%Y-%m-%d") + nowtime=time.strftime("%H:%M") + print """ + + + + %s + + + + +

%s - status on %s at %s

+
    +"""%(title,title,nowdate,nowtime) + + @staticmethod + def html_dump_middle(): + print "
" + + @staticmethod + def html_dump_footer(): + print "',self.html_href ('#'+self.friendly_name(),title),'' + + def html_dump_body(self): + if hasattr(self,'titles'): + for title in self.titles: + print '

',self.html_anchor(self.friendly_name(),title),'

' + if hasattr(self,'body'): + print self.body + print '

',self.html_href('#','Back to top'),'

' + + #################### + @staticmethod + def init_homedir (options): + topdir=options.workdir + if options.verbose and options.mode not in Main.silent_modes: + print 'Checking for',topdir + storage="%s/%s"%(topdir,Module.config_storage) + # sanity check. Either the topdir exists AND we have a config/storage + # or topdir does not exist and we create it + # to avoid people use their own daily svn repo + if os.path.isdir(topdir) and not os.path.isfile(storage): + print """The directory %s exists and has no CONFIG file +If this is your regular working directory, please provide another one as the +module-* commands need a fresh working dir. Make sure that you do not use +that for other purposes than tagging"""%topdir + sys.exit(1) + if not os.path.isdir (topdir): + print "Cannot find",topdir,"let's create it" + Module.prompt_config() + print "Checking ...", + Command("svn co -N %s %s"%(Module.config['svnpath'],topdir),options).run_fatal() + Command("svn co -N %s/%s %s/%s"%(Module.config['svnpath'], + Module.config['build'], + topdir, + Module.config['build']),options).run_fatal() + print "OK" + + # store config + f=file(storage,"w") + for (key,message,default) in Module.configKeys: + f.write("%s=%s\n"%(key,Module.config[key])) + f.close() + if options.debug: + print 'Stored',storage + Command("cat %s"%storage,options).run() + else: + # read config + f=open(storage) + for line in f.readlines(): + (key,value)=re.compile("^(.+)=(.+)$").match(line).groups() + Module.config[key]=value + f.close() + if options.verbose and options.mode not in Main.silent_modes: + print '******** Using config' + for (key,message,default) in Module.configKeys: + print '\t',key,'=',Module.config[key] + + def init_module_dir (self): + if self.options.verbose: + print 'Checking for',self.module_dir + if not os.path.isdir (self.module_dir): + self.run_fatal("svn update -N %s"%self.module_dir) + if not os.path.isdir (self.module_dir): + raise Exception, 'Cannot find %s - check module name'%self.module_dir + + def init_subdir (self,fullpath): + if self.options.verbose: + print 'Checking for',fullpath + if not os.path.isdir (fullpath): + self.run_fatal("svn update -N %s"%fullpath) + + def revert_subdir (self,fullpath): + if self.options.fast_checks: + if self.options.verbose: print 'Skipping revert of %s'%fullpath + return + if self.options.verbose: + print 'Checking whether',fullpath,'needs being reverted' + if Svnpath(fullpath,self.options).dir_needs_revert(): + self.run_fatal("svn revert -R %s"%fullpath) + + def update_subdir (self,fullpath): + if self.options.fast_checks: + if self.options.verbose: print 'Skipping update of %s'%fullpath + return + if self.options.verbose: + print 'Updating',fullpath + self.run_fatal("svn update -N %s"%fullpath) + + def init_edge_dir (self): + # if branch, edge_dir is two steps down + if hasattr(self,'branch'): + self.init_subdir("%s/branches"%self.module_dir) + elif hasattr(self,'tagname'): + self.init_subdir("%s/tags"%self.module_dir) + self.init_subdir(self.edge_dir()) + + def revert_edge_dir (self): + self.revert_subdir(self.edge_dir()) + + def update_edge_dir (self): + self.update_subdir(self.edge_dir()) + + def main_specname (self): + attempt="%s/%s.spec"%(self.edge_dir(),self.name) + if os.path.isfile (attempt): + return attempt + else: + pattern="%s/*.spec"%self.edge_dir() + try: + return glob(pattern)[0] + except: + raise Exception, 'Cannot guess specfile for module %s -- pattern was %s'%(self.name,pattern) + + def all_specnames (self): + return glob("%s/*.spec"%self.edge_dir()) + + def parse_spec (self, specfile, varnames): + if self.options.verbose: + print 'Parsing',specfile, + for var in varnames: + print "[%s]"%var, + print "" + result={} + f=open(specfile) + for line in f.readlines(): + attempt=Module.matcher_rpm_define.match(line) + if attempt: + (define,var,value)=attempt.groups() + if var in varnames: + result[var]=value + f.close() + if self.options.debug: + print 'found',len(result),'keys' + for (k,v) in result.iteritems(): + print k,'=',v + return result + + # stores in self.module_name_varname the rpm variable to be used for the module's name + # and the list of these names in self.varnames + def spec_dict (self): + specfile=self.main_specname() + redirector_keys = [ varname for (varname,default) in Module.redirectors] + redirect_dict = self.parse_spec(specfile,redirector_keys) + if self.options.debug: + print '1st pass parsing done, redirect_dict=',redirect_dict + varnames=[] + for (varname,default) in Module.redirectors: + if redirect_dict.has_key(varname): + setattr(self,varname,redirect_dict[varname]) + varnames += [redirect_dict[varname]] + else: + setattr(self,varname,default) + varnames += [ default ] + self.varnames = varnames + result = self.parse_spec (specfile,self.varnames) + if self.options.debug: + print '2st pass parsing done, varnames=',varnames,'result=',result + return result + + def patch_spec_var (self, patch_dict,define_missing=False): + for specfile in self.all_specnames(): + # record the keys that were changed + changed = dict ( [ (x,False) for x in patch_dict.keys() ] ) + newspecfile=specfile+".new" + if self.options.verbose: + print 'Patching',specfile,'for',patch_dict.keys() + spec=open (specfile) + new=open(newspecfile,"w") + + for line in spec.readlines(): + attempt=Module.matcher_rpm_define.match(line) + if attempt: + (define,var,value)=attempt.groups() + if var in patch_dict.keys(): + if self.options.debug: + print 'rewriting %s as %s'%(var,patch_dict[var]) + new.write('%%%s %s %s\n'%(define,var,patch_dict[var])) + changed[var]=True + continue + new.write(line) + if define_missing: + for (key,was_changed) in changed.iteritems(): + if not was_changed: + if self.options.debug: + print 'rewriting missing %s as %s'%(key,patch_dict[key]) + new.write('\n%%define %s %s\n'%(key,patch_dict[key])) + spec.close() + new.close() + os.rename(newspecfile,specfile) + + # returns all lines until the magic line + def unignored_lines (self, logfile): + result=[] + white_line_matcher = re.compile("\A\s*\Z") + for logline in file(logfile).readlines(): + if logline.strip() == Module.svn_magic_line: + break + elif white_line_matcher.match(logline): + continue + else: + result.append(logline.strip()+'\n') + return result + + # creates a copy of the input with only the unignored lines + def stripped_magic_line_filename (self, filein, fileout ,new_tag_name): + f=file(fileout,'w') + f.write(self.setting_tag_format%new_tag_name + '\n') + for line in self.unignored_lines(filein): + f.write(line) + f.close() + + def insert_changelog (self, logfile, oldtag, newtag): + for specfile in self.all_specnames(): + newspecfile=specfile+".new" + if self.options.verbose: + print 'Inserting changelog from %s into %s'%(logfile,specfile) + spec=open (specfile) + new=open(newspecfile,"w") + for line in spec.readlines(): + new.write(line) + if re.compile('%changelog').match(line): + dateformat="* %a %b %d %Y" + datepart=time.strftime(dateformat) + logpart="%s <%s> - %s"%(Module.config['username'], + Module.config['email'], + newtag) + new.write(datepart+" "+logpart+"\n") + for logline in self.unignored_lines(logfile): + new.write("- " + logline) + new.write("\n") + spec.close() + new.close() + os.rename(newspecfile,specfile) + + def show_dict (self, spec_dict): + if self.options.verbose: + for (k,v) in spec_dict.iteritems(): + print k,'=',v + + def mod_url (self): + return "%s/%s"%(Module.config['svnpath'],self.name) + + def edge_url (self): + if hasattr(self,'branch'): + return "%s/branches/%s"%(self.mod_url(),self.branch) + elif hasattr(self,'tagname'): + return "%s/tags/%s"%(self.mod_url(),self.tagname) + else: + return "%s/trunk"%(self.mod_url()) + + def last_tag (self, spec_dict): + return "%s-%s"%(spec_dict[self.module_version_varname],spec_dict[self.module_taglevel_varname]) + + def tag_name (self, spec_dict): + try: + return "%s-%s"%(self.name, + self.last_tag(spec_dict)) + except KeyError,err: + raise Exception, 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err) + + def tag_url (self, spec_dict): + return "%s/tags/%s"%(self.mod_url(),self.tag_name(spec_dict)) + + def check_svnpath_exists (self, url, message): + if self.options.fast_checks: + return + if self.options.verbose: + print 'Checking url (%s) %s'%(url,message), + ok=Svnpath(url,self.options).url_exists() + if ok: + if self.options.verbose: print 'exists - OK' + else: + if self.options.verbose: print 'KO' + raise Exception, 'Could not find %s URL %s'%(message,url) + + def check_svnpath_not_exists (self, url, message): + if self.options.fast_checks: + return + if self.options.verbose: + print 'Checking url (%s) %s'%(url,message), + ok=not Svnpath(url,self.options).url_exists() + if ok: + if self.options.verbose: print 'does not exist - OK' + else: + if self.options.verbose: print 'KO' + raise Exception, '%s URL %s already exists - exiting'%(message,url) + + # locate specfile, parse it, check it and show values + +############################## + def do_version (self): + self.init_module_dir() + self.init_edge_dir() + self.revert_edge_dir() + self.update_edge_dir() + spec_dict = self.spec_dict() + if self.options.www: + self.html_store_title('Version for module %s (%s)' % (self.friendly_name(),self.last_tag(spec_dict))) + for varname in self.varnames: + if not spec_dict.has_key(varname): + self.html_print ('Could not find %%define for %s'%varname) + return + else: + self.html_print ("%-16s %s"%(varname,spec_dict[varname])) + if self.options.show_urls: + self.html_print ("%-16s %s"%('edge url',self.edge_url())) + self.html_print ("%-16s %s"%('latest tag url',self.tag_url(spec_dict))) + if self.options.verbose: + self.html_print ("%-16s %s"%('main specfile:',self.main_specname())) + self.html_print ("%-16s %s"%('specfiles:',self.all_specnames())) + self.html_print_end() + +############################## + def do_list (self): +# print 'verbose',self.options.verbose +# print 'list_tags',self.options.list_tags +# print 'list_branches',self.options.list_branches +# print 'all_modules',self.options.all_modules + + (verbose,branches,pattern,exact) = (self.options.verbose,self.options.list_branches, + self.options.list_pattern,self.options.list_exact) + + extra_command="" + extra_message="" + if hasattr(self,'branch'): + pattern=self.branch + if pattern or exact: + if exact: + if verbose: grep="%s/$"%exact + else: grep="^%s$"%exact + else: + grep=pattern + extra_command=" | grep %s"%grep + extra_message=" matching %s"%grep + + if not branches: + message="==================== tags for %s"%self.friendly_name() + command="svn list " + if verbose: command+="--verbose " + command += "%s/tags"%self.mod_url() + command += extra_command + message += extra_message + if verbose: print message + self.run(command) + + else: + message="==================== branches for %s"%self.friendly_name() + command="svn list " + if verbose: command+="--verbose " + command += "%s/branches"%self.mod_url() + command += extra_command + message += extra_message + if verbose: print message + self.run(command) + +############################## + sync_warning="""*** WARNING +The module-sync function has the following limitations +* it does not handle changelogs +* it does not scan the -tags*.mk files to adopt the new tags""" + + def do_sync(self): + if self.options.verbose: + print Module.sync_warning + if not prompt('Want to proceed anyway'): + return + + self.init_module_dir() + self.init_edge_dir() + self.revert_edge_dir() + self.update_edge_dir() + spec_dict = self.spec_dict() + + edge_url=self.edge_url() + tag_name=self.tag_name(spec_dict) + tag_url=self.tag_url(spec_dict) + # check the tag does not exist yet + self.check_svnpath_not_exists(tag_url,"new tag") + + if self.options.message: + svnopt='--message "%s"'%self.options.message + else: + svnopt='--editor-cmd=%s'%self.options.editor + self.run_prompt("Create initial tag", + "svn copy %s %s %s"%(svnopt,edge_url,tag_url)) + +############################## + def do_diff (self,compute_only=False): + self.init_module_dir() + self.init_edge_dir() + self.revert_edge_dir() + self.update_edge_dir() + spec_dict = self.spec_dict() + self.show_dict(spec_dict) + + edge_url=self.edge_url() + tag_url=self.tag_url(spec_dict) + self.check_svnpath_exists(edge_url,"edge track") + self.check_svnpath_exists(tag_url,"latest tag") + command="svn diff %s %s"%(tag_url,edge_url) + if compute_only: + if self.options.verbose: + print 'Getting diff with %s'%command + diff_output = Command(command,self.options).output_of() + # if used as a utility + if compute_only: + return (spec_dict,edge_url,tag_url,diff_output) + # otherwise print the result + if self.options.list: + if diff_output: + print self.name + else: + thename=self.friendly_name() + do_print=False + if self.options.www and diff_output: + self.html_store_title("Diffs in module %s (%s) : %d chars"%(\ + thename,self.last_tag(spec_dict),len(diff_output))) + link=self.html_href(tag_url,tag_url) + self.html_store_raw ('

< (left) %s

'%link) + link=self.html_href(edge_url,edge_url) + self.html_store_raw ('

> (right) %s

'%link) + self.html_store_pre (diff_output) + elif not self.options.www: + print 'x'*30,'module',thename + print 'x'*20,'<',tag_url + print 'x'*20,'>',edge_url + print diff_output + +############################## + # using fine_grain means replacing only those instances that currently refer to this tag + # otherwise, -SVNPATH is replaced unconditionnally + def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True): + newtagsfile=tagsfile+".new" + tags=open (tagsfile) + new=open(newtagsfile,"w") + + matches=0 + # fine-grain : replace those lines that refer to oldname + if fine_grain: + if self.options.verbose: + print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile), + matcher=re.compile("^(.*)%s(.*)"%oldname) + for line in tags.readlines(): + if not matcher.match(line): + new.write(line) + else: + (begin,end)=matcher.match(line).groups() + new.write(begin+newname+end+"\n") + matches += 1 + # brute-force : change uncommented lines that define -SVNPATH + else: + if self.options.verbose: + print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile), + pattern="\A\s*(?P[^\s]+)-SVNPATH\s*(=|:=)\s*(?P[^\s]+)/%s/[^\s]+"\ + %(self.name) + matcher_module=re.compile(pattern) + for line in tags.readlines(): + attempt=matcher_module.match(line) + if attempt: + svnpath="%s-SVNPATH"%(attempt.group('make_name')) + if self.options.verbose: + print ' '+svnpath, + replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname) + new.write(replacement) + matches += 1 + else: + new.write(line) + tags.close() + new.close() + os.rename(newtagsfile,tagsfile) + if self.options.verbose: print "%d changes"%matches + return matches + + def do_tag (self): + self.init_module_dir() + self.init_edge_dir() + self.revert_edge_dir() + self.update_edge_dir() + # parse specfile + spec_dict = self.spec_dict() + self.show_dict(spec_dict) + + # side effects + edge_url=self.edge_url() + old_tag_name = self.tag_name(spec_dict) + old_tag_url=self.tag_url(spec_dict) + if (self.options.new_version): + # new version set on command line + spec_dict[self.module_version_varname] = self.options.new_version + spec_dict[self.module_taglevel_varname] = 0 + else: + # increment taglevel + new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1) + spec_dict[self.module_taglevel_varname] = new_taglevel + + # sanity check + new_tag_name = self.tag_name(spec_dict) + new_tag_url=self.tag_url(spec_dict) + self.check_svnpath_exists (edge_url,"edge track") + self.check_svnpath_exists (old_tag_url,"previous tag") + self.check_svnpath_not_exists (new_tag_url,"new tag") + + # checking for diffs + diff_output=Command("svn diff %s %s"%(old_tag_url,edge_url), + self.options).output_of() + if len(diff_output) == 0: + if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False): + return + + # side effect in trunk's specfile + self.patch_spec_var(spec_dict) + + # prepare changelog file + # we use the standard subversion magic string (see svn_magic_line) + # so we can provide useful information, such as version numbers and diff + # in the same file + changelog="/tmp/%s-%d.edit"%(self.name,os.getpid()) + changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid()) + setting_tag_line=Module.setting_tag_format%new_tag_name + file(changelog,"w").write(""" +%s +%s +Please write a changelog for this new tag in the section above +"""%(Module.svn_magic_line,setting_tag_line)) + + if not self.options.verbose or prompt('Want to see diffs while writing changelog',True): + file(changelog,"a").write('DIFF=========\n' + diff_output) + + if self.options.debug: + prompt('Proceed ?') + + # edit it + self.run("%s %s"%(self.options.editor,changelog)) + # strip magic line in second file - looks like svn has changed its magic line with 1.6 + # so we do the job ourselves + self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name) + # insert changelog in spec + if self.options.changelog: + self.insert_changelog (changelog,old_tag_name,new_tag_name) + + ## update build + try: + buildname=Module.config['build'] + except: + buildname="build" + if self.options.build_branch: + buildname+=":"+self.options.build_branch + build = Module(buildname,self.options) + build.init_module_dir() + build.init_edge_dir() + build.revert_edge_dir() + build.update_edge_dir() + + tagsfiles=glob(build.edge_dir()+"/*-tags*.mk") + tagsdict=dict( [ (x,'todo') for x in tagsfiles ] ) + default_answer = 'y' + while True: + for (tagsfile,status) in tagsdict.iteritems(): + basename=os.path.basename(tagsfile) + print ".................... Dealing with %s"%basename + while tagsdict[tagsfile] == 'todo' : + choice = prompt ("insert %s in %s "%(new_tag_name,basename),default_answer, + [ ('y','es'), ('n', 'ext'), ('f','orce'), + ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] , + allow_outside=True) + if choice == 'y': + self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True) + elif choice == 'n': + print 'Done with %s'%os.path.basename(tagsfile) + tagsdict[tagsfile]='done' + elif choice == 'f': + self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False) + elif choice == 'd': + self.run("svn diff %s"%tagsfile) + elif choice == 'r': + self.run("svn revert %s"%tagsfile) + elif choice == 'c': + self.run("cat %s"%tagsfile) + else: + name=self.name + print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s +f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s +d: show current diff for this tag file +r: revert that tag file +c: cat the current tag file +n: move to next file"""%locals() + + if prompt("Want to review changes on tags files",False): + tagsdict = dict ( [ (x, 'todo') for tagsfile in tagsfiles ] ) + default_answer='d' + else: + break + + paths="" + paths += self.edge_dir() + " " + paths += build.edge_dir() + " " + self.run_prompt("Review module and build","svn diff " + paths) + self.run_prompt("Commit module and build","svn commit --file %s %s"%(changelog_svn,paths)) + self.run_prompt("Create tag","svn copy --file %s %s %s"%(changelog_svn,edge_url,new_tag_url)) + + if self.options.debug: + print 'Preserving',changelog,'and stripped',changelog_svn + else: + os.unlink(changelog) + os.unlink(changelog_svn) + +############################## + def do_branch (self): + + # save self.branch if any, as a hint for the new branch + # do this before anything else and restore .branch to None, + # as this is part of the class's logic + new_trunk_name=None + if hasattr(self,'branch'): + new_trunk_name=self.branch + del self.branch + elif self.options.new_version: + new_trunk_name = self.options.new_version + + # compute diff - a way to initialize the whole stuff + # do_diff already does edge_dir initialization + # and it checks that edge_url and tag_url exist as well + (spec_dict,edge_url,tag_url,diff_listing) = self.do_diff(compute_only=True) + + # the version name in the trunk becomes the new branch name + branch_name = spec_dict[self.module_version_varname] + + # figure new branch name (the one for the trunk) if not provided on the command line + if not new_trunk_name: + # heuristic is to assume 'version' is a dot-separated name + # we isolate the rightmost part and try incrementing it by 1 + version=spec_dict[self.module_version_varname] + try: + m=re.compile("\A(?P.+)\.(?P[^\.]+)\Z") + (leftpart,rightmost)=m.match(version).groups() + incremented = int(rightmost)+1 + new_trunk_name="%s.%d"%(leftpart,incremented) + except: + raise Exception, 'Cannot figure next branch name from %s - exiting'%version + + # record starting point tagname + latest_tag_name = self.tag_name(spec_dict) + + print "**********" + print "Using starting point %s (%s)"%(tag_url,latest_tag_name) + print "Creating branch %s & moving trunk to %s"%(branch_name,new_trunk_name) + print "**********" + + # print warning if pending diffs + if diff_listing: + print """*** WARNING : Module %s has pending diffs on its trunk +It is safe to proceed, but please note that branch %s +will be based on latest tag %s and *not* on the current trunk"""%(self.name,branch_name,latest_tag_name) + while True: + answer = prompt ('Are you sure you want to proceed with branching',True,('d','iff')) + if answer is True: + break + elif answer is False: + raise Exception,"User quit" + elif answer == 'd': + print '<<<< %s'%tag_url + print '>>>> %s'%edge_url + print diff_listing + + branch_url = "%s/%s/branches/%s"%(Module.config['svnpath'],self.name,branch_name) + self.check_svnpath_not_exists (branch_url,"new branch") + + # patching trunk + spec_dict[self.module_version_varname]=new_trunk_name + spec_dict[self.module_taglevel_varname]='0' + # remember this in the trunk for easy location of the current branch + spec_dict['module_current_branch']=branch_name + self.patch_spec_var(spec_dict,True) + + # create commit log file + tmp="/tmp/branching-%d"%os.getpid() + f=open(tmp,"w") + f.write("Branch %s for module %s created (as new trunk) from tag %s\n"%(new_trunk_name,self.name,latest_tag_name)) + f.close() + + # we're done, let's commit the stuff + command="svn diff %s"%self.edge_dir() + self.run_prompt("Review changes in trunk",command) + command="svn copy --file %s %s %s"%(tmp,self.edge_url(),branch_url) + self.run_prompt("Create branch",command) + command="svn commit --file %s %s"%(tmp,self.edge_dir()) + self.run_prompt("Commit trunk",command) + new_tag_url=self.tag_url(spec_dict) + command="svn copy --file %s %s %s"%(tmp,self.edge_url(),new_tag_url) + self.run_prompt("Create initial tag in trunk",command) + os.unlink(tmp) + +############################## +class Package: + + def __init__(self, package, module, svnpath, spec): + self.package=package + self.module=module + self.svnpath=svnpath + self.spec=spec + self.specpath="%s/%s"%(svnpath,spec) + self.basename=os.path.basename(svnpath) + + # returns a http URL to the trac path where full diff can be viewed (between self and pkg) + # typically http://svn.planet-lab.org/changeset?old_path=Monitor%2Ftags%2FMonitor-1.0-7&new_path=Monitor%2Ftags%2FMonitor-1.0-13 + # xxx quick & dirty: rough url parsing + def trac_full_diff (self, pkg): + matcher=re.compile("\A(?P.*)://(?P[^/]+)/(svn/)?(?P.*)\Z") + self_match=matcher.match(self.svnpath) + pkg_match=matcher.match(pkg.svnpath) + if self_match and pkg_match: + (method,hostname,svn,path)=self_match.groups() + self_path=path.replace("/","%2F") + pkg_path=pkg_match.group('path').replace("/","%2F") + return "%s://%s/changeset?old_path=%s&new_path=%s"%(method,hostname,self_path,pkg_path) + else: + return None + + def details (self): + return "[%s %s] [%s (spec)]"%(self.svnpath,self.basename,self.specpath) + +class Build (Module): + + # we cannot get build's svnpath as for other packages as we'd get something in svn+ssh + # xxx quick & dirty + def __init__ (self, buildtag,options): + self.buildtag=buildtag + # if the buildtag start with a : (to use a branch rather than a tag) + if buildtag.find(':') == 0 : + module_name="build%(buildtag)s"%locals() + self.display=buildtag[1:] + self.svnpath="http://svn.planet-lab.org/svn/build/branches/%s"%self.display + else : + module_name="build@%(buildtag)s"%locals() + self.display=buildtag + self.svnpath="http://svn.planet-lab.org/svn/build/tags/%s"%self.buildtag + Module.__init__(self,module_name,options) + + @staticmethod + def get_distro_from_distrotag (distrotag): + # mhh: remove -tag* from distrotags to get distro + n=distrotag.find('-tag') + if n>0: + return distrotag[:n] + else: + return None + + def get_packages (self,distrotag): + result={} + distro=Build.get_distro_from_distrotag(distrotag) + if not distro: + return result + make_options="--no-print-directory -C %s stage1=true PLDISTRO=%s PLDISTROTAGS=%s 2> /dev/null"%(self.edge_dir(),distro,distrotag) + command="make %s packages"%make_options + make_packages=Command(command,self.options).output_of() + pkg_line=re.compile("\Apackage=(?P[^\s]+)\s+ref_module=(?P[^\s]+)\s.*\Z") + for line in make_packages.split("\n"): + if not line: + continue + attempt=pkg_line.match(line) + if line and not attempt: + print "=====" + print "WARNING: line not understood from make packages" + print "in dir %s"%self.edge_dir + print "with options",make_options + print 'line=',line + print "=====" + else: + (package,module) = (attempt.group('package'),attempt.group('module')) + command="make %s +%s-SVNPATH"%(make_options,module) + svnpath=Command(command,self.options).output_of().strip() + command="make %s +%s-SPEC"%(make_options,package) + spec=Command(command,self.options).output_of().strip() + result[package]=Package(package,module,svnpath,spec) + return result + + def get_distrotags (self): + return [os.path.basename(p) for p in glob("%s/*tags*mk"%self.edge_dir())] + +class DiffCache: + + def __init__ (self): + self._cache={} + + def key(self, frompath,topath): + return frompath+'-to-'+topath + + def fetch (self, frompath, topath): + key=self.key(frompath,topath) + if not self._cache.has_key(key): + return None + return self._cache[key] + + def store (self, frompath, topath, diff): + key=self.key(frompath,topath) + self._cache[key]=diff + +class Release: + + # header in diff output + discard_matcher=re.compile("\A(\+\+\+|---).*") + + @staticmethod + def do_changelog (buildtag_new,buildtag_old,options): + print "----" + print "----" + print "----" + (build_new,build_old) = (Build (buildtag_new,options), Build (buildtag_old,options)) + print "= build tag %s to %s = #build-%s"%(build_old.display,build_new.display,build_new.display) + for b in (build_new,build_old): + b.init_module_dir() + b.init_edge_dir() + b.update_edge_dir() + # find out the tags files that are common, unless option was specified + if options.distrotags: + distrotags=options.distrotags + else: + distrotags_new=build_new.get_distrotags() + distrotags_old=build_old.get_distrotags() + distrotags = list(set(distrotags_new).intersection(set(distrotags_old))) + distrotags.sort() + if options.verbose: print "Found distrotags",distrotags + first_distrotag=True + diffcache = DiffCache() + for distrotag in distrotags: + distro=Build.get_distro_from_distrotag(distrotag) + if not distro: + continue + if first_distrotag: + first_distrotag=False + else: + print '----' + print '== distro %s (%s to %s) == #distro-%s-%s'%(distrotag,build_old.display,build_new.display,distro,build_new.display) + print ' * from %s/%s'%(build_old.svnpath,distrotag) + print ' * to %s/%s'%(build_new.svnpath,distrotag) + + # parse make packages + packages_new=build_new.get_packages(distrotag) + pnames_new=set(packages_new.keys()) + if options.verbose: print 'got packages for ',build_new.display + packages_old=build_old.get_packages(distrotag) + pnames_old=set(packages_old.keys()) + if options.verbose: print 'got packages for ',build_old.display + + # get created, deprecated, and preserved package names + pnames_created = list(pnames_new-pnames_old) + pnames_created.sort() + pnames_deprecated = list(pnames_old-pnames_new) + pnames_deprecated.sort() + pnames = list(pnames_new.intersection(pnames_old)) + pnames.sort() + + if options.verbose: print "Found new/deprecated/preserved pnames",pnames_new,pnames_deprecated,pnames + + # display created and deprecated + for name in pnames_created: + print '=== %s : new package %s -- appeared in %s === #package-%s-%s-%s'%( + distrotag,name,build_new.display,name,distro,build_new.display) + pobj=packages_new[name] + print ' * %s'%pobj.details() + for name in pnames_deprecated: + print '=== %s : package %s -- deprecated, last occurrence in %s === #package-%s-%s-%s'%( + distrotag,name,build_old.display,name,distro,build_new.display) + pobj=packages_old[name] + if not pobj.svnpath: + print ' * codebase stored in CVS, specfile is %s'%pobj.spec + else: + print ' * %s'%pobj.details() + + # display other packages + for name in pnames: + (pobj_new,pobj_old)=(packages_new[name],packages_old[name]) + if options.verbose: print "Dealing with package",name + if pobj_old.specpath == pobj_new.specpath: + continue + specdiff = diffcache.fetch(pobj_old.specpath,pobj_new.specpath) + if specdiff is None: + command="svn diff %s %s"%(pobj_old.specpath,pobj_new.specpath) + specdiff=Command(command,options).output_of() + diffcache.store(pobj_old.specpath,pobj_new.specpath,specdiff) + else: + if options.verbose: print 'got diff from cache' + if not specdiff: + continue + print '=== %s - %s to %s : package %s === #package-%s-%s-%s'%( + distrotag,build_old.display,build_new.display,name,name,distro,build_new.display) + print ' * from %s to %s'%(pobj_old.details(),pobj_new.details()) + trac_diff_url=pobj_old.trac_full_diff(pobj_new) + if trac_diff_url: + print ' * [%s View full diff]'%trac_diff_url + print '{{{' + for line in specdiff.split('\n'): + if not line: + continue + if Release.discard_matcher.match(line): + continue + if line[0] in ['@']: + print '----------' + elif line[0] in ['+','-']: + print_fold(line) + print '}}}' + +############################## +class Main: + + module_usage="""Usage: %prog [options] module_desc [ .. module_desc ] +module-tools : a set of tools to manage subversion tags and specfile + requires the specfile to either + * define *version* and *taglevel* + OR alternatively + * define redirection variables module_version_varname / module_taglevel_varname +Trunk: + by default, the trunk of modules is taken into account + in this case, just mention the module name as +Branches: + if you wish to work on a branch rather than on the trunk, + you can use something like e.g. Mom:2.1 as +""" + release_usage="""Usage: %prog [options] tag1 .. tagn + Extract release notes from the changes in specfiles between several build tags, latest first + Examples: + release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22 + You can refer to a (build) branch by prepending a colon, like in + release-changelog :4.2 4.2-rc25 +""" + common_usage="""More help: + see http://svn.planet-lab.org/wiki/ModuleTools""" + + modes={ + 'list' : "displays a list of available tags or branches", + 'version' : "check latest specfile and print out details", + 'diff' : "show difference between module (trunk or branch) and latest tag", + 'tag' : """increment taglevel in specfile, insert changelog in specfile, + create new tag and and monitor its adoption in build/*-tags*.mk""", + 'branch' : """create a branch for this module, from the latest tag on the trunk, + and change trunk's version number to reflect the new branch name; + you can specify the new branch name by using module:branch""", + 'sync' : """create a tag from the module + this is a last resort option, mostly for repairs""", + 'changelog' : """extract changelog between build tags + expected arguments are a list of tags""", + } + + silent_modes = ['list'] + release_modes = ['changelog'] + + @staticmethod + def optparse_list (option, opt, value, parser): + try: + setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split()) + except: + setattr(parser.values,option.dest,value.split()) + + def run(self): + + mode=None + for function in Main.modes.keys(): + if sys.argv[0].find(function) >= 0: + mode = function + break + if not mode: + print "Unsupported command",sys.argv[0] + print "Supported commands:" + Modes.modes.keys.join(" ") + sys.exit(1) + + if mode not in Main.release_modes: + usage = Main.module_usage + usage += Main.common_usage + usage += "\nmodule-%s : %s"%(mode,Main.modes[mode]) + else: + usage = Main.release_usage + usage += Main.common_usage + + parser=OptionParser(usage=usage,version=subversion_id) + + if mode == 'list': + parser.add_option("-b","--branches",action="store_true",dest="list_branches",default=False, + help="list branches") + parser.add_option("-t","--tags",action="store_false",dest="list_branches", + help="list tags") + parser.add_option("-m","--match",action="store",dest="list_pattern",default=None, + help="grep pattern for filtering output") + parser.add_option("-x","--exact-match",action="store",dest="list_exact",default=None, + help="exact grep pattern for filtering output") + if mode == "tag" or mode == 'branch': + parser.add_option("-s","--set-version",action="store",dest="new_version",default=None, + help="set new version and reset taglevel to 0") + if mode == "tag" : + parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True, + help="do not update changelog section in specfile when tagging") + parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None, + help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place") + if mode == "tag" or mode == "sync" : + parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(), + help="specify editor") + if mode == "sync" : + parser.add_option("-m","--message", action="store", dest="message", default=None, + help="specify log message") + if mode in ["diff","version"] : + parser.add_option("-W","--www", action="store", dest="www", default=False, + help="export diff in html format, e.g. -W trunk") + if mode == "diff" : + parser.add_option("-l","--list", action="store_true", dest="list", default=False, + help="just list modules that exhibit differences") + + if mode == 'version': + parser.add_option("-u","--url", action="store_true", dest="show_urls", default=False, + help="display URLs") + + default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list" + if mode not in Main.release_modes: + parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, + help="run on all modules as found in %s"%default_modules_list) + parser.add_option("-f","--file",action="store",dest="modules_list",default=None, + help="run on all modules found in specified file") + else: + parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False, + help="dry run - shell commands are only displayed") + parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags", + default=[], nargs=1,type="string", + help="""specify distro-tags files, e.g. onelab-tags-4.2.mk +-- can be set multiple times, or use quotes""") + + parser.add_option("-w","--workdir", action="store", dest="workdir", + default="%s/%s"%(os.getenv("HOME"),"modules"), + help="""name for dedicated working dir - defaults to ~/modules +** THIS MUST NOT ** be your usual working directory""") + parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False, + help="skip safety checks, such as svn updates -- use with care") + + # default verbosity depending on function - temp + verbose_modes= ['tag','sync'] + + if mode not in verbose_modes: + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + help="run in verbose mode") + else: + parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True, + help="run in quiet (non-verbose) mode") +# parser.add_option("-d","--debug", action="store_true", dest="debug", default=False, +# help="debug mode - mostly more verbose") + (options, args) = parser.parse_args() + options.mode=mode + if not hasattr(options,'dry_run'): + options.dry_run=False + if not hasattr(options,'www'): + options.www=False + options.debug=False + + ########## release-* + if mode in Main.release_modes : + ########## changelog + if len(args) <= 1: + parser.print_help() + sys.exit(1) + Module.init_homedir(options) + for n in range(len(args)-1): + [t_new,t_old]=args[n:n+2] + Release.do_changelog (t_new,t_old,options) + else: + ########## module-* + if len(args) == 0: + if options.all_modules: + options.modules_list=default_modules_list + if options.modules_list: + args=Command("grep -v '#' %s"%options.modules_list,options).output_of().split() + else: + parser.print_help() + sys.exit(1) + Module.init_homedir(options) + + # 2 passes for www output + modules=[ Module(modname,options) for modname in args ] + # hack: create a dummy Module to store errors/warnings + error_module = Module('__errors__',options) + + # pass 1 : do it, except if options.www + for module in modules: + if len(args)>1 and mode not in Main.silent_modes and not options.www: + print '========================================',module.friendly_name() + # call the method called do_ + method=Module.__dict__["do_%s"%mode] + try: + method(module) + except Exception,e: + if options.www: + title=' Skipping module %s - failure: %s '%\ + (module.friendly_name(), str(e)) + error_module.html_store_title(title) + else: + print 'Skipping module %s: '%modname,e + + # in which case we do the actual printing in the second pass + if options.www: + if mode == "diff": + modetitle="Changes to tag in %s"%options.www + elif mode == "version": + modetitle="Latest tags in %s"%options.www + modules.append(error_module) + error_module.html_dump_header(modetitle) + for module in modules: + module.html_dump_toc() + Module.html_dump_middle() + for module in modules: + module.html_dump_body() + Module.html_dump_footer() + +#################### +if __name__ == "__main__" : + try: + Main().run() + except KeyboardInterrupt: + print '\nBye' diff --git a/trunk/module-version b/trunk/module-version new file mode 120000 index 00000000..323e9ad8 --- /dev/null +++ b/trunk/module-version @@ -0,0 +1 @@ +module-tools.py \ No newline at end of file diff --git a/trunk/modules.list b/trunk/modules.list new file mode 100644 index 00000000..ec46e366 --- /dev/null +++ b/trunk/modules.list @@ -0,0 +1,44 @@ +BootCD +BootManager +BootstrapFS +CoDemux +Mom +Monitor +MyPLC +NodeManager +NodeUpdate +PLCAPI +PLCRT +PLEWWW +PingOfDeath +VserverReference +comgt +drupal +dummynet_image +fprobe-ulog +inotify-tools +ipfw +iproute2 +iptables +iptables:1.3.8 +libnl +linux-2.6 +linux-2.6:22 +madwifi +madwifi:0.9.4 +nodeconfig +nozomi +pf2slice +pl_sshd +planetlab-umts-tools +pyopenssl +pypcilib +pyplnet +sfa +util-vserver-pl +util-vserver-pl:trellis +util-vserver:scholz +util-vserver:trellis +vsys-scripts +vsys:0.9 +www-register-wizard diff --git a/trunk/modules.update b/trunk/modules.update new file mode 100755 index 00000000..8630c487 --- /dev/null +++ b/trunk/modules.update @@ -0,0 +1,13 @@ +#!/bin/bash + +# redirect this into modules.list to get it refreshed +# + +# keep centos5 as the default target rather than - more accurate - f8 b/c we need more of our modules on centos +fcdistro=centos5 + +( \ + make stage1=true PLDISTRO=planetlab PLDISTROTAGS=planetlab-tags.mk DISTRONAME=${fcdistro} module-tools ; \ + make stage1=true PLDISTRO=planetlab PLDISTROTAGS=planetlab-k27-tags.mk DISTRONAME=${fcdistro} module-tools ; \ + make stage1=true PLDISTRO=onelab PLDISTROTAGS=onelab-tags.mk DISTRONAME=${fcdistro} module-tools ; \ +) | sort -u | grep -v '^build$' diff --git a/trunk/onelab-k27-tags.mk b/trunk/onelab-k27-tags.mk new file mode 100644 index 00000000..81f935ee --- /dev/null +++ b/trunk/onelab-k27-tags.mk @@ -0,0 +1,50 @@ +# $Id: onelab-tags.mk 15622 2009-11-03 21:19:17Z mef $ + +# build-SVNPATH is now set by vbuild-nightly.sh to avoid duplication + +linux-2.6-SVNBRANCH := 22 +# not adopting tag 40 about "VXC_PROC_WRITE support" - should be safe but Sapan advised keeping it out was safer still +linux-2.6-SVNPATH := http://svn.planet-lab.org/svn/linux-2.6/tags/linux-2.6-27-3 +ipfw-SVNPATH := http://svn.planet-lab.org/svn/ipfw/tags/ipfw-0.9-4 +madwifi-SVNPATH := http://svn.planet-lab.org/svn/madwifi/tags/madwifi-3878-0 +nozomi-SVNPATH := http://svn.onelab.eu/nozomi/tags/nozomi-2.21-1 +comgt-SVNPATH := http://svn.onelab.eu/comgt/imports/0.3 +planetlab-umts-tools-SVNPATH := http://svn.onelab.eu/planetlab-umts-tools/tags/planetlab-umts-tools-0.6-4 +util-vserver-SVNBRANCH := scholz +util-vserver-SVNPATH := http://svn.planet-lab.org/svn/util-vserver/tags/util-vserver-0.30.215-6 +libnl-SVNPATH := http://svn.planet-lab.org/svn/libnl/tags/libnl-1.1-2 +util-vserver-pl-SVNPATH := http://svn.planet-lab.org/svn/util-vserver-pl/tags/util-vserver-pl-0.3-19 +NodeUpdate-SVNPATH := http://svn.planet-lab.org/svn/NodeUpdate/tags/NodeUpdate-0.5-5 +PingOfDeath-SVNPATH := http://svn.planet-lab.org/svn/PingOfDeath/tags/PingOfDeath-2.2-1 +NodeManager-SVNPATH := http://svn.planet-lab.org/svn/NodeManager/tags/NodeManager-1.8-20 +pyplnet-SVNPATH := http://svn.planet-lab.org/svn/pyplnet/tags/pyplnet-4.3-4 +pl_sshd-SVNPATH := http://svn.planet-lab.org/svn/pl_sshd/tags/pl_sshd-1.0-11 +CoDemux-SVNPATH := http://svn.planet-lab.org/svn/CoDemux/tags/CoDemux-0.1-13 +fprobe-ulog-SVNPATH := http://svn.planet-lab.org/svn/fprobe-ulog/tags/fprobe-ulog-1.1.3-0 +pf2slice-SVNPATH := http://svn.planet-lab.org/svn/pf2slice/tags/pf2slice-1.0-2 +Mom-SVNPATH := http://svn.planet-lab.org/svn/Mom/tags/Mom-2.3-1 +iptables-SVNPATH := http://svn.planet-lab.org/svn//iptables/tags/iptables-1.4.1.1-1 +iproute2-SVNPATH := http://svn.planet-lab.org/svn/iproute2/tags/iproute2-2.6.16-2 +inotify-tools-SVNPATH := http://svn.planet-lab.org/svn/inotify-tools/tags/inotify-tools-3.13-2 +vsys-SVNBRANCH := 0.9 +vsys-SVNPATH := http://svn.planet-lab.org/svn/vsys/tags/vsys-0.9-3 +vsys-scripts-SVNPATH := http://svn.planet-lab.org/svn/vsys-scripts/tags/vsys-scripts-0.95-11 +dummynet_image-SVNPATH := http://svn.planet-lab.org/svn/dummynet_image/tags/dummynet_image-1.0-5 +PLCAPI-SVNPATH := http://svn.planet-lab.org/svn/PLCAPI/tags/PLCAPI-4.3-30 +drupal-SVNPATH := http://svn.planet-lab.org/svn/drupal/tags/drupal-4.7-13 +PLEWWW-SVNPATH := http://svn.planet-lab.org/svn/PLEWWW/tags/PLEWWW-4.3-33 +www-register-wizard-SVNPATH := http://svn.planet-lab.org/svn/www-register-wizard/tags/www-register-wizard-4.3-1 +Monitor-SVNPATH := http://svn.planet-lab.org/svn//Monitor/tags/Monitor-3.0-24 +nodeconfig-SVNPATH := http://svn.planet-lab.org/svn/nodeconfig/tags/nodeconfig-4.3-5 +BootManager-SVNPATH := http://svn.planet-lab.org/svn/BootManager/tags/BootManager-4.3-13 +pypcilib-SVNPATH := http://svn.planet-lab.org/svn/pypcilib/tags/pypcilib-0.2-8 +BootCD-SVNPATH := http://svn.planet-lab.org/svn/BootCD/tags/BootCD-4.2-15 +VserverReference-SVNPATH := http://svn.planet-lab.org/svn/VserverReference/tags/VserverReference-4.2-16 +BootstrapFS-SVNPATH := http://svn.planet-lab.org/svn/BootstrapFS/tags/BootstrapFS-1.0-10 +MyPLC-SVNPATH := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-4.3-29 +sfa-SVNPATH := http://svn.planet-lab.org/svn/sfa/tags/sfa-0.9-5 +pyopenssl-SVNPATH := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9-1 +PLCRT-SVNPATH := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-11 + +# locating the right test directory - see make tests_svnpath +tests-SVNPATH := http://svn.planet-lab.org/svn/tests/tags/tests-4.3-5 diff --git a/trunk/onelab-tags.mk b/trunk/onelab-tags.mk new file mode 100644 index 00000000..c3103f25 --- /dev/null +++ b/trunk/onelab-tags.mk @@ -0,0 +1,52 @@ +# $Id$ + +# build-SVNPATH is now set by vbuild-nightly.sh to avoid duplication + +linux-2.6-SVNBRANCH := 22 +# not adopting tag 40 about "VXC_PROC_WRITE support" - should be safe but Sapan advised keeping it out was safer still +linux-2.6-SVNPATH := http://svn.planet-lab.org/svn/linux-2.6/tags/linux-2.6-22-39-1 +ipfw-SVNPATH := http://svn.planet-lab.org/svn/ipfw/tags/ipfw-0.9-4 +madwifi-SVNBRANCH := 0.9.4 +madwifi-SVNPATH := http://svn.planet-lab.org/svn/madwifi/tags/madwifi-0.9.4-3 +nozomi-SVNPATH := http://svn.onelab.eu/nozomi/tags/nozomi-2.21-1 +comgt-SVNPATH := http://svn.onelab.eu/comgt/imports/0.3 +planetlab-umts-tools-SVNPATH := http://svn.onelab.eu/planetlab-umts-tools/tags/planetlab-umts-tools-0.6-4 +util-vserver-SVNBRANCH := scholz +util-vserver-SVNPATH := http://svn.planet-lab.org/svn/util-vserver/tags/util-vserver-0.30.215-6 +libnl-SVNPATH := http://svn.planet-lab.org/svn/libnl/tags/libnl-1.1-2 +util-vserver-pl-SVNPATH := http://svn.planet-lab.org/svn/util-vserver-pl/tags/util-vserver-pl-0.3-19 +NodeUpdate-SVNPATH := http://svn.planet-lab.org/svn/NodeUpdate/tags/NodeUpdate-0.5-5 +PingOfDeath-SVNPATH := http://svn.planet-lab.org/svn/PingOfDeath/tags/PingOfDeath-2.2-1 +NodeManager-SVNPATH := http://svn.planet-lab.org/svn/NodeManager/tags/NodeManager-1.8-20 +pyplnet-SVNPATH := http://svn.planet-lab.org/svn/pyplnet/tags/pyplnet-4.3-4 +pl_sshd-SVNPATH := http://svn.planet-lab.org/svn/pl_sshd/tags/pl_sshd-1.0-11 +CoDemux-SVNPATH := http://svn.planet-lab.org/svn/CoDemux/tags/CoDemux-0.1-13 +fprobe-ulog-SVNPATH := http://svn.planet-lab.org/svn/fprobe-ulog/tags/fprobe-ulog-1.1.3-0 +pf2slice-SVNPATH := http://svn.planet-lab.org/svn/pf2slice/tags/pf2slice-1.0-2 +Mom-SVNPATH := http://svn.planet-lab.org/svn/Mom/tags/Mom-2.3-1 +iptables-SVNBRANCH := 1.3.8 +iptables-SVNPATH := http://svn.planet-lab.org/svn/iptables/tags/iptables-1.3.8-9 +iproute2-SVNPATH := http://svn.planet-lab.org/svn/iproute2/tags/iproute2-2.6.16-2 +inotify-tools-SVNPATH := http://svn.planet-lab.org/svn/inotify-tools/tags/inotify-tools-3.13-2 +vsys-SVNBRANCH := 0.9 +vsys-SVNPATH := http://svn.planet-lab.org/svn/vsys/tags/vsys-0.9-3 +vsys-scripts-SVNPATH := http://svn.planet-lab.org/svn/vsys-scripts/tags/vsys-scripts-0.95-11 +dummynet_image-SVNPATH := http://svn.planet-lab.org/svn/dummynet_image/tags/dummynet_image-1.0-5 +PLCAPI-SVNPATH := http://svn.planet-lab.org/svn/PLCAPI/tags/PLCAPI-4.3-30 +drupal-SVNPATH := http://svn.planet-lab.org/svn/drupal/tags/drupal-4.7-13 +PLEWWW-SVNPATH := http://svn.planet-lab.org/svn/PLEWWW/tags/PLEWWW-4.3-33 +www-register-wizard-SVNPATH := http://svn.planet-lab.org/svn/www-register-wizard/tags/www-register-wizard-4.3-1 +Monitor-SVNPATH := http://svn.planet-lab.org/svn//Monitor/tags/Monitor-3.0-24 +nodeconfig-SVNPATH := http://svn.planet-lab.org/svn/nodeconfig/tags/nodeconfig-4.3-5 +BootManager-SVNPATH := http://svn.planet-lab.org/svn/BootManager/tags/BootManager-4.3-13 +pypcilib-SVNPATH := http://svn.planet-lab.org/svn/pypcilib/tags/pypcilib-0.2-8 +BootCD-SVNPATH := http://svn.planet-lab.org/svn/BootCD/tags/BootCD-4.2-15 +VserverReference-SVNPATH := http://svn.planet-lab.org/svn/VserverReference/tags/VserverReference-4.2-16 +BootstrapFS-SVNPATH := http://svn.planet-lab.org/svn/BootstrapFS/tags/BootstrapFS-1.0-10 +MyPLC-SVNPATH := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-4.3-29 +sfa-SVNPATH := http://svn.planet-lab.org/svn/sfa/tags/sfa-0.9-5 +pyopenssl-SVNPATH := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9-1 +PLCRT-SVNPATH := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-11 + +# locating the right test directory - see make tests_svnpath +tests-SVNPATH := http://svn.planet-lab.org/svn/tests/tags/tests-4.3-5 diff --git a/trunk/onelab.mk b/trunk/onelab.mk new file mode 100644 index 00000000..a7c46813 --- /dev/null +++ b/trunk/onelab.mk @@ -0,0 +1,461 @@ +# +# declare the packages to be built and their dependencies +# initial version from Mark Huang +# Mark Huang +# Copyright (C) 2003-2006 The Trustees of Princeton University +# rewritten by Thierry Parmentelat - INRIA Sophia Antipolis +# +# $Id$ +# +# see doc in Makefile +# + +# +# kernel +# +# use a package name with srpm in it: +# so the source rpm is created by running make srpm in the codebase +# + +kernel-MODULES := linux-2.6 +kernel-SPEC := kernel-2.6.spec +kernel-BUILD-FROM-SRPM := yes +ifeq "$(HOSTARCH)" "i386" +kernel-RPMFLAGS:= --target i686 +else +kernel-RPMFLAGS:= --target $(HOSTARCH) +endif +# this is useful for 2.6.22 but will not be needed anymore with 2.6.27 +kernel-SPECVARS := iwlwifi=1 +KERNELS += kernel + +kernels: $(KERNELS) +kernels-clean: $(foreach package,$(KERNELS),$(package)-clean) + +ALL += $(KERNELS) +# this is to mark on which image a given rpm is supposed to go +IN_BOOTCD += $(KERNELS) +IN_VSERVER += $(KERNELS) +IN_BOOTSTRAPFS += $(KERNELS) +# turns out myplc installs kernel-vserver +IN_MYPLC += $(KERNELS) + +# +# ipfw: root context module, and slice companion +# +ipfw-MODULES := ipfw +ipfw-SPEC := ipfw.spec +ipfw-DEPEND-DEVEL-RPMS := kernel-devel +ipfw-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +IN_BOOTSTRAPFS += ipfw +ALL += ipfw + +ipfwslice-MODULES := ipfw +ipfwslice-SPEC := ipfw-slice.spec +ipfwslice-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +IN_VSERVER += ipfwslice +ALL += ipfwslice + +# +# madwifi +# +madwifi-MODULES := madwifi +madwifi-SPEC := madwifi.spec +madwifi-BUILD-FROM-SRPM := yes +madwifi-DEPEND-DEVEL-RPMS := kernel-devel +madwifi-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +ALL += madwifi +IN_BOOTSTRAPFS += madwifi + +# +# nozomi +# +nozomi-MODULES := nozomi +nozomi-SPEC := nozomi.spec +nozomi-DEPEND-DEVEL-RPMS := kernel-devel +nozomi-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +IN_BOOTSTRAPFS += nozomi +ALL += nozomi + +# +# comgt +# +comgt-MODULES := comgt +comgt-SPEC := comgt.spec +IN_BOOTSTRAPFS += comgt +ALL += comgt + +# +# umts: root context stuff +# +umts-backend-MODULES := planetlab-umts-tools +umts-backend-SPEC := backend.spec +IN_BOOTSTRAPFS += umts-backend +ALL += umts-backend + +# +# umts: slice tools +# +umts-frontend-MODULES := planetlab-umts-tools +umts-frontend-SPEC := frontend.spec +IN_VSERVER += umts-frontend +ALL += umts-frontend + +# +# util-vserver +# +util-vserver-MODULES := util-vserver +util-vserver-SPEC := util-vserver.spec +util-vserver-RPMFLAGS:= --without dietlibc +ALL += util-vserver +IN_BOOTSTRAPFS += util-vserver + +# +# libnl - local import +# we need either 1.1 or at least 1.0.pre6 +# rebuild this on centos5 - see kexcludes in build.common +# +local_libnl=false +ifeq "$(DISTRONAME)" "centos5" +local_libnl=true +endif + +ifeq "$(local_libnl)" "true" +libnl-MODULES := libnl +libnl-SPEC := libnl.spec +libnl-BUILD-FROM-SRPM := yes +# this sounds like the thing to do, but in fact linux/if_vlan.h comes with kernel-headers +libnl-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += libnl +IN_BOOTSTRAPFS += libnl +endif + +# +# util-vserver-pl +# +util-vserver-pl-MODULES := util-vserver-pl +util-vserver-pl-SPEC := util-vserver-pl.spec +util-vserver-pl-DEPEND-DEVEL-RPMS := util-vserver-lib util-vserver-devel util-vserver-core +ifeq "$(local_libnl)" "true" +util-vserver-pl-DEPEND-DEVEL-RPMS += libnl libnl-devel +endif +ALL += util-vserver-pl +IN_BOOTSTRAPFS += util-vserver-pl + +# +# NodeUpdate +# +nodeupdate-MODULES := NodeUpdate +nodeupdate-SPEC := NodeUpdate.spec +ALL += nodeupdate +IN_BOOTSTRAPFS += nodeupdate + +# +# ipod +# +ipod-MODULES := PingOfDeath +ipod-SPEC := ipod.spec +ALL += ipod +IN_BOOTSTRAPFS += ipod + +# +# NodeManager +# +nodemanager-MODULES := NodeManager +nodemanager-SPEC := NodeManager.spec +ALL += nodemanager +IN_BOOTSTRAPFS += nodemanager + +# +# pl_sshd +# +sshd-MODULES := pl_sshd +sshd-SPEC := pl_sshd.spec +ALL += sshd +IN_BOOTSTRAPFS += sshd + +# +# codemux: Port 80 demux +# +codemux-MODULES := CoDemux +codemux-SPEC := codemux.spec +codemux-RPMBUILD := sudo bash ./rpmbuild.sh +ALL += codemux +IN_BOOTSTRAPFS += codemux + +# +# fprobe-ulog +# +fprobe-ulog-MODULES := fprobe-ulog +fprobe-ulog-SPEC := fprobe-ulog.spec +ALL += fprobe-ulog +IN_BOOTSTRAPFS += fprobe-ulog + +# +# pf2slice +# +pf2slice-MODULES := pf2slice +pf2slice-SPEC := pf2slice.spec +ALL += pf2slice + +# +# PlanetLab Mom: Cleans up your mess +# +mom-MODULES := Mom +mom-SPEC := pl_mom.spec +ALL += mom +IN_BOOTSTRAPFS += mom + +# +# iptables +# +iptables-MODULES := iptables +iptables-SPEC := iptables.spec +iptables-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += iptables +IN_BOOTSTRAPFS += iptables + +# +# iproute +# +iproute-MODULES := iproute2 +iproute-SPEC := iproute.spec +ALL += iproute +IN_BOOTSTRAPFS += iproute +IN_VSERVER += iproute +IN_BOOTCD += iproute + +# +# inotify-tools - local import +# rebuild this on centos5 (not found) - see kexcludes in build.common +# +local_inotify_tools=false +ifeq "$(DISTRONAME)" "centos5" +local_inotify_tools=true +endif + +ifeq "$(local_inotify_tools)" "true" +inotify-tools-MODULES := inotify-tools +inotify-tools-SPEC := inotify-tools.spec +inotify-tools-BUILD-FROM-SRPM := yes +IN_BOOTSTRAPFS += inotify-tools +ALL += inotify-tools +endif + +# +# vsys +# +vsys-MODULES := vsys +vsys-SPEC := vsys.spec +ifeq "$(local_inotify_tools)" "true" +vsys-DEPEND-DEVEL-RPMS := inotify-tools inotify-tools-devel +endif +IN_BOOTSTRAPFS += vsys +ALL += vsys + +# +# vsys-scripts +# +vsys-scripts-MODULES := vsys-scripts +vsys-scripts-SPEC := vsys-scripts.spec +IN_BOOTSTRAPFS += vsys-scripts +ALL += vsys-scripts + +# +# dummynet_image +# +dummynet_image-MODULES := dummynet_image +dummynet_image-SPEC := dummynet_image.spec +IN_MYPLC += dummynet_image +ALL += dummynet_image + +# +# PLCAPI +# +PLCAPI-MODULES := PLCAPI +PLCAPI-SPEC := PLCAPI.spec +ALL += PLCAPI +IN_MYPLC += PLCAPI + +# +# drupal +# +drupal-MODULES := drupal +drupal-SPEC := drupal.spec +drupal-BUILD-FROM-SRPM := yes +ALL += drupal +IN_MYPLC += drupal + +# +# use the plewww module instead +# +plewww-MODULES := PLEWWW +plewww-SPEC := plewww.spec +ALL += plewww +IN_MYPLC += plewww + +# +# www-register-wizard +# +www-register-wizard-MODULES := www-register-wizard +www-register-wizard-SPEC := www-register-wizard.spec +ALL += www-register-wizard +IN_MYPLC += www-register-wizard + +# +# monitor +# +monitor-MODULES := Monitor +monitor-SPEC := Monitor.spec +ALL += monitor +IN_BOOTSTRAPFS += monitor + +# +# PLC RT +# +plcrt-MODULES := PLCRT +plcrt-SPEC := plcrt.spec +ALL += plcrt + +# +# zabbix +# +zabbix-MODULES := Monitor +zabbix-SPEC := zabbix.spec +zabbix-BUILD-FROM-SRPM := yes +ALL += zabbix + +# +# pyopenssl +# +pyopenssl-MODULES := pyopenssl +pyopenssl-SPEC := pyOpenSSL.spec +pyopenssl-BUILD-FROM-SRPM := yes +ALL += pyopenssl + + +# +# sfa - Slice Facility Architecture +# +sfa-MODULES := sfa +sfa-SPEC := sfa.spec +ALL += sfa + +# +# nodeconfig +# +nodeconfig-MODULES := nodeconfig build +nodeconfig-SPEC := nodeconfig.spec +ALL += nodeconfig +IN_MYPLC += nodeconfig + +# +# bootmanager +# +bootmanager-MODULES := BootManager +bootmanager-SPEC := bootmanager.spec +ALL += bootmanager +IN_MYPLC += bootmanager + +# +# pypcilib : used in bootcd +# +pypcilib-MODULES := pypcilib +pypcilib-SPEC := pypcilib.spec +ALL += pypcilib +IN_BOOTCD += pypcilib + +# +# pyplnet +# +pyplnet-MODULES := pyplnet +pyplnet-SPEC := pyplnet.spec +ALL += pyplnet +IN_BOOTSTRAPFS += pyplnet +IN_MYPLC += pyplnet +IN_BOOTCD += pyplnet + +# +# bootcd +# +bootcd-MODULES := BootCD build +bootcd-SPEC := bootcd.spec +bootcd-RPMBUILD := sudo bash ./rpmbuild.sh +bootcd-DEPEND-PACKAGES := $(IN_BOOTCD) +bootcd-DEPEND-FILES := RPMS/yumgroups.xml +bootcd-RPMDATE := yes +ALL += bootcd +IN_MYPLC += bootcd + +# +# vserver : reference image for slices +# +vserver-MODULES := VserverReference build +vserver-SPEC := vserver-reference.spec +vserver-DEPEND-PACKAGES := $(IN_VSERVER) +vserver-DEPEND-FILES := RPMS/yumgroups.xml +vserver-RPMDATE := yes +ALL += vserver +IN_BOOTSTRAPFS += vserver + +# +# bootstrapfs +# +bootstrapfs-MODULES := BootstrapFS build +bootstrapfs-SPEC := bootstrapfs.spec +bootstrapfs-RPMBUILD := sudo bash ./rpmbuild.sh +bootstrapfs-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) +bootstrapfs-DEPEND-FILES := RPMS/yumgroups.xml +bootstrapfs-RPMDATE := yes +ALL += bootstrapfs +IN_MYPLC += bootstrapfs + +# +# noderepo +# +# all rpms resulting from packages marked as being in bootstrapfs and vserver +NODEREPO_RPMS = $(foreach package,$(IN_BOOTSTRAPFS) $(IN_VSERVER),$($(package).rpms)) +# replace space with +++ (specvars cannot deal with spaces) +SPACE=$(subst x, ,x) +NODEREPO_RPMS_3PLUS = $(subst $(SPACE),+++,$(NODEREPO_RPMS)) + +noderepo-MODULES := BootstrapFS +noderepo-SPEC := noderepo.spec +noderepo-RPMBUILD := sudo bash ./rpmbuild.sh +# package requires all regular packages +noderepo-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) $(IN_VSERVER) +noderepo-DEPEND-FILES := RPMS/yumgroups.xml +#export rpm list to the specfile +noderepo-SPECVARS = node_rpms_plus=$(NODEREPO_RPMS_3PLUS) +noderepo-RPMDATE := yes +ALL += noderepo +IN_MYPLC += noderepo + +# +# MyPLC : lightweight packaging, dependencies are yum-installed in a vserver +# +myplc-MODULES := MyPLC build +myplc-SPEC := myplc.spec +myplc-DEPEND-FILES := myplc-release RPMS/yumgroups.xml +ALL += myplc + +# myplc-docs only contains docs for PLCAPI and NMAPI, but +# we still need to pull MyPLC, as it is where the specfile lies, +# together with the utility script docbook2drupal.sh +myplc-docs-MODULES := MyPLC PLCAPI NodeManager Monitor +myplc-docs-SPEC := myplc-docs.spec +ALL += myplc-docs + +# using some other name than myplc-release, as this is a make target already +release-MODULES := MyPLC +release-SPEC := myplc-release.spec +release-RPMDATE := yes +ALL += release diff --git a/trunk/planetbridge-tags.mk b/trunk/planetbridge-tags.mk new file mode 100644 index 00000000..6fa76afa --- /dev/null +++ b/trunk/planetbridge-tags.mk @@ -0,0 +1,46 @@ +# $Id$ + +# build-SVNPATH is now set by vbuild-nightly.sh to avoid duplication + +linux-2.6-SVNBRANCH := 22 +linux-2.6-SVNPATH := http://svn.planet-lab.org/svn/linux-2.6/tags/linux-2.6-22-39 +madwifi-SVNBRANCH := 0.9.4 +madwifi-SVNPATH := http://svn.planet-lab.org/svn/madwifi/tags/madwifi-0.9.4-3 +util-vserver-SVNBRANCH := scholz +util-vserver-SVNPATH := http://svn.planet-lab.org/svn/util-vserver/tags/util-vserver-0.30.215-6 +libnl-SVNPATH := http://svn.planet-lab.org/svn/libnl/tags/libnl-1.1-2 +util-vserver-pl-SVNPATH := http://svn.planet-lab.org/svn/util-vserver-pl/tags/util-vserver-pl-0.3-16 +NodeUpdate-SVNPATH := http://svn.planet-lab.org/svn/NodeUpdate/tags/NodeUpdate-0.5-5 +PingOfDeath-SVNPATH := http://svn.planet-lab.org/svn/PingOfDeath/tags/PingOfDeath-2.2-1 +NodeManager-SVNPATH := http://svn.planet-lab.org/svn/NodeManager/tags/NodeManager-1.8-7 +pyplnet-SVNPATH := http://svn.planet-lab.org/svn/pyplnet/tags/pyplnet-4.3-4 +pl_sshd-SVNPATH := http://svn.planet-lab.org/svn/pl_sshd/tags/pl_sshd-1.0-11 +CoDemux-SVNPATH := http://svn.planet-lab.org/svn/CoDemux/tags/CoDemux-0.1-13 +fprobe-ulog-SVNPATH := http://svn.planet-lab.org/svn/fprobe-ulog/tags/fprobe-ulog-1.1.3-0 +pf2slice-SVNPATH := http://svn.planet-lab.org/svn/pf2slice/tags/pf2slice-1.0-2 +Mom-SVNPATH := http://svn.planet-lab.org/svn/Mom/tags/Mom-2.3-1 +iptables-SVNBRANCH := 1.3.8 +iptables-SVNPATH := http://svn.planet-lab.org/svn/iptables/tags/iptables-1.3.8-9 +iproute2-SVNPATH := http://svn.planet-lab.org/svn/iproute2/tags/iproute2-2.6.16-2 +inotify-tools-SVNPATH := http://svn.planet-lab.org/svn/inotify-tools/tags/inotify-tools-3.13-2 +vsys-SVNBRANCH := 0.9 +vsys-SVNPATH := http://svn.planet-lab.org/svn/vsys/tags/vsys-0.9-3 +vsys-scripts-SVNPATH := http://svn.planet-lab.org/svn/vsys-scripts/tags/vsys-scripts-0.95-11 +PLCAPI-SVNPATH := http://svn.planet-lab.org/svn/PLCAPI/tags/PLCAPI-4.3-30 +drupal-SVNPATH := http://svn.planet-lab.org/svn/drupal/tags/drupal-4.7-13 +PLEWWW-SVNPATH := http://svn.planet-lab.org/svn/PLEWWW/tags/PLEWWW-4.3-33 +www-register-wizard-SVNPATH := http://svn.planet-lab.org/svn/www-register-wizard/tags/www-register-wizard-4.3-1 +Monitor-SVNPATH := http://svn.planet-lab.org/svn/Monitor/tags/Monitor-3.0-24 +nodeconfig-SVNPATH := http://svn.planet-lab.org/svn/nodeconfig/tags/nodeconfig-4.3-5 +BootManager-SVNPATH := http://svn.planet-lab.org/svn/BootManager/tags/BootManager-4.3-13 +pypcilib-SVNPATH := http://svn.planet-lab.org/svn/pypcilib/tags/pypcilib-0.2-8 +BootCD-SVNPATH := http://svn.planet-lab.org/svn/BootCD/tags/BootCD-4.2-15 +VserverReference-SVNPATH := http://svn.planet-lab.org/svn/VserverReference/tags/VserverReference-4.2-16 +BootstrapFS-SVNPATH := http://svn.planet-lab.org/svn/BootstrapFS/tags/BootstrapFS-1.0-10 +MyPLC-SVNPATH := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-4.3-29 +sfa-SVNPATH := http://svn.planet-lab.org/svn/sfa/tags/sfa-0.9-5 +pyopenssl-SVNPATH := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9-1 +PLCRT-SVNPATH := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-10 + +# locating the right test directory - see make tests_svnpath +tests-SVNPATH := http://svn.planet-lab.org/svn/tests/tags/tests-4.3-5 diff --git a/trunk/planetbridge.mk b/trunk/planetbridge.mk new file mode 100644 index 00000000..51ceffac --- /dev/null +++ b/trunk/planetbridge.mk @@ -0,0 +1,396 @@ +# +# declare the packages to be built and their dependencies +# initial version from Mark Huang +# Mark Huang +# Copyright (C) 2003-2006 The Trustees of Princeton University +# rewritten by Thierry Parmentelat - INRIA Sophia Antipolis +# +# $Id$ +# +# see doc in Makefile +# + +# +# kernel +# +# use a package name with srpm in it: +# so the source rpm is created by running make srpm in the codebase +# + +kernel-MODULES := linux-2.6 +kernel-SPEC := kernel-2.6.spec +kernel-BUILD-FROM-SRPM := yes +ifeq "$(HOSTARCH)" "i386" +kernel-RPMFLAGS:= --target i686 +else +kernel-RPMFLAGS:= --target $(HOSTARCH) +endif +KERNELS += kernel + +kernels: $(KERNELS) +kernels-clean: $(foreach package,$(KERNELS),$(package)-clean) + +ALL += $(KERNELS) +# this is to mark on which image a given rpm is supposed to go +IN_BOOTCD += $(KERNELS) +IN_VSERVER += $(KERNELS) +IN_BOOTSTRAPFS += $(KERNELS) +# turns out myplc installs kernel-vserver +IN_MYPLC += $(KERNELS) + +# +# madwifi +# +madwifi-MODULES := madwifi +madwifi-SPEC := madwifi.spec +madwifi-BUILD-FROM-SRPM := yes +madwifi-DEPEND-DEVEL-RPMS := kernel-devel +madwifi-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +ALL += madwifi +IN_BOOTSTRAPFS += madwifi + +# +# util-vserver +# +util-vserver-MODULES := util-vserver +util-vserver-SPEC := util-vserver.spec +util-vserver-RPMFLAGS:= --without dietlibc +ALL += util-vserver +IN_BOOTSTRAPFS += util-vserver + +# +# libnl - local import +# we need either 1.1 or at least 1.0.pre6 +# rebuild this on centos5 - see kexcludes in build.common +# +local_libnl=false +ifeq "$(DISTRONAME)" "centos5" +local_libnl=true +endif + +ifeq "$(local_libnl)" "true" +libnl-MODULES := libnl +libnl-SPEC := libnl.spec +libnl-BUILD-FROM-SRPM := yes +# this sounds like the thing to do, but in fact linux/if_vlan.h comes with kernel-headers +libnl-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += libnl +IN_BOOTSTRAPFS += libnl +endif + +# +# util-vserver-pl +# +util-vserver-pl-MODULES := util-vserver-pl +util-vserver-pl-SPEC := util-vserver-pl.spec +util-vserver-pl-DEPEND-DEVEL-RPMS := util-vserver-lib util-vserver-devel util-vserver-core +ifeq "$(local_libnl)" "true" +util-vserver-pl-DEPEND-DEVEL-RPMS += libnl libnl-devel +endif +ALL += util-vserver-pl +IN_BOOTSTRAPFS += util-vserver-pl + +# +# NodeUpdate +# +nodeupdate-MODULES := NodeUpdate +nodeupdate-SPEC := NodeUpdate.spec +ALL += nodeupdate +IN_BOOTSTRAPFS += nodeupdate + +# +# ipod +# +ipod-MODULES := PingOfDeath +ipod-SPEC := ipod.spec +ALL += ipod +IN_BOOTSTRAPFS += ipod + +# +# NodeManager +# +nodemanager-MODULES := NodeManager +nodemanager-SPEC := NodeManager.spec +ALL += nodemanager +IN_BOOTSTRAPFS += nodemanager + +# +# pl_sshd +# +sshd-MODULES := pl_sshd +sshd-SPEC := pl_sshd.spec +ALL += sshd +IN_BOOTSTRAPFS += sshd + +# +# codemux: Port 80 demux +# +codemux-MODULES := CoDemux +codemux-SPEC := codemux.spec +codemux-RPMBUILD := sudo bash ./rpmbuild.sh +ALL += codemux +IN_BOOTSTRAPFS += codemux + +# +# fprobe-ulog +# +fprobe-ulog-MODULES := fprobe-ulog +fprobe-ulog-SPEC := fprobe-ulog.spec +ALL += fprobe-ulog +IN_BOOTSTRAPFS += fprobe-ulog + +# +# pf2slice +# +pf2slice-MODULES := pf2slice +pf2slice-SPEC := pf2slice.spec +ALL += pf2slice + +# +# PlanetLab Mom: Cleans up your mess +# +mom-MODULES := Mom +mom-SPEC := pl_mom.spec +ALL += mom +IN_BOOTSTRAPFS += mom + +# +# iptables +# +iptables-MODULES := iptables +iptables-SPEC := iptables.spec +iptables-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += iptables +IN_BOOTSTRAPFS += iptables + +# +# iproute +# +iproute-MODULES := iproute2 +iproute-SPEC := iproute.spec +ALL += iproute +IN_BOOTSTRAPFS += iproute +IN_VSERVER += iproute +IN_BOOTCD += iproute + +# +# inotify-tools - local import +# rebuild this on centos5 (not found) - see kexcludes in build.common +# +local_inotify_tools=false +ifeq "$(DISTRONAME)" "centos5" +local_inotify_tools=true +endif + +ifeq "$(local_inotify_tools)" "true" +inotify-tools-MODULES := inotify-tools +inotify-tools-SPEC := inotify-tools.spec +inotify-tools-BUILD-FROM-SRPM := yes +IN_BOOTSTRAPFS += inotify-tools +ALL += inotify-tools +endif + +# +# vsys +# +vsys-MODULES := vsys +vsys-SPEC := vsys.spec +ifeq "$(local_inotify_tools)" "true" +vsys-DEPEND-DEVEL-RPMS := inotify-tools inotify-tools-devel +endif +IN_BOOTSTRAPFS += vsys +ALL += vsys + +# +# vsys-scripts +# +vsys-scripts-MODULES := vsys-scripts +vsys-scripts-SPEC := vsys-scripts.spec +IN_BOOTSTRAPFS += vsys-scripts +ALL += vsys-scripts + +# +# PLCAPI +# +PLCAPI-MODULES := PLCAPI +PLCAPI-SPEC := PLCAPI.spec +ALL += PLCAPI +IN_MYPLC += PLCAPI + +# +# drupal +# +drupal-MODULES := drupal +drupal-SPEC := drupal.spec +drupal-BUILD-FROM-SRPM := yes +ALL += drupal +IN_MYPLC += drupal + +# +# use the plewww module instead +# +plewww-MODULES := PLEWWW +plewww-SPEC := plewww.spec +ALL += plewww +IN_MYPLC += plewww + +# +# www-register-wizard +# +www-register-wizard-MODULES := www-register-wizard +www-register-wizard-SPEC := www-register-wizard.spec +ALL += www-register-wizard +IN_MYPLC += www-register-wizard + +# +# monitor +# +monitor-MODULES := Monitor +monitor-SPEC := Monitor.spec +ALL += monitor +IN_BOOTSTRAPFS += monitor + +# +# PLC RT +# +plcrt-MODULES := PLCRT +plcrt-SPEC := plcrt.spec +ALL += plcrt + +# +# zabbix +# +zabbix-MODULES := Monitor +zabbix-SPEC := zabbix.spec +zabbix-BUILD-FROM-SRPM := yes +ALL += zabbix + +# +# pyopenssl +# +pyopenssl-MODULES := pyopenssl +pyopenssl-SPEC := pyOpenSSL.spec +pyopenssl-BUILD-FROM-SRPM := yes +ALL += pyopenssl + + +# +# sfa - Slice Facility Architecture +# +sfa-MODULES := sfa +sfa-SPEC := sfa.spec +ALL += sfa + +# +# nodeconfig +# +nodeconfig-MODULES := nodeconfig build +nodeconfig-SPEC := nodeconfig.spec +ALL += nodeconfig +IN_MYPLC += nodeconfig + +# +# bootmanager +# +bootmanager-MODULES := BootManager +bootmanager-SPEC := bootmanager.spec +ALL += bootmanager +IN_MYPLC += bootmanager + +# +# pypcilib : used in bootcd +# +pypcilib-MODULES := pypcilib +pypcilib-SPEC := pypcilib.spec +ALL += pypcilib +IN_BOOTCD += pypcilib + +# +# pyplnet +# +pyplnet-MODULES := pyplnet +pyplnet-SPEC := pyplnet.spec +ALL += pyplnet +IN_BOOTSTRAPFS += pyplnet +IN_MYPLC += pyplnet +IN_BOOTCD += pyplnet + +# +# bootcd +# +bootcd-MODULES := BootCD build +bootcd-SPEC := bootcd.spec +bootcd-RPMBUILD := sudo bash ./rpmbuild.sh +bootcd-DEPEND-PACKAGES := $(IN_BOOTCD) +bootcd-DEPEND-FILES := RPMS/yumgroups.xml +bootcd-RPMDATE := yes +ALL += bootcd +IN_MYPLC += bootcd + +# +# vserver : reference image for slices +# +vserver-MODULES := VserverReference build +vserver-SPEC := vserver-reference.spec +vserver-DEPEND-PACKAGES := $(IN_VSERVER) +vserver-DEPEND-FILES := RPMS/yumgroups.xml +vserver-RPMDATE := yes +ALL += vserver +IN_BOOTSTRAPFS += vserver + +# +# bootstrapfs +# +bootstrapfs-MODULES := BootstrapFS build +bootstrapfs-SPEC := bootstrapfs.spec +bootstrapfs-RPMBUILD := sudo bash ./rpmbuild.sh +bootstrapfs-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) +bootstrapfs-DEPEND-FILES := RPMS/yumgroups.xml +bootstrapfs-RPMDATE := yes +ALL += bootstrapfs +IN_MYPLC += bootstrapfs + +# +# noderepo +# +# all rpms resulting from packages marked as being in bootstrapfs and vserver +NODEREPO_RPMS = $(foreach package,$(IN_BOOTSTRAPFS) $(IN_VSERVER),$($(package).rpms)) +# replace space with +++ (specvars cannot deal with spaces) +SPACE=$(subst x, ,x) +NODEREPO_RPMS_3PLUS = $(subst $(SPACE),+++,$(NODEREPO_RPMS)) + +noderepo-MODULES := BootstrapFS +noderepo-SPEC := noderepo.spec +noderepo-RPMBUILD := sudo bash ./rpmbuild.sh +# package requires all regular packages +noderepo-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) $(IN_VSERVER) +noderepo-DEPEND-FILES := RPMS/yumgroups.xml +#export rpm list to the specfile +noderepo-SPECVARS = node_rpms_plus=$(NODEREPO_RPMS_3PLUS) +noderepo-RPMDATE := yes +ALL += noderepo +IN_MYPLC += noderepo + +# +# MyPLC : lightweight packaging, dependencies are yum-installed in a vserver +# +myplc-MODULES := MyPLC build +myplc-SPEC := myplc.spec +myplc-DEPEND-FILES := myplc-release RPMS/yumgroups.xml +ALL += myplc + +# myplc-docs only contains docs for PLCAPI and NMAPI, but +# we still need to pull MyPLC, as it is where the specfile lies, +# together with the utility script docbook2drupal.sh +myplc-docs-MODULES := MyPLC PLCAPI NodeManager Monitor +myplc-docs-SPEC := myplc-docs.spec +ALL += myplc-docs + +# using some other name than myplc-release, as this is a make target already +release-MODULES := MyPLC +release-SPEC := myplc-release.spec +release-RPMDATE := yes +ALL += release + diff --git a/trunk/planetlab-k27-tags.mk b/trunk/planetlab-k27-tags.mk new file mode 100644 index 00000000..15ab37e9 --- /dev/null +++ b/trunk/planetlab-k27-tags.mk @@ -0,0 +1,55 @@ +# $Id$ + +# build-SVNPATH is now set by vbuild-nightly.sh to avoid duplication + +# trying the 2.6.27 kernel +# VINI is running a prototype of a 2.6.27-based PlanetLab node (aka Trellis) +# Below we include a few Trellis versions of packages +linux-2.6-SVNPATH := http://svn.planet-lab.org/svn/linux-2.6/trunk +madwifi-SVNPATH := http://svn.planet-lab.org/svn/madwifi/tags/madwifi-3878-0 +# Trellis is using a modified util-vserver and util-vserver-pl with the 2.6.27 kernel +# this branch is numbered 0.30.216-pre2833 in the specfile +# initial tag would be util-vserver-0.30.216-pre2833-0 +util-vserver-SVNBRANCH := trellis +util-vserver-SVNPATH := http://svn.planet-lab.org/svn/util-vserver/branches/trellis +libnl-SVNPATH := http://svn.planet-lab.org/svn/libnl/tags/libnl-1.1-2 +# this branch is numbered 0.4 in the specfile +# initial tag would be tags/util-vserver-pl-0.4-3 +util-vserver-pl-SVNBRANCH := trellis +util-vserver-pl-SVNPATH := http://svn.planet-lab.org/svn/util-vserver-pl/branches/trellis +NodeUpdate-SVNPATH := http://svn.planet-lab.org/svn/NodeUpdate/tags/NodeUpdate-0.5-5 +PingOfDeath-SVNPATH := http://svn.planet-lab.org/svn/PingOfDeath/tags/PingOfDeath-2.2-1 +NodeManager-SVNPATH := http://svn.planet-lab.org/svn/NodeManager/tags/NodeManager-1.8-11 +# Trellis-specific NodeManager plugins +NodeManager-topo-SVNPATH := http://svn.planet-lab.org/svn/NodeManager-topo/trunk +NodeManager-optin-SVNPATH := http://svn.planet-lab.org/svn/NodeManager-optin/trunk +pyplnet-SVNPATH := http://svn.planet-lab.org/svn/pyplnet/tags/pyplnet-4.3-4 +pl_sshd-SVNPATH := http://svn.planet-lab.org/svn/pl_sshd/tags/pl_sshd-1.0-11 +CoDemux-SVNPATH := http://svn.planet-lab.org/svn/CoDemux/tags/CoDemux-0.1-13 +fprobe-ulog-SVNPATH := http://svn.planet-lab.org/svn/fprobe-ulog/tags/fprobe-ulog-1.1.3-0 +pf2slice-SVNPATH := http://svn.planet-lab.org/svn/pf2slice/tags/pf2slice-1.0-2 +Mom-SVNPATH := http://svn.planet-lab.org/svn/Mom/tags/Mom-2.3-1 +iptables-SVNPATH := http://svn.planet-lab.org/svn/iptables/trunk +iproute2-SVNPATH := http://svn.planet-lab.org/svn/iproute2/tags/iproute2-2.6.16-2 +inotify-tools-SVNPATH := http://svn.planet-lab.org/svn/inotify-tools/tags/inotify-tools-3.13-2 +vsys-SVNBRANCH := 0.9 +vsys-SVNPATH := http://svn.planet-lab.org/svn/vsys/tags/vsys-0.9-3 +vsys-scripts-SVNPATH := http://svn.planet-lab.org/svn/vsys-scripts/tags/vsys-scripts-0.95-11 +PLCAPI-SVNPATH := http://svn.planet-lab.org/svn/PLCAPI/tags/PLCAPI-4.3-30 +drupal-SVNPATH := http://svn.planet-lab.org/svn/drupal/tags/drupal-4.7-13 +PLEWWW-SVNPATH := http://svn.planet-lab.org/svn/PLEWWW/tags/PLEWWW-4.3-33 +www-register-wizard-SVNPATH := http://svn.planet-lab.org/svn/www-register-wizard/tags/www-register-wizard-4.3-1 +Monitor-SVNPATH := http://svn.planet-lab.org/svn/Monitor/tags/Monitor-3.0-24 +nodeconfig-SVNPATH := http://svn.planet-lab.org/svn/nodeconfig/tags/nodeconfig-4.3-5 +BootManager-SVNPATH := http://svn.planet-lab.org/svn/BootManager/tags/BootManager-4.3-13 +pypcilib-SVNPATH := http://svn.planet-lab.org/svn/pypcilib/tags/pypcilib-0.2-8 +BootCD-SVNPATH := http://svn.planet-lab.org/svn/BootCD/tags/BootCD-4.2-15 +VserverReference-SVNPATH := http://svn.planet-lab.org/svn/VserverReference/tags/VserverReference-4.2-16 +BootstrapFS-SVNPATH := http://svn.planet-lab.org/svn/BootstrapFS/tags/BootstrapFS-1.0-10 +MyPLC-SVNPATH := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-4.3-29 +sfa-SVNPATH := http://svn.planet-lab.org/svn/sfa/tags/sfa-0.9-5 +pyopenssl-SVNPATH := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9-1 +PLCRT-SVNPATH := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-10 + +# locating the right test directory - see make tests_svnpath +tests-SVNPATH := http://svn.planet-lab.org/svn/tests/tags/tests-4.3-5 diff --git a/trunk/planetlab-tags.mk b/trunk/planetlab-tags.mk new file mode 100644 index 00000000..d931bc3e --- /dev/null +++ b/trunk/planetlab-tags.mk @@ -0,0 +1,47 @@ +# $Id$ + +# build-SVNPATH is now set by vbuild-nightly.sh to avoid duplication + +linux-2.6-SVNBRANCH := 22 +# not adopting tag 40 about "VXC_PROC_WRITE support" - should be safe but Sapan advised keeping it out was safer still +linux-2.6-SVNPATH := http://svn.planet-lab.org/svn/linux-2.6/tags/linux-2.6-22-39 +madwifi-SVNBRANCH := 0.9.4 +madwifi-SVNPATH := http://svn.planet-lab.org/svn/madwifi/tags/madwifi-0.9.4-3 +util-vserver-SVNBRANCH := scholz +util-vserver-SVNPATH := http://svn.planet-lab.org/svn/util-vserver/tags/util-vserver-0.30.215-6 +libnl-SVNPATH := http://svn.planet-lab.org/svn/libnl/tags/libnl-1.1-2 +util-vserver-pl-SVNPATH := http://svn.planet-lab.org/svn/util-vserver-pl/tags/util-vserver-pl-0.3-19 +NodeUpdate-SVNPATH := http://svn.planet-lab.org/svn/NodeUpdate/tags/NodeUpdate-0.5-5 +PingOfDeath-SVNPATH := http://svn.planet-lab.org/svn/PingOfDeath/tags/PingOfDeath-2.2-1 +NodeManager-SVNPATH := http://svn.planet-lab.org/svn/NodeManager/tags/NodeManager-1.8-20 +pyplnet-SVNPATH := http://svn.planet-lab.org/svn/pyplnet/tags/pyplnet-4.3-4 +pl_sshd-SVNPATH := http://svn.planet-lab.org/svn/pl_sshd/tags/pl_sshd-1.0-11 +CoDemux-SVNPATH := http://svn.planet-lab.org/svn/CoDemux/tags/CoDemux-0.1-13 +fprobe-ulog-SVNPATH := http://svn.planet-lab.org/svn/fprobe-ulog/tags/fprobe-ulog-1.1.3-0 +pf2slice-SVNPATH := http://svn.planet-lab.org/svn/pf2slice/tags/pf2slice-1.0-2 +Mom-SVNPATH := http://svn.planet-lab.org/svn/Mom/tags/Mom-2.3-1 +iptables-SVNBRANCH := 1.3.8 +iptables-SVNPATH := http://svn.planet-lab.org/svn/iptables/tags/iptables-1.3.8-9 +iproute2-SVNPATH := http://svn.planet-lab.org/svn/iproute2/tags/iproute2-2.6.16-2 +inotify-tools-SVNPATH := http://svn.planet-lab.org/svn/inotify-tools/tags/inotify-tools-3.13-2 +vsys-SVNBRANCH := 0.9 +vsys-SVNPATH := http://svn.planet-lab.org/svn/vsys/tags/vsys-0.9-3 +vsys-scripts-SVNPATH := http://svn.planet-lab.org/svn/vsys-scripts/tags/vsys-scripts-0.95-11 +PLCAPI-SVNPATH := http://svn.planet-lab.org/svn/PLCAPI/tags/PLCAPI-4.3-30 +drupal-SVNPATH := http://svn.planet-lab.org/svn/drupal/tags/drupal-4.7-13 +PLEWWW-SVNPATH := http://svn.planet-lab.org/svn/PLEWWW/tags/PLEWWW-4.3-33 +www-register-wizard-SVNPATH := http://svn.planet-lab.org/svn/www-register-wizard/tags/www-register-wizard-4.3-1 +Monitor-SVNPATH := http://svn.planet-lab.org/svn/Monitor/tags/Monitor-3.0-24 +nodeconfig-SVNPATH := http://svn.planet-lab.org/svn/nodeconfig/tags/nodeconfig-4.3-5 +BootManager-SVNPATH := http://svn.planet-lab.org/svn/BootManager/tags/BootManager-4.3-13 +pypcilib-SVNPATH := http://svn.planet-lab.org/svn/pypcilib/tags/pypcilib-0.2-8 +BootCD-SVNPATH := http://svn.planet-lab.org/svn/BootCD/tags/BootCD-4.2-15 +VserverReference-SVNPATH := http://svn.planet-lab.org/svn/VserverReference/tags/VserverReference-4.2-16 +BootstrapFS-SVNPATH := http://svn.planet-lab.org/svn/BootstrapFS/tags/BootstrapFS-1.0-10 +MyPLC-SVNPATH := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-4.3-29 +sfa-SVNPATH := http://svn.planet-lab.org/svn/sfa/tags/sfa-0.9-5 +pyopenssl-SVNPATH := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9-1 +PLCRT-SVNPATH := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-11 + +# locating the right test directory - see make tests_svnpath +tests-SVNPATH := http://svn.planet-lab.org/svn/tests/tags/tests-4.3-5 diff --git a/trunk/planetlab.mk b/trunk/planetlab.mk new file mode 100644 index 00000000..51ceffac --- /dev/null +++ b/trunk/planetlab.mk @@ -0,0 +1,396 @@ +# +# declare the packages to be built and their dependencies +# initial version from Mark Huang +# Mark Huang +# Copyright (C) 2003-2006 The Trustees of Princeton University +# rewritten by Thierry Parmentelat - INRIA Sophia Antipolis +# +# $Id$ +# +# see doc in Makefile +# + +# +# kernel +# +# use a package name with srpm in it: +# so the source rpm is created by running make srpm in the codebase +# + +kernel-MODULES := linux-2.6 +kernel-SPEC := kernel-2.6.spec +kernel-BUILD-FROM-SRPM := yes +ifeq "$(HOSTARCH)" "i386" +kernel-RPMFLAGS:= --target i686 +else +kernel-RPMFLAGS:= --target $(HOSTARCH) +endif +KERNELS += kernel + +kernels: $(KERNELS) +kernels-clean: $(foreach package,$(KERNELS),$(package)-clean) + +ALL += $(KERNELS) +# this is to mark on which image a given rpm is supposed to go +IN_BOOTCD += $(KERNELS) +IN_VSERVER += $(KERNELS) +IN_BOOTSTRAPFS += $(KERNELS) +# turns out myplc installs kernel-vserver +IN_MYPLC += $(KERNELS) + +# +# madwifi +# +madwifi-MODULES := madwifi +madwifi-SPEC := madwifi.spec +madwifi-BUILD-FROM-SRPM := yes +madwifi-DEPEND-DEVEL-RPMS := kernel-devel +madwifi-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +ALL += madwifi +IN_BOOTSTRAPFS += madwifi + +# +# util-vserver +# +util-vserver-MODULES := util-vserver +util-vserver-SPEC := util-vserver.spec +util-vserver-RPMFLAGS:= --without dietlibc +ALL += util-vserver +IN_BOOTSTRAPFS += util-vserver + +# +# libnl - local import +# we need either 1.1 or at least 1.0.pre6 +# rebuild this on centos5 - see kexcludes in build.common +# +local_libnl=false +ifeq "$(DISTRONAME)" "centos5" +local_libnl=true +endif + +ifeq "$(local_libnl)" "true" +libnl-MODULES := libnl +libnl-SPEC := libnl.spec +libnl-BUILD-FROM-SRPM := yes +# this sounds like the thing to do, but in fact linux/if_vlan.h comes with kernel-headers +libnl-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += libnl +IN_BOOTSTRAPFS += libnl +endif + +# +# util-vserver-pl +# +util-vserver-pl-MODULES := util-vserver-pl +util-vserver-pl-SPEC := util-vserver-pl.spec +util-vserver-pl-DEPEND-DEVEL-RPMS := util-vserver-lib util-vserver-devel util-vserver-core +ifeq "$(local_libnl)" "true" +util-vserver-pl-DEPEND-DEVEL-RPMS += libnl libnl-devel +endif +ALL += util-vserver-pl +IN_BOOTSTRAPFS += util-vserver-pl + +# +# NodeUpdate +# +nodeupdate-MODULES := NodeUpdate +nodeupdate-SPEC := NodeUpdate.spec +ALL += nodeupdate +IN_BOOTSTRAPFS += nodeupdate + +# +# ipod +# +ipod-MODULES := PingOfDeath +ipod-SPEC := ipod.spec +ALL += ipod +IN_BOOTSTRAPFS += ipod + +# +# NodeManager +# +nodemanager-MODULES := NodeManager +nodemanager-SPEC := NodeManager.spec +ALL += nodemanager +IN_BOOTSTRAPFS += nodemanager + +# +# pl_sshd +# +sshd-MODULES := pl_sshd +sshd-SPEC := pl_sshd.spec +ALL += sshd +IN_BOOTSTRAPFS += sshd + +# +# codemux: Port 80 demux +# +codemux-MODULES := CoDemux +codemux-SPEC := codemux.spec +codemux-RPMBUILD := sudo bash ./rpmbuild.sh +ALL += codemux +IN_BOOTSTRAPFS += codemux + +# +# fprobe-ulog +# +fprobe-ulog-MODULES := fprobe-ulog +fprobe-ulog-SPEC := fprobe-ulog.spec +ALL += fprobe-ulog +IN_BOOTSTRAPFS += fprobe-ulog + +# +# pf2slice +# +pf2slice-MODULES := pf2slice +pf2slice-SPEC := pf2slice.spec +ALL += pf2slice + +# +# PlanetLab Mom: Cleans up your mess +# +mom-MODULES := Mom +mom-SPEC := pl_mom.spec +ALL += mom +IN_BOOTSTRAPFS += mom + +# +# iptables +# +iptables-MODULES := iptables +iptables-SPEC := iptables.spec +iptables-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += iptables +IN_BOOTSTRAPFS += iptables + +# +# iproute +# +iproute-MODULES := iproute2 +iproute-SPEC := iproute.spec +ALL += iproute +IN_BOOTSTRAPFS += iproute +IN_VSERVER += iproute +IN_BOOTCD += iproute + +# +# inotify-tools - local import +# rebuild this on centos5 (not found) - see kexcludes in build.common +# +local_inotify_tools=false +ifeq "$(DISTRONAME)" "centos5" +local_inotify_tools=true +endif + +ifeq "$(local_inotify_tools)" "true" +inotify-tools-MODULES := inotify-tools +inotify-tools-SPEC := inotify-tools.spec +inotify-tools-BUILD-FROM-SRPM := yes +IN_BOOTSTRAPFS += inotify-tools +ALL += inotify-tools +endif + +# +# vsys +# +vsys-MODULES := vsys +vsys-SPEC := vsys.spec +ifeq "$(local_inotify_tools)" "true" +vsys-DEPEND-DEVEL-RPMS := inotify-tools inotify-tools-devel +endif +IN_BOOTSTRAPFS += vsys +ALL += vsys + +# +# vsys-scripts +# +vsys-scripts-MODULES := vsys-scripts +vsys-scripts-SPEC := vsys-scripts.spec +IN_BOOTSTRAPFS += vsys-scripts +ALL += vsys-scripts + +# +# PLCAPI +# +PLCAPI-MODULES := PLCAPI +PLCAPI-SPEC := PLCAPI.spec +ALL += PLCAPI +IN_MYPLC += PLCAPI + +# +# drupal +# +drupal-MODULES := drupal +drupal-SPEC := drupal.spec +drupal-BUILD-FROM-SRPM := yes +ALL += drupal +IN_MYPLC += drupal + +# +# use the plewww module instead +# +plewww-MODULES := PLEWWW +plewww-SPEC := plewww.spec +ALL += plewww +IN_MYPLC += plewww + +# +# www-register-wizard +# +www-register-wizard-MODULES := www-register-wizard +www-register-wizard-SPEC := www-register-wizard.spec +ALL += www-register-wizard +IN_MYPLC += www-register-wizard + +# +# monitor +# +monitor-MODULES := Monitor +monitor-SPEC := Monitor.spec +ALL += monitor +IN_BOOTSTRAPFS += monitor + +# +# PLC RT +# +plcrt-MODULES := PLCRT +plcrt-SPEC := plcrt.spec +ALL += plcrt + +# +# zabbix +# +zabbix-MODULES := Monitor +zabbix-SPEC := zabbix.spec +zabbix-BUILD-FROM-SRPM := yes +ALL += zabbix + +# +# pyopenssl +# +pyopenssl-MODULES := pyopenssl +pyopenssl-SPEC := pyOpenSSL.spec +pyopenssl-BUILD-FROM-SRPM := yes +ALL += pyopenssl + + +# +# sfa - Slice Facility Architecture +# +sfa-MODULES := sfa +sfa-SPEC := sfa.spec +ALL += sfa + +# +# nodeconfig +# +nodeconfig-MODULES := nodeconfig build +nodeconfig-SPEC := nodeconfig.spec +ALL += nodeconfig +IN_MYPLC += nodeconfig + +# +# bootmanager +# +bootmanager-MODULES := BootManager +bootmanager-SPEC := bootmanager.spec +ALL += bootmanager +IN_MYPLC += bootmanager + +# +# pypcilib : used in bootcd +# +pypcilib-MODULES := pypcilib +pypcilib-SPEC := pypcilib.spec +ALL += pypcilib +IN_BOOTCD += pypcilib + +# +# pyplnet +# +pyplnet-MODULES := pyplnet +pyplnet-SPEC := pyplnet.spec +ALL += pyplnet +IN_BOOTSTRAPFS += pyplnet +IN_MYPLC += pyplnet +IN_BOOTCD += pyplnet + +# +# bootcd +# +bootcd-MODULES := BootCD build +bootcd-SPEC := bootcd.spec +bootcd-RPMBUILD := sudo bash ./rpmbuild.sh +bootcd-DEPEND-PACKAGES := $(IN_BOOTCD) +bootcd-DEPEND-FILES := RPMS/yumgroups.xml +bootcd-RPMDATE := yes +ALL += bootcd +IN_MYPLC += bootcd + +# +# vserver : reference image for slices +# +vserver-MODULES := VserverReference build +vserver-SPEC := vserver-reference.spec +vserver-DEPEND-PACKAGES := $(IN_VSERVER) +vserver-DEPEND-FILES := RPMS/yumgroups.xml +vserver-RPMDATE := yes +ALL += vserver +IN_BOOTSTRAPFS += vserver + +# +# bootstrapfs +# +bootstrapfs-MODULES := BootstrapFS build +bootstrapfs-SPEC := bootstrapfs.spec +bootstrapfs-RPMBUILD := sudo bash ./rpmbuild.sh +bootstrapfs-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) +bootstrapfs-DEPEND-FILES := RPMS/yumgroups.xml +bootstrapfs-RPMDATE := yes +ALL += bootstrapfs +IN_MYPLC += bootstrapfs + +# +# noderepo +# +# all rpms resulting from packages marked as being in bootstrapfs and vserver +NODEREPO_RPMS = $(foreach package,$(IN_BOOTSTRAPFS) $(IN_VSERVER),$($(package).rpms)) +# replace space with +++ (specvars cannot deal with spaces) +SPACE=$(subst x, ,x) +NODEREPO_RPMS_3PLUS = $(subst $(SPACE),+++,$(NODEREPO_RPMS)) + +noderepo-MODULES := BootstrapFS +noderepo-SPEC := noderepo.spec +noderepo-RPMBUILD := sudo bash ./rpmbuild.sh +# package requires all regular packages +noderepo-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) $(IN_VSERVER) +noderepo-DEPEND-FILES := RPMS/yumgroups.xml +#export rpm list to the specfile +noderepo-SPECVARS = node_rpms_plus=$(NODEREPO_RPMS_3PLUS) +noderepo-RPMDATE := yes +ALL += noderepo +IN_MYPLC += noderepo + +# +# MyPLC : lightweight packaging, dependencies are yum-installed in a vserver +# +myplc-MODULES := MyPLC build +myplc-SPEC := myplc.spec +myplc-DEPEND-FILES := myplc-release RPMS/yumgroups.xml +ALL += myplc + +# myplc-docs only contains docs for PLCAPI and NMAPI, but +# we still need to pull MyPLC, as it is where the specfile lies, +# together with the utility script docbook2drupal.sh +myplc-docs-MODULES := MyPLC PLCAPI NodeManager Monitor +myplc-docs-SPEC := myplc-docs.spec +ALL += myplc-docs + +# using some other name than myplc-release, as this is a make target already +release-MODULES := MyPLC +release-SPEC := myplc-release.spec +release-RPMDATE := yes +ALL += release + diff --git a/trunk/release-changelog b/trunk/release-changelog new file mode 120000 index 00000000..323e9ad8 --- /dev/null +++ b/trunk/release-changelog @@ -0,0 +1 @@ +module-tools.py \ No newline at end of file diff --git a/trunk/rpmbuild.sh b/trunk/rpmbuild.sh new file mode 100644 index 00000000..d65579e0 --- /dev/null +++ b/trunk/rpmbuild.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# We require this hack to tell rpmbuild where to find the .rpmmacros +# file. And for whatever reason the $HOME variable used by the build +# system is not set correctly when invoking rpmbuild with sudo. This +# is hopefully just a temporary hack until we can either figure out +# what to do with sudo or pass rpmbuild an option that points it at a +# specific .rpmmacros file. + +# Be verbose +set -x + +export HOME=${PWD} +rpmbuild "$@" diff --git a/trunk/run-nightlies.py b/trunk/run-nightlies.py new file mode 100755 index 00000000..01b30366 --- /dev/null +++ b/trunk/run-nightlies.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +# Script to read build configs in /etc/build_conf.py, turn the configuration into command lines and execute it + +import os +import re + +# Assemble a list of builds from a single build spec +def interpret_build(build, param_names, current_concrete_build={}, concrete_build_list=[]): + + if (param_names==[]): + concrete_build_list.extend([current_concrete_build]) + else: + (cur_param_name,remaining_param_names)=(param_names[0],param_names[1:]) + cur_param = build[cur_param_name] + + # If it's a list, produce a concrete build for each element of the list + if (type(cur_param)==type([])): + for value in cur_param: + new_concrete_build = current_concrete_build.copy() + new_concrete_build[cur_param_name] = value + concrete_build_list = interpret_build(build, remaining_param_names, new_concrete_build, concrete_build_list) + + # If not, just tack on the value and move on + else: + current_concrete_build[cur_param_name] = cur_param + concrete_build_list = interpret_build(build, remaining_param_names, current_concrete_build,concrete_build_list) + + return concrete_build_list + + +# Fill in parameters that are not defined in __default_build__ +def complete_build_spec_with_defaults (build, default_build): + for default_param in default_build.keys(): + if (not build.has_key(default_param)): + build[default_param]=default_build[default_param] + return build + + +# Turn a concrete build into a commandline + +def concrete_build_to_commandline(concrete_build): + + cmdline = '''%(sh)s + %(vbuildnightly)s + -b %(pldistro)s-%(fcdistro)s-%(arch)s-%(myplcversion)s-%(release)s-%(date)s + -f %(fcdistro)s + -m %(mailto)s + -p %(personality)s + -r %(webpath)s + -s %(svnpath)s + -t %(tags)s + -w %(webpath)s/%(pldistro)s/%(fcdistro)s + %(runtests)s'''.replace('\n','') + + cmdline = cmdline % concrete_build + + purge_spaces = re.compile('\s+') + + return purge_spaces.sub(' ', cmdline) + + +# reduce dependencies in a build +def reduce_dependencies(concrete_build): + for b in concrete_build.keys(): + val = concrete_build[b] + if (type(val)==type(lambda x:x)): + concrete_build[b] = val(concrete_build) + return concrete_build + + +# Turn build parameter dicts into commandlines and execute them +def process_builds (builds, build_names, default_build): + for build_name in build_names: + build = complete_build_spec_with_defaults (builds[build_name], default_build) + concrete_builds_without_deps = interpret_build (build, build.keys(), {}, []) + concrete_builds = map(reduce_dependencies, concrete_builds_without_deps) + commandlines = map(concrete_build_to_commandline, concrete_builds) + for commandline in commandlines: + os.system(commandline) + +def main(): + config_file = '/etc/build-conf-planetlab.py' + builds = {} + try: + execfile(config_file, builds) + except IOError, e: + raise IOError, "Could not open %s\n" % config_file + + + config_file_attributes = builds.keys() + build_names = [e for e in config_file_attributes if not e.startswith('__')] + + try: + default_build = builds['__default_build__'] + except KeyError: + raise KeyError, "Please define the default build config in %s\n" % config_file + + process_builds(builds, build_names, default_build) + + +if __name__ == "__main__": + main() + diff --git a/trunk/spec2make.c b/trunk/spec2make.c new file mode 100644 index 00000000..a32f4fdc --- /dev/null +++ b/trunk/spec2make.c @@ -0,0 +1,217 @@ +/* + * Parses RPM spec file into Makefile fragment. See + * + * http://fedora.redhat.com/docs/drafts/rpm-guide-en/ch-programming-c.html + * + * Mark Huang + * Copyright (C) 2006 The Trustees of Princeton University + * + * $Id$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* from f10 and up, Spec is renamed rpmSpec */ +#ifndef _RPMTYPES_H +#define rpmSpec Spec +#endif + +#ifndef PATH_MAX +#include +#endif + +extern size_t strnlen(const char *s, size_t maxlen); + +/* the structure describing the options we take and the defaults */ +static struct poptOption optionsTable[] = { + { NULL, '\0', POPT_ARG_INCLUDE_TABLE, rpmcliAllPoptTable, 0, + "Common options for all rpm modes and executables:", + NULL }, + + POPT_AUTOALIAS + POPT_AUTOHELP + POPT_TABLEEND +}; + +/* Stolen from rpm/build/spec.c:rpmspecQuery() */ +rpmSpec +rpmspecGet(rpmts ts, const char * arg) +{ + char * buildRoot = NULL; + int recursing = 0; + char * passPhrase = ""; + char *cookie = NULL; + int anyarch = 1; + int force = 1; + + if (parseSpec(ts, arg, "/", buildRoot, recursing, passPhrase, + cookie, anyarch, force)) { + fprintf(stderr, "query of specfile %s failed, can't parse\n", arg); + return NULL; + } + + return rpmtsSpec(ts); +} + +int +main(int argc, char *argv[]) +{ + poptContext context; + rpmts ts = NULL; + int ec = 0; + rpmSpec spec; + struct Source *source; + Package pkg; + const char *name, *version, *release, *arch, *unused; + const char *package_name; + + /* BEGIN: support to pull out --target from the args list */ + int alen, i; + char *target = NULL; + int args = 1; + int tlen = strlen("--target"); + + + /* walk argv list looking for --target */ + while ((args+1)sources; source; source = source->next) { + char fullSource[PATH_MAX]; + + strncpy(fullSource, source->fullSource, sizeof(fullSource)); + printf("%s.tarballs += SOURCES/%s\n", package_name, basename(fullSource)); + /* computes the SOURCEDIR variable by removing .tar.gz or .tar.bz2 */ + { + char *suffixes[] = {".tar.gz",".tgz",".tar.bz2", NULL}; + char **suffix; + char *suffix_index; + + for (suffix=suffixes ; *suffix ; suffix++) { + printf("# trying %s\n",*suffix); + suffix_index=strstr(fullSource,*suffix); + if (suffix_index) { + char sourcename[PATH_MAX]; + size_t len = (size_t)(suffix_index-fullSource); + strncpy(sourcename,fullSource,len); + sourcename[len]='\0'; + printf ("%s.source := SOURCES/%s\n",package_name,basename(sourcename)); + printf ("%s.codebase := CODEBASES/%s\n",package_name,package_name); + break; + } + } + } + + } + + /* Get SRPM name from name of first package */ + pkg = spec->packages; + name = version = release = NULL; + (void) headerNVR(pkg->header, &name, &version, &release); + if (name && version && release) + printf("%s.srpm := SRPMS/%s-%s-%s.src.rpm\n", + package_name, name, version, release); + + /* Print non-empty packages */ + for (pkg = spec->packages; pkg != NULL; pkg = pkg->next) { + name = version = release = arch = NULL; + (void) headerNEVRA(pkg->header, &name, &unused, &version, &release, &arch); + if (name && version && release && arch) { + if (target != NULL) { + if (strcmp(arch,target)!=0) { + arch=target; + } + } + /* skip empty packages */ + if (pkg->fileList) { + /* attach (add) rpm path to package */ + printf("%s.rpms += RPMS/%s/%s-%s-%s.%s.rpm\n", + package_name, arch, name, version, release, arch); + /* convenience */ + printf("%s.rpmnames += %s\n", + package_name, name); + /* attach path to rpm name */ + printf("%s.rpm-path := RPMS/%s/%s-%s-%s.%s.rpm\n", + name,arch, name, version, release, arch); + /* attach package to rpm name for backward resolution - should be unique */ + printf("%s.package := %s\n", + name,package_name); + } + } + } + + /* export some macros to make */ + /* note : this relies on pl-specific conventions and might be wrong */ + { + char *macros[] = { "release" , "name" , "version" , "taglevel" , NULL } ; + char **nav; + char *macro=malloc(32); + for (nav=macros; *nav; nav++) { + sprintf(macro,"%%{%s}",*nav); + char *value = rpmExpand(macro,NULL); + printf ("%s.rpm-%s := %s\n",package_name,*nav,value); + } + } + + /* export arch */ + printf ("%s.rpm-arch := %s\n",package_name,target); + + spec = freeSpec(spec); + + done: + ts = rpmtsFree(ts); + context = rpmcliFini(context); + return ec; +} diff --git a/trunk/trellis-tags.mk b/trunk/trellis-tags.mk new file mode 100644 index 00000000..96b5b441 --- /dev/null +++ b/trunk/trellis-tags.mk @@ -0,0 +1,51 @@ +# $Id$ + +# build-SVNPATH is now set by vbuild-nightly.sh to avoid duplication + +# trying the 2.6.27 kernel +# VINI is running a prototype of a 2.6.27-based PlanetLab node (aka Trellis) +# Below we include a few Trellis versions of packages +linux-2.6-SVNPATH := http://svn.planet-lab.org/svn/linux-2.6/trunk +madwifi-SVNPATH := http://svn.planet-lab.org/svn/madwifi/tags/madwifi-3878-0 +# Trellis is using a modified util-vserver and util-vserver-pl with the 2.6.27 kernel +util-vserver-SVNBRANCH := trellis +util-vserver-SVNPATH := http://svn.planet-lab.org/svn/util-vserver/branches/trellis +libnl-SVNPATH := http://svn.planet-lab.org/svn/libnl/tags/libnl-1.1-2 +util-vserver-pl-SVNBRANCH := trellis +util-vserver-pl-SVNPATH := http://svn.planet-lab.org/svn/util-vserver-pl/branches/trellis +NodeUpdate-SVNPATH := http://svn.planet-lab.org/svn/NodeUpdate/tags/NodeUpdate-0.5-5 +PingOfDeath-SVNPATH := http://svn.planet-lab.org/svn/PingOfDeath/tags/PingOfDeath-2.2-1 +NodeManager-SVNPATH := http://svn.planet-lab.org/svn/NodeManager/tags/NodeManager-1.8-11 +# Trellis-specific NodeManager plugins +NodeManager-topo-SVNPATH := http://svn.planet-lab.org/svn/NodeManager-topo/trunk +NodeManager-optin-SVNPATH := http://svn.planet-lab.org/svn/NodeManager-optin/trunk +pyplnet-SVNPATH := http://svn.planet-lab.org/svn/pyplnet/tags/pyplnet-4.3-4 +pl_sshd-SVNPATH := http://svn.planet-lab.org/svn/pl_sshd/tags/pl_sshd-1.0-11 +CoDemux-SVNPATH := http://svn.planet-lab.org/svn/CoDemux/tags/CoDemux-0.1-13 +fprobe-ulog-SVNPATH := http://svn.planet-lab.org/svn/fprobe-ulog/tags/fprobe-ulog-1.1.3-0 +pf2slice-SVNPATH := http://svn.planet-lab.org/svn/pf2slice/tags/pf2slice-1.0-2 +Mom-SVNPATH := http://svn.planet-lab.org/svn/Mom/tags/Mom-2.3-1 +iptables-SVNPATH := http://svn.planet-lab.org/svn/iptables/trunk +iproute2-SVNPATH := http://svn.planet-lab.org/svn/iproute2/tags/iproute2-2.6.16-2 +inotify-tools-SVNPATH := http://svn.planet-lab.org/svn/inotify-tools/tags/inotify-tools-3.13-2 +vsys-SVNBRANCH := 0.9 +vsys-SVNPATH := http://svn.planet-lab.org/svn/vsys/tags/vsys-0.9-3 +vsys-scripts-SVNPATH := http://svn.planet-lab.org/svn/vsys-scripts/tags/vsys-scripts-0.95-11 +PLCAPI-SVNPATH := http://svn.planet-lab.org/svn/PLCAPI/tags/PLCAPI-4.3-30 +drupal-SVNPATH := http://svn.planet-lab.org/svn/drupal/tags/drupal-4.7-13 +PLEWWW-SVNPATH := http://svn.planet-lab.org/svn/PLEWWW/tags/PLEWWW-4.3-33 +www-register-wizard-SVNPATH := http://svn.planet-lab.org/svn/www-register-wizard/tags/www-register-wizard-4.3-1 +Monitor-SVNPATH := http://svn.planet-lab.org/svn/Monitor/tags/Monitor-3.0-24 +nodeconfig-SVNPATH := http://svn.planet-lab.org/svn/nodeconfig/tags/nodeconfig-4.3-5 +BootManager-SVNPATH := http://svn.planet-lab.org/svn/BootManager/tags/BootManager-4.3-13 +pypcilib-SVNPATH := http://svn.planet-lab.org/svn/pypcilib/tags/pypcilib-0.2-8 +BootCD-SVNPATH := http://svn.planet-lab.org/svn/BootCD/tags/BootCD-4.2-15 +VserverReference-SVNPATH := http://svn.planet-lab.org/svn/VserverReference/tags/VserverReference-4.2-16 +BootstrapFS-SVNPATH := http://svn.planet-lab.org/svn/BootstrapFS/tags/BootstrapFS-1.0-10 +MyPLC-SVNPATH := http://svn.planet-lab.org/svn/MyPLC/tags/MyPLC-4.3-29 +sfa-SVNPATH := http://svn.planet-lab.org/svn/sfa/tags/sfa-0.9-5 +pyopenssl-SVNPATH := http://svn.planet-lab.org/svn/pyopenssl/tags/pyopenssl-0.9-1 +PLCRT-SVNPATH := http://svn.planet-lab.org/svn/PLCRT/tags/PLCRT-1.0-10 + +# locating the right test directory - see make tests_svnpath +tests-SVNPATH := http://svn.planet-lab.org/svn/tests/tags/tests-4.3-5 diff --git a/trunk/trellis.mk b/trunk/trellis.mk new file mode 100644 index 00000000..aec1f884 --- /dev/null +++ b/trunk/trellis.mk @@ -0,0 +1,396 @@ +# +# declare the packages to be built and their dependencies +# initial version from Mark Huang +# Mark Huang +# Copyright (C) 2003-2006 The Trustees of Princeton University +# rewritten by Thierry Parmentelat - INRIA Sophia Antipolis +# +# $Id: planetlab.mk 14860 2009-08-27 16:18:44Z acb $ +# +# see doc in Makefile +# + +# +# kernel +# +# use a package name with srpm in it: +# so the source rpm is created by running make srpm in the codebase +# + +kernel-MODULES := linux-2.6 +kernel-SPEC := kernel-2.6.spec +kernel-BUILD-FROM-SRPM := yes +ifeq "$(HOSTARCH)" "i386" +kernel-RPMFLAGS:= --target i686 +else +kernel-RPMFLAGS:= --target $(HOSTARCH) +endif +KERNELS += kernel + +kernels: $(KERNELS) +kernels-clean: $(foreach package,$(KERNELS),$(package)-clean) + +ALL += $(KERNELS) +# this is to mark on which image a given rpm is supposed to go +IN_BOOTCD += $(KERNELS) +IN_VSERVER += $(KERNELS) +IN_BOOTSTRAPFS += $(KERNELS) +# turns out myplc installs kernel-vserver +IN_MYPLC += $(KERNELS) + +# +# madwifi +# +madwifi-MODULES := madwifi +madwifi-SPEC := madwifi.spec +madwifi-BUILD-FROM-SRPM := yes +madwifi-DEPEND-DEVEL-RPMS := kernel-devel +madwifi-SPECVARS = kernel_version=$(kernel.rpm-version) \ + kernel_release=$(kernel.rpm-release) \ + kernel_arch=$(kernel.rpm-arch) +ALL += madwifi +IN_BOOTSTRAPFS += madwifi + +# +# util-vserver +# +util-vserver-MODULES := util-vserver +util-vserver-SPEC := util-vserver.spec +#util-vserver-RPMFLAGS:= --without dietlibc +ALL += util-vserver +IN_BOOTSTRAPFS += util-vserver + +# +# libnl - local import +# we need either 1.1 or at least 1.0.pre6 +# rebuild this on centos5 - see kexcludes in build.common +# +local_libnl=false +ifeq "$(DISTRONAME)" "centos5" +local_libnl=true +endif + +ifeq "$(local_libnl)" "true" +libnl-MODULES := libnl +libnl-SPEC := libnl.spec +libnl-BUILD-FROM-SRPM := yes +# this sounds like the thing to do, but in fact linux/if_vlan.h comes with kernel-headers +libnl-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += libnl +IN_BOOTSTRAPFS += libnl +endif + +# +# util-vserver-pl +# +util-vserver-pl-MODULES := util-vserver-pl +util-vserver-pl-SPEC := util-vserver-pl.spec +util-vserver-pl-DEPEND-DEVEL-RPMS := util-vserver-lib util-vserver-devel util-vserver-core +ifeq "$(local_libnl)" "true" +util-vserver-pl-DEPEND-DEVEL-RPMS += libnl libnl-devel +endif +ALL += util-vserver-pl +IN_BOOTSTRAPFS += util-vserver-pl + +# +# NodeUpdate +# +nodeupdate-MODULES := NodeUpdate +nodeupdate-SPEC := NodeUpdate.spec +ALL += nodeupdate +IN_BOOTSTRAPFS += nodeupdate + +# +# ipod +# +ipod-MODULES := PingOfDeath +ipod-SPEC := ipod.spec +ALL += ipod +IN_BOOTSTRAPFS += ipod + +# +# NodeManager +# +nodemanager-MODULES := NodeManager +nodemanager-SPEC := NodeManager.spec +ALL += nodemanager +IN_BOOTSTRAPFS += nodemanager + +# +# pl_sshd +# +sshd-MODULES := pl_sshd +sshd-SPEC := pl_sshd.spec +ALL += sshd +IN_BOOTSTRAPFS += sshd + +# +# codemux: Port 80 demux +# +codemux-MODULES := CoDemux +codemux-SPEC := codemux.spec +codemux-RPMBUILD := sudo bash ./rpmbuild.sh +ALL += codemux +IN_BOOTSTRAPFS += codemux + +# +# fprobe-ulog +# +fprobe-ulog-MODULES := fprobe-ulog +fprobe-ulog-SPEC := fprobe-ulog.spec +ALL += fprobe-ulog +IN_BOOTSTRAPFS += fprobe-ulog + +# +# pf2slice +# +pf2slice-MODULES := pf2slice +pf2slice-SPEC := pf2slice.spec +ALL += pf2slice + +# +# PlanetLab Mom: Cleans up your mess +# +mom-MODULES := Mom +mom-SPEC := pl_mom.spec +ALL += mom +IN_BOOTSTRAPFS += mom + +# +# iptables +# +iptables-MODULES := iptables +iptables-SPEC := iptables.spec +iptables-DEPEND-DEVEL-RPMS := kernel-devel kernel-headers +ALL += iptables +IN_BOOTSTRAPFS += iptables + +# +# iproute +# +iproute-MODULES := iproute2 +iproute-SPEC := iproute.spec +ALL += iproute +IN_BOOTSTRAPFS += iproute +IN_VSERVER += iproute +IN_BOOTCD += iproute + +# +# inotify-tools - local import +# rebuild this on centos5 (not found) - see kexcludes in build.common +# +local_inotify_tools=false +ifeq "$(DISTRONAME)" "centos5" +local_inotify_tools=true +endif + +ifeq "$(local_inotify_tools)" "true" +inotify-tools-MODULES := inotify-tools +inotify-tools-SPEC := inotify-tools.spec +inotify-tools-BUILD-FROM-SRPM := yes +IN_BOOTSTRAPFS += inotify-tools +ALL += inotify-tools +endif + +# +# vsys +# +vsys-MODULES := vsys +vsys-SPEC := vsys.spec +ifeq "$(local_inotify_tools)" "true" +vsys-DEPEND-DEVEL-RPMS := inotify-tools inotify-tools-devel +endif +IN_BOOTSTRAPFS += vsys +ALL += vsys + +# +# vsys-scripts +# +vsys-scripts-MODULES := vsys-scripts +vsys-scripts-SPEC := vsys-scripts.spec +IN_BOOTSTRAPFS += vsys-scripts +ALL += vsys-scripts + +# +# PLCAPI +# +PLCAPI-MODULES := PLCAPI +PLCAPI-SPEC := PLCAPI.spec +ALL += PLCAPI +IN_MYPLC += PLCAPI + +# +# drupal +# +drupal-MODULES := drupal +drupal-SPEC := drupal.spec +drupal-BUILD-FROM-SRPM := yes +ALL += drupal +IN_MYPLC += drupal + +# +# use the plewww module instead +# +plewww-MODULES := PLEWWW +plewww-SPEC := plewww.spec +ALL += plewww +IN_MYPLC += plewww + +# +# www-register-wizard +# +www-register-wizard-MODULES := www-register-wizard +www-register-wizard-SPEC := www-register-wizard.spec +ALL += www-register-wizard +IN_MYPLC += www-register-wizard + +# +# monitor +# +monitor-MODULES := Monitor +monitor-SPEC := Monitor.spec +ALL += monitor +IN_BOOTSTRAPFS += monitor + +# +# PLC RT +# +plcrt-MODULES := PLCRT +plcrt-SPEC := plcrt.spec +ALL += plcrt + +# +# zabbix +# +zabbix-MODULES := Monitor +zabbix-SPEC := zabbix.spec +zabbix-BUILD-FROM-SRPM := yes +ALL += zabbix + +# +# pyopenssl +# +pyopenssl-MODULES := pyopenssl +pyopenssl-SPEC := pyOpenSSL.spec +pyopenssl-BUILD-FROM-SRPM := yes +ALL += pyopenssl + + +# +# sfa - Slice Facility Architecture +# +sfa-MODULES := sfa +sfa-SPEC := sfa.spec +ALL += sfa + +# +# nodeconfig +# +nodeconfig-MODULES := nodeconfig build +nodeconfig-SPEC := nodeconfig.spec +ALL += nodeconfig +IN_MYPLC += nodeconfig + +# +# bootmanager +# +bootmanager-MODULES := BootManager +bootmanager-SPEC := bootmanager.spec +ALL += bootmanager +IN_MYPLC += bootmanager + +# +# pypcilib : used in bootcd +# +pypcilib-MODULES := pypcilib +pypcilib-SPEC := pypcilib.spec +ALL += pypcilib +IN_BOOTCD += pypcilib + +# +# pyplnet +# +pyplnet-MODULES := pyplnet +pyplnet-SPEC := pyplnet.spec +ALL += pyplnet +IN_BOOTSTRAPFS += pyplnet +IN_MYPLC += pyplnet +IN_BOOTCD += pyplnet + +# +# bootcd +# +bootcd-MODULES := BootCD build +bootcd-SPEC := bootcd.spec +bootcd-RPMBUILD := sudo bash ./rpmbuild.sh +bootcd-DEPEND-PACKAGES := $(IN_BOOTCD) +bootcd-DEPEND-FILES := RPMS/yumgroups.xml +bootcd-RPMDATE := yes +ALL += bootcd +IN_MYPLC += bootcd + +# +# vserver : reference image for slices +# +vserver-MODULES := VserverReference build +vserver-SPEC := vserver-reference.spec +vserver-DEPEND-PACKAGES := $(IN_VSERVER) +vserver-DEPEND-FILES := RPMS/yumgroups.xml +vserver-RPMDATE := yes +ALL += vserver +IN_BOOTSTRAPFS += vserver + +# +# bootstrapfs +# +bootstrapfs-MODULES := BootstrapFS build +bootstrapfs-SPEC := bootstrapfs.spec +bootstrapfs-RPMBUILD := sudo bash ./rpmbuild.sh +bootstrapfs-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) +bootstrapfs-DEPEND-FILES := RPMS/yumgroups.xml +bootstrapfs-RPMDATE := yes +ALL += bootstrapfs +IN_MYPLC += bootstrapfs + +# +# noderepo +# +# all rpms resulting from packages marked as being in bootstrapfs and vserver +NODEREPO_RPMS = $(foreach package,$(IN_BOOTSTRAPFS) $(IN_VSERVER),$($(package).rpms)) +# replace space with +++ (specvars cannot deal with spaces) +SPACE=$(subst x, ,x) +NODEREPO_RPMS_3PLUS = $(subst $(SPACE),+++,$(NODEREPO_RPMS)) + +noderepo-MODULES := BootstrapFS +noderepo-SPEC := noderepo.spec +noderepo-RPMBUILD := sudo bash ./rpmbuild.sh +# package requires all regular packages +noderepo-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) $(IN_VSERVER) +noderepo-DEPEND-FILES := RPMS/yumgroups.xml +#export rpm list to the specfile +noderepo-SPECVARS = node_rpms_plus=$(NODEREPO_RPMS_3PLUS) +noderepo-RPMDATE := yes +ALL += noderepo +IN_MYPLC += noderepo + +# +# MyPLC : lightweight packaging, dependencies are yum-installed in a vserver +# +myplc-MODULES := MyPLC build +myplc-SPEC := myplc.spec +myplc-DEPEND-FILES := myplc-release RPMS/yumgroups.xml +ALL += myplc + +# myplc-docs only contains docs for PLCAPI and NMAPI, but +# we still need to pull MyPLC, as it is where the specfile lies, +# together with the utility script docbook2drupal.sh +myplc-docs-MODULES := MyPLC PLCAPI NodeManager Monitor +myplc-docs-SPEC := myplc-docs.spec +ALL += myplc-docs + +# using some other name than myplc-release, as this is a make target already +release-MODULES := MyPLC +release-SPEC := myplc-release.spec +release-RPMDATE := yes +ALL += release + diff --git a/trunk/vbuild-fedora-mirror.sh b/trunk/vbuild-fedora-mirror.sh new file mode 100755 index 00000000..d87fa279 --- /dev/null +++ b/trunk/vbuild-fedora-mirror.sh @@ -0,0 +1,248 @@ + +#!/bin/bash +# this can help you create/update your fedora mirror +# $Id$ + +COMMAND=$(basename $0) +LOGDIR=/var/log/fedora-mirroring +DATE=$(date '+%Y-%m-%d-%H-%M') +LOG=${LOGDIR}/${DATE}.log + +dry_run= +verbose= +log= +skip_core=true +root=/mirror/ + + +dhozac_url=rsync://rpm.hozac.com/dhozac/centos/ + +us_fedora_url=rsync://mirrors.kernel.org/fedora +us_centos_url=rsync://mirrors.rit.edu/centos +us_epel_url=rsync://rsync.gtlib.gatech.edu/fedora-epel + +# ircam's fedora8 repo has been turned off +#eu_fedora_url=rsync://mirrors.ircam.fr/fedora-linux +eu_fedora_url=rsync://mirror.ovh.net/download.fedora.redhat.com/linux +eu_centos_url=rsync://mirrors.ircam.fr/CentOS +eu_epel_url=rsync://mirrors.ircam.fr/fedora-epel + +pl_fedora_url=$eu_fedora_url +pl_centos_url=rsync://ftp.tpnet.pl/centos +pl_epel_url=rsync://ftp.icm.edu.pl/pub/Linux/fedora/linux/epel + +jp_fedora_url="jp_fedora_url-needs-to-be-defined" +jp_centos_url="jp_centos_url-needs-to-be-defined" +jp_epel_url="jp_epel_url-needs-to-be-defined" + +default_distroname="centos5.4" +all_distronames="f10 f11 centos5.3 centos5.4 epel5" +default_arch="i386" +all_archs="i386 x86_64" + +case $(hostname) in + *.fr|*.de|*.uk) + fedora_url=$eu_fedora_url ; centos_url=$eu_centos_url ; epel_url=$eu_epel_url ;; + *.pl) + fedora_url=$pl_fedora_url ; centos_url=$pl_centos_url ; epel_url=$pl_epel_url ;; + *.jp) + fedora_url=$jp_fedora_url ; centos_url=$jp_centos_url ; epel_url=$jp_epel_url ;; + *) + fedora_url=$us_fedora_url ; centos_url=$us_centos_url ; epel_url=$us_epel_url ;; +esac + +function mirror_distro_arch () { + distroname=$1; shift + arch=$1; shift + + distroname=$(echo $distroname | tr '[A-Z]' '[a-z]') + case $distroname in + fc*[1-6]) + distroindex=$(echo $distroname | sed -e "s,fc,,g") + distro="Fedora Core" + rsyncurl=$fedora_url + ;; + f*[7-9]|f1?) + distroindex=$(echo $distroname | sed -e "s,f,,g") + distro="Fedora" + rsyncurl=$fedora_url + ;; + centos[4-5]|centos[4-5].[0-9]) + distroindex=$(echo $distroname | sed -e "s,centos,,g") + distro="CentOS" + rsyncurl=$centos_url + ;; + epel5) + distroindex=5 + distro=epel + rsyncurl=$epel_url + ;; + dhozac) + distroindex=5 + distro="dhozac" + rsyncurl=$dhozac_url + ;; + *) + echo "WARNING -- Unknown distribution $distroname -- skipped" + return 1 + ;; + esac + + excludelist="debug/ iso/ ppc/ source/" + options="--archive --compress --delete --delete-excluded $dry_run $verbose" + [ -n "$(rsync --help | grep no-motd)" ] && options="$options --no-motd" + for e in $excludelist; do + options="$options --exclude $e" + done + + echo ">>>>>>>>>>>>>>>>>>>> root=$root distroname=$distroname arch=$arch rsyncurl=$rsyncurl" + [ -n "$verbose" ] && echo "rsync options=$options" + + RES=1 + paths="" + case $distro in + [Ff]edora*) + case $distroindex in + 2|4|6) + [ -z "$skip_core" ] && paths="core/$distroindex/$arch/os/" + paths="$paths core/updates/$distroindex/$arch/ extras/$distroindex/$arch/" + RES=0 + ;; + 7|8|9|1?) + [ -z "$skip_core" ] && paths="releases/$distroindex/Everything/$arch/os/" + paths="$paths updates/$distroindex/$arch/" + # f8 and f9 have the additional newkey repo + case $distroindex in + 8|9) paths="$paths updates/$distroindex/${arch}.newkey/" ;; + esac + RES=0 + ;; + esac + localpath=fedora + ;; + + CentOS*) + case $distroindex in + 5*) + [ -z "$skip_core" ] && paths="$distroindex/os/$arch/" + paths="$paths $distroindex/updates/$arch/" + RES=0 + ;; + esac + localpath=centos + ;; + + epel*) + case $distroindex in + 5) + paths="$paths $distroindex/$arch/" + RES=0 + ;; + esac + localpath=epel + ;; + + dhozac*) + case $distroindex in + 5) + # leave off trailing '/' + paths="$paths $distroindex/vserver/$arch" + RES=0 + ;; + esac + localpath=dhozac + ;; + + esac + + if [ "$RES" = 1 ] ; then + echo "DISTRIBUTION $distro $distroindex CURRENTLY UNSUPPORTED - skipped" + else + for repopath in $paths; do + echo "===== $distro -> $distroindex $repopath" + [ -z "$dry_run" ] && mkdir -p ${root}/${localpath}/${repopath} + command="rsync $options ${rsyncurl}/${repopath} ${root}/${localpath}/${repopath}" + echo $command + $command + done + fi + + echo "<<<<<<<<<<<<<<<<<<<< $distroname $arch" + + return $RES +} + +function usage () { + echo "Usage: $COMMAND [-n] [-v] [-c] [-r root] [-u|U rsyncurl] [-e|-j] [-f distroname|-F] [-a arch|-A]" + echo "Defaults to -r $root -f $default_distroname -a $default_arch" + echo "Default urls : $fedora_url $centos_url" + echo "Options:" + echo " -n : dry run" + echo " -v : verbose" + echo " -l : turns on autologging in $LOGDIR" + echo " -c : skips core repository (default)" + echo " -C : force syncing core repository" + echo " -r root (default is $root)" + echo " -u rsyncurl for fedora (default is $fedora_url)" + echo " -U rsyncurl for centos (default is $centos_url)" + echo " -E rsyncurl for epel (default is $epel_url)" + echo " -s : uses standard (US) mirrors $us_fedora_url $us_centos_url $us_epel_url" + echo " -e : uses European mirrors $eu_fedora_url $eu_centos_url $eu_epel_url" + echo " -j : uses Japanese mirrors $jp_fedora_url $jp_centos_url $jp_epel_url" + echo " -f distroname - use vserver convention, e.g. f8 or centos5" + echo " -F : for distroname in $all_distronames" + echo " -a arch - use yum convention" + echo " -A : for arch in $all_archs" + exit 1 +} + +function run () { + RES=0 + for distroname in $distronames ; do + for arch in $archs; do + mirror_distro_arch "$distroname" "$arch" || RES=1 + done + done + return $RES +} + +function main () { + distronames="" + archs="" + while getopts "nvlcCr:u:U:E:sejf:Fa:Ah" opt ; do + case $opt in + n) dry_run=--dry-run ;; + v) verbose=--verbose ;; + l) log=true ;; + c) skip_core=true ;; + C) skip_core= ;; + r) root=$OPTARG ;; + u) fedora_url=$OPTARG ;; + U) centos_url=$OPTARG ;; + E) epel_url=$OPTARG ;; + s) fedora_url=$us_fedora_url ; centos_url=$us_centos_url ; epel_url=$us_epel_url;; + e) fedora_url=$eu_fedora_url ; centos_url=$eu_centos_url ; epel_url=$eu_epel_url ;; + j) fedora_url=$jp_fedora_url ; centos_url=$jp_centos_url ; epel_url=$jp_epel_url ;; + f) distronames="$distronames $OPTARG" ;; + F) distronames="$distronames $all_distronames" ;; + a) archs="$archs $OPTARG" ;; + A) archs="$archs $all_archs" ;; + h|*) usage ;; + esac + done + shift $(($OPTIND-1)) + [[ -n "$@" ]] && usage + [ -z "$distronames" ] && distronames=$default_distroname + [ -z "$archs" ] && archs=$default_arch + + # auto log : if specified + if [ -n "$log" ] ; then + mkdir -p $LOGDIR + run &> $LOG + else + run + fi + exit $? +} + +main "$@" diff --git a/trunk/vbuild-init-vserver.sh b/trunk/vbuild-init-vserver.sh new file mode 100755 index 00000000..fa1c4833 --- /dev/null +++ b/trunk/vbuild-init-vserver.sh @@ -0,0 +1,472 @@ +#!/bin/bash +# -*-shell-*- +# $Id$ + +#shopt -s huponexit + +COMMAND=$(basename $0) +DIRNAME=$(dirname $0) + +# pkgs parsing utilities +PATH=$(dirname $0):$PATH . build.common + +DEFAULT_FCDISTRO=f8 +DEFAULT_PLDISTRO=planetlab +DEFAULT_PERSONALITY=linux32 +DEFAULT_IFNAME=eth0 + +COMMAND_VBUILD="vbuild-init-vserver.sh" +COMMAND_MYPLC="vtest-init-vserver.sh" + +function failure () { + echo "$COMMAND : Bailing out" + exit 1 +} + +# overwrite vserver's internal yum config from what is in +# .distributions//yum/yum.conf and /yum.repos.d + +function configure_yum_in_vserver () { + set -x + set -e + trap failure ERR INT + + vserver=$1; shift + fcdistro=$1; shift + + templates=/etc/vservers/.distributions/${fcdistro} + if [ -f ${templates}/yum/yum.conf ] ; then + echo "Initializing yum.conf in $vserver from ${templates}/yum" + sed -e "s!@YUMETCDIR@!/etc!g; + s!@YUMCACHEDIR@!/var/cache/yum!g; + s!@YUMLOGDIR@!/var/log!g; + s!@YUMLOCKDIR@!/var/lock!g; + " ${templates}/yum/yum.conf > /vservers/$vserver/etc/yum.conf + + # post process the various @...@ variables from this yum.conf file. + else + echo "Using $fcdistro default for yum.conf" + fi + + if [ -d ${templates}/yum.repos.d ] ; then + echo "Initializing yum.repos.d in $vserver from ${templates}/yum.repos.d" + rm -rf /vservers/$vserver/etc/yum.repos.d + tar -C ${templates} -cf - yum.repos.d | tar -C /vservers/$vserver/etc -xvf - + else + echo "Cannot initialize yum.repos.d in $vserver" + fi + + if [ -n "$MYPLC_MODE" ] ; then + if [ ! -d /vservers/$vserver/etc/yum.repos.d ] ; then + echo "WARNING : cannot create myplc repo" + else + # exclude kernel from fedora repos + for repo in /vservers/$vserver/etc/yum.repos.d/* ; do + [ -f $repo ] && yumconf_exclude $repo "exclude=$pl_KEXCLUDES" + done + # the build repo is not signed at this stage + cat > /vservers/$vserver/etc/yum.repos.d/myplc.repo <> /etc/vservers/$vserver/personality + fi + fi + + if [ -n "$VBUILD_MODE" ] ; then + ### capabilities required for a build vserver + # set up appropriate vserver capabilities to mount, mknod and IPC_LOCK + BCAPFILE=/etc/vservers/$vserver/bcapabilities + touch $BCAPFILE + cap=$(grep ^CAP_SYS_ADMIN /etc/vservers/$vserver/bcapabilities | wc -l) + [ $cap -eq 0 ] && echo 'CAP_SYS_ADMIN' >> /etc/vservers/$vserver/bcapabilities + cap=$(grep ^CAP_MKNOD /etc/vservers/$vserver/bcapabilities | wc -l) + [ $cap -eq 0 ] && echo 'CAP_MKNOD' >> /etc/vservers/$vserver/bcapabilities + cap=$(grep ^CAP_IPC_LOCK /etc/vservers/$vserver/bcapabilities | wc -l) + [ $cap -eq 0 ] && echo 'CAP_IPC_LOCK' >> /etc/vservers/$vserver/bcapabilities + else + ### capabilities required for a myplc vserver + # for /etc/plc.d/gpg - need to init /dev/random + cap=$(grep ^CAP_MKNOD /etc/vservers/$vserver/bcapabilities | wc -l) + [ $cap -eq 0 ] && echo 'CAP_MKNOD' >> /etc/vservers/$vserver/bcapabilities + cap=$(grep ^CAP_NET_BIND_SERVICE /etc/vservers/$vserver/bcapabilities | wc -l) + [ $cap -eq 0 ] && echo 'CAP_NET_BIND_SERVICE' >> /etc/vservers/$vserver/bcapabilities + fi + + if [ "$pkg_method" = "yum" ] ; then + $personality vyum $vserver -- -y install yum + # ditto + for i in $(seq 20) ; do + $personality vserver $VERBOSE $vserver pkgmgmt internalize && break || true + echo "* ${i}-th attempt to 'vserver pkgmgmt internalize' failed - waiting for 3 seconds" + sleep 3 + done + fi + + # start the vserver so we can do the following operations + # redirect out/err to protect against the vserver's init sequence getting stalled + # mostly used for f10 vservers created remotely through ssh + $personality vserver $VERBOSE $vserver start >& /dev/null + [ "$pkg_method" = "yum" ] && $personality vserver $VERBOSE $vserver exec sh -c "rm -f /var/lib/rpm/__db*" + [ "$pkg_method" = "yum" ] && $personality vserver $VERBOSE $vserver exec rpm --rebuilddb + + # check if the vserver kernel is using VSERVER_DEVICE (vdevmap) support + need_vdevmap=$(grep "CONFIG_VSERVER_DEVICE=y" /boot/config-$(uname -r) | wc -l) + + if [ $need_vdevmap -eq 1 ] ; then + ctx=$(cat /etc/vservers/$vserver/context) + vdevmap --set --xid $ctx --open --create --target /dev/null + vdevmap --set --xid $ctx --open --create --target /dev/root + fi + + # minimal config in the vserver for yum to work + [ "$pkg_method" = "yum" ] && configure_yum_in_vserver $vserver $fcdistro + + # set up resolv.conf + cp /etc/resolv.conf /vservers/$vserver/etc/resolv.conf + # and /etc/hosts for at least localhost + [ -f /vservers/$vserver/etc/hosts ] || echo "127.0.0.1 localhost localhost.localdomain" > /vservers/$vserver/etc/hosts + +} + +function devel_or_vtest_tools () { + + set -x + set -e + trap failure ERR INT + + vserver=$1; shift + fcdistro=$1; shift + pldistro=$1; shift + personality=$1; shift + + pkg_method=$(package_method $fcdistro) + + # check for .pkgs file based on pldistro + if [ -n "$VBUILD_MODE" ] ; then + pkgsname=devel.pkgs + else + pkgsname=vtest.pkgs + fi + pkgsfile=$(pl_locateDistroFile $DIRNAME $pldistro $pkgsname) + + ### install individual packages, then groups + # get target arch - use uname -i here (we want either x86_64 or i386) + vserver_arch=$($personality vserver $vserver exec uname -i) + # on debian systems we get arch through the 'arch' command + [ "$vserver_arch" = "unknown" ] && vserver_arch=$($personality vserver $vserver exec arch) + + packages=$(pl_getPackages -a $vserver_arch $fcdistro $pldistro $pkgsfile) + groups=$(pl_getGroups -a $vserver_arch $fcdistro $pldistro $pkgsfile) + + [ "$pkg_method" = yum ] && [ -n "$packages" ] && $personality vserver $vserver exec yum -y install $packages + [ "$pkg_method" = yum ] && for group_plus in $groups; do + group=$(echo $group_plus | sed -e "s,+++, ,g") + $personality vserver $vserver exec yum -y groupinstall "$group" + done + + [ "$pkg_method" = debootstrap ] && $personality vserver $vserver exec apt-get update + [ "$pkg_method" = debootstrap ] && for package in $packages ; do + $personality vserver $vserver exec apt-get install -y $package + done + + return 0 +} + +function post_install () { + if [ -n "$VBUILD_MODE" ] ; then + post_install_vbuild "$@" + else + post_install_myplc "$@" + fi + # setup localtime from the host + vserver=$1; shift + cp /etc/localtime /vservers/$vserver/etc/localtime +} + +function post_install_vbuild () { + + set -x + set -e + trap failure ERR INT + + vserver=$1; shift + personality=$1; shift + +### From myplc-devel-native.spec +# be careful to backslash $ in this, otherwise it's the root context that's going to do the evaluation + cat << EOF | $personality vserver $VERBOSE $vserver exec bash -x + # set up /dev/loop* in vserver + for i in \$(seq 0 255) ; do + mknod -m 640 /dev/loop\$i b 7 \$i + done + + # create symlink for /dev/fd + [ ! -e "/dev/fd" ] && ln -s /proc/self/fd /dev/fd + + # modify /etc/rpm/macros to not use /sbin/new-kernel-pkg + sed -i 's,/sbin/new-kernel-pkg:,,' /etc/rpm/macros + if [ -h "/sbin/new-kernel-pkg" ] ; then + filename=\$(readlink -f /sbin/new-kernel-pkg) + if [ "\$filename" == "/sbin/true" ] ; then + echo "WARNING: /sbin/new-kernel-pkg symlinked to /sbin/true" + echo "\tmost likely /etc/rpm/macros has /sbin/new-kernel-pkg declared in _netsharedpath." + echo "\tPlease remove /sbin/new-kernel-pkg from _netsharedpath and reintall mkinitrd." + exit 1 + fi + fi + + # customize root's prompt + cat << PROFILE > /root/.profile +export PS1="[$vserver] \\w # " +PROFILE + + uid=2000 + gid=2000 + + # add a "build" user to the system + builduser=\$(grep "^build:" /etc/passwd | wc -l) + if [ \$builduser -eq 0 ] ; then + groupadd -o -g \$gid build; + useradd -o -c 'Automated Build' -u \$uid -g \$gid -n -M -s /bin/bash build; + fi + +# Allow build user to build certain RPMs as root + if [ -f /etc/sudoers ] ; then + buildsudo=\$(grep "^build.*ALL=(ALL).*NOPASSWD:.*ALL" /etc/sudoers | wc -l) + if [ \$buildsudo -eq 0 ] ; then + echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + fi + sed -i 's,^Defaults.*requiretty,#Defaults requiretty,' /etc/sudoers + fi +# +EOF + +} + +function post_install_myplc () { + set -x + set -e + trap failure ERR INT + + vserver=$1; shift + personality=$1; shift + +# be careful to backslash $ in this, otherwise it's the root context that's going to do the evaluation + cat << EOF | $personality vserver $VERBOSE $vserver exec bash -x + + # create /etc/sysconfig/network if missing + [ -f /etc/sysconfig/network ] || echo NETWORKING=yes > /etc/sysconfig/network + + # create symlink for /dev/fd + [ ! -e "/dev/fd" ] && ln -s /proc/self/fd /dev/fd + + # turn off regular crond, as plc invokes plc_crond + chkconfig crond off + + # take care of loginuid in /etc/pam.d + sed -i "s,#*\(.*loginuid.*\),#\1," /etc/pam.d/* + + # customize root's prompt + cat << PROFILE > /root/.profile +export PS1="[$vserver] \\w # " +PROFILE + +EOF +} + +# parses ifconfig's output to find out ip address and mask +# will then be passed to vserver as e.g. --interface 138.96.250.126/255.255.0.0 +# default is to use lo, that's enough for local mirrors +# use -i eth0 in case your fedora mirror is on a separate box on the network +function vserverIfconfig () { + ifname=$1; shift + local result="" + line=$(ifconfig $ifname 2> /dev/null | grep 'inet addr') + if [ -n "$line" ] ; then + set $line + for word in "$@" ; do + addr=$(echo $word | sed -e s,[aA][dD][dD][rR]:,,) + mask=$(echo $word | sed -e s,[mM][aA][sS][kK]:,,) + if [ "$word" != "$addr" ] ; then + result="${addr}" + elif [ "$word" != "$mask" ] ; then + result="${result}/${mask}" + fi + done + fi + if [ -z "$result" ] ; then + echo "vserverIfconfig failed to locate $ifname" + exit 1 + else + echo $result + fi +} + +function usage () { + set +x + echo "Usage: $COMMAND_VBUILD [options] vserver-name [ -- vserver-options ]" + echo "Usage: $COMMAND_MYPLC [options] vserver-name repo-url [ -- vserver-options ]" + echo "Requirements: you need to have a vserver-compliant kernel," + echo " as well as the util-vserver RPM installed" + echo "Description:" + echo " This command creates a fresh vserver instance, for building, or running, myplc" + echo "Supported options" + echo " -f fcdistro - for creating the root filesystem - defaults to $DEFAULT_FCDISTRO" + echo " -d pldistro - defaults to $DEFAULT_PLDISTRO" + echo " -p personality - defaults to $DEFAULT_PERSONALITY" + echo " -i ifname: determines ip and netmask attached to ifname, and passes it to the vserver" + echo " -v : verbose - passes -v to calls to vserver" + echo "vserver-options" + echo " all args after the optional -- are passed to vserver build " + echo " typical usage is e.g. --interface eth0:200.150.100.10/24" + exit 1 +} + +### parse args and +function main () { + + set -e + trap failure ERR INT + + case "$COMMAND" in + $COMMAND_VBUILD) + VBUILD_MODE=true ;; + $COMMAND_MYPLC) + MYPLC_MODE=true;; + *) + usage ;; + esac + + VERBOSE= + IFNAME="" + VSERVER_OPTIONS="" + while getopts "f:d:p:i:v" opt ; do + case $opt in + f) fcdistro=$OPTARG;; + d) pldistro=$OPTARG;; + p) personality=$OPTARG;; + i) IFNAME=$OPTARG;; + v) VERBOSE="-v" ;; + *) usage ;; + esac + done + + shift $(($OPTIND - 1)) + + # parse fixed arguments + [[ -z "$@" ]] && usage + vserver=$1 ; shift + if [ -n "$MYPLC_MODE" ] ; then + [[ -z "$@" ]] && usage + REPO_URL=$1 ; shift + fi + + # parse vserver options + if [[ -n "$@" ]] ; then + if [ "$1" == "--" ] ; then + shift + VSERVER_OPTIONS="$@" + else + usage + fi + fi + + # with new util-vserver, it is mandatory to provide an IP even for building + if [ -n "$VBUILD_MODE" ] ; then + [ -z "$IFNAME" ] && IFNAME=$DEFAULT_IFNAME + fi + if [ -n "$IFNAME" ] ; then + localip=$(vserverIfconfig $IFNAME) + VSERVER_OPTIONS="$VSERVER_OPTIONS --interface $localip" + fi + + [ -z "$fcdistro" ] && fcdistro=$DEFAULT_FCDISTRO + [ -z "$pldistro" ] && pldistro=$DEFAULT_PLDISTRO + [ -z "$personality" ] && personality=$DEFAULT_PERSONALITY + + setup_vserver $vserver $fcdistro $personality + devel_or_vtest_tools $vserver $fcdistro $pldistro $personality + post_install $vserver $personality + +} + +main "$@" diff --git a/trunk/vbuild-nightly.sh b/trunk/vbuild-nightly.sh new file mode 100755 index 00000000..62bae812 --- /dev/null +++ b/trunk/vbuild-nightly.sh @@ -0,0 +1,643 @@ +#!/bin/bash +REVISION=$(echo '$Revision$' | sed -e 's,\$,,g' -e 's,^\w*:\s,,' ) + +COMMANDPATH=$0 +COMMAND=$(basename $0) + +# default values, tunable with command-line options +DEFAULT_FCDISTRO=centos5 +DEFAULT_PLDISTRO=planetlab +DEFAULT_PERSONALITY=linux32 +DEFAULT_BASE="@DATE@--@PLDISTRO@-@FCDISTRO@-@PERSONALITY@" +DEFAULT_build_SVNPATH="http://svn.planet-lab.org/svn/build/trunk" +DEFAULT_IFNAME=eth0 + +# default gpg path used in signing yum repo +DEFAULT_GPGPATH="/etc/planetlab" +# default email to use in gpg secring +DEFAULT_GPGUID="root@$( /bin/hostname )" + +DEFAULT_TESTCONFIG="default" + +# for publishing results, and the tests settings +x=$(hostname) +y=$(hostname|sed -e s,inria,,) +# INRIA defaults +if [ "$x" != "$y" ] ; then + DEFAULT_WEBPATH="/build/@PLDISTRO@/" + DEFAULT_TESTBUILDURL="http://build.onelab.eu/" + # this is where the buildurl is pointing towards + DEFAULT_WEBROOT="/build/" + DEFAULT_TESTMASTER="testmaster.onelab.eu" +else + DEFAULT_WEBPATH="/build/@FCDISTRO@/@PLDISTRO@/" + DEFAULT_TESTBUILDURL="http://build.planet-lab.org/" + # this is where the buildurl is pointing towards + DEFAULT_WEBROOT="/build/" + DEFAULT_TESTMASTER="manager.test.planet-lab.org" +fi + +#################### +# assuming vserver runs in UTC +DATE=$(date +'%Y.%m.%d') + +# temporary - wrap a quick summary of suspicious stuff +# this is to focus on installation that go wrong +# use with care, a *lot* of other things can go bad as well +function summary () { + from=$1; shift + echo "******************** BEG SUMMARY" + python - $from <>>',line, + # an 'installing' line with messages afterwards : needs to be echoed + elif m_installing_err.match(line): + (installing,error)=m_installing_err.match(line).groups() + print '>>>',installing + print '>>>',error + echo=True + # closing an 'installing' section + elif m_installing_end.match(line): + echo=False + # any 'installing' line + elif m_installing_any.match(line): + if echo: + installing=m_installing_any.match(line).group(1) + print '>>>',installing + echo=False + # print lines when echo is true + else: + if echo: print '>>>',line, + f.close() + except: + print 'Failed to analyze',filename + +for arg in sys.argv[1:]: + summary(arg) +EOF + echo "******************** END SUMMARY" +} + + +# Notify recipient of failure or success, manage various stamps +function failure() { + set -x + # early stage ? - let's not create /build/@PLDISTRO@ + if [ ! -d ${WEBPATH} ] ; then + WEBPATH=/tmp + WEBLOG=/tmp/vbuild-early.log.txt + fi + cp $LOG ${WEBLOG} + summary $LOG >> ${WEBLOG} + (echo -n "============================== $COMMAND: failure at " ; date ; tail --lines=1000 $WEBLOG) > ${WEBLOG}.ko + if [ -n "$MAILTO" ] ; then + ( \ + echo "See full build log at ${LOG_URL}" ; \ + echo "and tail version at ${LOG_URL}.ko" ; \ + echo "See complete set of testlogs at ${TESTLOGS_URL}" ; \ + tail --lines=1000 ${WEBLOG} ) | mail -s "Failures with ${MAIL_SUBJECT} ${BASE}" $MAILTO + fi + exit 1 +} + +function success () { + set -x + # early stage ? - let's not create /build/@PLDISTRO@ + if [ ! -d ${WEBPATH} ] ; then + WEBPATH=/tmp + WEBLOG=/tmp/vbuild-early-$(date +%Y-%m-%d).log.txt + fi + cp $LOG ${WEBLOG} + summary $LOG >> ${WEBLOG} + if [ -n "$DO_TEST" ] ; then + ( \ + echo "Successfully built and tested" ; \ + echo "See full build log at ${LOG_URL}" ; \ + echo "See complete set of testlogs at ${TESTLOGS_URL}" ; \ + ) > ${WEBLOG}.pass + rm -f ${WEBLOG}.pkg-ok ${WEBLOG}.ko + else + ( \ + echo "Successful package-only build, no test requested" ; \ + echo "See full build log at ${LOG_URL}" ; \ + ) > ${WEBLOG}.pkg-ok + rm -f ${WEBLOG}.ko + fi + if [ -n "$MAILTO" ] ; then + ( \ + echo "$PLDISTRO ($BASE) build for $FCDISTRO completed on $(date)" ; \ + echo "See full build log at ${LOG_URL}" ; \ + [ -n "$DO_TEST" ] && echo "See complete set of testlogs at ${TESTLOGS_URL}" ) \ + | mail -s "Success with ${MAIL_SUBJECT} ${BASE}" $MAILTO + fi + # XXX For some reason, we haven't been getting this email for successful builds. If this sleep + # doesn't fix the problem, I'll remove it -- Sapan. + sleep 5 + exit 0 +} + +# run in the vserver - do not manage success/failure, will be done from the root ctx +function build () { + set -x + set -e + + echo -n "============================== Starting $COMMAND:build on " + date + + cd /build + show_env + + echo "Running make IN $(pwd)" + + # stuff our own variable settings + MAKEVARS=("PLDISTRO=${PLDISTRO}" "${MAKEVARS[@]}") + MAKEVARS=("PLDISTROTAGS=${PLDISTROTAGS}" "${MAKEVARS[@]}") + MAKEVARS=("build-SVNPATH=${build_SVNPATH}" "${MAKEVARS[@]}") + MAKEVARS=("PERSONALITY=${PERSONALITY}" "${MAKEVARS[@]}") + MAKEVARS=("MAILTO=${MAILTO}" "${MAKEVARS[@]}") + MAKEVARS=("WEBPATH=${WEBPATH}" "${MAKEVARS[@]}") + MAKEVARS=("TESTBUILDURL=${TESTBUILDURL}" "${MAKEVARS[@]}") + MAKEVARS=("WEBROOT=${WEBROOT}" "${MAKEVARS[@]}") + + MAKEVARS=("BASE=${BASE}" "${MAKEVARS[@]}") + + # stage1 + make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true + # store tests_svnpath + make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true tests_svnpath + # versions + make -C /build $DRY_RUN "${MAKEVARS[@]}" versions + # actual stuff + make -C /build $DRY_RUN "${MAKEVARS[@]}" "${MAKETARGETS[@]}" + +} + +# this was formerly run in the myplc-devel chroot but now is run in the root context, +# this is so that the .ssh config gets done manually, and once and for all +function run_log () { + set -x + set -e + trap failure ERR INT + + echo -n "============================== Starting $COMMAND:run_log on $(date)" + + # where to find TESTS_SVNPATH + stamp=/vservers/$BASE/build/tests_svnpath + if [ ! -f $stamp ] ; then + echo "$COMMAND: Cannot figure TESTS_SVNPATH from missing $stamp" + failure + exit 1 + fi + TESTS_SVNPATH=$(cat $stamp) + # don't need the tests fulltree anymore + TESTS_SYSTEM_SVNPATH=${TESTS_SVNPATH}/system + + ### the URL to the RPMS/ location + url="" + for a in i386 x86_64; do + archdir=/vservers/$BASE/build/RPMS/$a + if [ -d $archdir ] ; then + # where was that installed + url=$(echo $archdir | sed -e "s,/vservers/${BASE}/build,${WEBPATH}/${BASE},") + url=$(echo $url | sed -e "s,${WEBROOT},${TESTBUILDURL},") + break + fi + done + + if [ -z "$url" ] ; then + echo "$COMMAND: Cannot locate arch URL for testing" + failure + exit 1 + fi + + testmaster_ssh="root@${TESTMASTER}" + + # test directory name on test box + testdir=${BASE} + # clean it + ssh -n ${testmaster_ssh} rm -rf ${testdir} + # check it out + ssh -n ${testmaster_ssh} svn co ${TESTS_SYSTEM_SVNPATH} ${testdir} +### # check out the entire tests/ module (with system/ duplicated) as a subdir - see fulltree above +### ssh -n ${testmaster_ssh} svn co ${TESTS_SVNPATH} ${testdir}/tests + # invoke test on testbox - pass url and build url - so the tests can use vtest-init-vserver.sh + configs="" + for config in ${TESTCONFIG} ; do + configs="$configs --config $config" + done + test_env="-p $PERSONALITY -d $PLDISTRO -f $FCDISTRO" + + # need to proceed despite of set -e + success=true + ssh 2>&1 -n ${testmaster_ssh} ${testdir}/run_log --build ${build_SVNPATH} --url ${url} $configs $test_env --verbose --all || success= + + # gather logs in the vserver + mkdir -p /vservers/$BASE/build/testlogs + ssh 2>&1 -n ${testmaster_ssh} tar -C ${testdir}/logs -cf - . | tar -C /vservers/$BASE/build/testlogs -xf - || true + # push them to the build web + chmod -R a+r /vservers/$BASE/build/testlogs/ + rsync --archive --delete /vservers/$BASE/build/testlogs/ $WEBPATH/$BASE/testlogs/ + + if [ -z "$success" ] ; then + failure + fi + + echo -n "============================== End $COMMAND:run_log on $(date)" +} + +function in_root_context () { + rpm -q util-vserver > /dev/null +} + +# this part won't work with a remote(rsync) WEBPATH +function sign_node_packages () { + + echo "Signing node packages" + + need_createrepo="" + + repository=$WEBPATH/$BASE/RPMS/ + # the rpms that need signing + new_rpms= + # and the corresponding stamps + new_stamps= + + for package in $(find $repository/ -name '*.rpm') ; do + stamp=$repository/signed-stamps/$(basename $package).signed + # If package is newer than signature stamp + if [ $package -nt $stamp ] ; then + new_rpms="$new_rpms $package" + new_stamps="$new_stamps $stamp" + fi + # Or than createrepo database + [ $package -nt $repository/repodata/repomd.xml ] && need_createrepo=true + done + + if [ -n "$new_rpms" ] ; then + # Create a stamp once the package gets signed + mkdir $repository/signed-stamps 2> /dev/null + + # Sign RPMS. setsid detaches rpm from the terminal, + # allowing the (hopefully blank) GPG password to be + # entered from stdin instead of /dev/tty. + echo | setsid rpm \ + --define "_signature gpg" \ + --define "_gpg_path $GPGPATH" \ + --define "_gpg_name $GPGUID" \ + --resign $new_rpms && touch $new_stamps + fi + + # Update repository index / yum metadata. + if [ -n "$need_createrepo" ] ; then + echo "Indexing node packages after signing" + if [ -f $repository/yumgroups.xml ] ; then + createrepo --quiet -g yumgroups.xml $repository + else + createrepo --quiet $repository + fi + fi +} + +function show_env () { + set +x + echo FCDISTRO=$FCDISTRO + echo PLDISTRO=$PLDISTRO + echo PERSONALITY=$PERSONALITY + echo BASE=$BASE + echo build_SVNPATH=$build_SVNPATH + echo MAKEVARS="${MAKEVARS[@]}" + echo DRY_RUN="$DRY_RUN" + echo PLDISTROTAGS="$PLDISTROTAGS" + # this does not help, it's not yet set when we run show_env + #echo WEBPATH="$WEBPATH" + echo TESTBUILDURL="$TESTBUILDURL" + if in_root_context ; then + echo PLDISTROTAGS="$PLDISTROTAGS" + else + if [ -f /build/$PLDISTROTAGS ] ; then + echo "XXXXXXXXXXXXXXXXXXXX Contents of tags definition file /build/$PLDISTROTAGS" + cat /build/$PLDISTROTAGS + echo "XXXXXXXXXXXXXXXXXXXX end tags definition" + else + echo "XXXXXXXXXXXXXXXXXXXX Cannot find tags definition file /build/$PLDISTROTAGS, assuming remote pldistro" + fi + fi + set -x +} + +function setupssh () { + base=$1; shift + sshkey=$1; shift + + if [ -f ${sshkey} ] ; then + SSHDIR=/vservers/${base}/root/.ssh + mkdir -p ${SSHDIR} + cp $sshkey ${SSHDIR}/thekey + (echo "host *"; \ + echo " IdentityFile ~/.ssh/thekey"; \ + echo " StrictHostKeyChecking no" ) > ${SSHDIR}/config + chmod 700 ${SSHDIR} + chmod 400 ${SSHDIR}/* + else + echo "WARNING : could not find provided ssh key $sshkey - ignored" + fi +} + +function usage () { + echo "Usage: $COMMAND [option] [var=value...] make-targets" + echo "This is $REVISION" + echo "Supported options" + echo " -f fcdistro - defaults to $DEFAULT_FCDISTRO" + echo " -d pldistro - defaults to $DEFAULT_PLDISTRO" + echo " -p personality - defaults to $DEFAULT_PERSONALITY" + echo " -m mailto - no default" + echo " -s svnpath - where to fetch the build module - defaults to $DEFAULT_build_SVNPATH" + echo " -t pldistrotags - defaults to \${PLDISTRO}-tags.mk" + echo " -b base - defaults to $DEFAULT_BASE" + echo " @NAME@ replaced as appropriate" + echo " -o base: (overwrite) do not re-create vserver, re-use base instead" + echo " the -f/-d/-p/-m/-s/-t options are uneffective in this case" + echo " -c testconfig - defaults to $DEFAULT_TESTCONFIG" + echo " -w webpath - defaults to $DEFAULT_WEBPATH" + echo " -W testbuildurl - defaults to $DEFAULT_TESTBUILDURL" + echo " -r webroot - defaults to $DEFAULT_WEBROOT - the fs point where testbuildurl actually sits" + echo " -M testmaster - defaults to $DEFAULT_TESTMASTER" + echo " -y - sign yum repo in webpath" + echo " -g gpg_path - to the gpg secring used to sign rpms. Defaults to $DEFAULT_GPGPATH" + echo " -u gpg_uid - email used in secring. Defaults to $DEFAULT_GPGUID" + echo " -K svnsshkey - specify key to use when svn+ssh:// URLs are used for SVNPATH" + echo " -S - do not publish source rpms" + echo " -B - run build only" + echo " -T - run test only" + echo " -n - dry-run: -n passed to make - vserver gets created though - no mail sent" + echo " -v - be verbose" + echo " -7 - uses weekday-@FCDISTRO@ as base" + echo " -i ifname - defaults to $DEFAULT_IFNAME - used to determine local IP" + exit 1 +} + +function main () { + + set -e + + # parse arguments + MAKEVARS=() + MAKETARGETS=() + DRY_RUN= + DO_BUILD=true + DO_TEST=true + PUBLISH_SRPMS=true + SSH_KEY="" + SIGNYUMREPO="" + while getopts "f:d:p:m:s:t:b:o:c:w:W:r:M:yg:u:K:SBTnv7i:" opt ; do + case $opt in + f) FCDISTRO=$OPTARG ;; + d) PLDISTRO=$OPTARG ;; + p) PERSONALITY=$OPTARG ;; + m) MAILTO=$OPTARG ;; + s) build_SVNPATH=$OPTARG ;; + t) PLDISTROTAGS=$OPTARG ;; + b) BASE=$OPTARG ;; + o) OVERBASE=$OPTARG ;; + c) TESTCONFIG="$TESTCONFIG $OPTARG" ;; + w) WEBPATH=$OPTARG ;; + W) TESTBUILDURL=$OPTARG ;; + r) WEBROOT=$OPTARG ;; + M) TESTMASTER=$OPTARG ;; + y) SIGNYUMREPO=true ;; + g) GPGPATH=$OPTARG ;; + u) GPGUID=$OPTARG ;; + K) SSH_KEY=$OPTARG ;; + S) PUBLISH_SRPMS="" ;; + B) DO_TEST= ;; + T) DO_BUILD= ;; + n) DRY_RUN="-n" ;; + v) set -x ;; + 7) BASE="$(date +%a|tr A-Z a-z)-@FCDISTRO@" ;; + i) IFNAME=$OPTARG ;; + h|*) usage ;; + esac + done + + # preserve options for passing them again later, together with expanded base + declare -a options + toshift=$(($OPTIND - 1)) + arg=1; while [ $arg -le $toshift ] ; do options=(${options[@]} "$1") ; shift; arg=$(($arg+1)) ; done + + # allow var=value stuff; + for target in "$@" ; do + # check if contains '=' + target1=$(echo $target | sed -e s,=,,) + if [ "$target" = "$target1" ] ; then + MAKETARGETS=(${MAKETARGETS[@]} "$target") + else + MAKEVARS=(${MAKEVARS[@]} "$target") + fi + done + + # set defaults + [ -z "$FCDISTRO" ] && FCDISTRO=$DEFAULT_FCDISTRO + [ -z "$PLDISTRO" ] && PLDISTRO=$DEFAULT_PLDISTRO + [ -z "$PERSONALITY" ] && PERSONALITY=$DEFAULT_PERSONALITY + [ -z "$PLDISTROTAGS" ] && PLDISTROTAGS="${PLDISTRO}-tags.mk" + [ -z "$BASE" ] && BASE="$DEFAULT_BASE" + [ -z "$WEBPATH" ] && WEBPATH="$DEFAULT_WEBPATH" + [ -z "$TESTBUILDURL" ] && TESTBUILDURL="$DEFAULT_TESTBUILDURL" + [ -z "$WEBROOT" ] && WEBROOT="$DEFAULT_WEBROOT" + [ -z "$GPGPATH" ] && GPGPATH="$DEFAULT_GPGPATH" + [ -z "$GPGUID" ] && GPGUID="$DEFAULT_GPGUID" + [ -z "$IFNAME" ] && IFNAME="$DEFAULT_IFNAME" + [ -z "$build_SVNPATH" ] && build_SVNPATH="$DEFAULT_build_SVNPATH" + [ -z "$TESTCONFIG" ] && TESTCONFIG="$DEFAULT_TESTCONFIG" + [ -z "$TESTMASTER" ] && TESTMASTER="$DEFAULT_TESTMASTER" + + [ -n "$DRY_RUN" ] && MAILTO="" + + if [ -n "$OVERBASE" ] ; then + sedargs="-e s,@DATE@,${DATE},g" + BASE=$(echo ${OVERBASE} | sed $sedargs) + else + sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" + BASE=$(echo ${BASE} | sed $sedargs) + fi + + ### elaborate mail subject + if [ -n "$DO_BUILD" -a -n "$DO_TEST" ] ; then + MAIL_SUBJECT="complete" + elif [ -n "$DO_BUILD" ] ; then + MAIL_SUBJECT="package-only" + elif [ -n "$DO_TEST" ] ; then + MAIL_SUBJECT="test-only" + fi + if [ -n "$OVERBASE" ] ; then + MAIL_SUBJECT="$MAIL_SUBJECT incremental run on" + else + MAIL_SUBJECT="$MAIL_SUBJECT fresh build" + fi + + if ! in_root_context ; then + # in the vserver + echo "==================== Within vserver BEG $(date)" + build + echo "==================== Within vserver END $(date)" + + else + trap failure ERR INT + # we run in the root context : + # (*) create or check for the vserver to use + # (*) copy this command in the vserver + # (*) invoke it + + if [ -n "$OVERBASE" ] ; then + ### Re-use a vserver (finish an unfinished build..) + if [ ! -d /vservers/${BASE} ] ; then + echo $COMMAND : cannot find vserver $BASE + exit 1 + fi + # manage LOG - beware it might be a symlink so nuke it first + LOG=/vservers/${BASE}.log.txt + rm -f $LOG + exec > $LOG 2>&1 + set -x + echo "XXXXXXXXXX $COMMAND: using existing vserver $BASE" $(date) + # start in case e.g. we just rebooted + vserver ${BASE} start || : + # update build + [ -n "$SSH_KEY" ] && setupssh ${BASE} ${SSH_KEY} + vserver ${BASE} exec svn update /build + # get environment from the first run + FCDISTRO=$(vserver ${BASE} exec /build/getdistroname.sh) + + PLDISTRO=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +PLDISTRO) + PLDISTROTAGS=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +PLDISTROTAGS) + build_SVNPATH=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +build-SVNPATH) + PERSONALITY=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +PERSONALITY) + MAILTO=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +MAILTO) + WEBPATH=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +WEBPATH) + TESTBUILDURL=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +TESTBUILDURL) + WEBROOT=$(vserver ${BASE} exec make --no-print-directory -C /build stage1=skip +WEBROOT) + show_env + else + # create vserver: check it does not exist yet + i= + while [ -d /vservers/${BASE}${i} ] ; do + # we name subsequent builds -n so the logs and builds get sorted properly + [ -z ${i} ] && BASE=${BASE}-n + i=$((${i}+1)) + if [ $i -gt 100 ] ; then + echo "$COMMAND: Failed to create build vserver /vservers/${BASE}${i}" + exit 1 + fi + done + BASE=${BASE}${i} + # need update + # manage LOG - beware it might be a symlink so nuke it first + LOG=/vservers/${BASE}.log.txt + rm -f $LOG + exec > $LOG 2>&1 + set -x + echo "XXXXXXXXXX $COMMAND: creating vserver $BASE" $(date) + show_env + + ### extract the whole build - much simpler + tmpdir=/tmp/$COMMAND-$$ + svn export $build_SVNPATH $tmpdir + # Create vserver + cd $tmpdir + ./vbuild-init-vserver.sh -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} -i ${IFNAME} ${BASE} + # cleanup + cd - + rm -rf $tmpdir + # Extract build again - in the vserver + [ -n "$SSH_KEY" ] && setupssh ${BASE} ${SSH_KEY} + vserver ${BASE} exec svn checkout ${build_SVNPATH} /build + fi + # install ssh key in vserver + echo "XXXXXXXXXX $COMMAND: preparation of vserver $BASE done" $(date) + + # The log inside the vserver contains everything + LOG2=/vservers/${BASE}/log.txt + (echo "==================== BEG VSERVER Transcript of vserver creation" ; \ + cat $LOG ; \ + echo "==================== END VSERVER Transcript of vserver creation" ; \ + echo "xxxxxxxxxx Messing with logs, symlinking $LOG2 to $LOG" ) >> $LOG2 + ### not too nice : nuke the former log, symlink it to the new one + rm $LOG; ln -s $LOG2 $LOG + LOG=$LOG2 + # redirect log again + exec >> $LOG 2>&1 + + sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" + WEBPATH=$(echo ${WEBPATH} | sed $sedargs) + mkdir -p ${WEBPATH} + + # where to store the log for web access + WEBLOG=${WEBPATH}/${BASE}.log.txt + # compute the log URL - inserted in the mail messages for convenience + LOG_URL=$(echo ${WEBLOG} | sed -e "s,//,/,g" -e "s,${WEBROOT},${TESTBUILDURL},") + TESTLOGS_URL=$(echo ${WEBPATH}/${BASE}/testlogs | sed -e "s,//,/,g" -e "s,${WEBROOT},${TESTBUILDURL},") + + if [ -n "$DO_BUILD" ] ; then + + # invoke this command into the build directory of the vserver + cp $COMMANDPATH /vservers/${BASE}/build/ + + # invoke this command in the vserver for building (-T) + vserver ${BASE} exec chmod +x /build/$COMMAND + vserver ${BASE} exec /build/$COMMAND "${options[@]}" -b "${BASE}" "${MAKEVARS[@]}" "${MAKETARGETS[@]}" + fi + + # publish to the web so run_log can find them + rm -rf $WEBPATH/$BASE ; mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS} + rsync --archive --delete --verbose /vservers/$BASE/build/RPMS/ $WEBPATH/$BASE/RPMS/ + [[ -n "$PUBLISH_SRPMS" ]] && rsync --archive --delete --verbose /vservers/$BASE/build/SRPMS/ $WEBPATH/$BASE/SRPMS/ + # publish myplc-release if this exists + release=/vservers/$BASE/build/myplc-release + [ -f $release ] && rsync --verbose $release $WEBPATH/$BASE + + # create yum repo and sign packages. + if [ -n "$SIGNYUMREPO" ] ; then + sign_node_packages + fi + + if [ -n "$DO_TEST" ] ; then + run_log + fi + + success + + fi + +} + +########## +main "$@" diff --git a/trunk/vtest-init-vserver.sh b/trunk/vtest-init-vserver.sh new file mode 120000 index 00000000..1fa8674a --- /dev/null +++ b/trunk/vtest-init-vserver.sh @@ -0,0 +1 @@ +vbuild-init-vserver.sh \ No newline at end of file diff --git a/trunk/vtest-nightly.sh b/trunk/vtest-nightly.sh new file mode 100644 index 00000000..b50e621c --- /dev/null +++ b/trunk/vtest-nightly.sh @@ -0,0 +1,226 @@ +#!/bin/bash +REVISION=$(echo '$Revision$' | sed -e 's,\$,,g' -e 's,^\w*:\s,,' ) + +COMMANDPATH=$0 +COMMAND=$(basename $0) + +# default values, tunable with command-line options +DEFAULT_FCDISTRO=f7 +DEFAULT_PLDISTRO=planetlab +DEFAULT_PERSONALITY=linux32 +DEFAULT_BASE="@DATE@--test-@PLDISTRO@-@FCDISTRO@-@PERSONALITY@" +DEFAULT_SVNPATH="http://svn.planet-lab.org/svn/build/trunk" +DEFAULT_WEBPATH="/build/@PLDISTRO@/" + +DEFAULT_BUILDREPO="http://build.planet-lab.org/install-rpms/archive/" +DEFAULT_REPOURL="@BUILDREPO@/@PLDISTRO@/@FCDISTRO@/@DATE@--@PLDISTRO@-@FCDISTRO@-@PERSONALITY@/RPMS" + +# 10.1 subnet allocated for testing at Princeton +DEFAULT_IPPREFIX16="10.1" + +# default to eth0 +DEFAULT_TESTVSERVER_DEV=0 + +# for the test part +TESTSVNPATH="http://svn.planet-lab.org/svn/tests/trunk/system/" + +#################### +# assuming vserver runs in UTC +DATE=$(date +'%Y.%m.%d') + +# Notify recipient of failure or success, manage various stamps +function failure() { + set -x + WEBLOG=${WEBPATH}/${BASE}.test-log.txt + cp $LOG ${WEBLOG} + (echo -n "============================== $COMMAND: failure at " ; date ; tail -c 20k $WEBLOG) > ${WEBLOG}.ko + if [ -n "$MAILTO" ] ; then + tail -c 20k ${WEBLOG} | mail -s "Failures for test ${BASE}" $MAILTO + fi + exit 1 +} + +function success () { + set -x + WEBLOG=${WEBPATH}/${BASE}.test-log.txt + cp $LOG ${WEBLOG} + touch ${WEBLOG}.ok + if [ -n "$MAILTO" ] ; then + (echo "$PLDISTRO ($BASE) tests for $FCDISTRO completed on $(date)" ) | mail -s "Successful test for ${BASE}" $MAILTO + fi + exit 0 +} + +function show_env () { + set +x + echo FCDISTRO=$FCDISTRO + echo PLDISTRO=$PLDISTRO + echo BASE=$BASE + echo SVNPATH=$SVNPATH + echo MAKEVARS="${MAKEVARS[@]}" + echo MAKEOPTS="${MAKEOPTS[@]}" + echo PLDISTROTAGS="$PLDISTROTAGS" + echo TAGSRELEASE="$TAGSRELEASE" + echo -n "(might be unexpanded)" + echo WEBPATH="$WEBPATH" + if [ -d /vservers ] ; then + echo PLDISTROTAGS="$PLDISTROTAGS" + else + echo "XXXXXXXXXXXXXXXXXXXX Contents of tags definition file /build/$PLDISTROTAGS" + cat /build/$PLDISTROTAGS + echo "XXXXXXXXXXXXXXXXXXXX end tags definition" + fi + set -x +} + +function usage () { + ### set BASE from DISTRO, if unspecified + echo "Usage: $COMMAND [option] make-targets" + echo "This is $REVISION" + echo "Supported options" + echo " -n dry-run : -n passed to make - vserver gets created though - no mail sent" + echo " -f fcdistro - defaults to $DEFAULT_FCDISTRO" + echo " -d pldistro - defaults to $DEFAULT_PLDISTRO" + echo " -b base - defaults to $DEFAULT_BASE" + echo " @NAME@ replaced as appropriate, which currently defaults to $BASE" + echo " -p personality - defaults to $DEFAULT_PERSONALITY" + echo " -t pldistrotags - defaults to \${PLDISTRO}-tags.mk" + echo " -r tagsrelease - a release number that refers to PLDISTROTAGS - defaults to HEAD" + echo " -s svnpath - where to fetch the build module" + echo " -o : overwrite - re-run in base directory, do not create vserver" + echo " -m mailto" + echo " -a makevar=value - space in values are not supported" + echo " -u repourl -- defaults to $DEFAULT_REPOURL" + echo " @NAME@ replaced as appropriate, which currently defaults to $REPOURL" + echo " -w webpath - defaults to $DEFAULT_WEBPATH" + echo " @NAME@ replaced as appropriate, which currently defaults to $WEBPATH" + echo " -j ip address ; use this to give the vserver an explicit IP address; also see -i." + echo " -i /16 ip subnet for auto IP address selection within the subnet -- defaults to $DEFAULT_IPPREFIX16" + echo " This is only used when no explicit IP address is specified." + echo " -v : be verbose" + echo " -7 : uses weekday-@FCDISTRO@ as base -- defaults to $(date +%a|tr A-Z a-z)-$FCDISTRO" + exit 1 +} + +function main () { + + set -e + + # preserve arguments for passing them again later + declare -a argv + for arg in "$@"; do argv=(${argv[@]} "$arg") ; done + + # set defaults + [ -z "$FCDISTRO" ] && FCDISTRO=$DEFAULT_FCDISTRO + [ -z "$PLDISTRO" ] && PLDISTRO=$DEFAULT_PLDISTRO + [ -z "$PERSONALITY" ] && PERSONALITY=$DEFAULT_PERSONALITY + [ -z "$PLDISTROTAGS" ] && PLDISTROTAGS="${PLDISTRO}-tags.mk" + [ -z "$BASE" ] && BASE="$DEFAULT_BASE" + [ -z "$WEBPATH" ] && WEBPATH="$DEFAULT_WEBPATH" + [ -z "$SVNPATH" ] && SVNPATH="$DEFAULT_SVNPATH" + [ -z "$BUILDREPO" ] && BUILDREPO="$DEFAULT_BUILDREPO" + [ -z "$REPOURL" ] && REPOURL="$DEFAULT_REPOURL" + [ -z "$IPPREFIX16" ] && IPPREFIX16="$DEFAULT_IPPREFIX16" + [ -z "$TESTVSERVER_DEV" ] && TESTVSERVER_DEV="$DEFAULT_TESTVSERVER_DEV" + + # parse arguments + MAKEVARS=() + MAKEOPTS=() + while getopts "nf:d:b:p:t:r:s:om:a:u:w:i:j:vh7" opt ; do + case $opt in + n) DRY_RUN="true" ; MAKEOPTS=(${MAKEOPTS[@]} -n) ; MAILTO="";; + f) FCDISTRO=$OPTARG ;; + d) PLDISTRO=$OPTARG ;; + b) BASE=$OPTARG ;; + p) PERSONALITY=$OPTARG ;; + t) PLDISTROTAGS=$OPTARG ;; + r) TAGSRELEASE=$OPTARG ;; + s) SVNPATH=$OPTARG ;; + o) OVERWRITEMODE=true ;; + m) MAILTO=$OPTARG ;; + a) MAKEVARS=(${MAKEVARS[@]} "$OPTARG") ;; + u) REPOURL=$OPTARG ;; + w) WEBPATH=$OPTARG ;; + i) IPPREFIX16=$OPTARG ;; + j) TESTVSERVER_IP=$OPTARG ;; + v) set -x ;; + 7) BASE="$(date +%a|tr A-Z a-z)-@FCDISTRO@" ;; + h|*) usage ;; + esac + done + + shift $(($OPTIND - 1)) + MAKETARGETS="$@" + + ### set BASE from DISTRO, if unspecified + sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" + BASE=$(echo ${BASE} | sed $sedargs) + WEBPATH=$(echo ${WEBPATH} | sed $sedargs) + REPOURL=$(echo ${REPOURL} | sed $sedargs) + sedargs="-e s,@BUILDREPO@,${BUILDREPO},g" + REPOURL=$(echo ${REPOURL} | sed $sedargs) + + trap failure ERR INT + # create vserver: check it does not exist yet + i= + while [ -d /vservers/${BASE}${i} ] ; do + # we name subsequent builds -n so the logs and builds get sorted properly + [ -z ${i} ] && BASE=${BASE}-n + i=$((${i}+1)) + if [ $i -gt 100 ] ; then + echo "$COMMAND: Failed to create build vserver /vservers/${BASE}${i}" + exit 1 + fi + done + BASE=${BASE}${i} + # need update + # manage LOG - beware it might be a symlink so nuke it first + LOG=/vservers/${BASE}.log.txt + rm -f $LOG + exec > $LOG 2>&1 + set -x + echo "XXXXXXXXXX $COMMAND: creating vserver $BASE" $(date) + show_env + + ### extract the whole build - much simpler + tmpdir=/tmp/$COMMAND-$$ + svn export $SVNPATH $tmpdir + # Create vserver + cd $tmpdir + ./vtest-init-vserver.sh -i eth${TESTVSERVER_DEV} -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${BASE} ${REPOURL} + # cleanup + cd - + rm -rf $tmpdir + + vserver ${BASE} stop + rm -f /etc/vservers/${BASE}/interfaces/0/* + if [ -z "$TESTVSERVER_IP" ] ; then + xid=$(cat /etc/vservers/${BASE}/context) + TESTVSERVER_IP=$(python -c "context=int($xid); print '%s.%d.%d' % ($IPPREFIX16,(context&0xff00)>>8,context&0xff)") + fi + mkdir -p /etc/vservers/${BASE}/interfaces/0 + echo "${TESTVSERVER_IP}" > /etc/vservers/${BASE}/interfaces/0/ip + echo "eth${TESTVSERVER_DEV}" > /etc/vservers/${BASE}/interfaces/0/dev + vserver ${BASE} start + echo "XXXXXXXXXX $COMMAND: preparation of vserver $BASE done" $(date) + + # The log inside the vserver contains everything + LOG2=/vservers/${BASE}/log.txt + (echo "==================== BEG VSERVER Transcript of vserver creation" ; \ + cat $LOG ; \ + echo "==================== END VSERVER Transcript of vserver creation" ; \ + echo "xxxxxxxxxx Messing with logs, symlinking $LOG2 to $LOG" ) >> $LOG2 + ### not too nice : nuke the former log, symlink it to the new one + rm $LOG; ln -s $LOG2 $LOG + LOG=$LOG2 + # redirect log again + exec >> $LOG 2>&1 + + svn cat ${TESTSVNPATH}/selftest > /vservers/${BASE}/selftest + vserver ${BASE} exec chmod +x /selftest + vserver ${BASE} exec /selftest ${BASE}.$(hostname) ${TESTVSERVER_IP} + success +} + +########## +main "$@" diff --git a/trunk/yumgroups.sh b/trunk/yumgroups.sh new file mode 100755 index 00000000..1e8f506e --- /dev/null +++ b/trunk/yumgroups.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +. build.common + +function usage () { + echo "Usage: $0 pldistro" + exit 1 +} + +[ "$#" = 1 ] || usage +pldistro=$1; shift + +# gather all known pkgs files +here=$(pwd) +all_pkgs=$( (cd $here/config.planetlab ; ls -1 *.pkgs ; cd $here/config.$pldistro; ls *.pkgs) | sort -u) + +yumgroups_from_pkgs $(dirname $0) $pldistro $pl_DISTRO_NAME $all_pkgs -- 2.47.0