From 48f43cea02600d0f9015b192a61d16f08cf4dc66 Mon Sep 17 00:00:00 2001 From: parmentelat Date: Wed, 28 Nov 2018 14:01:15 +0100 Subject: [PATCH] f29 --- Makefile | 116 +++++++++++++++++++++++----------------------- lbuild-initvm.sh | 112 ++++++++++++++++++++++---------------------- lbuild-nightly.sh | 78 +++++++++++++++---------------- lxc.mk | 8 ++-- pkgs.py | 24 +++++----- 5 files changed, 169 insertions(+), 169 deletions(-) diff --git a/Makefile b/Makefile index 1c601a94..a214aa07 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # -# Thierry Parmentelat - INRIA Sophia Antipolis +# Thierry Parmentelat - INRIA Sophia Antipolis # #################### # invocation: @@ -15,33 +15,33 @@ # (*) as of nov. 2007, myplc-devel is deprecated # (*) instead, we create a fresh vserver that holds required tools (see e.g. planetlab-devel.lst) # (*) the build uses the current fedora version as a target for the produced images -# (*) so you simply need to create a fedora 8 build image for building fedora-8 images -# +# (*) so you simply need to create a fedora 8 build image for building fedora-8 images +# #################### (planetlab) distributions # # (*) see README-pldistros.txt -# (*) then you need to run +# (*) then you need to run # make stage1=true PLDISTRO=onelab # -#################### +#################### # This build deals with 3 kinds of objects -# +# # (*) packages are named upon the RPM name; they are mostly lowercase # Add a package to ALL if you want it built as part of the default set. # (*) modules are named after the subversion or git tree; we would wish to keep # these names lowercase as far as possible # (*) rpms are named in the spec files. A package typically defines several rpms; # rpms are used for defining LOCAL-DEVEL-RPMS. See also package.rpmnames -# +# # in simple cases, one package uses one module (in which case using the same name sounds right) -# but others might need several modules (e.g. image-creation packages like bootstrapfs need -# bootstrapfs and build); in this case the FIRST ONE is used for locating the specfile +# but others might need several modules (e.g. image-creation packages like bootstrapfs need +# bootstrapfs and build); in this case the FIRST ONE is used for locating the specfile # #################### packages # basics: how to build a package - you need to define the following variables -# +# # (*) package-MODULES -# a package needs one or several modules to build; the first one is used for +# a package needs one or several modules to build; the first one is used for # some special purposes, like locating the specfile # (*) package-SPEC # the package's specfile; this is relative to the FIRST module in package-MODULES @@ -65,23 +65,23 @@ # rpms in this area will *not* be uninstalled after the target is made, because that would ruin the build vm for good # (*) package-DEPEND-FILES # a set of files that the package depends on - and that make needs to know about -# if this contains RPMS/yumgroups.xml, then the toplevel RPMS's index +# if this contains RPMS/yumgroups.xml, then the toplevel RPMS's index # is refreshed with createrepo prior to running rpmbuild # (*) package-SPECVARS -# space-separated list of spec variable definitions, where you can reference make variables that -# belong to packages defined BEFORE the current one +# space-separated list of spec variable definitions, where you can reference make variables that +# belong to packages defined BEFORE the current one # note: you should use = to define these (as opposed to :=) -# e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release) -# would let you use the %release from the kernel's package when rpmbuild'ing mydriver +# e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release) +# would let you use the %release from the kernel's package when rpmbuild'ing mydriver # see automatic below # (*) package-RPMFLAGS: Miscellaneous RPM flags # this is passed to rpmbuild, as well as to spec2make for "exporting" various rpm variable # to make; beware that some features, like --with=debug or --define 'foo bar' are not # well handled by spec2make as of this writing, which can cause issues. # hopefully this will be fixed... -# (*) package-BUILD-FROM-SRPM: set this to any non-empty value, -# if your package is able to produce a source rpm. -# In this case the build will first invoke 'make srpm', and then rebuild binaries from that +# (*) package-BUILD-FROM-SRPM: set this to any non-empty value, +# if your package is able to produce a source rpm. +# In this case the build will first invoke 'make srpm', and then rebuild binaries from that # (*) package-RPMDATE: set this to any non-empty value to get the rpm package's release field hold the current date # this is useful for container packages, like e.g. bootstrapfs or vserver, that contains much more than the # correspondng module @@ -91,9 +91,9 @@ # # (*) module-SVNPATH # for svn modules -# the complete path where this module lies; +# the complete path where this module lies; # you can specify the trunk or a given tag with this variable -# +# # (*) module-GITPATH # for git modules # which by analogy with svn revision scheme, is expected to be @ @@ -115,7 +115,7 @@ PLANETLAB_RELEASE = 5.2 # # Default values # -# minimal compat with macos, just so this does not complain +# minimal compat with macos, just so this does not complain HOSTARCH := $(shell uname -i 2> /dev/null || uname -m 2> /dev/null) DISTRO := $(shell ./getdistro.sh) RELEASE := $(shell ./getrelease.sh) @@ -154,7 +154,7 @@ endif include $(PLDISTROTAGS) # this used to be set in the -tags.mk files, but that turned out to require -# error-prone duplicate changes +# error-prone duplicate changes # so now the nightly build script sets this to what it is currently using # in case we run this manually, i.e. if build-GITPATH is not set ifeq "$(build-GITPATH)" "" @@ -205,10 +205,10 @@ $(eval $(call remote_pldistro,wextoolbox,wextoolbox-tags,git)) $(eval $(call remote_pldistro,wexlxc,wexlxc-tags,git)) ########## stage1 and stage1iter -# extract specs and compute .mk files by running +# extract specs and compute .mk files by running # make stage1=true # entering stage1, we compute all the spec files -# then we use stage1iter to compute the .mk iteratively, +# then we use stage1iter to compute the .mk iteratively, # ensuring that the n-1 first makefiles are loaded when creating the n-th one # when stage1iter is set, it is supposed to be an index (starting at 1) in $(ALL) @@ -253,7 +253,7 @@ include $(ALLMKS) #all : modules #all : rpms #all : srpms -# mention $(ALL) here rather than rpms +# mention $(ALL) here rather than rpms # this is because the inter-package dependencies are expressed like # util-vserver: util-python all: rpms @@ -262,11 +262,11 @@ endif endif ### yumgroups.xml : compute from all known .pkgs files -RPMS/yumgroups.xml: +RPMS/yumgroups.xml: mkdir -p RPMS ./yumgroups.sh $(PLDISTRO) > $@ -createrepo = createrepo --quiet -g yumgroups.xml RPMS/ +createrepo = createrepo --quiet -g yumgroups.xml RPMS/ repo: RPMS/yumgroups.xml $(createrepo) @@ -274,7 +274,7 @@ repo: RPMS/yumgroups.xml .PHONY: repo #################### -# notes: +# notes: # * we always use the first module's location (SVNPATH/GITPATH) to extract the spec file # * no matter what SCM is used, SPEC should hold a relative path from the module's root # @@ -310,13 +310,13 @@ endef $(foreach module,$(ALL.modules), $(eval $(call stage1_module_vars,$(module)))) # -# for each package, compute whether we need to set date +# for each package, compute whether we need to set date # the heuristic is that we mention the date as part of the rpm release flag if # (*) the package has requested it by setting package-RPMDATE (container packages should do that) -# (*) or SVNPATH contains 'trunk' or 'branches' -# (*) or GITPATH has no '@' (trunk) +# (*) or SVNPATH contains 'trunk' or 'branches' +# (*) or GITPATH has no '@' (trunk) # (*) or GITPATH contains a '@', and the gittag part has no '-' (this is to tell a tag from a branch..) -# +# define package_hasdate $(1).has-date = $(if $($(1)-RPMDATE),yes, \ $(if $($($(1).module)-SVNPATH), \ @@ -351,7 +351,7 @@ header.spec: echo "%global _source_filedigest_algorithm 1" >> $@ echo "%global _source_payload w9.gzdio" >> $@ echo "%global _binary_payload w9.gzdio" >> $@ -ifeq "$(DISTRONAME)""$(filter $(DISTRONAME),f25 f27)" +ifeq "$(DISTRONAME)""$(filter $(DISTRONAME),f25 f27 f29)" echo "%define debug_package %{nil}" >> $@ endif @@ -446,7 +446,7 @@ endif ifeq "$(shell pwd)" "/build" rm -f $(FAKEROOT) ; ln -s $(REALROOT) $(FAKEROOT) endif - rm -f $@ + rm -f $@ echo "%_topdir $(HOME)" >> $@ echo "%_tmppath $(HOME)/tmp" >> $@ echo "%__spec_install_pre %{___build_pre}" >> $@ @@ -526,13 +526,13 @@ SOURCES/%.tgz: SOURCES/% ## URLS/%: url=$(subst @colon@,:,$(subst @slash@,/,$(notdir $@))) URLS/%: basename=$(notdir $(url)) -URLS/%: +URLS/%: echo curl $(url) -o SOURCES/$(basename) touch $@ -### the directory SOURCES/- is made +### the directory SOURCES/- is made # with a (set of) copy -rl from MODULES/ -# the former is $(package.source) +# the former is $(package.source) ALLSOURCES:=$(foreach package, $(ALL), $($(package).source)) # so that make does not use the rule below directly for creating the tarball files .SECONDARY: $(ALLSOURCES) @@ -574,13 +574,13 @@ RPMYUM-INSTALL-STOCK := yum -y install RPMYUM-UNINSTALL-STOCK := rpm -e ### these macro handles the LOCAL-DEVEL-RPMS and LOCAL-DEVEL-RPMS-CRUCIAL tags for a given package -# before building : rpm-install LOCAL-DEVEL-RPMS -define rpmyum_install_local_rpms - $(if $($(1).all-local-devel-rpm-paths), echo "Installing for $(1)-LOCAL-DEVEL-RPMS" ; $(RPMYUM-INSTALL-LOCAL) $($(1).all-local-devel-rpm-paths)) +# before building : rpm-install LOCAL-DEVEL-RPMS +define rpmyum_install_local_rpms + $(if $($(1).all-local-devel-rpm-paths), echo "Installing for $(1)-LOCAL-DEVEL-RPMS" ; $(RPMYUM-INSTALL-LOCAL) $($(1).all-local-devel-rpm-paths)) endef # install stock rpms if defined -define rpmyum_install_stock_rpms +define rpmyum_install_stock_rpms $(if $($(1)-STOCK-DEVEL-RPMS), echo "Installing for $(1)-STOCK-DEVEL-RPMS" ; $(RPMYUM-INSTALL-STOCK) $($(1)-STOCK-DEVEL-RPMS) || true) endef @@ -594,11 +594,11 @@ DPKGAPT-INSTALL-LOCAL := gdebi DPKGAPT-INSTALL-STOCK := apt-get -y install DPKGAPT-UNINSTALL-STOCK := echo WARNING uninstalling stock debs not implemented -define dpkgapt_install_local_debs +define dpkgapt_install_local_debs $(if $($(1)-LOCAL-DEVEL-DEBS), echo "Installing for $(1)-LOCAL-DEVEL-DEBS" ; $(foreach debname,$($(1)-LOCAL-DEVEL-DEBS),$(DPKGAPT-INSTALL-LOCAL) $(wildcard DEBIAN/$(debname)_*.deb);)) endef -define dpkgapt_install_stock_debs +define dpkgapt_install_stock_debs $(if $($(1)-STOCK-DEVEL-DEBS), echo "Installing for $(1)-STOCK-DEVEL-DEBS" ; $(DPKGAPT-INSTALL-STOCK) $($(1)-STOCK-DEVEL-DEBS) || true) endef @@ -609,9 +609,9 @@ endef #################### # usage: target_source_rpm package -define target_source_rpm +define target_source_rpm ifeq "$($(1)-BUILD-FROM-SRPM)" "" -$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).tarballs) +$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).tarballs) mkdir -p BUILD SRPMS tmp @(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using SOURCES) " ; date) $(call rpmyum_install_stock_rpms,$(1)) @@ -627,7 +627,7 @@ $($(1).srpm): $($(1).specpath) .rpmmacros $($(1).source) $(call rpmyum_install_local_rpms,$(1)) make -C $($(1).source) srpm SPECFILE=$(HOME)/$($(1).specpath) EXPECTED_SRPM=$(notdir $($(1).srpm)) && \ rm -f SRPMS/$(notdir $($(1).srpm)) && \ - ln $($(1).source)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm)) + ln $($(1).source)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm)) $(call rpmyum_uninstall_stock_rpms,$(1)) @(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date) endif @@ -642,14 +642,14 @@ rpms: $(ALLRPMS) @echo $(words $(ALLRPMS)) binary rpms OK .PHONY: rpms -# use tmp dirs when building binary rpm so make remains idempotent +# use tmp dirs when building binary rpm so make remains idempotent # otherwise SOURCES/ or SPEC gets touched again - which leads to rebuilding RPM-USE-TMP-DIRS = --define "_sourcedir $(HOME)/tmp" --define "_specdir $(HOME)/tmp" RPM-USE-COMPILE-DIRS = --define "_sourcedir $(HOME)/COMPILE" --define "_specdir $(HOME)/COMPILE" # usage: build_binary_rpm package # xxx hacky - invoke createrepo if DEPEND-FILES mentions RPMS/yumgroups.xml -define target_binary_rpm +define target_binary_rpm $($(1).rpms): $($(1).srpm) mkdir -p RPMS tmp @(echo -n "XXXXXXXXXXXXXXX -- BEG RPM $(1) " ; date) @@ -676,7 +676,7 @@ $(foreach package,$(ALL),$(eval $(call target_binary_rpm,$(package)))) ### shorthand target # e.g. make proper -> does propers rpms # usage shorthand_target package -define target_shorthand +define target_shorthand $(1): $($(1).rpms) .PHONY: $(1) $(1)-spec: $($(1)-SPEC) @@ -729,7 +729,7 @@ debian: $(ALL-DEBIAN) # very rough for now (one module per package), targets only sfa for now # the general idea here is, changing the specfile (for version number and all) is enough, and this # gets passed to "make debian" in the module -# PREFIX: at one point we had passed PREFIX=/usr to this 'make debian' +# PREFIX: at one point we had passed PREFIX=/usr to this 'make debian' # however it turned out we could manage this some other way (see manifold) # so I'm reverting to simplicity define target_debian @@ -776,7 +776,7 @@ $(1)-clean-debian: CLEANS += $(1)-clean-srpm $(1)-codeclean: $(1)-clean-source $(1)-clean-tarball $(1)-clean-build $(1)-clean-rpms $(1)-clean-srpm $(1)-clean-debian $(1)-clean: $(1)-clean-modules $(1)-codeclean -.PHONY: $(1)-codeclean $(1)-clean +.PHONY: $(1)-codeclean $(1)-clean $(1)-clean-spec: rm -rf $($(1).specpath) .PHONY: $(1)-clean-spec @@ -837,10 +837,10 @@ version-build: @echo "Build target-arch: $(HOSTARCH)" @echo "Build target-distro: $(DISTRO)" @echo "Build target-distroname: $(DISTRONAME)" - @echo "Build target-release: $(RELEASE)" - @echo "Build target-personality: $(PERSONALITY)" + @echo "Build target-release: $(RELEASE)" + @echo "Build target-personality: $(PERSONALITY)" -#################### +#################### # for a given module VFORMAT="%30s := %s\n" define print_tag_target @@ -869,12 +869,12 @@ versions: myplc-release version-build version-tags version-rpms #################### package info PKGKEYS := tarballs source srpm rpms rpmnames rpm-release rpm-name rpm-version rpm-subversion %-pkginfo: package=$(subst -pkginfo,,$@) -%-pkginfo: +%-pkginfo: @$(foreach key,$(PKGKEYS),echo "$(package).$(key)=$($(package).$(key))";) ## rpm info RPMKEYS := rpm-path package %-rpminfo: rpm=$(subst -rpminfo,,$@) -%-rpminfo: +%-rpminfo: @$(foreach key,$(RPMKEYS),echo "$(rpm).$(key)=$($(rpm).$(key))";) #################### various lists - designed to run with stage1=true @@ -899,7 +899,7 @@ module-tools: echo $(module):$($(module)-BRANCH); , \ echo $(module); ))) -info: info-packages info-modules info-branches +info: info-packages info-modules info-branches .PHONY: info info-packages info-modules info-branches module-tools diff --git a/lbuild-initvm.sh b/lbuild-initvm.sh index aa714608..ccbb7b16 100755 --- a/lbuild-initvm.sh +++ b/lbuild-initvm.sh @@ -13,7 +13,7 @@ BUILD_DIR=$(pwd) # pkgs parsing utilities + lbuild-bridge.sh export PATH=$(dirname $0):$PATH -# old guests have e.g. mount in /bin but this is no longer part of +# old guests have e.g. mount in /bin but this is no longer part of # the standard PATH in recent hosts after usrmove, so let's keep it simple export PATH=$PATH:/bin:/sbin @@ -30,7 +30,7 @@ function lxcroot () { # XXX fixme : when creating a 32bits VM we need to call linux32 as appropriate...s -DEFAULT_FCDISTRO=f27 +DEFAULT_FCDISTRO=f29 DEFAULT_PLDISTRO=lxc DEFAULT_PERSONALITY=linux64 DEFAULT_MEMORY=3072 @@ -60,7 +60,7 @@ function masklen_to_netmask () { python <=1 and masklen<=32): +if not (masklen>=1 and masklen<=32): print "Wrong masklen",masklen exit(1) result=[] @@ -72,7 +72,7 @@ for i in range(4): result.append(masklen) masklen=0 print ".".join([ str(256-2**(8-i)) for i in result ]) - + EOF } @@ -87,7 +87,7 @@ function package_method () { echo debootstrap ;; *) echo Unknown distro $fcdistro ;; - esac + esac } # return arch from debian distro and personality @@ -105,12 +105,12 @@ function canonical_arch () { } # the new test framework creates /timestamp in /vservers/ *before* populating it -function almost_empty () { - dir="$1"; shift ; +function almost_empty () { + dir="$1"; shift ; # non existing is fine - [ ! -d $dir ] && return 0; + [ ! -d $dir ] && return 0; # need to have at most one file - count=$(cd $dir; ls | wc -l); [ $count -le 1 ]; + count=$(cd $dir; ls | wc -l); [ $count -le 1 ]; } ############################## @@ -123,7 +123,7 @@ function fedora_install() { cache=/var/cache/lxc/fedora/$arch/${fedora_release} mkdir -p $cache - + ( flock --exclusive --timeout 60 200 || { echo "Cache repository is busy." ; return 1 ; } @@ -141,7 +141,7 @@ function fedora_install() { echo "Filling $lxc_root from $cache/rootfs ... " rsync -a $cache/rootfs/ $lxc_root/ - + return 0 ) 200> $cache/lock @@ -163,7 +163,7 @@ function fedora_download() { mkdir -p $INSTALL_ROOT || { echo "Failed to create '$INSTALL_ROOT' directory" ; return 1; } - mkdir -p $INSTALL_ROOT/etc/yum.repos.d + mkdir -p $INSTALL_ROOT/etc/yum.repos.d mkdir -p $INSTALL_ROOT/dev mknod -m 0444 $INSTALL_ROOT/dev/random c 1 8 mknod -m 0444 $INSTALL_ROOT/dev/urandom c 1 9 @@ -175,7 +175,7 @@ function fedora_download() { # append fedora repo files with desired ${fedora_release} and $basearch for f in $INSTALL_ROOT/etc/yum.repos.d/* ; do sed -i "s/\$basearch/$arch/g; s/\$releasever/${fedora_release}/g;" $f - done + done MIRROR_URL=$FEDORA_MIRROR_BASE/releases/${fedora_release}/Everything/$arch/os RELEASE_URL1="$MIRROR_URL/Packages/fedora-release-${fedora_release}-1.noarch.rpm" @@ -183,7 +183,7 @@ function fedora_download() { # first try the second version of fedora-release first RELEASE_URL2="$MIRROR_URL/Packages/f/fedora-release-${fedora_release}-2.noarch.rpm" RELEASE_URL3="$MIRROR_URL/Packages/f/fedora-release-${fedora_release}-1.noarch.rpm" - + RELEASE_TARGET=$INSTALL_ROOT/fedora-release-${fedora_release}.noarch.rpm found="" for attempt in $RELEASE_URL1 $RELEASE_URL2 $RELEASE_URL3; do @@ -196,7 +196,7 @@ function fedora_download() { fi done [ -n "$found" ] || { echo "Could not retrieve fedora-release rpm - exiting" ; exit 1; } - + mkdir -p $INSTALL_ROOT/var/lib/rpm rpm --root $INSTALL_ROOT --initdb # when installing f12 this apparently is already present, so ignore result @@ -204,9 +204,9 @@ function fedora_download() { # however f12 root images won't get created on a f18 host # (the issue here is the same as the one we ran into when dealing with a vs-box) # in a nutshell, in f12 the glibc-common and filesystem rpms have an apparent conflict - # >>> file /usr/lib/locale from install of glibc-common-2.11.2-3.x86_64 conflicts + # >>> file /usr/lib/locale from install of glibc-common-2.11.2-3.x86_64 conflicts # with file from package filesystem-2.4.30-2.fc12.x86_64 - # in fact this was - of course - allowed by f12's rpm but later on a fix was made + # in fact this was - of course - allowed by f12's rpm but later on a fix was made # http://rpm.org/gitweb?p=rpm.git;a=commitdiff;h=cf1095648194104a81a58abead05974a5bfa3b9a # So ideally if we want to be able to build f12 images from f18 we need an rpm that has # this patch undone, like we have in place on our f14 boxes (our f14 boxes need a f18-like rpm) @@ -235,7 +235,7 @@ function fedora_configure() { echo 0 > $lxc_root/selinux/enforce # set the hostname - case "$fcdistro" in + case "$fcdistro" in f18|f2?) cat < ${lxc_root}/etc/sysconfig/network NETWORKING=yes @@ -312,7 +312,7 @@ function fedora_configure_systemd() { lxc=$1; shift lxc_root=$(lxcroot $lxc) - # so ignore if we can't find /etc/systemd at all + # so ignore if we can't find /etc/systemd at all [ -d ${lxc_root}/etc/systemd ] || return 0 # otherwise let's proceed ln -sf /lib/systemd/system/multi-user.target ${lxc_root}/etc/systemd/system/default.target @@ -329,8 +329,8 @@ function fedora_configure_systemd() { # overwrite container yum config function fedora_configure_yum () { - set -x - set -e + set -x + set -e trap failure ERR INT lxc=$1; shift @@ -372,10 +372,10 @@ EOF if [ ! -d $lxc_root/etc/yum.repos.d ] ; then echo "WARNING : cannot create myplc repo" else - # exclude kernel from fedora repos + # exclude kernel from fedora repos yumexclude=$(pl_plcyumexclude $fcdistro $pldistro $DIRNAME) for repo in $lxc_root/etc/yum.repos.d/* ; do - [ -f $repo ] && yumconf_exclude $repo "exclude=$yumexclude" + [ -f $repo ] && yumconf_exclude $repo "exclude=$yumexclude" done # the build repo is not signed at this stage cat > $lxc_root/etc/yum.repos.d/myplc.repo < ${lxc_root}/etc/hostname $GUEST_HOSTNAME EOF - + } function debian_configure () { @@ -462,8 +462,8 @@ function setup_lxc() { lxc_root=$(lxcroot $lxc) - # create lxc container - + # create lxc container + pkg_method=$(package_method $fcdistro) case $pkg_method in yum) @@ -491,14 +491,14 @@ function setup_lxc() { # Enable cgroup -- xxx -- is this really useful ? [ -d $lxc_root/cgroup ] || mkdir $lxc_root/cgroup - + ### set up resolv.conf from host # ubuntu precise and on, /etc/resolv.conf is a symlink to ../run/resolvconf/resolv.conf [ -h $lxc_root/etc/resolv.conf ] && rm -f $lxc_root/etc/resolv.conf cp /etc/resolv.conf $lxc_root/etc/resolv.conf ### and /etc/hosts for at least localhost [ -f $lxc_root/etc/hosts ] || echo "127.0.0.1 localhost localhost.localdomain" > $lxc_root/etc/hosts - + # grant ssh access from host to guest mkdir -p $lxc_root/root/.ssh cat /root/.ssh/id_rsa.pub >> $lxc_root/root/.ssh/authorized_keys @@ -508,7 +508,7 @@ function setup_lxc() { # don't keep the input xml, this can be retrieved at all times with virsh dumpxml config_xml=/tmp/$lxc.xml ( [ -n "$NAT_MODE" ] && write_lxc_xml_natip $lxc || write_lxc_xml_publicip $lxc ) > $config_xml - + # define lxc container for libvirt virsh -c lxc:/// define $config_xml @@ -560,7 +560,7 @@ EOF } # grant build guests the ability to do mknods -function write_lxc_xml_natip () { +function write_lxc_xml_natip () { lxc=$1; shift lxc_root=$(lxcroot $lxc) cat < /root/.profile export PS1="[$lxc] \\w # " PROFILE EOF - + } function post_install_myplc () { - set -x - set -e + set -x + set -e trap failure ERR INT lxc=$1; shift @@ -802,16 +802,16 @@ function wait_for_ssh () { # if run in public_ip mode, we know the IP of the guest and it is specified here [ -n "$1" ] && { guest_ip=$1; shift; } - #wait max 2 min for sshd to start + #wait max 2 min for sshd to start success="" current_time=$(date +%s) stop_time=$(($current_time + 120)) - + counter=1 while [ "$current_time" -lt "$stop_time" ] ; do echo "$counter-th attempt to reach sshd in container $lxc ..." [ -z "$guest_ip" ] && guest_ip=$(guest_ipv4 $lxc) - [ -n "$guest_ip" ] && ssh -o "StrictHostKeyChecking no" $guest_ip 'uname -i' && { + [ -n "$guest_ip" ] && ssh -o "StrictHostKeyChecking no" $guest_ip 'uname -i' && { success=true; echo "SSHD in container $lxc is UP on IP $guest_ip"; break ; } || : counter=$(($counter+1)) sleep 10 @@ -819,7 +819,7 @@ function wait_for_ssh () { done # Thierry: this is fatal, let's just exit with a failure here - [ -z $success ] && { echo "SSHD in container $lxc could not be reached (guest_ip=$guest_ip)" ; exit 1 ; } + [ -z $success ] && { echo "SSHD in container $lxc could not be reached (guest_ip=$guest_ip)" ; exit 1 ; } return 0 } @@ -830,7 +830,7 @@ function failure () { } function usage () { - set +x + set +x echo "Usage: $COMMAND [options] lxc-name (aka build mode)" echo "Usage: $COMMAND -n hostname [options] lxc-name (aka test mode)" echo "Description:" @@ -853,7 +853,7 @@ function usage () { exit 1 } -### parse args and +### parse args and function main () { #set -e @@ -880,7 +880,7 @@ function main () { *) usage ;; esac done - + shift $(($OPTIND - 1)) # parse fixed arguments @@ -913,10 +913,10 @@ function main () { [ -z "$pldistro" ] && pldistro=$DEFAULT_PLDISTRO [ -z "$personality" ] && personality=$DEFAULT_PERSONALITY [ -z "$MEMORY" ] && MEMORY=$DEFAULT_MEMORY - + # set memory in KB MEMORY=$(($MEMORY * 1024)) - + # the set of preinstalled packages - depends on mode if [ -z "$PREINSTALLED" ] ; then if [ -n "$NAT_MODE" ] ; then @@ -936,7 +936,7 @@ function main () { if [ "$REPO_URL" == "none" ] ; then REPO_URL="" elif [ -z "$REPO_URL" ] ; then - echo "WARNING -- setting up a yum repo is recommended" + echo "WARNING -- setting up a yum repo is recommended" fi fi @@ -968,7 +968,7 @@ function main () { VIF_HOST="vif$(echo $GUEST_HOSTNAME | cut -d. -f1)" fi - setup_lxc $lxc $fcdistro $pldistro $personality + setup_lxc $lxc $fcdistro $pldistro $personality # historically this command is for setting up a build or a test VM # kind of patchy right now though @@ -976,7 +976,7 @@ function main () { # container gets started here post_install $lxc $personality - + echo $COMMAND Done exit 0 diff --git a/lbuild-nightly.sh b/lbuild-nightly.sh index 0b183128..ace4b474 100755 --- a/lbuild-nightly.sh +++ b/lbuild-nightly.sh @@ -6,12 +6,12 @@ COMMAND=$(basename $0) # close stdin, as with ubuntu and debian VMs this script tends to hang and wait for input .. 0<&- -# old guests have e.g. mount in /bin but this is no longer part of +# old guests have e.g. mount in /bin but this is no longer part of # the standard PATH in recent hosts after usrmove, so let's keep it simple export PATH=$PATH:/bin:/sbin # default values, tunable with command-line options -DEFAULT_FCDISTRO=f27 +DEFAULT_FCDISTRO=f29 DEFAULT_PLDISTRO=lxc DEFAULT_PERSONALITY=linux64 DEFAULT_MAILDEST="build at onelab.eu" @@ -71,7 +71,7 @@ function guest_ipv4() { # use with care, a *lot* of other things can go bad as well function summary () { from=$1; shift - echo "******************** BEG SUMMARY" + echo "******************** BEG SUMMARY" python - $from <>>',installing echo=False @@ -129,7 +129,7 @@ def summary (filename): for arg in sys.argv[1:]: summary(arg) EOF - echo "******************** END SUMMARY" + echo "******************** END SUMMARY" } ### we might build on a box other than the actual web server @@ -153,11 +153,11 @@ function pretty_duration () { total_minutes=$(($total_seconds/60)) minutes=$(($total_minutes%60)) hours=$(($total_minutes/60)) - + printf "%02d:%02d:%02d" $hours $minutes $seconds } -# Notify recipient of failure or success, manage various stamps +# Notify recipient of failure or success, manage various stamps function failure() { set -x # early stage ? - let's not create /build/@PLDISTRO@ @@ -242,7 +242,7 @@ function success () { ############################## # manage root / container contexts function in_root_context () { - rpm -q libvirt > /dev/null + rpm -q libvirt > /dev/null } # convenient for simple commands @@ -261,9 +261,9 @@ function build () { cd /build show_env - + echo "Running make IN $(pwd)" - + # stuff our own variable settings MAKEVARS=("build-GITPATH=${BUILD_SCM_URL}" "${MAKEVARS[@]}") MAKEVARS=("PLDISTRO=${PLDISTRO}" "${MAKEVARS[@]}") @@ -280,7 +280,7 @@ function build () { /build/latex-first-run.sh || : # stage1 - make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true + make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true # versions make -C /build $DRY_RUN "${MAKEVARS[@]}" versions # actual stuff @@ -326,7 +326,7 @@ function run_log () { # check it out in the build run_in_build_guest $BASE make -C /build tests-module ${MAKEVARS[@]} - + # push it onto the testmaster - just the 'system' subdir is enough rsync --verbose --archive $(rootdir $BASE)/build/MODULES/tests/system/ ${testmaster_ssh}:${BASE} # toss the build in the bargain, so the tests don't need to mess with extracting it @@ -347,7 +347,7 @@ function run_log () { case $retcod in 0) success=true; IGNORED="" ;; 2) success=true; IGNORED=true ;; - *) success=""; IGNORED="" ;; + *) success=""; IGNORED="" ;; esac # gather logs in the build vm @@ -363,7 +363,7 @@ function run_log () { echo "Tests have failed - bailing out" failure fi - + } # this part won't work if WEBHOST does not match the local host @@ -372,7 +372,7 @@ function run_log () { function sign_node_packages () { echo "Signing node packages" - + need_createrepo="" repository=$WEBPATH/$BASE/RPMS/ @@ -395,7 +395,7 @@ function sign_node_packages () { if [ -n "$new_rpms" ] ; then # Create a stamp once the package gets signed mkdir $repository/signed-stamps 2> /dev/null - + # Sign RPMS. setsid detaches rpm from the terminal, # allowing the (hopefully blank) GPG password to be # entered from stdin instead of /dev/tty. @@ -406,7 +406,7 @@ function sign_node_packages () { --resign $new_rpms && touch $new_stamps fi - # Update repository index / yum metadata. + # Update repository index / yum metadata. if [ -n "$need_createrepo" ] ; then echo "Indexing node packages after signing" if [ -f $repository/yumgroups.xml ] ; then @@ -448,7 +448,7 @@ function show_env () { function setupssh () { base=$1; shift sshkey=$1; shift - + if [ -f ${sshkey} ] ; then SSHDIR=$(rootdir ${base})/root/.ssh mkdir -p ${SSHDIR} @@ -458,7 +458,7 @@ function setupssh () { echo " StrictHostKeyChecking no" ) > ${SSHDIR}/config chmod 700 ${SSHDIR} chmod 400 ${SSHDIR}/* - else + else echo "WARNING : could not find provided ssh key $sshkey - ignored" fi } @@ -489,7 +489,7 @@ function usage () { echo " -r webroot - defaults to $DEFAULT_WEBROOT - the fs point where testbuildurl actually sits" echo " -M testmaster - defaults to $DEFAULT_TESTMASTER" echo " -Y - sign yum repo in webpath" - echo " -g gpg_path - to the gpg secring used to sign rpms. Defaults to $DEFAULT_GPGPATH" + echo " -g gpg_path - to the gpg secring used to sign rpms. Defaults to $DEFAULT_GPGPATH" echo " -u gpg_uid - email used in secring. Defaults to $DEFAULT_GPGUID" echo " -K sshkey - specify ssh key to use when reaching git over ssh" echo " -S - do not publish source rpms" @@ -572,7 +572,7 @@ function main () { # preserve options for passing them again later, together with expanded base options=$OPTS_ORIG - # allow var=value stuff; + # allow var=value stuff; for target in "$@" ; do # check if contains '=' target1=$(echo $target | sed -e s,=,,) @@ -582,7 +582,7 @@ function main () { MAKEVARS=(${MAKEVARS[@]} "$target") fi done - + # set defaults [ -z "$FCDISTRO" ] && FCDISTRO=$DEFAULT_FCDISTRO [ -z "$PLDISTRO" ] && PLDISTRO=$DEFAULT_PLDISTRO @@ -605,8 +605,8 @@ function main () { for config in ${TESTCONFIG} ; do RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --config $config" done - - + + if [ -n "$OVERBASE" ] ; then sedargs="-e s,@DATE@,${DATE},g" BASE=$(echo ${OVERBASE} | sed $sedargs) @@ -631,7 +631,7 @@ function main () { short_hostname=$(hostname | cut -d. -f1) MAIL_SUBJECT="on ${short_hostname} - ${MAIL_SUBJECT}" - ### compute WEBHOST from TESTBUILDURL + ### compute WEBHOST from TESTBUILDURL # this is to avoid having to change the builds configs everywhere # simplistic way to extract hostname from a URL WEBHOST=$(echo "$TESTBUILDURL" | cut -d/ -f 3) @@ -644,11 +644,11 @@ function main () { else trap failure ERR INT - # we run in the root context : + # we run in the root context : # (*) create or check for the vm to use # (*) copy this command in the vm # (*) invoke it - + if [ -n "$OVERBASE" ] ; then ### Re-use a vm (finish an unfinished build..) if [ ! -d $(rootdir ${BASE}) ] ; then @@ -701,7 +701,7 @@ function main () { # manage LOG - beware it might be a symlink so nuke it first LOG=$(logfile ${BASE}) rm -f $LOG - exec > $LOG 2>&1 + exec > $LOG 2>&1 set -x echo "XXXXXXXXXX $COMMAND: creating vm $BASE" $(date) show_env @@ -711,14 +711,14 @@ function main () { GIT_REPO=$(echo $BUILD_SCM_URL | cut -d@ -f1) GIT_TAG=$(echo $BUILD_SCM_URL | cut -s -d@ -f2) GIT_TAG=${GIT_TAG:-master} - mkdir -p $tmpdir + mkdir -p $tmpdir ( git archive --remote=$GIT_REPO $GIT_TAG | tar -C $tmpdir -xf -) || \ ( echo "==================== git archive FAILED, trying git clone instead" ; \ git clone $GIT_REPO $tmpdir && cd $tmpdir && git checkout $GIT_TAG && rm -rf .git) # Create lxc vm cd $tmpdir - ./lbuild-initvm.sh $VERBOSE -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${PREINSTALLED} ${BASE} + ./lbuild-initvm.sh $VERBOSE -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${PREINSTALLED} ${BASE} # cleanup cd - rm -rf $tmpdir @@ -738,7 +738,7 @@ function main () { rm $LOG; ln -s $LOG2 $LOG LOG=$LOG2 # redirect log again - exec >> $LOG 2>&1 + exec >> $LOG 2>&1 sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g" WEBPATH=$(echo ${WEBPATH} | sed $sedargs) @@ -749,8 +749,8 @@ function main () { WEBLOG=${WEBPATH}/${BASE}/log.txt # compute the log URL - inserted in the mail messages for convenience WEBBASE_URL=$(echo $WEBBASE | sed -e "s,//,/,g" -e "s,${WEBROOT},${TESTBUILDURL},") - - if [ -n "$DO_BUILD" ] ; then + + if [ -n "$DO_BUILD" ] ; then # invoke this command into the build directory of the vm cp $COMMANDPATH $(rootdir ${BASE})/build/ @@ -763,7 +763,7 @@ function main () { # publish to the web so run_log can find them set +e trap - ERR INT - webpublish rm -rf $WEBPATH/$BASE + webpublish rm -rf $WEBPATH/$BASE # guess if we've been doing any debian-related build if [ ! -f $(rootdir $BASE)/etc/debian_version ] ; then webpublish mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS} @@ -774,7 +774,7 @@ function main () { # (not needed on fedora b/c this is done by the regular build already) run_in_build_guest $BASE "(cd /build ; dpkg-scanpackages DEBIAN/ | gzip -9c > Packages.gz)" webpublish mkdir -p $WEBPATH/$BASE/DEBIAN - webpublish_rsync_files $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb + webpublish_rsync_files $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb webpublish_rsync_files $WEBPATH/$BASE/ $(rootdir $BASE)/build/Packages.gz fi # publish myplc-release if this exists @@ -796,16 +796,16 @@ function main () { fi fi - if [ -n "$DO_TEST" ] ; then + if [ -n "$DO_TEST" ] ; then run_log fi - success + success - echo "==================== MAIN END $(date)" + echo "==================== MAIN END $(date)" fi } ########## -main "$@" +main "$@" diff --git a/lxc.mk b/lxc.mk index 5a238de5..07aa552f 100644 --- a/lxc.mk +++ b/lxc.mk @@ -31,7 +31,7 @@ IN_NODEIMAGE += transforward # procprotect: root context module for protecting against weaknesses in /proc # ### remove procprotect from the nodes on f20 and above, needs more work starting with 3.19 -ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27)" +ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27 f29)" procprotect-MODULES := procprotect procprotect-SPEC := procprotect.spec ALL += procprotect @@ -43,7 +43,7 @@ endif # ### starting August 2015, ipfw module won't build against fedora22 # that comes with kernel 4.1.4 -ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f21 f22 f23 f24 f25 f27)" +ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f21 f22 f23 f24 f25 f27 f29)" ipfwroot-MODULES := ipfw ipfwroot-SPEC := planetlab/ipfwroot.spec ALL += ipfwroot @@ -130,7 +130,7 @@ IN_NODEIMAGE += codemux # fprobe-ulog # # xxx temporarily turning this off on f20 and above -ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27)" +ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27 f29)" fprobe-ulog-MODULES := fprobe-ulog fprobe-ulog-SPEC := fprobe-ulog.spec ALL += fprobe-ulog @@ -387,7 +387,7 @@ IN_NODEIMAGE += pyplnet IN_MYPLC += pyplnet IN_BOOTCD += pyplnet -ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f23 f24 f25 f27)" +ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f23 f24 f25 f27 f29)" # # OML measurement library # diff --git a/pkgs.py b/pkgs.py index c5a0bf2e..deb89064 100755 --- a/pkgs.py +++ b/pkgs.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -# This is a replacement for the formerly bash-written function pl_parsePkgs () -# +# This is a replacement for the formerly bash-written function pl_parsePkgs () +# # Usage: $0 [-a arch] default_arch keyword fcdistro pldistro pkgs-file[..s] # default_arch is $pl_DISTRO_ARCH, but can be overridden # @@ -35,14 +35,14 @@ import re default_arch='x86_64' known_arch = ['i386', 'i686', 'x86_64'] -default_fcdistro = 'f27' +default_fcdistro = 'f29' known_fcdistros = [ 'centos5', 'centos6', # oldies but we have references to that in the pkgs files 'f8', 'f10', 'f12', 'f16', # these ones are still relevant - 'f14', 'f18', 'f20', 'f21', 'f22', 'f23', 'f24', 'f25', 'f27', - 'sl6', + 'f14', 'f18', 'f20', 'f21', 'f22', 'f23', 'f24', 'f25', 'f27', 'f29', + 'sl6', # debians 'wheezy','jessie', # ubuntus @@ -56,8 +56,8 @@ known_fcdistros = [ default_pldistro='onelab' known_keywords = [ - 'group', 'groupname', 'groupdesc', - 'package', 'pip', 'gem', + 'group', 'groupname', 'groupdesc', + 'package', 'pip', 'gem', 'nodeyumexclude', 'plcyumexclude', 'yumexclude', 'precious', 'junk', 'mirror', ] @@ -121,7 +121,7 @@ class PkgsParser: re_old = '[a-z]+[+-][a-z]+[0-9]+' m_old = re.compile('\A{re_old}\Z'.format(**locals())) - + # returns a tuple (included, excluded) def parse(self, filename): ok = True @@ -159,7 +159,7 @@ class PkgsParser: # skip if another distro family if distro != self.distro: continue # skip if the qualifier does not fit - if not self.match (qual, version): + if not self.match (qual, version): if self.options.verbose: print('{filename}:{lineno}:qualifer {left} does not apply' .format(**locals()), file=stderr) @@ -180,7 +180,7 @@ class PkgsParser: format(**locals())) else: raise Exception('error in left expression {left}'.format(**locals())) - + except Exception as e: ok = False print("{filename}:{lineno}:syntax error: {e}".format(**locals()), file=stderr) @@ -200,7 +200,7 @@ class PkgsParser: ok = ok and o # avoid set operations that would not preserve order results = [ x for x in included if x not in excluded ] - + results = [ x.replace('@arch@', self.arch).\ replace('@fcdistro@', self.fcdistro).\ replace('@pldistro@', self.pldistro) for x in results] @@ -231,7 +231,7 @@ def main (): parser.add_option ('-u', '--no-sort', dest='sort_results', default=True, action='store_false', help='keep results in the same order as in the inputs') (options, args) = parser.parse_args() - + if len(args) <= 1 : parser.print_help(file=stderr) sys.exit(1) -- 2.43.0