#
-# Thierry Parmentelat - INRIA Sophia Antipolis
+# Thierry Parmentelat - INRIA Sophia Antipolis
#
####################
# invocation:
# (*) as of nov. 2007, myplc-devel is deprecated
# (*) instead, we create a fresh vserver that holds required tools (see e.g. planetlab-devel.lst)
# (*) the build uses the current fedora version as a target for the produced images
-# (*) so you simply need to create a fedora 8 build image for building fedora-8 images
-#
+# (*) so you simply need to create a fedora 8 build image for building fedora-8 images
+#
#################### (planetlab) distributions
#
# (*) see README-pldistros.txt
-# (*) then you need to run
+# (*) then you need to run
# make stage1=true PLDISTRO=onelab
#
-####################
+####################
# This build deals with 3 kinds of objects
-#
+#
# (*) packages are named upon the RPM name; they are mostly lowercase
# Add a package to ALL if you want it built as part of the default set.
# (*) modules are named after the subversion or git tree; we would wish to keep
# these names lowercase as far as possible
# (*) rpms are named in the spec files. A package typically defines several rpms;
# rpms are used for defining LOCAL-DEVEL-RPMS. See also package.rpmnames
-#
+#
# in simple cases, one package uses one module (in which case using the same name sounds right)
-# but others might need several modules (e.g. image-creation packages like bootstrapfs need
-# bootstrapfs and build); in this case the FIRST ONE is used for locating the specfile
+# but others might need several modules (e.g. image-creation packages like bootstrapfs need
+# bootstrapfs and build); in this case the FIRST ONE is used for locating the specfile
#
#################### packages
# basics: how to build a package - you need to define the following variables
-#
+#
# (*) package-MODULES
-# a package needs one or several modules to build; the first one is used for
+# a package needs one or several modules to build; the first one is used for
# some special purposes, like locating the specfile
# (*) package-SPEC
# the package's specfile; this is relative to the FIRST module in package-MODULES
# rpms in this area will *not* be uninstalled after the target is made, because that would ruin the build vm for good
# (*) package-DEPEND-FILES
# a set of files that the package depends on - and that make needs to know about
-# if this contains RPMS/yumgroups.xml, then the toplevel RPMS's index
+# if this contains RPMS/yumgroups.xml, then the toplevel RPMS's index
# is refreshed with createrepo prior to running rpmbuild
# (*) package-SPECVARS
-# space-separated list of spec variable definitions, where you can reference make variables that
-# belong to packages defined BEFORE the current one
+# space-separated list of spec variable definitions, where you can reference make variables that
+# belong to packages defined BEFORE the current one
# note: you should use = to define these (as opposed to :=)
-# e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release)
-# would let you use the %release from the kernel's package when rpmbuild'ing mydriver
+# e.g. mydriver-SPECVARS = foo=$(kernel-rpm-release)
+# would let you use the %release from the kernel's package when rpmbuild'ing mydriver
# see automatic below
# (*) package-RPMFLAGS: Miscellaneous RPM flags
# this is passed to rpmbuild, as well as to spec2make for "exporting" various rpm variable
# to make; beware that some features, like --with=debug or --define 'foo bar' are not
# well handled by spec2make as of this writing, which can cause issues.
# hopefully this will be fixed...
-# (*) package-BUILD-FROM-SRPM: set this to any non-empty value,
-# if your package is able to produce a source rpm.
-# In this case the build will first invoke 'make srpm', and then rebuild binaries from that
+# (*) package-BUILD-FROM-SRPM: set this to any non-empty value,
+# if your package is able to produce a source rpm.
+# In this case the build will first invoke 'make srpm', and then rebuild binaries from that
# (*) package-RPMDATE: set this to any non-empty value to get the rpm package's release field hold the current date
# this is useful for container packages, like e.g. bootstrapfs or vserver, that contains much more than the
# correspondng module
#
# (*) module-SVNPATH
# for svn modules
-# the complete path where this module lies;
+# the complete path where this module lies;
# you can specify the trunk or a given tag with this variable
-#
+#
# (*) module-GITPATH
# for git modules
# which by analogy with svn revision scheme, is expected to be <url>@<tag>
#
# Default values
#
-# minimal compat with macos, just so this does not complain
+# minimal compat with macos, just so this does not complain
HOSTARCH := $(shell uname -i 2> /dev/null || uname -m 2> /dev/null)
DISTRO := $(shell ./getdistro.sh)
RELEASE := $(shell ./getrelease.sh)
include $(PLDISTROTAGS)
# this used to be set in the -tags.mk files, but that turned out to require
-# error-prone duplicate changes
+# error-prone duplicate changes
# so now the nightly build script sets this to what it is currently using
# in case we run this manually, i.e. if build-GITPATH is not set
ifeq "$(build-GITPATH)" ""
$(eval $(call remote_pldistro,wexlxc,wexlxc-tags,git))
########## stage1 and stage1iter
-# extract specs and compute .mk files by running
+# extract specs and compute .mk files by running
# make stage1=true
# entering stage1, we compute all the spec files
-# then we use stage1iter to compute the .mk iteratively,
+# then we use stage1iter to compute the .mk iteratively,
# ensuring that the n-1 first makefiles are loaded when creating the n-th one
# when stage1iter is set, it is supposed to be an index (starting at 1) in $(ALL)
#all : modules
#all : rpms
#all : srpms
-# mention $(ALL) here rather than rpms
+# mention $(ALL) here rather than rpms
# this is because the inter-package dependencies are expressed like
# util-vserver: util-python
all: rpms
endif
### yumgroups.xml : compute from all known .pkgs files
-RPMS/yumgroups.xml:
+RPMS/yumgroups.xml:
mkdir -p RPMS
./yumgroups.sh $(PLDISTRO) > $@
-createrepo = createrepo --quiet -g yumgroups.xml RPMS/
+createrepo = createrepo --quiet -g yumgroups.xml RPMS/
repo: RPMS/yumgroups.xml
$(createrepo)
.PHONY: repo
####################
-# notes:
+# notes:
# * we always use the first module's location (SVNPATH/GITPATH) to extract the spec file
# * no matter what SCM is used, SPEC should hold a relative path from the module's root
#
$(foreach module,$(ALL.modules), $(eval $(call stage1_module_vars,$(module))))
#
-# for each package, compute whether we need to set date
+# for each package, compute whether we need to set date
# the heuristic is that we mention the date as part of the rpm release flag if
# (*) the package has requested it by setting package-RPMDATE (container packages should do that)
-# (*) or SVNPATH contains 'trunk' or 'branches'
-# (*) or GITPATH has no '@' (trunk)
+# (*) or SVNPATH contains 'trunk' or 'branches'
+# (*) or GITPATH has no '@' (trunk)
# (*) or GITPATH contains a '@', and the gittag part has no '-' (this is to tell a tag from a branch..)
-#
+#
define package_hasdate
$(1).has-date = $(if $($(1)-RPMDATE),yes, \
$(if $($($(1).module)-SVNPATH), \
echo "%global _source_filedigest_algorithm 1" >> $@
echo "%global _source_payload w9.gzdio" >> $@
echo "%global _binary_payload w9.gzdio" >> $@
-ifeq "$(DISTRONAME)""$(filter $(DISTRONAME),f25 f27)"
+ifeq "$(DISTRONAME)""$(filter $(DISTRONAME),f25 f27 f29)"
echo "%define debug_package %{nil}" >> $@
endif
ifeq "$(shell pwd)" "/build"
rm -f $(FAKEROOT) ; ln -s $(REALROOT) $(FAKEROOT)
endif
- rm -f $@
+ rm -f $@
echo "%_topdir $(HOME)" >> $@
echo "%_tmppath $(HOME)/tmp" >> $@
echo "%__spec_install_pre %{___build_pre}" >> $@
##
URLS/%: url=$(subst @colon@,:,$(subst @slash@,/,$(notdir $@)))
URLS/%: basename=$(notdir $(url))
-URLS/%:
+URLS/%:
echo curl $(url) -o SOURCES/$(basename)
touch $@
-### the directory SOURCES/<package>-<version> is made
+### the directory SOURCES/<package>-<version> is made
# with a (set of) copy -rl from MODULES/<module>
-# the former is $(package.source)
+# the former is $(package.source)
ALLSOURCES:=$(foreach package, $(ALL), $($(package).source))
# so that make does not use the rule below directly for creating the tarball files
.SECONDARY: $(ALLSOURCES)
RPMYUM-UNINSTALL-STOCK := rpm -e
### these macro handles the LOCAL-DEVEL-RPMS and LOCAL-DEVEL-RPMS-CRUCIAL tags for a given package
-# before building : rpm-install LOCAL-DEVEL-RPMS
-define rpmyum_install_local_rpms
- $(if $($(1).all-local-devel-rpm-paths), echo "Installing for $(1)-LOCAL-DEVEL-RPMS" ; $(RPMYUM-INSTALL-LOCAL) $($(1).all-local-devel-rpm-paths))
+# before building : rpm-install LOCAL-DEVEL-RPMS
+define rpmyum_install_local_rpms
+ $(if $($(1).all-local-devel-rpm-paths), echo "Installing for $(1)-LOCAL-DEVEL-RPMS" ; $(RPMYUM-INSTALL-LOCAL) $($(1).all-local-devel-rpm-paths))
endef
# install stock rpms if defined
-define rpmyum_install_stock_rpms
+define rpmyum_install_stock_rpms
$(if $($(1)-STOCK-DEVEL-RPMS), echo "Installing for $(1)-STOCK-DEVEL-RPMS" ; $(RPMYUM-INSTALL-STOCK) $($(1)-STOCK-DEVEL-RPMS) || true)
endef
DPKGAPT-INSTALL-STOCK := apt-get -y install
DPKGAPT-UNINSTALL-STOCK := echo WARNING uninstalling stock debs not implemented
-define dpkgapt_install_local_debs
+define dpkgapt_install_local_debs
$(if $($(1)-LOCAL-DEVEL-DEBS), echo "Installing for $(1)-LOCAL-DEVEL-DEBS" ; $(foreach debname,$($(1)-LOCAL-DEVEL-DEBS),$(DPKGAPT-INSTALL-LOCAL) $(wildcard DEBIAN/$(debname)_*.deb);))
endef
-define dpkgapt_install_stock_debs
+define dpkgapt_install_stock_debs
$(if $($(1)-STOCK-DEVEL-DEBS), echo "Installing for $(1)-STOCK-DEVEL-DEBS" ; $(DPKGAPT-INSTALL-STOCK) $($(1)-STOCK-DEVEL-DEBS) || true)
endef
####################
# usage: target_source_rpm package
-define target_source_rpm
+define target_source_rpm
ifeq "$($(1)-BUILD-FROM-SRPM)" ""
-$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).tarballs)
+$($(1).srpm): $($(1).specpath) .rpmmacros $($(1).tarballs)
mkdir -p BUILD SRPMS tmp
@(echo -n "XXXXXXXXXXXXXXX -- BEG SRPM $(1) (using SOURCES) " ; date)
$(call rpmyum_install_stock_rpms,$(1))
$(call rpmyum_install_local_rpms,$(1))
make -C $($(1).source) srpm SPECFILE=$(HOME)/$($(1).specpath) EXPECTED_SRPM=$(notdir $($(1).srpm)) && \
rm -f SRPMS/$(notdir $($(1).srpm)) && \
- ln $($(1).source)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm))
+ ln $($(1).source)/$(notdir $($(1).srpm)) SRPMS/$(notdir $($(1).srpm))
$(call rpmyum_uninstall_stock_rpms,$(1))
@(echo -n "XXXXXXXXXXXXXXX -- END SRPM $(1) " ; date)
endif
@echo $(words $(ALLRPMS)) binary rpms OK
.PHONY: rpms
-# use tmp dirs when building binary rpm so make remains idempotent
+# use tmp dirs when building binary rpm so make remains idempotent
# otherwise SOURCES/ or SPEC gets touched again - which leads to rebuilding
RPM-USE-TMP-DIRS = --define "_sourcedir $(HOME)/tmp" --define "_specdir $(HOME)/tmp"
RPM-USE-COMPILE-DIRS = --define "_sourcedir $(HOME)/COMPILE" --define "_specdir $(HOME)/COMPILE"
# usage: build_binary_rpm package
# xxx hacky - invoke createrepo if DEPEND-FILES mentions RPMS/yumgroups.xml
-define target_binary_rpm
+define target_binary_rpm
$($(1).rpms): $($(1).srpm)
mkdir -p RPMS tmp
@(echo -n "XXXXXXXXXXXXXXX -- BEG RPM $(1) " ; date)
### shorthand target
# e.g. make proper -> does propers rpms
# usage shorthand_target package
-define target_shorthand
+define target_shorthand
$(1): $($(1).rpms)
.PHONY: $(1)
$(1)-spec: $($(1)-SPEC)
# very rough for now (one module per package), targets only sfa for now
# the general idea here is, changing the specfile (for version number and all) is enough, and this
# gets passed to "make debian" in the module
-# PREFIX: at one point we had passed PREFIX=/usr to this 'make debian'
+# PREFIX: at one point we had passed PREFIX=/usr to this 'make debian'
# however it turned out we could manage this some other way (see manifold)
# so I'm reverting to simplicity
define target_debian
CLEANS += $(1)-clean-srpm
$(1)-codeclean: $(1)-clean-source $(1)-clean-tarball $(1)-clean-build $(1)-clean-rpms $(1)-clean-srpm $(1)-clean-debian
$(1)-clean: $(1)-clean-modules $(1)-codeclean
-.PHONY: $(1)-codeclean $(1)-clean
+.PHONY: $(1)-codeclean $(1)-clean
$(1)-clean-spec:
rm -rf $($(1).specpath)
.PHONY: $(1)-clean-spec
@echo "Build target-arch: $(HOSTARCH)"
@echo "Build target-distro: $(DISTRO)"
@echo "Build target-distroname: $(DISTRONAME)"
- @echo "Build target-release: $(RELEASE)"
- @echo "Build target-personality: $(PERSONALITY)"
+ @echo "Build target-release: $(RELEASE)"
+ @echo "Build target-personality: $(PERSONALITY)"
-####################
+####################
# for a given module
VFORMAT="%30s := %s\n"
define print_tag_target
#################### package info
PKGKEYS := tarballs source srpm rpms rpmnames rpm-release rpm-name rpm-version rpm-subversion
%-pkginfo: package=$(subst -pkginfo,,$@)
-%-pkginfo:
+%-pkginfo:
@$(foreach key,$(PKGKEYS),echo "$(package).$(key)=$($(package).$(key))";)
## rpm info
RPMKEYS := rpm-path package
%-rpminfo: rpm=$(subst -rpminfo,,$@)
-%-rpminfo:
+%-rpminfo:
@$(foreach key,$(RPMKEYS),echo "$(rpm).$(key)=$($(rpm).$(key))";)
#################### various lists - designed to run with stage1=true
echo $(module):$($(module)-BRANCH); , \
echo $(module); )))
-info: info-packages info-modules info-branches
+info: info-packages info-modules info-branches
.PHONY: info info-packages info-modules info-branches module-tools
# pkgs parsing utilities + lbuild-bridge.sh
export PATH=$(dirname $0):$PATH
-# old guests have e.g. mount in /bin but this is no longer part of
+# old guests have e.g. mount in /bin but this is no longer part of
# the standard PATH in recent hosts after usrmove, so let's keep it simple
export PATH=$PATH:/bin:/sbin
# XXX fixme : when creating a 32bits VM we need to call linux32 as appropriate...s
-DEFAULT_FCDISTRO=f27
+DEFAULT_FCDISTRO=f29
DEFAULT_PLDISTRO=lxc
DEFAULT_PERSONALITY=linux64
DEFAULT_MEMORY=3072
python <<EOF
import sys
masklen=$masklen
-if not (masklen>=1 and masklen<=32):
+if not (masklen>=1 and masklen<=32):
print "Wrong masklen",masklen
exit(1)
result=[]
result.append(masklen)
masklen=0
print ".".join([ str(256-2**(8-i)) for i in result ])
-
+
EOF
}
echo debootstrap ;;
*)
echo Unknown distro $fcdistro ;;
- esac
+ esac
}
# return arch from debian distro and personality
}
# the new test framework creates /timestamp in /vservers/<name> *before* populating it
-function almost_empty () {
- dir="$1"; shift ;
+function almost_empty () {
+ dir="$1"; shift ;
# non existing is fine
- [ ! -d $dir ] && return 0;
+ [ ! -d $dir ] && return 0;
# need to have at most one file
- count=$(cd $dir; ls | wc -l); [ $count -le 1 ];
+ count=$(cd $dir; ls | wc -l); [ $count -le 1 ];
}
##############################
cache=/var/cache/lxc/fedora/$arch/${fedora_release}
mkdir -p $cache
-
+
(
flock --exclusive --timeout 60 200 || { echo "Cache repository is busy." ; return 1 ; }
echo "Filling $lxc_root from $cache/rootfs ... "
rsync -a $cache/rootfs/ $lxc_root/
-
+
return 0
) 200> $cache/lock
mkdir -p $INSTALL_ROOT || { echo "Failed to create '$INSTALL_ROOT' directory" ; return 1; }
- mkdir -p $INSTALL_ROOT/etc/yum.repos.d
+ mkdir -p $INSTALL_ROOT/etc/yum.repos.d
mkdir -p $INSTALL_ROOT/dev
mknod -m 0444 $INSTALL_ROOT/dev/random c 1 8
mknod -m 0444 $INSTALL_ROOT/dev/urandom c 1 9
# append fedora repo files with desired ${fedora_release} and $basearch
for f in $INSTALL_ROOT/etc/yum.repos.d/* ; do
sed -i "s/\$basearch/$arch/g; s/\$releasever/${fedora_release}/g;" $f
- done
+ done
MIRROR_URL=$FEDORA_MIRROR_BASE/releases/${fedora_release}/Everything/$arch/os
RELEASE_URL1="$MIRROR_URL/Packages/fedora-release-${fedora_release}-1.noarch.rpm"
# first try the second version of fedora-release first
RELEASE_URL2="$MIRROR_URL/Packages/f/fedora-release-${fedora_release}-2.noarch.rpm"
RELEASE_URL3="$MIRROR_URL/Packages/f/fedora-release-${fedora_release}-1.noarch.rpm"
-
+
RELEASE_TARGET=$INSTALL_ROOT/fedora-release-${fedora_release}.noarch.rpm
found=""
for attempt in $RELEASE_URL1 $RELEASE_URL2 $RELEASE_URL3; do
fi
done
[ -n "$found" ] || { echo "Could not retrieve fedora-release rpm - exiting" ; exit 1; }
-
+
mkdir -p $INSTALL_ROOT/var/lib/rpm
rpm --root $INSTALL_ROOT --initdb
# when installing f12 this apparently is already present, so ignore result
# however f12 root images won't get created on a f18 host
# (the issue here is the same as the one we ran into when dealing with a vs-box)
# in a nutshell, in f12 the glibc-common and filesystem rpms have an apparent conflict
- # >>> file /usr/lib/locale from install of glibc-common-2.11.2-3.x86_64 conflicts
+ # >>> file /usr/lib/locale from install of glibc-common-2.11.2-3.x86_64 conflicts
# with file from package filesystem-2.4.30-2.fc12.x86_64
- # in fact this was - of course - allowed by f12's rpm but later on a fix was made
+ # in fact this was - of course - allowed by f12's rpm but later on a fix was made
# http://rpm.org/gitweb?p=rpm.git;a=commitdiff;h=cf1095648194104a81a58abead05974a5bfa3b9a
# So ideally if we want to be able to build f12 images from f18 we need an rpm that has
# this patch undone, like we have in place on our f14 boxes (our f14 boxes need a f18-like rpm)
echo 0 > $lxc_root/selinux/enforce
# set the hostname
- case "$fcdistro" in
+ case "$fcdistro" in
f18|f2?)
cat <<EOF > ${lxc_root}/etc/sysconfig/network
NETWORKING=yes
lxc=$1; shift
lxc_root=$(lxcroot $lxc)
- # so ignore if we can't find /etc/systemd at all
+ # so ignore if we can't find /etc/systemd at all
[ -d ${lxc_root}/etc/systemd ] || return 0
# otherwise let's proceed
ln -sf /lib/systemd/system/multi-user.target ${lxc_root}/etc/systemd/system/default.target
# overwrite container yum config
function fedora_configure_yum () {
- set -x
- set -e
+ set -x
+ set -e
trap failure ERR INT
lxc=$1; shift
if [ ! -d $lxc_root/etc/yum.repos.d ] ; then
echo "WARNING : cannot create myplc repo"
else
- # exclude kernel from fedora repos
+ # exclude kernel from fedora repos
yumexclude=$(pl_plcyumexclude $fcdistro $pldistro $DIRNAME)
for repo in $lxc_root/etc/yum.repos.d/* ; do
- [ -f $repo ] && yumconf_exclude $repo "exclude=$yumexclude"
+ [ -f $repo ] && yumconf_exclude $repo "exclude=$yumexclude"
done
# the build repo is not signed at this stage
cat > $lxc_root/etc/yum.repos.d/myplc.repo <<EOF
EOF
fi
fi
-}
+}
##############################
# apparently ubuntu exposes a mirrors list by country at
function debian_mirror () {
fcdistro=$1; shift
case $fcdistro in
- wheezy|jessie)
+ wheezy|jessie)
echo http://ftp2.fr.debian.org/debian/ ;;
- precise|trusty|utopic|vivid|wily|xenial)
+ precise|trusty|utopic|vivid|wily|xenial)
# echo http://mir1.ovh.net/ubuntu/ubuntu/ ;;
echo http://www-ftp.lip6.fr/pub/linux/distributions/Ubuntu/archive/ ;;
*) echo unknown distro $fcdistro; exit 1;;
cat <<EOF > ${lxc_root}/etc/hostname
$GUEST_HOSTNAME
EOF
-
+
}
function debian_configure () {
lxc_root=$(lxcroot $lxc)
- # create lxc container
-
+ # create lxc container
+
pkg_method=$(package_method $fcdistro)
case $pkg_method in
yum)
# Enable cgroup -- xxx -- is this really useful ?
[ -d $lxc_root/cgroup ] || mkdir $lxc_root/cgroup
-
+
### set up resolv.conf from host
# ubuntu precise and on, /etc/resolv.conf is a symlink to ../run/resolvconf/resolv.conf
[ -h $lxc_root/etc/resolv.conf ] && rm -f $lxc_root/etc/resolv.conf
cp /etc/resolv.conf $lxc_root/etc/resolv.conf
### and /etc/hosts for at least localhost
[ -f $lxc_root/etc/hosts ] || echo "127.0.0.1 localhost localhost.localdomain" > $lxc_root/etc/hosts
-
+
# grant ssh access from host to guest
mkdir -p $lxc_root/root/.ssh
cat /root/.ssh/id_rsa.pub >> $lxc_root/root/.ssh/authorized_keys
# don't keep the input xml, this can be retrieved at all times with virsh dumpxml
config_xml=/tmp/$lxc.xml
( [ -n "$NAT_MODE" ] && write_lxc_xml_natip $lxc || write_lxc_xml_publicip $lxc ) > $config_xml
-
+
# define lxc container for libvirt
virsh -c lxc:/// define $config_xml
}
# grant build guests the ability to do mknods
-function write_lxc_xml_natip () {
+function write_lxc_xml_natip () {
lxc=$1; shift
lxc_root=$(lxcroot $lxc)
cat <<EOF
function devel_or_test_tools () {
- set -x
- set -e
+ set -x
+ set -e
trap failure ERR INT
lxc=$1; shift
### install individual packages, then groups
# get target arch - use uname -i here (we want either x86_64 or i386)
-
+
lxc_arch=$(chroot ${lxc_root} $personality uname -i)
# on debian systems we get arch through the 'arch' command
[ "$lxc_arch" = "unknown" ] && lxc_arch=$(chroot ${lxc_root} $personality arch)
}
function post_install () {
- lxc=$1; shift
+ lxc=$1; shift
personality=$1; shift
lxc_root=$(lxcroot $lxc)
# setup localtime from the host
# just in case, let's stay on the safe side
function sshd_disable_password_auth () {
- lxc=$1; shift
+ lxc=$1; shift
lxc_root=$(lxcroot $lxc)
sed --in-place=.password -e 's,^#\?PasswordAuthentication.*,PasswordAuthentication no,' \
$lxc_root/etc/ssh/sshd_config
function post_install_natip () {
- set -x
- set -e
+ set -x
+ set -e
trap failure ERR INT
lxc=$1; shift
### From myplc-devel-native.spec
# be careful to backslash $ in this, otherwise it's the root context that's going to do the evaluation
cat << EOF | chroot ${lxc_root} $personality bash -x
-
+
# customize root's prompt
/bin/cat << PROFILE > /root/.profile
export PS1="[$lxc] \\w # "
PROFILE
EOF
-
+
}
function post_install_myplc () {
- set -x
- set -e
+ set -x
+ set -e
trap failure ERR INT
lxc=$1; shift
# if run in public_ip mode, we know the IP of the guest and it is specified here
[ -n "$1" ] && { guest_ip=$1; shift; }
- #wait max 2 min for sshd to start
+ #wait max 2 min for sshd to start
success=""
current_time=$(date +%s)
stop_time=$(($current_time + 120))
-
+
counter=1
while [ "$current_time" -lt "$stop_time" ] ; do
echo "$counter-th attempt to reach sshd in container $lxc ..."
[ -z "$guest_ip" ] && guest_ip=$(guest_ipv4 $lxc)
- [ -n "$guest_ip" ] && ssh -o "StrictHostKeyChecking no" $guest_ip 'uname -i' && {
+ [ -n "$guest_ip" ] && ssh -o "StrictHostKeyChecking no" $guest_ip 'uname -i' && {
success=true; echo "SSHD in container $lxc is UP on IP $guest_ip"; break ; } || :
counter=$(($counter+1))
sleep 10
done
# Thierry: this is fatal, let's just exit with a failure here
- [ -z $success ] && { echo "SSHD in container $lxc could not be reached (guest_ip=$guest_ip)" ; exit 1 ; }
+ [ -z $success ] && { echo "SSHD in container $lxc could not be reached (guest_ip=$guest_ip)" ; exit 1 ; }
return 0
}
}
function usage () {
- set +x
+ set +x
echo "Usage: $COMMAND [options] lxc-name (aka build mode)"
echo "Usage: $COMMAND -n hostname [options] lxc-name (aka test mode)"
echo "Description:"
exit 1
}
-### parse args and
+### parse args and
function main () {
#set -e
*) usage ;;
esac
done
-
+
shift $(($OPTIND - 1))
# parse fixed arguments
[ -z "$pldistro" ] && pldistro=$DEFAULT_PLDISTRO
[ -z "$personality" ] && personality=$DEFAULT_PERSONALITY
[ -z "$MEMORY" ] && MEMORY=$DEFAULT_MEMORY
-
+
# set memory in KB
MEMORY=$(($MEMORY * 1024))
-
+
# the set of preinstalled packages - depends on mode
if [ -z "$PREINSTALLED" ] ; then
if [ -n "$NAT_MODE" ] ; then
if [ "$REPO_URL" == "none" ] ; then
REPO_URL=""
elif [ -z "$REPO_URL" ] ; then
- echo "WARNING -- setting up a yum repo is recommended"
+ echo "WARNING -- setting up a yum repo is recommended"
fi
fi
VIF_HOST="vif$(echo $GUEST_HOSTNAME | cut -d. -f1)"
fi
- setup_lxc $lxc $fcdistro $pldistro $personality
+ setup_lxc $lxc $fcdistro $pldistro $personality
# historically this command is for setting up a build or a test VM
# kind of patchy right now though
# container gets started here
post_install $lxc $personality
-
+
echo $COMMAND Done
exit 0
# close stdin, as with ubuntu and debian VMs this script tends to hang and wait for input ..
0<&-
-# old guests have e.g. mount in /bin but this is no longer part of
+# old guests have e.g. mount in /bin but this is no longer part of
# the standard PATH in recent hosts after usrmove, so let's keep it simple
export PATH=$PATH:/bin:/sbin
# default values, tunable with command-line options
-DEFAULT_FCDISTRO=f27
+DEFAULT_FCDISTRO=f29
DEFAULT_PLDISTRO=lxc
DEFAULT_PERSONALITY=linux64
DEFAULT_MAILDEST="build at onelab.eu"
# use with care, a *lot* of other things can go bad as well
function summary () {
from=$1; shift
- echo "******************** BEG SUMMARY"
+ echo "******************** BEG SUMMARY"
python - $from <<EOF
#!/usr/bin/env python
# read a full log and tries to extract the interesting stuff
echo=False
# any 'installing' line
elif m_installing_any.match(line):
- if echo:
+ if echo:
installing=m_installing_any.match(line).group(1)
print '>>>',installing
echo=False
for arg in sys.argv[1:]:
summary(arg)
EOF
- echo "******************** END SUMMARY"
+ echo "******************** END SUMMARY"
}
### we might build on a box other than the actual web server
total_minutes=$(($total_seconds/60))
minutes=$(($total_minutes%60))
hours=$(($total_minutes/60))
-
+
printf "%02d:%02d:%02d" $hours $minutes $seconds
}
-# Notify recipient of failure or success, manage various stamps
+# Notify recipient of failure or success, manage various stamps
function failure() {
set -x
# early stage ? - let's not create /build/@PLDISTRO@
##############################
# manage root / container contexts
function in_root_context () {
- rpm -q libvirt > /dev/null
+ rpm -q libvirt > /dev/null
}
# convenient for simple commands
cd /build
show_env
-
+
echo "Running make IN $(pwd)"
-
+
# stuff our own variable settings
MAKEVARS=("build-GITPATH=${BUILD_SCM_URL}" "${MAKEVARS[@]}")
MAKEVARS=("PLDISTRO=${PLDISTRO}" "${MAKEVARS[@]}")
/build/latex-first-run.sh || :
# stage1
- make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true
+ make -C /build $DRY_RUN "${MAKEVARS[@]}" stage1=true
# versions
make -C /build $DRY_RUN "${MAKEVARS[@]}" versions
# actual stuff
# check it out in the build
run_in_build_guest $BASE make -C /build tests-module ${MAKEVARS[@]}
-
+
# push it onto the testmaster - just the 'system' subdir is enough
rsync --verbose --archive $(rootdir $BASE)/build/MODULES/tests/system/ ${testmaster_ssh}:${BASE}
# toss the build in the bargain, so the tests don't need to mess with extracting it
case $retcod in
0) success=true; IGNORED="" ;;
2) success=true; IGNORED=true ;;
- *) success=""; IGNORED="" ;;
+ *) success=""; IGNORED="" ;;
esac
# gather logs in the build vm
echo "Tests have failed - bailing out"
failure
fi
-
+
}
# this part won't work if WEBHOST does not match the local host
function sign_node_packages () {
echo "Signing node packages"
-
+
need_createrepo=""
repository=$WEBPATH/$BASE/RPMS/
if [ -n "$new_rpms" ] ; then
# Create a stamp once the package gets signed
mkdir $repository/signed-stamps 2> /dev/null
-
+
# Sign RPMS. setsid detaches rpm from the terminal,
# allowing the (hopefully blank) GPG password to be
# entered from stdin instead of /dev/tty.
--resign $new_rpms && touch $new_stamps
fi
- # Update repository index / yum metadata.
+ # Update repository index / yum metadata.
if [ -n "$need_createrepo" ] ; then
echo "Indexing node packages after signing"
if [ -f $repository/yumgroups.xml ] ; then
function setupssh () {
base=$1; shift
sshkey=$1; shift
-
+
if [ -f ${sshkey} ] ; then
SSHDIR=$(rootdir ${base})/root/.ssh
mkdir -p ${SSHDIR}
echo " StrictHostKeyChecking no" ) > ${SSHDIR}/config
chmod 700 ${SSHDIR}
chmod 400 ${SSHDIR}/*
- else
+ else
echo "WARNING : could not find provided ssh key $sshkey - ignored"
fi
}
echo " -r webroot - defaults to $DEFAULT_WEBROOT - the fs point where testbuildurl actually sits"
echo " -M testmaster - defaults to $DEFAULT_TESTMASTER"
echo " -Y - sign yum repo in webpath"
- echo " -g gpg_path - to the gpg secring used to sign rpms. Defaults to $DEFAULT_GPGPATH"
+ echo " -g gpg_path - to the gpg secring used to sign rpms. Defaults to $DEFAULT_GPGPATH"
echo " -u gpg_uid - email used in secring. Defaults to $DEFAULT_GPGUID"
echo " -K sshkey - specify ssh key to use when reaching git over ssh"
echo " -S - do not publish source rpms"
# preserve options for passing them again later, together with expanded base
options=$OPTS_ORIG
- # allow var=value stuff;
+ # allow var=value stuff;
for target in "$@" ; do
# check if contains '='
target1=$(echo $target | sed -e s,=,,)
MAKEVARS=(${MAKEVARS[@]} "$target")
fi
done
-
+
# set defaults
[ -z "$FCDISTRO" ] && FCDISTRO=$DEFAULT_FCDISTRO
[ -z "$PLDISTRO" ] && PLDISTRO=$DEFAULT_PLDISTRO
for config in ${TESTCONFIG} ; do
RUN_LOG_EXTRAS="$RUN_LOG_EXTRAS --config $config"
done
-
-
+
+
if [ -n "$OVERBASE" ] ; then
sedargs="-e s,@DATE@,${DATE},g"
BASE=$(echo ${OVERBASE} | sed $sedargs)
short_hostname=$(hostname | cut -d. -f1)
MAIL_SUBJECT="on ${short_hostname} - ${MAIL_SUBJECT}"
- ### compute WEBHOST from TESTBUILDURL
+ ### compute WEBHOST from TESTBUILDURL
# this is to avoid having to change the builds configs everywhere
# simplistic way to extract hostname from a URL
WEBHOST=$(echo "$TESTBUILDURL" | cut -d/ -f 3)
else
trap failure ERR INT
- # we run in the root context :
+ # we run in the root context :
# (*) create or check for the vm to use
# (*) copy this command in the vm
# (*) invoke it
-
+
if [ -n "$OVERBASE" ] ; then
### Re-use a vm (finish an unfinished build..)
if [ ! -d $(rootdir ${BASE}) ] ; then
# manage LOG - beware it might be a symlink so nuke it first
LOG=$(logfile ${BASE})
rm -f $LOG
- exec > $LOG 2>&1
+ exec > $LOG 2>&1
set -x
echo "XXXXXXXXXX $COMMAND: creating vm $BASE" $(date)
show_env
GIT_REPO=$(echo $BUILD_SCM_URL | cut -d@ -f1)
GIT_TAG=$(echo $BUILD_SCM_URL | cut -s -d@ -f2)
GIT_TAG=${GIT_TAG:-master}
- mkdir -p $tmpdir
+ mkdir -p $tmpdir
( git archive --remote=$GIT_REPO $GIT_TAG | tar -C $tmpdir -xf -) || \
( echo "==================== git archive FAILED, trying git clone instead" ; \
git clone $GIT_REPO $tmpdir && cd $tmpdir && git checkout $GIT_TAG && rm -rf .git)
# Create lxc vm
cd $tmpdir
- ./lbuild-initvm.sh $VERBOSE -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${PREINSTALLED} ${BASE}
+ ./lbuild-initvm.sh $VERBOSE -f ${FCDISTRO} -d ${PLDISTRO} -p ${PERSONALITY} ${PREINSTALLED} ${BASE}
# cleanup
cd -
rm -rf $tmpdir
rm $LOG; ln -s $LOG2 $LOG
LOG=$LOG2
# redirect log again
- exec >> $LOG 2>&1
+ exec >> $LOG 2>&1
sedargs="-e s,@DATE@,${DATE},g -e s,@FCDISTRO@,${FCDISTRO},g -e s,@PLDISTRO@,${PLDISTRO},g -e s,@PERSONALITY@,${PERSONALITY},g"
WEBPATH=$(echo ${WEBPATH} | sed $sedargs)
WEBLOG=${WEBPATH}/${BASE}/log.txt
# compute the log URL - inserted in the mail messages for convenience
WEBBASE_URL=$(echo $WEBBASE | sed -e "s,//,/,g" -e "s,${WEBROOT},${TESTBUILDURL},")
-
- if [ -n "$DO_BUILD" ] ; then
+
+ if [ -n "$DO_BUILD" ] ; then
# invoke this command into the build directory of the vm
cp $COMMANDPATH $(rootdir ${BASE})/build/
# publish to the web so run_log can find them
set +e
trap - ERR INT
- webpublish rm -rf $WEBPATH/$BASE
+ webpublish rm -rf $WEBPATH/$BASE
# guess if we've been doing any debian-related build
if [ ! -f $(rootdir $BASE)/etc/debian_version ] ; then
webpublish mkdir -p $WEBPATH/$BASE/{RPMS,SRPMS}
# (not needed on fedora b/c this is done by the regular build already)
run_in_build_guest $BASE "(cd /build ; dpkg-scanpackages DEBIAN/ | gzip -9c > Packages.gz)"
webpublish mkdir -p $WEBPATH/$BASE/DEBIAN
- webpublish_rsync_files $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb
+ webpublish_rsync_files $WEBPATH/$BASE/DEBIAN/ $(rootdir $BASE)/build/DEBIAN/*.deb
webpublish_rsync_files $WEBPATH/$BASE/ $(rootdir $BASE)/build/Packages.gz
fi
# publish myplc-release if this exists
fi
fi
- if [ -n "$DO_TEST" ] ; then
+ if [ -n "$DO_TEST" ] ; then
run_log
fi
- success
+ success
- echo "==================== MAIN END $(date)"
+ echo "==================== MAIN END $(date)"
fi
}
##########
-main "$@"
+main "$@"
# procprotect: root context module for protecting against weaknesses in /proc
#
### remove procprotect from the nodes on f20 and above, needs more work starting with 3.19
-ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27)"
+ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27 f29)"
procprotect-MODULES := procprotect
procprotect-SPEC := procprotect.spec
ALL += procprotect
#
### starting August 2015, ipfw module won't build against fedora22
# that comes with kernel 4.1.4
-ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f21 f22 f23 f24 f25 f27)"
+ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f21 f22 f23 f24 f25 f27 f29)"
ipfwroot-MODULES := ipfw
ipfwroot-SPEC := planetlab/ipfwroot.spec
ALL += ipfwroot
# fprobe-ulog
#
# xxx temporarily turning this off on f20 and above
-ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27)"
+ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f20 f21 f22 f23 f24 f25 f27 f29)"
fprobe-ulog-MODULES := fprobe-ulog
fprobe-ulog-SPEC := fprobe-ulog.spec
ALL += fprobe-ulog
IN_MYPLC += pyplnet
IN_BOOTCD += pyplnet
-ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f23 f24 f25 f27)"
+ifneq "$(DISTRONAME)" "$(filter $(DISTRONAME),f23 f24 f25 f27 f29)"
#
# OML measurement library
#
#!/usr/bin/python
#
-# This is a replacement for the formerly bash-written function pl_parsePkgs ()
-#
+# This is a replacement for the formerly bash-written function pl_parsePkgs ()
+#
# Usage: $0 [-a arch] default_arch keyword fcdistro pldistro pkgs-file[..s]
# default_arch is $pl_DISTRO_ARCH, but can be overridden
#
default_arch='x86_64'
known_arch = ['i386', 'i686', 'x86_64']
-default_fcdistro = 'f27'
+default_fcdistro = 'f29'
known_fcdistros = [
'centos5', 'centos6',
# oldies but we have references to that in the pkgs files
'f8', 'f10', 'f12', 'f16',
# these ones are still relevant
- 'f14', 'f18', 'f20', 'f21', 'f22', 'f23', 'f24', 'f25', 'f27',
- 'sl6',
+ 'f14', 'f18', 'f20', 'f21', 'f22', 'f23', 'f24', 'f25', 'f27', 'f29',
+ 'sl6',
# debians
'wheezy','jessie',
# ubuntus
default_pldistro='onelab'
known_keywords = [
- 'group', 'groupname', 'groupdesc',
- 'package', 'pip', 'gem',
+ 'group', 'groupname', 'groupdesc',
+ 'package', 'pip', 'gem',
'nodeyumexclude', 'plcyumexclude', 'yumexclude',
'precious', 'junk', 'mirror',
]
re_old = '[a-z]+[+-][a-z]+[0-9]+'
m_old = re.compile('\A{re_old}\Z'.format(**locals()))
-
+
# returns a tuple (included, excluded)
def parse(self, filename):
ok = True
# skip if another distro family
if distro != self.distro: continue
# skip if the qualifier does not fit
- if not self.match (qual, version):
+ if not self.match (qual, version):
if self.options.verbose:
print('{filename}:{lineno}:qualifer {left} does not apply'
.format(**locals()), file=stderr)
format(**locals()))
else:
raise Exception('error in left expression {left}'.format(**locals()))
-
+
except Exception as e:
ok = False
print("{filename}:{lineno}:syntax error: {e}".format(**locals()), file=stderr)
ok = ok and o
# avoid set operations that would not preserve order
results = [ x for x in included if x not in excluded ]
-
+
results = [ x.replace('@arch@', self.arch).\
replace('@fcdistro@', self.fcdistro).\
replace('@pldistro@', self.pldistro) for x in results]
parser.add_option ('-u', '--no-sort', dest='sort_results', default=True, action='store_false',
help='keep results in the same order as in the inputs')
(options, args) = parser.parse_args()
-
+
if len(args) <= 1 :
parser.print_help(file=stderr)
sys.exit(1)
git://git.onelab.eu / build.git / commitdiff