From: Marc Fiuczynski Date: Fri, 2 Nov 2007 21:37:24 +0000 (+0000) Subject: The changeset revamps the vserver-reference package by changing the X-Git-Tag: vserver-4.2-0~23 X-Git-Url: http://git.onelab.eu/?p=sliceimage.git;a=commitdiff_plain;h=90ed554d985b8d2de59024d29024fc78b4482346 The changeset revamps the vserver-reference package by changing the way system vservers -- such as planetflow -- are built. The old model involved copying the appropriate rpms required for, say, planetflow to the systems-packages directory. Upon boot the vserver-init script then construct the reference vserver for planetflow from those rpms. The new model creates the reference vserver at build time, reduces it disk footprint to just the newly installed packages for the system vserver, and places that into the /vservers/.vstub directory (e.g., /vservers/.vstub/planetflow). Upon boot the vserver-init script then simply merges the default vserver reference with this stub to create a /vservers/.vref/planetflow directory. One can define new system vservers by simply creating a .lst file in the reference-vservers/ directory. See the planetflow.lst file as an example. --- diff --git a/build.sh b/build.sh index 5859421..e777822 100755 --- a/build.sh +++ b/build.sh @@ -1,9 +1,16 @@ #!/bin/bash # -# Builds VServer reference image +# Builds all reference image for vservers. To optimize for space it +# will only build a complete base vserver reference image and then +# builds "stub" images that are just contain the additional files +# and/or changes for a given reference image. This is done to shrink +# the RPM itself. These will be pieced back together with the base +# vserver reference image by an init script that is installed on the +# node. # # Mark Huang -# Copyright (C) 2004-2006 The Trustees of Princeton University +# Marc E. Fiuczynski +# Copyright (C) 2004-2007 The Trustees of Princeton University # # $Id: build.sh,v 1.20 2007/09/06 20:41:23 faiyaza Exp $ # @@ -30,56 +37,66 @@ pl_process_fedora_options $@ shiftcount=$? shift $shiftcount -# XXX this should be coming from some configuration file -# Packages to install -packagelist=( -bash -coreutils -iputils -kernel-vserver -bzip2 -crontabs -diffutils -logrotate -openssh-clients -passwd -rsh -rsync -sudo -tcpdump -telnet -traceroute -time -vixie-cron -wget -which -yum -curl -gzip -perl -python -tar -jre -findutils -filesystem -) - -# vserver-reference packages used for reference image -for package in "${packagelist[@]}" ; do - packages="$packages -p $package" -done - # Do not tolerate errors set -e -# Make /vservers -vroot=$PWD/vservers/.vref/default -install -d -m 755 $vroot +# Path's to the vserver references images and stubs +vrefdir=$PWD/vservers/.vref +vstubdir=$PWD/vservers/.vstub + +# XXX: The vserver reference name should be passed in as an argument +# rather than being hardcoded. +vrefname=default + +# Make /vservers and default vserver reference image +vref=${vrefdir}/${vrefname} +install -d -m 755 ${vref} + +# "Parse" out the packages and groups for mkfedora +vrefpackages=$(grep "^package:.*" vserver-reference.lst | awk '{print $2}') +vrefgroups=$(grep "^group:.*" vserver-reference.lst | awk '{print $2}') +options="" +for package in ${vrefpackages} ; do options="$options -p $package"; done +for group in ${vrefgroups} ; do options="$options -g $group"; done # Populate a minimal /dev in the reference image -pl_makedevs $vroot +pl_makedevs ${vref} # Populate image with vserver-reference packages -pl_setup_chroot $vroot $packages +pl_setup_chroot ${vref} ${options} + +for systemvserver in reference-vservers/*.lst ; do + NAME=$(basename $systemvserver .lst) + + # "Parse" out the packages and groups for yum + systempackages=$(grep "^package:.*" $systemvserver | awk '{print $2}') + systemgroups=$(grep "^group:.*" $systemvserver | awk '{print $2}') + + vdir=${vstubdir}/${NAME} + install -d -m 755 ${vdir} + + # Clone the base vserver reference to the system vserver reference + # OPTIMIZATION: Consider using "cp -al" in the future + rsync -a ${vref}/ ${vdir}/ + rm -f ${vdir}/var/lib/rpm/__db* + + # Communicate to the initialization script from which vref this stub was cloned + echo ${vrefname} > ${vdir}.cloned + + # Construct the excludes & includes patterns for rsync + (cd ${vdir} && find *) > ${vdir}.excludes + echo "var/lib/rpm/*" > ${vdir}.includes + + # Install the system vserver specific packages + [ -n "$systempackages" ] && yum -c ${vdir}/etc/yum.conf --installroot=${vdir} -y install $systempackages + [ -n "$systemgroups" ] && yum -c ${vdir}/etc/yum.conf --installroot=${vdir} -y groupinstall $systemgroups + + # Create a copy of the system vserver w/o the vserver reference files + mkdir -p ${vdir}-tmp/ + rsync -a --include-from=${vdir}.includes --exclude-from=${vdir}.excludes ${vdir}/ ${vdir}-tmp/ + rm -rf ${vdir} + rm -f ${vdir}.excludes ${vdir}.includes + mv ${vdir}-tmp ${vdir} +done exit 0 diff --git a/reference-vservers/planetflow.lst b/reference-vservers/planetflow.lst new file mode 100644 index 0000000..6f7b3ad --- /dev/null +++ b/reference-vservers/planetflow.lst @@ -0,0 +1 @@ +package: netflow diff --git a/system-packages.sh b/system-packages.sh deleted file mode 100755 index d225ed5..0000000 --- a/system-packages.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash -# -# Download dependencies that would be necessary to build the -# pl_netflow and pl_conf root slices from the vserver-reference image. -# -# Mark Huang -# Copyright (C) 2004-2006 The Trustees of Princeton University -# -# $Id: system-packages.sh,v 1.5 2007/08/24 06:06:04 mef Exp $ -# - -export PATH=/sbin:/bin:/usr/sbin:/usr/bin - -vroot=$PWD/vservers/.vref/default -rpms=$PWD/vservers/system-packages -install -d -m 755 $rpms - -# curl can't list file:// URLs -list () -{ - url=$1 - if [ -e ${url#file://} ] ; then - /bin/ls ${url#file://} - else - curl --fail --silent --max-time 60 $url - fi -} - -# Space separated list of required packages -planetflow="netflow" - -for vref in planetflow ; do - packages=${!vref} - dependencies=() - - if yum --help | grep -q shell ; then - while read -a words ; do - if [ ${#words[*]} -lt 5 ] ; then - continue - fi - - # netflow i386 3.1-23.planetlab.2006.04.04 bootstrap 61 k - package=${words[0]} - arch=${words[1]} - version=${words[2]} - # Remove the epoch from the version - version=${version##*:} - repository=${words[3]} - - baseurl= - while read line ; do - if [ -z "$line" ] ; then - continue - elif grep -q "^\[$repository\]" <<<$line ; then - baseurl=$repository - elif [ "$baseurl" = "$repository" ] && grep -q "^baseurl=" <<<$line ; then - eval $line - - # We could parse headers/header.info and/or - # repodata/primary.xml.gz to figure out where the - # package actually is within the repository, but - # it would be too much trouble. Just try - # downloading it from one of the common - # subdirectories. - echo "* $vref: $repository $package-$version.$arch.rpm" - for subdirectory in "" Fedora/RPMS Fedora $arch ; do - if curl --fail --silent --max-time 60 $baseurl/$subdirectory/$package-$version.$arch.rpm \ - >$rpms/$package-$version.$arch.rpm ; then - break - fi - rm -f $rpms/$package-$version.$arch.rpm - done - - # Assert that we got it successfully - if [ ! -f $rpms/$package-$version.$arch.rpm ] ; then - echo "Failed to fetch $package-$version.$arch.rpm from $repository ($baseurl/$subdirectory)" >&2 - false - fi - - dependencies[${#dependencies[*]}]=$package-$version.$arch.rpm - break - fi - done <$vroot/etc/yum.conf - done < <((yum -c $vroot/etc/yum.conf --installroot=$vroot shell <$rpms/$(basename $path) - dependencies[${#dependencies[*]}]=$(basename $path) - fi - baseURL= - path= - fi - done < <(yum -d 3 -c $vroot/etc/yum.conf --installroot=$vroot -y --download-only install $packages 2>&1) - fi - - for dependency in "${dependencies[@]}" ; do - echo $dependency - done >$rpms/$vref.lst -done - -# Clean yum cache -yum -c $vroot/etc/yum.conf --installroot=$vroot -y \ - clean all - -# Clean RPM state -rm -f $vroot/var/lib/rpm/__db* - -exit 0 diff --git a/vserver-reference.init b/vserver-reference.init index 56c4de5..a9b7e4f 100755 --- a/vserver-reference.init +++ b/vserver-reference.init @@ -56,9 +56,8 @@ chmod 0000 "$__DEFAULT_VSERVERDIR" setattr --barrier "$__DEFAULT_VSERVERDIR" # Build reference images for system slices -# XXX Use Stork? -for lst in "$__DEFAULT_VSERVERDIR/system-packages/"*.lst ; do - NAME=$(basename $lst .lst) +for systemvserver in "$__DEFAULT_VSERVERDIR/.vstub/"*.cloned ; do + NAME=$(basename $systemvserver .cloned) # Copy base reference image if [ ! -d "$__DEFAULT_VSERVERDIR/.vref/$NAME" ] ; then @@ -68,15 +67,17 @@ for lst in "$__DEFAULT_VSERVERDIR/system-packages/"*.lst ; do mkdir -p "$__DEFAULT_VSERVERDIR/.vtmp" TMP=$(mktemp -d "$__DEFAULT_VSERVERDIR/.vtmp/$NAME.XXXXXX") mkdir -p "$__DEFAULT_VSERVERDIR/.vref" - "$__PKGLIBDIR/vbuild" "$__DEFAULT_VSERVERDIR/.vref/default" "$TMP" + + # build the systemvserver from the one it was originally cloned from + TYPE=$(cat $systemvserver) + REF="$__DEFAULT_VSERVERDIR/.vref/$TYPE" + "$_VCLONE" "$REF"/ "$TMP"/ RETVAL=$? - # Install/update additional packages + # merge the stub with the reference to get the system slice if [ $RETVAL -eq 0 ] ; then - pushd "$__DEFAULT_VSERVERDIR/system-packages" >/dev/null - xargs rpm --root "$TMP" --install < "$NAME.lst" + rsync -a "$__DEFAULT_VSERVERDIR/.vstub/$NAME"/ "$TMP"/ RETVAL=$? - popd >/dev/null fi # Clean RPM state diff --git a/vserver-reference.lst b/vserver-reference.lst new file mode 100644 index 0000000..cd71b7b --- /dev/null +++ b/vserver-reference.lst @@ -0,0 +1,30 @@ +# define packages +package: bash +package: coreutils +package: iputils +package: kernel-vserver +package: bzip2 +package: crontabs +package: diffutils +package: logrotate +package: openssh-clients +package: passwd +package: rsh +package: rsync +package: sudo +package: tcpdump +package: telnet +package: traceroute +package: time +package: vixie-cron +package: wget +package: which +package: yum +package: curl +package: gzip +package: perl +package: python +package: tar +package: jre +package: findutils +package: filesystem diff --git a/vserver-reference.spec b/vserver-reference.spec index 97f73d7..a5fb36a 100644 --- a/vserver-reference.spec +++ b/vserver-reference.spec @@ -1,11 +1,11 @@ %define name vserver -%define version 4.1 -%define release 3%{?pldistro:.%{pldistro}}%{?date:.%{date}} +%define version 4.2 +%define release 0%{?pldistro:.%{pldistro}}%{?date:.%{date}} Vendor: PlanetLab Packager: PlanetLab Central Distribution: PlanetLab 4.0 -URL: http://cvs.planet-lab.org/cvs/vserver-reference +URL: https://svn.planet-lab.org/svn/VserverReference/ Summary: VServer reference image Name: %{name} @@ -35,12 +35,13 @@ as the installation base for new PlanetLab slivers. %package system-packages Summary: System slice packages Group: Applications/System -Requires: vserver-reference = %{version}-%{release} +#Requires: vserver-reference = %{version}-%{release} +Requires: vserver-reference >= 4.2 AutoReqProv: no %description system-packages This package installs the RPMS necessary to create system ("root - resource") slices from the virtual server (VServer) reference image. +resource") slices from the virtual server (VServer) reference image. %prep %setup -q @@ -48,7 +49,6 @@ This package installs the RPMS necessary to create system ("root %build pushd VserverReference ./build.sh -./system-packages.sh popd %install @@ -84,7 +84,7 @@ fi %files system-packages %defattr(-,root,root) -/vservers/system-packages +/vservers/.vstub %define vcached_pid /var/run/vcached.pid