X-Git-Url: http://git.onelab.eu/?p=vserver-reference.git;a=blobdiff_plain;f=build.sh;h=53df7543df9ffd4c842880074b4f0ae4f843b229;hp=4bd4dd2d663a261c1fb34702c9b24372df5bb968;hb=HEAD;hpb=543702003be6be3b540fd631e634df2ecfe56914

diff --git a/build.sh b/build.sh
index 4bd4dd2..53df754 100755
--- a/build.sh
+++ b/build.sh
@@ -1,136 +1,175 @@
 #!/bin/bash
 #
-# Builds VServer reference image. Requires the web and boot servers to
-# be up, which complicates bootstrap. Alternatively, we could require
-# the build server to host a local yum repository. Already, it is
-# required to run the same major version of yum as the nodes.
+# Builds all reference image for slices.  To optimize for space it
+# will only build a complete base reference image and then
+# builds "stub" images that are just contain the additional files
+# and/or changes for a given reference image.  This is done to shrink
+# the RPM itself.  These will be pieced back together with the base
+# image by an init script that is installed on the node.
 #
 # Mark Huang <mlhuang@cs.princeton.edu>
-# Copyright (C) 2004-2005 The Trustees of Princeton University
-#
-# $Id: build.sh,v 1.3 2006/03/10 18:20:28 mlhuang Exp $
+# Marc E. Fiuczynski <mef@cs.princeton.edu>
+# Copyright (C) 2004-2007 The Trustees of Princeton University
 #
 
-# Get the production /etc/yum.conf file. XXX When MAs begin deploying
-# their own boot servers and/or code, this will have to change.
-curl --silent http://boot.planet-lab.org/$(curl --silent --insecure --form node_id=1 --form file=/etc/yum.conf https://boot.planet-lab.org/db/plnodeconf/getsinglefile.php) > yum.conf
-
-# Solve the bootstrap problem by including any just built packages in
-# the yum configuration. This cooperates with the PlanetLab build
-# system.
-if [ -n "$RPM_BUILD_DIR" ] ; then
-    # Remove any [PlanetLab*] sections
-    sed -i -f - yum.conf <<EOF
-# Match lines between [PlanetLab*] and the next [*
-/\[PlanetLab.*\]/I,/^\[/{
-# Delete [PlanetLab*]
-/\[PlanetLab.*\]/Id
-# Done when we see [*
-/^\[/b
-# Otherwise delete
-d
-}
-EOF
-
-    # And replace them with a section for the RPMS that were just built
-    yum-arch $(dirname $RPM_BUILD_DIR)/RPMS
-    createrepo $(dirname $RPM_BUILD_DIR)/RPMS || :
-    # If run under sudo, allow user to delete the headers/ and
-    # repodata/ directories.
-    if [ -n "$SUDO_USER" ] ; then
-	chown -R $SUDO_USER $(dirname $RPM_BUILD_DIR)/RPMS
-    fi
-    cat >> yum.conf <<EOF
-[Bootstrap]
-name=Bootstrap RPMS -- $(dirname $RPM_BUILD_DIR)/RPMS/
-baseurl=file://$(dirname $RPM_BUILD_DIR)/RPMS/
-EOF
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+
+# In both a normal CVS environment and a PlanetLab RPM
+# build environment, all of our dependencies are checked out into
+# directories at the same level as us.
+if [ -d ../build ] ; then
+    PATH=$PATH:../build
+    srcdir=..
+else
+    echo "Error: Could not find $(cd .. && pwd -P)/build/"
+    exit 1
 fi
 
-# Make /vservers
-VROOT=$PWD/vservers/vserver-reference
-install -d -m 755 $VROOT
-
-MAKEDEV ()
-{
-    rm -rf $VROOT/dev
-    mkdir -p $VROOT/dev
-    mknod -m 666 $VROOT/dev/null c 1 3
-    mknod -m 666 $VROOT/dev/zero c 1 5
-    mknod -m 666 $VROOT/dev/full c 1 7
-    mknod -m 644 $VROOT/dev/random c 1 8
-    mknod -m 644 $VROOT/dev/urandom c 1 9
-    mknod -m 666 $VROOT/dev/tty c 5 0
-    mknod -m 666 $VROOT/dev/ptmx c 5 2
-    # For bash command substitution
-    ln -nsf ../proc/self/fd /dev/fd
-    # For df and linuxconf
-    touch $VROOT/dev/hdv1
-    # For TUN/TAP
-    mkdir -p $VROOT/dev/net
-    mknod -m 600 $VROOT/dev/net/tun c 10 200
-}
-
-# Initialize /dev in reference image
-MAKEDEV
-
-# Mount /dev/pts in reference image
-mkdir -p $VROOT/dev/pts
-mount -t devpts none $VROOT/dev/pts
-
-# Mount /proc in reference image
-mkdir -p $VROOT/proc
-mount -t proc none $VROOT/proc
-
-# Clean up before exiting if anything goes wrong
-trap "umount $VROOT/proc ; umount $VROOT/dev/pts ; exit 255" ERR
-
-# Create a dummy /etc/fstab in reference image
-mkdir -p $VROOT/etc
-cat > $VROOT/etc/fstab <<EOF
-# This fake fstab exists only to please df and linuxconf.
-/dev/hdv1	/	ext2	defaults	1 1
-EOF
-cp $VROOT/etc/fstab $VROOT/etc/mtab
-
-# Prevent all locales from being installed in reference image
-mkdir -p $VROOT/etc/rpm
-cat > $VROOT/etc/rpm/macros <<EOF
-%_install_langs en_US:en
-%_excludedocs 1
-%__file_context_path /dev/null
-EOF
-
-# This tells the Boot Manager that it is okay to update
-# /etc/resolv.conf and /etc/hosts whenever the network configuration
-# changes. Users are free to delete this file.
-touch $VROOT/etc/AUTO_UPDATE_NET_FILES
-
-# Trick rpm and yum, who read the real root /etc/rpm/macros file
-# rather than the one installed in the reference image, despite what
-# you might expect the --root and --installroot options to mean. Both
-# programs always read $HOME/.rpmmacros.
-export HOME=$PWD
-ln -sf $VROOT/etc/rpm/macros $PWD/.rpmmacros
-
-# Initialize RPM database in reference image
-mkdir -p $VROOT/var/lib/rpm
-rpm --root $VROOT --initdb
-
-# Go, baby, go
-yum -c yum.conf --installroot=$VROOT -y groupinstall VServer
-
-# Remove stale RPM locks
-rm -f $VROOT/var/lib/rpm/__db*
-
-# Clean up /dev in reference image
-umount $VROOT/dev/pts
-MAKEDEV
-
-# Disable all services in reference image
-/usr/sbin/chroot $VROOT /bin/sh -c "/sbin/chkconfig --list | awk '{ print \$1 }' | xargs -i /sbin/chkconfig {} off"
-
-# Clean up
-umount $VROOT/proc
+export PATH
+
+# build.common comes from the build module
+. build.common
+
+pl_process_fedora_options $@
+shiftcount=$?
+shift $shiftcount
+
+# pldistro expected as $1 
+pldistro=$1 ; shift
+# this comes from spec's slicefamily
+slicefamily=$1; shift
+
+# Do not tolerate errors
+set -e
+
+# Path's to the reference images and stubs
+# This is inherited from util-vservers
+vrefdir=$PWD/vservers/.vref
+vref=${vrefdir}/${slicefamily}
+# stubs are created in a subdir per slicefamily
+vstubdir=$PWD/vservers/.vstub/${slicefamily}
+
+# Create paths
+install -d -m 755 ${vref}
+install -d -m 755 ${vstubdir}
+
+# Some of the PlanetLab RPMs attempt to (re)start themselves in %post,
+# unless the installation is running inside the BootCD environment. We
+# would like to pretend that we are.
+export PL_BOOTCD=1
+
+# Populate image with sliceimage packages
+pl_root_makedevs ${vref}
+# locate the packages and groups file
+pkgsfile=$(pl_locateDistroFile ../build/ ${pldistro} sliceimage.pkgs)
+pl_root_mkfedora ${vref} ${pldistro} $pkgsfile
+pl_root_tune_image ${vref}
+
+systemslice_count=$(ls ../build/config.${pldistro}/sliceimage-*.pkgs 2> /dev/null | wc -l)
+[ $systemslice_count -gt 0 ] && for systemslice in $(ls ../build/config.${pldistro}/sliceimage-*.pkgs) ; do
+    NAME=$(basename $systemslice .pkgs | sed -e s,sliceimage-,,)
+
+    echo " * --------START BUILDING system sliceimage ${NAME}: $(date)"
+
+    # "Parse" out the packages and groups for yum
+    systempackages=$(pl_getPackages ${pl_DISTRO_NAME} $pldistro $systemslice)
+    systemgroups=$(pl_getGroups ${pl_DISTRO_NAME} $pldistro $systemslice)
+    systempips=$(pl_getPips ${pl_DISTRO_NAME} $pldistro $systemslice)
+    systemgems=$(pl_getGems ${pl_DISTRO_NAME} $pldistro $systemslice)
+
+    vdir=${vstubdir}/${NAME}
+    rm -rf ${vdir}/*
+    install -d -m 755 ${vdir}
+
+    # Clone the base sliceimage reference to the system sliceimage reference
+    (cd ${vref} && find . | cpio -m -d -u -p ${vdir})
+    rm -f ${vdir}/var/lib/rpm/__db*
+
+    # Communicate to the initialization script from which vref this stub was cloned
+    echo ${slicefamily} > ${vdir}.cloned
+
+    # Install the system sliceimage specific packages
+    for yum_package in $systempackages; do
+	echo " * yum installing $yum_package"
+	yum -c ${vdir}/etc/mkfedora-yum.conf --installroot=${vdir} -y install $yum_package || \
+	    echo " * WARNING image $systemslice - yum install $yum_package failed"
+    done
+    for group_plus in $systemgroups; do
+	yum_group=$(echo $group_plus | sed -e "s,+++, ,g")
+	echo " * yum groupinstalling $yum_group"
+        yum -c ${vdir}/etc/mkfedora-yum.conf --installroot=${vdir} -y groupinstall "$yum_group" || \
+	    echo " * WARNING image $systemslice - yum groupinstall $yum_group failed"
+    done
+
+    # running pip or gem requires connectivity, and DNS
+    # so we expose the build-vm's /etc/resolv.conf to the current vdir 
+    mkdir -p ${vdir}/etc
+    cp /etc/resolv.conf ${vdir}/etc
+
+    # this requires pip to be available in sliceimage at that point
+    # fedora and debian -> python-pip
+    # on fedora the command is called pip-python (sigh.)
+    for pip in $systempips; do
+	echo " * pip installing $pip"
+	chroot ${vdir} pip -v install $pip || chroot ${vdir} pip-python -v $pip || \
+	    echo " * WARNING image $systemslice - pip install $pip failed"
+    done
+
+    # same for gems; comes with ruby in fedora but ruby-devel is most likely a good thing
+    # we add --no-rdoc --no-ri to keep it low
+    for gem in $systemgems; do
+	echo " * gem installing $gem"
+	chroot ${vdir} gem install --no-rdoc --no-ri $gem || \
+	    echo " * WARNING image $systemslice - gem install $gem failed"
+    done
+
+    # search e.g. sliceimage-planetflow.post in config.<pldistro> or in config.planetlab otherwise
+    postfile=$(pl_locateDistroFile ../build/ ${pldistro} sliceimage-${NAME}.post || : )
+
+    [ -f $postfile ] && /bin/bash $postfile ${vdir} || :
+
+    # Create a copy of the system sliceimage w/o the sliceimage reference files and make it smaller. 
+    # This is a three step process:
+
+    # step 1: clean out yum cache to reduce space requirements
+    yum -c ${vdir}/etc/mkfedora-yum.conf --installroot=${vdir} -y clean all
+
+    # step 2: figure out the new/changed files in ${vdir} vs. ${vref} and compute ${vdir}.changes
+    rsync -anv ${vdir}/ ${vref}/ > ${vdir}.changes
+    linecount=$(wc -l ${vdir}.changes | awk ' { print $1 } ')
+    let headcount=$linecount-3
+    let tailcount=$headcount-1
+    # get rid of the last 3 lines of the rsync output
+    head -${headcount} ${vdir}.changes > ${vdir}.changes.1
+    # get rid of the first line of the rsync output
+    tail -${tailcount} ${vdir}.changes.1 > ${vdir}.changes.2
+    # post process rsync output to get rid of symbolic link embellish output
+    awk ' { print $1 } ' ${vdir}.changes.2 > ${vdir}.changes
+    rm -f ${vdir}.changes.*
+
+    # step 3: create the ${vdir} with just the list given in ${vdir}.changes 
+    install -d -m 755 ${vdir}-tmp/
+    rm -rf ${vdir}-tmp/*
+    (cd ${vdir} && cpio -m -d -u -p ${vdir}-tmp < ${vdir}.changes)
+    rm -rf ${vdir}
+    rm -f  ${vdir}.changes
+    mv ${vdir}-tmp ${vdir}
+
+    # cleanup yum remainings
+    rm -rf ${vdir}/build ${vdir}/longbuildroot
+
+    echo " * --------DONE BUILDING system sliceimage ${NAME}: $(date)"
+done
+
+# search sliceimage.post in config.<pldistro> or in config.planetlab otherwise
+postfile=$(pl_locateDistroFile ../build/ ${pldistro} sliceimage.post)
+
+[ -f $postfile ] && /bin/bash $postfile ${vref} || :
+
+# fix sudoers config
+[ -f ${vref}/etc/sudoers ] && echo -e "\nDefaults\tlogfile=/var/log/sudo\n" >> ${vref}/etc/sudoers
+
+# cleanup yum remainings
+rm -rf ${vref}/build ${vref}/longbuildroot
 
 exit 0