X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=build.common;h=9cd25f2a8096e39d39d7d7875c78b37c557bc1e5;hb=f5ebf5952cf6c09af149ca8ca16403528dde4d66;hp=d9cfd68303c347860b9985da7cb67c5566c9b867;hpb=e135af715c8bc313c353f2eb4d87864bc296be27;p=build.git

diff --git a/build.common b/build.common
index d9cfd683..9cd25f2a 100644
--- a/build.common
+++ b/build.common
@@ -57,12 +57,12 @@ function pl_getReleaseName () {
 		releasename=f$release
 	    fi ;;
 	[Cc]entOS*)
-	    if [ "$release" = "4.5" ] ; then
-	    # centos 4.5 is just centos4 + enhancements
-		releasename=centos4
-	    else
-		releasename=centos$release
-	    fi ;;
+	    old_IFS="$IFS"
+	    IFS="."
+	    set -- $release
+	    IFS="$old_IFS"
+	    releasename=centos$1
+	    ;;
 	*)
 	    releasename="unknown-name-for-${pl_DISTRO}-please-edit-build.common"
 	    echo 1>&2 "build.common: WARNING - releasename not set for distro=$distro" 
@@ -76,11 +76,8 @@ function pl_getReleaseName () {
 # figure out which redhat distro we are using (fedora, centos, redhat)
 pl_DISTRO=$(pl_getDistro)
 
-# select basearch of the host devel environment
-pl_DISTRO_ARCH=$(uname -i)
-
-# let mkfedora select one of its mirrors
-pl_DISTRO_URL=""
+# select basearch of the host devel environment - protected for macos for local tests
+pl_DISTRO_ARCH=$(uname -i 2>/dev/null|| echo unknownarch)
 
 # the release number (plain number)
 pl_DISTRO_RELEASE=$(pl_getRelease)
@@ -124,7 +121,17 @@ function pl_process_fedora_options () {
     return $shiftcount
 }
 
-function pl_makedevs() {
+######################################## handling a root image
+function pl_root_rpm_macros () {
+    cat <<EOF
+%_install_langs C:en_US:en
+%_netsharedpath /proc:/dev/pts:/usr/share/info
+%_excludedocs 1
+%__file_context_path /dev/null
+EOF
+}
+
+function pl_root_makedevs() {
     vroot=$1
     # Clean ${vroot}/dev, but only when ${vroot}!=""
     [ -n $vroot ] && rm -rf $vroot/dev
@@ -162,26 +169,358 @@ function pl_makedevs() {
     done
 }
 
-function pl_mkfedora() {
-    root=$1
-    shift
-    options=$@
+function mkfedora_usage() {
+    echo "Usage: pl_root_mkfedora [OPTION]... basedir pldistro pkgsfile(s)"
+    echo "	-l url		Fedora mirror location."
+    echo "                      Defaults are searched in <pldistro>.mirrors"
+    echo "	-v		Be verbose"
+    echo "	-h		This message"
+    echo " target selection (defaults based on current build vserver)"
+    echo "	-r release	Fedora release number (default: $releasever)"
+    echo "	-a arch		Fedora architecture (default: $basearch)"
+    exit 1
+}
 
-    pl_makedevs $root
+function pl_root_mkfedora () {
 
-    [ -n "$pl_DISTRO_URL" ] && options="$options -l $pl_DISTRO_URL"
-    [ -n "$pl_DISTRO_ARCH" ] && options="$options -a $pl_DISTRO_ARCH"
-    [ -n "$pl_DISTRO_RELEASE" ] && options="$options -r $pl_DISTRO_RELEASE"
-    # echo "mkfedora -v $options $root"
-    eval mkfedora -v $options $root
-}
+    echo "* Entering pl_root_mkfedora" "$@"
 
-function pl_setup_chroot() {
-    root=$1
-    shift
-    options=$@
+    if [ $UID -ne 0 ] ; then
+	echo "Error: You must run this script as root."
+	exit 1
+    fi
+
+# Verbosity
+    verbose=0
+
+# Release and architecture to install : defaults to current vserver's settings or previously parsed fedora_options
+    releasever=$pl_DISTRO_RELEASE
+    basearch=$pl_DISTRO_ARCH
+
+# Get options
+    mirrors=""
+    while getopts "l:r:a:vh" opt ; do
+	case $opt in
+	    l)
+		mirrors="$mirrors $OPTARG"
+		;;
+	    r)
+		releasever=$OPTARG
+		;;
+	    a)
+		basearch=$OPTARG
+		;;
+	    v)
+		verbose=1
+		set -x
+		;;
+	    h|*)
+		mkfedora_usage
+		;;
+	esac
+    done
+
+    shift $(($OPTIND - 1))
+    [[ "$#" -lt 3 ]] && mkfedora_usage
+    vroot=$1 ; shift
+    pldistro=$1 ; shift
+    pkgsfile="$@"
+    vroot=$(cd $vroot && pwd -P)
+    [ -d $vroot ] || mkfedora_usage
+
+
+    # parse pkgsfile and add to local vars
+    fcdistro=${pl_DISTRO_NAME}
+    pkgs_packages=$(pl_parsePkgs package $fcdistro $pldistro $pkgsfile) 
+    pkgs_groups=$(pl_parsePkgs group $fcdistro $pldistro $pkgsfile)
+    # packages to exclude - obsolete, was maybe useful when installing a group
+    pkgs_excludes=$(pl_parsePkgs exclude $fcdistro $pldistro $pkgsfile) 
+    pkgs_junk=$(pl_parsePkgs junk $fcdistro $pldistro $pkgsfile)
+    pkgs_precious=$(pl_parsePkgs precious $fcdistro $pldistro $pkgsfile)
+    # formerly related to mkfedora -k
+    pkgs_kexcludes=$(pl_parsePkgs kexclude $fcdistro $pldistro $pkgsfile)
+    # get mirrors if not specified with -l
+    if [ -z "$mirrors" ] ; then
+	mirrorsfile=$(pl_locateDistroFile ../build/ $pldistro "$pldistro.mirrors")
+	mirrors=$(pl_parsePkgs mirror $fcdistro $pldistro $mirrorsfile)
+    fi
+
+    exclude_kernel=""
+    # add them manually as the output of pl_parsePkgs is line-separated
+    if [ -n "$pkgs_kexcludes" ] ; then
+	exclude_kernel="exclude="
+	for kexclude in $pkgs_kexcludes ; do
+	    exclude_kernel="$exclude_kernel $kexclude"
+	done
+    fi
+
+    function mkfedora_fetch ()
+    {
+	curl --fail --silent --max-time 60 "$1"
+    }
+
+    # set list of attempted locations according to target distro
+    # xxx all this should go into mirroring/ somehow
+    case $pl_DISTRO in
+	Fedora)
+	    if [ $releasever -ge 7 ] ; then
+	        # Plain Fedora comes with a new layout
+		attempts="
+fedora/releases/$releasever/Everything/$basearch/os
+linux/releases/$releasever/Everything/$basearch/os
+$releasever/Everything/$basearch/os
+core/$releasever/Everything/$basearch/os
+linux/core/$releasever/$basearch/os
+"
+	    else
+		# Fedora Core
+		attempts="
+fedora/core/$releasever/$basearch/os 
+linux/core/$releasever/$basearch/os 
+core/$releasever/$basearch/os 
+$releasever/$basearch/os
+"
+	    fi 
+	    public_gpg_key=RPM-GPG-KEY-fedora
+	    ;;
+	CentOS)
+	    # xxx hacky for now
+	    # you can use vbuild-fedora-mirror with -f centos5.1
+	    # which is hard-coded in mirroring/centos as well
+	    # for now I do the mapping here, mmhh
+	    case $releasever in
+		4) actual=4.6 ;;
+		5) actual=5.1 ;;
+		*) actual=$releasever ;;
+	    esac
+	    attempts="
+centos/$actual/os/$basearch
+"
+	    public_gpg_key=RPM-GPG-KEY-CentOS-$releasever
+	    ;;
+	*)
+	    echo "distro $pl_DISTRO not supported in build.common / mkfedora" ; exit 1
+    esac
+
+    echo "$0: candidate mirrors"
+    for mirror in $mirrors ; do
+	echo "* candidate mirror $mirror"
+    done
+    baseurl=""
+    for mirror in $mirrors ; do
+	for attempt in $attempts; do 
+	    attempturl=$mirror/$attempt
+	    if mkfedora_fetch $attempturl/repodata/repomd.xml >/dev/null ; then
+		baseurl=$attempturl
+		break 2
+	    fi
+	done
+    done
+
+    if [ -z "$baseurl" ] ; then
+	echo "Error: $releasever/$basearch/os/repodata/repomd.xml"
+	echo "       could not be found in any of the following locations:"
+	echo
+	for mirror in $mirrors ; do
+	    for attempt in $attempts ; do
+		echo $mirror/$attempt
+	    done
+	done
+	echo
+	mkfedora_usage
+    else
+	echo "* selecting mirror with baseurl=$baseurl"
+    fi
+
+    # Do not tolerate errors
+    set -e
+
+    ## make rpms ignore installing stuff to special fs entries like /proc
+    # Because of https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=52725
+    # you have to use at least one language beside 'C'
+    # Prevent all locales from being installed in reference image
+    mkdir -p $vroot/etc/rpm
+    pl_root_rpm_macros > $vroot/etc/rpm/macros 
+
+    # Trick rpm and yum, who read the real root /etc/rpm/macros file
+    # rather than the one installed in the reference image, despite what
+    # you might expect the --root and --installroot options to mean. Both
+    # programs always read $HOME/.rpmmacros.
+    export HOME=$vroot/tmp
+    mkdir -p $vroot/tmp
+    pl_root_rpm_macros > $vroot/tmp/.rpmmacros
+
+    function mkfedora_cleanup () {
+	umount -l $vroot/proc
+	umount -l $vroot/dev/shm
+	umount -l $vroot/dev/pts
+    }
+
+    # Clean up before exiting if anything goes wrong
+    trap "mkfedora_cleanup" ERR INT
+
+    # Mount in reference image
+    mount -t devpts none $vroot/dev/pts
+    mount -t tmpfs none $vroot/dev/shm
+    mkdir -p $vroot/proc
+    mount -t proc none $vroot/proc
+    
+    # Create a /var/lib dirs for yum & rpm
+    mkdir -p $vroot/var/lib/yum
+    mkdir -p $vroot/var/lib/rpm
+    mkdir -p $vroot/usr/share/info
+
+    # Create a dummy /etc/fstab in reference image
+    mkdir -p $vroot/etc
+    cat >$vroot/etc/fstab <<EOF
+# This fake fstab exists only to please df and linuxconf.
+/dev/hdv1	/	ext2	defaults	1 1
+EOF
+    cp $vroot/etc/fstab $vroot/etc/mtab
+
+    # Necessary for some scripts
+    mkdir -p $vroot/etc/sysconfig
+    echo "NETWORKING=yes" > $vroot/etc/sysconfig/network
+
+    # Initialize RPM database in reference image
+    mkdir -p $vroot/var/lib/rpm
+    rpm --root $vroot --initdb
+    rpm --root $vroot --import $baseurl/$public_gpg_key
+
+    # Initialize yum in reference image
+    mkdir -p $vroot/var/cache/yum $vroot/var/log
+    if [ $releasever -lt 7 ] ; then
+	corename="Core "
+    else
+	corename=""
+    fi
+
+    cat >$vroot/etc/yum.conf <<EOF
+[main]
+cachedir=/var/cache/yum
+debuglevel=2
+logfile=/var/log/yum.log
+pkgpolicy=newest
+distroverpkg=redhat-release
+tolerant=1
+exactarch=1
+retries=20
+obsoletes=1
+gpgcheck=0
+# Prevent yum-2.4 from loading additional repository definitions
+# (e.g., from /etc/yum.repos.d/)
+reposdir=/dev/null
+
+[base]
+name=Fedora ${corename}${releasever} - $basearch - base
+baseurl=$baseurl/
+$exclude_kernel
+EOF
+
+    for optional in updates extras ; do
+	for optionalurl in \
+	    $mirror/linux/core/$optional/$releasever/$basearch \
+	    $mirror/core/$optional/$releasever/$basearch \
+	    $mirror/linux/$optional/$releasever/$basearch \
+	    $mirror/$optional/$releasever/$basearch ; do
+            if mkfedora_fetch $optionalurl/repodata/repomd.xml ; then
+		cat >>$vroot/etc/yum.conf <<EOF
+
+[$(basename $optional)]
+name=Fedora ${corename}${releasever} - $basearch - $(basename $optional)
+baseurl=$optionalurl/
+$exclude_kernel
+EOF
+                break
+	    fi
+	done
+    done
+
+    # If we are being built as part of an automated RPM build, solve the
+    # bootstrap problem by including any just built packages in the yum
+    # configuration. This cooperates with the PlanetLab build system.
+    if [ -n "$RPM_BUILD_DIR" ] ; then
+	RPM_RPMS_DIR=$(cd $(dirname $RPM_BUILD_DIR)/RPMS && pwd -P)
+        # If run under sudo, allow user to delete the headers/ and
+        # repodata/ directories.
+	if [ -n "$SUDO_USER" ] ; then
+	    chown -R $SUDO_USER $RPM_RPMS_DIR
+	fi
+	cat >>$vroot/etc/yum.conf <<EOF
+
+[bootstrap]
+name=Bootstrap - $basearch - $RPM_RPMS_DIR/
+baseurl=file://$RPM_RPMS_DIR/
+EOF
+fi
+
+    exclude_arg=""
+    for exclude in $pkgs_excludes; do
+	exclude_arg="$exclude_arg --exclude $exclude"
+    done
+
+    # glibc must be specified explicitly for the correct arch to be
+    # chosen.
+    echo "* Installing glibc"
+    yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg install glibc
+
+    # Go, baby, go
+    if [ -n "$pkgs_packages" ] ; then
+	echo "* Installing optional packages" $pkgs_packages
+	yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg \
+	    install $pkgs_packages
+	if ! rpm --root $vroot -q $pkgs_packages >/dev/null ; then
+	    echo "* Warning: Missing packages"
+	    rpm --root $vroot -q $pkgs_packages | grep "not installed"
+	fi
+    fi
+
+    if [ -n "$pkgs_groups" ] ; then
+       ## call yum sequentially to get finer-grained info on dependencies
+	for grp in $pkgs_groups ; do
+	    echo "* Installing optional group $grp" 
+	    yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg \
+		groupinstall "$grp"
+	done
+    fi
+
+    # formerly in bootcd/prep.sh : to optimize footprint
+    if [ -n "$pkgs_junk" ] ; then
+	echo "* Removing unnecessary junk"
+	pushd $vroot
+        # Save precious files
+	[ -n "$pkgs_precious" ] && tar --ignore-failed-read -cpf precious.tar $pkgs_precious
+        # Remove unnecessary junk
+	[ -n "$pkgs_junk" ] && rm -rf $pkgs_junk
+        # Restore precious files
+	[ -n "$pkgs_precious" ] && tar -xpf precious.tar && rm -f precious.tar
+	popd
+    fi
+
+    # Clean yum cache
+    echo "* Cleaning up"
+    yum -c $vroot/etc/yum.conf --installroot=$vroot -y clean all
+
+    # Clean RPM state
+    rm -f $vroot/var/lib/rpm/__db*
+
+    # Set time zone to UTC
+    if [ -f $vroot/usr/share/zoneinfo/UTC -a -f $vroot/etc/localtime ] ; then
+	rm -f $vroot/etc/localtime
+	ln -s /usr/share/zoneinfo/UTC $vroot/etc/localtime
+    fi
+
+    # remove trap handler, as we are about to call it directly.
+    trap - ERR INT
 
-    pl_mkfedora $root $options
+    # Clean up
+    mkfedora_cleanup
+
+    return 0
+}
+
+function pl_root_tune_image () {
+    root=$1; shift
 
     # Disable all services in reference image
     chroot $root sh -c "/sbin/chkconfig --list | awk '{ print \$1 }' | xargs -i /sbin/chkconfig {} off"
@@ -194,7 +533,7 @@ function pl_setup_chroot() {
     # This tells the Boot Manager that it is okay to update
     # /etc/resolv.conf and /etc/hosts whenever the network configuration
     # changes. Users are free to delete this file.
-    touch $vroot/etc/AUTO_UPDATE_NET_FILES
+    touch $root/etc/AUTO_UPDATE_NET_FILES
 }
 
 # Move specified directories out of a src tree into a dst tree, and
@@ -213,7 +552,7 @@ function pl_move_dirs() {
 	fi
 	rm -rf ${root}/${datadir}
 	mkdir -p $(dirname ${root}/${datadir})
-	ln -nsf /${store}/${datadir} ${root}/${datadir}
+	ln -nsf ${store}/${datadir} ${root}/${datadir}
     done
 }
 
@@ -240,7 +579,7 @@ function pl_make_image() {
     (cd $root && tar cpf - .) | (cd $tmp && tar xpf -)
 
     # Unmount it
-    umount -l $tmp
+    umount $tmp
     rmdir $tmp
     trap - ERR INT
 }
@@ -264,41 +603,48 @@ function pl_fixdirs() {
 
 ########## .pkgs format
 # comments start with a # - this is needed only if you use a keyword in a comment
+
+function pl_getPkgsAttribute () {
+    keyword=$1; shift
+    file=$1; shift
+    # remove any initial white spaces from the result
+    grep -v '^#' $file | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,," -e "s,^[[:space:]][[:space:]]*,,"
+}
+
 # for a given keyword like 'package' :
 # we support fcdistro-dependant format, for tokens (pkgname) without whitespace
 # you can e.g. use
 # package: pkg1 .. pkgn 
 # package+f8: pkg1 .. pkgn
 # package-f8: pkg1 .. pkgn
-
+# 
+# values can contain @arch@, @fcdistro@ or @pldistro@ that are replaced with the current values
+#
 function pl_parsePkgs () {
 
     keyword=$1;shift
     fcdistro=$1; shift
+    pldistro=$1; shift
     # remaining arguments are paths to the pkgs files
 
     # grab regular descriptions
     all=$(grep -v '^#' "$@" | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,,")
     # grab additions
-    add=$(grep -v '^#' "$@" | grep --regexp="^${keyword}+${fcdistro}:" | sed -e "s,${keyword}\+${fcdistro}:,,")
+    add=$(grep -v '^#' "$@" | grep --regexp="^${keyword}+${fcdistro}:" | sed -e "s,${keyword}+${fcdistro}:,,")
     # grab exclusions
-    sub=$(grep -v '^#' "$@" | grep --regexp="^${keyword}\-${fcdistro}:" | sed -e "s,${keyword}\-${fcdistro}:,,")
+    sub=$(grep -v '^#' "$@" | grep --regexp="^${keyword}-${fcdistro}:" | sed -e "s,${keyword}-${fcdistro}:,,")
 
     for i in $all $add; do
 	for exclude in $sub; do
 	    [ "$i" = "$exclude" ] && continue 2
 	done
-	echo "$i "
+	echo "$i " | sed -e "s,@arch@,$pl_DISTRO_ARCH,g" -e "s,@fcdistro@,$fcdistro,g" -e "s,@pldistro@,$pldistro,g"
     done
     return 0
 }
 
-function pl_getPackages() { fcdistro=$1; shift ; pl_parsePkgs package $fcdistro "$@" ; }
-function pl_getGroups() { fcdistro=$1; shift ; pl_parsePkgs group $fcdistro "$@" ; }
-### # add -p before each package for mkfedora
-### function pl_getPackagesOptions2 () { pl_getPackages2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-p " $i}}' ; }
-### # add -g before each group for mkfedora
-### function pl_getGroupsOptions2 () { pl_getGroups2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-g " $i}}' ; }
+function pl_getPackages() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs package $fcdistro $pldistro "$@" ; }
+function pl_getGroups() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs group $fcdistro $pldistro "$@" ; }
 
 # locates a pldistro-dependant file
 # tries first in build/<pldistro>/, then in build/planetlab/
@@ -327,3 +673,50 @@ function pl_locateDistroFile () {
 	return 0
     fi
 }
+
+function yumgroups_from_pkgs () {
+    builddir=$1; shift
+    pldistro=$1; shift
+    fcdistro=$1; shift
+    pkgsnames=$@
+
+    sedargs="-e s,@FCDISTRO@,$fcdistro,g"
+
+   cat <<__header
+<?xml version="1.0"?>
+<!DOCTYPE comps PUBLIC "-//Red Hat, Inc.//DTD Comps info//EN" "comps.dtd">
+<comps>
+__header
+
+    for pkgsname in $pkgsnames; do
+	pkgsfile=$(pl_locateDistroFile $builddir $pldistro $pkgsname)
+	packages=$(pl_getPackages $fcdistro $pldistro $pkgsfile)
+
+	groupname=$(pl_getPkgsAttribute groupname $pkgsfile | sed $sedargs)
+	groupdesc=$(pl_getPkgsAttribute groupdesc $pkgsfile | sed $sedargs)
+
+	if [ -z "$groupname" -o -z "$groupdesc" ] ; then
+	    echo "Cannot find groupname: and groupdesc: in $pkgsfile -- skipped" 1>&2
+	    continue
+	fi
+	
+	cat << __group_header
+  <group>
+    <id>$(echo $groupname|tr A-Z a-z)</id>
+    <name>$groupname</name>
+    <description>$groupdesc</description>
+    <uservisible>true</uservisible>
+    <packagelist>
+__group_header
+        for package in $packages; do 
+	    echo "<packagereq type=\"mandatory\">$package</packagereq>"
+	done
+	cat << __group_footer
+    </packagelist>
+  </group>
+__group_footer
+    done
+cat <<__footer
+</comps>
+__footer
+}