X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=build.common;h=983ed75a058c343d5586ed5f1d3d27fec43717fe;hb=c290186b9cad1ff9df0b85d05dff3ca0a39cc036;hp=541d5cd464dff6634c635bcebbd33459c91ed5aa;hpb=b00b9e1c3772da040198bd2ffb752bfcd119d528;p=build.git

diff --git a/build.common b/build.common
index 541d5cd4..983ed75a 100644
--- a/build.common
+++ b/build.common
@@ -65,7 +65,7 @@ function pl_getReleaseName () {
 	    fi ;;
 	*)
 	    releasename="unknown-name-for-${pl_DISTRO}-please-edit-build.common"
-	    echo 1>&2 "build.common: WARNING - releasename not set for distro=$pl_DISTRO" 
+	    echo 1>&2 "build.common: WARNING - releasename not set for distro=$distro" 
 	    return 1
 	    ;;
     esac
@@ -76,8 +76,8 @@ function pl_getReleaseName () {
 # figure out which redhat distro we are using (fedora, centos, redhat)
 pl_DISTRO=$(pl_getDistro)
 
-# select basearch of the host devel environment
-pl_DISTRO_ARCH=$(uname -i)
+# select basearch of the host devel environment - protected for macos for local tests
+pl_DISTRO_ARCH=$(uname -i 2>/dev/null|| echo unknownarch)
 
 # let mkfedora select one of its mirrors
 pl_DISTRO_URL=""
@@ -86,7 +86,7 @@ pl_DISTRO_URL=""
 pl_DISTRO_RELEASE=$(pl_getRelease)
 
 # the release name - for vserver build - like fc4, f8 or centos4
-pl_DISTRO_NAME=$(pl_getReleaseName)
+pl_DISTRO_NAME=$(pl_getReleaseName $pl_DISTRO $pl_DISTRO_RELEASE)
 
 # get path to appropriate yumgroups.xml file
 # Thierry: quick & dirty improvement 
@@ -124,7 +124,17 @@ function pl_process_fedora_options () {
     return $shiftcount
 }
 
-function pl_makedevs() {
+######################################## handling a root image
+function pl_root_rpm_macros () {
+    cat <<EOF
+%_install_langs C:en_US:en
+%_netsharedpath /proc:/dev/pts:/usr/share/info
+%_excludedocs 1
+%__file_context_path /dev/null
+EOF
+}
+
+function pl_root_makedevs() {
     vroot=$1
     # Clean ${vroot}/dev, but only when ${vroot}!=""
     [ -n $vroot ] && rm -rf $vroot/dev
@@ -162,26 +172,412 @@ function pl_makedevs() {
     done
 }
 
-function pl_mkfedora() {
-    root=$1
-    shift
-    options=$@
+# Default yum repositories to try
+mirrors=(
+    file:///data/fedora
+    http://localhost/fedora
+    http://build.planet-lab.org/fedora
+    http://coblitz.codeen.org/coblitz.planet-lab.org/pub/fedora
+    ftp://mirror.cs.princeton.edu/pub/mirrors/fedora
+    ftp://mirror.stanford.edu/pub/mirrors/fedora
+    ftp://rpmfind.net/linux/fedora
+    )
+
+function mkfedora_usage() {
+    echo "Usage: pl_root_mkfedora [OPTION]... basedir"
+    echo "	-l url		Fedora mirror location. Default is to try:"
+    for mirror in "${mirrors[@]}" ; do
+	echo "			$mirror"
+    done
+    echo "	-f pkgsfile	use this pkgs file for packages, groups, excludes.."
+    echo "	-k		Exclude kernel* packages from all repositories except bootstrap"
+    echo "	-v		Be verbose"
+    echo "	-h		This message"
+    echo " target selection (defaults based on current build vserver)"
+    echo "	-r release	Fedora release number (default: $releasever)"
+    echo "	-a arch		Fedora architecture (default: $basearch)"
+    echo " legacy (use -f instead)"
+    echo "	-g group1 -g group2 ..."
+    echo "			Yumgroups to install (default: none)"
+    echo "	-p package1 -p package2 ..."
+    echo "			Additional packages to install (default: none)"
+    echo "	-x package1 -x package2 ..."
+    echo "			Packages to exclude (default: none)"
+    exit 1
+}
+
+function pl_root_mkfedora () {
+
+    echo "* Entering pl_root_mkfedora" "$@"
+
+    if [ $UID -ne 0 ] ; then
+	echo "Error: You must run this script as root."
+	exit 1
+    fi
+
+# Verbosity
+    verbose=0
+
+# Release and architecture to install : defaults to current vserver's settings or previously parsed fedora_options
+    releasever=$pl_DISTRO_RELEASE
+    basearch=$pl_DISTRO_ARCH
+    [ -n "$pl_DISTRO_URL" ] && mirrors=($pl_DISTRO_URL)
+
+# Yum groups to install
+    groups=()
+
+# Packages to install
+    packages=()
+
+# Packages to exclude
+    exclude=()
+
+# Exclude kernel* (and related) packages from all repositories except bootstrap
+    exclude_kernel=
+
+# Get options
+    while getopts "l:r:a:g:p:x:f:kvh" opt ; do
+	case $opt in
+	    l)
+		if echo $OPTARG | grep -q -i '^\(file\|http[s]*\)://' ; then
+		    mirrors=($OPTARG)
+		else
+		    # xxx rather use this as a .mirrors file, searched in config.pldistro/
+		    # that would use the pkgs syntax with mirror:
+		    #mirrors=($(pl_parsePkgs mirror <fcdistro> $(pl_locateDistroFile ../build/ <pldistro> $OPTARG.mirrors)))
+		    mirrors=(file://$OPTARG)
+		fi
+		;;
+	    r)
+		releasever=$OPTARG
+		;;
+	    a)
+		basearch=$OPTARG
+		;;
+	    g)
+		groups[${#groups[*]}]="$OPTARG"
+		;;
+	    p)
+		packages[${#packages[*]}]="$OPTARG"
+		;;
+	    x)
+		exclude[${#exclude[*]}]="$OPTARG"
+		;;
+	    f)
+		pkgsfile=$OPTARG
+		;;
+	    k)
+		exclude_kernel="exclude=kernel* ulogd iptables"
+		;;
+	    v)
+		verbose=1
+		set -x
+		;;
+	    h|*)
+		mkfedora_usage
+		;;
+	esac
+    done
+
+    shift $(($OPTIND - 1))
+    [[ -n "$@" ]] || mkfedora_usage
+    vroot=$1 ; shift
+    vroot=$(cd $vroot && pwd -P)
+    [[ -z "$@" ]] || mkfedora_usage
+    [ -d $vroot ] || mkfedora_usage
+
+    function mkfedora_fetch ()
+    {
+	curl --fail --silent --max-time 60 "$1"
+    }
+
+    # set list of attempted locations according to releasever
+    if [ $releasever -ge 7 ] ; then
+	attempts="
+linux/releases/$releasever/Everything/$basearch/os
+$releasever/Everything/$basearch/os
+core/$releasever/Everything/$basearch/os
+linux/core/$releasever/$basearch/os
+"
+    else
+	attempts="
+	linux/core/$releasever/$basearch/os 
+	core/$releasever/$basearch/os 
+	$releasever/$basearch/os
+	"
+    fi
+
+    echo "$0: candidate mirrors"
+    for mirror in "${mirrors[@]}" ; do
+	echo "* candidate mirror $mirror"
+    done
+    baseurl=""
+    for mirror in "${mirrors[@]}" ; do
+	for attempt in $attempts; do 
+	    attempturl=$mirror/$attempt
+	    if mkfedora_fetch $attempturl/repodata/repomd.xml >/dev/null ; then
+		baseurl=$attempturl
+		break 2
+	    fi
+	done
+    done
+
+    if [ -z "$baseurl" ] ; then
+	echo "Error: $releasever/$basearch/os/repodata/repomd.xml"
+	echo "       could not be found in any of the following locations:"
+	echo
+	for mirror in ${mirrors[@]} ; do
+	    for attempt in $attempts ; do
+		echo $mirror/$attempt
+	    done
+	done
+	echo
+	mkfedora_usage
+    else
+	echo "* selecting mirror with baseurl=$baseurl"
+    fi
+
+    # Do not tolerate errors
+    set -e
+
+    ## make rpms ignore installing stuff to special fs entries like /proc
+    # Because of https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=52725
+    # you have to use at least one language beside 'C'
+    # Prevent all locales from being installed in reference image
+    mkdir -p $vroot/etc/rpm
+    pl_root_rpm_macros > $vroot/etc/rpm/macros 
+
+    # Trick rpm and yum, who read the real root /etc/rpm/macros file
+    # rather than the one installed in the reference image, despite what
+    # you might expect the --root and --installroot options to mean. Both
+    # programs always read $HOME/.rpmmacros.
+    export HOME=$vroot/tmp
+    mkdir -p $vroot/tmp
+    pl_root_rpm_macros > $vroot/tmp/.rpmmacros
+
+    function mkfedora_cleanup () {
+	umount -l $vroot/proc
+	umount -l $vroot/dev/shm
+	umount -l $vroot/dev/pts
+    }
+
+    # Clean up before exiting if anything goes wrong
+    trap "mkfedora_cleanup" ERR INT
+
+    # Mount in reference image
+    mount -t devpts none $vroot/dev/pts
+    mount -t tmpfs none $vroot/dev/shm
+    mkdir -p $vroot/proc
+    mount -t proc none $vroot/proc
+    
+    # Create a /var/lib dirs for yum & rpm
+    mkdir -p $vroot/var/lib/yum
+    mkdir -p $vroot/var/lib/rpm
+    mkdir -p $vroot/usr/share/info
+
+    # Create a dummy /etc/fstab in reference image
+    mkdir -p $vroot/etc
+    cat >$vroot/etc/fstab <<EOF
+# This fake fstab exists only to please df and linuxconf.
+/dev/hdv1	/	ext2	defaults	1 1
+EOF
+    cp $vroot/etc/fstab $vroot/etc/mtab
+
+    # Necessary for some scripts
+    mkdir -p $vroot/etc/sysconfig
+    echo "NETWORKING=yes" > $vroot/etc/sysconfig/network
+
+    # Initialize RPM database in reference image
+    mkdir -p $vroot/var/lib/rpm
+    rpm --root $vroot --initdb
+    rpm --root $vroot --import $baseurl/RPM-GPG-KEY-fedora
+
+    # Initialize yum in reference image
+    mkdir -p $vroot/var/cache/yum $vroot/var/log
+    if [ $releasever -lt 7 ] ; then
+	corename="Core "
+    else
+	corename=""
+    fi
+
+    cat >$vroot/etc/yum.conf <<EOF
+[main]
+cachedir=/var/cache/yum
+debuglevel=2
+logfile=/var/log/yum.log
+pkgpolicy=newest
+distroverpkg=redhat-release
+tolerant=1
+exactarch=1
+retries=20
+obsoletes=1
+gpgcheck=0
+# Prevent yum-2.4 from loading additional repository definitions
+# (e.g., from /etc/yum.repos.d/)
+reposdir=/dev/null
+
+[base]
+name=Fedora ${corename}${releasever} - $basearch - base
+baseurl=$baseurl/
+$exclude_kernel
+EOF
+
+    for optional in updates extras ; do
+	for optionalurl in \
+	    $mirror/linux/core/$optional/$releasever/$basearch \
+	    $mirror/core/$optional/$releasever/$basearch \
+	    $mirror/linux/$optional/$releasever/$basearch \
+	    $mirror/$optional/$releasever/$basearch ; do
+            if mkfedora_fetch $optionalurl/repodata/repomd.xml ; then
+		cat >>$vroot/etc/yum.conf <<EOF
+
+[$(basename $optional)]
+name=Fedora ${corename}${releasever} - $basearch - $(basename $optional)
+baseurl=$optionalurl/
+$exclude_kernel
+EOF
+                break
+	    fi
+	done
+    done
+
+    # If we are being built as part of an automated RPM build, solve the
+    # bootstrap problem by including any just built packages in the yum
+    # configuration. This cooperates with the PlanetLab build system.
+### make takes care of that
+    if [ -n "$RPM_BUILD_DIR" ] ; then
+	RPM_RPMS_DIR=$(cd $(dirname $RPM_BUILD_DIR)/RPMS && pwd -P)
+###        # yum-2.0.x
+###	if [ -x /usr/bin/yum-arch ] ; then
+###	    yum-arch -q $RPM_RPMS_DIR
+###	fi
+###        # yum-2.4.x
+###	if [ -x /usr/bin/createrepo ] ; then
+###	    if [ -f $RPM_RPMS_DIR/yumgroups.xml ] ; then
+###		groupfile="-g yumgroups.xml"
+###	    fi
+###	    createrepo --quiet $groupfile $RPM_RPMS_DIR
+###	fi
+        # If run under sudo, allow user to delete the headers/ and
+        # repodata/ directories.
+	if [ -n "$SUDO_USER" ] ; then
+	    chown -R $SUDO_USER $RPM_RPMS_DIR
+	fi
+	cat >>$vroot/etc/yum.conf <<EOF
+
+[bootstrap]
+name=Bootstrap - $basearch - $RPM_RPMS_DIR/
+baseurl=file://$RPM_RPMS_DIR/
+EOF
+fi
+
+    # pkgs file
+    if [ -n "$pkgsfile" ] ; then
+        # parse pkgsfile and add to local vars
+	fcdistro=$(pl_getReleaseName "Fedora" $releasever)
+	for i in $(pl_parsePkgs package $fcdistro $pkgsfile)  ; do
+	    packages[${#packages[*]}]="$i"
+	done
+	for i in $(pl_parsePkgs group $fcdistro $pkgsfile) ; do
+	    groups[${#groups[*]}]="$i"
+	done
+	for i in $(pl_parsePkgs exclude $fcdistro $pkgsfile) ; do
+	    exclude[${#exclude[*]}]="$i"
+	done
+	junk=$(pl_parsePkgs junk $fcdistro $pkgsfile)
+	precious=$(pl_parsePkgs precious $fcdistro $pkgsfile)
+    fi
 
-    pl_makedevs $root
+    excludes=
+    for package in "${exclude[@]}" ; do
+	excludes="$excludes --exclude=$package"
+    done
+
+    # glibc must be specified explicitly for the correct arch to be
+    # chosen.
+    echo "* Installing glibc"
+    yum -c $vroot/etc/yum.conf --installroot=$vroot -y $excludes install glibc
+
+    # Go, baby, go
+    if [ ${#packages[*]} -gt 0 ] ; then
+	echo "* Installing optional packages" "${packages[@]}" 
+	yum -c $vroot/etc/yum.conf --installroot=$vroot -y $excludes \
+	    install "${packages[@]}"
+	if ! rpm --root $vroot -q "${packages[@]}" >/dev/null ; then
+	    echo "* Warning: Missing packages"
+	    rpm --root $vroot -q "${packages[@]}" | grep "not installed"
+	fi
+    fi
+
+    if [ ${#groups[*]} -gt 0 ] ; then
+       ## call yum sequentially to get finer-grained info on dependencies
+	for grp in "${groups[@]}" ; do
+	    echo "* Installing optional group $grp" 
+	    yum -c $vroot/etc/yum.conf --installroot=$vroot -y $excludes \
+		groupinstall "$grp"
+	done
+    fi
+
+    # formerly in bootcd/prep.sh : to optimize footprint
+    echo "* Removing unnecessary junk"
+
+    pushd $vroot
+
+    # Save precious files
+    [ -n "$precious" ] && tar --ignore-failed-read -cpf precious.tar $precious
+
+    # Remove unnecessary junk
+    [ -n "$junk" ] && rm -rf $junk
+
+    # Restore precious files
+    [ -n "$precious" ] && tar -xpf precious.tar && rm -f precious.tar
+
+    popd
+
+###    # FC2 dev %preinstall checks /proc/mounts to make sure that /dev is
+###    # not currently mounted as devfs. If it thinks it is, it will refuse
+###    # to install the package. On a modern system running udev that mounts
+###    # /dev as tmpfs, this check fails. Since we are installing into a
+###    # chroot, whether /dev is mounted on the host system or not doesn't
+###    # matter. If dev was explicitly mentioned in the packages list, force
+###    # its installation.
+###    if [ "$releasever" = "2" ] ; then
+###	for package in "${packages[@]}" ; do
+###	    if [ "$package" = "dev" ] && ! rpm --root $vroot -q dev >/dev/null 2>&1 ; then
+###		rpm --root $vroot -Uvh --noscripts $baseurl/Fedora/RPMS/dev-3.3.13-1.i386.rpm
+###		break
+###	    fi
+###	done
+###    fi
+
+    # Clean yum cache
+    echo "* Cleaning up"
+    yum -c $vroot/etc/yum.conf --installroot=$vroot -y clean all
+
+    # Clean RPM state
+    rm -f $vroot/var/lib/rpm/__db*
+
+    # Set time zone to UTC
+    if [ -f $vroot/usr/share/zoneinfo/UTC -a -f $vroot/etc/localtime ] ; then
+	rm -f $vroot/etc/localtime
+	ln -s /usr/share/zoneinfo/UTC $vroot/etc/localtime
+    fi
+
+    # remove trap handler, as we are about to call it directly.
+    trap - ERR INT
 
-    [ -n "$pl_DISTRO_URL" ] && options="$options -l $pl_DISTRO_URL"
-    [ -n "$pl_DISTRO_ARCH" ] && options="$options -a $pl_DISTRO_ARCH"
-    [ -n "$pl_DISTRO_RELEASE" ] && options="$options -r $pl_DISTRO_RELEASE"
-    # echo "mkfedora -v $options $root"
-    eval mkfedora -v $options $root
+    # Clean up
+    mkfedora_cleanup
+
+    return 0
 }
 
-function pl_setup_chroot() {
-    root=$1
-    shift
-    options=$@
+function pl_root_setup_chroot() {
+    root=$1; shift
+    options="$@"
 
-    pl_mkfedora $root $options
+    pl_root_makedevs $root
+    pl_root_mkfedora $options $root 
 
     # Disable all services in reference image
     chroot $root sh -c "/sbin/chkconfig --list | awk '{ print \$1 }' | xargs -i /sbin/chkconfig {} off"
@@ -213,7 +609,7 @@ function pl_move_dirs() {
 	fi
 	rm -rf ${root}/${datadir}
 	mkdir -p $(dirname ${root}/${datadir})
-	ln -nsf /${store}/${datadir} ${root}/${datadir}
+	ln -nsf ${store}/${datadir} ${root}/${datadir}
     done
 }
 
@@ -240,7 +636,7 @@ function pl_make_image() {
     (cd $root && tar cpf - .) | (cd $tmp && tar xpf -)
 
     # Unmount it
-    umount -l $tmp
+    umount $tmp
     rmdir $tmp
     trap - ERR INT
 }
@@ -264,6 +660,14 @@ function pl_fixdirs() {
 
 ########## .pkgs format
 # comments start with a # - this is needed only if you use a keyword in a comment
+
+function pl_getPkgsAttribute () {
+    keyword=$1; shift
+    file=$1; shift
+    grep -v '^#' $file | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,,"
+}
+
+
 # for a given keyword like 'package' :
 # we support fcdistro-dependant format, for tokens (pkgname) without whitespace
 # you can e.g. use
@@ -280,9 +684,9 @@ function pl_parsePkgs () {
     # grab regular descriptions
     all=$(grep -v '^#' "$@" | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,,")
     # grab additions
-    add=$(grep -v '^#' "$@" | grep --regexp="^${keyword}+${fcdistro}:" | sed -e "s,${keyword}\+${fcdistro}:,,")
+    add=$(grep -v '^#' "$@" | grep --regexp="^${keyword}+${fcdistro}:" | sed -e "s,${keyword}+${fcdistro}:,,")
     # grab exclusions
-    sub=$(grep -v '^#' "$@" | grep --regexp="^${keyword}\-${fcdistro}:" | sed -e "s,${keyword}\-${fcdistro}:,,")
+    sub=$(grep -v '^#' "$@" | grep --regexp="^${keyword}-${fcdistro}:" | sed -e "s,${keyword}-${fcdistro}:,,")
 
     for i in $all $add; do
 	for exclude in $sub; do
@@ -295,10 +699,6 @@ function pl_parsePkgs () {
 
 function pl_getPackages() { fcdistro=$1; shift ; pl_parsePkgs package $fcdistro "$@" ; }
 function pl_getGroups() { fcdistro=$1; shift ; pl_parsePkgs group $fcdistro "$@" ; }
-### # add -p before each package for mkfedora
-### function pl_getPackagesOptions2 () { pl_getPackages2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-p " $i}}' ; }
-### # add -g before each group for mkfedora
-### function pl_getGroupsOptions2 () { pl_getGroups2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-g " $i}}' ; }
 
 # locates a pldistro-dependant file
 # tries first in build/<pldistro>/, then in build/planetlab/
@@ -311,13 +711,14 @@ function pl_locateDistroFile () {
     # locate it
     paths="$builddir/config.$pldistro/$pkgsfile $builddir/config.planetlab/$pkgsfile"
     for path in $paths; do
-	if [ -f $path/$pkgsfile ] ; then
+	if [ -f $path ] ; then
 	    pkgspath=$path
 	    break
 	fi
     done
     if [ -z "$pkgspath" ] ; then
-	echo 1>&2 "pl_locateDistroFile: cannot locate $pkgsfile in $builddir"
+	echo 1>&2 "pl_locateDistroFile - in $(pwd) : cannot locate $pkgsfile in $builddir"
+	echo 1>&2 "candidates were $paths"
 	echo "not-found-by-pl_locateDistroFile"
 	return 1
     else
@@ -326,3 +727,54 @@ function pl_locateDistroFile () {
 	return 0
     fi
 }
+
+# experimental
+function yumgroups_from_pkgs () {
+    builddir=$1; shift
+    pldistro=$1; shift
+    fcdistro=$1; shift
+    pkgsnames=$@
+
+sedargs="-e s,@FCDISTRO@,$fcdistro,g"
+
+   cat <<__header
+<?xml version="1.0"?>
+<!DOCTYPE comps PUBLIC "-//Red Hat, Inc.//DTD Comps info//EN" "comps.dtd">
+<comps>
+__header
+
+    for pkgsname in $pkgsnames; do
+	pkgsfile=$(pl_locateDistroFile $builddir $pldistro $pkgsname)
+	packages=$(pl_getPackages $fcdistro $pkgsfile)
+
+	groupname=$(pl_getPkgsAttribute groupname $pkgsfile | sed $sedargs)
+	groupdesc=$(pl_getPkgsAttribute groupdesc $pkgsfile | sed $sedargs)
+
+	if [ -z "$groupname" -o -z "$groupdesc" ] ; then
+	    echo "Cannot find groupname: and groupdesc: in $pkgsfile -- skipped" 1>&2
+	    continue
+	fi
+	
+	cat << __group_header
+  <group>
+    <id>$(echo $groupname|tr A-Z a-z)</id>
+    <name>$groupname</name>
+    <default>true</default>
+    <description>$groupdesc</description>
+    <uservisible>false</uservisible>
+    <packagelist>
+__group_header
+        for package in $packages; do 
+	    echo "<packagereq type=\"mandatory\">$package</packagereq>"
+	done
+	cat << __group_footer
+    </packagelist>
+  </group>
+__group_footer
+    done
+cat <<__footer
+</comps>
+__footer
+}
+
+