X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=build.common;h=8d2e66490396cb824a538c370e36abc2a308db53;hb=010274a3332e3fe543c907e0b379cc7d338f2da9;hp=d566e09daf6f7f54f1c9ee99665899869ec3742b;hpb=4ef448e9f6f7593e6e9e8987a3658d0ee5483f99;p=build.git

diff --git a/build.common b/build.common
index d566e09d..8d2e6649 100644
--- a/build.common
+++ b/build.common
@@ -9,53 +9,38 @@
 # $Id$
 #
 
-function pl_getDefaultDistro() {
-    # FC4 is currently the default release
-    echo "Fedora"
-    return 0
-}
-
-function pl_getDefaultRelease() {
-    # FC4 is currently the default release
-    echo "4"
-    return 0
-}
+# support for fedora and centos only for now
 
 function pl_getDistro() {
-    defaultDistro=$(pl_getDefaultDistro)
     if [ -f "/etc/redhat-release" ] ; then
 	distro=$(awk ' { print $1 } ' /etc/redhat-release)
     else
-	distro=$defaultDistro
+	echo "build.common.pl_getDistro-unknown"
+	exit 1
     fi
     echo "$distro"
     return 0
 }
 
 function pl_getRelease() {
-    defaultRelease=$(pl_getDefaultRelease)
     if [ -f "/etc/redhat-release" ] ; then
 	release=$(awk ' { if ($1=="Fedora" && $2=="Core") print $4 ; if (($1=="Fedora" && $2!="Core")||$1=="CentOS") print $3 } ' /etc/redhat-release)
-	[ $release -lt $defaultRelease ] && release=$defaultRelease
     else
-	release=$defaultRelease
+	echo "build.common.pl_getRelease-unknown"
+	exit 1
     fi
     echo "$release"
     return 0
 }
 
-# vserver expects something like fc4 or f7
-# for fedora and centos only for now
+# vserver expects something like f8 or centos5
 function pl_getReleaseName () {
     distro=$1; shift
     release=$1; shift
     case $distro in
 	[Ff]edora*)
-	    if [ "$release" -le 6 ] ; then
-		releasename=fc$release
-	    else
-		releasename=f$release
-	    fi ;;
+	    releasename=f$release
+	    ;;
 	[Cc]entOS*)
 	    old_IFS="$IFS"
 	    IFS="."
@@ -73,6 +58,24 @@ function pl_getReleaseName () {
     return 0
 }
 
+# on fedora 8 or above, we use libnl from the stock repos
+# on centos5 we build it locally
+function pl_getKexcludes () {
+    distroname=$1; shift
+    case $distroname in 
+	f?|f1?)
+	    echo 'kernel* util-vserver* iptables iproute drupal'
+	    ;;
+	centos5)
+	    echo 'kernel* util-vserver* iptables iproute inotify-tools* libnl* drupal'
+	    ;;
+	*)
+	    echo "pl_DISTRO_NAME=$pl_DISTRO_NAME not supported in getexcludes.sh"
+	    exit 1
+	    ;;
+    esac
+}
+
 # figure out which redhat distro we are using (fedora, centos, redhat)
 pl_DISTRO=$(pl_getDistro)
 
@@ -82,12 +85,15 @@ pl_DISTRO_ARCH=$(uname -i 2>/dev/null|| echo unknownarch)
 # the release number (plain number)
 pl_DISTRO_RELEASE=$(pl_getRelease)
 
-# the release name - for vserver build - like fc4, f8 or centos4
+# the release name - for vserver build ...
 pl_DISTRO_NAME=$(pl_getReleaseName $pl_DISTRO $pl_DISTRO_RELEASE)
 
+# the packages to exclude
+pl_KEXCLUDES=$(pl_getKexcludes $pl_DISTRO_NAME)
+
 # get path to appropriate yumgroups.xml file
 # Thierry: quick & dirty improvement 
-# this file is updated by the toplevel build, from groups/<pldistro>.xml
+# this file is updated by the toplevel build, from the .pkgs files
 pl_DISTRO_YUMGROUPS="../../../RPMS/yumgroups.xml"
 
 function pl_process_fedora_options () {
@@ -198,25 +204,10 @@ function pl_root_mkfedora () {
     basearch=$pl_DISTRO_ARCH
 
 # Get options
-    mirrors=""
-    while getopts "l:r:a:vh" opt ; do
+    while getopts "vh" opt ; do
 	case $opt in
-	    l)
-		mirrors="$mirrors $OPTARG"
-		;;
-	    r)
-		releasever=$OPTARG
-		;;
-	    a)
-		basearch=$OPTARG
-		;;
-	    v)
-		verbose=1
-		set -x
-		;;
-	    h|*)
-		mkfedora_usage
-		;;
+	    v) verbose=1; set -x ;;
+	    h|*) mkfedora_usage ;;
 	esac
     done
 
@@ -233,104 +224,42 @@ function pl_root_mkfedora () {
     fcdistro=${pl_DISTRO_NAME}
     pkgs_packages=$(pl_parsePkgs package $fcdistro $pldistro $pkgsfile) 
     pkgs_groups=$(pl_parsePkgs group $fcdistro $pldistro $pkgsfile)
-    # packages to exclude - obsolete, was maybe useful when installing a group
+    # packages to exclude 
     pkgs_excludes=$(pl_parsePkgs exclude $fcdistro $pldistro $pkgsfile) 
+    # what can get trashed to save space
     pkgs_junk=$(pl_parsePkgs junk $fcdistro $pldistro $pkgsfile)
+    # but not this
     pkgs_precious=$(pl_parsePkgs precious $fcdistro $pldistro $pkgsfile)
-    # formerly related to mkfedora -k
-    pkgs_kexcludes=$(pl_parsePkgs kexclude $fcdistro $pldistro $pkgsfile)
+    # formerly related to mkfedora -k : packages to take from our own build 
+    # and thus need be excluded frem the stock repos
+    pkgs_kexcludes=$(pl_parsePkgs kexclude $fcdistro $pldistro $pkgsfile | sed -e s,@KEXCLUDE@,"$pl_KEXCLUDES",)
     # get mirrors if not specified with -l
     if [ -z "$mirrors" ] ; then
 	mirrorsfile=$(pl_locateDistroFile ../build/ $pldistro "$pldistro.mirrors")
 	mirrors=$(pl_parsePkgs mirror $fcdistro $pldistro $mirrorsfile)
     fi
 
-    exclude_kernel=""
+    kexclude_line=""
     # add them manually as the output of pl_parsePkgs is line-separated
     if [ -n "$pkgs_kexcludes" ] ; then
-	exclude_kernel="exclude="
+	kexclude_line="exclude="
 	for kexclude in $pkgs_kexcludes ; do
-	    exclude_kernel="$exclude_kernel $kexclude"
+	    kexclude_line="$kexclude_line $kexclude"
 	done
     fi
 
-    function mkfedora_fetch ()
-    {
-	curl --fail --silent --max-time 60 "$1"
-    }
-
-    # set list of attempted locations according to target distro
-    # xxx all this should go into mirroring/ somehow
-    case $pl_DISTRO in
-	Fedora)
-	    if [ $releasever -ge 7 ] ; then
-	        # Plain Fedora comes with a new layout
-		attempts="
-fedora/releases/$releasever/Everything/$basearch/os
-linux/releases/$releasever/Everything/$basearch/os
-$releasever/Everything/$basearch/os
-core/$releasever/Everything/$basearch/os
-linux/core/$releasever/$basearch/os
-"
-	    else
-		# Fedora Core
-		attempts="
-fedora/core/$releasever/$basearch/os 
-linux/core/$releasever/$basearch/os 
-core/$releasever/$basearch/os 
-$releasever/$basearch/os
-"
-	    fi 
-	    public_gpg_key=RPM-GPG-KEY-fedora
-	    ;;
-	CentOS)
-	    # xxx hacky for now
-	    # you can use vbuild-fedora-mirror with -f centos5.1
-	    # which is hard-coded in mirroring/centos as well
-	    # for now I do the mapping here, mmhh
-	    case $releasever in
-		4) actual=4.6 ;;
-		5) actual=5.1 ;;
-		*) actual=$releasever ;;
-	    esac
-	    attempts="
-centos/$actual/os/$basearch
-"
-	    public_gpg_key=RPM-GPG-KEY-CentOS-$releasever
-	    ;;
-	*)
-	    echo "distro $pl_DISTRO not supported in build.common / mkfedora" ; exit 1
-    esac
-
     echo "$0: candidate mirrors"
     for mirror in $mirrors ; do
 	echo "* candidate mirror $mirror"
     done
-    baseurl=""
-    for mirror in $mirrors ; do
-	for attempt in $attempts; do 
-	    attempturl=$mirror/$attempt
-	    if mkfedora_fetch $attempturl/repodata/repomd.xml >/dev/null ; then
-		baseurl=$attempturl
-		break 2
-	    fi
-	done
-    done
 
-    if [ -z "$baseurl" ] ; then
-	echo "Error: $releasever/$basearch/os/repodata/repomd.xml"
-	echo "       could not be found in any of the following locations:"
-	echo
-	for mirror in $mirrors ; do
-	    for attempt in $attempts ; do
-		echo $mirror/$attempt
-	    done
-	done
-	echo
-	mkfedora_usage
-    else
-	echo "* selecting mirror with baseurl=$baseurl"
+    # the repo part of the final yum.conf
+    yum_conf_repos=$vroot/xxxmkfedora-repos.confxxx
+    if ! yumconf_mirrors $yum_conf_repos ../build/ $fcdistro "$kexclude_line" $mirrors ; then
+	echo xxx -- error ; return 1
     fi
+    
+    public_gpg_key=$(yumconf_gpgkey $yum_conf_repos)
 
     # Do not tolerate errors
     set -e
@@ -385,15 +314,10 @@ EOF
     # Initialize RPM database in reference image
     mkdir -p $vroot/var/lib/rpm
     rpm --root $vroot --initdb
-    rpm --root $vroot --import $baseurl/$public_gpg_key
+    rpm --root $vroot --import $public_gpg_key
 
     # Initialize yum in reference image
     mkdir -p $vroot/var/cache/yum $vroot/var/log
-    if [ $releasever -lt 7 ] ; then
-	corename="Core "
-    else
-	corename=""
-    fi
 
 # yum.conf is for building only - store in different location than /etc/yum.conf
     yum_conf=$vroot/etc/mkfedora-yum.conf
@@ -412,31 +336,9 @@ gpgcheck=0
 # Prevent yum-2.4 from loading additional repository definitions
 # (e.g., from /etc/yum.repos.d/)
 reposdir=/dev/null
-
-[base]
-name=Fedora ${corename}${releasever} - $basearch - base
-baseurl=$baseurl/
-$exclude_kernel
-EOF
-
-    for optional in updates extras ; do
-	for optionalurl in \
-	    $mirror/linux/core/$optional/$releasever/$basearch \
-	    $mirror/core/$optional/$releasever/$basearch \
-	    $mirror/linux/$optional/$releasever/$basearch \
-	    $mirror/$optional/$releasever/$basearch ; do
-            if mkfedora_fetch $optionalurl/repodata/repomd.xml ; then
-		cat >> $yum_conf <<EOF
-
-[$(basename $optional)]
-name=Fedora ${corename}${releasever} - $basearch - $(basename $optional)
-baseurl=$optionalurl/
-$exclude_kernel
 EOF
-                break
-	    fi
-	done
-    done
+    
+    cat $yum_conf_repos >> $yum_conf
 
     # If we are being built as part of an automated RPM build, solve the
     # bootstrap problem by including any just built packages in the yum
@@ -450,8 +352,8 @@ EOF
 	fi
 	cat >> $yum_conf <<EOF
 
-[bootstrap]
-name=Bootstrap - $basearch - $RPM_RPMS_DIR/
+[building]
+name=Building - $basearch - $RPM_RPMS_DIR/
 baseurl=file://$RPM_RPMS_DIR/
 EOF
 fi
@@ -633,9 +535,16 @@ function pl_getPkgsAttribute () {
 # 
 # values can contain @arch@, @fcdistro@ or @pldistro@ that are replaced with the current values
 #
+# Usage: pl_parsePkgs keyword [-a arch] fcdistro pldistro pkgs-file[..s]
+# the reason for the -a option is for when we build the build vserver itself; in this case
+# pl_DISTRO_ARCH is the one we obtain from the root context, and that's wrong
+# specify -sa arch AFTER keyword, so as to keep pl_getPackages and pl_getGroups simple
+#
 function pl_parsePkgs () {
 
+    target_arch=$pl_DISTRO_ARCH
     keyword=$1;shift
+    [ "$1" == "-a" ] && { shift; target_arch="$1"; shift; }
     fcdistro=$1; shift
     pldistro=$1; shift
     # remaining arguments are paths to the pkgs files
@@ -647,17 +556,18 @@ function pl_parsePkgs () {
     # grab exclusions
     sub=$(grep -v '^#' "$@" | grep --regexp="^${keyword}-${fcdistro}:" | sed -e "s,${keyword}-${fcdistro}:,,")
 
-    for i in $all $add; do
+    for rpm in $all $add; do
 	for exclude in $sub; do
-	    [ "$i" = "$exclude" ] && continue 2
+	    [ "$rpm" = "$exclude" ] && continue 2
 	done
-	echo "$i " | sed -e "s,@arch@,$pl_DISTRO_ARCH,g" -e "s,@fcdistro@,$fcdistro,g" -e "s,@pldistro@,$pldistro,g"
+	echo "${rpm} " | sed -e "s,@arch@,${target_arch},g" -e "s,@fcdistro@,$fcdistro,g" -e "s,@pldistro@,$pldistro,g"
     done
     return 0
 }
 
-function pl_getPackages() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs package $fcdistro $pldistro "$@" ; }
-function pl_getGroups() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs group $fcdistro $pldistro "$@" ; }
+# usage: pl_getPackages [-a arch] fcdistro pldistro pkg-file[..s]
+function pl_getPackages() { pl_parsePkgs package "$@" ; }
+function pl_getGroups() { pl_parsePkgs group "$@" ; }
 
 # locates a pldistro-dependant file
 # tries first in build/<pldistro>/, then in build/planetlab/
@@ -667,6 +577,11 @@ function pl_locateDistroFile () {
     pkgsfile=$1; shift
 
     pkgspath=""
+    # if config dir is missing but a .svnpath exists
+    if [[ -f "$builddir/config.${pldistro}.svnpath" && ! -d "$builddir/config.${pldistro}" ]] ; then
+	echo 1>&2 "Invoking make to extract remote config.${pldistro}"
+	make 1>&2 --silent --no-print-directory -C $builddir stage1=true config.${pldistro}
+    fi
     # locate it
     paths="$builddir/config.$pldistro/$pkgsfile $builddir/config.planetlab/$pkgsfile"
     for path in $paths; do
@@ -733,3 +648,90 @@ cat <<__footer
 </comps>
 __footer
 }
+
+
+function build_fetch () {
+    curl --fail --silent --max-time 60 --output /dev/null "$1" 
+}
+
+# tries to compute a valid yum.conf for that pldistro from the template in mirroring/
+# returns 0 and writes <dest_yumconf> on success
+# returns 1 on failure, in which case <dest_yumconf> is deleted
+function yumconf_mirrors () {
+    dest_yumconf=$1; shift
+    builddir=$1; shift
+    fcdistro=$1; shift
+    kexclude_line="$1" ; shift
+    mirrors="$@"
+
+    template=$builddir/mirroring/$fcdistro/yum.repos.d/building.repo.in
+    
+    if [ ! -f $template ] ; then
+	echo "yumconf_mirrors: cannot locate template $template"
+	rm -f $dest_yumconf
+	return 1
+    fi
+
+    for mirror in $mirrors; do
+	if yumconf_mirror $dest_yumconf $template "$kexclude_line" $mirror; then
+	    return 0
+	fi
+    done
+    rm -f $dest_yumconf
+    return 1
+}
+
+# computes a yum.conf from the template, and checks that all baseurl defined in there are valid repos
+# returns 0 on success and 1 on failure
+function yumconf_mirror () {
+    dest_yumconf=$1; shift
+    template=$1; shift
+    kexclude_line="$1" ; shift
+    mirror=$1; shift
+
+    sed -e "s,@MIRRORURL@,$mirror,g" \
+	-e "/baseurl=/i\\
+$kexclude_line" $template > $dest_yumconf
+    
+    # capture all lines defining baseurl
+    baseurl_defs=$(grep '^baseurl=' $dest_yumconf)
+    if [ -z "$baseurl_defs" ] ; then
+	return 1
+    fi
+
+    for baseurl_def in $baseurl_defs; do
+	baseurl=$(echo $baseurl_def | sed \
+	    -e s,baseurl=,, \
+	    -e 's,$basearch,'"$pl_DISTRO_ARCH",g)
+	repomd=$baseurl/repodata/repomd.xml
+
+	echo "* Trying to fetch $repomd"
+	if ! build_fetch $repomd ; then
+	    echo "* Failed to fetch $repomd"
+	    return 1
+	fi
+    done
+    echo "* Selected mirror $mirror"
+    return 0
+}
+
+# from a yum.conf as generated above, computes the (first) gpgkey url
+function yumconf_gpgkey () {
+    dest_yumconf=$1; shift
+
+    first_line=$(grep '^gpgkey=' $dest_yumconf | head -1)
+    values=$(echo $first_line | sed -e s,gpgkey=,,)
+    value=$(echo $values | awk '{print $1;}' | sed -e 's,$basearch,'"$pl_DISTRO_ARCH",g)
+    [ -n "$value" ] || return 1
+    echo $value
+    return 0
+}
+
+# patches a yum conf to insert an exclude line in each declared repo
+function yumconf_exclude () {
+    repo=$1; shift
+    kexclude_line="$1" ; shift
+    
+    sed -i -e "/baseurl=.*$/i\\
+$kexclude_line" $repo
+}