X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=build.common;h=153a010d0fcba8402f94b9e9d9b665d2ea59bc53;hb=1517ff234915ac3ba5f81813ca2572e64e63fd2f;hp=d45df3c7255ecaf43c47594ae57013e64a2736c9;hpb=695c71891b9a7f840697aea43bcf9473c86a5e36;p=build.git diff --git a/build.common b/build.common index d45df3c7..153a010d 100644 --- a/build.common +++ b/build.common @@ -9,43 +9,31 @@ # $Id$ # -function pl_getDefaultDistro() { - # FC4 is currently the default release - echo "Fedora" - return 0 -} - -function pl_getDefaultRelease() { - # FC4 is currently the default release - echo "4" - return 0 -} +# support for fedora and centos only for now function pl_getDistro() { - defaultDistro=$(pl_getDefaultDistro) if [ -f "/etc/redhat-release" ] ; then distro=$(awk ' { print $1 } ' /etc/redhat-release) else - distro=$defaultDistro + echo "build.common.pl_getDistro-unknown" + exit 1 fi echo "$distro" return 0 } function pl_getRelease() { - defaultRelease=$(pl_getDefaultRelease) if [ -f "/etc/redhat-release" ] ; then release=$(awk ' { if ($1=="Fedora" && $2=="Core") print $4 ; if (($1=="Fedora" && $2!="Core")||$1=="CentOS") print $3 } ' /etc/redhat-release) - [ $release -lt $defaultRelease ] && release=$defaultRelease else - release=$defaultRelease + echo "build.common.pl_getRelease-unknown" + exit 1 fi echo "$release" return 0 } # vserver expects something like fc4 or f7 -# for fedora and centos only for now function pl_getReleaseName () { distro=$1; shift release=$1; shift @@ -57,12 +45,12 @@ function pl_getReleaseName () { releasename=f$release fi ;; [Cc]entOS*) - if [ "$release" = "4.5" ] ; then - # centos 4.5 is just centos4 + enhancements - releasename=centos4 - else - releasename=centos$release - fi ;; + old_IFS="$IFS" + IFS="." + set -- $release + IFS="$old_IFS" + releasename=centos$1 + ;; *) releasename="unknown-name-for-${pl_DISTRO}-please-edit-build.common" echo 1>&2 "build.common: WARNING - releasename not set for distro=$distro" @@ -198,25 +186,15 @@ function pl_root_mkfedora () { basearch=$pl_DISTRO_ARCH # Get options - mirrors="" - while getopts "l:r:a:vh" opt ; do +### mirrors="" +### while getopts "l:r:a:vh" opt ; do + while getopts "vh" opt ; do case $opt in - l) - mirrors="$mirrors $OPTARG" - ;; - r) - releasever=$OPTARG - ;; - a) - basearch=$OPTARG - ;; - v) - verbose=1 - set -x - ;; - h|*) - mkfedora_usage - ;; +### l) mirrors="$mirrors $OPTARG" ;; +### r) releasever=$OPTARG ;; +### a) basearch=$OPTARG ;; + v) verbose=1; set -x ;; + h|*) mkfedora_usage ;; esac done @@ -245,59 +223,27 @@ function pl_root_mkfedora () { mirrors=$(pl_parsePkgs mirror $fcdistro $pldistro $mirrorsfile) fi - exclude_kernel="" - [ -n "$pkgs_kexcludes" ] && exclude_kernel="exclude=$pkgs_excludes" - - function mkfedora_fetch () - { - curl --fail --silent --max-time 60 "$1" - } - - # set list of attempted locations according to releasever - if [ $releasever -ge 7 ] ; then - attempts=" -linux/releases/$releasever/Everything/$basearch/os -$releasever/Everything/$basearch/os -core/$releasever/Everything/$basearch/os -linux/core/$releasever/$basearch/os -" - else - attempts=" - linux/core/$releasever/$basearch/os - core/$releasever/$basearch/os - $releasever/$basearch/os - " + exclude_line="" + # add them manually as the output of pl_parsePkgs is line-separated + if [ -n "$pkgs_kexcludes" ] ; then + exclude_line="exclude=" + for kexclude in $pkgs_kexcludes ; do + exclude_line="$exclude_line $kexclude" + done fi echo "$0: candidate mirrors" for mirror in $mirrors ; do echo "* candidate mirror $mirror" done - baseurl="" - for mirror in $mirrors ; do - for attempt in $attempts; do - attempturl=$mirror/$attempt - if mkfedora_fetch $attempturl/repodata/repomd.xml >/dev/null ; then - baseurl=$attempturl - break 2 - fi - done - done - if [ -z "$baseurl" ] ; then - echo "Error: $releasever/$basearch/os/repodata/repomd.xml" - echo " could not be found in any of the following locations:" - echo - for mirror in $mirrors ; do - for attempt in $attempts ; do - echo $mirror/$attempt - done - done - echo - mkfedora_usage - else - echo "* selecting mirror with baseurl=$baseurl" + # the repo part of the final yum.conf + yum_conf_repos=$vroot/xxxmkfedora-repos.confxxx + if ! yumconf_mirrors $yum_conf_repos ../build/ $fcdistro "$exclude_line" $mirrors ; then + echo xxx -- error ; return 1 fi + + public_gpg_key=$(yumconf_gpgkey $yum_conf_repos) # Do not tolerate errors set -e @@ -352,17 +298,14 @@ EOF # Initialize RPM database in reference image mkdir -p $vroot/var/lib/rpm rpm --root $vroot --initdb - rpm --root $vroot --import $baseurl/RPM-GPG-KEY-fedora + rpm --root $vroot --import $public_gpg_key # Initialize yum in reference image mkdir -p $vroot/var/cache/yum $vroot/var/log - if [ $releasever -lt 7 ] ; then - corename="Core " - else - corename="" - fi - cat >$vroot/etc/yum.conf < $yum_conf <>$vroot/etc/yum.conf <> $yum_conf # If we are being built as part of an automated RPM build, solve the # bootstrap problem by including any just built packages in the yum @@ -413,14 +334,24 @@ EOF if [ -n "$SUDO_USER" ] ; then chown -R $SUDO_USER $RPM_RPMS_DIR fi - cat >>$vroot/etc/yum.conf <> $yum_conf < /dev/null && yum_options="$yum_options --verbose" + yum_options="$yum_options -y" + yum_options="$yum_options -c $yum_conf" + yum_options="$yum_options --installroot=$vroot" + exclude_arg="" for exclude in $pkgs_excludes; do exclude_arg="$exclude_arg --exclude $exclude" @@ -429,13 +360,12 @@ fi # glibc must be specified explicitly for the correct arch to be # chosen. echo "* Installing glibc" - yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg install glibc + yum $yum_options $exclude_arg install glibc # Go, baby, go if [ -n "$pkgs_packages" ] ; then echo "* Installing optional packages" $pkgs_packages - yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg \ - install $pkgs_packages + yum $yum_options $exclude_arg install $pkgs_packages if ! rpm --root $vroot -q $pkgs_packages >/dev/null ; then echo "* Warning: Missing packages" rpm --root $vroot -q $pkgs_packages | grep "not installed" @@ -446,8 +376,7 @@ fi ## call yum sequentially to get finer-grained info on dependencies for grp in $pkgs_groups ; do echo "* Installing optional group $grp" - yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg \ - groupinstall "$grp" + yum $yum_options $exclude_arg groupinstall "$grp" done fi @@ -466,7 +395,7 @@ fi # Clean yum cache echo "* Cleaning up" - yum -c $vroot/etc/yum.conf --installroot=$vroot -y clean all + yum $yum_options clean all # Clean RPM state rm -f $vroot/var/lib/rpm/__db* @@ -477,6 +406,9 @@ fi ln -s /usr/share/zoneinfo/UTC $vroot/etc/localtime fi + echo "Dumping current list of rpms in /etc/mkfedora-rpms.txt" + chroot $vroot rpm -aq | sort > $vroot/etc/mkfedora-rpms.txt + # remove trap handler, as we are about to call it directly. trap - ERR INT @@ -574,7 +506,8 @@ function pl_fixdirs() { function pl_getPkgsAttribute () { keyword=$1; shift file=$1; shift - grep -v '^#' $file | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,," + # remove any initial white spaces from the result + grep -v '^#' $file | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,," -e "s,^[[:space:]][[:space:]]*,," } # for a given keyword like 'package' : @@ -586,9 +519,16 @@ function pl_getPkgsAttribute () { # # values can contain @arch@, @fcdistro@ or @pldistro@ that are replaced with the current values # +# Usage: pl_parsePkgs keyword [-a arch] fcdistro pldistro pkgs-file[..s] +# the reason for the -a option is for when we build the build vserver itself; in this case +# pl_DISTRO_ARCH is the one we obtain from the root context, and that's wrong +# specify -sa arch AFTER keyword, so as to keep pl_getPackages and pl_getGroups simple +# function pl_parsePkgs () { + target_arch=$pl_DISTRO_ARCH keyword=$1;shift + [ "$1" == "-a" ] && { shift; target_arch="$1"; shift; } fcdistro=$1; shift pldistro=$1; shift # remaining arguments are paths to the pkgs files @@ -600,17 +540,18 @@ function pl_parsePkgs () { # grab exclusions sub=$(grep -v '^#' "$@" | grep --regexp="^${keyword}-${fcdistro}:" | sed -e "s,${keyword}-${fcdistro}:,,") - for i in $all $add; do + for rpm in $all $add; do for exclude in $sub; do - [ "$i" = "$exclude" ] && continue 2 + [ "$rpm" = "$exclude" ] && continue 2 done - echo "$i " | sed -e "s,@arch@,$pl_DISTRO_ARCH,g" -e "s,@fcdistro@,$fcdistro,g" -e "s,@pldistro@,$pldistro,g" + echo "${rpm} " | sed -e "s,@arch@,${target_arch},g" -e "s,@fcdistro@,$fcdistro,g" -e "s,@pldistro@,$pldistro,g" done return 0 } -function pl_getPackages() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs package $fcdistro $pldistro "$@" ; } -function pl_getGroups() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs group $fcdistro $pldistro "$@" ; } +# usage: pl_getPackages [-a arch] fcdistro pldistro pkg-file[..s] +function pl_getPackages() { pl_parsePkgs package "$@" ; } +function pl_getGroups() { pl_parsePkgs group "$@" ; } # locates a pldistro-dependant file # tries first in build//, then in build/planetlab/ @@ -640,14 +581,13 @@ function pl_locateDistroFile () { fi } -# experimental function yumgroups_from_pkgs () { builddir=$1; shift pldistro=$1; shift fcdistro=$1; shift pkgsnames=$@ -sedargs="-e s,@FCDISTRO@,$fcdistro,g" + sedargs="-e s,@FCDISTRO@,$fcdistro,g" cat <<__header @@ -671,9 +611,8 @@ __header $(echo $groupname|tr A-Z a-z) $groupname - true $groupdesc - false + true __group_header for package in $packages; do @@ -688,3 +627,81 @@ cat <<__footer __footer } + + +function build_fetch () { + curl --fail --silent --max-time 60 --output /dev/null "$1" +} + +# tries to compute a valid yum.conf for that pldistro from the template in mirroring/ +# returns 0 and writes on success +# returns 1 on failure, in which case is deleted +function yumconf_mirrors () { + dest_yumconf=$1; shift + builddir=$1; shift + fcdistro=$1; shift + exclude_line="$1" ; shift + mirrors="$@" + + template=$builddir/mirroring/$fcdistro/yum.repos.d/building.repo.in + + if [ ! -f $template ] ; then + echo "yumconf_mirrors: cannot locate template $template" + rm -f $dest_yumconf + return 1 + fi + + for mirror in $mirrors; do + if yumconf_mirror $dest_yumconf $template "$exclude_line" $mirror; then + return 0 + fi + done + rm -f $dest_yumconf + return 1 +} + +# computes a yum.conf from the template, and checks that all baseurl defined in there are valid repos +# returns 0 on success and 1 on failure +function yumconf_mirror () { + dest_yumconf=$1; shift + template=$1; shift + exclude_line="$1" ; shift + mirror=$1; shift + + sed -e "s,@MIRRORURL@,$mirror,g" \ + -e "/baseurl=/i\\ +$exclude_line" $template > $dest_yumconf + + # capture all lines defining baseurl + baseurl_defs=$(grep '^baseurl=' $dest_yumconf) + if [ -z "$baseurl_defs" ] ; then + return 1 + fi + + for baseurl_def in $baseurl_defs; do + baseurl=$(echo $baseurl_def | sed \ + -e s,baseurl=,, \ + -e 's,$basearch,'"$pl_DISTRO_ARCH",g) + repomd=$baseurl/repodata/repomd.xml + + echo "* Trying to fetch $repomd" + if ! build_fetch $repomd ; then + echo "* Failed to fetch $repomd" + return 1 + fi + done + echo "* Selected mirror $mirror" + return 0 +} + +# from a yum.conf as generated above, computes the (first) gpgkey url +function yumconf_gpgkey () { + dest_yumconf=$1; shift + + first_line=$(grep '^gpgkey=' $dest_yumconf | head -1) + values=$(echo $first_line | sed -e s,gpgkey=,,) + value=$(echo $values | awk '{print $1;}' | sed -e 's,$basearch,'"$pl_DISTRO_ARCH",g) + [ -n "$value" ] || return 1 + echo $value + return 0 +}