From 6b30afa6411089b55f294a4453a6435b1b0ce95e Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Mon, 13 Feb 2012 11:24:04 +0100 Subject: [PATCH] first attempt at a rework of kexclude -> {node,plc}yumexclude defined in yumexclude.pkgs --- build.common | 64 +++++++++++++++---------------- config.onelab/bootstrapfs.pkgs | 2 +- config.onelab/vserver.pkgs | 2 +- config.planetlab/bootcd.pkgs | 2 +- config.planetlab/bootstrapfs.pkgs | 2 +- config.planetlab/vserver.pkgs | 2 +- config.planetlab/yumexclude.pkgs | 10 +++++ getkexcludes.sh | 32 ---------------- nodeyumexclude.sh | 20 ++++++++++ onelab-k32-tags.mk | 4 +- onelab.mk | 4 +- pkgs.py | 4 +- planetlab-k32-tags.mk | 2 +- planetlab.mk | 4 +- vbuild-init-vserver.sh | 9 +++-- 15 files changed, 82 insertions(+), 81 deletions(-) create mode 100644 config.planetlab/yumexclude.pkgs delete mode 100755 getkexcludes.sh create mode 100755 nodeyumexclude.sh diff --git a/build.common b/build.common index 00665fc8..075064c6 100644 --- a/build.common +++ b/build.common @@ -5,8 +5,9 @@ # # Marc E. Fiuczynski # Copyright (C) 2007 The Trustees of Princeton University +# Thierry Parmentelat INRIA # -# supported distros f8, f10, f12, centos5 +# supported distros f8, f10, f12, f14, f16, centos5, sl6 # # for locating pkgs.py export PATH=.:$PATH @@ -65,31 +66,30 @@ function pl_getReleaseName () { return 0 } -# on fedora 8 or above, we use libnl from the stock repos -# on centos5 we build it locally -function pl_getKexcludes () { - distroname=$1; shift - case $distroname in - # when using k27, we need our own flavour of iproute - # for now we build only for k32, regardless of fcdistro - f8) - echo 'kernel* util-vserver* iptables iptables-ipv6 drupal' ;; - f*) - echo 'kernel* util-vserver* iptables iptables-ipv6 drupal' ;; - centos5) - echo 'kernel kernel-vserver kernel-devel kernel-headers kernel-debuginfo util-vserver* iptables drupal inotify-tools* libnl' ;; - sl*) - echo 'kernel kernel-debug kernel-debug-devel kernel-devel kernel-doc kernel-firmware kernel-headers iptables' ;; - *) - echo "pl_DISTRO_NAME=$pl_DISTRO_NAME not supported in getexcludes.sh" ; exit 1 ;; - esac +# yum exclusions are now defined in yumexclude.pkgs +# so they can now depend both on the linux distro and the pl distro +function pl_yumexclude () { + keyword=$1; shift + fcdistro=$1; shift + pldistro=$1; shift + # search for file "yumexclude.pkgs" + yumexclude_file=$(pl_locateDistroFile ../build/ $pldistro "yumexclude.pkgs") + # parse it + pl_parsePkgs keyword $fcdistro $pldistro $yumexclude_file || \ + { echo pl_yumexclude failed with fcdistro=$fcdistro and pldistro=$pldistro; return 1 ; } } +# node side : use the 'nodeyumexclude' keywork in yumexclude.pkgs +function pl_nodeyumexclude () { pl_yumexclude 'nodeyumexclude' "$@" ; } +function pl_plcyumexclude () { pl_yumexclude 'plcyumexclude' "$@" ; } + + # figure out which redhat distro we are using (fedora, centos, redhat) pl_DISTRO=$(pl_getDistro) # select basearch of the host devel environment - protected for macos for local tests -pl_DISTRO_ARCH=$(uname -i 2>/dev/null|| echo unknownarch) +# try arch for testing stuff on a mac +pl_DISTRO_ARCH=$(uname -i 2>/dev/null || arch 2> /dev/null || echo unknownarch) # the release number (plain number) pl_DISTRO_RELEASE=$(pl_getRelease) @@ -97,9 +97,6 @@ pl_DISTRO_RELEASE=$(pl_getRelease) # the release name - for vserver build ... pl_DISTRO_NAME=$(pl_getReleaseName $pl_DISTRO $pl_DISTRO_RELEASE) -# the packages to exclude -pl_KEXCLUDES=$(pl_getKexcludes $pl_DISTRO_NAME) - # get path to appropriate yumgroups.xml file # Thierry: quick & dirty improvement # this file is updated by the toplevel build, from the .pkgs files @@ -239,15 +236,16 @@ function pl_root_mkfedora () { pkgs_precious=$(pl_parsePkgs precious $fcdistro $pldistro $pkgsfile) # formerly related to mkfedora -k : packages to take from our own build # and thus need be excluded frem the stock repos - pkgs_kexcludes=$(pl_parsePkgs kexclude $fcdistro $pldistro $pkgsfile | sed -e s,@KEXCLUDE@,"$pl_KEXCLUDES",) + SUBST_NODEYUMEXCLUDE=$(pl_nodeyumexclude $fcdistro $pldistro) + pkgs_yumexcludes=$(pl_parsePkgs yumexclude $fcdistro $pldistro $pkgsfile | sed -e s,@NODEYUMEXCLUDE@,"$SUBST_NODEYUMEXCLUDE",) # get mirrors if not specified with -l if [ -z "$mirrors" ] ; then mirrorsfile=$(pl_locateDistroFile ../build/ $pldistro "$pldistro.mirrors") mirrors=$(pl_parsePkgs mirror $fcdistro $pldistro $mirrorsfile) fi - kexclude_line="" - [ -n "$pkgs_kexcludes" ] && kexclude_line="exclude=$pkgs_kexcludes" + yumexclude_line="" + [ -n "$pkgs_yumexcludes" ] && yumexclude_line="exclude=$pkgs_yumexcludes" echo "$0: candidate mirrors" for mirror in $mirrors ; do @@ -256,7 +254,7 @@ function pl_root_mkfedora () { # the repo part of the final yum.conf yum_conf_repos=$vroot/xxxmkfedora-repos.confxxx - if ! yumconf_mirrors $yum_conf_repos ../build/ $fcdistro "$kexclude_line" $mirrors ; then + if ! yumconf_mirrors $yum_conf_repos ../build/ $fcdistro "$yumexclude_line" $mirrors ; then echo xxx -- error ; return 1 fi @@ -689,7 +687,7 @@ function yumconf_mirrors () { dest_yumconf=$1; shift builddir=$1; shift fcdistro=$1; shift - kexclude_line="$1" ; shift + yumexclude_line="$1" ; shift mirrors="$@" template=$builddir/mirroring/$fcdistro/yum.repos.d/building.repo.in @@ -701,7 +699,7 @@ function yumconf_mirrors () { fi for mirror in $mirrors; do - if yumconf_mirror $dest_yumconf $template "$kexclude_line" $mirror; then + if yumconf_mirror $dest_yumconf $template "$yumexclude_line" $mirror; then return 0 fi done @@ -715,12 +713,12 @@ function yumconf_mirrors () { function yumconf_mirror () { dest_yumconf=$1; shift template=$1; shift - kexclude_line="$1" ; shift + yumexclude_line="$1" ; shift mirror=$1; shift sed -e "s,@MIRRORURL@,$mirror,g" \ -e "/baseurl=/i\\ -$kexclude_line" $template > $dest_yumconf +$yumexclude_line" $template > $dest_yumconf # capture all lines defining baseurl baseurl_defs=$(grep '^baseurl=' $dest_yumconf) @@ -757,8 +755,8 @@ function yumconf_gpgkey () { # patches a yum conf to insert an exclude line in each declared repo function yumconf_exclude () { repo=$1; shift - kexclude_line="$1" ; shift + yumexclude_line="$1" ; shift sed -i -e "/#baseurl=.*$/i\\ -$kexclude_line" $repo +$yumexclude_line" $repo } diff --git a/config.onelab/bootstrapfs.pkgs b/config.onelab/bootstrapfs.pkgs index 79e394b6..e5375ccb 100644 --- a/config.onelab/bootstrapfs.pkgs +++ b/config.onelab/bootstrapfs.pkgs @@ -4,7 +4,7 @@ groupname: PlanetLab groupdesc: PlanetLab Node Root # packages to exclude from stock repositories -kexclude: @KEXCLUDE@ +yumexclude: @NODEYUMEXCLUDE@ # fedora-release or centos-release are pulled from deps # define packages package: kernel diff --git a/config.onelab/vserver.pkgs b/config.onelab/vserver.pkgs index 17953c49..ca264475 100644 --- a/config.onelab/vserver.pkgs +++ b/config.onelab/vserver.pkgs @@ -2,7 +2,7 @@ groupname: PlanetLabSlice groupdesc: PlanetLab Slice Reference Image # packages to exclude from stock repositories -kexclude: @KEXCLUDE@ +yumexclude: @NODEYUMEXCLUDE@ # define packages package: bash package: coreutils diff --git a/config.planetlab/bootcd.pkgs b/config.planetlab/bootcd.pkgs index f8206023..b88f0f56 100644 --- a/config.planetlab/bootcd.pkgs +++ b/config.planetlab/bootcd.pkgs @@ -1,7 +1,7 @@ groupname: PlanetLabBootcd groupdesc: PlanetLab BootCD # packages to exclude from stock repositories -kexclude: @KEXCLUDE@ +yumexclude: @NODEYUMEXCLUDE@ # define packages package: filesystem package: udev diff --git a/config.planetlab/bootstrapfs.pkgs b/config.planetlab/bootstrapfs.pkgs index d38ba95d..185f524b 100644 --- a/config.planetlab/bootstrapfs.pkgs +++ b/config.planetlab/bootstrapfs.pkgs @@ -4,7 +4,7 @@ groupname: PlanetLab groupdesc: PlanetLab Node Root # packages to exclude from stock repositories -kexclude: @KEXCLUDE@ +yumexclude: @NODEYUMEXCLUDE@ # fedora-release or centos-release are pulled from deps # define packages package: kernel diff --git a/config.planetlab/vserver.pkgs b/config.planetlab/vserver.pkgs index a6e07283..58b19623 100644 --- a/config.planetlab/vserver.pkgs +++ b/config.planetlab/vserver.pkgs @@ -2,7 +2,7 @@ groupname: PlanetLabSlice groupdesc: PlanetLab Slice Reference Image # packages to exclude from stock repositories -kexclude: @KEXCLUDE@ +yumexclude: @NODEYUMEXCLUDE@ # define packages package: bash package: coreutils diff --git a/config.planetlab/yumexclude.pkgs b/config.planetlab/yumexclude.pkgs new file mode 100644 index 00000000..98c77abc --- /dev/null +++ b/config.planetlab/yumexclude.pkgs @@ -0,0 +1,10 @@ +## feb. 2012 -- this used to be hard-wired in build.common + +# plexclude: to exclude on the myplc side - typially drupal +plcyumexclude: drupal + +# excludes on the node side +nodeyumexclude=f8: kernel* util-vserver* iptables iptables-ipv6 +nodeyumexclude>=f12: kernel* util-vserver* iptables iptables-ipv6 +nodeyumexclude=centos5: kernel kernel-vserver kernel-devel kernel-headers kernel-debuginfo util-vserver* iptables inotify-tools* libnl +nodeyumexclude=sl6: kernel kernel-debug kernel-debug-devel kernel-devel kernel-doc kernel-firmware kernel-headers iptables diff --git a/getkexcludes.sh b/getkexcludes.sh deleted file mode 100755 index d45524c7..00000000 --- a/getkexcludes.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -COMMAND=$(basename $0) -. $(dirname $0)/build.common - -function usage () { - echo "Usage: $COMMAND [-f fcdistro]" - echo "outputs the list of packages to exclude from the stock repositories" - echo "this is set in build.common, and needs to fit the set of packages" - echo "that we override in the planetlab build" - exit 1 -} - -while getopts "f:" opt ; do - case $opt in - f) FCDISTRO=$OPTARG ;; - h|*) usage ;; - esac -done - -toshift=$(($OPTIND - 1)) -shift $toshift - -[[ -n "$@" ]] && usage - -# if the fcdistro is passed in argument -if [ -n "$FCDISTRO" ] ; then - pl_getKexcludes "$FCDISTRO" -# otherwise use the value for the current system -else - echo "$pl_KEXCLUDES" -fi diff --git a/nodeyumexclude.sh b/nodeyumexclude.sh new file mode 100755 index 00000000..79965922 --- /dev/null +++ b/nodeyumexclude.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +COMMAND=$(basename $0) +. $(dirname $0)/build.common + +function usage () { + echo "Usage: $COMMAND fcdistro pldistro" + echo "outputs the list of packages to exclude from the stock repo" + echo "this is set in yumexclude.pkgs, and needs to match the set of packages" + echo "that are produced by the planetlab build and meant to replace the stock ones" + exit 1 +} + +[[ -z "$@" ]] && usage +FCDISTRO=$1; shift +[[ -z "$@" ]] && usage +PLDISTRO=$1; shift +[[ -n "$@" ]] && usage + +pl_nodeyumexclude "$FCDISTRO" "$PLDISTRO" diff --git a/onelab-k32-tags.mk b/onelab-k32-tags.mk index 59b6b0f0..f71d820d 100644 --- a/onelab-k32-tags.mk +++ b/onelab-k32-tags.mk @@ -13,7 +13,7 @@ kernel-DEVEL-RPMS += elfutils-libelf-devel madwifi-GITPATH := git://git.onelab.eu/madwifi.git@madwifi-4132-5 iptables-GITPATH := git://git.onelab.eu/iptables.git@iptables-1.4.10-5 # we use the stock iproute2 with 2.6.32, since our gre patch is not needed anymore with that kernel -# note that this should be consistently reflected in pl_getKexcludes in build.common +# note that this should be consistently reflected in nodeyumexclude ALL := $(filter-out iproute,$(ALL)) ### ipfw-GITPATH := git://git.onelab.eu/ipfw.git@ipfw-0.9-23 @@ -57,7 +57,7 @@ pypcilib-GITPATH := git://git.onelab.eu/pypcilib.git@pypcilib-0.2-10 pyplnet-GITPATH := git://git.onelab.eu/pyplnet.git@pyplnet-4.3-11 bootcd-GITPATH := git://git.onelab.eu/bootcd.git@bootcd-5.0-11 vserver-reference-GITPATH := git://git.onelab.eu/vserver-reference.git@vserver-reference-5.0-6 -bootstrapfs-GITPATH := git://git.onelab.eu/bootstrapfs.git@bootstrapfs-2.0-13 +bootstrapfs-GITPATH := git://git.onelab.eu/bootstrapfs.git@master myplc-GITPATH := git://git.onelab.eu/myplc.git@myplc-5.0-19 DistributedRateLimiting-SVNPATH := http://svn.planet-lab.org/svn/DistributedRateLimiting/tags/DistributedRateLimiting-0.1-1 diff --git a/onelab.mk b/onelab.mk index 109fb730..2b332238 100644 --- a/onelab.mk +++ b/onelab.mk @@ -131,7 +131,7 @@ IN_BOOTSTRAPFS += util-vserver # # libnl - local import # we need either 1.1 or at least 1.0.pre6 -# rebuild this on centos5 - see kexcludes in build.common +# rebuild this on centos5 - see yumexclude # local_libnl=false ifeq "$(DISTRONAME)" "centos5" @@ -236,7 +236,7 @@ IN_BOOTSTRAPFS += mom # # inotify-tools - local import -# rebuild this on centos5 (not found) - see kexcludes in build.common +# rebuild this on centos5 (not found) - see yumexclude # local_inotify_tools=false ifeq "$(DISTRONAME)" "centos5" diff --git a/pkgs.py b/pkgs.py index cccd7ea4..5a35e135 100755 --- a/pkgs.py +++ b/pkgs.py @@ -37,7 +37,9 @@ default_fcdistro='f12' known_fcdistros = [ 'centos5','centos6','f8', 'f9','f10','f11','f12', 'f13', 'f14', 'f16', 'sl6'] default_pldistro='onelab' -known_keywords=['groupname', 'groupdesc', 'kexclude', 'package', 'group', 'precious', 'junk', 'mirror', ] +known_keywords=['groupname', 'groupdesc', + 'nodeyumexclude', 'plcyumexclude', + 'yumexclude', 'package', 'group', 'precious', 'junk', 'mirror', ] m_fcdistro_cutter = re.compile('([a-z]+)([0-9]+)') diff --git a/planetlab-k32-tags.mk b/planetlab-k32-tags.mk index 9a98c9b1..14377a8b 100644 --- a/planetlab-k32-tags.mk +++ b/planetlab-k32-tags.mk @@ -11,7 +11,7 @@ kernel-DEVEL-RPMS += elfutils-libelf-devel madwifi-GITPATH := git://git.planet-lab.org/madwifi.git@madwifi-4132-5 iptables-GITPATH := git://git.planet-lab.org/iptables.git@iptables-1.4.10-5 # we use the stock iproute2 with 2.6.32, since our gre patch is not needed anymore with that kernel -# note that this should be consistently reflected in pl_getKexcludes in build.common +# note that this should be consistently reflected in yumexclude ALL := $(filter-out iproute,$(ALL)) util-vserver-GITPATH := git://git.planet-lab.org/util-vserver.git@util-vserver-0.30.216-19 libnl-GITPATH := git://git.planet-lab.org/libnl.git@libnl-1.1-2 diff --git a/planetlab.mk b/planetlab.mk index 59191c51..1b464051 100644 --- a/planetlab.mk +++ b/planetlab.mk @@ -104,7 +104,7 @@ IN_BOOTSTRAPFS += util-vserver # # libnl - local import # we need either 1.1 or at least 1.0.pre6 -# rebuild this on centos5 - see kexcludes in build.common +# rebuild this on centos5 - see yumexclude # local_libnl=false ifeq "$(DISTRONAME)" "centos5" @@ -206,7 +206,7 @@ IN_BOOTSTRAPFS += mom # # inotify-tools - local import -# rebuild this on centos5 (not found) - see kexcludes in build.common +# rebuild this on centos5 (not found) - see yumexclude # local_inotify_tools=false ifeq "$(DISTRONAME)" "centos5" diff --git a/vbuild-init-vserver.sh b/vbuild-init-vserver.sh index 7aed8179..6d49b321 100755 --- a/vbuild-init-vserver.sh +++ b/vbuild-init-vserver.sh @@ -33,6 +33,7 @@ function configure_yum_in_vserver () { vserver=$1; shift fcdistro=$1; shift + pldistro=$1; shift templates=/etc/vservers/.distributions/${fcdistro} if [ -f ${templates}/yum/yum.conf ] ; then @@ -63,8 +64,9 @@ function configure_yum_in_vserver () { echo "WARNING : cannot create myplc repo" else # exclude kernel from fedora repos + yumexclude=$(pl_plcyumexclude $fcdistro $pldistro) for repo in /vservers/$vserver/etc/yum.repos.d/* ; do - [ -f $repo ] && yumconf_exclude $repo "exclude=$pl_KEXCLUDES" + [ -f $repo ] && yumconf_exclude $repo "exclude=$yumexclude" done # the build repo is not signed at this stage cat > /vservers/$vserver/etc/yum.repos.d/myplc.repo <