From: Thierry Parmentelat Date: Mon, 10 Mar 2008 11:45:24 +0000 (+0000) Subject: Second step towards multiarch myplc https://svn.planet-lab.org/ticket/214 X-Git-Tag: 4.2-rc2~155 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=46cb991a23c0e0d7b79f04833e68456a77c53280;p=build.git Second step towards multiarch myplc https://svn.planet-lab.org/ticket/214 -- design * new notion of a 'nodefamily' - that for now is - * standard rpm repository is now named /var/www/html/install-rpms/ a legacy symlink install-rpms/planetlab is created my myplc packaging * bootstrafs rpm name is now bootstrapfs- so that several instances from various builds can be installed in the same myplc * noderepo rpm (named noderepo-) built as part of the planetlab distro this is for installing /var/www/html/intall-rpms/ from another build -- other changes * extensive cleanup of mkfedora: ** obsolete options -p -g -x removed ** option -p removed, pkgsfile is a mandatory argument ** option -k removed, use kexclude: in the pkgs file instead * also cleaned up pl_root_* functions - no wrapper to pl_root_mkfedora anymore -- status * not tested at all - likely to break * next step planned for storing nodefamily and pass it to yum.php.conf (impacted would be bootmanager, nodemanager and nodeconfig) * a final cleanup in mkfedora would be to define mirrors in a separate .pkgs-like file * kupdate.sh, as well as possibly other convenience scripts, might be impacted --- diff --git a/build.common b/build.common index 217fa1a2..255344a1 100644 --- a/build.common +++ b/build.common @@ -184,25 +184,16 @@ mirrors=( ) function mkfedora_usage() { - echo "Usage: pl_root_mkfedora [OPTION]... basedir" + echo "Usage: pl_root_mkfedora [OPTION]... basedir pldistro pkgsfile(s)" echo " -l url Fedora mirror location. Default is to try:" for mirror in "${mirrors[@]}" ; do echo " $mirror" done - echo " -f pkgsfile use this pkgs file for packages, groups, excludes.." - echo " -k Exclude kernel* packages from all repositories except bootstrap" echo " -v Be verbose" echo " -h This message" echo " target selection (defaults based on current build vserver)" echo " -r release Fedora release number (default: $releasever)" echo " -a arch Fedora architecture (default: $basearch)" - echo " legacy (use -f instead)" - echo " -g group1 -g group2 ..." - echo " Yumgroups to install (default: none)" - echo " -p package1 -p package2 ..." - echo " Additional packages to install (default: none)" - echo " -x package1 -x package2 ..." - echo " Packages to exclude (default: none)" exit 1 } @@ -223,20 +214,8 @@ function pl_root_mkfedora () { basearch=$pl_DISTRO_ARCH [ -n "$pl_DISTRO_URL" ] && mirrors=($pl_DISTRO_URL) -# Yum groups to install - groups=() - -# Packages to install - packages=() - -# Packages to exclude - exclude=() - -# Exclude kernel* (and related) packages from all repositories except bootstrap - exclude_kernel= - # Get options - while getopts "l:r:a:g:p:x:f:kvh" opt ; do + while getopts "l:r:a:vh" opt ; do case $opt in l) if echo $OPTARG | grep -q -i '^\(file\|http[s]*\)://' ; then @@ -244,7 +223,7 @@ function pl_root_mkfedora () { else # xxx rather use this as a .mirrors file, searched in config.pldistro/ # that would use the pkgs syntax with mirror: - #mirrors=($(pl_parsePkgs mirror $(pl_locateDistroFile ../build/ $OPTARG.mirrors))) + #mirrors=($(pl_parsePkgs mirror $(pl_locateDistroFile ../build/ $OPTARG.mirrors))) mirrors=(file://$OPTARG) fi ;; @@ -254,21 +233,6 @@ function pl_root_mkfedora () { a) basearch=$OPTARG ;; - g) - groups[${#groups[*]}]="$OPTARG" - ;; - p) - packages[${#packages[*]}]="$OPTARG" - ;; - x) - exclude[${#exclude[*]}]="$OPTARG" - ;; - f) - pkgsfile=$OPTARG - ;; - k) - exclude_kernel="exclude=kernel* ulogd iptables" - ;; v) verbose=1 set -x @@ -280,12 +244,28 @@ function pl_root_mkfedora () { done shift $(($OPTIND - 1)) - [[ -n "$@" ]] || mkfedora_usage + [[ "$#" -lt 3 ]] && mkfedora_usage vroot=$1 ; shift + pldistro=$1 ; shift + pkgsfile="$@" vroot=$(cd $vroot && pwd -P) - [[ -z "$@" ]] || mkfedora_usage [ -d $vroot ] || mkfedora_usage + + # parse pkgsfile and add to local vars + fcdistro=${pl_DISTRO_NAME} + pkgs_packages=(pl_parsePkgs package $fcdistro $pldistro $pkgsfile) + pkgs_groups=(pl_parsePkgs group $fcdistro $pldistro $pkgsfile) + # packages to exclude - obsolete, was maybe useful when installing a group + pkgs_excludes=$(pl_parsePkgs exclude $fcdistro $pldistro $pkgsfile) + pkgs_junk=$(pl_parsePkgs junk $fcdistro $pldistro $pkgsfile) + pkgs_precious=$(pl_parsePkgs precious $fcdistro $pldistro $pkgsfile) + # formerly related to mkfedora -k + pkgs_kexcludes=$(pl_parsePkgs kexclude $fcdistro $pldistro $pkgsfile) + + exclude_kernel="" + [ -n "$pkgs_kexcludes" ] && exclude_kernel="exclude=$pkgs_excludes" + function mkfedora_fetch () { curl --fail --silent --max-time 60 "$1" @@ -444,20 +424,8 @@ EOF # If we are being built as part of an automated RPM build, solve the # bootstrap problem by including any just built packages in the yum # configuration. This cooperates with the PlanetLab build system. -### make takes care of that if [ -n "$RPM_BUILD_DIR" ] ; then RPM_RPMS_DIR=$(cd $(dirname $RPM_BUILD_DIR)/RPMS && pwd -P) -### # yum-2.0.x -### if [ -x /usr/bin/yum-arch ] ; then -### yum-arch -q $RPM_RPMS_DIR -### fi -### # yum-2.4.x -### if [ -x /usr/bin/createrepo ] ; then -### if [ -f $RPM_RPMS_DIR/yumgroups.xml ] ; then -### groupfile="-g yumgroups.xml" -### fi -### createrepo --quiet $groupfile $RPM_RPMS_DIR -### fi # If run under sudo, allow user to delete the headers/ and # repodata/ directories. if [ -n "$SUDO_USER" ] ; then @@ -471,84 +439,48 @@ baseurl=file://$RPM_RPMS_DIR/ EOF fi - # pkgs file - if [ -n "$pkgsfile" ] ; then - # parse pkgsfile and add to local vars - fcdistro=$(pl_getReleaseName "Fedora" $releasever) - for i in $(pl_parsePkgs package $fcdistro $pkgsfile) ; do - packages[${#packages[*]}]="$i" - done - for i in $(pl_parsePkgs group $fcdistro $pkgsfile) ; do - groups[${#groups[*]}]="$i" - done - for i in $(pl_parsePkgs exclude $fcdistro $pkgsfile) ; do - exclude[${#exclude[*]}]="$i" - done - junk=$(pl_parsePkgs junk $fcdistro $pkgsfile) - precious=$(pl_parsePkgs precious $fcdistro $pkgsfile) - fi - - excludes= - for package in "${exclude[@]}" ; do - excludes="$excludes --exclude=$package" + exclude_arg="" + for exclude in $pkgs_excludes; do + exclude_arg="$exclude_arg --exclude $exclude" done # glibc must be specified explicitly for the correct arch to be # chosen. echo "* Installing glibc" - yum -c $vroot/etc/yum.conf --installroot=$vroot -y $excludes install glibc + yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg install glibc # Go, baby, go - if [ ${#packages[*]} -gt 0 ] ; then - echo "* Installing optional packages" "${packages[@]}" - yum -c $vroot/etc/yum.conf --installroot=$vroot -y $excludes \ - install "${packages[@]}" - if ! rpm --root $vroot -q "${packages[@]}" >/dev/null ; then + if [ -n "$pkgs_packages" ] ; then + echo "* Installing optional packages" "$pkgs_packages" + yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg \ + install "$pkgs_packages" + if ! rpm --root $vroot -q "$pkgs_packages" >/dev/null ; then echo "* Warning: Missing packages" - rpm --root $vroot -q "${packages[@]}" | grep "not installed" + rpm --root $vroot -q "$pkgs_packages" | grep "not installed" fi fi - if [ ${#groups[*]} -gt 0 ] ; then + if [ -n "$pkgs_groups" ] ; then ## call yum sequentially to get finer-grained info on dependencies - for grp in "${groups[@]}" ; do + for grp in "$pkgs_groups" ; do echo "* Installing optional group $grp" - yum -c $vroot/etc/yum.conf --installroot=$vroot -y $excludes \ + yum -c $vroot/etc/yum.conf --installroot=$vroot -y $exclude_arg \ groupinstall "$grp" done fi # formerly in bootcd/prep.sh : to optimize footprint - echo "* Removing unnecessary junk" - - pushd $vroot - - # Save precious files - [ -n "$precious" ] && tar --ignore-failed-read -cpf precious.tar $precious - - # Remove unnecessary junk - [ -n "$junk" ] && rm -rf $junk - - # Restore precious files - [ -n "$precious" ] && tar -xpf precious.tar && rm -f precious.tar - - popd - -### # FC2 dev %preinstall checks /proc/mounts to make sure that /dev is -### # not currently mounted as devfs. If it thinks it is, it will refuse -### # to install the package. On a modern system running udev that mounts -### # /dev as tmpfs, this check fails. Since we are installing into a -### # chroot, whether /dev is mounted on the host system or not doesn't -### # matter. If dev was explicitly mentioned in the packages list, force -### # its installation. -### if [ "$releasever" = "2" ] ; then -### for package in "${packages[@]}" ; do -### if [ "$package" = "dev" ] && ! rpm --root $vroot -q dev >/dev/null 2>&1 ; then -### rpm --root $vroot -Uvh --noscripts $baseurl/Fedora/RPMS/dev-3.3.13-1.i386.rpm -### break -### fi -### done -### fi + if [ -n "$pkgs_junk" ] ; then + echo "* Removing unnecessary junk" + pushd $vroot + # Save precious files + [ -n "$pkgs_precious" ] && tar --ignore-failed-read -cpf precious.tar $pkgs_precious + # Remove unnecessary junk + [ -n "$pkgs_junk" ] && rm -rf $pkgs_junk + # Restore precious files + [ -n "$pkgs_precious" ] && tar -xpf precious.tar && rm -f precious.tar + popd + fi # Clean yum cache echo "* Cleaning up" @@ -572,12 +504,8 @@ fi return 0 } -function pl_root_setup_chroot() { +function pl_root_tune_image () { root=$1; shift - options="$@" - - pl_root_makedevs $root - pl_root_mkfedora $options $root # Disable all services in reference image chroot $root sh -c "/sbin/chkconfig --list | awk '{ print \$1 }' | xargs -i /sbin/chkconfig {} off" @@ -667,18 +595,20 @@ function pl_getPkgsAttribute () { grep -v '^#' $file | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,," } - # for a given keyword like 'package' : # we support fcdistro-dependant format, for tokens (pkgname) without whitespace # you can e.g. use # package: pkg1 .. pkgn # package+f8: pkg1 .. pkgn # package-f8: pkg1 .. pkgn - +# +# values can contain @arch@, @fcdistro@ or @pldistro@ that are replaced with the current values +# function pl_parsePkgs () { keyword=$1;shift fcdistro=$1; shift + pldistro=$1; shift # remaining arguments are paths to the pkgs files # grab regular descriptions @@ -692,13 +622,13 @@ function pl_parsePkgs () { for exclude in $sub; do [ "$i" = "$exclude" ] && continue 2 done - echo "$i " + echo "$i " | sed -e "s,@arch@,$pl_DISTRO_ARCH,g" -e "s,@fcdistro@,$fcdistro,g" -e "s,@pldistro@,$pldistro,g" done return 0 } -function pl_getPackages() { fcdistro=$1; shift ; pl_parsePkgs package $fcdistro "$@" ; } -function pl_getGroups() { fcdistro=$1; shift ; pl_parsePkgs group $fcdistro "$@" ; } +function pl_getPackages() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs package $fcdistro $pldistro "$@" ; } +function pl_getGroups() { fcdistro=$1; shift ; pldistro=$1; shift ; pl_parsePkgs group $fcdistro $pldistro "$@" ; } # locates a pldistro-dependant file # tries first in build//, then in build/planetlab/ @@ -745,7 +675,7 @@ __header for pkgsname in $pkgsnames; do pkgsfile=$(pl_locateDistroFile $builddir $pldistro $pkgsname) - packages=$(pl_getPackages $fcdistro $pkgsfile) + packages=$(pl_getPackages $fcdistro $pldistro $pkgsfile) groupname=$(pl_getPkgsAttribute groupname $pkgsfile | sed $sedargs) groupdesc=$(pl_getPkgsAttribute groupdesc $pkgsfile | sed $sedargs) @@ -776,5 +706,3 @@ cat <<__footer __footer } - - diff --git a/config.planetlab/bootcd.pkgs b/config.planetlab/bootcd.pkgs index 3a0c9709..94d3daf0 100644 --- a/config.planetlab/bootcd.pkgs +++ b/config.planetlab/bootcd.pkgs @@ -1,5 +1,8 @@ groupname: PlanetLabBootcd groupdesc: PlanetLab BootCD +# This is the replacement for the former mkfedora -k. The following packages +# will get excluded from the stock fedora repos, as we need to use ours +kexclude: kernel* ulogd iptables # from prep.sh, was hardwired as packagelist package: filesystem package: udev diff --git a/config.planetlab/bootstrapfs.pkgs b/config.planetlab/bootstrapfs.pkgs index d8dde682..4f05ccce 100644 --- a/config.planetlab/bootstrapfs.pkgs +++ b/config.planetlab/bootstrapfs.pkgs @@ -2,6 +2,10 @@ # groupname: Bootstrapfs groupname: PlanetLab groupdesc: PlanetLab Node Root +# This is the replacement for the former mkfedora -k. The following packages +# will get excluded from the stock fedora repos, as we need to use ours +kexclude: kernel* ulogd iptables +# package: kernel package: udev package: basesystem diff --git a/config.planetlab/myplc.pkgs b/config.planetlab/myplc.pkgs index e1e656dc..61a8f7c1 100644 --- a/config.planetlab/myplc.pkgs +++ b/config.planetlab/myplc.pkgs @@ -53,5 +53,8 @@ package:gd php-gd perl-GD # plc-map.py uses this package:python-imaging # planetlab stuff -package:bootcd bootmanager bootstrapfs +package:bootcd bootmanager +# embed the bootstrafs rpm for this distribution & arch +# use a qualified name so other variants can be installed as well +package: bootstrapfs-@pldistro@-@arch@ package:PLCAPI PLCWWW nodeconfig diff --git a/config.planetlab/vserver.pkgs b/config.planetlab/vserver.pkgs index 35770a2f..1a20d1a1 100644 --- a/config.planetlab/vserver.pkgs +++ b/config.planetlab/vserver.pkgs @@ -1,5 +1,8 @@ groupname: PlanetLabSlice groupdesc: PlanetLab Slice Reference Image +# This is the replacement for the former mkfedora -k. The following packages +# will get excluded from the stock fedora repos, as we need to use ours +kexclude: kernel* ulogd iptables # define packages package: bash package: coreutils diff --git a/onelab-install.mk b/onelab-install.mk index b5422d90..0d54a306 100644 --- a/onelab-install.mk +++ b/onelab-install.mk @@ -6,6 +6,11 @@ # Installation rules # +install: + @echo WARNING: this target is deprecated + @echo you might siwh to use the noderepo rpm instead + @echo former behaviour is available throuhg make install-obsolete + # make install : # uses $(PLCSSH) as an ssh address for the destination host # first copies everything on a separate rpm repo, prepares it, @@ -35,11 +40,11 @@ RSYNC_COND_DRY_RUN := $(if $(findstring n,$(MAKEFLAGS)),--dry-run,) RSYNC := rsync $(RSYNC_COND_DRY_RUN) INSTALL-TARGETS := install-rpms install-index install-adopt install-bootstrap -install: $(INSTALL-TARGETS) -.PHONY: install $(INSTALL-TARGETS) +install-obsolete: $(INSTALL-TARGETS) +.PHONY: install install-obsolete $(INSTALL-TARGETS) install-help: - @echo install: $(INSTALL-TARGETS) + @echo install-obsolete: $(INSTALL-TARGETS) # compute the exact set of rpms to install - we do not need bootstrapfs nor myplc here node_packages=$(sort $(IN_VSERVER) $(IN_BOOTSTRAPFS)) diff --git a/onelab.mk b/onelab.mk index d7d8eae0..7e4dcde8 100644 --- a/onelab.mk +++ b/onelab.mk @@ -353,9 +353,8 @@ noderepo-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) $(IN_VSERVER) noderepo-DEPEND-FILES := RPMS/yumgroups.xml #export rpm list to the specfile noderepo-SPECVARS = node_rpms_plus=$(NODEREPO_RPMS_3PLUS) -# not yet ALL += noderepo -#IN_MYPLC += noderepo +IN_MYPLC += noderepo # # myplc : initial, chroot-based packaging diff --git a/planetlab.mk b/planetlab.mk index 7d236683..6314e1d0 100644 --- a/planetlab.mk +++ b/planetlab.mk @@ -315,6 +315,26 @@ bootstrapfs-DEPEND-FILES := RPMS/yumgroups.xml ALL += bootstrapfs IN_MYPLC += bootstrapfs +# +# noderepo +# +# all rpms resulting from packages marked as being in bootstrapfs and vserver +NODEREPO_RPMS = $(foreach package,$(IN_BOOTSTRAPFS) $(IN_VSERVER),$($(package).rpms)) +# replace space with +++ (specvars cannot deal with spaces) +SPACE=$(subst x, ,x) +NODEREPO_RPMS_3PLUS = $(subst $(SPACE),+++,$(NODEREPO_RPMS)) + +noderepo-MODULES := BootstrapFS +noderepo-SPEC := noderepo.spec +noderepo-RPMBUILD := sudo bash ./rpmbuild.sh +# package requires all regular packages +noderepo-DEPEND-PACKAGES := $(IN_BOOTSTRAPFS) $(IN_VSERVER) +noderepo-DEPEND-FILES := RPMS/yumgroups.xml +#export rpm list to the specfile +noderepo-SPECVARS = node_rpms_plus=$(NODEREPO_RPMS_3PLUS) +ALL += noderepo +IN_MYPLC += noderepo + # # myplc : initial, chroot-based packaging # diff --git a/vbuild-init-vserver.sh b/vbuild-init-vserver.sh index 3a886450..cc7b10b4 100755 --- a/vbuild-init-vserver.sh +++ b/vbuild-init-vserver.sh @@ -148,7 +148,7 @@ function setup_vserver () { cp /etc/resolv.conf /vservers/$vserver/etc/resolv.conf } -function devel_tools () { +function devel_or_vtest_tools () { set -x set -e @@ -168,8 +168,8 @@ function devel_tools () { pkgsfile=$(pl_locateDistroFile $DIRNAME $pldistro $pkgsname) # install individual packages, then groups - packages=$(pl_getPackages ${fcdistro} $pkgsfile) - groups=$(pl_getGroups ${fcdistro} $pkgsfile) + packages=$(pl_getPackages $fcdistro $pldistro $pkgsfile) + groups=$(pl_getGroups $fcdistro $pldistro $pkgsfile) [ -n "$packages" ] && $personality vserver $vserver exec yum -y install $packages [ -n "$groups" ] && $personality vserver $vserver exec yum -y groupinstall $groups @@ -338,7 +338,7 @@ function main () { [ -z "$personality" ] && personality=$DEFAULT_PERSONALITY setup_vserver $vserver $fcdistro $personality - devel_tools $vserver $fcdistro $pldistro $personality + devel_or_vtest_tools $vserver $fcdistro $pldistro $personality post_install $vserver $personality } diff --git a/vbuild-nightly.sh b/vbuild-nightly.sh index 101b3403..1941dee2 100755 --- a/vbuild-nightly.sh +++ b/vbuild-nightly.sh @@ -174,7 +174,6 @@ function runtest () { ssh ${TESTBOXSSH} svn co ${TESTSVNPATH} ${testdir} # invoke test on testbox - pass url and build url - so the tests can use vtest-init-vserver.sh ssh 2>&1 ${TESTBOXSSH} python -u ${testdir}/runtest --build ${SVNPATH} --url ${url} --all - # still missing - need to populate /var/www/html/install-rpms on the myplc if [ "$?" != 0 ] ; then failure