From c4e209060103076da66862f1315212c2471da26a Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Thu, 6 Dec 2007 08:55:20 +0000 Subject: [PATCH] (*) groups distro-dependent files in build/config. (*) renamed .lst files into .pkgs (lst suffix commonly used for other/tmp purposes) (*) distro-dependent files are (mostly) optional, defaults to planetlab (*) mkfedora accepts new -f argument : to pass a pkgs file rather than -p .. -g .. (*) mkfedora supports junk: and precious: in pkgs file this is from the former bootcd/prep.sh - to reduce footprint Notes. (*) this is untested (*) mkfedora still supports -x (exclude) on the command line and in pkgs file however I suspect this feature is never used (*) mkfedora is still a separate command - could/should be merged into buld.common (*) myplc/build.functions could/should be merged into build.common as well --- build.common | 108 ++++++++++++------ config.planetlab/bootcd.pkgs | 74 ++++++++++++ config.planetlab/bootstrapfs-production.pkgs | 13 +++ config.planetlab/bootstrapfs.pkgs | 7 ++ .../devel.pkgs | 0 config.planetlab/myplc.pkgs | 56 +++++++++ .../native-shell.pkgs | 0 config.planetlab/vserver-planetflow.pkgs | 1 + config.planetlab/vserver.pkgs | 29 +++++ mkfedora | 42 ++++++- vbuild-init-vserver.sh | 54 ++++----- 11 files changed, 319 insertions(+), 65 deletions(-) create mode 100644 config.planetlab/bootcd.pkgs create mode 100644 config.planetlab/bootstrapfs-production.pkgs create mode 100644 config.planetlab/bootstrapfs.pkgs rename planetlab-devel.lst => config.planetlab/devel.pkgs (100%) create mode 100644 config.planetlab/myplc.pkgs rename planetlab-shell.lst => config.planetlab/native-shell.pkgs (100%) create mode 100644 config.planetlab/vserver-planetflow.pkgs create mode 100644 config.planetlab/vserver.pkgs diff --git a/build.common b/build.common index c08d7ee0..463aaf6d 100644 --- a/build.common +++ b/build.common @@ -12,11 +12,13 @@ function pl_getDefaultDistro() { # FC4 is currently the default release echo "Fedora" + return 0 } function pl_getDefaultRelease() { # FC4 is currently the default release echo "4" + return 0 } function pl_getDistro() { @@ -27,6 +29,7 @@ function pl_getDistro() { distro=$defaultDistro fi echo "$distro" + return 0 } function pl_getRelease() { @@ -38,6 +41,36 @@ function pl_getRelease() { release=$defaultRelease fi echo "$release" + return 0 +} + +# vserver expects something like fc4 or f7 +# for fedora and centos only for now +function pl_getReleaseName () { + distro=$1; shift + release=$1; shift + case $distro in + [Ff]edora*) + if [ "$release" -le 6 ] ; then + releasename=fc$release + else + releasename=f$release + fi ;; + [Cc]entOS*) + if [ "$release" = "4.5" ] ; then + # centos 4.5 is just centos4 + enhancements + releasename=centos4 + else + releasename=centos$release + fi ;; + *) + releasename="unknown-name-for-${pl_DISTRO}-please-edit-build.common" + echo 1>&2 "build.common: WARNING - releasename not set for distro=$pl_DISTRO" + return 1 + ;; + esac + echo "$releasename" + return 0 } # figure out which redhat distro we are using (fedora, centos, redhat) @@ -49,33 +82,15 @@ pl_DISTRO_ARCH=$(uname -i) # let mkfedora select one of its mirrors pl_DISTRO_URL="" +# the release number (plain number) pl_DISTRO_RELEASE=$(pl_getRelease) -# vserver expects something like fc4 or f7 -# for fedora only as of now -case $pl_DISTRO in - [Ff]edora*) - if [ "$pl_DISTRO_RELEASE" -le 6 ] ; then - pl_DISTRO_NAME=fc$pl_DISTRO_RELEASE - else - pl_DISTRO_NAME=f$pl_DISTRO_RELEASE - fi ;; - [Cc]entOS*) - if [ "$pl_DISTRO_RELEASE" = "4.5" ] ; then - # centos 4.5 is just centos4 + enhancements - pl_DISTRO_NAME=centos4 - else - pl_DISTRO_NAME=centos$pl_DISTRO_RELEASE - fi ;; - *) - pl_DISTRO_NAME="unknown-name-for-${pl_DISTRO}-please-edit-build.common" - echo 1>&2 "build.common: WARNING - pl_DISTRO_NAME not set for distro=$pl_DISTRO" - ;; -esac - -# get patch to appropriate yumgroups.xml file +# the release name - for vserver build - like fc4, f8 or centos4 +pl_DISTRO_NAME=$(pl_getReleaseName) + +# get path to appropriate yumgroups.xml file # Thierry: quick & dirty improvement -# this file is updated by the toplevel build, from groups/pldistro>.xml +# this file is updated by the toplevel build, from groups/.xml pl_DISTRO_YUMGROUPS="../../../RPMS/yumgroups.xml" function pl_process_fedora_options () { @@ -247,9 +262,8 @@ function pl_fixdirs() { done } -########## .lst format +########## .pkgs format # comments start with a # - this is needed only if you use a keyword in a comment -# lst_parse keyword fcdistro lst1 .. lstn # for a given keyword like 'package' : # we support fcdistro-dependant format, for tokens (pkgname) without whitespace # you can e.g. use @@ -257,10 +271,11 @@ function pl_fixdirs() { # package+f8: pkg1 .. pkgn # package-f8: pkg1 .. pkgn -function lst_parse () { +function pl_parsePkgs () { keyword=$1;shift fcdistro=$1; shift + # remaining arguments are paths to the pkgs files # grab regular descriptions all=$(grep -v '^#' "$@" | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,,") @@ -278,9 +293,36 @@ function lst_parse () { return 0 } -function pl_getPackages2() { fcdistro=$1; shift ; lst_parse package $fcdistro "$@" ; } -function pl_getGroups2() { fcdistro=$1; shift ; lst_parse group $fcdistro "$@" ; } -# add -p before each package for mkfedora -function pl_getPackagesOptions2 () { pl_getPackages2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-p " $i}}' ; } -# add -g before each group for mkfedora -function pl_getGroupsOptions2 () { pl_getGroups2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-g " $i}}' ; } +function pl_getPackages() { fcdistro=$1; shift ; pl_parsePkgs package $fcdistro "$@" ; } +function pl_getGroups() { fcdistro=$1; shift ; pl_parsePkgs group $fcdistro "$@" ; } +### # add -p before each package for mkfedora +### function pl_getPackagesOptions2 () { pl_getPackages2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-p " $i}}' ; } +### # add -g before each group for mkfedora +### function pl_getGroupsOptions2 () { pl_getGroups2 "$@" | awk '{for (i=1;i<=NF;i++) {print "-g " $i}}' ; } + +# locates a pldistro-dependant file +# tries first in build//, then in build/planetlab/ +function pl_locateDistroFile () { + buildir=$1; shift + pldistro=$1; shift + pkgsfile=$1; shift + + pkgspath="" + # locate it + paths="$builddir/config.$pldistro/$pkgsfile $builddir/config.planetlab/$pkgsfile" + for path in $paths; do + if [ -f $path/$pkgsfile ] ; then + pkgspath=$path + break + fi + done + if [ -z "$pkgspath" ] ; then + echo 1>&2 "pl_locateDistroFile: cannot locate $pkgsfile in $builddir" + echo "not-found-by-pl_locateDistroFile" + return 1 + else + echo 1>&2 "pl_locateDistroFile: using $pkgspath" + echo $pkgspath + return 0 + fi +} diff --git a/config.planetlab/bootcd.pkgs b/config.planetlab/bootcd.pkgs new file mode 100644 index 00000000..f5147148 --- /dev/null +++ b/config.planetlab/bootcd.pkgs @@ -0,0 +1,74 @@ +# from prep.sh, was hardwired as packagelist +package: filesystem +package: udev +package: dhclient +package: bash +package: coreutils +package: iputils +package: kernel +package: bzip2 +package: diffutils +package: logrotate +package: passwd +#package: rsh +package: rsync +package: sudo +package: tcpdump +package: telnet +package: traceroute +package: time +package: wget +package: yum +package: curl +package: gzip +package: python +package: tar +package: pciutils +package: kbd +package: authconfig +package: hdparm +package: lvm2 +package: kexec-tools +package: gnupg +package: nano +package: parted +package: pyparted +package: openssh-server +package: openssh-clients +package: ncftp +package: dosfstools +package: dos2unix +package: bind-utils +package: sharutils +package: vconfig +package: ntp +package: pypcilib +package: openvpn +#################### formerly in prep.sh +junk: lib/obsolete +junk: lib/tls +junk: usr/share/cracklib +junk: usr/share/emacs +junk: usr/share/gnupg +junk: usr/share/i18n +junk: usr/share/locale +junk: usr/share/terminfo +junk: usr/share/zoneinfo +junk: usr/sbin/build-locale-archive +junk: usr/sbin/dbconverter-2 +junk: usr/sbin/sasl* +junk: usr/sbin/tcpslice +junk: usr/lib/perl* +junk: usr/lib/locale +junk: usr/lib/sasl* +junk: usr/lib/gconv +junk: usr/lib/tls +# +precious: usr/share/i18n/locales/en_US +precious: usr/share/i18n/charmaps/UTF-8.gz +precious: usr/share/locale/en +precious: usr/share/terminfo/l/linux +precious: usr/share/terminfo/v/vt100 +precious: usr/share/terminfo/x/xterm +precious: usr/share/zoneinfo/UTC +precious: usr/lib/locale/en_US.utf8 diff --git a/config.planetlab/bootstrapfs-production.pkgs b/config.planetlab/bootstrapfs-production.pkgs new file mode 100644 index 00000000..114d87bf --- /dev/null +++ b/config.planetlab/bootstrapfs-production.pkgs @@ -0,0 +1,13 @@ +# Install the "PlanetLab" group. This requires that the PlanetLab +# build system install the appropriate yumgroups.xml file (currently +# build/groups/v4_yumgroups.xml) in $RPM_BUILD_DIR/../RPMS/ and that +# mkfedora runs either yum-arch or createrepo on that directory. dev +# is specified explicitly because of a stupid bug in its %post script +# that causes its installation to fail; see the mkfedora script for a +# full explanation. coreutils and python are specified explicitly +# because groupinstall does not honor Requires(pre) dependencies +# properly, most %pre scripts require coreutils to be installed first, +# and some of our %post scripts require python. + +package: python +group: PlanetLab \ No newline at end of file diff --git a/config.planetlab/bootstrapfs.pkgs b/config.planetlab/bootstrapfs.pkgs new file mode 100644 index 00000000..73d42d3f --- /dev/null +++ b/config.planetlab/bootstrapfs.pkgs @@ -0,0 +1,7 @@ +package: bash +package: filesystem +package: udev +package: coreutils +# see former bootmanager/support-files/buildnode.sh +package: python +group: PlanetLab diff --git a/planetlab-devel.lst b/config.planetlab/devel.pkgs similarity index 100% rename from planetlab-devel.lst rename to config.planetlab/devel.pkgs diff --git a/config.planetlab/myplc.pkgs b/config.planetlab/myplc.pkgs new file mode 100644 index 00000000..cdfa0c28 --- /dev/null +++ b/config.planetlab/myplc.pkgs @@ -0,0 +1,56 @@ +# first draft, extracted from plc_config.xml with --packages +#### +# kernel-vserver is intended for the vserver-reference, but +# serves the same useful purpose for MyPLC, namely, to +# Provide: kernel without actually installing anything. +package:kernel-vserver +package:dev +# utilities +package:bzip2 +package:cpio +package:tar +package:rsync +package:diffutils +package:gzip +package:findutils +package:less +package:curl wget +package:vim-minimal +# for mkpasswd +package:expect +# package management +package:rpm +package:yum createrepo +# cron jobs +package:vixie-cron +# sendmail +package:sendmail sendmail-cf +# dns server +package:dnsmasq +# python +package:python python-devel SOAPpy PyXML +# add ons for plcapi +package: python-pycurl python-psycopg2 +# but not on fc4 as we provide ours +package-fc4: python-pycurl python-psycopg2 +# postgresql +package:postgresql postgresql-server postgresql-python +# apache +package:httpd mod_python mod_ssl +# ssl +package:openssl xmlsec1 xmlsec1-openssl +# php +package:php php-pgsql +# apache user needs root access for building node-dependent images +package:sudo +# for ssh-keygen +package:openssh +# ssh - easily reach nodes from the chroot/vserver context +package:openssh-clients +# for ImageCreate +package:gd php-gd perl-GD +# plc-map.py uses this +package:python-imaging +# planetlab stuff +package:bootcd bootmanager bootstrapfs +package:PLCAPI PLCWWW diff --git a/planetlab-shell.lst b/config.planetlab/native-shell.pkgs similarity index 100% rename from planetlab-shell.lst rename to config.planetlab/native-shell.pkgs diff --git a/config.planetlab/vserver-planetflow.pkgs b/config.planetlab/vserver-planetflow.pkgs new file mode 100644 index 00000000..6f7b3ada --- /dev/null +++ b/config.planetlab/vserver-planetflow.pkgs @@ -0,0 +1 @@ +package: netflow diff --git a/config.planetlab/vserver.pkgs b/config.planetlab/vserver.pkgs new file mode 100644 index 00000000..5f90b72c --- /dev/null +++ b/config.planetlab/vserver.pkgs @@ -0,0 +1,29 @@ +# define packages +package: bash +package: coreutils +package: iputils +package: kernel-vserver +package: bzip2 +package: crontabs +package: diffutils +package: logrotate +package: openssh-clients +package: passwd +package: rsh +package: rsync +package: sudo +package: tcpdump +package: telnet +package: traceroute +package: time +package: vixie-cron +package: wget +package: which +package: yum +package: curl +package: gzip +package: perl +package: python +package: tar +package: findutils +package: filesystem diff --git a/mkfedora b/mkfedora index dd46179a..468b9d3d 100755 --- a/mkfedora +++ b/mkfedora @@ -16,6 +16,9 @@ # $Id$ # +# for pkgs parsing functions +. build.common + export PATH=/sbin:/bin:/usr/sbin:/usr/bin echo "* Entering mkfedora " "$@" @@ -35,7 +38,7 @@ ftp://rpmfind.net/linux/fedora ) # Release and architecture to install -releasever=4 +releasever=7 basearch=i386 # Yum groups to install @@ -68,6 +71,7 @@ usage() done echo " -r release Fedora release number (default: $releasever)" echo " -a arch Fedora architecture (default: $basearch)" + echo " -f pkgsfile use this pkgs file for packages, groups, excludes.." echo " -g group1 -g group2 ..." echo " Yumgroups to install (default: none)" echo " -p package1 -p package2 ..." @@ -105,6 +109,9 @@ while getopts "l:r:a:g:p:x:kvh" opt ; do x) exclude[${#exclude[*]}]="$OPTARG" ;; + f) + pkgsfile=$OPTARG + ;; k) exclude_kernel="exclude=kernel* ulogd iptables" ;; @@ -331,6 +338,23 @@ baseurl=file://$RPM_RPMS_DIR/ EOF fi +# pkgs file +if [ -n "$pkgsfile" ] ; then + # parse pkgsfile and add to local vars + fcdistro=$(pl_getReleaseName "Fedora" $releasever) + for i in $(pl_parsePkgs package $fcdistro $pkgsfile) ; do + packages[${#packages[*]}]="$i" + done + for i in $(pl_parsePkgs group $fcdistro $pkgsfile) ; do + groups[${#groups[*]}]="$i" + done + for i in $(pl_parsePkgs exclude $fcdistro $pkgsfile) ; do + exclude[${#exclude[*]}]="$i" + done + junk=$(pl_parsePkgs junk $fcdistro $pkgsfile) + precious=$(pl_parsePkgs precious $fcdistro $pkgsfile) +fi + excludes= for package in "${exclude[@]}" ; do excludes="$excludes --exclude=$package" @@ -391,6 +415,22 @@ if [ -f $vroot/usr/share/zoneinfo/UTC -a -f $vroot/etc/localtime ] ; then ln -s /usr/share/zoneinfo/UTC $vroot/etc/localtime fi +# formerly in bootcd/prep.sh : to optimize footprint +echo "* Removing unnecessary junk" >&3 + +pushd $vroot + +# Save precious files +tar --ignore-failed-read -cpf precious.tar $precious + +# Remove unnecessary junk +rm -rf $junk + +# Restore precious files +tar -xpf precious.tar +rm -f precious.tar + +popd # remove trap handler, as we are about to call it directly. trap - ERR INT diff --git a/vbuild-init-vserver.sh b/vbuild-init-vserver.sh index 409af154..ddf2f124 100755 --- a/vbuild-init-vserver.sh +++ b/vbuild-init-vserver.sh @@ -4,7 +4,7 @@ COMMAND=$(basename $0) DIRNAME=$(dirname $0) -# lst parsing utilities +# pkgs parsing utilities PATH=$(dirname $0):$PATH . build.common DEFAULT_FCDISTRO=f7 @@ -86,11 +86,7 @@ function setup_vserver () { fi # create it - options="" - [ -n "$V_IPADDR" ] && options="$options --interface $V_IPADDR" - [ -n "$V_HOSTNAME" ] && options="$options --hostname $V_HOSTNAME" - [ -n "$V_NET_DEV" ] && options="$options --netdev $V_NET_DEV" - $personality vserver $VERBOSE $vserver build $options -m yum -- -d $fcdistro + $personality vserver $VERBOSE $vserver build $VSERVER_OPTIONS -m yum -- -d $fcdistro if [ ! -z "$personality" ] ; then registered_personality=$(grep $personality /etc/vservers/$vserver/personality | wc -l) @@ -137,25 +133,17 @@ function devel_tools () { pldistro=$1; shift personality=$1; shift - # check for .lst file based on pldistro + # check for .pkgs file based on pldistro if [ -n "$VBUILD_MODE" ] ; then - lst=${pldistro}-devel.lst + pkgsname=devel.pkgs else - lst=${pldistro}-shell.lst - fi - if [ -f $lst ] ; then - echo "$COMMAND: Using $lst" - elif [ -f $DIRNAME/$lst ] ; then - lst=$DIRNAME/$lst - echo "$COMMAND: Using $lst" - else - echo "$COMMAND : Cannot locate $lst - exiting" - usage + pkgsname=native-shell.pkgs fi + pkgsfile=$(pl_locateDistroFile $(DIRNAME) $pldistro $pkgsname) # install individual packages, then groups - packages=$(pl_getPackages2 ${fcdistro} $lst) - groups=$(pl_getGroups2 ${fcdistro} $lst) + packages=$(pl_getPackages ${fcdistro} $pkgsfile) + groups=$(pl_getGroups ${fcdistro} $pkgsfile) [ -n "$packages" ] && $personality vserver $vserver exec yum -y install $packages [ -n "$groups" ] && $personality vserver $vserver exec yum -y groupinstall $groups @@ -253,8 +241,8 @@ COMMAND_VBUILD="vbuild-init-vserver.sh" COMMAND_MYPLC="myplc-init-vserver.sh" function usage () { set +x - echo "Usage: $COMMAND_VBUILD [-v] vserver-name" - echo "Usage: $COMMAND_MYPLC [-v] vserver-name repo-url" + echo "Usage: $COMMAND_VBUILD [-v] vserver-name [ -- vserver-options ]" + echo "Usage: $COMMAND_MYPLC [-v] vserver-name repo-url [ -- vserver-options ]" echo "Requirements: you need to have a vserver-compliant kernel," echo " as well as the util-vserver RPM installed" echo "Description:" @@ -264,10 +252,6 @@ function usage () { echo " -d pldistro - defaults to $DEFAULT_PLDISTRO" echo " -p personality - defaults to $DEFAULT_PERSONALITY" echo " -v : passes -v to calls to vserver" - echo "$COMMAND_MYPLC only:" - echo " -h hostname: passed as vserver build --hostname" - echo " -i ip-address: passed as vserver build --interface" - echo " -e dev: passed as vserver build --netdev" exit 1 } @@ -287,27 +271,35 @@ function main () { esac VERBOSE= - while getopts "f:d:p:vu:h:i:e:" opt ; do + while getopts "f:d:p:v" opt ; do case $opt in f) fcdistro=$OPTARG;; d) pldistro=$OPTARG;; p) personality=$OPTARG;; - i) [ -z "$MYPLC_MODE" ] && usage ; V_IPADDR=$OPTARG;; - h) [ -z "$MYPLC_MODE" ] && usage ; V_HOSTNAME=$OPTARG;; - e) [ -z "$MYPLC_MODE" ] && usage ; V_NET_DEV==$OPTARG;; v) VERBOSE="-v" ;; *) usage ;; esac done shift $(($OPTIND - 1)) + + # parse fixed arguments [[ -z "$@" ]] && usage vserver=$1 ; shift if [ -n "$MYPLC_MODE" ] ; then [[ -z "$@" ]] && usage REPO_URL=$1 ; shift fi - [[ -n "$@" ]] && usage + + # parse vserver options + if [[ -n "$@" ]] ; then + if [ "$1" == "--" ] ; then + shift + VSERVER_OPTIONS="$@" + else + usage + fi + fi [ -z "$fcdistro" ] && fcdistro=$DEFAULT_FCDISTRO [ -z "$pldistro" ] && pldistro=$DEFAULT_PLDISTRO -- 2.43.0