From: Thierry Parmentelat Date: Sat, 26 Dec 2009 16:04:52 +0000 (+0000) Subject: a new format for pkgs files X-Git-Tag: coblitz-2.01-rc8~15 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;ds=sidebyside;h=201d0f864916b35c797b68e0a295033132e71045;p=build.git a new format for pkgs files --- diff --git a/build.common b/build.common index 3d5c703e..74683c4f 100644 --- a/build.common +++ b/build.common @@ -222,8 +222,6 @@ function pl_root_mkfedora () { fcdistro=${pl_DISTRO_NAME} pkgs_packages=$(pl_parsePkgs package $fcdistro $pldistro $pkgsfile) pkgs_groups=$(pl_parsePkgs group $fcdistro $pldistro $pkgsfile) - # packages to exclude - pkgs_excludes=$(pl_parsePkgs exclude $fcdistro $pldistro $pkgsfile) # what can get trashed to save space pkgs_junk=$(pl_parsePkgs junk $fcdistro $pldistro $pkgsfile) # but not this @@ -238,13 +236,7 @@ function pl_root_mkfedora () { fi kexclude_line="" - # add them manually as the output of pl_parsePkgs is line-separated - if [ -n "$pkgs_kexcludes" ] ; then - kexclude_line="exclude=" - for kexclude in $pkgs_kexcludes ; do - kexclude_line="$kexclude_line $kexclude" - done - fi + [ -n "$pkgs_kexcludes" ] && kexclude_line="exclude=$pkgs_kexcludes" echo "$0: candidate mirrors" for mirror in $mirrors ; do @@ -366,22 +358,17 @@ fi yum_options="$yum_options -c $yum_conf" yum_options="$yum_options --installroot=$vroot" - exclude_arg="" - for exclude in $pkgs_excludes; do - exclude_arg="$exclude_arg --exclude $exclude" - done - # glibc must be specified explicitly for the correct arch to be # chosen. echo "* Installing glibc" # ignore yum's return code that is basically undefined - yum $yum_options $exclude_arg install glibc || : + yum $yum_options install glibc || : # Go, baby, go if [ -n "$pkgs_packages" ] ; then echo "* Installing optional packages" $pkgs_packages # ignore yum's return code that is basically undefined - yum $yum_options $exclude_arg install $pkgs_packages || : + yum $yum_options install $pkgs_packages || : if ! rpm --root $vroot -q $pkgs_packages >/dev/null ; then echo "* Warning: Missing packages" rpm --root $vroot -q $pkgs_packages | grep "not installed" @@ -394,7 +381,7 @@ fi group=$(echo $group_plus | sed -e "s,+++, ,g") echo "* Installing optional group $group" # ignore yum's return code that is basically undefined - yum $yum_options $exclude_arg groupinstall "$group" || : + yum $yum_options groupinstall "$group" || : done fi @@ -520,15 +507,23 @@ function pl_fixdirs() { } ########## .pkgs format -# comments start with a # - this is needed only if you use a keyword in a comment +# Usage: pl_parsePkgs keyword [-a arch] fcdistro pldistro pkgs-file[..s] +# pkgs.py sohuld be found in PATH, like this file build.common +function pl_parsePkgs () { + target_arch=$pl_DISTRO_ARCH + keyword=$1;shift + [ "$1" == "-a" ] && { shift; target_arch="$1"; shift; } + fcdistro=$1; shift + pldistro=$1; shift -function pl_getPkgsAttribute () { - keyword=$1; shift - file=$1; shift - # remove any initial white spaces from the result - grep -v '^#' $file | grep --regexp="^${keyword}:" | sed -e "s,${keyword}:,," -e "s,^[[:space:]][[:space:]]*,," + pkgs.py -a $target_arch -f $fcdistro -d $pldistro $keyword "$@" } +# usage: pl_getPackages [-a arch] fcdistro pldistro pkg-file[..s] +function pl_getPackages() { pl_parsePkgs package "$@" ; } +function pl_getGroups() { pl_parsePkgs group "$@" ; } +############################## +### temporary - only for checking the new python version # for a given keyword like 'package' : # we support fcdistro-dependant format, for tokens (pkgname) without whitespace # you can e.g. use @@ -538,12 +533,11 @@ function pl_getPkgsAttribute () { # # values can contain @arch@, @fcdistro@ or @pldistro@ that are replaced with the current values # -# Usage: pl_parsePkgs keyword [-a arch] fcdistro pldistro pkgs-file[..s] # the reason for the -a option is for when we build the build vserver itself; in this case # pl_DISTRO_ARCH is the one we obtain from the root context, and that's wrong # specify -sa arch AFTER keyword, so as to keep pl_getPackages and pl_getGroups simple # -function pl_parsePkgs () { +function pl_parsePkgs_old () { target_arch=$pl_DISTRO_ARCH keyword=$1;shift @@ -567,10 +561,7 @@ function pl_parsePkgs () { done return 0 } - -# usage: pl_getPackages [-a arch] fcdistro pldistro pkg-file[..s] -function pl_getPackages() { pl_parsePkgs package "$@" ; } -function pl_getGroups() { pl_parsePkgs group "$@" ; } +############################## # locates a pldistro-dependant file # tries first in build//, then in build/planetlab/ @@ -623,8 +614,8 @@ __header pkgsfile=$(pl_locateDistroFile $builddir $pldistro $pkgsname) packages=$(pl_getPackages $fcdistro $pldistro $pkgsfile) - groupname=$(pl_getPkgsAttribute groupname $pkgsfile | sed $sedargs) - groupdesc=$(pl_getPkgsAttribute groupdesc $pkgsfile | sed $sedargs) + groupname=$(pkgs.py groupname $pkgsfile | sed $sedargs) + groupdesc=$(pkgs.py groupdesc $pkgsfile | sed $sedargs) if [ -z "$groupname" -o -z "$groupdesc" ] ; then echo "Cannot find groupname: and groupdesc: in $pkgsfile -- skipped" 1>&2 diff --git a/config.coblitz/bootstrapfs.pkgs b/config.coblitz/bootstrapfs.pkgs index f4926c0e..6d6f7f20 100644 --- a/config.coblitz/bootstrapfs.pkgs +++ b/config.coblitz/bootstrapfs.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# # Keeping the old name as it might be referenced elsewhere (node update..) # groupname: Bootstrapfs groupname: PlanetLab @@ -78,22 +81,16 @@ package: openvpn package: vconfig package: PyXML -package: termcap libtermcap -package: vixie-cron - # # platform-dependent # -package: rsyslog -package-centos5: rsyslog -package+centos5: sysklogd +package>=f8: rsyslog +package=centos5: sysklogd -package: util-linux-ng -package-centos5: util-linux-ng -package+centos5: util-linux +package>=f8: util-linux-ng +package=centos5: util-linux -package-f10: termcap libtermcap -package-f10: vixie-cron +package<=f8 package=centos5: termcap libtermcap vixie-cron # # planetlab # diff --git a/config.coblitz/devel.pkgs b/config.coblitz/devel.pkgs index 38695c25..8a3c807b 100644 --- a/config.coblitz/devel.pkgs +++ b/config.coblitz/devel.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# groupname: PlanetLabDevel groupdesc: Building PlanetLab # general utilities @@ -37,7 +40,7 @@ package: ncurses-devel package: readline-devel package: dnsmasq # for spec2make / rpmlib -package+f8: popt-devel +package<=f8: popt-devel # kernel package: gnupg package: diffutils @@ -79,16 +82,14 @@ package: pciutils-devel # for vsys - same as for php-devel above package: ocaml.@arch@ ocaml-docs # use local inotify-tools on centos -package:inotify-tools-devel -package-centos5: inotify-tools-devel +package>=f8:inotify-tools-devel ########## ## for util-vserver-pl # use local libnl on centos -package: libnl-devel -package-centos5: libnl-devel +package>=f8: libnl-devel # zabbix/monitor package: net-snmp net-snmp-devel -package+centos5: krb5-devel.@arch@ e2fsprogs-devel.@arch@ libidn-devel.@arch@ +package>=centos5: krb5-devel.@arch@ e2fsprogs-devel.@arch@ libidn-devel.@arch@ package: python-cherrypy ########## # for sfa : rebuilding wsdl index at build-time diff --git a/config.f12/devel.pkgs b/config.f12/devel.pkgs index 0b37aac7..a53aeb62 100644 --- a/config.f12/devel.pkgs +++ b/config.f12/devel.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# groupname: PlanetLabDevel groupdesc: Building PlanetLab # general utilities diff --git a/config.onelab/bootstrapfs.pkgs b/config.onelab/bootstrapfs.pkgs index 7b0e1a46..e44d0ec3 100644 --- a/config.onelab/bootstrapfs.pkgs +++ b/config.onelab/bootstrapfs.pkgs @@ -82,22 +82,16 @@ package: openvpn package: vconfig package: PyXML -package: termcap libtermcap -package: vixie-cron - # # platform-dependent # -package: rsyslog -package-centos5: rsyslog -package+centos5: sysklogd +package>=f8: rsyslog +package=centos5: sysklogd -package: util-linux-ng -package-centos5: util-linux-ng -package+centos5: util-linux +package>=f8: util-linux-ng +package=centos5: util-linux -package-f10: termcap libtermcap -package-f10: vixie-cron +package<=f8 package=centos5: termcap libtermcap vixie-cron # for nagios package: nrpe diff --git a/config.onelab/vserver.pkgs b/config.onelab/vserver.pkgs index 7ce901be..d0cf0cc6 100644 --- a/config.onelab/vserver.pkgs +++ b/config.onelab/vserver.pkgs @@ -34,7 +34,7 @@ package: python-devel package: tar package: findutils package: filesystem -package: vixie-cron -package-f10: vixie-cron package: planetlab-umts-tools-frontend package: ipfw-slice + +package<=f8 package=centos5: vixie-cron diff --git a/config.planetbridge/bootstrapfs.pkgs b/config.planetbridge/bootstrapfs.pkgs index 591b45ed..73b28043 100644 --- a/config.planetbridge/bootstrapfs.pkgs +++ b/config.planetbridge/bootstrapfs.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# # Keeping the old name as it might be referenced elsewhere (node update..) # groupname: Bootstrapfs groupname: PlanetLab @@ -79,22 +82,17 @@ package: openvpn package: vconfig package: PyXML -package: termcap libtermcap -package: vixie-cron # # platform-dependent # -package: rsyslog -package-centos5: rsyslog -package+centos5: sysklogd +package>=f8: rsyslog +package>=centos5: sysklogd -package: util-linux-ng -package-centos5: util-linux-ng -package+centos5: util-linux +package>=f8: util-linux-ng +package>=centos5: util-linux -package-f10: termcap libtermcap -package-f10: vixie-cron +package<=f8 package<=centos5: termcap libtermcap vixie-cron # # planetlab # diff --git a/config.planetlab/bootcd.pkgs b/config.planetlab/bootcd.pkgs index b670023f..73af53ae 100644 --- a/config.planetlab/bootcd.pkgs +++ b/config.planetlab/bootcd.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# groupname: PlanetLabBootcd groupdesc: PlanetLab BootCD # packages to exclude from stock repositories @@ -74,5 +77,4 @@ precious: usr/share/terminfo/l/linux precious: usr/share/terminfo/v/vt100 precious: usr/share/terminfo/x/xterm precious: usr/share/zoneinfo/UTC -precious+f8: usr/lib/locale/en_US.utf8 -precious+centos5: usr/lib/locale/en_US.utf8 +precious<=f8 precious<=centos5: usr/lib/locale/en_US.utf8 diff --git a/config.planetlab/bootstrapfs.pkgs b/config.planetlab/bootstrapfs.pkgs index 591b45ed..80dfc8ff 100644 --- a/config.planetlab/bootstrapfs.pkgs +++ b/config.planetlab/bootstrapfs.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# # Keeping the old name as it might be referenced elsewhere (node update..) # groupname: Bootstrapfs groupname: PlanetLab @@ -79,22 +82,17 @@ package: openvpn package: vconfig package: PyXML -package: termcap libtermcap -package: vixie-cron - # # platform-dependent # -package: rsyslog -package-centos5: rsyslog -package+centos5: sysklogd +package>=f8: rsyslog +package=centos5: sysklogd + +package>=f8: util-linux-ng +package=centos5: util-linux -package: util-linux-ng -package-centos5: util-linux-ng -package+centos5: util-linux +package<=f8 package=centos5: termcap libtermcap vixie-cron -package-f10: termcap libtermcap -package-f10: vixie-cron # # planetlab # diff --git a/config.planetlab/devel.pkgs b/config.planetlab/devel.pkgs index 4f22a92e..de40c9c8 100644 --- a/config.planetlab/devel.pkgs +++ b/config.planetlab/devel.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# groupname: PlanetLabDevel groupdesc: Building PlanetLab # general utilities @@ -37,7 +40,7 @@ package: ncurses-devel package: readline-devel package: dnsmasq # for spec2make / rpmlib -package+f8: popt-devel +package=f8: popt-devel # kernel package: gnupg package: diffutils @@ -72,7 +75,7 @@ package: postgresql postgresql-devel postgresql-python postgresql-server # in an undetermined order, and php-config --extension-dir might return the wrong place package: php php-devel.@arch@ php-gd php-pgsql # now this seems to have been fixed in f12 -package+f12: php-devel +package>=f12: php-devel package: PyXML # used to reference SOAPpy as well # for pypcilib @@ -81,19 +84,17 @@ package: pciutils-devel # for vsys - same as for php-devel above package: ocaml.@arch@ ocaml-docs # use local inotify-tools on centos -package:inotify-tools-devel -package-centos5: inotify-tools-devel +package>=f8:inotify-tools-devel ########## ## for util-vserver-pl # use local libnl on centos -package: libnl-devel -package-centos5: libnl-devel +package>=f8: libnl-devel # for util-vserver package: ctags -package+f12: e2fsprogs-devel +package>=f12: e2fsprogs-devel # zabbix/monitor package: net-snmp net-snmp-devel -package+centos5: krb5-devel.@arch@ e2fsprogs-devel.@arch@ libidn-devel.@arch@ +package=centos5: krb5-devel.@arch@ e2fsprogs-devel.@arch@ libidn-devel.@arch@ package: python-cherrypy ########## # for sfa : rebuilding wsdl index at build-time @@ -101,4 +102,3 @@ package: python-uuid pyOpenSSL m2crypto package: libxml2-python libxslt-python package: python-ZSI package: python-psycopg2 - diff --git a/config.planetlab/vserver.pkgs b/config.planetlab/vserver.pkgs index c114ef8e..c803bc49 100644 --- a/config.planetlab/vserver.pkgs +++ b/config.planetlab/vserver.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# groupname: PlanetLabSlice groupdesc: PlanetLab Slice Reference Image # packages to exclude from stock repositories @@ -31,5 +34,4 @@ package: python-devel package: tar package: findutils package: filesystem -package: vixie-cron -package-f10: vixie-cron +package<=f8 package=centos5: vixie-cron diff --git a/config.planetlab/vtest.pkgs b/config.planetlab/vtest.pkgs index 19ba1aab..d661ea9a 100644 --- a/config.planetlab/vtest.pkgs +++ b/config.planetlab/vtest.pkgs @@ -1,3 +1,6 @@ +# $Id$ +# $URL$ +# groupname: PlanetLabNative groupdesc: Test Vserver for MyPlc Native package: openssh-clients curl diff --git a/pkgs.py b/pkgs.py new file mode 100755 index 00000000..d31e57d2 --- /dev/null +++ b/pkgs.py @@ -0,0 +1,222 @@ +#!/usr/bin/python +# +# $Id$ +# $URL$ +# +# This is a replacement for the formerly bash-written function pl_parsePkgs () +# +# Usage: $0 [-a arch] default_arch keyword fcdistro pldistro pkgs-file[..s] +# default_arch is $pl_DISTRO_ARCH, but can be overridden +# + +#################### original language was (in this example, keyword=package) +## to add to all distros +# package: p1 p2 +## to add in one distro +# package+f12: p1 p2 +## to remove in one distro +# package-f10: p1 p2 +#################### replacement language +## add in distro f10 +# +package=f10: p1 p2 +# or simply +# package=f10: p1 p2 +## add in fedora distros starting with f10 +# +package>=f10: p1 p2 +# or simply +# package>=f10: p1 p2 +## ditto but remove instead +# -package=centos5: p1 p2 +# -package<=centos5: p1 p2 + +import sys +from sys import stderr +from optparse import OptionParser +import re + +default_arch='x86_64' +known_arch = ['i386','x86_64'] +default_fcdistro='f12' +known_fcdistros = [ 'centos5','centos6','f8', 'f9','f10','f11','f12', 'f13' ] +default_pldistro='onelab' + +known_keywords=['groupname', 'groupdesc', 'kexclude', 'package', 'group', 'precious', 'junk', 'mirror', ] + + +m_fcdistro_cutter = re.compile('([a-z]+)([0-9]+)') +re_ident='[a-z]+' + +class PkgsParser: + + def __init__ (self,arch,fcdistro,pldistro,keyword,inputs,options): + self.arch=arch + self.fcdistro=fcdistro + self.pldistro=pldistro + self.keyword=keyword + self.inputs=inputs + # for verbose, new_line, and the like + self.options=options + ok=False + for known in known_fcdistros: + if fcdistro == known: + (distro,version)=m_fcdistro_cutter.match(fcdistro).groups() + ok=True + if ok: + self.distro=distro + self.version=int(version) + else: + print >> stderr, 'unrecognized fcdistro', fcdistro + sys.exit(1) + + # qualifier is either '>=','<=', or '=' + def match (self, qualifier, version): + if qualifier == '=': + return self.version == version + elif qualifier == '>=': + return self.version >= version + elif qualifier == '<=': + return self.version <= version + else: + raise Exception, 'Internal error - unexpected qualifier %r' % qualifier + + m_comment=re.compile('\A\s*#') + m_blank=re.compile('\A\s*\Z') + + m_ident=re.compile('\A'+re_ident+'\Z') + re_qualified = '\s*' + re_qualified += '(?P[+-]?)' + re_qualified += '\s*' + re_qualified += '(?P%s)'%re_ident + re_qualified += '\s*' + re_qualified += '(?P>=|<=|=)' + re_qualified += '\s*' + re_qualified += '(?P%s[0-9]+)'%re_ident + re_qualified += '\s*' + m_qualified = re.compile('\A%s\Z'%re_qualified) + + re_old = '[a-z]+[+-][a-z]+[0-9]+' + m_old = re.compile ('\A%s\Z'%re_old) + + # returns a tuple (included,excluded) + def parse (self,filename): + ok=True + included=[] + excluded=[] + lineno=0 + try: + for line in file(filename).readlines(): + lineno += 1 + line=line.strip() + if self.m_comment.match(line) or self.m_blank.match(line): + continue + try: + [lefts,rights] = line.split(':') + for left in lefts.split(): + ########## single ident + if self.m_ident.match(left): + if left not in known_keywords: + raise Exception,"Unknown keyword %r"%left + elif left == self.keyword: + included += rights.split() + else: + m=self.m_qualified.match(left) + if m: + (plus_minus,kw,qual,fcdistro) = m.groups() + if kw not in known_keywords: + raise Exception,"Unknown keyword in %r"%left + if fcdistro not in known_fcdistros: + raise Exception, 'Unknown fcdistro %r'%fcdistro + # skip if another keyword + if kw != self.keyword: continue + # does this fcdistro match ? + (distro,version)=m_fcdistro_cutter.match(fcdistro).groups() + version = int (version) + # skip if another distro family + if distro != self.distro: continue + # skip if the qualifier does not fit + if not self.match (qual, version): + if self.options.verbose: print >> stderr,'%s:%d:qualifer %s does not apply'%(filename,lineno,left) + continue + # we're in, let's add (default) or remove (if plus_minus is minus) + if plus_minus == '-': + if self.options.verbose: print >> stderr,'%s:%d: from %s, excluding %r'%(filename,lineno,left,rights) + excluded += rights.split() + else: + if self.options.verbose: print >> stderr,'%s:%d: from %s, including %r'%(filename,lineno,left,rights) + included += rights.split() + elif self.m_old.match(left): + raise Exception,'Old-fashioned syntax not supported anymore %r'%left + else: + raise Exception,'error in left expression %r'%left + + except Exception,e: + ok=False + print >> stderr, "%s:%d:syntax error: %r"%(filename,lineno,e) + except Exception,e: + ok=False + print >> stderr, 'Could not parse file',filename,e + return (ok,included,excluded) + + def run (self): + ok=True + included=[] + excluded=[] + for input in self.inputs: + (o,i,e) = self.parse (input) + included += i + excluded += e + ok = ok and o + results = list (set(included).difference(set(excluded))) + + results = [ x.replace('@arch@',self.arch).\ + replace('@fcdistro@',self.fcdistro).\ + replace('@pldistro@',self.pldistro) for x in results] + results.sort() + # default is space-separated + if not self.options.new_line: + print " ".join(results) + # but for tests results are printed each on a line + else: + for result in results : print result + return ok + +def main (): + usage="Usage: %prog [options] keyword input[...]" + parser=OptionParser (usage=usage) + parser.add_option ('-a','--arch',dest='arch',action='store',default=default_arch, + help='target arch, e.g. i386 or x86_64') + parser.add_option ('-f','--fcdistro',dest='fcdistro',action='store', default=default_fcdistro, + help='fcdistro, e.g. f12 or centos5') + parser.add_option ('-d','--pldistro',dest='pldistro',action='store', default=default_pldistro, + help='pldistro, e.g. onelab or planetlab') + parser.add_option ('-v', '--verbose',dest='verbose',action='store_true',default=False, + help='verbose when using qualifiers') + parser.add_option ('-n', '--new-line',dest='new_line',action='store_true',default=False, + help='print outputs separated with newlines rather than with a space') + (options,args) = parser.parse_args() + + if len(args) <=1 : + parser.print_help(file=stderr) + sys.exit(1) + keyword=args[0] + inputs=args[1:] + if not options.arch in known_arch: + print >> stderr, 'Unsupported arch',options.arch + parser.print_help(file=stderr) + sys.exit(1) + if not options.fcdistro in known_fcdistros: + print >> stderr, 'Unsupported fcdistro',options.fcdistro + parser.print_help(file=stderr) + sys.exit(1) + + pkgs = PkgsParser (options.arch,options.fcdistro,options.pldistro,keyword,inputs,options) + + pkgs.run() + + sys.exit(0) + +if __name__ == '__main__': + if main(): + sys.exit(0) + else: + sys.exit(1)