no longer build myplc-docs

[build.git] / pkgs.py

diff --git a/pkgs.py b/pkgs.py
index 495369b..4ba02af 100755 (executable)
--- a/pkgs.py
+++ b/pkgs.py
@@ -1,7 +1,7 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
+#
+# This is a replacement for the formerly bash-written function pl_parsePkgs ()

 #
 #

-# This is a replacement for the formerly bash-written function pl_parsePkgs () 
-# 

 # Usage: $0  [-a arch] default_arch keyword fcdistro pldistro pkgs-file[..s]
 # default_arch is $pl_DISTRO_ARCH, but can be overridden
 #
 # Usage: $0  [-a arch] default_arch keyword fcdistro pldistro pkgs-file[..s]
 # default_arch is $pl_DISTRO_ARCH, but can be overridden
 #


@@ -26,49 +26,80 @@
 # -package=centos5: p1 p2
 # -package<=centos5: p1 p2
 
 # -package=centos5: p1 p2
 # -package<=centos5: p1 p2
 

+# pylint: disable=c0111
+
+

 import sys
 from sys import stderr
 from optparse import OptionParser
 import re
 
 import sys
 from sys import stderr
 from optparse import OptionParser
 import re
 

-default_arch='x86_64'
-known_arch = ['i386','x86_64']
-default_fcdistro='f12'
-known_fcdistros = [ 'centos5','centos6','f8', 'f9','f10','f11','f12', 'f13', 'f14', 'f16', 'sl6']
-default_pldistro='onelab'
+default_arch = 'x86_64'
+known_archs = ['i386', 'i686', 'x86_64']
+default_fcdistro = 'f43'
+known_fcdistros = [
+    'centos5', 'centos6',
+    # oldies but we have references to that in the pkgs files
+    'f8', 'f10', 'f12', 'f14', 'f16', 'f18',
+    'f20', 'f21', 'f22', 'f23', 'f24', 'f25', 'f27',
+    # these ones are still relevant;
+    # f32 is mentioned to be able to use create-vms with that distro
+    # as we're running into issues to build a minimal f33 from a f29 host
+    'f29', 'f31', 'f32', 'f33', 'f35', 'f37', 'f39', 'f41', 'f43',
+    # scientific linux
+    'sl6',
+    # debians
+    'wheezy', 'jessie',
+    # ubuntus
+    'trusty',  # 14.04 LTS
+    'xenial',  # 16.04 LTS
+    'bionic',  # 18.04 LTS
+    'focal',   # 20.04 LTS
+    'jammy',   # 22.04 LTS
+    'noble',   # 24.04 LTS
+]
+default_pldistro = 'onelab'

 
 

-known_keywords=['groupname', 'groupdesc', 
-                'nodeyumexclude', 'plcyumexclude', 
-                'yumexclude', 'package', 'group', 'precious', 'junk', 'mirror', ]
+known_keywords = [
+    'group', 'groupname', 'groupdesc',
+    'package', 'pip', 'gem',
+    'nodeyumexclude', 'plcyumexclude', 'yumexclude',
+    'precious', 'junk', 'mirror',
+]

 
 
 m_fcdistro_cutter = re.compile('([a-z]+)([0-9]+)')
 
 
 m_fcdistro_cutter = re.compile('([a-z]+)([0-9]+)')

-re_ident='[a-z]+'
+re_ident = '[a-z]+'

 
 class PkgsParser:
 
 
 class PkgsParser:
 

-    def __init__ (self,arch,fcdistro,pldistro,keyword,inputs,options):
-        self.arch=arch
-        self.fcdistro=fcdistro
-        self.pldistro=pldistro
-        self.keyword=keyword
-        self.inputs=inputs
+    def __init__(self, arch, fcdistro, pldistro, keyword, inputs, options):
+        self.arch = arch
+        self.fcdistro = fcdistro
+        self.pldistro = pldistro
+        self.keyword = keyword
+        self.inputs = inputs

         # for verbose, new_line, and the like
         # for verbose, new_line, and the like

-        self.options=options
-        ok=False
+        self.options = options
+        ok = False

         for known in known_fcdistros:
             if fcdistro == known:
         for known in known_fcdistros:
             if fcdistro == known:

-                (distro,version)=m_fcdistro_cutter.match(fcdistro).groups()
-                ok=True
+                try:
+                    (distro, version) = m_fcdistro_cutter.match(fcdistro).groups()
+                # debian-like names can't use numbering
+                except:
+                    distro = fcdistro
+                    version = 0
+                ok = True

         if ok:
         if ok:

-            self.distro=distro
-            self.version=int(version)
+            self.distro = distro
+            self.version = int(version)

         else:
         else:

-            print >> stderr, 'unrecognized fcdistro', fcdistro
+            print('unrecognized fcdistro', fcdistro, file=stderr)

             sys.exit(1)
 
     # qualifier is either '>=','<=', or '='
             sys.exit(1)
 
     # qualifier is either '>=','<=', or '='

-    def match (self, qualifier, version):
+    def match(self, qualifier, version):

         if qualifier == '=':
             return self.version == version
         elif qualifier == '>=':
         if qualifier == '=':
             return self.version == version
         elif qualifier == '>=':


@@ -76,143 +107,159 @@ class PkgsParser:
         elif qualifier == '<=':
             return self.version <= version
         else:
         elif qualifier == '<=':
             return self.version <= version
         else:

-            raise Exception, 'Internal error - unexpected qualifier %r' % qualifier
-
-    m_comment=re.compile('\A\s*#')
-    m_blank=re.compile('\A\s*\Z')
-
-    m_ident=re.compile('\A'+re_ident+'\Z')
-    re_qualified = '\s*'
-    re_qualified += '(?P<plus_minus>[+-]?)'
-    re_qualified += '\s*'
-    re_qualified += '(?P<keyword>%s)'%re_ident
-    re_qualified += '\s*'
-    re_qualified += '(?P<qualifier>>=|<=|=)'
-    re_qualified += '\s*'
-    re_qualified += '(?P<fcdistro>%s[0-9]+)'%re_ident
-    re_qualified += '\s*'
-    m_qualified = re.compile('\A%s\Z'%re_qualified)
+            raise Exception(
+                f'Internal error - unexpected qualifier {qualifier}')
+
+    m_comment = re.compile(r'\A\s*#')
+    m_blank = re.compile(r'\A\s*\Z')
+
+    m_ident = re.compile(r'\A'+re_ident+r'\Z')
+    re_qualified = r'\s*'
+    re_qualified += rf'(?P<plus_minus>[+-]?)'
+    re_qualified += rf'\s*'
+    re_qualified += rf'(?P<keyword>{re_ident})'
+    re_qualified += rf'\s*'
+    re_qualified += rf'(?P<qualifier>=|<=|>=)'
+    re_qualified += rf'\s*'
+    re_qualified += rf'(?P<fcdistro>{re_ident}[0-9]+)'
+    re_qualified += rf'\s*'
+    m_qualified = re.compile(rf'\A{re_qualified}\Z')

 
     re_old = '[a-z]+[+-][a-z]+[0-9]+'
 
     re_old = '[a-z]+[+-][a-z]+[0-9]+'

-    m_old = re.compile ('\A%s\Z'%re_old)
-    
-    # returns a tuple (included,excluded)
-    def parse (self,filename):
-        ok=True
-        included=[]
-        excluded=[]
-        lineno=0
+    m_old = re.compile(rf'\A{re_old}\Z')
+
+    # returns a tuple (included, excluded)
+    def parse(self, filename):
+        ok = True
+        included = []
+        excluded = []

         try:
         try:

-            for line in file(filename).readlines():
-                lineno += 1
-                line=line.strip()
-                if self.m_comment.match(line) or self.m_blank.match(line):
-                    continue
-                try:
-                    [lefts,rights] = line.split(':',1)
-                    for left in lefts.split():
-                        ########## single ident
-                        if self.m_ident.match(left):
-                            if left not in known_keywords:
-                                raise Exception,"Unknown keyword %r"%left
-                            elif left == self.keyword:
-                                included += rights.split()
-                        else:
-                            m=self.m_qualified.match(left)
-                            if m:
-                                (plus_minus,kw,qual,fcdistro) = m.groups()
-                                if kw not in known_keywords:
-                                    raise Exception,"Unknown keyword in %r"%left
-                                if fcdistro not in known_fcdistros:
-                                    raise Exception, 'Unknown fcdistro %r'%fcdistro
-                                # skip if another keyword
-                                if kw != self.keyword: continue
-                                # does this fcdistro match ?
-                                (distro,version)=m_fcdistro_cutter.match(fcdistro).groups()
-                                version = int (version)
-                                # skip if another distro family
-                                if distro != self.distro: continue
-                                # skip if the qualifier does not fit
-                                if not self.match (qual, version): 
-                                    if self.options.verbose: print >> stderr,'%s:%d:qualifer %s does not apply'%(filename,lineno,left)
-                                    continue
-                                # we're in, let's add (default) or remove (if plus_minus is minus)
-                                if plus_minus == '-':
-                                    if self.options.verbose: print >> stderr,'%s:%d: from %s, excluding %r'%(filename,lineno,left,rights)
-                                    excluded += rights.split()
-                                else:
-                                    if self.options.verbose: print >> stderr,'%s:%d: from %s, including %r'%(filename,lineno,left,rights)
+            with open(filename) as feed:
+                for lineno, line in enumerate(feed, 1):
+                    line = line.strip()
+                    if self.m_comment.match(line) or self.m_blank.match(line):
+                        continue
+                    try:
+                        lefts, rights = line.split(':', 1)
+                        for left in lefts.split():
+                            ########## single ident
+                            if self.m_ident.match(left):
+                                if left not in known_keywords:
+                                    raise Exception(f"Unknown keyword {left}")
+                                elif left == self.keyword:

                                     included += rights.split()
                                     included += rights.split()

-                            elif self.m_old.match(left):
-                                raise Exception,'Old-fashioned syntax not supported anymore %r'%left

                             else:
                             else:

-                                raise Exception,'error in left expression %r'%left
-                                
-                except Exception,e:
-                    ok=False
-                    print >> stderr, "%s:%d:syntax error: %r"%(filename,lineno,e)
-        except Exception,e:
-            ok=False
-            print >> stderr, 'Could not parse file',filename,e
-        return (ok,included,excluded)
+                                m = self.m_qualified.match(left)
+                                if m:
+                                    (plus_minus, kw, qual, fcdistro) = m.groups()
+                                    if kw not in known_keywords:
+                                        raise Exception(f"Unknown keyword in {left}")
+                                    if fcdistro not in known_fcdistros:
+                                        raise Exception(f'Unknown fcdistro {fcdistro}')
+                                    # skip if another keyword
+                                    if kw != self.keyword: continue
+                                    # does this fcdistro match ?
+                                    (distro, version) = m_fcdistro_cutter.match(fcdistro).groups()
+                                    version = int (version)
+                                    # skip if another distro family
+                                    if distro != self.distro: continue
+                                    # skip if the qualifier does not fit
+                                    if not self.match (qual, version):
+                                        if self.options.verbose:
+                                            print(f'{filename}:{lineno}:qualifer {left} does not apply',
+                                                  file=stderr)
+                                        continue
+                                    # we're in, let's add (default) or remove (if plus_minus is minus)
+                                    if plus_minus == '-':
+                                        if self.options.verbose:
+                                            print(f'{filename}:{lineno}: from {left}, excluding {rights}',
+                                                  file=stderr)
+                                        excluded += rights.split()
+                                    else:
+                                        if self.options.verbose:
+                                            print(f'{filename}:{lineno}: from {left}, including {rights}',
+                                                  file=stderr)
+                                        included += rights.split()
+                                elif self.m_old.match(left):
+                                    raise Exception(f'Old-fashioned syntax not supported anymore {left}')
+                                else:
+                                    raise Exception(f'error in left expression {left}')
+
+                    except Exception as e:
+                        ok = False
+                        print(f"{filename}:{lineno}:syntax error: {e}", file=stderr)
+        except Exception as exc:
+            ok = False
+            print(f'Could not parse file {filename} {exc=}', file=stderr)
+        return (ok, included, excluded)

 
     def run (self):
 
     def run (self):

-        ok=True
-        included=[]
-        excluded=[]
+        ok = True
+        included = []
+        excluded = []

         for input in self.inputs:
         for input in self.inputs:

-            (o,i,e) = self.parse (input)
+            (o, i, e) = self.parse (input)

             included += i
             excluded += e
             ok = ok and o
         # avoid set operations that would not preserve order
             included += i
             excluded += e
             ok = ok and o
         # avoid set operations that would not preserve order

-        results = [ x for x in included if x not in excluded ]
-        
-        results = [ x.replace('@arch@',self.arch).\
-                        replace('@fcdistro@',self.fcdistro).\
-                        replace('@pldistro@',self.pldistro) for x in results]
+        results = [x for x in included if x not in excluded]
+
+        results = [x.replace('@arch@', self.arch)
+                   .replace('@fcdistro@', self.fcdistro)
+                   .replace('@pldistro@', self.pldistro) for x in results]

         if self.options.sort_results:
             results.sort()
         # default is space-separated
         if not self.options.new_line:
         if self.options.sort_results:
             results.sort()
         # default is space-separated
         if not self.options.new_line:

-            print " ".join(results)
+            print(" ".join(results))

         # but for tests results are printed each on a line
         else:
         # but for tests results are printed each on a line
         else:

-            for result in results : print result
+            for result in results:
+                print(result)

         return ok
 
 def main ():
         return ok
 
 def main ():

-    usage="Usage: %prog [options] keyword input[...]"
-    parser=OptionParser (usage=usage)
-    parser.add_option ('-a','--arch',dest='arch',action='store',default=default_arch,
-                       help='target arch, e.g. i386 or x86_64, default=%s'%default_arch)
-    parser.add_option ('-f','--fcdistro',dest='fcdistro',action='store', default=default_fcdistro,
-                       help='fcdistro, e.g. f12 or centos5')
-    parser.add_option ('-d','--pldistro',dest='pldistro',action='store', default=default_pldistro,
-                       help='pldistro, e.g. onelab or planetlab')
-    parser.add_option ('-v', '--verbose',dest='verbose',action='store_true',default=False,
-                       help='verbose when using qualifiers')
-    parser.add_option ('-n', '--new-line',dest='new_line',action='store_true',default=False,
-                       help='print outputs separated with newlines rather than with a space')
-    parser.add_option ('-u', '--no-sort',dest='sort_results',default=True,action='store_false',
-                       help='keep results in the same order as in the inputs')
-    (options,args) = parser.parse_args()
-    
-    if len(args) <=1 :
+    usage = "Usage: %prog [options] keyword input[...]"
+    parser = OptionParser(usage=usage)
+    parser.add_option(
+        '-a', '--arch', dest='arch', action='store', default=default_arch,
+        help=f'target arch, e.g. i386 or x86_64, default={default_arch}')
+    parser.add_option(
+        '-f', '--fcdistro', dest='fcdistro', action='store', default=default_fcdistro,
+        help='fcdistro, e.g. f12 or centos5')
+    parser.add_option(
+        '-d', '--pldistro', dest='pldistro', action='store', default=default_pldistro,
+        help='pldistro, e.g. onelab or planetlab')
+    parser.add_option(
+        '-v', '--verbose', dest='verbose', action='store_true', default=False,
+        help='verbose when using qualifiers')
+    parser.add_option(
+        '-n', '--new-line', dest='new_line', action='store_true', default=False,
+        help='print outputs separated with newlines rather than with a space')
+    parser.add_option(
+        '-u', '--no-sort', dest='sort_results', default=True, action='store_false',
+        help='keep results in the same order as in the inputs')
+    (options, args) = parser.parse_args()
+
+    if len(args) <= 1:

         parser.print_help(file=stderr)
         sys.exit(1)
         parser.print_help(file=stderr)
         sys.exit(1)

-    keyword=args[0]
-    inputs=args[1:]
-    if not options.arch in known_arch:
-        print >> stderr, 'Unsupported arch',options.arch
+    keyword = args[0]
+    inputs = args[1:]
+    if options.arch not in known_archs:
+        print('Unsupported arch', options.arch, file=stderr)

         parser.print_help(file=stderr)
         sys.exit(1)
         parser.print_help(file=stderr)
         sys.exit(1)

-    if not options.fcdistro in known_fcdistros:
-        print >> stderr, 'Unsupported fcdistro',options.fcdistro
+    if options.arch == 'i686':
+        options.arch = 'i386'
+    if options.fcdistro not in known_fcdistros:
+        print('Unsupported fcdistro', options.fcdistro, file=stderr)

         parser.print_help(file=stderr)
         sys.exit(1)
 
         parser.print_help(file=stderr)
         sys.exit(1)
 

-    pkgs = PkgsParser (options.arch,options.fcdistro,options.pldistro,keyword,inputs,options)
+    pkgs = PkgsParser(options.arch, options.fcdistro, options.pldistro,
+                      keyword, inputs, options)

 
     if pkgs.run():
         sys.exit(0)
 
     if pkgs.run():
         sys.exit(0)