module-tools.py

   1 #!/usr/bin/python -u
   2
   3 subversion_id = "$Id$"
   4
   5 import sys, os
   6 import re
   7 import time
   8 from glob import glob
   9 from optparse import OptionParser
  10
  11 # HARDCODED NAME CHANGES
  12 #
  13 # Moving to git we decided to rename some of the repositories. Here is
  14 # a map of name changes applied in git repositories.
  15 RENAMED_SVN_MODULES = {
  16     "PLEWWW": "plewww"
  17     }
  18
  19 def svn_to_git_name(module):
  20     if RENAMED_SVN_MODULES.has_key(module):
  21         return RENAMED_SVN_MODULES[module]
  22     return module
  23
  24 def git_to_svn_name(module):
  25     for key in RENAMED_SVN_MODULES:
  26         if module == RENAMED_SVN_MODULES[key]:
  27             return key
  28     return module
  29
  30
  31 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  32 def prompt (question,default=True,other_choices=[],allow_outside=False):
  33     if not isinstance (other_choices,list):
  34         other_choices = [ other_choices ]
  35     chars = [ c for (c,rest) in other_choices ]
  36
  37     choices = []
  38     if 'y' not in chars:
  39         if default is True: choices.append('[y]')
  40         else : choices.append('y')
  41     if 'n' not in chars:
  42         if default is False: choices.append('[n]')
  43         else : choices.append('n')
  44
  45     for (char,choice) in other_choices:
  46         if default == char:
  47             choices.append("["+char+"]"+choice)
  48         else:
  49             choices.append("<"+char+">"+choice)
  50     try:
  51         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  52         if not answer:
  53             return default
  54         answer=answer[0].lower()
  55         if answer == 'y':
  56             if 'y' in chars: return 'y'
  57             else: return True
  58         elif answer == 'n':
  59             if 'n' in chars: return 'n'
  60             else: return False
  61         elif other_choices:
  62             for (char,choice) in other_choices:
  63                 if answer == char:
  64                     return char
  65             if allow_outside:
  66                 return answer
  67         return prompt(question,default,other_choices)
  68     except:
  69         raise
  70
  71 def default_editor():
  72     try:
  73         editor = os.environ['EDITOR']
  74     except:
  75         editor = "emacs"
  76     return editor
  77
  78 ### fold long lines
  79 fold_length=132
  80
  81 def print_fold (line):
  82     while len(line) >= fold_length:
  83         print line[:fold_length],'\\'
  84         line=line[fold_length:]
  85     print line
  86
  87 class Command:
  88     def __init__ (self,command,options):
  89         self.command=command
  90         self.options=options
  91         self.tmp="/tmp/command-%d"%os.getpid()
  92
  93     def run (self):
  94         if self.options.dry_run:
  95             print 'dry_run',self.command
  96             return 0
  97         if self.options.verbose and self.options.mode not in Main.silent_modes:
  98             print '+',self.command
  99             sys.stdout.flush()
 100         return os.system(self.command)
 101
 102     def run_silent (self):
 103         if self.options.dry_run:
 104             print 'dry_run',self.command
 105             return 0
 106         if self.options.verbose:
 107             print '+',self.command,' .. ',
 108             sys.stdout.flush()
 109         retcod=os.system(self.command + " &> " + self.tmp)
 110         if retcod != 0:
 111             print "FAILED ! -- out+err below (command was %s)"%self.command
 112             os.system("cat " + self.tmp)
 113             print "FAILED ! -- end of quoted output"
 114         elif self.options.verbose:
 115             print "OK"
 116         os.unlink(self.tmp)
 117         return retcod
 118
 119     def run_fatal(self):
 120         if self.run_silent() !=0:
 121             raise Exception,"Command %s failed"%self.command
 122
 123     # returns stdout, like bash's $(mycommand)
 124     def output_of (self,with_stderr=False):
 125         if self.options.dry_run:
 126             print 'dry_run',self.command
 127             return 'dry_run output'
 128         tmp="/tmp/status-%d"%os.getpid()
 129         if self.options.debug:
 130             print '+',self.command,' .. ',
 131             sys.stdout.flush()
 132         command=self.command
 133         if with_stderr:
 134             command += " &> "
 135         else:
 136             command += " > "
 137         command += tmp
 138         os.system(command)
 139         result=file(tmp).read()
 140         os.unlink(tmp)
 141         if self.options.debug:
 142             print 'Done',
 143         return result
 144
 145
 146 class SvnRepository:
 147     type = "svn"
 148
 149     def __init__(self, path, options):
 150         self.path = path
 151         self.options = options
 152
 153     def name(self):
 154         return os.path.basename(self.path)
 155
 156     def url(self):
 157         out = Command("svn info %s" % self.path, self.options).output_of()
 158         for line in out.split('\n'):
 159             if line.startswith("URL:"):
 160                 return line.split()[1].strip()
 161
 162     def repo_root(self):
 163         out = Command("svn info %s" % self.path, self.options).output_of()
 164         for line in out.split('\n'):
 165             if line.startswith("Repository Root:"):
 166                 repo_root = line.split()[2].strip()
 167                 return "%s/svn/%s" (repo_root, self.name)
 168
 169     @classmethod
 170     def checkout(cls, remote, local, options, recursive=False):
 171         if recursive:
 172             svncommand = "svn co %s %s" % (remote, local)
 173         else:
 174             svncommand = "svn co -N %s %s" % (remote, local)
 175         Command("rm -rf %s" % local, options).run_silent()
 176         Command(svncommand, options).run_fatal()
 177
 178         return SvnRepository(local, options)
 179
 180     @classmethod
 181     def remote_exists(cls, remote):
 182         return os.system("svn list %s &> /dev/null" % remote) == 0
 183
 184     def tag_exists(self, tagname):
 185         url = "%s/tags/%s" % (self.repo_root(), tagname)
 186         return SvnRepository.remote_exists(url)
 187
 188     def update(self, subdir="", recursive=True):
 189         path = os.path.join(self.path, subdir)
 190         if recursive:
 191             svncommand = "svn up %s" % path
 192         else:
 193             svncommand = "svn up -N %s" % path
 194         Command(svncommand, self.options).run_fatal()
 195
 196     def commit(self, logfile):
 197         # add all new files to the repository
 198         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 199                 self.path, self.options).run_silent()
 200         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 201
 202     def to_branch(self, branch):
 203         remote = "%s/branches/%s" % (self.repo_root(), branch)
 204         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 205
 206     def to_tag(self, tag):
 207         remote = "%s/tags/%s" % (self.repo_root(), branch)
 208         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 209
 210     def tag(self, tagname, logfile):
 211         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 212         self_url = self.url()
 213         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 214
 215     def diff(self):
 216         return Command("svn diff %s" % self.path, self.options).output_of(True)
 217
 218     def diff_with_tag(self, tagname):
 219         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 220         return Command("svn diff %s %s" % (tag_url, self.url()),
 221                        self.options).output_of(True)
 222
 223     def revert(self):
 224         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 225         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 226                 self.path, self.options).run_silent()
 227
 228     def is_clean(self):
 229         command="svn status %s" % self.path
 230         return len(Command(command,self.options).output_of(True)) == 0
 231
 232     def is_valid(self):
 233         return os.path.exists(os.path.join(self.path, ".svn"))
 234
 235
 236 class GitRepository:
 237     type = "git"
 238
 239     def __init__(self, path, options):
 240         self.path = path
 241         self.options = options
 242
 243     def name(self):
 244         return os.path.basename(self.path)
 245
 246     def url(self):
 247         self.repo_root()
 248
 249     def repo_root(self):
 250         c = Command("git remote show origin", self.options)
 251         out = self.__run_in_repo(c.output_of)
 252         for line in out.split('\n'):
 253             if line.strip().startswith("Fetch URL:"):
 254                 repo = line.split()[2]
 255
 256     @classmethod
 257     def checkout(cls, remote, local, options, depth=1):
 258         Command("rm -rf %s" % local, options).run_silent()
 259         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 260         return GitRepository(local, options)
 261
 262     @classmethod
 263     def remote_exists(cls, remote):
 264         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 265
 266     def tag_exists(self, tagname):
 267         command = 'git tag -l | grep "^%s$"' % tagname
 268         c = Command(command, self.options)
 269         out = self.__run_in_repo(c.output_of, with_stderr=True)
 270         return len(out) > 0
 271
 272     def __run_in_repo(self, fun, *args, **kwargs):
 273         cwd = os.getcwd()
 274         os.chdir(self.path)
 275         ret = fun(*args, **kwargs)
 276         os.chdir(cwd)
 277         return ret
 278
 279     def __run_command_in_repo(self, command):
 280         c = Command(command, self.options)
 281         return self.__run_in_repo(c.run_fatal)
 282
 283     def update(self, subdir=None, recursive=None):
 284         return self.__run_command_in_repo("git pull")
 285
 286     def to_branch(self, branch, remote=True):
 287         if remote:
 288             branch = "origin/%s" % branch
 289         return self.__run_command_in_repo("git checkout %s" % branch)
 290
 291     def to_tag(self, tag):
 292         return self.__run_command_in_repo("git checkout %s" % tag)
 293
 294     def tag(self, tagname, logfile):
 295         return self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 296
 297     def diff(self):
 298         c = Command("git diff", self.options)
 299         return self.__run_in_repo(c.output_of, with_stderr=True)
 300
 301     def diff_with_tag(self, tagname):
 302         c = Command("git diff %s" % tagname, self.options)
 303         return self.__run_in_repo(c.output_of, with_stderr=True)
 304
 305     def commit(self, logfile):
 306         self.__run_command_in_repo("git add -A")
 307         self.__run_command_in_repo("git commit -F  %s" % logfile)
 308         self.__run_command_in_repo("git push --tags")
 309
 310     def revert(self):
 311         self.__run_command_in_repo("git --no-pager reset --hard")
 312         self.__run_command_in_repo("git --no-pager clean -f")
 313
 314     def is_clean(self):
 315         def check_commit():
 316             command="git status"
 317             s="nothing to commit (working directory clean)"
 318             return Command(command, self.options).output_of(True).find(s) >= 0
 319         return self.__run_in_repo(check_commit)
 320
 321     def is_valid(self):
 322         return os.path.exists(os.path.join(self.path, ".git"))
 323
 324
 325 class Repository:
 326     """ Generic repository """
 327     supported_repo_types = [SvnRepository, GitRepository]
 328
 329     def __init__(self, path, options):
 330         self.path = path
 331         self.options = options
 332         for repo in self.supported_repo_types:
 333             self.repo = repo(self.path, self.options)
 334             if self.repo.is_valid():
 335                 break
 336
 337     @classmethod
 338     def has_moved_to_git(cls, module, svnpath):
 339         module = git_to_svn_name(module)
 340         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 341
 342     @classmethod
 343     def remote_exists(cls, remote):
 344         for repo in Repository.supported_repo_types:
 345             if repo.remote_exists(remote):
 346                 return True
 347         return False
 348
 349     def __getattr__(self, attr):
 350         return getattr(self.repo, attr)
 351
 352
 353
 354 # support for tagged module is minimal, and is for the Build class only
 355 class Module:
 356
 357     svn_magic_line="--This line, and those below, will be ignored--"
 358     setting_tag_format = "Setting tag %s"
 359
 360     redirectors=[ # ('module_name_varname','name'),
 361                   ('module_version_varname','version'),
 362                   ('module_taglevel_varname','taglevel'), ]
 363
 364     # where to store user's config
 365     config_storage="CONFIG"
 366     #
 367     config={}
 368
 369     import commands
 370     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 371                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 372                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 373                  ("build", "Enter the name of your build module","build"),
 374                  ('username',"Enter your firstname and lastname for changelogs",""),
 375                  ("email","Enter your email address for changelogs",""),
 376                  ]
 377
 378     @classmethod
 379     def prompt_config (cls):
 380         for (key,message,default) in cls.configKeys:
 381             cls.config[key]=""
 382             while not cls.config[key]:
 383                 cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 384
 385
 386     # for parsing module spec name:branch
 387     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 388     # special form for tagged module - for Build
 389     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 390     # parsing specfiles
 391     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 392
 393     def __init__ (self,module_spec,options):
 394         # parse module spec
 395         attempt=Module.matcher_branch_spec.match(module_spec)
 396         if attempt:
 397             self.name=attempt.group('name')
 398             self.branch=attempt.group('branch')
 399         else:
 400             attempt=Module.matcher_tag_spec.match(module_spec)
 401             if attempt:
 402                 self.name=attempt.group('name')
 403                 self.tagname=attempt.group('tagname')
 404             else:
 405                 self.name=module_spec
 406
 407         # when available prefer to use git module name internally
 408         self.name = svn_to_git_name(self.name)
 409
 410         self.options=options
 411         self.module_dir="%s/%s"%(options.workdir,self.name)
 412         self.repository = None
 413         self.build = None
 414
 415     def run (self,command):
 416         return Command(command,self.options).run()
 417     def run_fatal (self,command):
 418         return Command(command,self.options).run_fatal()
 419     def run_prompt (self,message,fun, *args):
 420         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 421         if not self.options.verbose:
 422             while True:
 423                 choice=prompt(message,True,('s','how'))
 424                 if choice is True:
 425                     fun(*args)
 426                     return
 427                 elif choice is False:
 428                     print 'About to run function:', fun_msg
 429         else:
 430             question=message+" - want to run function: " + fun_msg
 431             if prompt(question,True):
 432                 fun(*args)
 433
 434     def friendly_name (self):
 435         if hasattr(self,'branch'):
 436             return "%s:%s"%(self.name,self.branch)
 437         elif hasattr(self,'tagname'):
 438             return "%s@%s"%(self.name,self.tagname)
 439         else:
 440             return self.name
 441
 442     @classmethod
 443     def git_remote_dir (cls, name):
 444         return "%s:/git/%s.git" % (cls.config['gitserver'], name)
 445
 446     @classmethod
 447     def svn_remote_dir (cls, name):
 448         name = git_to_svn_name(name)
 449         svn = cls.config['svnpath']
 450         if svn.endswith('/'):
 451             return "%s%s" % (svn, name)
 452         return "%s/%s" % (svn, name)
 453
 454     def svn_selected_remote(self):
 455         svn_name = git_to_svn_name(self.name)
 456         remote = self.svn_remote_dir(svn_name)
 457         if hasattr(self,'branch'):
 458             remote = "%s/branches/%s" % (remote, self.branch)
 459         elif hasattr(self,'tagname'):
 460             remote = "%s/tags/%s" % (remote, self.tagname)
 461         else:
 462             remote = "%s/trunk" % remote
 463         return remote
 464
 465     ####################
 466     @classmethod
 467     def init_homedir (cls, options):
 468         if options.verbose and options.mode not in Main.silent_modes:
 469             print 'Checking for', options.workdir
 470         storage="%s/%s"%(options.workdir, cls.config_storage)
 471         # sanity check. Either the topdir exists AND we have a config/storage
 472         # or topdir does not exist and we create it
 473         # to avoid people use their own daily svn repo
 474         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 475             print """The directory %s exists and has no CONFIG file
 476 If this is your regular working directory, please provide another one as the
 477 module-* commands need a fresh working dir. Make sure that you do not use
 478 that for other purposes than tagging""" % options.workdir
 479             sys.exit(1)
 480         if not os.path.isdir (options.workdir):
 481             print "Cannot find",options.workdir,"let's create it"
 482             cls.prompt_config()
 483             print "Checking ...",
 484             remote = cls.git_remote_dir(cls.config['build'])
 485             local = os.path.join(options.workdir, cls.config['build'])
 486             build = GitRepository.checkout(remote, local, options, depth=1)
 487             if not build.is_clean():
 488                 build.revert()
 489             print "OK"
 490
 491             # store config
 492             f=file(storage,"w")
 493             for (key,message,default) in Module.configKeys:
 494                 f.write("%s=%s\n"%(key,Module.config[key]))
 495             f.close()
 496             if options.debug:
 497                 print 'Stored',storage
 498                 Command("cat %s"%storage,options).run()
 499         else:
 500             # read config
 501             f=open(storage)
 502             for line in f.readlines():
 503                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 504                 Module.config[key]=value
 505             f.close()
 506
 507             build_dir = os.path.join(options.workdir, cls.config['build'])
 508             build = Repository(build_dir, options)
 509             if not build.is_clean():
 510                 print "build module needs a revert"
 511                 build.revert()
 512                 print "OK"
 513
 514         if options.verbose and options.mode not in Main.silent_modes:
 515             print '******** Using config'
 516             for (key,message,default) in Module.configKeys:
 517                 print '\t',key,'=',Module.config[key]
 518
 519     def init_module_dir (self):
 520         if self.options.verbose:
 521             print 'Checking for',self.module_dir
 522
 523         if not os.path.isdir (self.module_dir):
 524             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 525                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 526                                                          self.module_dir,
 527                                                          self.options)
 528             else:
 529                 remote = self.svn_selected_remote()
 530                 self.repository = SvnRepository.checkout(remote,
 531                                                          self.module_dir,
 532                                                          self.options, recursive=False)
 533
 534         self.repository = Repository(self.module_dir, self.options)
 535         if self.repository.type == "svn":
 536             # check if module has moved to git
 537             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 538                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 539                 self.init_module_dir()
 540             # check if we have the required branch/tag
 541             if self.repository.url() != self.svn_selected_remote():
 542                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 543                 self.init_module_dir()
 544
 545         elif self.repository.type == "git":
 546             if hasattr(self,'branch'):
 547                 self.repository.to_branch(self.branch)
 548             elif hasattr(self,'tagname'):
 549                 self.repository.to_tag(self.tagname)
 550
 551         else:
 552             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 553
 554
 555     def revert_module_dir (self):
 556         if self.options.fast_checks:
 557             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 558             return
 559         if self.options.verbose:
 560             print 'Checking whether', self.module_dir, 'needs being reverted'
 561
 562         if not self.repository.is_clean():
 563             self.repository.revert()
 564
 565     def update_module_dir (self):
 566         if self.options.fast_checks:
 567             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 568             return
 569         if self.options.verbose:
 570             print 'Updating', self.module_dir
 571         self.repository.update()
 572
 573     def main_specname (self):
 574         attempt="%s/%s.spec"%(self.module_dir,self.name)
 575         if os.path.isfile (attempt):
 576             return attempt
 577         pattern1="%s/*.spec"%self.module_dir
 578         level1=glob(pattern1)
 579         if level1:
 580             return level1[0]
 581         pattern2="%s/*/*.spec"%self.module_dir
 582         level2=glob(pattern2)
 583
 584         if level2:
 585             return level2[0]
 586         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 587
 588     def all_specnames (self):
 589         level1=glob("%s/*.spec" % self.module_dir)
 590         if level1: return level1
 591         level2=glob("%s/*/*.spec" % self.module_dir)
 592         return level2
 593
 594     def parse_spec (self, specfile, varnames):
 595         if self.options.verbose:
 596             print 'Parsing',specfile,
 597             for var in varnames:
 598                 print "[%s]"%var,
 599             print ""
 600         result={}
 601         f=open(specfile)
 602         for line in f.readlines():
 603             attempt=Module.matcher_rpm_define.match(line)
 604             if attempt:
 605                 (define,var,value)=attempt.groups()
 606                 if var in varnames:
 607                     result[var]=value
 608         f.close()
 609         if self.options.debug:
 610             print 'found',len(result),'keys'
 611             for (k,v) in result.iteritems():
 612                 print k,'=',v
 613         return result
 614
 615     # stores in self.module_name_varname the rpm variable to be used for the module's name
 616     # and the list of these names in self.varnames
 617     def spec_dict (self):
 618         specfile=self.main_specname()
 619         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 620         redirect_dict = self.parse_spec(specfile,redirector_keys)
 621         if self.options.debug:
 622             print '1st pass parsing done, redirect_dict=',redirect_dict
 623         varnames=[]
 624         for (varname,default) in Module.redirectors:
 625             if redirect_dict.has_key(varname):
 626                 setattr(self,varname,redirect_dict[varname])
 627                 varnames += [redirect_dict[varname]]
 628             else:
 629                 setattr(self,varname,default)
 630                 varnames += [ default ]
 631         self.varnames = varnames
 632         result = self.parse_spec (specfile,self.varnames)
 633         if self.options.debug:
 634             print '2st pass parsing done, varnames=',varnames,'result=',result
 635         return result
 636
 637     def patch_spec_var (self, patch_dict,define_missing=False):
 638         for specfile in self.all_specnames():
 639             # record the keys that were changed
 640             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 641             newspecfile=specfile+".new"
 642             if self.options.verbose:
 643                 print 'Patching',specfile,'for',patch_dict.keys()
 644             spec=open (specfile)
 645             new=open(newspecfile,"w")
 646
 647             for line in spec.readlines():
 648                 attempt=Module.matcher_rpm_define.match(line)
 649                 if attempt:
 650                     (define,var,value)=attempt.groups()
 651                     if var in patch_dict.keys():
 652                         if self.options.debug:
 653                             print 'rewriting %s as %s'%(var,patch_dict[var])
 654                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 655                         changed[var]=True
 656                         continue
 657                 new.write(line)
 658             if define_missing:
 659                 for (key,was_changed) in changed.iteritems():
 660                     if not was_changed:
 661                         if self.options.debug:
 662                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 663                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 664             spec.close()
 665             new.close()
 666             os.rename(newspecfile,specfile)
 667
 668     # returns all lines until the magic line
 669     def unignored_lines (self, logfile):
 670         result=[]
 671         white_line_matcher = re.compile("\A\s*\Z")
 672         for logline in file(logfile).readlines():
 673             if logline.strip() == Module.svn_magic_line:
 674                 break
 675             elif white_line_matcher.match(logline):
 676                 continue
 677             else:
 678                 result.append(logline.strip()+'\n')
 679         return result
 680
 681     # creates a copy of the input with only the unignored lines
 682     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 683        f=file(fileout,'w')
 684        f.write(self.setting_tag_format%new_tag_name + '\n')
 685        for line in self.unignored_lines(filein):
 686            f.write(line)
 687        f.close()
 688
 689     def insert_changelog (self, logfile, oldtag, newtag):
 690         for specfile in self.all_specnames():
 691             newspecfile=specfile+".new"
 692             if self.options.verbose:
 693                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 694             spec=open (specfile)
 695             new=open(newspecfile,"w")
 696             for line in spec.readlines():
 697                 new.write(line)
 698                 if re.compile('%changelog').match(line):
 699                     dateformat="* %a %b %d %Y"
 700                     datepart=time.strftime(dateformat)
 701                     logpart="%s <%s> - %s"%(Module.config['username'],
 702                                                  Module.config['email'],
 703                                                  newtag)
 704                     new.write(datepart+" "+logpart+"\n")
 705                     for logline in self.unignored_lines(logfile):
 706                         new.write("- " + logline)
 707                     new.write("\n")
 708             spec.close()
 709             new.close()
 710             os.rename(newspecfile,specfile)
 711
 712     def show_dict (self, spec_dict):
 713         if self.options.verbose:
 714             for (k,v) in spec_dict.iteritems():
 715                 print k,'=',v
 716
 717     def last_tag (self, spec_dict):
 718         try:
 719             return "%s-%s" % (spec_dict[self.module_version_varname],
 720                               spec_dict[self.module_taglevel_varname])
 721         except KeyError,err:
 722             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 723
 724     def tag_name (self, spec_dict, old_svn_name=False):
 725         base_tag_name = self.name
 726         if old_svn_name:
 727             base_tag_name = git_to_svn_name(self.name)
 728         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 729
 730
 731 ##############################
 732     # using fine_grain means replacing only those instances that currently refer to this tag
 733     # otherwise, <module>-SVNPATH is replaced unconditionnally
 734     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 735         newtagsfile=tagsfile+".new"
 736         tags=open (tagsfile)
 737         new=open(newtagsfile,"w")
 738
 739         matches=0
 740         # fine-grain : replace those lines that refer to oldname
 741         if fine_grain:
 742             if self.options.verbose:
 743                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 744             matcher=re.compile("^(.*)%s(.*)"%oldname)
 745             for line in tags.readlines():
 746                 if not matcher.match(line):
 747                     new.write(line)
 748                 else:
 749                     (begin,end)=matcher.match(line).groups()
 750                     new.write(begin+newname+end+"\n")
 751                     matches += 1
 752         # brute-force : change uncommented lines that define <module>-SVNPATH
 753         else:
 754             if self.options.verbose:
 755                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 756             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 757                                           %(self.name)
 758             matcher_module=re.compile(pattern)
 759             for line in tags.readlines():
 760                 attempt=matcher_module.match(line)
 761                 if attempt:
 762                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 763                     if self.options.verbose:
 764                         print ' '+svnpath,
 765                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 766                     new.write(replacement)
 767                     matches += 1
 768                 else:
 769                     new.write(line)
 770         tags.close()
 771         new.close()
 772         os.rename(newtagsfile,tagsfile)
 773         if self.options.verbose: print "%d changes"%matches
 774         return matches
 775
 776     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 777         if self.options.verbose:
 778             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 779
 780         found_tagname = tagname
 781         found = self.repository.tag_exists(tagname)
 782         if not found and old_svn_tag_name:
 783             if self.options.verbose:
 784                 print "KO"
 785                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 786             found = self.repository.tag_exists(old_svn_tag_name)
 787             if found:
 788                 found_tagname = old_svn_tag_name
 789
 790         if (found and need_it) or (not found and not need_it):
 791             print "OK "
 792         else:
 793             print "KO"
 794             if found:
 795                 raise Exception, "tag (%s) is already there" % tagname
 796             else:
 797                 raise Exception, "can not find required tag (%s)" % tagname
 798
 799         return found_tagname
 800
 801
 802     def do_tag (self):
 803         self.init_module_dir()
 804         self.revert_module_dir()
 805         self.update_module_dir()
 806         # parse specfile
 807         spec_dict = self.spec_dict()
 808         self.show_dict(spec_dict)
 809
 810         # side effects
 811         old_tag_name = self.tag_name(spec_dict)
 812         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 813
 814         if (self.options.new_version):
 815             # new version set on command line
 816             spec_dict[self.module_version_varname] = self.options.new_version
 817             spec_dict[self.module_taglevel_varname] = 0
 818         else:
 819             # increment taglevel
 820             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 821             spec_dict[self.module_taglevel_varname] = new_taglevel
 822
 823         new_tag_name = self.tag_name(spec_dict)
 824
 825         # sanity check
 826         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 827         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 828
 829         # checking for diffs
 830         diff_output = self.repository.diff_with_tag(old_tag_name)
 831         if len(diff_output) == 0:
 832             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 833                 return
 834
 835         # side effect in trunk's specfile
 836         self.patch_spec_var(spec_dict)
 837
 838         # prepare changelog file
 839         # we use the standard subversion magic string (see svn_magic_line)
 840         # so we can provide useful information, such as version numbers and diff
 841         # in the same file
 842         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 843         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 844         setting_tag_line=Module.setting_tag_format%new_tag_name
 845         file(changelog,"w").write("""
 846 %s
 847 %s
 848 Please write a changelog for this new tag in the section above
 849 """%(Module.svn_magic_line,setting_tag_line))
 850
 851         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 852             file(changelog,"a").write('DIFF=========\n' + diff_output)
 853
 854         if self.options.debug:
 855             prompt('Proceed ?')
 856
 857         # edit it
 858         self.run("%s %s"%(self.options.editor,changelog))
 859         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 860         # so we do the job ourselves
 861         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 862         # insert changelog in spec
 863         if self.options.changelog:
 864             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 865
 866         ## update build
 867         build_path = os.path.join(self.options.workdir,
 868                                   Module.config['build'])
 869         build = Repository(build_path, self.options)
 870         if self.options.build_branch:
 871             build.to_branch(self.options.build_branch)
 872         if not build.is_clean():
 873             build.revert()
 874
 875         tagsfiles=glob(build.path+"/*-tags*.mk")
 876         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 877         default_answer = 'y'
 878         tagsfiles.sort()
 879         while True:
 880             for tagsfile in tagsfiles:
 881                 status=tagsdict[tagsfile]
 882                 basename=os.path.basename(tagsfile)
 883                 print ".................... Dealing with %s"%basename
 884                 while tagsdict[tagsfile] == 'todo' :
 885                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 886                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 887                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 888                                      allow_outside=True)
 889                     if choice == 'y':
 890                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 891                     elif choice == 'n':
 892                         print 'Done with %s'%os.path.basename(tagsfile)
 893                         tagsdict[tagsfile]='done'
 894                     elif choice == 'f':
 895                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 896                     elif choice == 'd':
 897                         self.run("svn diff %s"%tagsfile)
 898                     elif choice == 'r':
 899                         self.run("svn revert %s"%tagsfile)
 900                     elif choice == 'c':
 901                         self.run("cat %s"%tagsfile)
 902                     else:
 903                         name=self.name
 904                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 905 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 906 d: show current diff for this tag file
 907 r: revert that tag file
 908 c: cat the current tag file
 909 n: move to next file"""%locals()
 910
 911             if prompt("Want to review changes on tags files",False):
 912                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 913                 default_answer='d'
 914             else:
 915                 break
 916
 917         def diff_all_changes():
 918             print build.diff()
 919             print self.repository.diff()
 920
 921         def commit_all_changes(log):
 922             self.repository.commit(log)
 923             build.commit(log)
 924
 925         self.run_prompt("Review module and build", diff_all_changes)
 926         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 927         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 928
 929         if self.options.debug:
 930             print 'Preserving',changelog,'and stripped',changelog_svn
 931         else:
 932             os.unlink(changelog)
 933             os.unlink(changelog_svn)
 934
 935
 936 ##############################
 937 class Main:
 938
 939     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 940
 941 module-tools : a set of tools to manage subversion tags and specfile
 942   requires the specfile to either
 943   * define *version* and *taglevel*
 944   OR alternatively
 945   * define redirection variables module_version_varname / module_taglevel_varname
 946 Trunk:
 947   by default, the trunk of modules is taken into account
 948   in this case, just mention the module name as <module_desc>
 949 Branches:
 950   if you wish to work on a branch rather than on the trunk,
 951   you can use something like e.g. Mom:2.1 as <module_desc>
 952 """
 953     release_usage="""Usage: %prog [options] tag1 .. tagn
 954   Extract release notes from the changes in specfiles between several build tags, latest first
 955   Examples:
 956       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 957   You can refer to a (build) branch by prepending a colon, like in
 958       release-changelog :4.2 4.2-rc25
 959   You can refer to the build trunk by just mentioning 'trunk', e.g.
 960       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 961 """
 962     common_usage="""More help:
 963   see http://svn.planet-lab.org/wiki/ModuleTools"""
 964
 965     modes={
 966         'list' : "displays a list of available tags or branches",
 967         'version' : "check latest specfile and print out details",
 968         'diff' : "show difference between module (trunk or branch) and latest tag",
 969         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 970                 create new tag and and monitor its adoption in build/*-tags*.mk""",
 971         'branch' : """create a branch for this module, from the latest tag on the trunk,
 972                   and change trunk's version number to reflect the new branch name;
 973                   you can specify the new branch name by using module:branch""",
 974         'sync' : """create a tag from the module
 975                 this is a last resort option, mostly for repairs""",
 976         'changelog' : """extract changelog between build tags
 977                 expected arguments are a list of tags""",
 978         }
 979
 980     silent_modes = ['list']
 981     release_modes = ['changelog']
 982
 983     @staticmethod
 984     def optparse_list (option, opt, value, parser):
 985         try:
 986             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
 987         except:
 988             setattr(parser.values,option.dest,value.split())
 989
 990     def run(self):
 991
 992         mode=None
 993         for function in Main.modes.keys():
 994             if sys.argv[0].find(function) >= 0:
 995                 mode = function
 996                 break
 997         if not mode:
 998             print "Unsupported command",sys.argv[0]
 999             print "Supported commands:" + " ".join(Main.modes.keys())
1000             sys.exit(1)
1001
1002         if mode not in Main.release_modes:
1003             usage = Main.module_usage
1004             usage += Main.common_usage
1005             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1006         else:
1007             usage = Main.release_usage
1008             usage += Main.common_usage
1009
1010         parser=OptionParser(usage=usage,version=subversion_id)
1011
1012         if mode == "tag" or mode == 'branch':
1013             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1014                               help="set new version and reset taglevel to 0")
1015         if mode == "tag" :
1016             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1017                               help="do not update changelog section in specfile when tagging")
1018             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1019                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1020         if mode == "tag" or mode == "sync" :
1021             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1022                               help="specify editor")
1023
1024         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1025         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1026                           help="dry run - shell commands are only displayed")
1027         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1028                           default=[], nargs=1,type="string",
1029                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1030 -- can be set multiple times, or use quotes""")
1031
1032         parser.add_option("-w","--workdir", action="store", dest="workdir",
1033                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1034                           help="""name for dedicated working dir - defaults to ~/modules
1035 ** THIS MUST NOT ** be your usual working directory""")
1036         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1037                           help="skip safety checks, such as svn updates -- use with care")
1038
1039         # default verbosity depending on function - temp
1040         verbose_modes= ['tag', 'sync', 'branch']
1041
1042         if mode not in verbose_modes:
1043             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1044                               help="run in verbose mode")
1045         else:
1046             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1047                               help="run in quiet (non-verbose) mode")
1048         (options, args) = parser.parse_args()
1049         options.mode=mode
1050         if not hasattr(options,'dry_run'):
1051             options.dry_run=False
1052         if not hasattr(options,'www'):
1053             options.www=False
1054         options.debug=False
1055
1056         ########## module-*
1057         if len(args) == 0:
1058             parser.print_help()
1059             sys.exit(1)
1060         Module.init_homedir(options)
1061
1062         modules=[ Module(modname,options) for modname in args ]
1063         for module in modules:
1064             if len(args)>1 and mode not in Main.silent_modes:
1065                 print '========================================',module.friendly_name()
1066             # call the method called do_<mode>
1067             method=Module.__dict__["do_%s"%mode]
1068             try:
1069                 method(module)
1070             except Exception,e:
1071                 import traceback
1072                 traceback.print_exc()
1073                 print 'Skipping module %s: '%modname,e
1074
1075 ####################
1076 if __name__ == "__main__" :
1077     try:
1078         Main().run()
1079     except KeyboardInterrupt:
1080         print '\nBye'