module-tools.py

   1 #!/usr/bin/python -u
   2
   3 subversion_id = "$Id$"
   4
   5 import sys, os
   6 import re
   7 import time
   8 from glob import glob
   9 from optparse import OptionParser
  10
  11 # HARDCODED NAME CHANGES
  12 #
  13 # Moving to git we decided to rename some of the repositories. Here is
  14 # a map of name changes applied in git repositories.
  15 RENAMED_SVN_MODULES = {
  16     "PLEWWW": "plewww"
  17     }
  18
  19 def svn_to_git_name(module):
  20     if RENAMED_SVN_MODULES.has_key(module):
  21         return RENAMED_SVN_MODULES[module]
  22     return module
  23
  24 def git_to_svn_name(module):
  25     for key in RENAMED_SVN_MODULES:
  26         if module == RENAMED_SVN_MODULES[key]:
  27             return key
  28     return module
  29
  30
  31 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  32 def prompt (question,default=True,other_choices=[],allow_outside=False):
  33     if not isinstance (other_choices,list):
  34         other_choices = [ other_choices ]
  35     chars = [ c for (c,rest) in other_choices ]
  36
  37     choices = []
  38     if 'y' not in chars:
  39         if default is True: choices.append('[y]')
  40         else : choices.append('y')
  41     if 'n' not in chars:
  42         if default is False: choices.append('[n]')
  43         else : choices.append('n')
  44
  45     for (char,choice) in other_choices:
  46         if default == char:
  47             choices.append("["+char+"]"+choice)
  48         else:
  49             choices.append("<"+char+">"+choice)
  50     try:
  51         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  52         if not answer:
  53             return default
  54         answer=answer[0].lower()
  55         if answer == 'y':
  56             if 'y' in chars: return 'y'
  57             else: return True
  58         elif answer == 'n':
  59             if 'n' in chars: return 'n'
  60             else: return False
  61         elif other_choices:
  62             for (char,choice) in other_choices:
  63                 if answer == char:
  64                     return char
  65             if allow_outside:
  66                 return answer
  67         return prompt(question,default,other_choices)
  68     except:
  69         raise
  70
  71 def default_editor():
  72     try:
  73         editor = os.environ['EDITOR']
  74     except:
  75         editor = "emacs"
  76     return editor
  77
  78 ### fold long lines
  79 fold_length=132
  80
  81 def print_fold (line):
  82     while len(line) >= fold_length:
  83         print line[:fold_length],'\\'
  84         line=line[fold_length:]
  85     print line
  86
  87 class Command:
  88     def __init__ (self,command,options):
  89         self.command=command
  90         self.options=options
  91         self.tmp="/tmp/command-%d"%os.getpid()
  92
  93     def run (self):
  94         if self.options.dry_run:
  95             print 'dry_run',self.command
  96             return 0
  97         if self.options.verbose and self.options.mode not in Main.silent_modes:
  98             print '+',self.command
  99             sys.stdout.flush()
 100         return os.system(self.command)
 101
 102     def run_silent (self):
 103         if self.options.dry_run:
 104             print 'dry_run',self.command
 105             return 0
 106         if self.options.verbose:
 107             print '+',self.command,' .. ',
 108             sys.stdout.flush()
 109         retcod=os.system(self.command + " &> " + self.tmp)
 110         if retcod != 0:
 111             print "FAILED ! -- out+err below (command was %s)"%self.command
 112             os.system("cat " + self.tmp)
 113             print "FAILED ! -- end of quoted output"
 114         elif self.options.verbose:
 115             print "OK"
 116         os.unlink(self.tmp)
 117         return retcod
 118
 119     def run_fatal(self):
 120         if self.run_silent() !=0:
 121             raise Exception,"Command %s failed"%self.command
 122
 123     # returns stdout, like bash's $(mycommand)
 124     def output_of (self,with_stderr=False):
 125         if self.options.dry_run:
 126             print 'dry_run',self.command
 127             return 'dry_run output'
 128         tmp="/tmp/status-%d"%os.getpid()
 129         if self.options.debug:
 130             print '+',self.command,' .. ',
 131             sys.stdout.flush()
 132         command=self.command
 133         if with_stderr:
 134             command += " &> "
 135         else:
 136             command += " > "
 137         command += tmp
 138         os.system(command)
 139         result=file(tmp).read()
 140         os.unlink(tmp)
 141         if self.options.debug:
 142             print 'Done',
 143         return result
 144
 145
 146 class SvnRepository:
 147     type = "svn"
 148
 149     def __init__(self, path, options):
 150         self.path = path
 151         self.options = options
 152
 153     def name(self):
 154         return os.path.basename(self.path)
 155
 156     def url(self):
 157         out = Command("svn info %s" % self.path, self.options).output_of()
 158         for line in out.split('\n'):
 159             if line.startswith("URL:"):
 160                 return line.split()[1].strip()
 161
 162     def repo_root(self):
 163         out = Command("svn info %s" % self.path, self.options).output_of()
 164         for line in out.split('\n'):
 165             if line.startswith("Repository Root:"):
 166                 repo_root = line.split()[2].strip()
 167                 return "%s/svn/%s" (repo_root, self.name)
 168
 169     @classmethod
 170     def checkout(cls, remote, local, options, recursive=False):
 171         if recursive:
 172             svncommand = "svn co %s %s" % (remote, local)
 173         else:
 174             svncommand = "svn co -N %s %s" % (remote, local)
 175         Command("rm -rf %s" % local, options).run_silent()
 176         Command(svncommand, options).run_fatal()
 177
 178         return SvnRepository(local, options)
 179
 180     @classmethod
 181     def remote_exists(cls, remote):
 182         return os.system("svn list %s &> /dev/null" % remote) == 0
 183
 184     def tag_exists(self, tagname):
 185         url = "%s/tags/%s" % (self.repo_root(), tagname)
 186         return SvnRepository.remote_exists(url)
 187
 188     def update(self, subdir="", recursive=True):
 189         path = os.path.join(self.path, subdir)
 190         if recursive:
 191             svncommand = "svn up %s" % path
 192         else:
 193             svncommand = "svn up -N %s" % path
 194         Command(svncommand, self.options).run_fatal()
 195
 196     def commit(self, logfile):
 197         # add all new files to the repository
 198         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 199                 self.path, self.options).run_silent()
 200         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 201
 202     def to_branch(self, branch):
 203         remote = "%s/branches/%s" % (self.repo_root(), branch)
 204         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 205
 206     def to_tag(self, tag):
 207         remote = "%s/tags/%s" % (self.repo_root(), branch)
 208         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 209
 210     def tag(self, tagname, logfile):
 211         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 212         self_url = self.url()
 213         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 214
 215     def diff(self):
 216         return Command("svn diff %s" % self.path, self.options).output_of(True)
 217
 218     def diff_with_tag(self, tagname):
 219         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 220         return Command("svn diff %s %s" % (tag_url, self.url()),
 221                        self.options).output_of(True)
 222
 223     def revert(self):
 224         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 225         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 226                 self.path, self.options).run_silent()
 227
 228     def is_clean(self):
 229         command="svn status %s" % self.path
 230         return len(Command(command,self.options).output_of(True)) == 0
 231
 232     def is_valid(self):
 233         return os.path.exists(os.path.join(self.path, ".svn"))
 234
 235
 236 class GitRepository:
 237     type = "git"
 238
 239     def __init__(self, path, options):
 240         self.path = path
 241         self.options = options
 242
 243     def name(self):
 244         return os.path.basename(self.path)
 245
 246     def url(self):
 247         self.repo_root()
 248
 249     def repo_root(self):
 250         c = Command("git remote show origin", self.options)
 251         out = self.__run_in_repo(c.output_of)
 252         for line in out.split('\n'):
 253             if line.strip().startswith("Fetch URL:"):
 254                 repo = line.split()[2]
 255
 256     @classmethod
 257     def checkout(cls, remote, local, options, depth=1):
 258         Command("rm -rf %s" % local, options).run_silent()
 259         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 260         return GitRepository(local, options)
 261
 262     @classmethod
 263     def remote_exists(cls, remote):
 264         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 265
 266     def tag_exists(self, tagname):
 267         command = 'git tag -l | grep "^%s$"' % tagname
 268         c = Command(command, self.options)
 269         out = self.__run_in_repo(c.output_of, with_stderr=True)
 270         return len(out) > 0
 271
 272     def __run_in_repo(self, fun, *args, **kwargs):
 273         cwd = os.getcwd()
 274         os.chdir(self.path)
 275         ret = fun(*args, **kwargs)
 276         os.chdir(cwd)
 277         return ret
 278
 279     def __run_command_in_repo(self, command, ignore_errors=False):
 280         c = Command(command, self.options)
 281         if ignore_errors:
 282             return self.__run_in_repo(c.output_of)
 283         else:
 284             return self.__run_in_repo(c.run_fatal)
 285
 286     def update(self, subdir=None, recursive=None):
 287         return self.__run_command_in_repo("git pull")
 288
 289     def to_branch(self, branch, remote=True):
 290         if remote:
 291             branch = "origin/%s" % branch
 292         return self.__run_command_in_repo("git checkout %s" % branch)
 293
 294     def to_tag(self, tag):
 295         return self.__run_command_in_repo("git checkout %s" % tag)
 296
 297     def tag(self, tagname, logfile):
 298         self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 299         self.commit(logfile)
 300
 301     def diff(self):
 302         c = Command("git diff", self.options)
 303         return self.__run_in_repo(c.output_of, with_stderr=True)
 304
 305     def diff_with_tag(self, tagname):
 306         c = Command("git diff %s" % tagname, self.options)
 307         return self.__run_in_repo(c.output_of, with_stderr=True)
 308
 309     def commit(self, logfile):
 310         self.__run_command_in_repo("git add -A", ignore_errors=True)
 311         self.__run_command_in_repo("git commit -F  %s" % logfile, ignore_errors=True)
 312         self.__run_command_in_repo("git push --tags")
 313
 314     def revert(self):
 315         self.__run_command_in_repo("git --no-pager reset --hard")
 316         self.__run_command_in_repo("git --no-pager clean -f")
 317
 318     def is_clean(self):
 319         def check_commit():
 320             command="git status"
 321             s="nothing to commit (working directory clean)"
 322             return Command(command, self.options).output_of(True).find(s) >= 0
 323         return self.__run_in_repo(check_commit)
 324
 325     def is_valid(self):
 326         return os.path.exists(os.path.join(self.path, ".git"))
 327
 328
 329 class Repository:
 330     """ Generic repository """
 331     supported_repo_types = [SvnRepository, GitRepository]
 332
 333     def __init__(self, path, options):
 334         self.path = path
 335         self.options = options
 336         for repo in self.supported_repo_types:
 337             self.repo = repo(self.path, self.options)
 338             if self.repo.is_valid():
 339                 break
 340
 341     @classmethod
 342     def has_moved_to_git(cls, module, svnpath):
 343         module = git_to_svn_name(module)
 344         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 345
 346     @classmethod
 347     def remote_exists(cls, remote):
 348         for repo in Repository.supported_repo_types:
 349             if repo.remote_exists(remote):
 350                 return True
 351         return False
 352
 353     def __getattr__(self, attr):
 354         return getattr(self.repo, attr)
 355
 356
 357
 358 # support for tagged module is minimal, and is for the Build class only
 359 class Module:
 360
 361     svn_magic_line="--This line, and those below, will be ignored--"
 362     setting_tag_format = "Setting tag %s"
 363
 364     redirectors=[ # ('module_name_varname','name'),
 365                   ('module_version_varname','version'),
 366                   ('module_taglevel_varname','taglevel'), ]
 367
 368     # where to store user's config
 369     config_storage="CONFIG"
 370     #
 371     config={}
 372
 373     import commands
 374     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 375                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 376                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 377                  ("build", "Enter the name of your build module","build"),
 378                  ('username',"Enter your firstname and lastname for changelogs",""),
 379                  ("email","Enter your email address for changelogs",""),
 380                  ]
 381
 382     @classmethod
 383     def prompt_config (cls):
 384         for (key,message,default) in cls.configKeys:
 385             cls.config[key]=""
 386             while not cls.config[key]:
 387                 cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 388
 389
 390     # for parsing module spec name:branch
 391     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 392     # special form for tagged module - for Build
 393     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 394     # parsing specfiles
 395     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 396
 397     def __init__ (self,module_spec,options):
 398         # parse module spec
 399         attempt=Module.matcher_branch_spec.match(module_spec)
 400         if attempt:
 401             self.name=attempt.group('name')
 402             self.branch=attempt.group('branch')
 403         else:
 404             attempt=Module.matcher_tag_spec.match(module_spec)
 405             if attempt:
 406                 self.name=attempt.group('name')
 407                 self.tagname=attempt.group('tagname')
 408             else:
 409                 self.name=module_spec
 410
 411         # when available prefer to use git module name internally
 412         self.name = svn_to_git_name(self.name)
 413
 414         self.options=options
 415         self.module_dir="%s/%s"%(options.workdir,self.name)
 416         self.repository = None
 417         self.build = None
 418
 419     def run (self,command):
 420         return Command(command,self.options).run()
 421     def run_fatal (self,command):
 422         return Command(command,self.options).run_fatal()
 423     def run_prompt (self,message,fun, *args):
 424         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 425         if not self.options.verbose:
 426             while True:
 427                 choice=prompt(message,True,('s','how'))
 428                 if choice is True:
 429                     fun(*args)
 430                     return
 431                 elif choice is False:
 432                     print 'About to run function:', fun_msg
 433         else:
 434             question=message+" - want to run function: " + fun_msg
 435             if prompt(question,True):
 436                 fun(*args)
 437
 438     def friendly_name (self):
 439         if hasattr(self,'branch'):
 440             return "%s:%s"%(self.name,self.branch)
 441         elif hasattr(self,'tagname'):
 442             return "%s@%s"%(self.name,self.tagname)
 443         else:
 444             return self.name
 445
 446     @classmethod
 447     def git_remote_dir (cls, name):
 448         return "%s:/git/%s.git" % (cls.config['gitserver'], name)
 449
 450     @classmethod
 451     def svn_remote_dir (cls, name):
 452         name = git_to_svn_name(name)
 453         svn = cls.config['svnpath']
 454         if svn.endswith('/'):
 455             return "%s%s" % (svn, name)
 456         return "%s/%s" % (svn, name)
 457
 458     def svn_selected_remote(self):
 459         svn_name = git_to_svn_name(self.name)
 460         remote = self.svn_remote_dir(svn_name)
 461         if hasattr(self,'branch'):
 462             remote = "%s/branches/%s" % (remote, self.branch)
 463         elif hasattr(self,'tagname'):
 464             remote = "%s/tags/%s" % (remote, self.tagname)
 465         else:
 466             remote = "%s/trunk" % remote
 467         return remote
 468
 469     ####################
 470     @classmethod
 471     def init_homedir (cls, options):
 472         if options.verbose and options.mode not in Main.silent_modes:
 473             print 'Checking for', options.workdir
 474         storage="%s/%s"%(options.workdir, cls.config_storage)
 475         # sanity check. Either the topdir exists AND we have a config/storage
 476         # or topdir does not exist and we create it
 477         # to avoid people use their own daily svn repo
 478         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 479             print """The directory %s exists and has no CONFIG file
 480 If this is your regular working directory, please provide another one as the
 481 module-* commands need a fresh working dir. Make sure that you do not use
 482 that for other purposes than tagging""" % options.workdir
 483             sys.exit(1)
 484         if not os.path.isdir (options.workdir):
 485             print "Cannot find",options.workdir,"let's create it"
 486             cls.prompt_config()
 487             print "Checking ...",
 488             remote = cls.git_remote_dir(cls.config['build'])
 489             local = os.path.join(options.workdir, cls.config['build'])
 490             GitRepository.checkout(remote, local, options, depth=1)
 491             print "OK"
 492
 493             # store config
 494             f=file(storage,"w")
 495             for (key,message,default) in Module.configKeys:
 496                 f.write("%s=%s\n"%(key,Module.config[key]))
 497             f.close()
 498             if options.debug:
 499                 print 'Stored',storage
 500                 Command("cat %s"%storage,options).run()
 501         else:
 502             # read config
 503             f=open(storage)
 504             for line in f.readlines():
 505                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 506                 Module.config[key]=value
 507             f.close()
 508
 509             build_dir = os.path.join(options.workdir, cls.config['build'])
 510             build = Repository(build_dir, options)
 511             if not build.is_clean():
 512                 print "build module needs a revert"
 513                 build.revert()
 514                 print "OK"
 515             build.update()
 516
 517         if options.verbose and options.mode not in Main.silent_modes:
 518             print '******** Using config'
 519             for (key,message,default) in Module.configKeys:
 520                 print '\t',key,'=',Module.config[key]
 521
 522     def init_module_dir (self):
 523         if self.options.verbose:
 524             print 'Checking for',self.module_dir
 525
 526         if not os.path.isdir (self.module_dir):
 527             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 528                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 529                                                          self.module_dir,
 530                                                          self.options)
 531             else:
 532                 remote = self.svn_selected_remote()
 533                 self.repository = SvnRepository.checkout(remote,
 534                                                          self.module_dir,
 535                                                          self.options, recursive=False)
 536
 537         self.repository = Repository(self.module_dir, self.options)
 538         if self.repository.type == "svn":
 539             # check if module has moved to git
 540             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 541                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 542                 self.init_module_dir()
 543             # check if we have the required branch/tag
 544             if self.repository.url() != self.svn_selected_remote():
 545                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 546                 self.init_module_dir()
 547
 548         elif self.repository.type == "git":
 549             if hasattr(self,'branch'):
 550                 self.repository.to_branch(self.branch)
 551             elif hasattr(self,'tagname'):
 552                 self.repository.to_tag(self.tagname)
 553
 554         else:
 555             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 556
 557
 558     def revert_module_dir (self):
 559         if self.options.fast_checks:
 560             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 561             return
 562         if self.options.verbose:
 563             print 'Checking whether', self.module_dir, 'needs being reverted'
 564
 565         if not self.repository.is_clean():
 566             self.repository.revert()
 567
 568     def update_module_dir (self):
 569         if self.options.fast_checks:
 570             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 571             return
 572         if self.options.verbose:
 573             print 'Updating', self.module_dir
 574         self.repository.update()
 575
 576     def main_specname (self):
 577         attempt="%s/%s.spec"%(self.module_dir,self.name)
 578         if os.path.isfile (attempt):
 579             return attempt
 580         pattern1="%s/*.spec"%self.module_dir
 581         level1=glob(pattern1)
 582         if level1:
 583             return level1[0]
 584         pattern2="%s/*/*.spec"%self.module_dir
 585         level2=glob(pattern2)
 586
 587         if level2:
 588             return level2[0]
 589         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 590
 591     def all_specnames (self):
 592         level1=glob("%s/*.spec" % self.module_dir)
 593         if level1: return level1
 594         level2=glob("%s/*/*.spec" % self.module_dir)
 595         return level2
 596
 597     def parse_spec (self, specfile, varnames):
 598         if self.options.verbose:
 599             print 'Parsing',specfile,
 600             for var in varnames:
 601                 print "[%s]"%var,
 602             print ""
 603         result={}
 604         f=open(specfile)
 605         for line in f.readlines():
 606             attempt=Module.matcher_rpm_define.match(line)
 607             if attempt:
 608                 (define,var,value)=attempt.groups()
 609                 if var in varnames:
 610                     result[var]=value
 611         f.close()
 612         if self.options.debug:
 613             print 'found',len(result),'keys'
 614             for (k,v) in result.iteritems():
 615                 print k,'=',v
 616         return result
 617
 618     # stores in self.module_name_varname the rpm variable to be used for the module's name
 619     # and the list of these names in self.varnames
 620     def spec_dict (self):
 621         specfile=self.main_specname()
 622         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 623         redirect_dict = self.parse_spec(specfile,redirector_keys)
 624         if self.options.debug:
 625             print '1st pass parsing done, redirect_dict=',redirect_dict
 626         varnames=[]
 627         for (varname,default) in Module.redirectors:
 628             if redirect_dict.has_key(varname):
 629                 setattr(self,varname,redirect_dict[varname])
 630                 varnames += [redirect_dict[varname]]
 631             else:
 632                 setattr(self,varname,default)
 633                 varnames += [ default ]
 634         self.varnames = varnames
 635         result = self.parse_spec (specfile,self.varnames)
 636         if self.options.debug:
 637             print '2st pass parsing done, varnames=',varnames,'result=',result
 638         return result
 639
 640     def patch_spec_var (self, patch_dict,define_missing=False):
 641         for specfile in self.all_specnames():
 642             # record the keys that were changed
 643             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 644             newspecfile=specfile+".new"
 645             if self.options.verbose:
 646                 print 'Patching',specfile,'for',patch_dict.keys()
 647             spec=open (specfile)
 648             new=open(newspecfile,"w")
 649
 650             for line in spec.readlines():
 651                 attempt=Module.matcher_rpm_define.match(line)
 652                 if attempt:
 653                     (define,var,value)=attempt.groups()
 654                     if var in patch_dict.keys():
 655                         if self.options.debug:
 656                             print 'rewriting %s as %s'%(var,patch_dict[var])
 657                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 658                         changed[var]=True
 659                         continue
 660                 new.write(line)
 661             if define_missing:
 662                 for (key,was_changed) in changed.iteritems():
 663                     if not was_changed:
 664                         if self.options.debug:
 665                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 666                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 667             spec.close()
 668             new.close()
 669             os.rename(newspecfile,specfile)
 670
 671     # returns all lines until the magic line
 672     def unignored_lines (self, logfile):
 673         result=[]
 674         white_line_matcher = re.compile("\A\s*\Z")
 675         for logline in file(logfile).readlines():
 676             if logline.strip() == Module.svn_magic_line:
 677                 break
 678             elif white_line_matcher.match(logline):
 679                 continue
 680             else:
 681                 result.append(logline.strip()+'\n')
 682         return result
 683
 684     # creates a copy of the input with only the unignored lines
 685     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 686        f=file(fileout,'w')
 687        f.write(self.setting_tag_format%new_tag_name + '\n')
 688        for line in self.unignored_lines(filein):
 689            f.write(line)
 690        f.close()
 691
 692     def insert_changelog (self, logfile, oldtag, newtag):
 693         for specfile in self.all_specnames():
 694             newspecfile=specfile+".new"
 695             if self.options.verbose:
 696                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 697             spec=open (specfile)
 698             new=open(newspecfile,"w")
 699             for line in spec.readlines():
 700                 new.write(line)
 701                 if re.compile('%changelog').match(line):
 702                     dateformat="* %a %b %d %Y"
 703                     datepart=time.strftime(dateformat)
 704                     logpart="%s <%s> - %s"%(Module.config['username'],
 705                                                  Module.config['email'],
 706                                                  newtag)
 707                     new.write(datepart+" "+logpart+"\n")
 708                     for logline in self.unignored_lines(logfile):
 709                         new.write("- " + logline)
 710                     new.write("\n")
 711             spec.close()
 712             new.close()
 713             os.rename(newspecfile,specfile)
 714
 715     def show_dict (self, spec_dict):
 716         if self.options.verbose:
 717             for (k,v) in spec_dict.iteritems():
 718                 print k,'=',v
 719
 720     def last_tag (self, spec_dict):
 721         try:
 722             return "%s-%s" % (spec_dict[self.module_version_varname],
 723                               spec_dict[self.module_taglevel_varname])
 724         except KeyError,err:
 725             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 726
 727     def tag_name (self, spec_dict, old_svn_name=False):
 728         base_tag_name = self.name
 729         if old_svn_name:
 730             base_tag_name = git_to_svn_name(self.name)
 731         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 732
 733
 734 ##############################
 735     # using fine_grain means replacing only those instances that currently refer to this tag
 736     # otherwise, <module>-SVNPATH is replaced unconditionnally
 737     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 738         newtagsfile=tagsfile+".new"
 739         tags=open (tagsfile)
 740         new=open(newtagsfile,"w")
 741
 742         matches=0
 743         # fine-grain : replace those lines that refer to oldname
 744         if fine_grain:
 745             if self.options.verbose:
 746                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 747             matcher=re.compile("^(.*)%s(.*)"%oldname)
 748             for line in tags.readlines():
 749                 if not matcher.match(line):
 750                     new.write(line)
 751                 else:
 752                     (begin,end)=matcher.match(line).groups()
 753                     new.write(begin+newname+end+"\n")
 754                     matches += 1
 755         # brute-force : change uncommented lines that define <module>-SVNPATH
 756         else:
 757             if self.options.verbose:
 758                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 759             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 760                                           %(self.name)
 761             matcher_module=re.compile(pattern)
 762             for line in tags.readlines():
 763                 attempt=matcher_module.match(line)
 764                 if attempt:
 765                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 766                     if self.options.verbose:
 767                         print ' '+svnpath,
 768                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 769                     new.write(replacement)
 770                     matches += 1
 771                 else:
 772                     new.write(line)
 773         tags.close()
 774         new.close()
 775         os.rename(newtagsfile,tagsfile)
 776         if self.options.verbose: print "%d changes"%matches
 777         return matches
 778
 779     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 780         if self.options.verbose:
 781             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 782
 783         found_tagname = tagname
 784         found = self.repository.tag_exists(tagname)
 785         if not found and old_svn_tag_name:
 786             if self.options.verbose:
 787                 print "KO"
 788                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 789             found = self.repository.tag_exists(old_svn_tag_name)
 790             if found:
 791                 found_tagname = old_svn_tag_name
 792
 793         if (found and need_it) or (not found and not need_it):
 794             print "OK "
 795         else:
 796             print "KO"
 797             if found:
 798                 raise Exception, "tag (%s) is already there" % tagname
 799             else:
 800                 raise Exception, "can not find required tag (%s)" % tagname
 801
 802         return found_tagname
 803
 804
 805     def do_tag (self):
 806         self.init_module_dir()
 807         self.revert_module_dir()
 808         self.update_module_dir()
 809         # parse specfile
 810         spec_dict = self.spec_dict()
 811         self.show_dict(spec_dict)
 812
 813         # side effects
 814         old_tag_name = self.tag_name(spec_dict)
 815         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 816
 817         if (self.options.new_version):
 818             # new version set on command line
 819             spec_dict[self.module_version_varname] = self.options.new_version
 820             spec_dict[self.module_taglevel_varname] = 0
 821         else:
 822             # increment taglevel
 823             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 824             spec_dict[self.module_taglevel_varname] = new_taglevel
 825
 826         new_tag_name = self.tag_name(spec_dict)
 827
 828         # sanity check
 829         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 830         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 831
 832         # checking for diffs
 833         diff_output = self.repository.diff_with_tag(old_tag_name)
 834         if len(diff_output) == 0:
 835             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 836                 return
 837
 838         # side effect in trunk's specfile
 839         self.patch_spec_var(spec_dict)
 840
 841         # prepare changelog file
 842         # we use the standard subversion magic string (see svn_magic_line)
 843         # so we can provide useful information, such as version numbers and diff
 844         # in the same file
 845         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 846         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 847         setting_tag_line=Module.setting_tag_format%new_tag_name
 848         file(changelog,"w").write("""
 849 %s
 850 %s
 851 Please write a changelog for this new tag in the section above
 852 """%(Module.svn_magic_line,setting_tag_line))
 853
 854         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 855             file(changelog,"a").write('DIFF=========\n' + diff_output)
 856
 857         if self.options.debug:
 858             prompt('Proceed ?')
 859
 860         # edit it
 861         self.run("%s %s"%(self.options.editor,changelog))
 862         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 863         # so we do the job ourselves
 864         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 865         # insert changelog in spec
 866         if self.options.changelog:
 867             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 868
 869         ## update build
 870         build_path = os.path.join(self.options.workdir,
 871                                   Module.config['build'])
 872         build = Repository(build_path, self.options)
 873         if self.options.build_branch:
 874             build.to_branch(self.options.build_branch)
 875         if not build.is_clean():
 876             build.revert()
 877
 878         tagsfiles=glob(build.path+"/*-tags*.mk")
 879         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 880         default_answer = 'y'
 881         tagsfiles.sort()
 882         while True:
 883             for tagsfile in tagsfiles:
 884                 status=tagsdict[tagsfile]
 885                 basename=os.path.basename(tagsfile)
 886                 print ".................... Dealing with %s"%basename
 887                 while tagsdict[tagsfile] == 'todo' :
 888                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 889                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 890                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 891                                      allow_outside=True)
 892                     if choice == 'y':
 893                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 894                     elif choice == 'n':
 895                         print 'Done with %s'%os.path.basename(tagsfile)
 896                         tagsdict[tagsfile]='done'
 897                     elif choice == 'f':
 898                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 899                     elif choice == 'd':
 900                         self.run("svn diff %s"%tagsfile)
 901                     elif choice == 'r':
 902                         self.run("svn revert %s"%tagsfile)
 903                     elif choice == 'c':
 904                         self.run("cat %s"%tagsfile)
 905                     else:
 906                         name=self.name
 907                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 908 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 909 d: show current diff for this tag file
 910 r: revert that tag file
 911 c: cat the current tag file
 912 n: move to next file"""%locals()
 913
 914             if prompt("Want to review changes on tags files",False):
 915                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 916                 default_answer='d'
 917             else:
 918                 break
 919
 920         def diff_all_changes():
 921             print build.diff()
 922             print self.repository.diff()
 923
 924         def commit_all_changes(log):
 925             self.repository.commit(log)
 926             build.commit(log)
 927
 928         self.run_prompt("Review module and build", diff_all_changes)
 929         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 930         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 931
 932         if self.options.debug:
 933             print 'Preserving',changelog,'and stripped',changelog_svn
 934         else:
 935             os.unlink(changelog)
 936             os.unlink(changelog_svn)
 937
 938
 939 ##############################
 940 class Main:
 941
 942     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 943
 944 module-tools : a set of tools to manage subversion tags and specfile
 945   requires the specfile to either
 946   * define *version* and *taglevel*
 947   OR alternatively
 948   * define redirection variables module_version_varname / module_taglevel_varname
 949 Trunk:
 950   by default, the trunk of modules is taken into account
 951   in this case, just mention the module name as <module_desc>
 952 Branches:
 953   if you wish to work on a branch rather than on the trunk,
 954   you can use something like e.g. Mom:2.1 as <module_desc>
 955 """
 956     release_usage="""Usage: %prog [options] tag1 .. tagn
 957   Extract release notes from the changes in specfiles between several build tags, latest first
 958   Examples:
 959       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 960   You can refer to a (build) branch by prepending a colon, like in
 961       release-changelog :4.2 4.2-rc25
 962   You can refer to the build trunk by just mentioning 'trunk', e.g.
 963       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 964 """
 965     common_usage="""More help:
 966   see http://svn.planet-lab.org/wiki/ModuleTools"""
 967
 968     modes={
 969         'list' : "displays a list of available tags or branches",
 970         'version' : "check latest specfile and print out details",
 971         'diff' : "show difference between module (trunk or branch) and latest tag",
 972         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 973                 create new tag and and monitor its adoption in build/*-tags*.mk""",
 974         'branch' : """create a branch for this module, from the latest tag on the trunk,
 975                   and change trunk's version number to reflect the new branch name;
 976                   you can specify the new branch name by using module:branch""",
 977         'sync' : """create a tag from the module
 978                 this is a last resort option, mostly for repairs""",
 979         'changelog' : """extract changelog between build tags
 980                 expected arguments are a list of tags""",
 981         }
 982
 983     silent_modes = ['list']
 984     release_modes = ['changelog']
 985
 986     @staticmethod
 987     def optparse_list (option, opt, value, parser):
 988         try:
 989             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
 990         except:
 991             setattr(parser.values,option.dest,value.split())
 992
 993     def run(self):
 994
 995         mode=None
 996         for function in Main.modes.keys():
 997             if sys.argv[0].find(function) >= 0:
 998                 mode = function
 999                 break
1000         if not mode:
1001             print "Unsupported command",sys.argv[0]
1002             print "Supported commands:" + " ".join(Main.modes.keys())
1003             sys.exit(1)
1004
1005         if mode not in Main.release_modes:
1006             usage = Main.module_usage
1007             usage += Main.common_usage
1008             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1009         else:
1010             usage = Main.release_usage
1011             usage += Main.common_usage
1012
1013         parser=OptionParser(usage=usage,version=subversion_id)
1014
1015         if mode == "tag" or mode == 'branch':
1016             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1017                               help="set new version and reset taglevel to 0")
1018         if mode == "tag" :
1019             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1020                               help="do not update changelog section in specfile when tagging")
1021             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1022                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1023         if mode == "tag" or mode == "sync" :
1024             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1025                               help="specify editor")
1026
1027         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1028         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1029                           help="dry run - shell commands are only displayed")
1030         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1031                           default=[], nargs=1,type="string",
1032                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1033 -- can be set multiple times, or use quotes""")
1034
1035         parser.add_option("-w","--workdir", action="store", dest="workdir",
1036                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1037                           help="""name for dedicated working dir - defaults to ~/modules
1038 ** THIS MUST NOT ** be your usual working directory""")
1039         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1040                           help="skip safety checks, such as svn updates -- use with care")
1041
1042         # default verbosity depending on function - temp
1043         verbose_modes= ['tag', 'sync', 'branch']
1044
1045         if mode not in verbose_modes:
1046             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1047                               help="run in verbose mode")
1048         else:
1049             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1050                               help="run in quiet (non-verbose) mode")
1051         (options, args) = parser.parse_args()
1052         options.mode=mode
1053         if not hasattr(options,'dry_run'):
1054             options.dry_run=False
1055         if not hasattr(options,'www'):
1056             options.www=False
1057         options.debug=False
1058
1059         ########## module-*
1060         if len(args) == 0:
1061             parser.print_help()
1062             sys.exit(1)
1063         Module.init_homedir(options)
1064
1065         modules=[ Module(modname,options) for modname in args ]
1066         for module in modules:
1067             if len(args)>1 and mode not in Main.silent_modes:
1068                 print '========================================',module.friendly_name()
1069             # call the method called do_<mode>
1070             method=Module.__dict__["do_%s"%mode]
1071             try:
1072                 method(module)
1073             except Exception,e:
1074                 import traceback
1075                 traceback.print_exc()
1076                 print 'Skipping module %s: '%modname,e
1077
1078 ####################
1079 if __name__ == "__main__" :
1080     try:
1081         Main().run()
1082     except KeyboardInterrupt:
1083         print '\nBye'