module-tools.py

   1 #!/usr/bin/python -u
   2
   3 subversion_id = "$Id$"
   4
   5 import sys, os
   6 import re
   7 import time
   8 from glob import glob
   9 from optparse import OptionParser
  10
  11 # HARDCODED NAME CHANGES
  12 #
  13 # Moving to git we decided to rename some of the repositories. Here is
  14 # a map of name changes applied in git repositories.
  15 RENAMED_SVN_MODULES = {
  16     "PLEWWW": "plewww"
  17     }
  18
  19 def svn_to_git_name(module):
  20     if RENAMED_SVN_MODULES.has_key(module):
  21         return RENAMED_SVN_MODULES[module]
  22     return module
  23
  24 def git_to_svn_name(module):
  25     for key in RENAMED_SVN_MODULES:
  26         if module == RENAMED_SVN_MODULES[key]:
  27             return key
  28     return module
  29
  30
  31 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  32 def prompt (question,default=True,other_choices=[],allow_outside=False):
  33     if not isinstance (other_choices,list):
  34         other_choices = [ other_choices ]
  35     chars = [ c for (c,rest) in other_choices ]
  36
  37     choices = []
  38     if 'y' not in chars:
  39         if default is True: choices.append('[y]')
  40         else : choices.append('y')
  41     if 'n' not in chars:
  42         if default is False: choices.append('[n]')
  43         else : choices.append('n')
  44
  45     for (char,choice) in other_choices:
  46         if default == char:
  47             choices.append("["+char+"]"+choice)
  48         else:
  49             choices.append("<"+char+">"+choice)
  50     try:
  51         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  52         if not answer:
  53             return default
  54         answer=answer[0].lower()
  55         if answer == 'y':
  56             if 'y' in chars: return 'y'
  57             else: return True
  58         elif answer == 'n':
  59             if 'n' in chars: return 'n'
  60             else: return False
  61         elif other_choices:
  62             for (char,choice) in other_choices:
  63                 if answer == char:
  64                     return char
  65             if allow_outside:
  66                 return answer
  67         return prompt(question,default,other_choices)
  68     except:
  69         raise
  70
  71 def default_editor():
  72     try:
  73         editor = os.environ['EDITOR']
  74     except:
  75         editor = "emacs"
  76     return editor
  77
  78 ### fold long lines
  79 fold_length=132
  80
  81 def print_fold (line):
  82     while len(line) >= fold_length:
  83         print line[:fold_length],'\\'
  84         line=line[fold_length:]
  85     print line
  86
  87 class Command:
  88     def __init__ (self,command,options):
  89         self.command=command
  90         self.options=options
  91         self.tmp="/tmp/command-%d"%os.getpid()
  92
  93     def run (self):
  94         if self.options.dry_run:
  95             print 'dry_run',self.command
  96             return 0
  97         if self.options.verbose and self.options.mode not in Main.silent_modes:
  98             print '+',self.command
  99             sys.stdout.flush()
 100         return os.system(self.command)
 101
 102     def run_silent (self):
 103         if self.options.dry_run:
 104             print 'dry_run',self.command
 105             return 0
 106         if self.options.verbose:
 107             print '+',self.command,' .. ',
 108             sys.stdout.flush()
 109         retcod=os.system(self.command + " &> " + self.tmp)
 110         if retcod != 0:
 111             print "FAILED ! -- out+err below (command was %s)"%self.command
 112             os.system("cat " + self.tmp)
 113             print "FAILED ! -- end of quoted output"
 114         elif self.options.verbose:
 115             print "OK"
 116         os.unlink(self.tmp)
 117         return retcod
 118
 119     def run_fatal(self):
 120         if self.run_silent() !=0:
 121             raise Exception,"Command %s failed"%self.command
 122
 123     # returns stdout, like bash's $(mycommand)
 124     def output_of (self,with_stderr=False):
 125         if self.options.dry_run:
 126             print 'dry_run',self.command
 127             return 'dry_run output'
 128         tmp="/tmp/status-%d"%os.getpid()
 129         if self.options.debug:
 130             print '+',self.command,' .. ',
 131             sys.stdout.flush()
 132         command=self.command
 133         if with_stderr:
 134             command += " &> "
 135         else:
 136             command += " > "
 137         command += tmp
 138         os.system(command)
 139         result=file(tmp).read()
 140         os.unlink(tmp)
 141         if self.options.debug:
 142             print 'Done',
 143         return result
 144
 145
 146 class SvnRepository:
 147     type = "svn"
 148
 149     def __init__(self, path, options):
 150         self.path = path
 151         self.options = options
 152
 153     def name(self):
 154         return os.path.basename(self.path)
 155
 156     def url(self):
 157         out = Command("svn info %s" % self.path, self.options).output_of()
 158         for line in out.split('\n'):
 159             if line.startswith("URL:"):
 160                 return line.split()[1].strip()
 161
 162     def repo_root(self):
 163         out = Command("svn info %s" % self.path, self.options).output_of()
 164         for line in out.split('\n'):
 165             if line.startswith("Repository Root:"):
 166                 repo_root = line.split()[2].strip()
 167                 return "%s/svn/%s" (repo_root, self.name)
 168
 169     @classmethod
 170     def checkout(cls, remote, local, options, recursive=False):
 171         if recursive:
 172             svncommand = "svn co %s %s" % (remote, local)
 173         else:
 174             svncommand = "svn co -N %s %s" % (remote, local)
 175         Command("rm -rf %s" % local, options).run_silent()
 176         Command(svncommand, options).run_fatal()
 177
 178         return SvnRepository(local, options)
 179
 180     @classmethod
 181     def remote_exists(cls, remote):
 182         return os.system("svn list %s &> /dev/null" % remote) == 0
 183
 184     def tag_exists(self, tagname):
 185         url = "%s/tags/%s" % (self.repo_root(), tagname)
 186         return SvnRepository.remote_exists(url)
 187
 188     def update(self, subdir="", recursive=True):
 189         path = os.path.join(self.path, subdir)
 190         if recursive:
 191             svncommand = "svn up %s" % path
 192         else:
 193             svncommand = "svn up -N %s" % path
 194         Command(svncommand, self.options).run_fatal()
 195
 196     def commit(self, logfile):
 197         # add all new files to the repository
 198         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 199                 self.path, self.options).run_silent()
 200         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 201
 202     def to_branch(self, branch):
 203         remote = "%s/branches/%s" % (self.repo_root(), branch)
 204         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 205
 206     def to_tag(self, tag):
 207         remote = "%s/tags/%s" % (self.repo_root(), branch)
 208         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 209
 210     def tag(self, tagname, logfile):
 211         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 212         self_url = self.url()
 213         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 214
 215     def diff(self):
 216         return Command("svn diff %s" % self.path, self.options).output_of(True)
 217
 218     def diff_with_tag(self, tagname):
 219         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 220         return Command("svn diff %s %s" % (tag_url, self.url()),
 221                        self.options).output_of(True)
 222
 223     def revert(self):
 224         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 225         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 226                 self.path, self.options).run_silent()
 227
 228     def is_clean(self):
 229         command="svn status %s" % self.path
 230         return len(Command(command,self.options).output_of(True)) == 0
 231
 232     def is_valid(self):
 233         return os.path.exists(os.path.join(self.path, ".svn"))
 234
 235
 236 class GitRepository:
 237     type = "git"
 238
 239     def __init__(self, path, options):
 240         self.path = path
 241         self.options = options
 242
 243     def name(self):
 244         return os.path.basename(self.path)
 245
 246     def url(self):
 247         self.repo_root()
 248
 249     def repo_root(self):
 250         c = Command("git remote show origin", self.options)
 251         out = self.__run_in_repo(c.output_of)
 252         for line in out.split('\n'):
 253             if line.strip().startswith("Fetch URL:"):
 254                 repo = line.split()[2]
 255
 256     @classmethod
 257     def checkout(cls, remote, local, options, depth=1):
 258         Command("rm -rf %s" % local, options).run_silent()
 259         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 260         return GitRepository(local, options)
 261
 262     @classmethod
 263     def remote_exists(cls, remote):
 264         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 265
 266     def tag_exists(self, tagname):
 267         command = 'git tag -l | grep "^%s$"' % tagname
 268         c = Command(command, self.options)
 269         out = self.__run_in_repo(c.output_of, with_stderr=True)
 270         return len(out) > 0
 271
 272     def __run_in_repo(self, fun, *args, **kwargs):
 273         cwd = os.getcwd()
 274         os.chdir(self.path)
 275         ret = fun(*args, **kwargs)
 276         os.chdir(cwd)
 277         return ret
 278
 279     def __run_command_in_repo(self, command, ignore_errors=False):
 280         c = Command(command, self.options)
 281         if ignore_errors:
 282             return self.__run_in_repo(c.output_of)
 283         else:
 284             return self.__run_in_repo(c.run_fatal)
 285
 286     def update(self, subdir=None, recursive=None):
 287         return self.__run_command_in_repo("git pull")
 288
 289     def to_branch(self, branch, remote=True):
 290         if remote:
 291             branch = "origin/%s" % branch
 292         return self.__run_command_in_repo("git checkout %s" % branch)
 293
 294     def to_tag(self, tag):
 295         return self.__run_command_in_repo("git checkout %s" % tag)
 296
 297     def tag(self, tagname, logfile):
 298         self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 299         self.commit(logfile)
 300
 301     def diff(self):
 302         c = Command("git diff", self.options)
 303         return self.__run_in_repo(c.output_of, with_stderr=True)
 304
 305     def diff_with_tag(self, tagname):
 306         c = Command("git diff %s" % tagname, self.options)
 307         return self.__run_in_repo(c.output_of, with_stderr=True)
 308
 309     def commit(self, logfile):
 310         self.__run_command_in_repo("git add -A", ignore_errors=True)
 311         self.__run_command_in_repo("git commit -F  %s" % logfile, ignore_errors=True)
 312         self.__run_command_in_repo("git push")
 313         self.__run_command_in_repo("git push --tags")
 314
 315     def revert(self):
 316         self.__run_command_in_repo("git --no-pager reset --hard")
 317         self.__run_command_in_repo("git --no-pager clean -f")
 318
 319     def is_clean(self):
 320         def check_commit():
 321             command="git status"
 322             s="nothing to commit (working directory clean)"
 323             return Command(command, self.options).output_of(True).find(s) >= 0
 324         return self.__run_in_repo(check_commit)
 325
 326     def is_valid(self):
 327         return os.path.exists(os.path.join(self.path, ".git"))
 328
 329
 330 class Repository:
 331     """ Generic repository """
 332     supported_repo_types = [SvnRepository, GitRepository]
 333
 334     def __init__(self, path, options):
 335         self.path = path
 336         self.options = options
 337         for repo in self.supported_repo_types:
 338             self.repo = repo(self.path, self.options)
 339             if self.repo.is_valid():
 340                 break
 341
 342     @classmethod
 343     def has_moved_to_git(cls, module, svnpath):
 344         module = git_to_svn_name(module)
 345         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 346
 347     @classmethod
 348     def remote_exists(cls, remote):
 349         for repo in Repository.supported_repo_types:
 350             if repo.remote_exists(remote):
 351                 return True
 352         return False
 353
 354     def __getattr__(self, attr):
 355         return getattr(self.repo, attr)
 356
 357
 358
 359 # support for tagged module is minimal, and is for the Build class only
 360 class Module:
 361
 362     svn_magic_line="--This line, and those below, will be ignored--"
 363     setting_tag_format = "Setting tag %s"
 364
 365     redirectors=[ # ('module_name_varname','name'),
 366                   ('module_version_varname','version'),
 367                   ('module_taglevel_varname','taglevel'), ]
 368
 369     # where to store user's config
 370     config_storage="CONFIG"
 371     #
 372     config={}
 373
 374     import commands
 375     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 376                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 377                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 378                  ("build", "Enter the name of your build module","build"),
 379                  ('username',"Enter your firstname and lastname for changelogs",""),
 380                  ("email","Enter your email address for changelogs",""),
 381                  ]
 382
 383     @classmethod
 384     def prompt_config (cls):
 385         for (key,message,default) in cls.configKeys:
 386             cls.config[key]=""
 387             while not cls.config[key]:
 388                 cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 389
 390
 391     # for parsing module spec name:branch
 392     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 393     # special form for tagged module - for Build
 394     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 395     # parsing specfiles
 396     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 397
 398     def __init__ (self,module_spec,options):
 399         # parse module spec
 400         attempt=Module.matcher_branch_spec.match(module_spec)
 401         if attempt:
 402             self.name=attempt.group('name')
 403             self.branch=attempt.group('branch')
 404         else:
 405             attempt=Module.matcher_tag_spec.match(module_spec)
 406             if attempt:
 407                 self.name=attempt.group('name')
 408                 self.tagname=attempt.group('tagname')
 409             else:
 410                 self.name=module_spec
 411
 412         # when available prefer to use git module name internally
 413         self.name = svn_to_git_name(self.name)
 414
 415         self.options=options
 416         self.module_dir="%s/%s"%(options.workdir,self.name)
 417         self.repository = None
 418         self.build = None
 419
 420     def run (self,command):
 421         return Command(command,self.options).run()
 422     def run_fatal (self,command):
 423         return Command(command,self.options).run_fatal()
 424     def run_prompt (self,message,fun, *args):
 425         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 426         if not self.options.verbose:
 427             while True:
 428                 choice=prompt(message,True,('s','how'))
 429                 if choice is True:
 430                     fun(*args)
 431                     return
 432                 elif choice is False:
 433                     print 'About to run function:', fun_msg
 434         else:
 435             question=message+" - want to run function: " + fun_msg
 436             if prompt(question,True):
 437                 fun(*args)
 438
 439     def friendly_name (self):
 440         if hasattr(self,'branch'):
 441             return "%s:%s"%(self.name,self.branch)
 442         elif hasattr(self,'tagname'):
 443             return "%s@%s"%(self.name,self.tagname)
 444         else:
 445             return self.name
 446
 447     @classmethod
 448     def git_remote_dir (cls, name):
 449         return "%s:/git/%s.git" % (cls.config['gitserver'], name)
 450
 451     @classmethod
 452     def svn_remote_dir (cls, name):
 453         name = git_to_svn_name(name)
 454         svn = cls.config['svnpath']
 455         if svn.endswith('/'):
 456             return "%s%s" % (svn, name)
 457         return "%s/%s" % (svn, name)
 458
 459     def svn_selected_remote(self):
 460         svn_name = git_to_svn_name(self.name)
 461         remote = self.svn_remote_dir(svn_name)
 462         if hasattr(self,'branch'):
 463             remote = "%s/branches/%s" % (remote, self.branch)
 464         elif hasattr(self,'tagname'):
 465             remote = "%s/tags/%s" % (remote, self.tagname)
 466         else:
 467             remote = "%s/trunk" % remote
 468         return remote
 469
 470     ####################
 471     @classmethod
 472     def init_homedir (cls, options):
 473         if options.verbose and options.mode not in Main.silent_modes:
 474             print 'Checking for', options.workdir
 475         storage="%s/%s"%(options.workdir, cls.config_storage)
 476         # sanity check. Either the topdir exists AND we have a config/storage
 477         # or topdir does not exist and we create it
 478         # to avoid people use their own daily svn repo
 479         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 480             print """The directory %s exists and has no CONFIG file
 481 If this is your regular working directory, please provide another one as the
 482 module-* commands need a fresh working dir. Make sure that you do not use
 483 that for other purposes than tagging""" % options.workdir
 484             sys.exit(1)
 485         if not os.path.isdir (options.workdir):
 486             print "Cannot find",options.workdir,"let's create it"
 487             cls.prompt_config()
 488             print "Checking ...",
 489             remote = cls.git_remote_dir(cls.config['build'])
 490             local = os.path.join(options.workdir, cls.config['build'])
 491             GitRepository.checkout(remote, local, options, depth=1)
 492             print "OK"
 493
 494             # store config
 495             f=file(storage,"w")
 496             for (key,message,default) in Module.configKeys:
 497                 f.write("%s=%s\n"%(key,Module.config[key]))
 498             f.close()
 499             if options.debug:
 500                 print 'Stored',storage
 501                 Command("cat %s"%storage,options).run()
 502         else:
 503             # read config
 504             f=open(storage)
 505             for line in f.readlines():
 506                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 507                 Module.config[key]=value
 508             f.close()
 509
 510             build_dir = os.path.join(options.workdir, cls.config['build'])
 511             build = Repository(build_dir, options)
 512             if not build.is_clean():
 513                 print "build module needs a revert"
 514                 build.revert()
 515                 print "OK"
 516             build.update()
 517
 518         if options.verbose and options.mode not in Main.silent_modes:
 519             print '******** Using config'
 520             for (key,message,default) in Module.configKeys:
 521                 print '\t',key,'=',Module.config[key]
 522
 523     def init_module_dir (self):
 524         if self.options.verbose:
 525             print 'Checking for',self.module_dir
 526
 527         if not os.path.isdir (self.module_dir):
 528             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 529                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 530                                                          self.module_dir,
 531                                                          self.options)
 532             else:
 533                 remote = self.svn_selected_remote()
 534                 self.repository = SvnRepository.checkout(remote,
 535                                                          self.module_dir,
 536                                                          self.options, recursive=False)
 537
 538         self.repository = Repository(self.module_dir, self.options)
 539         if self.repository.type == "svn":
 540             # check if module has moved to git
 541             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 542                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 543                 self.init_module_dir()
 544             # check if we have the required branch/tag
 545             if self.repository.url() != self.svn_selected_remote():
 546                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 547                 self.init_module_dir()
 548
 549         elif self.repository.type == "git":
 550             if hasattr(self,'branch'):
 551                 self.repository.to_branch(self.branch)
 552             elif hasattr(self,'tagname'):
 553                 self.repository.to_tag(self.tagname)
 554
 555         else:
 556             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 557
 558
 559     def revert_module_dir (self):
 560         if self.options.fast_checks:
 561             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 562             return
 563         if self.options.verbose:
 564             print 'Checking whether', self.module_dir, 'needs being reverted'
 565
 566         if not self.repository.is_clean():
 567             self.repository.revert()
 568
 569     def update_module_dir (self):
 570         if self.options.fast_checks:
 571             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 572             return
 573         if self.options.verbose:
 574             print 'Updating', self.module_dir
 575         self.repository.update()
 576
 577     def main_specname (self):
 578         attempt="%s/%s.spec"%(self.module_dir,self.name)
 579         if os.path.isfile (attempt):
 580             return attempt
 581         pattern1="%s/*.spec"%self.module_dir
 582         level1=glob(pattern1)
 583         if level1:
 584             return level1[0]
 585         pattern2="%s/*/*.spec"%self.module_dir
 586         level2=glob(pattern2)
 587
 588         if level2:
 589             return level2[0]
 590         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 591
 592     def all_specnames (self):
 593         level1=glob("%s/*.spec" % self.module_dir)
 594         if level1: return level1
 595         level2=glob("%s/*/*.spec" % self.module_dir)
 596         return level2
 597
 598     def parse_spec (self, specfile, varnames):
 599         if self.options.verbose:
 600             print 'Parsing',specfile,
 601             for var in varnames:
 602                 print "[%s]"%var,
 603             print ""
 604         result={}
 605         f=open(specfile)
 606         for line in f.readlines():
 607             attempt=Module.matcher_rpm_define.match(line)
 608             if attempt:
 609                 (define,var,value)=attempt.groups()
 610                 if var in varnames:
 611                     result[var]=value
 612         f.close()
 613         if self.options.debug:
 614             print 'found',len(result),'keys'
 615             for (k,v) in result.iteritems():
 616                 print k,'=',v
 617         return result
 618
 619     # stores in self.module_name_varname the rpm variable to be used for the module's name
 620     # and the list of these names in self.varnames
 621     def spec_dict (self):
 622         specfile=self.main_specname()
 623         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 624         redirect_dict = self.parse_spec(specfile,redirector_keys)
 625         if self.options.debug:
 626             print '1st pass parsing done, redirect_dict=',redirect_dict
 627         varnames=[]
 628         for (varname,default) in Module.redirectors:
 629             if redirect_dict.has_key(varname):
 630                 setattr(self,varname,redirect_dict[varname])
 631                 varnames += [redirect_dict[varname]]
 632             else:
 633                 setattr(self,varname,default)
 634                 varnames += [ default ]
 635         self.varnames = varnames
 636         result = self.parse_spec (specfile,self.varnames)
 637         if self.options.debug:
 638             print '2st pass parsing done, varnames=',varnames,'result=',result
 639         return result
 640
 641     def patch_spec_var (self, patch_dict,define_missing=False):
 642         for specfile in self.all_specnames():
 643             # record the keys that were changed
 644             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 645             newspecfile=specfile+".new"
 646             if self.options.verbose:
 647                 print 'Patching',specfile,'for',patch_dict.keys()
 648             spec=open (specfile)
 649             new=open(newspecfile,"w")
 650
 651             for line in spec.readlines():
 652                 attempt=Module.matcher_rpm_define.match(line)
 653                 if attempt:
 654                     (define,var,value)=attempt.groups()
 655                     if var in patch_dict.keys():
 656                         if self.options.debug:
 657                             print 'rewriting %s as %s'%(var,patch_dict[var])
 658                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 659                         changed[var]=True
 660                         continue
 661                 new.write(line)
 662             if define_missing:
 663                 for (key,was_changed) in changed.iteritems():
 664                     if not was_changed:
 665                         if self.options.debug:
 666                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 667                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 668             spec.close()
 669             new.close()
 670             os.rename(newspecfile,specfile)
 671
 672     # returns all lines until the magic line
 673     def unignored_lines (self, logfile):
 674         result=[]
 675         white_line_matcher = re.compile("\A\s*\Z")
 676         for logline in file(logfile).readlines():
 677             if logline.strip() == Module.svn_magic_line:
 678                 break
 679             elif white_line_matcher.match(logline):
 680                 continue
 681             else:
 682                 result.append(logline.strip()+'\n')
 683         return result
 684
 685     # creates a copy of the input with only the unignored lines
 686     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 687        f=file(fileout,'w')
 688        f.write(self.setting_tag_format%new_tag_name + '\n')
 689        for line in self.unignored_lines(filein):
 690            f.write(line)
 691        f.close()
 692
 693     def insert_changelog (self, logfile, oldtag, newtag):
 694         for specfile in self.all_specnames():
 695             newspecfile=specfile+".new"
 696             if self.options.verbose:
 697                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 698             spec=open (specfile)
 699             new=open(newspecfile,"w")
 700             for line in spec.readlines():
 701                 new.write(line)
 702                 if re.compile('%changelog').match(line):
 703                     dateformat="* %a %b %d %Y"
 704                     datepart=time.strftime(dateformat)
 705                     logpart="%s <%s> - %s"%(Module.config['username'],
 706                                                  Module.config['email'],
 707                                                  newtag)
 708                     new.write(datepart+" "+logpart+"\n")
 709                     for logline in self.unignored_lines(logfile):
 710                         new.write("- " + logline)
 711                     new.write("\n")
 712             spec.close()
 713             new.close()
 714             os.rename(newspecfile,specfile)
 715
 716     def show_dict (self, spec_dict):
 717         if self.options.verbose:
 718             for (k,v) in spec_dict.iteritems():
 719                 print k,'=',v
 720
 721     def last_tag (self, spec_dict):
 722         try:
 723             return "%s-%s" % (spec_dict[self.module_version_varname],
 724                               spec_dict[self.module_taglevel_varname])
 725         except KeyError,err:
 726             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 727
 728     def tag_name (self, spec_dict, old_svn_name=False):
 729         base_tag_name = self.name
 730         if old_svn_name:
 731             base_tag_name = git_to_svn_name(self.name)
 732         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 733
 734
 735 ##############################
 736     # using fine_grain means replacing only those instances that currently refer to this tag
 737     # otherwise, <module>-SVNPATH is replaced unconditionnally
 738     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 739         newtagsfile=tagsfile+".new"
 740         tags=open (tagsfile)
 741         new=open(newtagsfile,"w")
 742
 743         matches=0
 744         # fine-grain : replace those lines that refer to oldname
 745         if fine_grain:
 746             if self.options.verbose:
 747                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 748             matcher=re.compile("^(.*)%s(.*)"%oldname)
 749             for line in tags.readlines():
 750                 if not matcher.match(line):
 751                     new.write(line)
 752                 else:
 753                     (begin,end)=matcher.match(line).groups()
 754                     new.write(begin+newname+end+"\n")
 755                     matches += 1
 756         # brute-force : change uncommented lines that define <module>-SVNPATH
 757         else:
 758             if self.options.verbose:
 759                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 760             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 761                                           %(self.name)
 762             matcher_module=re.compile(pattern)
 763             for line in tags.readlines():
 764                 attempt=matcher_module.match(line)
 765                 if attempt:
 766                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 767                     if self.options.verbose:
 768                         print ' '+svnpath,
 769                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 770                     new.write(replacement)
 771                     matches += 1
 772                 else:
 773                     new.write(line)
 774         tags.close()
 775         new.close()
 776         os.rename(newtagsfile,tagsfile)
 777         if self.options.verbose: print "%d changes"%matches
 778         return matches
 779
 780     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 781         if self.options.verbose:
 782             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 783
 784         found_tagname = tagname
 785         found = self.repository.tag_exists(tagname)
 786         if not found and old_svn_tag_name:
 787             if self.options.verbose:
 788                 print "KO"
 789                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 790             found = self.repository.tag_exists(old_svn_tag_name)
 791             if found:
 792                 found_tagname = old_svn_tag_name
 793
 794         if (found and need_it) or (not found and not need_it):
 795             print "OK "
 796         else:
 797             print "KO"
 798             if found:
 799                 raise Exception, "tag (%s) is already there" % tagname
 800             else:
 801                 raise Exception, "can not find required tag (%s)" % tagname
 802
 803         return found_tagname
 804
 805
 806     def do_tag (self):
 807         self.init_module_dir()
 808         self.revert_module_dir()
 809         self.update_module_dir()
 810         # parse specfile
 811         spec_dict = self.spec_dict()
 812         self.show_dict(spec_dict)
 813
 814         # side effects
 815         old_tag_name = self.tag_name(spec_dict)
 816         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 817
 818         if (self.options.new_version):
 819             # new version set on command line
 820             spec_dict[self.module_version_varname] = self.options.new_version
 821             spec_dict[self.module_taglevel_varname] = 0
 822         else:
 823             # increment taglevel
 824             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 825             spec_dict[self.module_taglevel_varname] = new_taglevel
 826
 827         new_tag_name = self.tag_name(spec_dict)
 828
 829         # sanity check
 830         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 831         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 832
 833         # checking for diffs
 834         diff_output = self.repository.diff_with_tag(old_tag_name)
 835         if len(diff_output) == 0:
 836             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 837                 return
 838
 839         # side effect in trunk's specfile
 840         self.patch_spec_var(spec_dict)
 841
 842         # prepare changelog file
 843         # we use the standard subversion magic string (see svn_magic_line)
 844         # so we can provide useful information, such as version numbers and diff
 845         # in the same file
 846         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 847         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 848         setting_tag_line=Module.setting_tag_format%new_tag_name
 849         file(changelog,"w").write("""
 850 %s
 851 %s
 852 Please write a changelog for this new tag in the section above
 853 """%(Module.svn_magic_line,setting_tag_line))
 854
 855         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 856             file(changelog,"a").write('DIFF=========\n' + diff_output)
 857
 858         if self.options.debug:
 859             prompt('Proceed ?')
 860
 861         # edit it
 862         self.run("%s %s"%(self.options.editor,changelog))
 863         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 864         # so we do the job ourselves
 865         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 866         # insert changelog in spec
 867         if self.options.changelog:
 868             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 869
 870         ## update build
 871         build_path = os.path.join(self.options.workdir,
 872                                   Module.config['build'])
 873         build = Repository(build_path, self.options)
 874         if self.options.build_branch:
 875             build.to_branch(self.options.build_branch)
 876         if not build.is_clean():
 877             build.revert()
 878
 879         tagsfiles=glob(build.path+"/*-tags*.mk")
 880         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 881         default_answer = 'y'
 882         tagsfiles.sort()
 883         while True:
 884             for tagsfile in tagsfiles:
 885                 status=tagsdict[tagsfile]
 886                 basename=os.path.basename(tagsfile)
 887                 print ".................... Dealing with %s"%basename
 888                 while tagsdict[tagsfile] == 'todo' :
 889                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 890                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 891                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 892                                      allow_outside=True)
 893                     if choice == 'y':
 894                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 895                     elif choice == 'n':
 896                         print 'Done with %s'%os.path.basename(tagsfile)
 897                         tagsdict[tagsfile]='done'
 898                     elif choice == 'f':
 899                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 900                     elif choice == 'd':
 901                         self.run("svn diff %s"%tagsfile)
 902                     elif choice == 'r':
 903                         self.run("svn revert %s"%tagsfile)
 904                     elif choice == 'c':
 905                         self.run("cat %s"%tagsfile)
 906                     else:
 907                         name=self.name
 908                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 909 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 910 d: show current diff for this tag file
 911 r: revert that tag file
 912 c: cat the current tag file
 913 n: move to next file"""%locals()
 914
 915             if prompt("Want to review changes on tags files",False):
 916                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 917                 default_answer='d'
 918             else:
 919                 break
 920
 921         def diff_all_changes():
 922             print build.diff()
 923             print self.repository.diff()
 924
 925         def commit_all_changes(log):
 926             self.repository.commit(log)
 927             build.commit(log)
 928
 929         self.run_prompt("Review module and build", diff_all_changes)
 930         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 931         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 932
 933         if self.options.debug:
 934             print 'Preserving',changelog,'and stripped',changelog_svn
 935         else:
 936             os.unlink(changelog)
 937             os.unlink(changelog_svn)
 938
 939
 940 ##############################
 941 class Main:
 942
 943     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 944
 945 module-tools : a set of tools to manage subversion tags and specfile
 946   requires the specfile to either
 947   * define *version* and *taglevel*
 948   OR alternatively
 949   * define redirection variables module_version_varname / module_taglevel_varname
 950 Trunk:
 951   by default, the trunk of modules is taken into account
 952   in this case, just mention the module name as <module_desc>
 953 Branches:
 954   if you wish to work on a branch rather than on the trunk,
 955   you can use something like e.g. Mom:2.1 as <module_desc>
 956 """
 957     release_usage="""Usage: %prog [options] tag1 .. tagn
 958   Extract release notes from the changes in specfiles between several build tags, latest first
 959   Examples:
 960       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 961   You can refer to a (build) branch by prepending a colon, like in
 962       release-changelog :4.2 4.2-rc25
 963   You can refer to the build trunk by just mentioning 'trunk', e.g.
 964       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 965 """
 966     common_usage="""More help:
 967   see http://svn.planet-lab.org/wiki/ModuleTools"""
 968
 969     modes={
 970         'list' : "displays a list of available tags or branches",
 971         'version' : "check latest specfile and print out details",
 972         'diff' : "show difference between module (trunk or branch) and latest tag",
 973         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 974                 create new tag and and monitor its adoption in build/*-tags*.mk""",
 975         'branch' : """create a branch for this module, from the latest tag on the trunk,
 976                   and change trunk's version number to reflect the new branch name;
 977                   you can specify the new branch name by using module:branch""",
 978         'sync' : """create a tag from the module
 979                 this is a last resort option, mostly for repairs""",
 980         'changelog' : """extract changelog between build tags
 981                 expected arguments are a list of tags""",
 982         }
 983
 984     silent_modes = ['list']
 985     release_modes = ['changelog']
 986
 987     @staticmethod
 988     def optparse_list (option, opt, value, parser):
 989         try:
 990             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
 991         except:
 992             setattr(parser.values,option.dest,value.split())
 993
 994     def run(self):
 995
 996         mode=None
 997         for function in Main.modes.keys():
 998             if sys.argv[0].find(function) >= 0:
 999                 mode = function
1000                 break
1001         if not mode:
1002             print "Unsupported command",sys.argv[0]
1003             print "Supported commands:" + " ".join(Main.modes.keys())
1004             sys.exit(1)
1005
1006         if mode not in Main.release_modes:
1007             usage = Main.module_usage
1008             usage += Main.common_usage
1009             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1010         else:
1011             usage = Main.release_usage
1012             usage += Main.common_usage
1013
1014         parser=OptionParser(usage=usage,version=subversion_id)
1015
1016         if mode == "tag" or mode == 'branch':
1017             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1018                               help="set new version and reset taglevel to 0")
1019         if mode == "tag" :
1020             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1021                               help="do not update changelog section in specfile when tagging")
1022             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1023                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1024         if mode == "tag" or mode == "sync" :
1025             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1026                               help="specify editor")
1027
1028         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1029         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1030                           help="dry run - shell commands are only displayed")
1031         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1032                           default=[], nargs=1,type="string",
1033                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1034 -- can be set multiple times, or use quotes""")
1035
1036         parser.add_option("-w","--workdir", action="store", dest="workdir",
1037                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1038                           help="""name for dedicated working dir - defaults to ~/modules
1039 ** THIS MUST NOT ** be your usual working directory""")
1040         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1041                           help="skip safety checks, such as svn updates -- use with care")
1042
1043         # default verbosity depending on function - temp
1044         verbose_modes= ['tag', 'sync', 'branch']
1045
1046         if mode not in verbose_modes:
1047             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1048                               help="run in verbose mode")
1049         else:
1050             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1051                               help="run in quiet (non-verbose) mode")
1052         (options, args) = parser.parse_args()
1053         options.mode=mode
1054         if not hasattr(options,'dry_run'):
1055             options.dry_run=False
1056         if not hasattr(options,'www'):
1057             options.www=False
1058         options.debug=False
1059
1060         ########## module-*
1061         if len(args) == 0:
1062             parser.print_help()
1063             sys.exit(1)
1064         Module.init_homedir(options)
1065
1066         modules=[ Module(modname,options) for modname in args ]
1067         for module in modules:
1068             if len(args)>1 and mode not in Main.silent_modes:
1069                 print '========================================',module.friendly_name()
1070             # call the method called do_<mode>
1071             method=Module.__dict__["do_%s"%mode]
1072             try:
1073                 method(module)
1074             except Exception,e:
1075                 import traceback
1076                 traceback.print_exc()
1077                 print 'Skipping module %s: '%modname,e
1078
1079 ####################
1080 if __name__ == "__main__" :
1081     try:
1082         Main().run()
1083     except KeyboardInterrupt:
1084         print '\nBye'