module-tools.py

   1 #!/usr/bin/python -u
   2
   3 subversion_id = "$Id$"
   4
   5 import sys, os
   6 import re
   7 import time
   8 from glob import glob
   9 from optparse import OptionParser
  10
  11 # HARDCODED NAME CHANGES
  12 #
  13 # Moving to git we decided to rename some of the repositories. Here is
  14 # a map of name changes applied in git repositories.
  15 RENAMED_SVN_MODULES = {
  16     "PLEWWW": "plewww"
  17     }
  18
  19 def svn_to_git_name(module):
  20     if RENAMED_SVN_MODULES.has_key(module):
  21         return RENAMED_SVN_MODULES[module]
  22     return module
  23
  24 def git_to_svn_name(module):
  25     for key in RENAMED_SVN_MODULES:
  26         if module == RENAMED_SVN_MODULES[key]:
  27             return key
  28     return module
  29
  30
  31 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  32 def prompt (question,default=True,other_choices=[],allow_outside=False):
  33     if not isinstance (other_choices,list):
  34         other_choices = [ other_choices ]
  35     chars = [ c for (c,rest) in other_choices ]
  36
  37     choices = []
  38     if 'y' not in chars:
  39         if default is True: choices.append('[y]')
  40         else : choices.append('y')
  41     if 'n' not in chars:
  42         if default is False: choices.append('[n]')
  43         else : choices.append('n')
  44
  45     for (char,choice) in other_choices:
  46         if default == char:
  47             choices.append("["+char+"]"+choice)
  48         else:
  49             choices.append("<"+char+">"+choice)
  50     try:
  51         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  52         if not answer:
  53             return default
  54         answer=answer[0].lower()
  55         if answer == 'y':
  56             if 'y' in chars: return 'y'
  57             else: return True
  58         elif answer == 'n':
  59             if 'n' in chars: return 'n'
  60             else: return False
  61         elif other_choices:
  62             for (char,choice) in other_choices:
  63                 if answer == char:
  64                     return char
  65             if allow_outside:
  66                 return answer
  67         return prompt(question,default,other_choices)
  68     except:
  69         raise
  70
  71 def default_editor():
  72     try:
  73         editor = os.environ['EDITOR']
  74     except:
  75         editor = "emacs"
  76     return editor
  77
  78 ### fold long lines
  79 fold_length=132
  80
  81 def print_fold (line):
  82     while len(line) >= fold_length:
  83         print line[:fold_length],'\\'
  84         line=line[fold_length:]
  85     print line
  86
  87 class Command:
  88     def __init__ (self,command,options):
  89         self.command=command
  90         self.options=options
  91         self.tmp="/tmp/command-%d"%os.getpid()
  92
  93     def run (self):
  94         if self.options.dry_run:
  95             print 'dry_run',self.command
  96             return 0
  97         if self.options.verbose and self.options.mode not in Main.silent_modes:
  98             print '+',self.command
  99             sys.stdout.flush()
 100         return os.system(self.command)
 101
 102     def run_silent (self):
 103         if self.options.dry_run:
 104             print 'dry_run',self.command
 105             return 0
 106         if self.options.verbose:
 107             print '+',self.command,' .. ',
 108             sys.stdout.flush()
 109         retcod=os.system(self.command + " &> " + self.tmp)
 110         if retcod != 0:
 111             print "FAILED ! -- out+err below (command was %s)"%self.command
 112             os.system("cat " + self.tmp)
 113             print "FAILED ! -- end of quoted output"
 114         elif self.options.verbose:
 115             print "OK"
 116         os.unlink(self.tmp)
 117         return retcod
 118
 119     def run_fatal(self):
 120         if self.run_silent() !=0:
 121             raise Exception,"Command %s failed"%self.command
 122
 123     # returns stdout, like bash's $(mycommand)
 124     def output_of (self,with_stderr=False):
 125         if self.options.dry_run:
 126             print 'dry_run',self.command
 127             return 'dry_run output'
 128         tmp="/tmp/status-%d"%os.getpid()
 129         if self.options.debug:
 130             print '+',self.command,' .. ',
 131             sys.stdout.flush()
 132         command=self.command
 133         if with_stderr:
 134             command += " &> "
 135         else:
 136             command += " > "
 137         command += tmp
 138         os.system(command)
 139         result=file(tmp).read()
 140         os.unlink(tmp)
 141         if self.options.debug:
 142             print 'Done',
 143         return result
 144
 145
 146 class SvnRepository:
 147     type = "svn"
 148
 149     def __init__(self, path, options):
 150         self.path = path
 151         self.options = options
 152
 153     def name(self):
 154         return os.path.basename(self.path)
 155
 156     def url(self):
 157         out = Command("svn info %s" % self.path, self.options).output_of()
 158         for line in out.split('\n'):
 159             if line.startswith("URL:"):
 160                 return line.split()[1].strip()
 161
 162     def repo_root(self):
 163         out = Command("svn info %s" % self.path, self.options).output_of()
 164         for line in out.split('\n'):
 165             if line.startswith("Repository Root:"):
 166                 repo_root = line.split()[2].strip()
 167                 return "%s/svn/%s" (repo_root, self.name)
 168
 169     @classmethod
 170     def checkout(cls, remote, local, options, recursive=False):
 171         if recursive:
 172             svncommand = "svn co %s %s" % (remote, local)
 173         else:
 174             svncommand = "svn co -N %s %s" % (remote, local)
 175         Command("rm -rf %s" % local, options).run_silent()
 176         Command(svncommand, options).run_fatal()
 177
 178         return SvnRepository(local, options)
 179
 180     @classmethod
 181     def remote_exists(cls, remote):
 182         return os.system("svn list %s &> /dev/null" % remote) == 0
 183
 184     def tag_exists(self, tagname):
 185         url = "%s/tags/%s" % (self.repo_root(), tagname)
 186         return SvnRepository.remote_exists(url)
 187
 188     def update(self, subdir="", recursive=True):
 189         path = os.path.join(self.path, subdir)
 190         if recursive:
 191             svncommand = "svn up %s" % path
 192         else:
 193             svncommand = "svn up -N %s" % path
 194         Command(svncommand, self.options).run_fatal()
 195
 196     def commit(self, logfile):
 197         # add all new files to the repository
 198         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 199                 self.path, self.options).run_silent()
 200         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 201
 202     def to_branch(self, branch):
 203         remote = "%s/branches/%s" % (self.repo_root(), branch)
 204         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 205
 206     def to_tag(self, tag):
 207         remote = "%s/tags/%s" % (self.repo_root(), branch)
 208         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 209
 210     def tag(self, tagname, logfile):
 211         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 212         self_url = self.url()
 213         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 214
 215     def diff(self):
 216         return Command("svn diff %s" % self.path, self.options).output_of(True)
 217
 218     def diff_with_tag(self, tagname):
 219         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 220         return Command("svn diff %s %s" % (tag_url, self.url()),
 221                        self.options).output_of(True)
 222
 223     def revert(self):
 224         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 225         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 226                 self.path, self.options).run_silent()
 227
 228     def is_clean(self):
 229         command="svn status %s" % self.path
 230         return len(Command(command,self.options).output_of(True)) == 0
 231
 232     def is_valid(self):
 233         return os.path.exists(os.path.join(self.path, ".svn"))
 234
 235
 236 class GitRepository:
 237     type = "git"
 238
 239     def __init__(self, path, options):
 240         self.path = path
 241         self.options = options
 242
 243     def name(self):
 244         return os.path.basename(self.path)
 245
 246     def url(self):
 247         self.repo_root()
 248
 249     def repo_root(self):
 250         c = Command("git remote show origin", self.options)
 251         out = self.__run_in_repo(c.output_of)
 252         for line in out.split('\n'):
 253             if line.strip().startswith("Fetch URL:"):
 254                 repo = line.split()[2]
 255
 256     @classmethod
 257     def checkout(cls, remote, local, options, depth=1):
 258         Command("rm -rf %s" % local, options).run_silent()
 259         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 260         return GitRepository(local, options)
 261
 262     @classmethod
 263     def remote_exists(cls, remote):
 264         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 265
 266     def tag_exists(self, tagname):
 267         command = 'git tag -l | grep "^%s$"' % tagname
 268         c = Command(command, self.options)
 269         out = self.__run_in_repo(c.output_of, with_stderr=True)
 270         return len(out) > 0
 271
 272     def __run_in_repo(self, fun, *args, **kwargs):
 273         cwd = os.getcwd()
 274         os.chdir(self.path)
 275         ret = fun(*args, **kwargs)
 276         os.chdir(cwd)
 277         return ret
 278
 279     def __run_command_in_repo(self, command):
 280         c = Command(command, self.options)
 281         return self.__run_in_repo(c.run_fatal)
 282
 283     def update(self, subdir=None, recursive=None):
 284         return self.__run_command_in_repo("git pull")
 285
 286     def to_branch(self, branch, remote=True):
 287         if remote:
 288             branch = "origin/%s" % branch
 289         return self.__run_command_in_repo("git checkout %s" % branch)
 290
 291     def to_tag(self, tag):
 292         return self.__run_command_in_repo("git checkout %s" % tag)
 293
 294     def tag(self, tagname, logfile):
 295         self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 296         self.commit(logfile)
 297
 298     def diff(self):
 299         c = Command("git diff", self.options)
 300         return self.__run_in_repo(c.output_of, with_stderr=True)
 301
 302     def diff_with_tag(self, tagname):
 303         c = Command("git diff %s" % tagname, self.options)
 304         return self.__run_in_repo(c.output_of, with_stderr=True)
 305
 306     def commit(self, logfile):
 307         self.__run_command_in_repo("git add -A")
 308         self.__run_command_in_repo("git commit -F  %s" % logfile)
 309         self.__run_command_in_repo("git push --tags")
 310
 311     def revert(self):
 312         self.__run_command_in_repo("git --no-pager reset --hard")
 313         self.__run_command_in_repo("git --no-pager clean -f")
 314
 315     def is_clean(self):
 316         def check_commit():
 317             command="git status"
 318             s="nothing to commit (working directory clean)"
 319             return Command(command, self.options).output_of(True).find(s) >= 0
 320         return self.__run_in_repo(check_commit)
 321
 322     def is_valid(self):
 323         return os.path.exists(os.path.join(self.path, ".git"))
 324
 325
 326 class Repository:
 327     """ Generic repository """
 328     supported_repo_types = [SvnRepository, GitRepository]
 329
 330     def __init__(self, path, options):
 331         self.path = path
 332         self.options = options
 333         for repo in self.supported_repo_types:
 334             self.repo = repo(self.path, self.options)
 335             if self.repo.is_valid():
 336                 break
 337
 338     @classmethod
 339     def has_moved_to_git(cls, module, svnpath):
 340         module = git_to_svn_name(module)
 341         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 342
 343     @classmethod
 344     def remote_exists(cls, remote):
 345         for repo in Repository.supported_repo_types:
 346             if repo.remote_exists(remote):
 347                 return True
 348         return False
 349
 350     def __getattr__(self, attr):
 351         return getattr(self.repo, attr)
 352
 353
 354
 355 # support for tagged module is minimal, and is for the Build class only
 356 class Module:
 357
 358     svn_magic_line="--This line, and those below, will be ignored--"
 359     setting_tag_format = "Setting tag %s"
 360
 361     redirectors=[ # ('module_name_varname','name'),
 362                   ('module_version_varname','version'),
 363                   ('module_taglevel_varname','taglevel'), ]
 364
 365     # where to store user's config
 366     config_storage="CONFIG"
 367     #
 368     config={}
 369
 370     import commands
 371     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 372                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 373                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 374                  ("build", "Enter the name of your build module","build"),
 375                  ('username',"Enter your firstname and lastname for changelogs",""),
 376                  ("email","Enter your email address for changelogs",""),
 377                  ]
 378
 379     @classmethod
 380     def prompt_config (cls):
 381         for (key,message,default) in cls.configKeys:
 382             cls.config[key]=""
 383             while not cls.config[key]:
 384                 cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 385
 386
 387     # for parsing module spec name:branch
 388     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 389     # special form for tagged module - for Build
 390     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 391     # parsing specfiles
 392     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 393
 394     def __init__ (self,module_spec,options):
 395         # parse module spec
 396         attempt=Module.matcher_branch_spec.match(module_spec)
 397         if attempt:
 398             self.name=attempt.group('name')
 399             self.branch=attempt.group('branch')
 400         else:
 401             attempt=Module.matcher_tag_spec.match(module_spec)
 402             if attempt:
 403                 self.name=attempt.group('name')
 404                 self.tagname=attempt.group('tagname')
 405             else:
 406                 self.name=module_spec
 407
 408         # when available prefer to use git module name internally
 409         self.name = svn_to_git_name(self.name)
 410
 411         self.options=options
 412         self.module_dir="%s/%s"%(options.workdir,self.name)
 413         self.repository = None
 414         self.build = None
 415
 416     def run (self,command):
 417         return Command(command,self.options).run()
 418     def run_fatal (self,command):
 419         return Command(command,self.options).run_fatal()
 420     def run_prompt (self,message,fun, *args):
 421         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 422         if not self.options.verbose:
 423             while True:
 424                 choice=prompt(message,True,('s','how'))
 425                 if choice is True:
 426                     fun(*args)
 427                     return
 428                 elif choice is False:
 429                     print 'About to run function:', fun_msg
 430         else:
 431             question=message+" - want to run function: " + fun_msg
 432             if prompt(question,True):
 433                 fun(*args)
 434
 435     def friendly_name (self):
 436         if hasattr(self,'branch'):
 437             return "%s:%s"%(self.name,self.branch)
 438         elif hasattr(self,'tagname'):
 439             return "%s@%s"%(self.name,self.tagname)
 440         else:
 441             return self.name
 442
 443     @classmethod
 444     def git_remote_dir (cls, name):
 445         return "%s:/git/%s.git" % (cls.config['gitserver'], name)
 446
 447     @classmethod
 448     def svn_remote_dir (cls, name):
 449         name = git_to_svn_name(name)
 450         svn = cls.config['svnpath']
 451         if svn.endswith('/'):
 452             return "%s%s" % (svn, name)
 453         return "%s/%s" % (svn, name)
 454
 455     def svn_selected_remote(self):
 456         svn_name = git_to_svn_name(self.name)
 457         remote = self.svn_remote_dir(svn_name)
 458         if hasattr(self,'branch'):
 459             remote = "%s/branches/%s" % (remote, self.branch)
 460         elif hasattr(self,'tagname'):
 461             remote = "%s/tags/%s" % (remote, self.tagname)
 462         else:
 463             remote = "%s/trunk" % remote
 464         return remote
 465
 466     ####################
 467     @classmethod
 468     def init_homedir (cls, options):
 469         if options.verbose and options.mode not in Main.silent_modes:
 470             print 'Checking for', options.workdir
 471         storage="%s/%s"%(options.workdir, cls.config_storage)
 472         # sanity check. Either the topdir exists AND we have a config/storage
 473         # or topdir does not exist and we create it
 474         # to avoid people use their own daily svn repo
 475         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 476             print """The directory %s exists and has no CONFIG file
 477 If this is your regular working directory, please provide another one as the
 478 module-* commands need a fresh working dir. Make sure that you do not use
 479 that for other purposes than tagging""" % options.workdir
 480             sys.exit(1)
 481         if not os.path.isdir (options.workdir):
 482             print "Cannot find",options.workdir,"let's create it"
 483             cls.prompt_config()
 484             print "Checking ...",
 485             remote = cls.git_remote_dir(cls.config['build'])
 486             local = os.path.join(options.workdir, cls.config['build'])
 487             build = GitRepository.checkout(remote, local, options, depth=1)
 488             if not build.is_clean():
 489                 build.revert()
 490             print "OK"
 491
 492             # store config
 493             f=file(storage,"w")
 494             for (key,message,default) in Module.configKeys:
 495                 f.write("%s=%s\n"%(key,Module.config[key]))
 496             f.close()
 497             if options.debug:
 498                 print 'Stored',storage
 499                 Command("cat %s"%storage,options).run()
 500         else:
 501             # read config
 502             f=open(storage)
 503             for line in f.readlines():
 504                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 505                 Module.config[key]=value
 506             f.close()
 507
 508             build_dir = os.path.join(options.workdir, cls.config['build'])
 509             build = Repository(build_dir, options)
 510             if not build.is_clean():
 511                 print "build module needs a revert"
 512                 build.revert()
 513                 print "OK"
 514
 515         if options.verbose and options.mode not in Main.silent_modes:
 516             print '******** Using config'
 517             for (key,message,default) in Module.configKeys:
 518                 print '\t',key,'=',Module.config[key]
 519
 520     def init_module_dir (self):
 521         if self.options.verbose:
 522             print 'Checking for',self.module_dir
 523
 524         if not os.path.isdir (self.module_dir):
 525             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 526                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 527                                                          self.module_dir,
 528                                                          self.options)
 529             else:
 530                 remote = self.svn_selected_remote()
 531                 self.repository = SvnRepository.checkout(remote,
 532                                                          self.module_dir,
 533                                                          self.options, recursive=False)
 534
 535         self.repository = Repository(self.module_dir, self.options)
 536         if self.repository.type == "svn":
 537             # check if module has moved to git
 538             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 539                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 540                 self.init_module_dir()
 541             # check if we have the required branch/tag
 542             if self.repository.url() != self.svn_selected_remote():
 543                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 544                 self.init_module_dir()
 545
 546         elif self.repository.type == "git":
 547             if hasattr(self,'branch'):
 548                 self.repository.to_branch(self.branch)
 549             elif hasattr(self,'tagname'):
 550                 self.repository.to_tag(self.tagname)
 551
 552         else:
 553             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 554
 555
 556     def revert_module_dir (self):
 557         if self.options.fast_checks:
 558             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 559             return
 560         if self.options.verbose:
 561             print 'Checking whether', self.module_dir, 'needs being reverted'
 562
 563         if not self.repository.is_clean():
 564             self.repository.revert()
 565
 566     def update_module_dir (self):
 567         if self.options.fast_checks:
 568             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 569             return
 570         if self.options.verbose:
 571             print 'Updating', self.module_dir
 572         self.repository.update()
 573
 574     def main_specname (self):
 575         attempt="%s/%s.spec"%(self.module_dir,self.name)
 576         if os.path.isfile (attempt):
 577             return attempt
 578         pattern1="%s/*.spec"%self.module_dir
 579         level1=glob(pattern1)
 580         if level1:
 581             return level1[0]
 582         pattern2="%s/*/*.spec"%self.module_dir
 583         level2=glob(pattern2)
 584
 585         if level2:
 586             return level2[0]
 587         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 588
 589     def all_specnames (self):
 590         level1=glob("%s/*.spec" % self.module_dir)
 591         if level1: return level1
 592         level2=glob("%s/*/*.spec" % self.module_dir)
 593         return level2
 594
 595     def parse_spec (self, specfile, varnames):
 596         if self.options.verbose:
 597             print 'Parsing',specfile,
 598             for var in varnames:
 599                 print "[%s]"%var,
 600             print ""
 601         result={}
 602         f=open(specfile)
 603         for line in f.readlines():
 604             attempt=Module.matcher_rpm_define.match(line)
 605             if attempt:
 606                 (define,var,value)=attempt.groups()
 607                 if var in varnames:
 608                     result[var]=value
 609         f.close()
 610         if self.options.debug:
 611             print 'found',len(result),'keys'
 612             for (k,v) in result.iteritems():
 613                 print k,'=',v
 614         return result
 615
 616     # stores in self.module_name_varname the rpm variable to be used for the module's name
 617     # and the list of these names in self.varnames
 618     def spec_dict (self):
 619         specfile=self.main_specname()
 620         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 621         redirect_dict = self.parse_spec(specfile,redirector_keys)
 622         if self.options.debug:
 623             print '1st pass parsing done, redirect_dict=',redirect_dict
 624         varnames=[]
 625         for (varname,default) in Module.redirectors:
 626             if redirect_dict.has_key(varname):
 627                 setattr(self,varname,redirect_dict[varname])
 628                 varnames += [redirect_dict[varname]]
 629             else:
 630                 setattr(self,varname,default)
 631                 varnames += [ default ]
 632         self.varnames = varnames
 633         result = self.parse_spec (specfile,self.varnames)
 634         if self.options.debug:
 635             print '2st pass parsing done, varnames=',varnames,'result=',result
 636         return result
 637
 638     def patch_spec_var (self, patch_dict,define_missing=False):
 639         for specfile in self.all_specnames():
 640             # record the keys that were changed
 641             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 642             newspecfile=specfile+".new"
 643             if self.options.verbose:
 644                 print 'Patching',specfile,'for',patch_dict.keys()
 645             spec=open (specfile)
 646             new=open(newspecfile,"w")
 647
 648             for line in spec.readlines():
 649                 attempt=Module.matcher_rpm_define.match(line)
 650                 if attempt:
 651                     (define,var,value)=attempt.groups()
 652                     if var in patch_dict.keys():
 653                         if self.options.debug:
 654                             print 'rewriting %s as %s'%(var,patch_dict[var])
 655                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 656                         changed[var]=True
 657                         continue
 658                 new.write(line)
 659             if define_missing:
 660                 for (key,was_changed) in changed.iteritems():
 661                     if not was_changed:
 662                         if self.options.debug:
 663                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 664                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 665             spec.close()
 666             new.close()
 667             os.rename(newspecfile,specfile)
 668
 669     # returns all lines until the magic line
 670     def unignored_lines (self, logfile):
 671         result=[]
 672         white_line_matcher = re.compile("\A\s*\Z")
 673         for logline in file(logfile).readlines():
 674             if logline.strip() == Module.svn_magic_line:
 675                 break
 676             elif white_line_matcher.match(logline):
 677                 continue
 678             else:
 679                 result.append(logline.strip()+'\n')
 680         return result
 681
 682     # creates a copy of the input with only the unignored lines
 683     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 684        f=file(fileout,'w')
 685        f.write(self.setting_tag_format%new_tag_name + '\n')
 686        for line in self.unignored_lines(filein):
 687            f.write(line)
 688        f.close()
 689
 690     def insert_changelog (self, logfile, oldtag, newtag):
 691         for specfile in self.all_specnames():
 692             newspecfile=specfile+".new"
 693             if self.options.verbose:
 694                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 695             spec=open (specfile)
 696             new=open(newspecfile,"w")
 697             for line in spec.readlines():
 698                 new.write(line)
 699                 if re.compile('%changelog').match(line):
 700                     dateformat="* %a %b %d %Y"
 701                     datepart=time.strftime(dateformat)
 702                     logpart="%s <%s> - %s"%(Module.config['username'],
 703                                                  Module.config['email'],
 704                                                  newtag)
 705                     new.write(datepart+" "+logpart+"\n")
 706                     for logline in self.unignored_lines(logfile):
 707                         new.write("- " + logline)
 708                     new.write("\n")
 709             spec.close()
 710             new.close()
 711             os.rename(newspecfile,specfile)
 712
 713     def show_dict (self, spec_dict):
 714         if self.options.verbose:
 715             for (k,v) in spec_dict.iteritems():
 716                 print k,'=',v
 717
 718     def last_tag (self, spec_dict):
 719         try:
 720             return "%s-%s" % (spec_dict[self.module_version_varname],
 721                               spec_dict[self.module_taglevel_varname])
 722         except KeyError,err:
 723             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 724
 725     def tag_name (self, spec_dict, old_svn_name=False):
 726         base_tag_name = self.name
 727         if old_svn_name:
 728             base_tag_name = git_to_svn_name(self.name)
 729         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 730
 731
 732 ##############################
 733     # using fine_grain means replacing only those instances that currently refer to this tag
 734     # otherwise, <module>-SVNPATH is replaced unconditionnally
 735     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 736         newtagsfile=tagsfile+".new"
 737         tags=open (tagsfile)
 738         new=open(newtagsfile,"w")
 739
 740         matches=0
 741         # fine-grain : replace those lines that refer to oldname
 742         if fine_grain:
 743             if self.options.verbose:
 744                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 745             matcher=re.compile("^(.*)%s(.*)"%oldname)
 746             for line in tags.readlines():
 747                 if not matcher.match(line):
 748                     new.write(line)
 749                 else:
 750                     (begin,end)=matcher.match(line).groups()
 751                     new.write(begin+newname+end+"\n")
 752                     matches += 1
 753         # brute-force : change uncommented lines that define <module>-SVNPATH
 754         else:
 755             if self.options.verbose:
 756                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 757             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 758                                           %(self.name)
 759             matcher_module=re.compile(pattern)
 760             for line in tags.readlines():
 761                 attempt=matcher_module.match(line)
 762                 if attempt:
 763                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 764                     if self.options.verbose:
 765                         print ' '+svnpath,
 766                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 767                     new.write(replacement)
 768                     matches += 1
 769                 else:
 770                     new.write(line)
 771         tags.close()
 772         new.close()
 773         os.rename(newtagsfile,tagsfile)
 774         if self.options.verbose: print "%d changes"%matches
 775         return matches
 776
 777     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 778         if self.options.verbose:
 779             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 780
 781         found_tagname = tagname
 782         found = self.repository.tag_exists(tagname)
 783         if not found and old_svn_tag_name:
 784             if self.options.verbose:
 785                 print "KO"
 786                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 787             found = self.repository.tag_exists(old_svn_tag_name)
 788             if found:
 789                 found_tagname = old_svn_tag_name
 790
 791         if (found and need_it) or (not found and not need_it):
 792             print "OK "
 793         else:
 794             print "KO"
 795             if found:
 796                 raise Exception, "tag (%s) is already there" % tagname
 797             else:
 798                 raise Exception, "can not find required tag (%s)" % tagname
 799
 800         return found_tagname
 801
 802
 803     def do_tag (self):
 804         self.init_module_dir()
 805         self.revert_module_dir()
 806         self.update_module_dir()
 807         # parse specfile
 808         spec_dict = self.spec_dict()
 809         self.show_dict(spec_dict)
 810
 811         # side effects
 812         old_tag_name = self.tag_name(spec_dict)
 813         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 814
 815         if (self.options.new_version):
 816             # new version set on command line
 817             spec_dict[self.module_version_varname] = self.options.new_version
 818             spec_dict[self.module_taglevel_varname] = 0
 819         else:
 820             # increment taglevel
 821             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 822             spec_dict[self.module_taglevel_varname] = new_taglevel
 823
 824         new_tag_name = self.tag_name(spec_dict)
 825
 826         # sanity check
 827         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 828         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 829
 830         # checking for diffs
 831         diff_output = self.repository.diff_with_tag(old_tag_name)
 832         if len(diff_output) == 0:
 833             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 834                 return
 835
 836         # side effect in trunk's specfile
 837         self.patch_spec_var(spec_dict)
 838
 839         # prepare changelog file
 840         # we use the standard subversion magic string (see svn_magic_line)
 841         # so we can provide useful information, such as version numbers and diff
 842         # in the same file
 843         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 844         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 845         setting_tag_line=Module.setting_tag_format%new_tag_name
 846         file(changelog,"w").write("""
 847 %s
 848 %s
 849 Please write a changelog for this new tag in the section above
 850 """%(Module.svn_magic_line,setting_tag_line))
 851
 852         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 853             file(changelog,"a").write('DIFF=========\n' + diff_output)
 854
 855         if self.options.debug:
 856             prompt('Proceed ?')
 857
 858         # edit it
 859         self.run("%s %s"%(self.options.editor,changelog))
 860         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 861         # so we do the job ourselves
 862         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 863         # insert changelog in spec
 864         if self.options.changelog:
 865             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 866
 867         ## update build
 868         build_path = os.path.join(self.options.workdir,
 869                                   Module.config['build'])
 870         build = Repository(build_path, self.options)
 871         if self.options.build_branch:
 872             build.to_branch(self.options.build_branch)
 873         if not build.is_clean():
 874             build.revert()
 875
 876         tagsfiles=glob(build.path+"/*-tags*.mk")
 877         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 878         default_answer = 'y'
 879         tagsfiles.sort()
 880         while True:
 881             for tagsfile in tagsfiles:
 882                 status=tagsdict[tagsfile]
 883                 basename=os.path.basename(tagsfile)
 884                 print ".................... Dealing with %s"%basename
 885                 while tagsdict[tagsfile] == 'todo' :
 886                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 887                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 888                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 889                                      allow_outside=True)
 890                     if choice == 'y':
 891                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 892                     elif choice == 'n':
 893                         print 'Done with %s'%os.path.basename(tagsfile)
 894                         tagsdict[tagsfile]='done'
 895                     elif choice == 'f':
 896                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 897                     elif choice == 'd':
 898                         self.run("svn diff %s"%tagsfile)
 899                     elif choice == 'r':
 900                         self.run("svn revert %s"%tagsfile)
 901                     elif choice == 'c':
 902                         self.run("cat %s"%tagsfile)
 903                     else:
 904                         name=self.name
 905                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 906 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 907 d: show current diff for this tag file
 908 r: revert that tag file
 909 c: cat the current tag file
 910 n: move to next file"""%locals()
 911
 912             if prompt("Want to review changes on tags files",False):
 913                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 914                 default_answer='d'
 915             else:
 916                 break
 917
 918         def diff_all_changes():
 919             print build.diff()
 920             print self.repository.diff()
 921
 922         def commit_all_changes(log):
 923             self.repository.commit(log)
 924             build.commit(log)
 925
 926         self.run_prompt("Review module and build", diff_all_changes)
 927         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 928         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 929
 930         if self.options.debug:
 931             print 'Preserving',changelog,'and stripped',changelog_svn
 932         else:
 933             os.unlink(changelog)
 934             os.unlink(changelog_svn)
 935
 936
 937 ##############################
 938 class Main:
 939
 940     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 941
 942 module-tools : a set of tools to manage subversion tags and specfile
 943   requires the specfile to either
 944   * define *version* and *taglevel*
 945   OR alternatively
 946   * define redirection variables module_version_varname / module_taglevel_varname
 947 Trunk:
 948   by default, the trunk of modules is taken into account
 949   in this case, just mention the module name as <module_desc>
 950 Branches:
 951   if you wish to work on a branch rather than on the trunk,
 952   you can use something like e.g. Mom:2.1 as <module_desc>
 953 """
 954     release_usage="""Usage: %prog [options] tag1 .. tagn
 955   Extract release notes from the changes in specfiles between several build tags, latest first
 956   Examples:
 957       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 958   You can refer to a (build) branch by prepending a colon, like in
 959       release-changelog :4.2 4.2-rc25
 960   You can refer to the build trunk by just mentioning 'trunk', e.g.
 961       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 962 """
 963     common_usage="""More help:
 964   see http://svn.planet-lab.org/wiki/ModuleTools"""
 965
 966     modes={
 967         'list' : "displays a list of available tags or branches",
 968         'version' : "check latest specfile and print out details",
 969         'diff' : "show difference between module (trunk or branch) and latest tag",
 970         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 971                 create new tag and and monitor its adoption in build/*-tags*.mk""",
 972         'branch' : """create a branch for this module, from the latest tag on the trunk,
 973                   and change trunk's version number to reflect the new branch name;
 974                   you can specify the new branch name by using module:branch""",
 975         'sync' : """create a tag from the module
 976                 this is a last resort option, mostly for repairs""",
 977         'changelog' : """extract changelog between build tags
 978                 expected arguments are a list of tags""",
 979         }
 980
 981     silent_modes = ['list']
 982     release_modes = ['changelog']
 983
 984     @staticmethod
 985     def optparse_list (option, opt, value, parser):
 986         try:
 987             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
 988         except:
 989             setattr(parser.values,option.dest,value.split())
 990
 991     def run(self):
 992
 993         mode=None
 994         for function in Main.modes.keys():
 995             if sys.argv[0].find(function) >= 0:
 996                 mode = function
 997                 break
 998         if not mode:
 999             print "Unsupported command",sys.argv[0]
1000             print "Supported commands:" + " ".join(Main.modes.keys())
1001             sys.exit(1)
1002
1003         if mode not in Main.release_modes:
1004             usage = Main.module_usage
1005             usage += Main.common_usage
1006             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1007         else:
1008             usage = Main.release_usage
1009             usage += Main.common_usage
1010
1011         parser=OptionParser(usage=usage,version=subversion_id)
1012
1013         if mode == "tag" or mode == 'branch':
1014             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1015                               help="set new version and reset taglevel to 0")
1016         if mode == "tag" :
1017             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1018                               help="do not update changelog section in specfile when tagging")
1019             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1020                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1021         if mode == "tag" or mode == "sync" :
1022             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1023                               help="specify editor")
1024
1025         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1026         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1027                           help="dry run - shell commands are only displayed")
1028         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1029                           default=[], nargs=1,type="string",
1030                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1031 -- can be set multiple times, or use quotes""")
1032
1033         parser.add_option("-w","--workdir", action="store", dest="workdir",
1034                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1035                           help="""name for dedicated working dir - defaults to ~/modules
1036 ** THIS MUST NOT ** be your usual working directory""")
1037         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1038                           help="skip safety checks, such as svn updates -- use with care")
1039
1040         # default verbosity depending on function - temp
1041         verbose_modes= ['tag', 'sync', 'branch']
1042
1043         if mode not in verbose_modes:
1044             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1045                               help="run in verbose mode")
1046         else:
1047             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1048                               help="run in quiet (non-verbose) mode")
1049         (options, args) = parser.parse_args()
1050         options.mode=mode
1051         if not hasattr(options,'dry_run'):
1052             options.dry_run=False
1053         if not hasattr(options,'www'):
1054             options.www=False
1055         options.debug=False
1056
1057         ########## module-*
1058         if len(args) == 0:
1059             parser.print_help()
1060             sys.exit(1)
1061         Module.init_homedir(options)
1062
1063         modules=[ Module(modname,options) for modname in args ]
1064         for module in modules:
1065             if len(args)>1 and mode not in Main.silent_modes:
1066                 print '========================================',module.friendly_name()
1067             # call the method called do_<mode>
1068             method=Module.__dict__["do_%s"%mode]
1069             try:
1070                 method(module)
1071             except Exception,e:
1072                 import traceback
1073                 traceback.print_exc()
1074                 print 'Skipping module %s: '%modname,e
1075
1076 ####################
1077 if __name__ == "__main__" :
1078     try:
1079         Main().run()
1080     except KeyboardInterrupt:
1081         print '\nBye'