module-tools.py

   1 #!/usr/bin/python -u
   2
   3 subversion_id = "$Id$"
   4
   5 import sys, os
   6 import re
   7 import time
   8 from glob import glob
   9 from optparse import OptionParser
  10
  11 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  12 def prompt (question,default=True,other_choices=[],allow_outside=False):
  13     if not isinstance (other_choices,list):
  14         other_choices = [ other_choices ]
  15     chars = [ c for (c,rest) in other_choices ]
  16
  17     choices = []
  18     if 'y' not in chars:
  19         if default is True: choices.append('[y]')
  20         else : choices.append('y')
  21     if 'n' not in chars:
  22         if default is False: choices.append('[n]')
  23         else : choices.append('n')
  24
  25     for (char,choice) in other_choices:
  26         if default == char:
  27             choices.append("["+char+"]"+choice)
  28         else:
  29             choices.append("<"+char+">"+choice)
  30     try:
  31         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  32         if not answer:
  33             return default
  34         answer=answer[0].lower()
  35         if answer == 'y':
  36             if 'y' in chars: return 'y'
  37             else: return True
  38         elif answer == 'n':
  39             if 'n' in chars: return 'n'
  40             else: return False
  41         elif other_choices:
  42             for (char,choice) in other_choices:
  43                 if answer == char:
  44                     return char
  45             if allow_outside:
  46                 return answer
  47         return prompt(question,default,other_choices)
  48     except:
  49         raise
  50
  51 def default_editor():
  52     try:
  53         editor = os.environ['EDITOR']
  54     except:
  55         editor = "emacs"
  56     return editor
  57
  58 ### fold long lines
  59 fold_length=132
  60
  61 def print_fold (line):
  62     while len(line) >= fold_length:
  63         print line[:fold_length],'\\'
  64         line=line[fold_length:]
  65     print line
  66
  67 class Command:
  68     def __init__ (self,command,options):
  69         self.command=command
  70         self.options=options
  71         self.tmp="/tmp/command-%d"%os.getpid()
  72
  73     def run (self):
  74         if self.options.dry_run:
  75             print 'dry_run',self.command
  76             return 0
  77         if self.options.verbose and self.options.mode not in Main.silent_modes:
  78             print '+',self.command
  79             sys.stdout.flush()
  80         return os.system(self.command)
  81
  82     def run_silent (self):
  83         if self.options.dry_run:
  84             print 'dry_run',self.command
  85             return 0
  86         if self.options.verbose:
  87             print '+',self.command,' .. ',
  88             sys.stdout.flush()
  89         retcod=os.system(self.command + " &> " + self.tmp)
  90         if retcod != 0:
  91             print "FAILED ! -- out+err below (command was %s)"%self.command
  92             os.system("cat " + self.tmp)
  93             print "FAILED ! -- end of quoted output"
  94         elif self.options.verbose:
  95             print "OK"
  96         os.unlink(self.tmp)
  97         return retcod
  98
  99     def run_fatal(self):
 100         if self.run_silent() !=0:
 101             raise Exception,"Command %s failed"%self.command
 102
 103     # returns stdout, like bash's $(mycommand)
 104     def output_of (self,with_stderr=False):
 105         if self.options.dry_run:
 106             print 'dry_run',self.command
 107             return 'dry_run output'
 108         tmp="/tmp/status-%d"%os.getpid()
 109         if self.options.debug:
 110             print '+',self.command,' .. ',
 111             sys.stdout.flush()
 112         command=self.command
 113         if with_stderr:
 114             command += " &> "
 115         else:
 116             command += " > "
 117         command += tmp
 118         os.system(command)
 119         result=file(tmp).read()
 120         os.unlink(tmp)
 121         if self.options.debug:
 122             print 'Done',
 123         return result
 124
 125
 126 class SvnRepository:
 127     type = "svn"
 128
 129     def __init__(self, path, options):
 130         self.path = path
 131         self.options = options
 132
 133     @classmethod
 134     def checkout(cls, module, config, options, recursive=False):
 135         remote = "%s/%s" % (config['svnpath'], module)
 136         local = os.path.join(options.workdir, module)
 137
 138         if recursive:
 139             svncommand = "svn co %s %s" % (remote, local)
 140         else:
 141             svncommand = "svn co -N %s %s" % (remote, local)
 142         Command("rm -rf %s" % local, options).run_silent()
 143         Command(svncommand, options).run_fatal()
 144
 145         return SvnRepository(local, options)
 146
 147     @classmethod
 148     def remote_exists(cls, remote):
 149         return os.system("svn list %s &> /dev/null" % remote) == 0
 150
 151     def update(self, subdir="", recursive=True):
 152         path = os.path.join(self.path, subdir)
 153         if recursive:
 154             svncommand = "svn up %s" % path
 155         else:
 156             svncommand = "svn up -N %s" % path
 157         Command(svncommand, self.options).run_fatal()
 158
 159     def commit(self, logfile):
 160         # add all new files to the repository
 161         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 162                 self.path, self.options).run_silent()
 163         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 164
 165     def tag(self, from_url, to_url, logfile):
 166         # TODO: get svn url from svn info output
 167         # and only require tagname and logfile parameters
 168         Command("svn copy -F %s %s %s" % (logfile, from_url, to_url), self.options).run_fatal()
 169
 170     def diff(self):
 171         return Command("svn diff %s" % self.path, self.options).output_of(True)
 172
 173     def revert(self):
 174         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 175
 176     def is_clean(self):
 177         command="svn status %s" % self.path
 178         return len(Command(command,self.options).output_of(True)) == 0
 179
 180     def is_valid(self):
 181         return os.path.exists(os.path.join(self.path, ".svn"))
 182
 183
 184
 185 class GitRepository:
 186     type = "git"
 187
 188     def __init__(self, path, options):
 189         self.path = path
 190         self.options = options
 191
 192     @classmethod
 193     def checkout(cls, module, config, options, depth=1):
 194         remote = "%s:/git/%s.git" % (config['gitserver'], module)
 195         local = os.path.join(options.workdir, module)
 196
 197         Command("rm -rf %s" % local, options).run_silent()
 198         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 199
 200         return GitRepository(local, options)
 201
 202     @classmethod
 203     def remote_exists(cls, remote):
 204         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 205
 206     def __run_in_repo(self, fun, *args, **kwargs):
 207         cwd = os.getcwd()
 208         os.chdir(self.path)
 209         ret = fun(*args, **kwargs)
 210         os.chdir(cwd)
 211         return ret
 212
 213     def __run_command_in_repo(self, command):
 214         c = Command(command, self.options)
 215         return self.__run_in_repo(c.run_fatal)
 216
 217     def update(self, subdir=None, recursive=None):
 218         return self.__run_command_in_repo("git pull")
 219
 220     def to_branch(self, branch, remote=True):
 221         if remote:
 222             branch = "origin/%s" % branch
 223         self.__run_command_in_repo("git checkout %s" % branch)
 224
 225     def diff(self):
 226         c = Command("git diff", self.options)
 227         return self.__run_in_repo(c.output_of, with_stderr=True)
 228
 229     def commit(self, logfile):
 230         self.__run_command_in_repo("git add -A")
 231         self.__run_command_in_repo("git commit -F  %s" % logfile)
 232         self.__run_command_in_repo("git push")
 233
 234     def revert(self):
 235         self.__run_command_in_repo("git --no-pager reset --hard")
 236         self.__run_command_in_repo("git --no-pager clean -f")
 237
 238     def is_clean(self):
 239         def check_commit():
 240             command="git status"
 241             s="nothing to commit (working directory clean)"
 242             return Command(command, self.options).output_of(True).find(s) >= 0
 243         return self.__run_in_repo(check_commit)
 244
 245     def is_valid(self):
 246         return os.path.exists(os.path.join(self.path, ".git"))
 247
 248
 249 class Repository:
 250     """ Generic repository """
 251     supported_repo_types = [SvnRepository, GitRepository]
 252
 253     def __init__(self, path, options):
 254         self.path = path
 255         self.options = options
 256         for repo in self.supported_repo_types:
 257             self.repo = repo(self.path, self.options)
 258             if self.repo.is_valid():
 259                 break
 260
 261     @classmethod
 262     def has_moved_to_git(cls, module, config):
 263         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (config['svnpath'], module))
 264
 265     @classmethod
 266     def remote_exists(cls, remote):
 267         for repo in Repository.supported_repo_types:
 268             if repo.remote_exists(remote):
 269                 return True
 270         return False
 271
 272     def __getattr__(self, attr):
 273         return getattr(self.repo, attr)
 274
 275
 276
 277 # support for tagged module is minimal, and is for the Build class only
 278 class Module:
 279
 280     svn_magic_line="--This line, and those below, will be ignored--"
 281     setting_tag_format = "Setting tag %s"
 282
 283     redirectors=[ # ('module_name_varname','name'),
 284                   ('module_version_varname','version'),
 285                   ('module_taglevel_varname','taglevel'), ]
 286
 287     # where to store user's config
 288     config_storage="CONFIG"
 289     #
 290     config={}
 291
 292     import commands
 293     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 294                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 295                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 296                  ("build", "Enter the name of your build module","build"),
 297                  ('username',"Enter your firstname and lastname for changelogs",""),
 298                  ("email","Enter your email address for changelogs",""),
 299                  ]
 300
 301     @staticmethod
 302     def prompt_config ():
 303         for (key,message,default) in Module.configKeys:
 304             Module.config[key]=""
 305             while not Module.config[key]:
 306                 Module.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 307
 308
 309     # for parsing module spec name:branch
 310     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 311     # special form for tagged module - for Build
 312     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 313     # parsing specfiles
 314     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 315
 316     def __init__ (self,module_spec,options):
 317         # parse module spec
 318         attempt=Module.matcher_branch_spec.match(module_spec)
 319         if attempt:
 320             self.name=attempt.group('name')
 321             self.branch=attempt.group('branch')
 322         else:
 323             attempt=Module.matcher_tag_spec.match(module_spec)
 324             if attempt:
 325                 self.name=attempt.group('name')
 326                 self.tagname=attempt.group('tagname')
 327             else:
 328                 self.name=module_spec
 329
 330         self.options=options
 331         self.module_dir="%s/%s"%(options.workdir,self.name)
 332         self.repository = None
 333         self.build = None
 334
 335     def friendly_name (self):
 336         if hasattr(self,'branch'):
 337             return "%s:%s"%(self.name,self.branch)
 338         elif hasattr(self,'tagname'):
 339             return "%s@%s"%(self.name,self.tagname)
 340         else:
 341             return self.name
 342
 343     def edge_dir (self):
 344         if hasattr(self,'branch'):
 345             return "%s/branches/%s"%(self.module_dir,self.branch)
 346         elif hasattr(self,'tagname'):
 347             return "%s/tags/%s"%(self.module_dir,self.tagname)
 348         else:
 349             return "%s/trunk"%(self.module_dir)
 350
 351     def tags_dir (self):
 352         return "%s/tags"%(self.module_dir)
 353
 354     def run (self,command):
 355         return Command(command,self.options).run()
 356     def run_fatal (self,command):
 357         return Command(command,self.options).run_fatal()
 358     def run_prompt (self,message,fun, *args):
 359         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 360         if not self.options.verbose:
 361             while True:
 362                 choice=prompt(message,True,('s','how'))
 363                 if choice is True:
 364                     fun(*args)
 365                     return
 366                 elif choice is False:
 367                     print 'About to run function:', fun_msg
 368         else:
 369             question=message+" - want to run function: " + fun_msg
 370             if prompt(question,True):
 371                 fun(*args)
 372
 373
 374     ####################
 375     @staticmethod
 376     def init_homedir (options):
 377         topdir=options.workdir
 378         if options.verbose and options.mode not in Main.silent_modes:
 379             print 'Checking for',topdir
 380         storage="%s/%s"%(topdir,Module.config_storage)
 381         # sanity check. Either the topdir exists AND we have a config/storage
 382         # or topdir does not exist and we create it
 383         # to avoid people use their own daily svn repo
 384         if os.path.isdir(topdir) and not os.path.isfile(storage):
 385             print """The directory %s exists and has no CONFIG file
 386 If this is your regular working directory, please provide another one as the
 387 module-* commands need a fresh working dir. Make sure that you do not use
 388 that for other purposes than tagging"""%topdir
 389             sys.exit(1)
 390         if not os.path.isdir (topdir):
 391             print "Cannot find",topdir,"let's create it"
 392             Module.prompt_config()
 393             print "Checking ...",
 394             GitRepository.checkout(Module.config['build'], Module.config,
 395                                    options, depth=1)
 396             print "OK"
 397
 398             # store config
 399             f=file(storage,"w")
 400             for (key,message,default) in Module.configKeys:
 401                 f.write("%s=%s\n"%(key,Module.config[key]))
 402             f.close()
 403             if options.debug:
 404                 print 'Stored',storage
 405                 Command("cat %s"%storage,options).run()
 406         else:
 407             # read config
 408             f=open(storage)
 409             for line in f.readlines():
 410                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 411                 Module.config[key]=value
 412             f.close()
 413         if options.verbose and options.mode not in Main.silent_modes:
 414             print '******** Using config'
 415             for (key,message,default) in Module.configKeys:
 416                 print '\t',key,'=',Module.config[key]
 417
 418     def init_module_dir (self):
 419         if self.options.verbose:
 420             print 'Checking for',self.module_dir
 421
 422         if not os.path.isdir (self.module_dir):
 423             if Repository.has_moved_to_git(self.name, Module.config):
 424                 self.repository = GitRepository.checkout(self.name, Module.config,
 425                                                          self.options, depth=1)
 426             else:
 427                 self.repository = SvnRepository.checkout(self.name, Module.config,
 428                                                          self.options, recursive=False)
 429         else:
 430             try:
 431                 self.repository = Repository(self.module_dir, self.options)
 432             except:
 433                 raise Exception, 'Cannot find %s - check module name'%self.module_dir
 434
 435     def init_subdir (self,subdir, recursive=True):
 436         path = os.path.join(self.repository.path, subdir)
 437         if self.options.verbose:
 438             print 'Checking for', path
 439         if not os.path.isdir(path):
 440             self.repository.update(recursive=recursive, subdir=subdir)
 441
 442     def revert_subdir (self,fullpath):
 443         if self.options.fast_checks:
 444             if self.options.verbose: print 'Skipping revert of %s'%fullpath
 445             return
 446         if self.options.verbose:
 447             print 'Checking whether',fullpath,'needs being reverted'
 448
 449         repo=Repository(fullpath, self.options)
 450         if not repo.is_clean():
 451             repo.revert()
 452
 453     def update_subdir (self,fullpath):
 454         if self.options.fast_checks:
 455             if self.options.verbose: print 'Skipping update of %s'%fullpath
 456             return
 457         if self.options.verbose:
 458             print 'Updating',fullpath
 459         Repository(fullpath, self.options).update()
 460
 461     def init_edge_dir (self):
 462         # if branch, edge_dir is two steps down
 463         if hasattr(self,'branch'):
 464             self.init_subdir("%s/branches"%self.module_dir,recursive=False)
 465         elif hasattr(self,'tagname'):
 466             self.init_subdir("%s/tags"%self.module_dir,recursive=False)
 467         self.init_subdir(self.edge_dir(),recursive=True)
 468
 469     def revert_edge_dir (self):
 470         self.revert_subdir(self.edge_dir())
 471
 472     def update_edge_dir (self):
 473         self.update_subdir(self.edge_dir())
 474
 475     def main_specname (self):
 476         attempt="%s/%s.spec"%(self.edge_dir(),self.name)
 477         if os.path.isfile (attempt):
 478             return attempt
 479         pattern1="%s/*.spec"%self.edge_dir()
 480         level1=glob(pattern1)
 481         if level1:
 482             return level1[0]
 483         pattern2="%s/*/*.spec"%self.edge_dir()
 484         level2=glob(pattern2)
 485         if level2:
 486             return level2[0]
 487         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 488
 489     def all_specnames (self):
 490         level1=glob("%s/*.spec"%self.edge_dir())
 491         if level1: return level1
 492         level2=glob("%s/*/*.spec"%self.edge_dir())
 493         return level2
 494
 495     def parse_spec (self, specfile, varnames):
 496         if self.options.verbose:
 497             print 'Parsing',specfile,
 498             for var in varnames:
 499                 print "[%s]"%var,
 500             print ""
 501         result={}
 502         f=open(specfile)
 503         for line in f.readlines():
 504             attempt=Module.matcher_rpm_define.match(line)
 505             if attempt:
 506                 (define,var,value)=attempt.groups()
 507                 if var in varnames:
 508                     result[var]=value
 509         f.close()
 510         if self.options.debug:
 511             print 'found',len(result),'keys'
 512             for (k,v) in result.iteritems():
 513                 print k,'=',v
 514         return result
 515
 516     # stores in self.module_name_varname the rpm variable to be used for the module's name
 517     # and the list of these names in self.varnames
 518     def spec_dict (self):
 519         specfile=self.main_specname()
 520         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 521         redirect_dict = self.parse_spec(specfile,redirector_keys)
 522         if self.options.debug:
 523             print '1st pass parsing done, redirect_dict=',redirect_dict
 524         varnames=[]
 525         for (varname,default) in Module.redirectors:
 526             if redirect_dict.has_key(varname):
 527                 setattr(self,varname,redirect_dict[varname])
 528                 varnames += [redirect_dict[varname]]
 529             else:
 530                 setattr(self,varname,default)
 531                 varnames += [ default ]
 532         self.varnames = varnames
 533         result = self.parse_spec (specfile,self.varnames)
 534         if self.options.debug:
 535             print '2st pass parsing done, varnames=',varnames,'result=',result
 536         return result
 537
 538     def patch_spec_var (self, patch_dict,define_missing=False):
 539         for specfile in self.all_specnames():
 540             # record the keys that were changed
 541             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 542             newspecfile=specfile+".new"
 543             if self.options.verbose:
 544                 print 'Patching',specfile,'for',patch_dict.keys()
 545             spec=open (specfile)
 546             new=open(newspecfile,"w")
 547
 548             for line in spec.readlines():
 549                 attempt=Module.matcher_rpm_define.match(line)
 550                 if attempt:
 551                     (define,var,value)=attempt.groups()
 552                     if var in patch_dict.keys():
 553                         if self.options.debug:
 554                             print 'rewriting %s as %s'%(var,patch_dict[var])
 555                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 556                         changed[var]=True
 557                         continue
 558                 new.write(line)
 559             if define_missing:
 560                 for (key,was_changed) in changed.iteritems():
 561                     if not was_changed:
 562                         if self.options.debug:
 563                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 564                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 565             spec.close()
 566             new.close()
 567             os.rename(newspecfile,specfile)
 568
 569     # returns all lines until the magic line
 570     def unignored_lines (self, logfile):
 571         result=[]
 572         white_line_matcher = re.compile("\A\s*\Z")
 573         for logline in file(logfile).readlines():
 574             if logline.strip() == Module.svn_magic_line:
 575                 break
 576             elif white_line_matcher.match(logline):
 577                 continue
 578             else:
 579                 result.append(logline.strip()+'\n')
 580         return result
 581
 582     # creates a copy of the input with only the unignored lines
 583     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 584        f=file(fileout,'w')
 585        f.write(self.setting_tag_format%new_tag_name + '\n')
 586        for line in self.unignored_lines(filein):
 587            f.write(line)
 588        f.close()
 589
 590     def insert_changelog (self, logfile, oldtag, newtag):
 591         for specfile in self.all_specnames():
 592             newspecfile=specfile+".new"
 593             if self.options.verbose:
 594                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 595             spec=open (specfile)
 596             new=open(newspecfile,"w")
 597             for line in spec.readlines():
 598                 new.write(line)
 599                 if re.compile('%changelog').match(line):
 600                     dateformat="* %a %b %d %Y"
 601                     datepart=time.strftime(dateformat)
 602                     logpart="%s <%s> - %s"%(Module.config['username'],
 603                                                  Module.config['email'],
 604                                                  newtag)
 605                     new.write(datepart+" "+logpart+"\n")
 606                     for logline in self.unignored_lines(logfile):
 607                         new.write("- " + logline)
 608                     new.write("\n")
 609             spec.close()
 610             new.close()
 611             os.rename(newspecfile,specfile)
 612
 613     def show_dict (self, spec_dict):
 614         if self.options.verbose:
 615             for (k,v) in spec_dict.iteritems():
 616                 print k,'=',v
 617
 618     def mod_svn_url (self):
 619         return "%s/%s"%(Module.config['svnpath'],self.name)
 620
 621     def edge_url (self):
 622         if hasattr(self,'branch'):
 623             return "%s/branches/%s"%(self.mod_svn_url(),self.branch)
 624         elif hasattr(self,'tagname'):
 625             return "%s/tags/%s"%(self.mod_svn_url(),self.tagname)
 626         else:
 627             return "%s/trunk"%(self.mod_svn_url())
 628
 629     def last_tag (self, spec_dict):
 630         return "%s-%s"%(spec_dict[self.module_version_varname],spec_dict[self.module_taglevel_varname])
 631
 632     def tag_name (self, spec_dict):
 633         try:
 634             return "%s-%s"%(self.name,
 635                             self.last_tag(spec_dict))
 636         except KeyError,err:
 637             raise Exception, 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 638
 639     def tag_url (self, spec_dict):
 640         return "%s/tags/%s"%(self.mod_svn_url(),self.tag_name(spec_dict))
 641
 642     def check_svnpath_exists (self, url, message):
 643         if self.options.fast_checks:
 644             return
 645         if self.options.verbose:
 646             print 'Checking url (%s) %s'%(url,message),
 647
 648         if SvnRepository.remote_exists(url):
 649             if self.options.verbose: print 'exists - OK'
 650         else:
 651             if self.options.verbose: print 'KO'
 652             raise Exception, 'Could not find %s URL %s'%(message,url)
 653
 654     def check_svnpath_not_exists (self, url, message):
 655         if self.options.fast_checks:
 656             return
 657         if self.options.verbose:
 658             print 'Checking url (%s) %s'%(url,message),
 659
 660         if not SvnRepository.remote_exists(url):
 661             if self.options.verbose: print 'does not exist - OK'
 662         else:
 663             if self.options.verbose: print 'KO'
 664             raise Exception, '%s URL %s already exists - exiting'%(message,url)
 665
 666     # locate specfile, parse it, check it and show values
 667
 668
 669 ##############################
 670     # using fine_grain means replacing only those instances that currently refer to this tag
 671     # otherwise, <module>-SVNPATH is replaced unconditionnally
 672     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 673         newtagsfile=tagsfile+".new"
 674         tags=open (tagsfile)
 675         new=open(newtagsfile,"w")
 676
 677         matches=0
 678         # fine-grain : replace those lines that refer to oldname
 679         if fine_grain:
 680             if self.options.verbose:
 681                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 682             matcher=re.compile("^(.*)%s(.*)"%oldname)
 683             for line in tags.readlines():
 684                 if not matcher.match(line):
 685                     new.write(line)
 686                 else:
 687                     (begin,end)=matcher.match(line).groups()
 688                     new.write(begin+newname+end+"\n")
 689                     matches += 1
 690         # brute-force : change uncommented lines that define <module>-SVNPATH
 691         else:
 692             if self.options.verbose:
 693                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 694             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 695                                           %(self.name)
 696             matcher_module=re.compile(pattern)
 697             for line in tags.readlines():
 698                 attempt=matcher_module.match(line)
 699                 if attempt:
 700                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 701                     if self.options.verbose:
 702                         print ' '+svnpath,
 703                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 704                     new.write(replacement)
 705                     matches += 1
 706                 else:
 707                     new.write(line)
 708         tags.close()
 709         new.close()
 710         os.rename(newtagsfile,tagsfile)
 711         if self.options.verbose: print "%d changes"%matches
 712         return matches
 713
 714     def do_tag (self):
 715         self.init_module_dir()
 716         self.init_edge_dir()
 717         self.revert_edge_dir()
 718         self.update_edge_dir()
 719         # parse specfile
 720         spec_dict = self.spec_dict()
 721         self.show_dict(spec_dict)
 722
 723         # side effects
 724         edge_url=self.edge_url()
 725         old_tag_name = self.tag_name(spec_dict)
 726         old_tag_url=self.tag_url(spec_dict)
 727         if (self.options.new_version):
 728             # new version set on command line
 729             spec_dict[self.module_version_varname] = self.options.new_version
 730             spec_dict[self.module_taglevel_varname] = 0
 731         else:
 732             # increment taglevel
 733             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 734             spec_dict[self.module_taglevel_varname] = new_taglevel
 735
 736         # sanity check
 737         new_tag_name = self.tag_name(spec_dict)
 738         new_tag_url=self.tag_url(spec_dict)
 739         self.check_svnpath_exists (edge_url,"edge track")
 740         self.check_svnpath_exists (old_tag_url,"previous tag")
 741         self.check_svnpath_not_exists (new_tag_url,"new tag")
 742
 743         # checking for diffs
 744         diff_output=Command("svn diff %s %s"%(old_tag_url,edge_url),
 745                             self.options).output_of()
 746         if len(diff_output) == 0:
 747             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 748                 return
 749
 750         # side effect in trunk's specfile
 751         self.patch_spec_var(spec_dict)
 752
 753         # prepare changelog file
 754         # we use the standard subversion magic string (see svn_magic_line)
 755         # so we can provide useful information, such as version numbers and diff
 756         # in the same file
 757         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 758         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 759         setting_tag_line=Module.setting_tag_format%new_tag_name
 760         file(changelog,"w").write("""
 761 %s
 762 %s
 763 Please write a changelog for this new tag in the section above
 764 """%(Module.svn_magic_line,setting_tag_line))
 765
 766         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 767             file(changelog,"a").write('DIFF=========\n' + diff_output)
 768
 769         if self.options.debug:
 770             prompt('Proceed ?')
 771
 772         # edit it
 773         self.run("%s %s"%(self.options.editor,changelog))
 774         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 775         # so we do the job ourselves
 776         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 777         # insert changelog in spec
 778         if self.options.changelog:
 779             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 780
 781         ## update build
 782         build_path = os.path.join(self.options.workdir,
 783                                   Module.config['build'])
 784         build = Repository(build_path, self.options)
 785         if self.options.build_branch:
 786             build.to_branch(self.options.build_branch)
 787
 788         tagsfiles=glob(build.path+"/*-tags*.mk")
 789         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 790         default_answer = 'y'
 791         tagsfiles.sort()
 792         while True:
 793             for tagsfile in tagsfiles:
 794                 status=tagsdict[tagsfile]
 795                 basename=os.path.basename(tagsfile)
 796                 print ".................... Dealing with %s"%basename
 797                 while tagsdict[tagsfile] == 'todo' :
 798                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 799                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 800                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 801                                      allow_outside=True)
 802                     if choice == 'y':
 803                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 804                     elif choice == 'n':
 805                         print 'Done with %s'%os.path.basename(tagsfile)
 806                         tagsdict[tagsfile]='done'
 807                     elif choice == 'f':
 808                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 809                     elif choice == 'd':
 810                         self.run("svn diff %s"%tagsfile)
 811                     elif choice == 'r':
 812                         self.run("svn revert %s"%tagsfile)
 813                     elif choice == 'c':
 814                         self.run("cat %s"%tagsfile)
 815                     else:
 816                         name=self.name
 817                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 818 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 819 d: show current diff for this tag file
 820 r: revert that tag file
 821 c: cat the current tag file
 822 n: move to next file"""%locals()
 823
 824             if prompt("Want to review changes on tags files",False):
 825                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 826                 default_answer='d'
 827             else:
 828                 break
 829
 830         def diff_all_changes():
 831             print build.diff()
 832             print self.repository.diff()
 833
 834         def commit_all_changes(log):
 835             self.repository.commit(log)
 836             build.commit(log)
 837
 838         self.run_prompt("Review module and build", diff_all_changes)
 839         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 840         # TODO: implement repository.tag() for git
 841         self.run_prompt("Create tag", self.repository.tag, edge_url, new_tag_url, changelog_svn)
 842
 843         if self.options.debug:
 844             print 'Preserving',changelog,'and stripped',changelog_svn
 845         else:
 846             os.unlink(changelog)
 847             os.unlink(changelog_svn)
 848
 849
 850 ##############################
 851 class Main:
 852
 853     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 854
 855 module-tools : a set of tools to manage subversion tags and specfile
 856   requires the specfile to either
 857   * define *version* and *taglevel*
 858   OR alternatively
 859   * define redirection variables module_version_varname / module_taglevel_varname
 860 Trunk:
 861   by default, the trunk of modules is taken into account
 862   in this case, just mention the module name as <module_desc>
 863 Branches:
 864   if you wish to work on a branch rather than on the trunk,
 865   you can use something like e.g. Mom:2.1 as <module_desc>
 866 """
 867     release_usage="""Usage: %prog [options] tag1 .. tagn
 868   Extract release notes from the changes in specfiles between several build tags, latest first
 869   Examples:
 870       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 871   You can refer to a (build) branch by prepending a colon, like in
 872       release-changelog :4.2 4.2-rc25
 873   You can refer to the build trunk by just mentioning 'trunk', e.g.
 874       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 875 """
 876     common_usage="""More help:
 877   see http://svn.planet-lab.org/wiki/ModuleTools"""
 878
 879     modes={
 880         'list' : "displays a list of available tags or branches",
 881         'version' : "check latest specfile and print out details",
 882         'diff' : "show difference between module (trunk or branch) and latest tag",
 883         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 884                 create new tag and and monitor its adoption in build/*-tags*.mk""",
 885         'branch' : """create a branch for this module, from the latest tag on the trunk,
 886                   and change trunk's version number to reflect the new branch name;
 887                   you can specify the new branch name by using module:branch""",
 888         'sync' : """create a tag from the module
 889                 this is a last resort option, mostly for repairs""",
 890         'changelog' : """extract changelog between build tags
 891                 expected arguments are a list of tags""",
 892         }
 893
 894     silent_modes = ['list']
 895     release_modes = ['changelog']
 896
 897     @staticmethod
 898     def optparse_list (option, opt, value, parser):
 899         try:
 900             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
 901         except:
 902             setattr(parser.values,option.dest,value.split())
 903
 904     def run(self):
 905
 906         mode=None
 907         for function in Main.modes.keys():
 908             if sys.argv[0].find(function) >= 0:
 909                 mode = function
 910                 break
 911         if not mode:
 912             print "Unsupported command",sys.argv[0]
 913             print "Supported commands:" + " ".join(Main.modes.keys())
 914             sys.exit(1)
 915
 916         if mode not in Main.release_modes:
 917             usage = Main.module_usage
 918             usage += Main.common_usage
 919             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
 920         else:
 921             usage = Main.release_usage
 922             usage += Main.common_usage
 923
 924         parser=OptionParser(usage=usage,version=subversion_id)
 925
 926         if mode == "tag" or mode == 'branch':
 927             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
 928                               help="set new version and reset taglevel to 0")
 929         if mode == "tag" :
 930             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
 931                               help="do not update changelog section in specfile when tagging")
 932             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
 933                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
 934         if mode == "tag" or mode == "sync" :
 935             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
 936                               help="specify editor")
 937
 938         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
 939         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
 940                           help="dry run - shell commands are only displayed")
 941         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
 942                           default=[], nargs=1,type="string",
 943                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
 944 -- can be set multiple times, or use quotes""")
 945
 946         parser.add_option("-w","--workdir", action="store", dest="workdir",
 947                           default="%s/%s"%(os.getenv("HOME"),"modules"),
 948                           help="""name for dedicated working dir - defaults to ~/modules
 949 ** THIS MUST NOT ** be your usual working directory""")
 950         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
 951                           help="skip safety checks, such as svn updates -- use with care")
 952
 953         # default verbosity depending on function - temp
 954         verbose_modes= ['tag', 'sync', 'branch']
 955
 956         if mode not in verbose_modes:
 957             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 958                               help="run in verbose mode")
 959         else:
 960             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
 961                               help="run in quiet (non-verbose) mode")
 962         (options, args) = parser.parse_args()
 963         options.mode=mode
 964         if not hasattr(options,'dry_run'):
 965             options.dry_run=False
 966         if not hasattr(options,'www'):
 967             options.www=False
 968         options.debug=False
 969
 970         ########## module-*
 971         if len(args) == 0:
 972             parser.print_help()
 973             sys.exit(1)
 974         Module.init_homedir(options)
 975
 976         modules=[ Module(modname,options) for modname in args ]
 977         for module in modules:
 978             if len(args)>1 and mode not in Main.silent_modes:
 979                 print '========================================',module.friendly_name()
 980             # call the method called do_<mode>
 981             method=Module.__dict__["do_%s"%mode]
 982             try:
 983                 method(module)
 984             except Exception,e:
 985                 import traceback
 986                 traceback.print_exc()
 987                 print 'Skipping module %s: '%modname,e
 988
 989 ####################
 990 if __name__ == "__main__" :
 991     try:
 992         Main().run()
 993     except KeyboardInterrupt:
 994         print '\nBye'