module-tools.py

   1 #!/usr/bin/python -u
   2
   3 subversion_id = "$Id$"
   4
   5 import sys, os
   6 import re
   7 import time
   8 from glob import glob
   9 from optparse import OptionParser
  10
  11 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  12 def prompt (question,default=True,other_choices=[],allow_outside=False):
  13     if not isinstance (other_choices,list):
  14         other_choices = [ other_choices ]
  15     chars = [ c for (c,rest) in other_choices ]
  16
  17     choices = []
  18     if 'y' not in chars:
  19         if default is True: choices.append('[y]')
  20         else : choices.append('y')
  21     if 'n' not in chars:
  22         if default is False: choices.append('[n]')
  23         else : choices.append('n')
  24
  25     for (char,choice) in other_choices:
  26         if default == char:
  27             choices.append("["+char+"]"+choice)
  28         else:
  29             choices.append("<"+char+">"+choice)
  30     try:
  31         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  32         if not answer:
  33             return default
  34         answer=answer[0].lower()
  35         if answer == 'y':
  36             if 'y' in chars: return 'y'
  37             else: return True
  38         elif answer == 'n':
  39             if 'n' in chars: return 'n'
  40             else: return False
  41         elif other_choices:
  42             for (char,choice) in other_choices:
  43                 if answer == char:
  44                     return char
  45             if allow_outside:
  46                 return answer
  47         return prompt(question,default,other_choices)
  48     except:
  49         raise
  50
  51 def default_editor():
  52     try:
  53         editor = os.environ['EDITOR']
  54     except:
  55         editor = "emacs"
  56     return editor
  57
  58 ### fold long lines
  59 fold_length=132
  60
  61 def print_fold (line):
  62     while len(line) >= fold_length:
  63         print line[:fold_length],'\\'
  64         line=line[fold_length:]
  65     print line
  66
  67 class Command:
  68     def __init__ (self,command,options):
  69         self.command=command
  70         self.options=options
  71         self.tmp="/tmp/command-%d"%os.getpid()
  72
  73     def run (self):
  74         if self.options.dry_run:
  75             print 'dry_run',self.command
  76             return 0
  77         if self.options.verbose and self.options.mode not in Main.silent_modes:
  78             print '+',self.command
  79             sys.stdout.flush()
  80         return os.system(self.command)
  81
  82     def run_silent (self):
  83         if self.options.dry_run:
  84             print 'dry_run',self.command
  85             return 0
  86         if self.options.verbose:
  87             print '+',self.command,' .. ',
  88             sys.stdout.flush()
  89         retcod=os.system(self.command + " &> " + self.tmp)
  90         if retcod != 0:
  91             print "FAILED ! -- out+err below (command was %s)"%self.command
  92             os.system("cat " + self.tmp)
  93             print "FAILED ! -- end of quoted output"
  94         elif self.options.verbose:
  95             print "OK"
  96         os.unlink(self.tmp)
  97         return retcod
  98
  99     def run_fatal(self):
 100         if self.run_silent() !=0:
 101             raise Exception,"Command %s failed"%self.command
 102
 103     # returns stdout, like bash's $(mycommand)
 104     def output_of (self,with_stderr=False):
 105         if self.options.dry_run:
 106             print 'dry_run',self.command
 107             return 'dry_run output'
 108         tmp="/tmp/status-%d"%os.getpid()
 109         if self.options.debug:
 110             print '+',self.command,' .. ',
 111             sys.stdout.flush()
 112         command=self.command
 113         if with_stderr:
 114             command += " &> "
 115         else:
 116             command += " > "
 117         command += tmp
 118         os.system(command)
 119         result=file(tmp).read()
 120         os.unlink(tmp)
 121         if self.options.debug:
 122             print 'Done',
 123         return result
 124
 125
 126 class SvnRepository:
 127     type = "svn"
 128
 129     def __init__(self, path, options):
 130         self.path = path
 131         self.options = options
 132
 133     def name(self):
 134         return os.path.basename(self.path)
 135
 136     def url(self):
 137         out = Command("svn info %s" % self.path, self.options).output_of()
 138         for line in out.split('\n'):
 139             if line.startswith("URL:"):
 140                 return line.split()[1].strip()
 141
 142     def repo_root(self):
 143         out = Command("svn info %s" % self.path, self.options).output_of()
 144         for line in out.split('\n'):
 145             if line.startswith("Repository Root:"):
 146                 repo_root = line.split()[2].strip()
 147                 return "%s/svn/%s" (repo_root, self.name)
 148
 149     @classmethod
 150     def checkout(cls, remote, local, options, recursive=False):
 151         if recursive:
 152             svncommand = "svn co %s %s" % (remote, local)
 153         else:
 154             svncommand = "svn co -N %s %s" % (remote, local)
 155         Command("rm -rf %s" % local, options).run_silent()
 156         Command(svncommand, options).run_fatal()
 157
 158         return SvnRepository(local, options)
 159
 160     @classmethod
 161     def remote_exists(cls, remote):
 162         return os.system("svn list %s &> /dev/null" % remote) == 0
 163
 164     def update(self, subdir="", recursive=True):
 165         path = os.path.join(self.path, subdir)
 166         if recursive:
 167             svncommand = "svn up %s" % path
 168         else:
 169             svncommand = "svn up -N %s" % path
 170         Command(svncommand, self.options).run_fatal()
 171
 172     def commit(self, logfile):
 173         # add all new files to the repository
 174         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 175                 self.path, self.options).run_silent()
 176         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 177
 178     def to_branch(self, branch):
 179         remote = "%s/branches/%s" % (self.repo_root(), branch)
 180         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 181
 182     def to_tag(self, tag):
 183         remote = "%s/tags/%s" % (self.repo_root(), branch)
 184         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 185
 186     def tag(self, from_url, to_url, logfile):
 187         # TODO: get svn url from svn info output
 188         # and only require tagname and logfile parameters
 189         Command("svn copy -F %s %s %s" % (logfile, from_url, to_url), self.options).run_fatal()
 190
 191     def diff(self):
 192         return Command("svn diff %s" % self.path, self.options).output_of(True)
 193
 194     def revert(self):
 195         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 196         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 197                 self.path, self.options).run_silent()
 198
 199     def is_clean(self):
 200         command="svn status %s" % self.path
 201         return len(Command(command,self.options).output_of(True)) == 0
 202
 203     def is_valid(self):
 204         return os.path.exists(os.path.join(self.path, ".svn"))
 205
 206
 207 class GitRepository:
 208     type = "git"
 209
 210     def __init__(self, path, options):
 211         self.path = path
 212         self.options = options
 213
 214     def name(self):
 215         return os.path.basename(self.path)
 216
 217     def url(self):
 218         self.repo_root()
 219
 220     def repo_root(self):
 221         c = Command("git remote show origin", self.options)
 222         out = self.__run_in_repo(c.output_of)
 223         for line in out.split('\n'):
 224             if line.strip().startswith("Fetch URL:"):
 225                 repo = line.split()[2]
 226
 227     @classmethod
 228     def checkout(cls, remote, local, options, depth=1):
 229         Command("rm -rf %s" % local, options).run_silent()
 230         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 231         return GitRepository(local, options)
 232
 233     @classmethod
 234     def remote_exists(cls, remote):
 235         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 236
 237     def __run_in_repo(self, fun, *args, **kwargs):
 238         cwd = os.getcwd()
 239         os.chdir(self.path)
 240         ret = fun(*args, **kwargs)
 241         os.chdir(cwd)
 242         return ret
 243
 244     def __run_command_in_repo(self, command):
 245         c = Command(command, self.options)
 246         return self.__run_in_repo(c.run_fatal)
 247
 248     def update(self, subdir=None, recursive=None):
 249         return self.__run_command_in_repo("git pull")
 250
 251     def to_branch(self, branch, remote=True):
 252         if remote:
 253             branch = "origin/%s" % branch
 254         return self.__run_command_in_repo("git checkout %s" % branch)
 255
 256     def to_tag(self, tag):
 257         return self.__run_command_in_repo("git checkout %s" % tag)
 258
 259     def tag(self, tagname, logfile):
 260         return self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 261
 262     def diff(self):
 263         c = Command("git diff", self.options)
 264         return self.__run_in_repo(c.output_of, with_stderr=True)
 265
 266     def commit(self, logfile):
 267         self.__run_command_in_repo("git add -A")
 268         self.__run_command_in_repo("git commit -F  %s" % logfile)
 269         self.__run_command_in_repo("git push --tags")
 270
 271     def revert(self):
 272         self.__run_command_in_repo("git --no-pager reset --hard")
 273         self.__run_command_in_repo("git --no-pager clean -f")
 274
 275     def is_clean(self):
 276         def check_commit():
 277             command="git status"
 278             s="nothing to commit (working directory clean)"
 279             return Command(command, self.options).output_of(True).find(s) >= 0
 280         return self.__run_in_repo(check_commit)
 281
 282     def is_valid(self):
 283         return os.path.exists(os.path.join(self.path, ".git"))
 284
 285
 286 class Repository:
 287     """ Generic repository """
 288     supported_repo_types = [SvnRepository, GitRepository]
 289
 290     def __init__(self, path, options):
 291         self.path = path
 292         self.options = options
 293         for repo in self.supported_repo_types:
 294             self.repo = repo(self.path, self.options)
 295             if self.repo.is_valid():
 296                 break
 297
 298     @classmethod
 299     def has_moved_to_git(cls, module, svnpath):
 300         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 301
 302     @classmethod
 303     def remote_exists(cls, remote):
 304         for repo in Repository.supported_repo_types:
 305             if repo.remote_exists(remote):
 306                 return True
 307         return False
 308
 309     def __getattr__(self, attr):
 310         return getattr(self.repo, attr)
 311
 312
 313
 314 # support for tagged module is minimal, and is for the Build class only
 315 class Module:
 316
 317     svn_magic_line="--This line, and those below, will be ignored--"
 318     setting_tag_format = "Setting tag %s"
 319
 320     redirectors=[ # ('module_name_varname','name'),
 321                   ('module_version_varname','version'),
 322                   ('module_taglevel_varname','taglevel'), ]
 323
 324     # where to store user's config
 325     config_storage="CONFIG"
 326     #
 327     config={}
 328
 329     import commands
 330     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 331                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 332                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 333                  ("build", "Enter the name of your build module","build"),
 334                  ('username',"Enter your firstname and lastname for changelogs",""),
 335                  ("email","Enter your email address for changelogs",""),
 336                  ]
 337
 338     @classmethod
 339     def prompt_config (cls):
 340         for (key,message,default) in cls.configKeys:
 341             cls.config[key]=""
 342             while not cls.config[key]:
 343                 cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 344
 345
 346     # for parsing module spec name:branch
 347     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 348     # special form for tagged module - for Build
 349     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 350     # parsing specfiles
 351     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 352
 353     def __init__ (self,module_spec,options):
 354         # parse module spec
 355         attempt=Module.matcher_branch_spec.match(module_spec)
 356         if attempt:
 357             self.name=attempt.group('name')
 358             self.branch=attempt.group('branch')
 359         else:
 360             attempt=Module.matcher_tag_spec.match(module_spec)
 361             if attempt:
 362                 self.name=attempt.group('name')
 363                 self.tagname=attempt.group('tagname')
 364             else:
 365                 self.name=module_spec
 366
 367         self.options=options
 368         self.module_dir="%s/%s"%(options.workdir,self.name)
 369         self.repository = None
 370         self.build = None
 371
 372     def run (self,command):
 373         return Command(command,self.options).run()
 374     def run_fatal (self,command):
 375         return Command(command,self.options).run_fatal()
 376     def run_prompt (self,message,fun, *args):
 377         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 378         if not self.options.verbose:
 379             while True:
 380                 choice=prompt(message,True,('s','how'))
 381                 if choice is True:
 382                     fun(*args)
 383                     return
 384                 elif choice is False:
 385                     print 'About to run function:', fun_msg
 386         else:
 387             question=message+" - want to run function: " + fun_msg
 388             if prompt(question,True):
 389                 fun(*args)
 390
 391     def friendly_name (self):
 392         if hasattr(self,'branch'):
 393             return "%s:%s"%(self.name,self.branch)
 394         elif hasattr(self,'tagname'):
 395             return "%s@%s"%(self.name,self.tagname)
 396         else:
 397             return self.name
 398
 399     @classmethod
 400     def git_remote_dir (cls, name):
 401         return "%s:/git/%s.git" % (cls.config['gitserver'], name)
 402
 403     @classmethod
 404     def svn_remote_dir (cls, name):
 405         svn = cls.config['svnpath']
 406         if svn.endswith('/'):
 407             return "%s%s" % (svn, name)
 408         return "%s/%s" % (svn, name)
 409
 410     ####################
 411     @classmethod
 412     def init_homedir (cls, options):
 413         if options.verbose and options.mode not in Main.silent_modes:
 414             print 'Checking for', options.workdir
 415         storage="%s/%s"%(options.workdir, cls.config_storage)
 416         # sanity check. Either the topdir exists AND we have a config/storage
 417         # or topdir does not exist and we create it
 418         # to avoid people use their own daily svn repo
 419         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 420             print """The directory %s exists and has no CONFIG file
 421 If this is your regular working directory, please provide another one as the
 422 module-* commands need a fresh working dir. Make sure that you do not use
 423 that for other purposes than tagging""" % options.workdir
 424             sys.exit(1)
 425         if not os.path.isdir (options.workdir):
 426             print "Cannot find",options.workdir,"let's create it"
 427             cls.prompt_config()
 428             print "Checking ...",
 429             remote = cls.git_remote_dir(cls.config['build'])
 430             local = os.path.join(options.workdir, cls.config['build'])
 431             GitRepository.checkout(remote, local, options, depth=1)
 432             print "OK"
 433
 434             # store config
 435             f=file(storage,"w")
 436             for (key,message,default) in Module.configKeys:
 437                 f.write("%s=%s\n"%(key,Module.config[key]))
 438             f.close()
 439             if options.debug:
 440                 print 'Stored',storage
 441                 Command("cat %s"%storage,options).run()
 442         else:
 443             # read config
 444             f=open(storage)
 445             for line in f.readlines():
 446                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 447                 Module.config[key]=value
 448             f.close()
 449         if options.verbose and options.mode not in Main.silent_modes:
 450             print '******** Using config'
 451             for (key,message,default) in Module.configKeys:
 452                 print '\t',key,'=',Module.config[key]
 453
 454     def svn_selected_remote(self):
 455         remote = self.svn_remote_dir(self.name)
 456         if hasattr(self,'branch'):
 457             remote = "%s/branches/%s" % (remote, self.branch)
 458         elif hasattr(self,'tagname'):
 459             remote = "%s/tags/%s" % (remote, self.tagname)
 460         else:
 461             remote = "%s/trunk" % remote
 462         return remote
 463
 464     def init_module_dir (self):
 465         if self.options.verbose:
 466             print 'Checking for',self.module_dir
 467
 468         if not os.path.isdir (self.module_dir):
 469             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 470                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 471                                                          self.module_dir,
 472                                                          self.options)
 473             else:
 474                 remote = self.svn_selected_remote()
 475                 self.repository = SvnRepository.checkout(remote,
 476                                                          self.module_dir,
 477                                                          self.options, recursive=False)
 478
 479         self.repository = Repository(self.module_dir, self.options)
 480         if self.repository.type == "svn":
 481             # check if module has moved to git
 482             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 483                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 484                 self.init_module_dir()
 485             # check if we have the required branch/tag
 486             if self.repository.url() != self.svn_selected_remote():
 487                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 488                 self.init_module_dir()
 489
 490         elif self.repository.type == "git":
 491             if hasattr(self,'branch'):
 492                 self.repository.to_branch(self.branch)
 493             elif hasattr(self,'tagname'):
 494                 self.repository.to_tag(self.tagname)
 495
 496         else:
 497             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 498
 499
 500     def revert_module_dir (self):
 501         if self.options.fast_checks:
 502             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 503             return
 504         if self.options.verbose:
 505             print 'Checking whether', self.module_dir, 'needs being reverted'
 506
 507         if not self.repository.is_clean():
 508             self.repository.revert()
 509
 510     def update_module_dir (self):
 511         if self.options.fast_checks:
 512             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 513             return
 514         if self.options.verbose:
 515             print 'Updating', self.module_dir
 516         self.repository.update()
 517
 518     def main_specname (self):
 519         attempt="%s/%s.spec"%(self.module_dir,self.name)
 520         if os.path.isfile (attempt):
 521             return attempt
 522         pattern1="%s/*.spec"%self.module_dir
 523         level1=glob(pattern1)
 524         if level1:
 525             return level1[0]
 526         pattern2="%s/*/*.spec"%self.module_dir
 527         level2=glob(pattern2)
 528
 529         if level2:
 530             return level2[0]
 531         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 532
 533     def all_specnames (self):
 534         level1=glob("%s/*.spec" % self.module_dir)
 535         if level1: return level1
 536         level2=glob("%s/*/*.spec" % self.module_dir)
 537         return level2
 538
 539     def parse_spec (self, specfile, varnames):
 540         if self.options.verbose:
 541             print 'Parsing',specfile,
 542             for var in varnames:
 543                 print "[%s]"%var,
 544             print ""
 545         result={}
 546         f=open(specfile)
 547         for line in f.readlines():
 548             attempt=Module.matcher_rpm_define.match(line)
 549             if attempt:
 550                 (define,var,value)=attempt.groups()
 551                 if var in varnames:
 552                     result[var]=value
 553         f.close()
 554         if self.options.debug:
 555             print 'found',len(result),'keys'
 556             for (k,v) in result.iteritems():
 557                 print k,'=',v
 558         return result
 559
 560     # stores in self.module_name_varname the rpm variable to be used for the module's name
 561     # and the list of these names in self.varnames
 562     def spec_dict (self):
 563         specfile=self.main_specname()
 564         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 565         redirect_dict = self.parse_spec(specfile,redirector_keys)
 566         if self.options.debug:
 567             print '1st pass parsing done, redirect_dict=',redirect_dict
 568         varnames=[]
 569         for (varname,default) in Module.redirectors:
 570             if redirect_dict.has_key(varname):
 571                 setattr(self,varname,redirect_dict[varname])
 572                 varnames += [redirect_dict[varname]]
 573             else:
 574                 setattr(self,varname,default)
 575                 varnames += [ default ]
 576         self.varnames = varnames
 577         result = self.parse_spec (specfile,self.varnames)
 578         if self.options.debug:
 579             print '2st pass parsing done, varnames=',varnames,'result=',result
 580         return result
 581
 582     def patch_spec_var (self, patch_dict,define_missing=False):
 583         for specfile in self.all_specnames():
 584             # record the keys that were changed
 585             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 586             newspecfile=specfile+".new"
 587             if self.options.verbose:
 588                 print 'Patching',specfile,'for',patch_dict.keys()
 589             spec=open (specfile)
 590             new=open(newspecfile,"w")
 591
 592             for line in spec.readlines():
 593                 attempt=Module.matcher_rpm_define.match(line)
 594                 if attempt:
 595                     (define,var,value)=attempt.groups()
 596                     if var in patch_dict.keys():
 597                         if self.options.debug:
 598                             print 'rewriting %s as %s'%(var,patch_dict[var])
 599                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 600                         changed[var]=True
 601                         continue
 602                 new.write(line)
 603             if define_missing:
 604                 for (key,was_changed) in changed.iteritems():
 605                     if not was_changed:
 606                         if self.options.debug:
 607                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 608                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 609             spec.close()
 610             new.close()
 611             os.rename(newspecfile,specfile)
 612
 613     # returns all lines until the magic line
 614     def unignored_lines (self, logfile):
 615         result=[]
 616         white_line_matcher = re.compile("\A\s*\Z")
 617         for logline in file(logfile).readlines():
 618             if logline.strip() == Module.svn_magic_line:
 619                 break
 620             elif white_line_matcher.match(logline):
 621                 continue
 622             else:
 623                 result.append(logline.strip()+'\n')
 624         return result
 625
 626     # creates a copy of the input with only the unignored lines
 627     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 628        f=file(fileout,'w')
 629        f.write(self.setting_tag_format%new_tag_name + '\n')
 630        for line in self.unignored_lines(filein):
 631            f.write(line)
 632        f.close()
 633
 634     def insert_changelog (self, logfile, oldtag, newtag):
 635         for specfile in self.all_specnames():
 636             newspecfile=specfile+".new"
 637             if self.options.verbose:
 638                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 639             spec=open (specfile)
 640             new=open(newspecfile,"w")
 641             for line in spec.readlines():
 642                 new.write(line)
 643                 if re.compile('%changelog').match(line):
 644                     dateformat="* %a %b %d %Y"
 645                     datepart=time.strftime(dateformat)
 646                     logpart="%s <%s> - %s"%(Module.config['username'],
 647                                                  Module.config['email'],
 648                                                  newtag)
 649                     new.write(datepart+" "+logpart+"\n")
 650                     for logline in self.unignored_lines(logfile):
 651                         new.write("- " + logline)
 652                     new.write("\n")
 653             spec.close()
 654             new.close()
 655             os.rename(newspecfile,specfile)
 656
 657     def show_dict (self, spec_dict):
 658         if self.options.verbose:
 659             for (k,v) in spec_dict.iteritems():
 660                 print k,'=',v
 661
 662     def mod_svn_url (self):
 663         return "%s/%s"%(Module.config['svnpath'],self.name)
 664
 665     def edge_url (self):
 666         if hasattr(self,'branch'):
 667             return "%s/branches/%s"%(self.mod_svn_url(),self.branch)
 668         elif hasattr(self,'tagname'):
 669             return "%s/tags/%s"%(self.mod_svn_url(),self.tagname)
 670         else:
 671             return "%s/trunk"%(self.mod_svn_url())
 672
 673     def last_tag (self, spec_dict):
 674         return "%s-%s"%(spec_dict[self.module_version_varname],spec_dict[self.module_taglevel_varname])
 675
 676     def tag_name (self, spec_dict):
 677         try:
 678             return "%s-%s"%(self.name,
 679                             self.last_tag(spec_dict))
 680         except KeyError,err:
 681             raise Exception, 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 682
 683     def tag_url (self, spec_dict):
 684         return "%s/tags/%s"%(self.mod_svn_url(),self.tag_name(spec_dict))
 685
 686     def check_svnpath_exists (self, url, message):
 687         if self.options.fast_checks:
 688             return
 689         if self.options.verbose:
 690             print 'Checking url (%s) %s'%(url,message),
 691
 692         if SvnRepository.remote_exists(url):
 693             if self.options.verbose: print 'exists - OK'
 694         else:
 695             if self.options.verbose: print 'KO'
 696             raise Exception, 'Could not find %s URL %s'%(message,url)
 697
 698     def check_svnpath_not_exists (self, url, message):
 699         if self.options.fast_checks:
 700             return
 701         if self.options.verbose:
 702             print 'Checking url (%s) %s'%(url,message),
 703
 704         if not SvnRepository.remote_exists(url):
 705             if self.options.verbose: print 'does not exist - OK'
 706         else:
 707             if self.options.verbose: print 'KO'
 708             raise Exception, '%s URL %s already exists - exiting'%(message,url)
 709
 710     # locate specfile, parse it, check it and show values
 711
 712
 713 ##############################
 714     # using fine_grain means replacing only those instances that currently refer to this tag
 715     # otherwise, <module>-SVNPATH is replaced unconditionnally
 716     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 717         newtagsfile=tagsfile+".new"
 718         tags=open (tagsfile)
 719         new=open(newtagsfile,"w")
 720
 721         matches=0
 722         # fine-grain : replace those lines that refer to oldname
 723         if fine_grain:
 724             if self.options.verbose:
 725                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 726             matcher=re.compile("^(.*)%s(.*)"%oldname)
 727             for line in tags.readlines():
 728                 if not matcher.match(line):
 729                     new.write(line)
 730                 else:
 731                     (begin,end)=matcher.match(line).groups()
 732                     new.write(begin+newname+end+"\n")
 733                     matches += 1
 734         # brute-force : change uncommented lines that define <module>-SVNPATH
 735         else:
 736             if self.options.verbose:
 737                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 738             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 739                                           %(self.name)
 740             matcher_module=re.compile(pattern)
 741             for line in tags.readlines():
 742                 attempt=matcher_module.match(line)
 743                 if attempt:
 744                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 745                     if self.options.verbose:
 746                         print ' '+svnpath,
 747                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 748                     new.write(replacement)
 749                     matches += 1
 750                 else:
 751                     new.write(line)
 752         tags.close()
 753         new.close()
 754         os.rename(newtagsfile,tagsfile)
 755         if self.options.verbose: print "%d changes"%matches
 756         return matches
 757
 758     def do_tag (self):
 759         self.init_module_dir()
 760         self.revert_module_dir()
 761         self.update_module_dir()
 762         # parse specfile
 763         spec_dict = self.spec_dict()
 764         self.show_dict(spec_dict)
 765
 766         # side effects
 767         edge_url=self.edge_url()
 768         old_tag_name = self.tag_name(spec_dict)
 769         old_tag_url=self.tag_url(spec_dict)
 770         if (self.options.new_version):
 771             # new version set on command line
 772             spec_dict[self.module_version_varname] = self.options.new_version
 773             spec_dict[self.module_taglevel_varname] = 0
 774         else:
 775             # increment taglevel
 776             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 777             spec_dict[self.module_taglevel_varname] = new_taglevel
 778
 779         # sanity check
 780         new_tag_name = self.tag_name(spec_dict)
 781         new_tag_url=self.tag_url(spec_dict)
 782         self.check_svnpath_exists (edge_url,"edge track")
 783         self.check_svnpath_exists (old_tag_url,"previous tag")
 784         self.check_svnpath_not_exists (new_tag_url,"new tag")
 785
 786         # checking for diffs
 787         diff_output=Command("svn diff %s %s"%(old_tag_url,edge_url),
 788                             self.options).output_of()
 789         if len(diff_output) == 0:
 790             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 791                 return
 792
 793         # side effect in trunk's specfile
 794         self.patch_spec_var(spec_dict)
 795
 796         # prepare changelog file
 797         # we use the standard subversion magic string (see svn_magic_line)
 798         # so we can provide useful information, such as version numbers and diff
 799         # in the same file
 800         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 801         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 802         setting_tag_line=Module.setting_tag_format%new_tag_name
 803         file(changelog,"w").write("""
 804 %s
 805 %s
 806 Please write a changelog for this new tag in the section above
 807 """%(Module.svn_magic_line,setting_tag_line))
 808
 809         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 810             file(changelog,"a").write('DIFF=========\n' + diff_output)
 811
 812         if self.options.debug:
 813             prompt('Proceed ?')
 814
 815         # edit it
 816         self.run("%s %s"%(self.options.editor,changelog))
 817         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 818         # so we do the job ourselves
 819         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 820         # insert changelog in spec
 821         if self.options.changelog:
 822             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 823
 824         ## update build
 825         build_path = os.path.join(self.options.workdir,
 826                                   Module.config['build'])
 827         build = Repository(build_path, self.options)
 828         if self.options.build_branch:
 829             build.to_branch(self.options.build_branch)
 830
 831         tagsfiles=glob(build.path+"/*-tags*.mk")
 832         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 833         default_answer = 'y'
 834         tagsfiles.sort()
 835         while True:
 836             for tagsfile in tagsfiles:
 837                 status=tagsdict[tagsfile]
 838                 basename=os.path.basename(tagsfile)
 839                 print ".................... Dealing with %s"%basename
 840                 while tagsdict[tagsfile] == 'todo' :
 841                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 842                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 843                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 844                                      allow_outside=True)
 845                     if choice == 'y':
 846                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 847                     elif choice == 'n':
 848                         print 'Done with %s'%os.path.basename(tagsfile)
 849                         tagsdict[tagsfile]='done'
 850                     elif choice == 'f':
 851                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 852                     elif choice == 'd':
 853                         self.run("svn diff %s"%tagsfile)
 854                     elif choice == 'r':
 855                         self.run("svn revert %s"%tagsfile)
 856                     elif choice == 'c':
 857                         self.run("cat %s"%tagsfile)
 858                     else:
 859                         name=self.name
 860                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 861 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 862 d: show current diff for this tag file
 863 r: revert that tag file
 864 c: cat the current tag file
 865 n: move to next file"""%locals()
 866
 867             if prompt("Want to review changes on tags files",False):
 868                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 869                 default_answer='d'
 870             else:
 871                 break
 872
 873         def diff_all_changes():
 874             print build.diff()
 875             print self.repository.diff()
 876
 877         def commit_all_changes(log):
 878             self.repository.commit(log)
 879             build.commit(log)
 880
 881         self.run_prompt("Review module and build", diff_all_changes)
 882         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 883
 884         if self.repository.type == 'svn':
 885             self.run_prompt("Create tag", self.repository.tag, edge_url, new_tag_url, changelog_svn)
 886         elif self.repository.type == 'git':
 887             tagname = os.path.basename(new_tag_url)
 888             self.run_prompt("Create tag", self.repository.tag, tagname, changelog_svn)
 889
 890         if self.options.debug:
 891             print 'Preserving',changelog,'and stripped',changelog_svn
 892         else:
 893             os.unlink(changelog)
 894             os.unlink(changelog_svn)
 895
 896
 897 ##############################
 898 class Main:
 899
 900     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 901
 902 module-tools : a set of tools to manage subversion tags and specfile
 903   requires the specfile to either
 904   * define *version* and *taglevel*
 905   OR alternatively
 906   * define redirection variables module_version_varname / module_taglevel_varname
 907 Trunk:
 908   by default, the trunk of modules is taken into account
 909   in this case, just mention the module name as <module_desc>
 910 Branches:
 911   if you wish to work on a branch rather than on the trunk,
 912   you can use something like e.g. Mom:2.1 as <module_desc>
 913 """
 914     release_usage="""Usage: %prog [options] tag1 .. tagn
 915   Extract release notes from the changes in specfiles between several build tags, latest first
 916   Examples:
 917       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 918   You can refer to a (build) branch by prepending a colon, like in
 919       release-changelog :4.2 4.2-rc25
 920   You can refer to the build trunk by just mentioning 'trunk', e.g.
 921       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 922 """
 923     common_usage="""More help:
 924   see http://svn.planet-lab.org/wiki/ModuleTools"""
 925
 926     modes={
 927         'list' : "displays a list of available tags or branches",
 928         'version' : "check latest specfile and print out details",
 929         'diff' : "show difference between module (trunk or branch) and latest tag",
 930         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 931                 create new tag and and monitor its adoption in build/*-tags*.mk""",
 932         'branch' : """create a branch for this module, from the latest tag on the trunk,
 933                   and change trunk's version number to reflect the new branch name;
 934                   you can specify the new branch name by using module:branch""",
 935         'sync' : """create a tag from the module
 936                 this is a last resort option, mostly for repairs""",
 937         'changelog' : """extract changelog between build tags
 938                 expected arguments are a list of tags""",
 939         }
 940
 941     silent_modes = ['list']
 942     release_modes = ['changelog']
 943
 944     @staticmethod
 945     def optparse_list (option, opt, value, parser):
 946         try:
 947             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
 948         except:
 949             setattr(parser.values,option.dest,value.split())
 950
 951     def run(self):
 952
 953         mode=None
 954         for function in Main.modes.keys():
 955             if sys.argv[0].find(function) >= 0:
 956                 mode = function
 957                 break
 958         if not mode:
 959             print "Unsupported command",sys.argv[0]
 960             print "Supported commands:" + " ".join(Main.modes.keys())
 961             sys.exit(1)
 962
 963         if mode not in Main.release_modes:
 964             usage = Main.module_usage
 965             usage += Main.common_usage
 966             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
 967         else:
 968             usage = Main.release_usage
 969             usage += Main.common_usage
 970
 971         parser=OptionParser(usage=usage,version=subversion_id)
 972
 973         if mode == "tag" or mode == 'branch':
 974             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
 975                               help="set new version and reset taglevel to 0")
 976         if mode == "tag" :
 977             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
 978                               help="do not update changelog section in specfile when tagging")
 979             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
 980                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
 981         if mode == "tag" or mode == "sync" :
 982             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
 983                               help="specify editor")
 984
 985         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
 986         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
 987                           help="dry run - shell commands are only displayed")
 988         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
 989                           default=[], nargs=1,type="string",
 990                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
 991 -- can be set multiple times, or use quotes""")
 992
 993         parser.add_option("-w","--workdir", action="store", dest="workdir",
 994                           default="%s/%s"%(os.getenv("HOME"),"modules"),
 995                           help="""name for dedicated working dir - defaults to ~/modules
 996 ** THIS MUST NOT ** be your usual working directory""")
 997         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
 998                           help="skip safety checks, such as svn updates -- use with care")
 999
1000         # default verbosity depending on function - temp
1001         verbose_modes= ['tag', 'sync', 'branch']
1002
1003         if mode not in verbose_modes:
1004             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1005                               help="run in verbose mode")
1006         else:
1007             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1008                               help="run in quiet (non-verbose) mode")
1009         (options, args) = parser.parse_args()
1010         options.mode=mode
1011         if not hasattr(options,'dry_run'):
1012             options.dry_run=False
1013         if not hasattr(options,'www'):
1014             options.www=False
1015         options.debug=False
1016
1017         ########## module-*
1018         if len(args) == 0:
1019             parser.print_help()
1020             sys.exit(1)
1021         Module.init_homedir(options)
1022
1023         modules=[ Module(modname,options) for modname in args ]
1024         for module in modules:
1025             if len(args)>1 and mode not in Main.silent_modes:
1026                 print '========================================',module.friendly_name()
1027             # call the method called do_<mode>
1028             method=Module.__dict__["do_%s"%mode]
1029             try:
1030                 method(module)
1031             except Exception,e:
1032                 import traceback
1033                 traceback.print_exc()
1034                 print 'Skipping module %s: '%modname,e
1035
1036 ####################
1037 if __name__ == "__main__" :
1038     try:
1039         Main().run()
1040     except KeyboardInterrupt:
1041         print '\nBye'