module-tools.py

   1 #!/usr/bin/python -u
   2
   3 import sys, os
   4 import re
   5 import time
   6 from glob import glob
   7 from optparse import OptionParser
   8
   9 # HARDCODED NAME CHANGES
  10 #
  11 # Moving to git we decided to rename some of the repositories. Here is
  12 # a map of name changes applied in git repositories.
  13 RENAMED_SVN_MODULES = {
  14     "PLEWWW": "plewww"
  15     }
  16
  17 def svn_to_git_name(module):
  18     if RENAMED_SVN_MODULES.has_key(module):
  19         return RENAMED_SVN_MODULES[module]
  20     return module
  21
  22 def git_to_svn_name(module):
  23     for key in RENAMED_SVN_MODULES:
  24         if module == RENAMED_SVN_MODULES[key]:
  25             return key
  26     return module
  27
  28
  29 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  30 def prompt (question,default=True,other_choices=[],allow_outside=False):
  31     if not isinstance (other_choices,list):
  32         other_choices = [ other_choices ]
  33     chars = [ c for (c,rest) in other_choices ]
  34
  35     choices = []
  36     if 'y' not in chars:
  37         if default is True: choices.append('[y]')
  38         else : choices.append('y')
  39     if 'n' not in chars:
  40         if default is False: choices.append('[n]')
  41         else : choices.append('n')
  42
  43     for (char,choice) in other_choices:
  44         if default == char:
  45             choices.append("["+char+"]"+choice)
  46         else:
  47             choices.append("<"+char+">"+choice)
  48     try:
  49         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  50         if not answer:
  51             return default
  52         answer=answer[0].lower()
  53         if answer == 'y':
  54             if 'y' in chars: return 'y'
  55             else: return True
  56         elif answer == 'n':
  57             if 'n' in chars: return 'n'
  58             else: return False
  59         elif other_choices:
  60             for (char,choice) in other_choices:
  61                 if answer == char:
  62                     return char
  63             if allow_outside:
  64                 return answer
  65         return prompt(question,default,other_choices)
  66     except:
  67         raise
  68
  69 def default_editor():
  70     try:
  71         editor = os.environ['EDITOR']
  72     except:
  73         editor = "emacs"
  74     return editor
  75
  76 ### fold long lines
  77 fold_length=132
  78
  79 def print_fold (line):
  80     while len(line) >= fold_length:
  81         print line[:fold_length],'\\'
  82         line=line[fold_length:]
  83     print line
  84
  85 class Command:
  86     def __init__ (self,command,options):
  87         self.command=command
  88         self.options=options
  89         self.tmp="/tmp/command-%d"%os.getpid()
  90
  91     def run (self):
  92         if self.options.dry_run:
  93             print 'dry_run',self.command
  94             return 0
  95         if self.options.verbose and self.options.mode not in Main.silent_modes:
  96             print '+',self.command
  97             sys.stdout.flush()
  98         return os.system(self.command)
  99
 100     def run_silent (self):
 101         if self.options.dry_run:
 102             print 'dry_run',self.command
 103             return 0
 104         if self.options.verbose:
 105             print '+',self.command,' .. ',
 106             sys.stdout.flush()
 107         retcod=os.system(self.command + " &> " + self.tmp)
 108         if retcod != 0:
 109             print "FAILED ! -- out+err below (command was %s)"%self.command
 110             os.system("cat " + self.tmp)
 111             print "FAILED ! -- end of quoted output"
 112         elif self.options.verbose:
 113             print "OK"
 114         os.unlink(self.tmp)
 115         return retcod
 116
 117     def run_fatal(self):
 118         if self.run_silent() !=0:
 119             raise Exception,"Command %s failed"%self.command
 120
 121     # returns stdout, like bash's $(mycommand)
 122     def output_of (self,with_stderr=False):
 123         if self.options.dry_run:
 124             print 'dry_run',self.command
 125             return 'dry_run output'
 126         tmp="/tmp/status-%d"%os.getpid()
 127         if self.options.debug:
 128             print '+',self.command,' .. ',
 129             sys.stdout.flush()
 130         command=self.command
 131         if with_stderr:
 132             command += " &> "
 133         else:
 134             command += " > "
 135         command += tmp
 136         os.system(command)
 137         result=file(tmp).read()
 138         os.unlink(tmp)
 139         if self.options.debug:
 140             print 'Done',
 141         return result
 142
 143
 144 class SvnRepository:
 145     type = "svn"
 146
 147     def __init__(self, path, options):
 148         self.path = path
 149         self.options = options
 150
 151     def name(self):
 152         return os.path.basename(self.path)
 153
 154     def url(self):
 155         out = Command("svn info %s" % self.path, self.options).output_of()
 156         for line in out.split('\n'):
 157             if line.startswith("URL:"):
 158                 return line.split()[1].strip()
 159
 160     def repo_root(self):
 161         out = Command("svn info %s" % self.path, self.options).output_of()
 162         for line in out.split('\n'):
 163             if line.startswith("Repository Root:"):
 164                 repo_root = line.split()[2].strip()
 165                 return "%s/svn/%s" (repo_root, self.name)
 166
 167     @classmethod
 168     def checkout(cls, remote, local, options, recursive=False):
 169         if recursive:
 170             svncommand = "svn co %s %s" % (remote, local)
 171         else:
 172             svncommand = "svn co -N %s %s" % (remote, local)
 173         Command("rm -rf %s" % local, options).run_silent()
 174         Command(svncommand, options).run_fatal()
 175
 176         return SvnRepository(local, options)
 177
 178     @classmethod
 179     def remote_exists(cls, remote):
 180         return os.system("svn list %s &> /dev/null" % remote) == 0
 181
 182     def tag_exists(self, tagname):
 183         url = "%s/tags/%s" % (self.repo_root(), tagname)
 184         return SvnRepository.remote_exists(url)
 185
 186     def update(self, subdir="", recursive=True):
 187         path = os.path.join(self.path, subdir)
 188         if recursive:
 189             svncommand = "svn up %s" % path
 190         else:
 191             svncommand = "svn up -N %s" % path
 192         Command(svncommand, self.options).run_fatal()
 193
 194     def commit(self, logfile):
 195         # add all new files to the repository
 196         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 197                 self.path, self.options).run_silent()
 198         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 199
 200     def to_branch(self, branch):
 201         remote = "%s/branches/%s" % (self.repo_root(), branch)
 202         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 203
 204     def to_tag(self, tag):
 205         remote = "%s/tags/%s" % (self.repo_root(), branch)
 206         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 207
 208     def tag(self, tagname, logfile):
 209         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 210         self_url = self.url()
 211         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 212
 213     def diff(self):
 214         return Command("svn diff %s" % self.path, self.options).output_of(True)
 215
 216     def diff_with_tag(self, tagname):
 217         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 218         return Command("svn diff %s %s" % (tag_url, self.url()),
 219                        self.options).output_of(True)
 220
 221     def revert(self):
 222         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 223         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 224                 self.path, self.options).run_silent()
 225
 226     def is_clean(self):
 227         command="svn status %s" % self.path
 228         return len(Command(command,self.options).output_of(True)) == 0
 229
 230     def is_valid(self):
 231         return os.path.exists(os.path.join(self.path, ".svn"))
 232
 233
 234 class GitRepository:
 235     type = "git"
 236
 237     def __init__(self, path, options):
 238         self.path = path
 239         self.options = options
 240
 241     def name(self):
 242         return os.path.basename(self.path)
 243
 244     def url(self):
 245         self.repo_root()
 246
 247     def repo_root(self):
 248         c = Command("git remote show origin", self.options)
 249         out = self.__run_in_repo(c.output_of)
 250         for line in out.split('\n'):
 251             if line.strip().startswith("Fetch URL:"):
 252                 repo = line.split()[2]
 253
 254     @classmethod
 255     def checkout(cls, remote, local, options, depth=1):
 256         Command("rm -rf %s" % local, options).run_silent()
 257         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 258         return GitRepository(local, options)
 259
 260     @classmethod
 261     def remote_exists(cls, remote):
 262         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 263
 264     def tag_exists(self, tagname):
 265         command = 'git tag -l | grep "^%s$"' % tagname
 266         c = Command(command, self.options)
 267         out = self.__run_in_repo(c.output_of, with_stderr=True)
 268         return len(out) > 0
 269
 270     def __run_in_repo(self, fun, *args, **kwargs):
 271         cwd = os.getcwd()
 272         os.chdir(self.path)
 273         ret = fun(*args, **kwargs)
 274         os.chdir(cwd)
 275         return ret
 276
 277     def __run_command_in_repo(self, command, ignore_errors=False):
 278         c = Command(command, self.options)
 279         if ignore_errors:
 280             return self.__run_in_repo(c.output_of)
 281         else:
 282             return self.__run_in_repo(c.run_fatal)
 283
 284     def update(self, subdir=None, recursive=None):
 285         return self.__run_command_in_repo("git pull")
 286
 287     def to_branch(self, branch, remote=True):
 288         if remote:
 289             branch = "origin/%s" % branch
 290         return self.__run_command_in_repo("git checkout %s" % branch)
 291
 292     def to_tag(self, tag):
 293         return self.__run_command_in_repo("git checkout %s" % tag)
 294
 295     def tag(self, tagname, logfile):
 296         self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 297         self.commit(logfile)
 298
 299     def diff(self):
 300         c = Command("git diff", self.options)
 301         return self.__run_in_repo(c.output_of, with_stderr=True)
 302
 303     def diff_with_tag(self, tagname):
 304         c = Command("git diff %s" % tagname, self.options)
 305         return self.__run_in_repo(c.output_of, with_stderr=True)
 306
 307     def commit(self, logfile):
 308         self.__run_command_in_repo("git add -A", ignore_errors=True)
 309         self.__run_command_in_repo("git commit -F  %s" % logfile, ignore_errors=True)
 310         self.__run_command_in_repo("git push")
 311         self.__run_command_in_repo("git push --tags")
 312
 313     def revert(self):
 314         self.__run_command_in_repo("git --no-pager reset --hard")
 315         self.__run_command_in_repo("git --no-pager clean -f")
 316
 317     def is_clean(self):
 318         def check_commit():
 319             command="git status"
 320             s="nothing to commit (working directory clean)"
 321             return Command(command, self.options).output_of(True).find(s) >= 0
 322         return self.__run_in_repo(check_commit)
 323
 324     def is_valid(self):
 325         return os.path.exists(os.path.join(self.path, ".git"))
 326
 327
 328 class Repository:
 329     """ Generic repository """
 330     supported_repo_types = [SvnRepository, GitRepository]
 331
 332     def __init__(self, path, options):
 333         self.path = path
 334         self.options = options
 335         for repo in self.supported_repo_types:
 336             self.repo = repo(self.path, self.options)
 337             if self.repo.is_valid():
 338                 break
 339
 340     @classmethod
 341     def has_moved_to_git(cls, module, svnpath):
 342         module = git_to_svn_name(module)
 343         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 344
 345     @classmethod
 346     def remote_exists(cls, remote):
 347         for repo in Repository.supported_repo_types:
 348             if repo.remote_exists(remote):
 349                 return True
 350         return False
 351
 352     def __getattr__(self, attr):
 353         return getattr(self.repo, attr)
 354
 355
 356
 357 # support for tagged module is minimal, and is for the Build class only
 358 class Module:
 359
 360     svn_magic_line="--This line, and those below, will be ignored--"
 361     setting_tag_format = "Setting tag %s"
 362
 363     redirectors=[ # ('module_name_varname','name'),
 364                   ('module_version_varname','version'),
 365                   ('module_taglevel_varname','taglevel'), ]
 366
 367     # where to store user's config
 368     config_storage="CONFIG"
 369     #
 370     config={}
 371
 372     import commands
 373     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 374                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 375                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 376                  ("build", "Enter the name of your build module","build"),
 377                  ('username',"Enter your firstname and lastname for changelogs",""),
 378                  ("email","Enter your email address for changelogs",""),
 379                  ]
 380
 381     @classmethod
 382     def prompt_config_option(cls, key, message, default):
 383         cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 384
 385     @classmethod
 386     def prompt_config (cls):
 387         for (key,message,default) in cls.configKeys:
 388             cls.config[key]=""
 389             while not cls.config[key]:
 390                 cls.prompt_config_option(key, message, default)
 391
 392
 393     # for parsing module spec name:branch
 394     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 395     # special form for tagged module - for Build
 396     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 397     # parsing specfiles
 398     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 399
 400     def __init__ (self,module_spec,options):
 401         # parse module spec
 402         attempt=Module.matcher_branch_spec.match(module_spec)
 403         if attempt:
 404             self.name=attempt.group('name')
 405             self.branch=attempt.group('branch')
 406         else:
 407             attempt=Module.matcher_tag_spec.match(module_spec)
 408             if attempt:
 409                 self.name=attempt.group('name')
 410                 self.tagname=attempt.group('tagname')
 411             else:
 412                 self.name=module_spec
 413
 414         # when available prefer to use git module name internally
 415         self.name = svn_to_git_name(self.name)
 416
 417         self.options=options
 418         self.module_dir="%s/%s"%(options.workdir,self.name)
 419         self.repository = None
 420         self.build = None
 421
 422     def run (self,command):
 423         return Command(command,self.options).run()
 424     def run_fatal (self,command):
 425         return Command(command,self.options).run_fatal()
 426     def run_prompt (self,message,fun, *args):
 427         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 428         if not self.options.verbose:
 429             while True:
 430                 choice=prompt(message,True,('s','how'))
 431                 if choice is True:
 432                     fun(*args)
 433                     return
 434                 elif choice is False:
 435                     print 'About to run function:', fun_msg
 436         else:
 437             question=message+" - want to run function: " + fun_msg
 438             if prompt(question,True):
 439                 fun(*args)
 440
 441     def friendly_name (self):
 442         if hasattr(self,'branch'):
 443             return "%s:%s"%(self.name,self.branch)
 444         elif hasattr(self,'tagname'):
 445             return "%s@%s"%(self.name,self.tagname)
 446         else:
 447             return self.name
 448
 449     @classmethod
 450     def git_remote_dir (cls, name):
 451         return "%s:/git/%s.git" % (cls.config['gitserver'], name)
 452
 453     @classmethod
 454     def svn_remote_dir (cls, name):
 455         name = git_to_svn_name(name)
 456         svn = cls.config['svnpath']
 457         if svn.endswith('/'):
 458             return "%s%s" % (svn, name)
 459         return "%s/%s" % (svn, name)
 460
 461     def svn_selected_remote(self):
 462         svn_name = git_to_svn_name(self.name)
 463         remote = self.svn_remote_dir(svn_name)
 464         if hasattr(self,'branch'):
 465             remote = "%s/branches/%s" % (remote, self.branch)
 466         elif hasattr(self,'tagname'):
 467             remote = "%s/tags/%s" % (remote, self.tagname)
 468         else:
 469             remote = "%s/trunk" % remote
 470         return remote
 471
 472     ####################
 473     @classmethod
 474     def init_homedir (cls, options):
 475         if options.verbose and options.mode not in Main.silent_modes:
 476             print 'Checking for', options.workdir
 477         storage="%s/%s"%(options.workdir, cls.config_storage)
 478         # sanity check. Either the topdir exists AND we have a config/storage
 479         # or topdir does not exist and we create it
 480         # to avoid people use their own daily svn repo
 481         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 482             print """The directory %s exists and has no CONFIG file
 483 If this is your regular working directory, please provide another one as the
 484 module-* commands need a fresh working dir. Make sure that you do not use
 485 that for other purposes than tagging""" % options.workdir
 486             sys.exit(1)
 487
 488         def checkout_build():
 489             print "Checking out build module..."
 490             remote = cls.git_remote_dir(cls.config['build'])
 491             local = os.path.join(options.workdir, cls.config['build'])
 492             GitRepository.checkout(remote, local, options, depth=1)
 493             print "OK"
 494
 495         def store_config():
 496             f=file(storage,"w")
 497             for (key,message,default) in Module.configKeys:
 498                 f.write("%s=%s\n"%(key,Module.config[key]))
 499             f.close()
 500             if options.debug:
 501                 print 'Stored',storage
 502                 Command("cat %s"%storage,options).run()
 503
 504         def read_config():
 505             # read config
 506             f=open(storage)
 507             for line in f.readlines():
 508                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 509                 Module.config[key]=value
 510             f.close()
 511
 512         if not os.path.isdir (options.workdir):
 513             print "Cannot find",options.workdir,"let's create it"
 514             Command("mkdir -p %s" % options.workdir, options).run_silent()
 515             cls.prompt_config()
 516             checkout_build()
 517             store_config()
 518         else:
 519             read_config()
 520             # check missing config options
 521             old_layout = False
 522             for (key,message,default) in cls.configKeys:
 523                 if not Module.config.has_key(key):
 524                     print "Configuration changed for module-tools"
 525                     cls.prompt_config_option(key, message, default)
 526                     old_layout = True
 527
 528             if old_layout:
 529                 Command("rm -rf %s" % options.workdir, options).run_silent()
 530                 Command("mkdir -p %s" % options.workdir, options).run_silent()
 531                 checkout_build()
 532                 store_config()
 533
 534             build_dir = os.path.join(options.workdir, cls.config['build'])
 535             build = Repository(build_dir, options)
 536             if not build.is_clean():
 537                 print "build module needs a revert"
 538                 build.revert()
 539                 print "OK"
 540             build.update()
 541
 542         if options.verbose and options.mode not in Main.silent_modes:
 543             print '******** Using config'
 544             for (key,message,default) in Module.configKeys:
 545                 print '\t',key,'=',Module.config[key]
 546
 547     def init_module_dir (self):
 548         if self.options.verbose:
 549             print 'Checking for',self.module_dir
 550
 551         if not os.path.isdir (self.module_dir):
 552             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 553                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 554                                                          self.module_dir,
 555                                                          self.options)
 556             else:
 557                 remote = self.svn_selected_remote()
 558                 self.repository = SvnRepository.checkout(remote,
 559                                                          self.module_dir,
 560                                                          self.options, recursive=False)
 561
 562         self.repository = Repository(self.module_dir, self.options)
 563         if self.repository.type == "svn":
 564             # check if module has moved to git
 565             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 566                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 567                 self.init_module_dir()
 568             # check if we have the required branch/tag
 569             if self.repository.url() != self.svn_selected_remote():
 570                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 571                 self.init_module_dir()
 572
 573         elif self.repository.type == "git":
 574             if hasattr(self,'branch'):
 575                 self.repository.to_branch(self.branch)
 576             elif hasattr(self,'tagname'):
 577                 self.repository.to_tag(self.tagname)
 578
 579         else:
 580             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 581
 582
 583     def revert_module_dir (self):
 584         if self.options.fast_checks:
 585             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 586             return
 587         if self.options.verbose:
 588             print 'Checking whether', self.module_dir, 'needs being reverted'
 589
 590         if not self.repository.is_clean():
 591             self.repository.revert()
 592
 593     def update_module_dir (self):
 594         if self.options.fast_checks:
 595             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 596             return
 597         if self.options.verbose:
 598             print 'Updating', self.module_dir
 599         self.repository.update()
 600
 601     def main_specname (self):
 602         attempt="%s/%s.spec"%(self.module_dir,self.name)
 603         if os.path.isfile (attempt):
 604             return attempt
 605         pattern1="%s/*.spec"%self.module_dir
 606         level1=glob(pattern1)
 607         if level1:
 608             return level1[0]
 609         pattern2="%s/*/*.spec"%self.module_dir
 610         level2=glob(pattern2)
 611
 612         if level2:
 613             return level2[0]
 614         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 615
 616     def all_specnames (self):
 617         level1=glob("%s/*.spec" % self.module_dir)
 618         if level1: return level1
 619         level2=glob("%s/*/*.spec" % self.module_dir)
 620         return level2
 621
 622     def parse_spec (self, specfile, varnames):
 623         if self.options.verbose:
 624             print 'Parsing',specfile,
 625             for var in varnames:
 626                 print "[%s]"%var,
 627             print ""
 628         result={}
 629         f=open(specfile)
 630         for line in f.readlines():
 631             attempt=Module.matcher_rpm_define.match(line)
 632             if attempt:
 633                 (define,var,value)=attempt.groups()
 634                 if var in varnames:
 635                     result[var]=value
 636         f.close()
 637         if self.options.debug:
 638             print 'found',len(result),'keys'
 639             for (k,v) in result.iteritems():
 640                 print k,'=',v
 641         return result
 642
 643     # stores in self.module_name_varname the rpm variable to be used for the module's name
 644     # and the list of these names in self.varnames
 645     def spec_dict (self):
 646         specfile=self.main_specname()
 647         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 648         redirect_dict = self.parse_spec(specfile,redirector_keys)
 649         if self.options.debug:
 650             print '1st pass parsing done, redirect_dict=',redirect_dict
 651         varnames=[]
 652         for (varname,default) in Module.redirectors:
 653             if redirect_dict.has_key(varname):
 654                 setattr(self,varname,redirect_dict[varname])
 655                 varnames += [redirect_dict[varname]]
 656             else:
 657                 setattr(self,varname,default)
 658                 varnames += [ default ]
 659         self.varnames = varnames
 660         result = self.parse_spec (specfile,self.varnames)
 661         if self.options.debug:
 662             print '2st pass parsing done, varnames=',varnames,'result=',result
 663         return result
 664
 665     def patch_spec_var (self, patch_dict,define_missing=False):
 666         for specfile in self.all_specnames():
 667             # record the keys that were changed
 668             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 669             newspecfile=specfile+".new"
 670             if self.options.verbose:
 671                 print 'Patching',specfile,'for',patch_dict.keys()
 672             spec=open (specfile)
 673             new=open(newspecfile,"w")
 674
 675             for line in spec.readlines():
 676                 attempt=Module.matcher_rpm_define.match(line)
 677                 if attempt:
 678                     (define,var,value)=attempt.groups()
 679                     if var in patch_dict.keys():
 680                         if self.options.debug:
 681                             print 'rewriting %s as %s'%(var,patch_dict[var])
 682                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 683                         changed[var]=True
 684                         continue
 685                 new.write(line)
 686             if define_missing:
 687                 for (key,was_changed) in changed.iteritems():
 688                     if not was_changed:
 689                         if self.options.debug:
 690                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 691                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 692             spec.close()
 693             new.close()
 694             os.rename(newspecfile,specfile)
 695
 696     # returns all lines until the magic line
 697     def unignored_lines (self, logfile):
 698         result=[]
 699         white_line_matcher = re.compile("\A\s*\Z")
 700         for logline in file(logfile).readlines():
 701             if logline.strip() == Module.svn_magic_line:
 702                 break
 703             elif white_line_matcher.match(logline):
 704                 continue
 705             else:
 706                 result.append(logline.strip()+'\n')
 707         return result
 708
 709     # creates a copy of the input with only the unignored lines
 710     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 711        f=file(fileout,'w')
 712        f.write(self.setting_tag_format%new_tag_name + '\n')
 713        for line in self.unignored_lines(filein):
 714            f.write(line)
 715        f.close()
 716
 717     def insert_changelog (self, logfile, oldtag, newtag):
 718         for specfile in self.all_specnames():
 719             newspecfile=specfile+".new"
 720             if self.options.verbose:
 721                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 722             spec=open (specfile)
 723             new=open(newspecfile,"w")
 724             for line in spec.readlines():
 725                 new.write(line)
 726                 if re.compile('%changelog').match(line):
 727                     dateformat="* %a %b %d %Y"
 728                     datepart=time.strftime(dateformat)
 729                     logpart="%s <%s> - %s"%(Module.config['username'],
 730                                                  Module.config['email'],
 731                                                  newtag)
 732                     new.write(datepart+" "+logpart+"\n")
 733                     for logline in self.unignored_lines(logfile):
 734                         new.write("- " + logline)
 735                     new.write("\n")
 736             spec.close()
 737             new.close()
 738             os.rename(newspecfile,specfile)
 739
 740     def show_dict (self, spec_dict):
 741         if self.options.verbose:
 742             for (k,v) in spec_dict.iteritems():
 743                 print k,'=',v
 744
 745     def last_tag (self, spec_dict):
 746         try:
 747             return "%s-%s" % (spec_dict[self.module_version_varname],
 748                               spec_dict[self.module_taglevel_varname])
 749         except KeyError,err:
 750             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 751
 752     def tag_name (self, spec_dict, old_svn_name=False):
 753         base_tag_name = self.name
 754         if old_svn_name:
 755             base_tag_name = git_to_svn_name(self.name)
 756         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 757
 758
 759 ##############################
 760     # using fine_grain means replacing only those instances that currently refer to this tag
 761     # otherwise, <module>-SVNPATH is replaced unconditionnally
 762     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 763         newtagsfile=tagsfile+".new"
 764         tags=open (tagsfile)
 765         new=open(newtagsfile,"w")
 766
 767         matches=0
 768         # fine-grain : replace those lines that refer to oldname
 769         if fine_grain:
 770             if self.options.verbose:
 771                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 772             matcher=re.compile("^(.*)%s(.*)"%oldname)
 773             for line in tags.readlines():
 774                 if not matcher.match(line):
 775                     new.write(line)
 776                 else:
 777                     (begin,end)=matcher.match(line).groups()
 778                     new.write(begin+newname+end+"\n")
 779                     matches += 1
 780         # brute-force : change uncommented lines that define <module>-SVNPATH
 781         else:
 782             if self.options.verbose:
 783                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 784             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 785                                           %(self.name)
 786             matcher_module=re.compile(pattern)
 787             for line in tags.readlines():
 788                 attempt=matcher_module.match(line)
 789                 if attempt:
 790                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 791                     if self.options.verbose:
 792                         print ' '+svnpath,
 793                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 794                     new.write(replacement)
 795                     matches += 1
 796                 else:
 797                     new.write(line)
 798         tags.close()
 799         new.close()
 800         os.rename(newtagsfile,tagsfile)
 801         if self.options.verbose: print "%d changes"%matches
 802         return matches
 803
 804     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 805         if self.options.verbose:
 806             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 807
 808         found_tagname = tagname
 809         found = self.repository.tag_exists(tagname)
 810         if not found and old_svn_tag_name:
 811             if self.options.verbose:
 812                 print "KO"
 813                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 814             found = self.repository.tag_exists(old_svn_tag_name)
 815             if found:
 816                 found_tagname = old_svn_tag_name
 817
 818         if (found and need_it) or (not found and not need_it):
 819             print "OK "
 820         else:
 821             print "KO"
 822             if found:
 823                 raise Exception, "tag (%s) is already there" % tagname
 824             else:
 825                 raise Exception, "can not find required tag (%s)" % tagname
 826
 827         return found_tagname
 828
 829
 830     def do_tag (self):
 831         self.init_module_dir()
 832         self.revert_module_dir()
 833         self.update_module_dir()
 834         # parse specfile
 835         spec_dict = self.spec_dict()
 836         self.show_dict(spec_dict)
 837
 838         # side effects
 839         old_tag_name = self.tag_name(spec_dict)
 840         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 841
 842         if (self.options.new_version):
 843             # new version set on command line
 844             spec_dict[self.module_version_varname] = self.options.new_version
 845             spec_dict[self.module_taglevel_varname] = 0
 846         else:
 847             # increment taglevel
 848             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 849             spec_dict[self.module_taglevel_varname] = new_taglevel
 850
 851         new_tag_name = self.tag_name(spec_dict)
 852
 853         # sanity check
 854         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 855         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 856
 857         # checking for diffs
 858         diff_output = self.repository.diff_with_tag(old_tag_name)
 859         if len(diff_output) == 0:
 860             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 861                 return
 862
 863         # side effect in trunk's specfile
 864         self.patch_spec_var(spec_dict)
 865
 866         # prepare changelog file
 867         # we use the standard subversion magic string (see svn_magic_line)
 868         # so we can provide useful information, such as version numbers and diff
 869         # in the same file
 870         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 871         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 872         setting_tag_line=Module.setting_tag_format%new_tag_name
 873         file(changelog,"w").write("""
 874 %s
 875 %s
 876 Please write a changelog for this new tag in the section above
 877 """%(Module.svn_magic_line,setting_tag_line))
 878
 879         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 880             file(changelog,"a").write('DIFF=========\n' + diff_output)
 881
 882         if self.options.debug:
 883             prompt('Proceed ?')
 884
 885         # edit it
 886         self.run("%s %s"%(self.options.editor,changelog))
 887         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 888         # so we do the job ourselves
 889         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 890         # insert changelog in spec
 891         if self.options.changelog:
 892             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 893
 894         ## update build
 895         build_path = os.path.join(self.options.workdir,
 896                                   Module.config['build'])
 897         build = Repository(build_path, self.options)
 898         if self.options.build_branch:
 899             build.to_branch(self.options.build_branch)
 900         if not build.is_clean():
 901             build.revert()
 902
 903         tagsfiles=glob(build.path+"/*-tags*.mk")
 904         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 905         default_answer = 'y'
 906         tagsfiles.sort()
 907         while True:
 908             for tagsfile in tagsfiles:
 909                 status=tagsdict[tagsfile]
 910                 basename=os.path.basename(tagsfile)
 911                 print ".................... Dealing with %s"%basename
 912                 while tagsdict[tagsfile] == 'todo' :
 913                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 914                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 915                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 916                                      allow_outside=True)
 917                     if choice == 'y':
 918                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 919                     elif choice == 'n':
 920                         print 'Done with %s'%os.path.basename(tagsfile)
 921                         tagsdict[tagsfile]='done'
 922                     elif choice == 'f':
 923                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 924                     elif choice == 'd':
 925                         self.run("svn diff %s"%tagsfile)
 926                     elif choice == 'r':
 927                         self.run("svn revert %s"%tagsfile)
 928                     elif choice == 'c':
 929                         self.run("cat %s"%tagsfile)
 930                     else:
 931                         name=self.name
 932                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 933 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 934 d: show current diff for this tag file
 935 r: revert that tag file
 936 c: cat the current tag file
 937 n: move to next file"""%locals()
 938
 939             if prompt("Want to review changes on tags files",False):
 940                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 941                 default_answer='d'
 942             else:
 943                 break
 944
 945         def diff_all_changes():
 946             print build.diff()
 947             print self.repository.diff()
 948
 949         def commit_all_changes(log):
 950             self.repository.commit(log)
 951             build.commit(log)
 952
 953         self.run_prompt("Review module and build", diff_all_changes)
 954         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 955         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 956
 957         if self.options.debug:
 958             print 'Preserving',changelog,'and stripped',changelog_svn
 959         else:
 960             os.unlink(changelog)
 961             os.unlink(changelog_svn)
 962
 963
 964 ##############################
 965 class Main:
 966
 967     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 968
 969 module-tools : a set of tools to manage subversion tags and specfile
 970   requires the specfile to either
 971   * define *version* and *taglevel*
 972   OR alternatively
 973   * define redirection variables module_version_varname / module_taglevel_varname
 974 Trunk:
 975   by default, the trunk of modules is taken into account
 976   in this case, just mention the module name as <module_desc>
 977 Branches:
 978   if you wish to work on a branch rather than on the trunk,
 979   you can use something like e.g. Mom:2.1 as <module_desc>
 980 """
 981     release_usage="""Usage: %prog [options] tag1 .. tagn
 982   Extract release notes from the changes in specfiles between several build tags, latest first
 983   Examples:
 984       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 985   You can refer to a (build) branch by prepending a colon, like in
 986       release-changelog :4.2 4.2-rc25
 987   You can refer to the build trunk by just mentioning 'trunk', e.g.
 988       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 989 """
 990     common_usage="""More help:
 991   see http://svn.planet-lab.org/wiki/ModuleTools"""
 992
 993     modes={
 994         'list' : "displays a list of available tags or branches",
 995         'version' : "check latest specfile and print out details",
 996         'diff' : "show difference between module (trunk or branch) and latest tag",
 997         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 998                 create new tag and and monitor its adoption in build/*-tags*.mk""",
 999         'branch' : """create a branch for this module, from the latest tag on the trunk,
1000                   and change trunk's version number to reflect the new branch name;
1001                   you can specify the new branch name by using module:branch""",
1002         'sync' : """create a tag from the module
1003                 this is a last resort option, mostly for repairs""",
1004         'changelog' : """extract changelog between build tags
1005                 expected arguments are a list of tags""",
1006         }
1007
1008     silent_modes = ['list']
1009     release_modes = ['changelog']
1010
1011     @staticmethod
1012     def optparse_list (option, opt, value, parser):
1013         try:
1014             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
1015         except:
1016             setattr(parser.values,option.dest,value.split())
1017
1018     def run(self):
1019
1020         mode=None
1021         for function in Main.modes.keys():
1022             if sys.argv[0].find(function) >= 0:
1023                 mode = function
1024                 break
1025         if not mode:
1026             print "Unsupported command",sys.argv[0]
1027             print "Supported commands:" + " ".join(Main.modes.keys())
1028             sys.exit(1)
1029
1030         if mode not in Main.release_modes:
1031             usage = Main.module_usage
1032             usage += Main.common_usage
1033             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1034         else:
1035             usage = Main.release_usage
1036             usage += Main.common_usage
1037
1038         parser=OptionParser(usage=usage)
1039
1040         if mode == "tag" or mode == 'branch':
1041             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1042                               help="set new version and reset taglevel to 0")
1043         if mode == "tag" :
1044             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1045                               help="do not update changelog section in specfile when tagging")
1046             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1047                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1048         if mode == "tag" or mode == "sync" :
1049             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1050                               help="specify editor")
1051
1052         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1053         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1054                           help="dry run - shell commands are only displayed")
1055         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1056                           default=[], nargs=1,type="string",
1057                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1058 -- can be set multiple times, or use quotes""")
1059
1060         parser.add_option("-w","--workdir", action="store", dest="workdir",
1061                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1062                           help="""name for dedicated working dir - defaults to ~/modules
1063 ** THIS MUST NOT ** be your usual working directory""")
1064         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1065                           help="skip safety checks, such as svn updates -- use with care")
1066
1067         # default verbosity depending on function - temp
1068         verbose_modes= ['tag', 'sync', 'branch']
1069
1070         if mode not in verbose_modes:
1071             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1072                               help="run in verbose mode")
1073         else:
1074             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1075                               help="run in quiet (non-verbose) mode")
1076         (options, args) = parser.parse_args()
1077         options.mode=mode
1078         if not hasattr(options,'dry_run'):
1079             options.dry_run=False
1080         if not hasattr(options,'www'):
1081             options.www=False
1082         options.debug=False
1083
1084         ########## module-*
1085         if len(args) == 0:
1086             parser.print_help()
1087             sys.exit(1)
1088         Module.init_homedir(options)
1089
1090         modules=[ Module(modname,options) for modname in args ]
1091         for module in modules:
1092             if len(args)>1 and mode not in Main.silent_modes:
1093                 print '========================================',module.friendly_name()
1094             # call the method called do_<mode>
1095             method=Module.__dict__["do_%s"%mode]
1096             try:
1097                 method(module)
1098             except Exception,e:
1099                 import traceback
1100                 traceback.print_exc()
1101                 print 'Skipping module %s: '%modname,e
1102
1103 ####################
1104 if __name__ == "__main__" :
1105     try:
1106         Main().run()
1107     except KeyboardInterrupt:
1108         print '\nBye'