module-tools.py

   1 #!/usr/bin/python -u
   2
   3 import sys, os
   4 import re
   5 import time
   6 from glob import glob
   7 from optparse import OptionParser
   8
   9 # HARDCODED NAME CHANGES
  10 #
  11 # Moving to git we decided to rename some of the repositories. Here is
  12 # a map of name changes applied in git repositories.
  13 RENAMED_SVN_MODULES = {
  14     "PLEWWW": "plewww"
  15     }
  16
  17 def svn_to_git_name(module):
  18     if RENAMED_SVN_MODULES.has_key(module):
  19         return RENAMED_SVN_MODULES[module]
  20     return module
  21
  22 def git_to_svn_name(module):
  23     for key in RENAMED_SVN_MODULES:
  24         if module == RENAMED_SVN_MODULES[key]:
  25             return key
  26     return module
  27
  28
  29 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  30 def prompt (question,default=True,other_choices=[],allow_outside=False):
  31     if not isinstance (other_choices,list):
  32         other_choices = [ other_choices ]
  33     chars = [ c for (c,rest) in other_choices ]
  34
  35     choices = []
  36     if 'y' not in chars:
  37         if default is True: choices.append('[y]')
  38         else : choices.append('y')
  39     if 'n' not in chars:
  40         if default is False: choices.append('[n]')
  41         else : choices.append('n')
  42
  43     for (char,choice) in other_choices:
  44         if default == char:
  45             choices.append("["+char+"]"+choice)
  46         else:
  47             choices.append("<"+char+">"+choice)
  48     try:
  49         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  50         if not answer:
  51             return default
  52         answer=answer[0].lower()
  53         if answer == 'y':
  54             if 'y' in chars: return 'y'
  55             else: return True
  56         elif answer == 'n':
  57             if 'n' in chars: return 'n'
  58             else: return False
  59         elif other_choices:
  60             for (char,choice) in other_choices:
  61                 if answer == char:
  62                     return char
  63             if allow_outside:
  64                 return answer
  65         return prompt(question,default,other_choices)
  66     except:
  67         raise
  68
  69 def default_editor():
  70     try:
  71         editor = os.environ['EDITOR']
  72     except:
  73         editor = "emacs"
  74     return editor
  75
  76 ### fold long lines
  77 fold_length=132
  78
  79 def print_fold (line):
  80     while len(line) >= fold_length:
  81         print line[:fold_length],'\\'
  82         line=line[fold_length:]
  83     print line
  84
  85 class Command:
  86     def __init__ (self,command,options):
  87         self.command=command
  88         self.options=options
  89         self.tmp="/tmp/command-%d"%os.getpid()
  90
  91     def run (self):
  92         if self.options.dry_run:
  93             print 'dry_run',self.command
  94             return 0
  95         if self.options.verbose and self.options.mode not in Main.silent_modes:
  96             print '+',self.command
  97             sys.stdout.flush()
  98         return os.system(self.command)
  99
 100     def run_silent (self):
 101         if self.options.dry_run:
 102             print 'dry_run',self.command
 103             return 0
 104         if self.options.verbose:
 105             print '+',self.command,' .. ',
 106             sys.stdout.flush()
 107         retcod=os.system(self.command + " &> " + self.tmp)
 108         if retcod != 0:
 109             print "FAILED ! -- out+err below (command was %s)"%self.command
 110             os.system("cat " + self.tmp)
 111             print "FAILED ! -- end of quoted output"
 112         elif self.options.verbose:
 113             print "OK"
 114         os.unlink(self.tmp)
 115         return retcod
 116
 117     def run_fatal(self):
 118         if self.run_silent() !=0:
 119             raise Exception,"Command %s failed"%self.command
 120
 121     # returns stdout, like bash's $(mycommand)
 122     def output_of (self,with_stderr=False):
 123         if self.options.dry_run:
 124             print 'dry_run',self.command
 125             return 'dry_run output'
 126         tmp="/tmp/status-%d"%os.getpid()
 127         if self.options.debug:
 128             print '+',self.command,' .. ',
 129             sys.stdout.flush()
 130         command=self.command
 131         if with_stderr:
 132             command += " &> "
 133         else:
 134             command += " > "
 135         command += tmp
 136         os.system(command)
 137         result=file(tmp).read()
 138         os.unlink(tmp)
 139         if self.options.debug:
 140             print 'Done',
 141         return result
 142
 143
 144 class SvnRepository:
 145     type = "svn"
 146
 147     def __init__(self, path, options):
 148         self.path = path
 149         self.options = options
 150
 151     def name(self):
 152         return os.path.basename(self.path)
 153
 154     def url(self):
 155         out = Command("svn info %s" % self.path, self.options).output_of()
 156         for line in out.split('\n'):
 157             if line.startswith("URL:"):
 158                 return line.split()[1].strip()
 159
 160     def repo_root(self):
 161         out = Command("svn info %s" % self.path, self.options).output_of()
 162         for line in out.split('\n'):
 163             if line.startswith("Repository Root:"):
 164                 root = line.split()[2].strip()
 165                 return "%s/%s" % (root, self.name())
 166
 167     @classmethod
 168     def checkout(cls, remote, local, options, recursive=False):
 169         if recursive:
 170             svncommand = "svn co %s %s" % (remote, local)
 171         else:
 172             svncommand = "svn co -N %s %s" % (remote, local)
 173         Command("rm -rf %s" % local, options).run_silent()
 174         Command(svncommand, options).run_fatal()
 175
 176         return SvnRepository(local, options)
 177
 178     @classmethod
 179     def remote_exists(cls, remote):
 180         return os.system("svn list %s &> /dev/null" % remote) == 0
 181
 182     def tag_exists(self, tagname):
 183         url = "%s/tags/%s" % (self.repo_root(), tagname)
 184         return SvnRepository.remote_exists(url)
 185
 186     def update(self, subdir="", recursive=True):
 187         path = os.path.join(self.path, subdir)
 188         if recursive:
 189             svncommand = "svn up %s" % path
 190         else:
 191             svncommand = "svn up -N %s" % path
 192         Command(svncommand, self.options).run_fatal()
 193
 194     def commit(self, logfile):
 195         # add all new files to the repository
 196         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 197                 self.path, self.options).run_silent()
 198         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 199
 200     def to_branch(self, branch):
 201         remote = "%s/branches/%s" % (self.repo_root(), branch)
 202         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 203
 204     def to_tag(self, tag):
 205         remote = "%s/tags/%s" % (self.repo_root(), branch)
 206         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 207
 208     def tag(self, tagname, logfile):
 209         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 210         self_url = self.url()
 211         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 212
 213     def diff(self):
 214         return Command("svn diff %s" % self.path, self.options).output_of(True)
 215
 216     def diff_with_tag(self, tagname):
 217         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 218         return Command("svn diff %s %s" % (tag_url, self.url()),
 219                        self.options).output_of(True)
 220
 221     def revert(self):
 222         Command("svn revert %s -R" % self.path, self.options).run_fatal()
 223         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 224                 self.path, self.options).run_silent()
 225
 226     def is_clean(self):
 227         command="svn status %s" % self.path
 228         return len(Command(command,self.options).output_of(True)) == 0
 229
 230     def is_valid(self):
 231         return os.path.exists(os.path.join(self.path, ".svn"))
 232
 233
 234 class GitRepository:
 235     type = "git"
 236
 237     def __init__(self, path, options):
 238         self.path = path
 239         self.options = options
 240
 241     def name(self):
 242         return os.path.basename(self.path)
 243
 244     def url(self):
 245         self.repo_root()
 246
 247     def repo_root(self):
 248         c = Command("git remote show origin", self.options)
 249         out = self.__run_in_repo(c.output_of)
 250         for line in out.split('\n'):
 251             if line.strip().startswith("Fetch URL:"):
 252                 repo = line.split()[2]
 253
 254     @classmethod
 255     def checkout(cls, remote, local, options, depth=1):
 256         Command("rm -rf %s" % local, options).run_silent()
 257         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 258         return GitRepository(local, options)
 259
 260     @classmethod
 261     def remote_exists(cls, remote):
 262         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 263
 264     def tag_exists(self, tagname):
 265         command = 'git tag -l | grep "^%s$"' % tagname
 266         c = Command(command, self.options)
 267         out = self.__run_in_repo(c.output_of, with_stderr=True)
 268         return len(out) > 0
 269
 270     def __run_in_repo(self, fun, *args, **kwargs):
 271         cwd = os.getcwd()
 272         os.chdir(self.path)
 273         ret = fun(*args, **kwargs)
 274         os.chdir(cwd)
 275         return ret
 276
 277     def __run_command_in_repo(self, command, ignore_errors=False):
 278         c = Command(command, self.options)
 279         if ignore_errors:
 280             return self.__run_in_repo(c.output_of)
 281         else:
 282             return self.__run_in_repo(c.run_fatal)
 283
 284     def update(self, subdir=None, recursive=None):
 285         return self.__run_command_in_repo("git pull")
 286
 287     def to_branch(self, branch, remote=True):
 288         if remote:
 289             branch = "origin/%s" % branch
 290         return self.__run_command_in_repo("git checkout %s" % branch)
 291
 292     def to_tag(self, tag):
 293         return self.__run_command_in_repo("git checkout %s" % tag)
 294
 295     def tag(self, tagname, logfile):
 296         self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 297         self.commit(logfile)
 298
 299     def diff(self):
 300         c = Command("git diff", self.options)
 301         return self.__run_in_repo(c.output_of, with_stderr=True)
 302
 303     def diff_with_tag(self, tagname):
 304         c = Command("git diff %s" % tagname, self.options)
 305         return self.__run_in_repo(c.output_of, with_stderr=True)
 306
 307     def commit(self, logfile):
 308         self.__run_command_in_repo("git add -A", ignore_errors=True)
 309         self.__run_command_in_repo("git commit -F  %s" % logfile, ignore_errors=True)
 310         self.__run_command_in_repo("git push")
 311         self.__run_command_in_repo("git push --tags")
 312
 313     def revert(self):
 314         self.__run_command_in_repo("git --no-pager reset --hard")
 315         self.__run_command_in_repo("git --no-pager clean -f")
 316
 317     def is_clean(self):
 318         def check_commit():
 319             command="git status"
 320             s="nothing to commit (working directory clean)"
 321             return Command(command, self.options).output_of(True).find(s) >= 0
 322         return self.__run_in_repo(check_commit)
 323
 324     def is_valid(self):
 325         return os.path.exists(os.path.join(self.path, ".git"))
 326
 327
 328 class Repository:
 329     """ Generic repository """
 330     supported_repo_types = [SvnRepository, GitRepository]
 331
 332     def __init__(self, path, options):
 333         self.path = path
 334         self.options = options
 335         for repo in self.supported_repo_types:
 336             self.repo = repo(self.path, self.options)
 337             if self.repo.is_valid():
 338                 break
 339
 340     @classmethod
 341     def has_moved_to_git(cls, module, svnpath):
 342         module = git_to_svn_name(module)
 343         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 344
 345     @classmethod
 346     def remote_exists(cls, remote):
 347         for repo in Repository.supported_repo_types:
 348             if repo.remote_exists(remote):
 349                 return True
 350         return False
 351
 352     def __getattr__(self, attr):
 353         return getattr(self.repo, attr)
 354
 355
 356
 357 # support for tagged module is minimal, and is for the Build class only
 358 class Module:
 359
 360     svn_magic_line="--This line, and those below, will be ignored--"
 361     setting_tag_format = "Setting tag %s"
 362
 363     redirectors=[ # ('module_name_varname','name'),
 364                   ('module_version_varname','version'),
 365                   ('module_taglevel_varname','taglevel'), ]
 366
 367     # where to store user's config
 368     config_storage="CONFIG"
 369     #
 370     config={}
 371
 372     import commands
 373     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 374                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 375                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 376                  ('gituser', "Enter your user name (login name) on git server", os.getlogin()),
 377                  ("build", "Enter the name of your build module","build"),
 378                  ('username',"Enter your firstname and lastname for changelogs",""),
 379                  ("email","Enter your email address for changelogs",""),
 380                  ]
 381
 382     @classmethod
 383     def prompt_config_option(cls, key, message, default):
 384         cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 385
 386     @classmethod
 387     def prompt_config (cls):
 388         for (key,message,default) in cls.configKeys:
 389             cls.config[key]=""
 390             while not cls.config[key]:
 391                 cls.prompt_config_option(key, message, default)
 392
 393
 394     # for parsing module spec name:branch
 395     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 396     # special form for tagged module - for Build
 397     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 398     # parsing specfiles
 399     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 400
 401     def __init__ (self,module_spec,options):
 402         # parse module spec
 403         attempt=Module.matcher_branch_spec.match(module_spec)
 404         if attempt:
 405             self.name=attempt.group('name')
 406             self.branch=attempt.group('branch')
 407         else:
 408             attempt=Module.matcher_tag_spec.match(module_spec)
 409             if attempt:
 410                 self.name=attempt.group('name')
 411                 self.tagname=attempt.group('tagname')
 412             else:
 413                 self.name=module_spec
 414
 415         # when available prefer to use git module name internally
 416         self.name = svn_to_git_name(self.name)
 417
 418         self.options=options
 419         self.module_dir="%s/%s"%(options.workdir,self.name)
 420         self.repository = None
 421         self.build = None
 422
 423     def run (self,command):
 424         return Command(command,self.options).run()
 425     def run_fatal (self,command):
 426         return Command(command,self.options).run_fatal()
 427     def run_prompt (self,message,fun, *args):
 428         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 429         if not self.options.verbose:
 430             while True:
 431                 choice=prompt(message,True,('s','how'))
 432                 if choice is True:
 433                     fun(*args)
 434                     return
 435                 elif choice is False:
 436                     print 'About to run function:', fun_msg
 437         else:
 438             question=message+" - want to run function: " + fun_msg
 439             if prompt(question,True):
 440                 fun(*args)
 441
 442     def friendly_name (self):
 443         if hasattr(self,'branch'):
 444             return "%s:%s"%(self.name,self.branch)
 445         elif hasattr(self,'tagname'):
 446             return "%s@%s"%(self.name,self.tagname)
 447         else:
 448             return self.name
 449
 450     @classmethod
 451     def git_remote_dir (cls, name):
 452         return "%s@%s:/git/%s.git" % (cls.config['gituser'], cls.config['gitserver'], name)
 453
 454     @classmethod
 455     def svn_remote_dir (cls, name):
 456         name = git_to_svn_name(name)
 457         svn = cls.config['svnpath']
 458         if svn.endswith('/'):
 459             return "%s%s" % (svn, name)
 460         return "%s/%s" % (svn, name)
 461
 462     def svn_selected_remote(self):
 463         svn_name = git_to_svn_name(self.name)
 464         remote = self.svn_remote_dir(svn_name)
 465         if hasattr(self,'branch'):
 466             remote = "%s/branches/%s" % (remote, self.branch)
 467         elif hasattr(self,'tagname'):
 468             remote = "%s/tags/%s" % (remote, self.tagname)
 469         else:
 470             remote = "%s/trunk" % remote
 471         return remote
 472
 473     ####################
 474     @classmethod
 475     def init_homedir (cls, options):
 476         if options.verbose and options.mode not in Main.silent_modes:
 477             print 'Checking for', options.workdir
 478         storage="%s/%s"%(options.workdir, cls.config_storage)
 479         # sanity check. Either the topdir exists AND we have a config/storage
 480         # or topdir does not exist and we create it
 481         # to avoid people use their own daily svn repo
 482         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 483             print """The directory %s exists and has no CONFIG file
 484 If this is your regular working directory, please provide another one as the
 485 module-* commands need a fresh working dir. Make sure that you do not use
 486 that for other purposes than tagging""" % options.workdir
 487             sys.exit(1)
 488
 489         def checkout_build():
 490             print "Checking out build module..."
 491             remote = cls.git_remote_dir(cls.config['build'])
 492             local = os.path.join(options.workdir, cls.config['build'])
 493             GitRepository.checkout(remote, local, options, depth=1)
 494             print "OK"
 495
 496         def store_config():
 497             f=file(storage,"w")
 498             for (key,message,default) in Module.configKeys:
 499                 f.write("%s=%s\n"%(key,Module.config[key]))
 500             f.close()
 501             if options.debug:
 502                 print 'Stored',storage
 503                 Command("cat %s"%storage,options).run()
 504
 505         def read_config():
 506             # read config
 507             f=open(storage)
 508             for line in f.readlines():
 509                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 510                 Module.config[key]=value
 511             f.close()
 512
 513         if not os.path.isdir (options.workdir):
 514             print "Cannot find",options.workdir,"let's create it"
 515             Command("mkdir -p %s" % options.workdir, options).run_silent()
 516             cls.prompt_config()
 517             checkout_build()
 518             store_config()
 519         else:
 520             read_config()
 521             # check missing config options
 522             old_layout = False
 523             for (key,message,default) in cls.configKeys:
 524                 if not Module.config.has_key(key):
 525                     print "Configuration changed for module-tools"
 526                     cls.prompt_config_option(key, message, default)
 527                     old_layout = True
 528
 529             if old_layout:
 530                 Command("rm -rf %s" % options.workdir, options).run_silent()
 531                 Command("mkdir -p %s" % options.workdir, options).run_silent()
 532                 checkout_build()
 533                 store_config()
 534
 535             build_dir = os.path.join(options.workdir, cls.config['build'])
 536             build = Repository(build_dir, options)
 537             if not build.is_clean():
 538                 print "build module needs a revert"
 539                 build.revert()
 540                 print "OK"
 541             build.update()
 542
 543         if options.verbose and options.mode not in Main.silent_modes:
 544             print '******** Using config'
 545             for (key,message,default) in Module.configKeys:
 546                 print '\t',key,'=',Module.config[key]
 547
 548     def init_module_dir (self):
 549         if self.options.verbose:
 550             print 'Checking for',self.module_dir
 551
 552         if not os.path.isdir (self.module_dir):
 553             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 554                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 555                                                          self.module_dir,
 556                                                          self.options)
 557             else:
 558                 remote = self.svn_selected_remote()
 559                 self.repository = SvnRepository.checkout(remote,
 560                                                          self.module_dir,
 561                                                          self.options, recursive=False)
 562
 563         self.repository = Repository(self.module_dir, self.options)
 564         if self.repository.type == "svn":
 565             # check if module has moved to git
 566             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 567                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 568                 self.init_module_dir()
 569             # check if we have the required branch/tag
 570             if self.repository.url() != self.svn_selected_remote():
 571                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 572                 self.init_module_dir()
 573
 574         elif self.repository.type == "git":
 575             if hasattr(self,'branch'):
 576                 self.repository.to_branch(self.branch)
 577             elif hasattr(self,'tagname'):
 578                 self.repository.to_tag(self.tagname)
 579
 580         else:
 581             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 582
 583
 584     def revert_module_dir (self):
 585         if self.options.fast_checks:
 586             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 587             return
 588         if self.options.verbose:
 589             print 'Checking whether', self.module_dir, 'needs being reverted'
 590
 591         if not self.repository.is_clean():
 592             self.repository.revert()
 593
 594     def update_module_dir (self):
 595         if self.options.fast_checks:
 596             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 597             return
 598         if self.options.verbose:
 599             print 'Updating', self.module_dir
 600         self.repository.update()
 601
 602     def main_specname (self):
 603         attempt="%s/%s.spec"%(self.module_dir,self.name)
 604         if os.path.isfile (attempt):
 605             return attempt
 606         pattern1="%s/*.spec"%self.module_dir
 607         level1=glob(pattern1)
 608         if level1:
 609             return level1[0]
 610         pattern2="%s/*/*.spec"%self.module_dir
 611         level2=glob(pattern2)
 612
 613         if level2:
 614             return level2[0]
 615         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 616
 617     def all_specnames (self):
 618         level1=glob("%s/*.spec" % self.module_dir)
 619         if level1: return level1
 620         level2=glob("%s/*/*.spec" % self.module_dir)
 621         return level2
 622
 623     def parse_spec (self, specfile, varnames):
 624         if self.options.verbose:
 625             print 'Parsing',specfile,
 626             for var in varnames:
 627                 print "[%s]"%var,
 628             print ""
 629         result={}
 630         f=open(specfile)
 631         for line in f.readlines():
 632             attempt=Module.matcher_rpm_define.match(line)
 633             if attempt:
 634                 (define,var,value)=attempt.groups()
 635                 if var in varnames:
 636                     result[var]=value
 637         f.close()
 638         if self.options.debug:
 639             print 'found',len(result),'keys'
 640             for (k,v) in result.iteritems():
 641                 print k,'=',v
 642         return result
 643
 644     # stores in self.module_name_varname the rpm variable to be used for the module's name
 645     # and the list of these names in self.varnames
 646     def spec_dict (self):
 647         specfile=self.main_specname()
 648         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 649         redirect_dict = self.parse_spec(specfile,redirector_keys)
 650         if self.options.debug:
 651             print '1st pass parsing done, redirect_dict=',redirect_dict
 652         varnames=[]
 653         for (varname,default) in Module.redirectors:
 654             if redirect_dict.has_key(varname):
 655                 setattr(self,varname,redirect_dict[varname])
 656                 varnames += [redirect_dict[varname]]
 657             else:
 658                 setattr(self,varname,default)
 659                 varnames += [ default ]
 660         self.varnames = varnames
 661         result = self.parse_spec (specfile,self.varnames)
 662         if self.options.debug:
 663             print '2st pass parsing done, varnames=',varnames,'result=',result
 664         return result
 665
 666     def patch_spec_var (self, patch_dict,define_missing=False):
 667         for specfile in self.all_specnames():
 668             # record the keys that were changed
 669             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 670             newspecfile=specfile+".new"
 671             if self.options.verbose:
 672                 print 'Patching',specfile,'for',patch_dict.keys()
 673             spec=open (specfile)
 674             new=open(newspecfile,"w")
 675
 676             for line in spec.readlines():
 677                 attempt=Module.matcher_rpm_define.match(line)
 678                 if attempt:
 679                     (define,var,value)=attempt.groups()
 680                     if var in patch_dict.keys():
 681                         if self.options.debug:
 682                             print 'rewriting %s as %s'%(var,patch_dict[var])
 683                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 684                         changed[var]=True
 685                         continue
 686                 new.write(line)
 687             if define_missing:
 688                 for (key,was_changed) in changed.iteritems():
 689                     if not was_changed:
 690                         if self.options.debug:
 691                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 692                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 693             spec.close()
 694             new.close()
 695             os.rename(newspecfile,specfile)
 696
 697     # returns all lines until the magic line
 698     def unignored_lines (self, logfile):
 699         result=[]
 700         white_line_matcher = re.compile("\A\s*\Z")
 701         for logline in file(logfile).readlines():
 702             if logline.strip() == Module.svn_magic_line:
 703                 break
 704             elif white_line_matcher.match(logline):
 705                 continue
 706             else:
 707                 result.append(logline.strip()+'\n')
 708         return result
 709
 710     # creates a copy of the input with only the unignored lines
 711     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 712        f=file(fileout,'w')
 713        f.write(self.setting_tag_format%new_tag_name + '\n')
 714        for line in self.unignored_lines(filein):
 715            f.write(line)
 716        f.close()
 717
 718     def insert_changelog (self, logfile, oldtag, newtag):
 719         for specfile in self.all_specnames():
 720             newspecfile=specfile+".new"
 721             if self.options.verbose:
 722                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 723             spec=open (specfile)
 724             new=open(newspecfile,"w")
 725             for line in spec.readlines():
 726                 new.write(line)
 727                 if re.compile('%changelog').match(line):
 728                     dateformat="* %a %b %d %Y"
 729                     datepart=time.strftime(dateformat)
 730                     logpart="%s <%s> - %s"%(Module.config['username'],
 731                                                  Module.config['email'],
 732                                                  newtag)
 733                     new.write(datepart+" "+logpart+"\n")
 734                     for logline in self.unignored_lines(logfile):
 735                         new.write("- " + logline)
 736                     new.write("\n")
 737             spec.close()
 738             new.close()
 739             os.rename(newspecfile,specfile)
 740
 741     def show_dict (self, spec_dict):
 742         if self.options.verbose:
 743             for (k,v) in spec_dict.iteritems():
 744                 print k,'=',v
 745
 746     def last_tag (self, spec_dict):
 747         try:
 748             return "%s-%s" % (spec_dict[self.module_version_varname],
 749                               spec_dict[self.module_taglevel_varname])
 750         except KeyError,err:
 751             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 752
 753     def tag_name (self, spec_dict, old_svn_name=False):
 754         base_tag_name = self.name
 755         if old_svn_name:
 756             base_tag_name = git_to_svn_name(self.name)
 757         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 758
 759
 760 ##############################
 761     # using fine_grain means replacing only those instances that currently refer to this tag
 762     # otherwise, <module>-SVNPATH is replaced unconditionnally
 763     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 764         newtagsfile=tagsfile+".new"
 765         tags=open (tagsfile)
 766         new=open(newtagsfile,"w")
 767
 768         matches=0
 769         # fine-grain : replace those lines that refer to oldname
 770         if fine_grain:
 771             if self.options.verbose:
 772                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 773             matcher=re.compile("^(.*)%s(.*)"%oldname)
 774             for line in tags.readlines():
 775                 if not matcher.match(line):
 776                     new.write(line)
 777                 else:
 778                     (begin,end)=matcher.match(line).groups()
 779                     new.write(begin+newname+end+"\n")
 780                     matches += 1
 781         # brute-force : change uncommented lines that define <module>-SVNPATH
 782         else:
 783             if self.options.verbose:
 784                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 785             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 786                                           %(self.name)
 787             matcher_module=re.compile(pattern)
 788             for line in tags.readlines():
 789                 attempt=matcher_module.match(line)
 790                 if attempt:
 791                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 792                     if self.options.verbose:
 793                         print ' '+svnpath,
 794                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 795                     new.write(replacement)
 796                     matches += 1
 797                 else:
 798                     new.write(line)
 799         tags.close()
 800         new.close()
 801         os.rename(newtagsfile,tagsfile)
 802         if self.options.verbose: print "%d changes"%matches
 803         return matches
 804
 805     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 806         if self.options.verbose:
 807             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 808
 809         found_tagname = tagname
 810         found = self.repository.tag_exists(tagname)
 811         if not found and old_svn_tag_name:
 812             if self.options.verbose:
 813                 print "KO"
 814                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 815             found = self.repository.tag_exists(old_svn_tag_name)
 816             if found:
 817                 found_tagname = old_svn_tag_name
 818
 819         if (found and need_it) or (not found and not need_it):
 820             print "OK "
 821         else:
 822             print "KO"
 823             if found:
 824                 raise Exception, "tag (%s) is already there" % tagname
 825             else:
 826                 raise Exception, "can not find required tag (%s)" % tagname
 827
 828         return found_tagname
 829
 830
 831     def do_tag (self):
 832         self.init_module_dir()
 833         self.revert_module_dir()
 834         self.update_module_dir()
 835         # parse specfile
 836         spec_dict = self.spec_dict()
 837         self.show_dict(spec_dict)
 838
 839         # side effects
 840         old_tag_name = self.tag_name(spec_dict)
 841         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 842
 843         if (self.options.new_version):
 844             # new version set on command line
 845             spec_dict[self.module_version_varname] = self.options.new_version
 846             spec_dict[self.module_taglevel_varname] = 0
 847         else:
 848             # increment taglevel
 849             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 850             spec_dict[self.module_taglevel_varname] = new_taglevel
 851
 852         new_tag_name = self.tag_name(spec_dict)
 853
 854         # sanity check
 855         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 856         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 857
 858         # checking for diffs
 859         diff_output = self.repository.diff_with_tag(old_tag_name)
 860         if len(diff_output) == 0:
 861             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 862                 return
 863
 864         # side effect in trunk's specfile
 865         self.patch_spec_var(spec_dict)
 866
 867         # prepare changelog file
 868         # we use the standard subversion magic string (see svn_magic_line)
 869         # so we can provide useful information, such as version numbers and diff
 870         # in the same file
 871         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 872         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 873         setting_tag_line=Module.setting_tag_format%new_tag_name
 874         file(changelog,"w").write("""
 875 %s
 876 %s
 877 Please write a changelog for this new tag in the section above
 878 """%(Module.svn_magic_line,setting_tag_line))
 879
 880         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 881             file(changelog,"a").write('DIFF=========\n' + diff_output)
 882
 883         if self.options.debug:
 884             prompt('Proceed ?')
 885
 886         # edit it
 887         self.run("%s %s"%(self.options.editor,changelog))
 888         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 889         # so we do the job ourselves
 890         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 891         # insert changelog in spec
 892         if self.options.changelog:
 893             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 894
 895         ## update build
 896         build_path = os.path.join(self.options.workdir,
 897                                   Module.config['build'])
 898         build = Repository(build_path, self.options)
 899         if self.options.build_branch:
 900             build.to_branch(self.options.build_branch)
 901         if not build.is_clean():
 902             build.revert()
 903
 904         tagsfiles=glob(build.path+"/*-tags*.mk")
 905         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 906         default_answer = 'y'
 907         tagsfiles.sort()
 908         while True:
 909             for tagsfile in tagsfiles:
 910                 status=tagsdict[tagsfile]
 911                 basename=os.path.basename(tagsfile)
 912                 print ".................... Dealing with %s"%basename
 913                 while tagsdict[tagsfile] == 'todo' :
 914                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 915                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 916                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 917                                      allow_outside=True)
 918                     if choice == 'y':
 919                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 920                     elif choice == 'n':
 921                         print 'Done with %s'%os.path.basename(tagsfile)
 922                         tagsdict[tagsfile]='done'
 923                     elif choice == 'f':
 924                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 925                     elif choice == 'd':
 926                         self.run("svn diff %s"%tagsfile)
 927                     elif choice == 'r':
 928                         self.run("svn revert %s"%tagsfile)
 929                     elif choice == 'c':
 930                         self.run("cat %s"%tagsfile)
 931                     else:
 932                         name=self.name
 933                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 934 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 935 d: show current diff for this tag file
 936 r: revert that tag file
 937 c: cat the current tag file
 938 n: move to next file"""%locals()
 939
 940             if prompt("Want to review changes on tags files",False):
 941                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 942                 default_answer='d'
 943             else:
 944                 break
 945
 946         def diff_all_changes():
 947             print build.diff()
 948             print self.repository.diff()
 949
 950         def commit_all_changes(log):
 951             self.repository.commit(log)
 952             build.commit(log)
 953
 954         self.run_prompt("Review module and build", diff_all_changes)
 955         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 956         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 957
 958         if self.options.debug:
 959             print 'Preserving',changelog,'and stripped',changelog_svn
 960         else:
 961             os.unlink(changelog)
 962             os.unlink(changelog_svn)
 963
 964
 965 ##############################
 966 class Main:
 967
 968     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 969
 970 module-tools : a set of tools to manage subversion tags and specfile
 971   requires the specfile to either
 972   * define *version* and *taglevel*
 973   OR alternatively
 974   * define redirection variables module_version_varname / module_taglevel_varname
 975 Trunk:
 976   by default, the trunk of modules is taken into account
 977   in this case, just mention the module name as <module_desc>
 978 Branches:
 979   if you wish to work on a branch rather than on the trunk,
 980   you can use something like e.g. Mom:2.1 as <module_desc>
 981 """
 982     release_usage="""Usage: %prog [options] tag1 .. tagn
 983   Extract release notes from the changes in specfiles between several build tags, latest first
 984   Examples:
 985       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
 986   You can refer to a (build) branch by prepending a colon, like in
 987       release-changelog :4.2 4.2-rc25
 988   You can refer to the build trunk by just mentioning 'trunk', e.g.
 989       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
 990 """
 991     common_usage="""More help:
 992   see http://svn.planet-lab.org/wiki/ModuleTools"""
 993
 994     modes={
 995         'list' : "displays a list of available tags or branches",
 996         'version' : "check latest specfile and print out details",
 997         'diff' : "show difference between module (trunk or branch) and latest tag",
 998         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
 999                 create new tag and and monitor its adoption in build/*-tags*.mk""",
1000         'branch' : """create a branch for this module, from the latest tag on the trunk,
1001                   and change trunk's version number to reflect the new branch name;
1002                   you can specify the new branch name by using module:branch""",
1003         'sync' : """create a tag from the module
1004                 this is a last resort option, mostly for repairs""",
1005         'changelog' : """extract changelog between build tags
1006                 expected arguments are a list of tags""",
1007         }
1008
1009     silent_modes = ['list']
1010     release_modes = ['changelog']
1011
1012     @staticmethod
1013     def optparse_list (option, opt, value, parser):
1014         try:
1015             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
1016         except:
1017             setattr(parser.values,option.dest,value.split())
1018
1019     def run(self):
1020
1021         mode=None
1022         for function in Main.modes.keys():
1023             if sys.argv[0].find(function) >= 0:
1024                 mode = function
1025                 break
1026         if not mode:
1027             print "Unsupported command",sys.argv[0]
1028             print "Supported commands:" + " ".join(Main.modes.keys())
1029             sys.exit(1)
1030
1031         if mode not in Main.release_modes:
1032             usage = Main.module_usage
1033             usage += Main.common_usage
1034             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1035         else:
1036             usage = Main.release_usage
1037             usage += Main.common_usage
1038
1039         parser=OptionParser(usage=usage)
1040
1041         if mode == "tag" or mode == 'branch':
1042             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1043                               help="set new version and reset taglevel to 0")
1044         if mode == "tag" :
1045             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1046                               help="do not update changelog section in specfile when tagging")
1047             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1048                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1049         if mode == "tag" or mode == "sync" :
1050             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1051                               help="specify editor")
1052
1053         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1054         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1055                           help="dry run - shell commands are only displayed")
1056         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1057                           default=[], nargs=1,type="string",
1058                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1059 -- can be set multiple times, or use quotes""")
1060
1061         parser.add_option("-w","--workdir", action="store", dest="workdir",
1062                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1063                           help="""name for dedicated working dir - defaults to ~/modules
1064 ** THIS MUST NOT ** be your usual working directory""")
1065         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1066                           help="skip safety checks, such as svn updates -- use with care")
1067
1068         # default verbosity depending on function - temp
1069         verbose_modes= ['tag', 'sync', 'branch']
1070
1071         if mode not in verbose_modes:
1072             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1073                               help="run in verbose mode")
1074         else:
1075             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1076                               help="run in quiet (non-verbose) mode")
1077         (options, args) = parser.parse_args()
1078         options.mode=mode
1079         if not hasattr(options,'dry_run'):
1080             options.dry_run=False
1081         if not hasattr(options,'www'):
1082             options.www=False
1083         options.debug=False
1084
1085         ########## module-*
1086         if len(args) == 0:
1087             parser.print_help()
1088             sys.exit(1)
1089         Module.init_homedir(options)
1090
1091         modules=[ Module(modname,options) for modname in args ]
1092         for module in modules:
1093             if len(args)>1 and mode not in Main.silent_modes:
1094                 print '========================================',module.friendly_name()
1095             # call the method called do_<mode>
1096             method=Module.__dict__["do_%s"%mode]
1097             try:
1098                 method(module)
1099             except Exception,e:
1100                 import traceback
1101                 traceback.print_exc()
1102                 print 'Skipping module %s: '%modname,e
1103
1104 ####################
1105 if __name__ == "__main__" :
1106     try:
1107         Main().run()
1108     except KeyboardInterrupt:
1109         print '\nBye'