module-tools.py

   1 #!/usr/bin/python -u
   2
   3 import sys, os
   4 import re
   5 import time
   6 from glob import glob
   7 from optparse import OptionParser
   8
   9 # HARDCODED NAME CHANGES
  10 #
  11 # Moving to git we decided to rename some of the repositories. Here is
  12 # a map of name changes applied in git repositories.
  13 RENAMED_SVN_MODULES = {
  14     "PLEWWW": "plewww"
  15     }
  16
  17 def svn_to_git_name(module):
  18     if RENAMED_SVN_MODULES.has_key(module):
  19         return RENAMED_SVN_MODULES[module]
  20     return module
  21
  22 def git_to_svn_name(module):
  23     for key in RENAMED_SVN_MODULES:
  24         if module == RENAMED_SVN_MODULES[key]:
  25             return key
  26     return module
  27
  28
  29 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  30 def prompt (question,default=True,other_choices=[],allow_outside=False):
  31     if not isinstance (other_choices,list):
  32         other_choices = [ other_choices ]
  33     chars = [ c for (c,rest) in other_choices ]
  34
  35     choices = []
  36     if 'y' not in chars:
  37         if default is True: choices.append('[y]')
  38         else : choices.append('y')
  39     if 'n' not in chars:
  40         if default is False: choices.append('[n]')
  41         else : choices.append('n')
  42
  43     for (char,choice) in other_choices:
  44         if default == char:
  45             choices.append("["+char+"]"+choice)
  46         else:
  47             choices.append("<"+char+">"+choice)
  48     try:
  49         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  50         if not answer:
  51             return default
  52         answer=answer[0].lower()
  53         if answer == 'y':
  54             if 'y' in chars: return 'y'
  55             else: return True
  56         elif answer == 'n':
  57             if 'n' in chars: return 'n'
  58             else: return False
  59         elif other_choices:
  60             for (char,choice) in other_choices:
  61                 if answer == char:
  62                     return char
  63             if allow_outside:
  64                 return answer
  65         return prompt(question,default,other_choices)
  66     except:
  67         raise
  68
  69 def default_editor():
  70     try:
  71         editor = os.environ['EDITOR']
  72     except:
  73         editor = "emacs"
  74     return editor
  75
  76 ### fold long lines
  77 fold_length=132
  78
  79 def print_fold (line):
  80     while len(line) >= fold_length:
  81         print line[:fold_length],'\\'
  82         line=line[fold_length:]
  83     print line
  84
  85 class Command:
  86     def __init__ (self,command,options):
  87         self.command=command
  88         self.options=options
  89         self.tmp="/tmp/command-%d"%os.getpid()
  90
  91     def run (self):
  92         if self.options.dry_run:
  93             print 'dry_run',self.command
  94             return 0
  95         if self.options.verbose and self.options.mode not in Main.silent_modes:
  96             print '+',self.command
  97             sys.stdout.flush()
  98         return os.system(self.command)
  99
 100     def run_silent (self):
 101         if self.options.dry_run:
 102             print 'dry_run',self.command
 103             return 0
 104         if self.options.verbose:
 105             print '+',self.command,' .. ',
 106             sys.stdout.flush()
 107         retcod=os.system(self.command + " &> " + self.tmp)
 108         if retcod != 0:
 109             print "FAILED ! -- out+err below (command was %s)"%self.command
 110             os.system("cat " + self.tmp)
 111             print "FAILED ! -- end of quoted output"
 112         elif self.options.verbose:
 113             print "OK"
 114         os.unlink(self.tmp)
 115         return retcod
 116
 117     def run_fatal(self):
 118         if self.run_silent() !=0:
 119             raise Exception,"Command %s failed"%self.command
 120
 121     # returns stdout, like bash's $(mycommand)
 122     def output_of (self,with_stderr=False):
 123         if self.options.dry_run:
 124             print 'dry_run',self.command
 125             return 'dry_run output'
 126         tmp="/tmp/status-%d"%os.getpid()
 127         if self.options.debug:
 128             print '+',self.command,' .. ',
 129             sys.stdout.flush()
 130         command=self.command
 131         if with_stderr:
 132             command += " &> "
 133         else:
 134             command += " > "
 135         command += tmp
 136         os.system(command)
 137         result=file(tmp).read()
 138         os.unlink(tmp)
 139         if self.options.debug:
 140             print 'Done',
 141         return result
 142
 143
 144 class SvnRepository:
 145     type = "svn"
 146
 147     def __init__(self, path, options):
 148         self.path = path
 149         self.options = options
 150
 151     def name(self):
 152         return os.path.basename(self.path)
 153
 154     def url(self):
 155         out = Command("svn info %s" % self.path, self.options).output_of()
 156         for line in out.split('\n'):
 157             if line.startswith("URL:"):
 158                 return line.split()[1].strip()
 159
 160     def repo_root(self):
 161         out = Command("svn info %s" % self.path, self.options).output_of()
 162         for line in out.split('\n'):
 163             if line.startswith("Repository Root:"):
 164                 root = line.split()[2].strip()
 165                 return "%s/%s" % (root, self.name())
 166
 167     @classmethod
 168     def checkout(cls, remote, local, options, recursive=False):
 169         if recursive:
 170             svncommand = "svn co %s %s" % (remote, local)
 171         else:
 172             svncommand = "svn co -N %s %s" % (remote, local)
 173         Command("rm -rf %s" % local, options).run_silent()
 174         Command(svncommand, options).run_fatal()
 175
 176         return SvnRepository(local, options)
 177
 178     @classmethod
 179     def remote_exists(cls, remote):
 180         return os.system("svn list %s &> /dev/null" % remote) == 0
 181
 182     def tag_exists(self, tagname):
 183         url = "%s/tags/%s" % (self.repo_root(), tagname)
 184         return SvnRepository.remote_exists(url)
 185
 186     def update(self, subdir="", recursive=True):
 187         path = os.path.join(self.path, subdir)
 188         if recursive:
 189             svncommand = "svn up %s" % path
 190         else:
 191             svncommand = "svn up -N %s" % path
 192         Command(svncommand, self.options).run_fatal()
 193
 194     def commit(self, logfile):
 195         # add all new files to the repository
 196         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 197                 self.path, self.options).run_silent()
 198         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 199
 200     def to_branch(self, branch):
 201         remote = "%s/branches/%s" % (self.repo_root(), branch)
 202         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 203
 204     def to_tag(self, tag):
 205         remote = "%s/tags/%s" % (self.repo_root(), branch)
 206         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 207
 208     def tag(self, tagname, logfile):
 209         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 210         self_url = self.url()
 211         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 212
 213     def diff(self, f=""):
 214         if f:
 215             f = os.path.join(self.path, f)
 216         else:
 217             f = self.path
 218         return Command("svn diff %s" % f, self.options).output_of(True)
 219
 220     def diff_with_tag(self, tagname):
 221         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 222         return Command("svn diff %s %s" % (tag_url, self.url()),
 223                        self.options).output_of(True)
 224
 225     def revert(self, f=""):
 226         if f:
 227             Command("svn revert %s" % os.path.join(self.path, f), self.options).run_fatal()
 228         else:
 229             # revert all
 230             Command("svn revert %s -R" % self.path, self.options).run_fatal()
 231             Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 232                     self.path, self.options).run_silent()
 233
 234     def is_clean(self):
 235         command="svn status %s" % self.path
 236         return len(Command(command,self.options).output_of(True)) == 0
 237
 238     def is_valid(self):
 239         return os.path.exists(os.path.join(self.path, ".svn"))
 240
 241
 242 class GitRepository:
 243     type = "git"
 244
 245     def __init__(self, path, options):
 246         self.path = path
 247         self.options = options
 248
 249     def name(self):
 250         return os.path.basename(self.path)
 251
 252     def url(self):
 253         self.repo_root()
 254
 255     def repo_root(self):
 256         c = Command("git remote show origin", self.options)
 257         out = self.__run_in_repo(c.output_of)
 258         for line in out.split('\n'):
 259             if line.strip().startswith("Fetch URL:"):
 260                 repo = line.split()[2]
 261
 262     @classmethod
 263     def checkout(cls, remote, local, options, depth=1):
 264         Command("rm -rf %s" % local, options).run_silent()
 265         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 266         return GitRepository(local, options)
 267
 268     @classmethod
 269     def remote_exists(cls, remote):
 270         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 271
 272     def tag_exists(self, tagname):
 273         command = 'git tag -l | grep "^%s$"' % tagname
 274         c = Command(command, self.options)
 275         out = self.__run_in_repo(c.output_of, with_stderr=True)
 276         return len(out) > 0
 277
 278     def __run_in_repo(self, fun, *args, **kwargs):
 279         cwd = os.getcwd()
 280         os.chdir(self.path)
 281         ret = fun(*args, **kwargs)
 282         os.chdir(cwd)
 283         return ret
 284
 285     def __run_command_in_repo(self, command, ignore_errors=False):
 286         c = Command(command, self.options)
 287         if ignore_errors:
 288             return self.__run_in_repo(c.output_of)
 289         else:
 290             return self.__run_in_repo(c.run_fatal)
 291
 292     def update(self, subdir=None, recursive=None):
 293         return self.__run_command_in_repo("git pull")
 294
 295     def to_branch(self, branch, remote=True):
 296         if remote:
 297             branch = "origin/%s" % branch
 298         return self.__run_command_in_repo("git checkout %s" % branch)
 299
 300     def to_tag(self, tag):
 301         return self.__run_command_in_repo("git checkout %s" % tag)
 302
 303     def tag(self, tagname, logfile):
 304         self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 305         self.commit(logfile)
 306
 307     def diff(self, f=""):
 308         c = Command("git diff %s" % f, self.options)
 309         return self.__run_in_repo(c.output_of, with_stderr=True)
 310
 311     def diff_with_tag(self, tagname):
 312         c = Command("git diff %s" % tagname, self.options)
 313         return self.__run_in_repo(c.output_of, with_stderr=True)
 314
 315     def commit(self, logfile):
 316         self.__run_command_in_repo("git add -A", ignore_errors=True)
 317         self.__run_command_in_repo("git commit -F  %s" % logfile, ignore_errors=True)
 318         self.__run_command_in_repo("git push")
 319         self.__run_command_in_repo("git push --tags")
 320
 321     def revert(self, f=""):
 322         if f:
 323             self.__run_command_in_repo("git checkout %s" % f)
 324         else:
 325             # revert all
 326             self.__run_command_in_repo("git --no-pager reset --hard")
 327             self.__run_command_in_repo("git --no-pager clean -f")
 328
 329     def is_clean(self):
 330         def check_commit():
 331             command="git status"
 332             s="nothing to commit (working directory clean)"
 333             return Command(command, self.options).output_of(True).find(s) >= 0
 334         return self.__run_in_repo(check_commit)
 335
 336     def is_valid(self):
 337         return os.path.exists(os.path.join(self.path, ".git"))
 338
 339
 340 class Repository:
 341     """ Generic repository """
 342     supported_repo_types = [SvnRepository, GitRepository]
 343
 344     def __init__(self, path, options):
 345         self.path = path
 346         self.options = options
 347         for repo in self.supported_repo_types:
 348             self.repo = repo(self.path, self.options)
 349             if self.repo.is_valid():
 350                 break
 351
 352     @classmethod
 353     def has_moved_to_git(cls, module, svnpath):
 354         module = git_to_svn_name(module)
 355         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 356
 357     @classmethod
 358     def remote_exists(cls, remote):
 359         for repo in Repository.supported_repo_types:
 360             if repo.remote_exists(remote):
 361                 return True
 362         return False
 363
 364     def __getattr__(self, attr):
 365         return getattr(self.repo, attr)
 366
 367
 368
 369 # support for tagged module is minimal, and is for the Build class only
 370 class Module:
 371
 372     svn_magic_line="--This line, and those below, will be ignored--"
 373     setting_tag_format = "Setting tag %s"
 374
 375     redirectors=[ # ('module_name_varname','name'),
 376                   ('module_version_varname','version'),
 377                   ('module_taglevel_varname','taglevel'), ]
 378
 379     # where to store user's config
 380     config_storage="CONFIG"
 381     #
 382     config={}
 383
 384     import commands
 385     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 386                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 387                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 388                  ('gituser', "Enter your user name (login name) on git server", os.getlogin()),
 389                  ("build", "Enter the name of your build module","build"),
 390                  ('username',"Enter your firstname and lastname for changelogs",""),
 391                  ("email","Enter your email address for changelogs",""),
 392                  ]
 393
 394     @classmethod
 395     def prompt_config_option(cls, key, message, default):
 396         cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 397
 398     @classmethod
 399     def prompt_config (cls):
 400         for (key,message,default) in cls.configKeys:
 401             cls.config[key]=""
 402             while not cls.config[key]:
 403                 cls.prompt_config_option(key, message, default)
 404
 405
 406     # for parsing module spec name:branch
 407     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 408     # special form for tagged module - for Build
 409     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 410     # parsing specfiles
 411     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 412
 413     def __init__ (self,module_spec,options):
 414         # parse module spec
 415         attempt=Module.matcher_branch_spec.match(module_spec)
 416         if attempt:
 417             self.name=attempt.group('name')
 418             self.branch=attempt.group('branch')
 419         else:
 420             attempt=Module.matcher_tag_spec.match(module_spec)
 421             if attempt:
 422                 self.name=attempt.group('name')
 423                 self.tagname=attempt.group('tagname')
 424             else:
 425                 self.name=module_spec
 426
 427         # when available prefer to use git module name internally
 428         self.name = svn_to_git_name(self.name)
 429
 430         self.options=options
 431         self.module_dir="%s/%s"%(options.workdir,self.name)
 432         self.repository = None
 433         self.build = None
 434
 435     def run (self,command):
 436         return Command(command,self.options).run()
 437     def run_fatal (self,command):
 438         return Command(command,self.options).run_fatal()
 439     def run_prompt (self,message,fun, *args):
 440         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 441         if not self.options.verbose:
 442             while True:
 443                 choice=prompt(message,True,('s','how'))
 444                 if choice is True:
 445                     fun(*args)
 446                     return
 447                 elif choice is False:
 448                     print 'About to run function:', fun_msg
 449         else:
 450             question=message+" - want to run function: " + fun_msg
 451             if prompt(question,True):
 452                 fun(*args)
 453
 454     def friendly_name (self):
 455         if hasattr(self,'branch'):
 456             return "%s:%s"%(self.name,self.branch)
 457         elif hasattr(self,'tagname'):
 458             return "%s@%s"%(self.name,self.tagname)
 459         else:
 460             return self.name
 461
 462     @classmethod
 463     def git_remote_dir (cls, name):
 464         return "%s@%s:/git/%s.git" % (cls.config['gituser'], cls.config['gitserver'], name)
 465
 466     @classmethod
 467     def svn_remote_dir (cls, name):
 468         name = git_to_svn_name(name)
 469         svn = cls.config['svnpath']
 470         if svn.endswith('/'):
 471             return "%s%s" % (svn, name)
 472         return "%s/%s" % (svn, name)
 473
 474     def svn_selected_remote(self):
 475         svn_name = git_to_svn_name(self.name)
 476         remote = self.svn_remote_dir(svn_name)
 477         if hasattr(self,'branch'):
 478             remote = "%s/branches/%s" % (remote, self.branch)
 479         elif hasattr(self,'tagname'):
 480             remote = "%s/tags/%s" % (remote, self.tagname)
 481         else:
 482             remote = "%s/trunk" % remote
 483         return remote
 484
 485     ####################
 486     @classmethod
 487     def init_homedir (cls, options):
 488         if options.verbose and options.mode not in Main.silent_modes:
 489             print 'Checking for', options.workdir
 490         storage="%s/%s"%(options.workdir, cls.config_storage)
 491         # sanity check. Either the topdir exists AND we have a config/storage
 492         # or topdir does not exist and we create it
 493         # to avoid people use their own daily svn repo
 494         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 495             print """The directory %s exists and has no CONFIG file
 496 If this is your regular working directory, please provide another one as the
 497 module-* commands need a fresh working dir. Make sure that you do not use
 498 that for other purposes than tagging""" % options.workdir
 499             sys.exit(1)
 500
 501         def checkout_build():
 502             print "Checking out build module..."
 503             remote = cls.git_remote_dir(cls.config['build'])
 504             local = os.path.join(options.workdir, cls.config['build'])
 505             GitRepository.checkout(remote, local, options, depth=1)
 506             print "OK"
 507
 508         def store_config():
 509             f=file(storage,"w")
 510             for (key,message,default) in Module.configKeys:
 511                 f.write("%s=%s\n"%(key,Module.config[key]))
 512             f.close()
 513             if options.debug:
 514                 print 'Stored',storage
 515                 Command("cat %s"%storage,options).run()
 516
 517         def read_config():
 518             # read config
 519             f=open(storage)
 520             for line in f.readlines():
 521                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 522                 Module.config[key]=value
 523             f.close()
 524
 525         if not os.path.isdir (options.workdir):
 526             print "Cannot find",options.workdir,"let's create it"
 527             Command("mkdir -p %s" % options.workdir, options).run_silent()
 528             cls.prompt_config()
 529             checkout_build()
 530             store_config()
 531         else:
 532             read_config()
 533             # check missing config options
 534             old_layout = False
 535             for (key,message,default) in cls.configKeys:
 536                 if not Module.config.has_key(key):
 537                     print "Configuration changed for module-tools"
 538                     cls.prompt_config_option(key, message, default)
 539                     old_layout = True
 540
 541             if old_layout:
 542                 Command("rm -rf %s" % options.workdir, options).run_silent()
 543                 Command("mkdir -p %s" % options.workdir, options).run_silent()
 544                 checkout_build()
 545                 store_config()
 546
 547             build_dir = os.path.join(options.workdir, cls.config['build'])
 548             if not os.path.isdir(build_dir):
 549                 checkout_build()
 550             else:
 551                 build = Repository(build_dir, options)
 552                 if not build.is_clean():
 553                     print "build module needs a revert"
 554                     build.revert()
 555                     print "OK"
 556                 build.update()
 557
 558         if options.verbose and options.mode not in Main.silent_modes:
 559             print '******** Using config'
 560             for (key,message,default) in Module.configKeys:
 561                 print '\t',key,'=',Module.config[key]
 562
 563     def init_module_dir (self):
 564         if self.options.verbose:
 565             print 'Checking for',self.module_dir
 566
 567         if not os.path.isdir (self.module_dir):
 568             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 569                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 570                                                          self.module_dir,
 571                                                          self.options)
 572             else:
 573                 remote = self.svn_selected_remote()
 574                 self.repository = SvnRepository.checkout(remote,
 575                                                          self.module_dir,
 576                                                          self.options, recursive=False)
 577
 578         self.repository = Repository(self.module_dir, self.options)
 579         if self.repository.type == "svn":
 580             # check if module has moved to git
 581             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 582                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 583                 self.init_module_dir()
 584             # check if we have the required branch/tag
 585             if self.repository.url() != self.svn_selected_remote():
 586                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 587                 self.init_module_dir()
 588
 589         elif self.repository.type == "git":
 590             if hasattr(self,'branch'):
 591                 self.repository.to_branch(self.branch)
 592             elif hasattr(self,'tagname'):
 593                 self.repository.to_tag(self.tagname)
 594
 595         else:
 596             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 597
 598
 599     def revert_module_dir (self):
 600         if self.options.fast_checks:
 601             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 602             return
 603         if self.options.verbose:
 604             print 'Checking whether', self.module_dir, 'needs being reverted'
 605
 606         if not self.repository.is_clean():
 607             self.repository.revert()
 608
 609     def update_module_dir (self):
 610         if self.options.fast_checks:
 611             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 612             return
 613         if self.options.verbose:
 614             print 'Updating', self.module_dir
 615         self.repository.update()
 616
 617     def main_specname (self):
 618         attempt="%s/%s.spec"%(self.module_dir,self.name)
 619         if os.path.isfile (attempt):
 620             return attempt
 621         pattern1="%s/*.spec"%self.module_dir
 622         level1=glob(pattern1)
 623         if level1:
 624             return level1[0]
 625         pattern2="%s/*/*.spec"%self.module_dir
 626         level2=glob(pattern2)
 627
 628         if level2:
 629             return level2[0]
 630         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 631
 632     def all_specnames (self):
 633         level1=glob("%s/*.spec" % self.module_dir)
 634         if level1: return level1
 635         level2=glob("%s/*/*.spec" % self.module_dir)
 636         return level2
 637
 638     def parse_spec (self, specfile, varnames):
 639         if self.options.verbose:
 640             print 'Parsing',specfile,
 641             for var in varnames:
 642                 print "[%s]"%var,
 643             print ""
 644         result={}
 645         f=open(specfile)
 646         for line in f.readlines():
 647             attempt=Module.matcher_rpm_define.match(line)
 648             if attempt:
 649                 (define,var,value)=attempt.groups()
 650                 if var in varnames:
 651                     result[var]=value
 652         f.close()
 653         if self.options.debug:
 654             print 'found',len(result),'keys'
 655             for (k,v) in result.iteritems():
 656                 print k,'=',v
 657         return result
 658
 659     # stores in self.module_name_varname the rpm variable to be used for the module's name
 660     # and the list of these names in self.varnames
 661     def spec_dict (self):
 662         specfile=self.main_specname()
 663         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 664         redirect_dict = self.parse_spec(specfile,redirector_keys)
 665         if self.options.debug:
 666             print '1st pass parsing done, redirect_dict=',redirect_dict
 667         varnames=[]
 668         for (varname,default) in Module.redirectors:
 669             if redirect_dict.has_key(varname):
 670                 setattr(self,varname,redirect_dict[varname])
 671                 varnames += [redirect_dict[varname]]
 672             else:
 673                 setattr(self,varname,default)
 674                 varnames += [ default ]
 675         self.varnames = varnames
 676         result = self.parse_spec (specfile,self.varnames)
 677         if self.options.debug:
 678             print '2st pass parsing done, varnames=',varnames,'result=',result
 679         return result
 680
 681     def patch_spec_var (self, patch_dict,define_missing=False):
 682         for specfile in self.all_specnames():
 683             # record the keys that were changed
 684             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 685             newspecfile=specfile+".new"
 686             if self.options.verbose:
 687                 print 'Patching',specfile,'for',patch_dict.keys()
 688             spec=open (specfile)
 689             new=open(newspecfile,"w")
 690
 691             for line in spec.readlines():
 692                 attempt=Module.matcher_rpm_define.match(line)
 693                 if attempt:
 694                     (define,var,value)=attempt.groups()
 695                     if var in patch_dict.keys():
 696                         if self.options.debug:
 697                             print 'rewriting %s as %s'%(var,patch_dict[var])
 698                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 699                         changed[var]=True
 700                         continue
 701                 new.write(line)
 702             if define_missing:
 703                 for (key,was_changed) in changed.iteritems():
 704                     if not was_changed:
 705                         if self.options.debug:
 706                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 707                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 708             spec.close()
 709             new.close()
 710             os.rename(newspecfile,specfile)
 711
 712     # returns all lines until the magic line
 713     def unignored_lines (self, logfile):
 714         result=[]
 715         white_line_matcher = re.compile("\A\s*\Z")
 716         for logline in file(logfile).readlines():
 717             if logline.strip() == Module.svn_magic_line:
 718                 break
 719             elif white_line_matcher.match(logline):
 720                 continue
 721             else:
 722                 result.append(logline.strip()+'\n')
 723         return result
 724
 725     # creates a copy of the input with only the unignored lines
 726     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 727        f=file(fileout,'w')
 728        f.write(self.setting_tag_format%new_tag_name + '\n')
 729        for line in self.unignored_lines(filein):
 730            f.write(line)
 731        f.close()
 732
 733     def insert_changelog (self, logfile, oldtag, newtag):
 734         for specfile in self.all_specnames():
 735             newspecfile=specfile+".new"
 736             if self.options.verbose:
 737                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 738             spec=open (specfile)
 739             new=open(newspecfile,"w")
 740             for line in spec.readlines():
 741                 new.write(line)
 742                 if re.compile('%changelog').match(line):
 743                     dateformat="* %a %b %d %Y"
 744                     datepart=time.strftime(dateformat)
 745                     logpart="%s <%s> - %s"%(Module.config['username'],
 746                                                  Module.config['email'],
 747                                                  newtag)
 748                     new.write(datepart+" "+logpart+"\n")
 749                     for logline in self.unignored_lines(logfile):
 750                         new.write("- " + logline)
 751                     new.write("\n")
 752             spec.close()
 753             new.close()
 754             os.rename(newspecfile,specfile)
 755
 756     def show_dict (self, spec_dict):
 757         if self.options.verbose:
 758             for (k,v) in spec_dict.iteritems():
 759                 print k,'=',v
 760
 761     def last_tag (self, spec_dict):
 762         try:
 763             return "%s-%s" % (spec_dict[self.module_version_varname],
 764                               spec_dict[self.module_taglevel_varname])
 765         except KeyError,err:
 766             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 767
 768     def tag_name (self, spec_dict, old_svn_name=False):
 769         base_tag_name = self.name
 770         if old_svn_name:
 771             base_tag_name = git_to_svn_name(self.name)
 772         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 773
 774
 775 ##############################
 776     # using fine_grain means replacing only those instances that currently refer to this tag
 777     # otherwise, <module>-SVNPATH is replaced unconditionnally
 778     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 779         newtagsfile=tagsfile+".new"
 780         tags=open (tagsfile)
 781         new=open(newtagsfile,"w")
 782
 783         matches=0
 784         # fine-grain : replace those lines that refer to oldname
 785         if fine_grain:
 786             if self.options.verbose:
 787                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 788             matcher=re.compile("^(.*)%s(.*)"%oldname)
 789             for line in tags.readlines():
 790                 if not matcher.match(line):
 791                     new.write(line)
 792                 else:
 793                     (begin,end)=matcher.match(line).groups()
 794                     new.write(begin+newname+end+"\n")
 795                     matches += 1
 796         # brute-force : change uncommented lines that define <module>-SVNPATH
 797         else:
 798             if self.options.verbose:
 799                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 800             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 801                                           %(self.name)
 802             matcher_module=re.compile(pattern)
 803             for line in tags.readlines():
 804                 attempt=matcher_module.match(line)
 805                 if attempt:
 806                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 807                     if self.options.verbose:
 808                         print ' '+svnpath,
 809                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 810                     new.write(replacement)
 811                     matches += 1
 812                 else:
 813                     new.write(line)
 814         tags.close()
 815         new.close()
 816         os.rename(newtagsfile,tagsfile)
 817         if self.options.verbose: print "%d changes"%matches
 818         return matches
 819
 820     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 821         if self.options.verbose:
 822             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 823
 824         found_tagname = tagname
 825         found = self.repository.tag_exists(tagname)
 826         if not found and old_svn_tag_name:
 827             if self.options.verbose:
 828                 print "KO"
 829                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 830             found = self.repository.tag_exists(old_svn_tag_name)
 831             if found:
 832                 found_tagname = old_svn_tag_name
 833
 834         if (found and need_it) or (not found and not need_it):
 835             print "OK "
 836         else:
 837             print "KO"
 838             if found:
 839                 raise Exception, "tag (%s) is already there" % tagname
 840             else:
 841                 raise Exception, "can not find required tag (%s)" % tagname
 842
 843         return found_tagname
 844
 845
 846     def do_tag (self):
 847         self.init_module_dir()
 848         self.revert_module_dir()
 849         self.update_module_dir()
 850         # parse specfile
 851         spec_dict = self.spec_dict()
 852         self.show_dict(spec_dict)
 853
 854         # side effects
 855         old_tag_name = self.tag_name(spec_dict)
 856         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 857
 858         if (self.options.new_version):
 859             # new version set on command line
 860             spec_dict[self.module_version_varname] = self.options.new_version
 861             spec_dict[self.module_taglevel_varname] = 0
 862         else:
 863             # increment taglevel
 864             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 865             spec_dict[self.module_taglevel_varname] = new_taglevel
 866
 867         new_tag_name = self.tag_name(spec_dict)
 868
 869         # sanity check
 870         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 871         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 872
 873         # checking for diffs
 874         diff_output = self.repository.diff_with_tag(old_tag_name)
 875         if len(diff_output) == 0:
 876             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 877                 return
 878
 879         # side effect in trunk's specfile
 880         self.patch_spec_var(spec_dict)
 881
 882         # prepare changelog file
 883         # we use the standard subversion magic string (see svn_magic_line)
 884         # so we can provide useful information, such as version numbers and diff
 885         # in the same file
 886         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 887         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 888         setting_tag_line=Module.setting_tag_format%new_tag_name
 889         file(changelog,"w").write("""
 890 %s
 891 %s
 892 Please write a changelog for this new tag in the section above
 893 """%(Module.svn_magic_line,setting_tag_line))
 894
 895         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 896             file(changelog,"a").write('DIFF=========\n' + diff_output)
 897
 898         if self.options.debug:
 899             prompt('Proceed ?')
 900
 901         # edit it
 902         self.run("%s %s"%(self.options.editor,changelog))
 903         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 904         # so we do the job ourselves
 905         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 906         # insert changelog in spec
 907         if self.options.changelog:
 908             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 909
 910         ## update build
 911         build_path = os.path.join(self.options.workdir,
 912                                   Module.config['build'])
 913         build = Repository(build_path, self.options)
 914         if self.options.build_branch:
 915             build.to_branch(self.options.build_branch)
 916         if not build.is_clean():
 917             build.revert()
 918
 919         tagsfiles=glob(build.path+"/*-tags*.mk")
 920         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 921         default_answer = 'y'
 922         tagsfiles.sort()
 923         while True:
 924             for tagsfile in tagsfiles:
 925                 status=tagsdict[tagsfile]
 926                 basename=os.path.basename(tagsfile)
 927                 print ".................... Dealing with %s"%basename
 928                 while tagsdict[tagsfile] == 'todo' :
 929                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 930                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 931                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 932                                      allow_outside=True)
 933                     if choice == 'y':
 934                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 935                     elif choice == 'n':
 936                         print 'Done with %s'%os.path.basename(tagsfile)
 937                         tagsdict[tagsfile]='done'
 938                     elif choice == 'f':
 939                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 940                     elif choice == 'd':
 941                         print build.diff(f=tagsfile)
 942                     elif choice == 'r':
 943                         build.revert(f=tagsfile)
 944                     elif choice == 'c':
 945                         self.run("cat %s"%tagsfile)
 946                     else:
 947                         name=self.name
 948                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 949 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 950 d: show current diff for this tag file
 951 r: revert that tag file
 952 c: cat the current tag file
 953 n: move to next file"""%locals()
 954
 955             if prompt("Want to review changes on tags files",False):
 956                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 957                 default_answer='d'
 958             else:
 959                 break
 960
 961         def diff_all_changes():
 962             print build.diff()
 963             print self.repository.diff()
 964
 965         def commit_all_changes(log):
 966             self.repository.commit(log)
 967             build.commit(log)
 968
 969         self.run_prompt("Review module and build", diff_all_changes)
 970         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 971         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 972
 973         if self.options.debug:
 974             print 'Preserving',changelog,'and stripped',changelog_svn
 975         else:
 976             os.unlink(changelog)
 977             os.unlink(changelog_svn)
 978
 979
 980 ##############################
 981 class Main:
 982
 983     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
 984
 985 module-tools : a set of tools to manage subversion tags and specfile
 986   requires the specfile to either
 987   * define *version* and *taglevel*
 988   OR alternatively
 989   * define redirection variables module_version_varname / module_taglevel_varname
 990 Trunk:
 991   by default, the trunk of modules is taken into account
 992   in this case, just mention the module name as <module_desc>
 993 Branches:
 994   if you wish to work on a branch rather than on the trunk,
 995   you can use something like e.g. Mom:2.1 as <module_desc>
 996 """
 997     release_usage="""Usage: %prog [options] tag1 .. tagn
 998   Extract release notes from the changes in specfiles between several build tags, latest first
 999   Examples:
1000       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
1001   You can refer to a (build) branch by prepending a colon, like in
1002       release-changelog :4.2 4.2-rc25
1003   You can refer to the build trunk by just mentioning 'trunk', e.g.
1004       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
1005 """
1006     common_usage="""More help:
1007   see http://svn.planet-lab.org/wiki/ModuleTools"""
1008
1009     modes={
1010         'list' : "displays a list of available tags or branches",
1011         'version' : "check latest specfile and print out details",
1012         'diff' : "show difference between module (trunk or branch) and latest tag",
1013         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
1014                 create new tag and and monitor its adoption in build/*-tags*.mk""",
1015         'branch' : """create a branch for this module, from the latest tag on the trunk,
1016                   and change trunk's version number to reflect the new branch name;
1017                   you can specify the new branch name by using module:branch""",
1018         'sync' : """create a tag from the module
1019                 this is a last resort option, mostly for repairs""",
1020         'changelog' : """extract changelog between build tags
1021                 expected arguments are a list of tags""",
1022         }
1023
1024     silent_modes = ['list']
1025     release_modes = ['changelog']
1026
1027     @staticmethod
1028     def optparse_list (option, opt, value, parser):
1029         try:
1030             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
1031         except:
1032             setattr(parser.values,option.dest,value.split())
1033
1034     def run(self):
1035
1036         mode=None
1037         for function in Main.modes.keys():
1038             if sys.argv[0].find(function) >= 0:
1039                 mode = function
1040                 break
1041         if not mode:
1042             print "Unsupported command",sys.argv[0]
1043             print "Supported commands:" + " ".join(Main.modes.keys())
1044             sys.exit(1)
1045
1046         if mode not in Main.release_modes:
1047             usage = Main.module_usage
1048             usage += Main.common_usage
1049             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1050         else:
1051             usage = Main.release_usage
1052             usage += Main.common_usage
1053
1054         parser=OptionParser(usage=usage)
1055
1056         if mode == "tag" or mode == 'branch':
1057             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1058                               help="set new version and reset taglevel to 0")
1059         if mode == "tag" :
1060             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1061                               help="do not update changelog section in specfile when tagging")
1062             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1063                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1064         if mode == "tag" or mode == "sync" :
1065             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1066                               help="specify editor")
1067
1068         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1069         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1070                           help="dry run - shell commands are only displayed")
1071         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1072                           default=[], nargs=1,type="string",
1073                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1074 -- can be set multiple times, or use quotes""")
1075
1076         parser.add_option("-w","--workdir", action="store", dest="workdir",
1077                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1078                           help="""name for dedicated working dir - defaults to ~/modules
1079 ** THIS MUST NOT ** be your usual working directory""")
1080         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1081                           help="skip safety checks, such as svn updates -- use with care")
1082
1083         # default verbosity depending on function - temp
1084         verbose_modes= ['tag', 'sync', 'branch']
1085
1086         if mode not in verbose_modes:
1087             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1088                               help="run in verbose mode")
1089         else:
1090             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1091                               help="run in quiet (non-verbose) mode")
1092         (options, args) = parser.parse_args()
1093         options.mode=mode
1094         if not hasattr(options,'dry_run'):
1095             options.dry_run=False
1096         if not hasattr(options,'www'):
1097             options.www=False
1098         options.debug=False
1099
1100         ########## module-*
1101         if len(args) == 0:
1102             parser.print_help()
1103             sys.exit(1)
1104         Module.init_homedir(options)
1105
1106         modules=[ Module(modname,options) for modname in args ]
1107         for module in modules:
1108             if len(args)>1 and mode not in Main.silent_modes:
1109                 print '========================================',module.friendly_name()
1110             # call the method called do_<mode>
1111             method=Module.__dict__["do_%s"%mode]
1112             try:
1113                 method(module)
1114             except Exception,e:
1115                 import traceback
1116                 traceback.print_exc()
1117                 print 'Skipping module %s: '%modname,e
1118
1119 ####################
1120 if __name__ == "__main__" :
1121     try:
1122         Main().run()
1123     except KeyboardInterrupt:
1124         print '\nBye'