module-tools.py

   1 #!/usr/bin/python -u
   2
   3 import sys, os
   4 import re
   5 import time
   6 from glob import glob
   7 from optparse import OptionParser
   8
   9 # HARDCODED NAME CHANGES
  10 #
  11 # Moving to git we decided to rename some of the repositories. Here is
  12 # a map of name changes applied in git repositories.
  13 RENAMED_SVN_MODULES = {
  14     "PLEWWW": "plewww"
  15     }
  16
  17 def svn_to_git_name(module):
  18     if RENAMED_SVN_MODULES.has_key(module):
  19         return RENAMED_SVN_MODULES[module]
  20     return module
  21
  22 def git_to_svn_name(module):
  23     for key in RENAMED_SVN_MODULES:
  24         if module == RENAMED_SVN_MODULES[key]:
  25             return key
  26     return module
  27
  28
  29 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
  30 def prompt (question,default=True,other_choices=[],allow_outside=False):
  31     if not isinstance (other_choices,list):
  32         other_choices = [ other_choices ]
  33     chars = [ c for (c,rest) in other_choices ]
  34
  35     choices = []
  36     if 'y' not in chars:
  37         if default is True: choices.append('[y]')
  38         else : choices.append('y')
  39     if 'n' not in chars:
  40         if default is False: choices.append('[n]')
  41         else : choices.append('n')
  42
  43     for (char,choice) in other_choices:
  44         if default == char:
  45             choices.append("["+char+"]"+choice)
  46         else:
  47             choices.append("<"+char+">"+choice)
  48     try:
  49         answer=raw_input(question + " " + "/".join(choices) + " ? ")
  50         if not answer:
  51             return default
  52         answer=answer[0].lower()
  53         if answer == 'y':
  54             if 'y' in chars: return 'y'
  55             else: return True
  56         elif answer == 'n':
  57             if 'n' in chars: return 'n'
  58             else: return False
  59         elif other_choices:
  60             for (char,choice) in other_choices:
  61                 if answer == char:
  62                     return char
  63             if allow_outside:
  64                 return answer
  65         return prompt(question,default,other_choices)
  66     except:
  67         raise
  68
  69 def default_editor():
  70     try:
  71         editor = os.environ['EDITOR']
  72     except:
  73         editor = "emacs"
  74     return editor
  75
  76 ### fold long lines
  77 fold_length=132
  78
  79 def print_fold (line):
  80     while len(line) >= fold_length:
  81         print line[:fold_length],'\\'
  82         line=line[fold_length:]
  83     print line
  84
  85 class Command:
  86     def __init__ (self,command,options):
  87         self.command=command
  88         self.options=options
  89         self.tmp="/tmp/command-%d"%os.getpid()
  90
  91     def run (self):
  92         if self.options.dry_run:
  93             print 'dry_run',self.command
  94             return 0
  95         if self.options.verbose and self.options.mode not in Main.silent_modes:
  96             print '+',self.command
  97             sys.stdout.flush()
  98         return os.system(self.command)
  99
 100     def run_silent (self):
 101         if self.options.dry_run:
 102             print 'dry_run',self.command
 103             return 0
 104         if self.options.verbose:
 105             print '+',self.command,' .. ',
 106             sys.stdout.flush()
 107         retcod=os.system(self.command + " &> " + self.tmp)
 108         if retcod != 0:
 109             print "FAILED ! -- out+err below (command was %s)"%self.command
 110             os.system("cat " + self.tmp)
 111             print "FAILED ! -- end of quoted output"
 112         elif self.options.verbose:
 113             print "OK"
 114         os.unlink(self.tmp)
 115         return retcod
 116
 117     def run_fatal(self):
 118         if self.run_silent() !=0:
 119             raise Exception,"Command %s failed"%self.command
 120
 121     # returns stdout, like bash's $(mycommand)
 122     def output_of (self,with_stderr=False):
 123         if self.options.dry_run:
 124             print 'dry_run',self.command
 125             return 'dry_run output'
 126         tmp="/tmp/status-%d"%os.getpid()
 127         if self.options.debug:
 128             print '+',self.command,' .. ',
 129             sys.stdout.flush()
 130         command=self.command
 131         if with_stderr:
 132             command += " &> "
 133         else:
 134             command += " > "
 135         command += tmp
 136         os.system(command)
 137         result=file(tmp).read()
 138         os.unlink(tmp)
 139         if self.options.debug:
 140             print 'Done',
 141         return result
 142
 143
 144 class SvnRepository:
 145     type = "svn"
 146
 147     def __init__(self, path, options):
 148         self.path = path
 149         self.options = options
 150
 151     def name(self):
 152         return os.path.basename(self.path)
 153
 154     def url(self):
 155         out = Command("svn info %s" % self.path, self.options).output_of()
 156         for line in out.split('\n'):
 157             if line.startswith("URL:"):
 158                 return line.split()[1].strip()
 159
 160     def repo_root(self):
 161         out = Command("svn info %s" % self.path, self.options).output_of()
 162         for line in out.split('\n'):
 163             if line.startswith("Repository Root:"):
 164                 root = line.split()[2].strip()
 165                 return "%s/%s" % (root, self.name())
 166
 167     @classmethod
 168     def checkout(cls, remote, local, options, recursive=False):
 169         if recursive:
 170             svncommand = "svn co %s %s" % (remote, local)
 171         else:
 172             svncommand = "svn co -N %s %s" % (remote, local)
 173         Command("rm -rf %s" % local, options).run_silent()
 174         Command(svncommand, options).run_fatal()
 175
 176         return SvnRepository(local, options)
 177
 178     @classmethod
 179     def remote_exists(cls, remote):
 180         return os.system("svn list %s &> /dev/null" % remote) == 0
 181
 182     def tag_exists(self, tagname):
 183         url = "%s/tags/%s" % (self.repo_root(), tagname)
 184         return SvnRepository.remote_exists(url)
 185
 186     def update(self, subdir="", recursive=True):
 187         path = os.path.join(self.path, subdir)
 188         if recursive:
 189             svncommand = "svn up %s" % path
 190         else:
 191             svncommand = "svn up -N %s" % path
 192         Command(svncommand, self.options).run_fatal()
 193
 194     def commit(self, logfile):
 195         # add all new files to the repository
 196         Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" %
 197                 self.path, self.options).run_silent()
 198         Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal()
 199
 200     def to_branch(self, branch):
 201         remote = "%s/branches/%s" % (self.repo_root(), branch)
 202         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 203
 204     def to_tag(self, tag):
 205         remote = "%s/tags/%s" % (self.repo_root(), branch)
 206         SvnRepository.checkout(remote, self.path, self.options, recursive=True)
 207
 208     def tag(self, tagname, logfile):
 209         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 210         self_url = self.url()
 211         Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal()
 212
 213     def diff(self, f=""):
 214         if f:
 215             f = os.path.join(self.path, f)
 216         else:
 217             f = self.path
 218         return Command("svn diff %s" % f, self.options).output_of(True)
 219
 220     def diff_with_tag(self, tagname):
 221         tag_url = "%s/tags/%s" % (self.repo_root(), tagname)
 222         return Command("svn diff %s %s" % (tag_url, self.url()),
 223                        self.options).output_of(True)
 224
 225     def revert(self, f=""):
 226         if f:
 227             Command("svn revert %s" % os.path.join(self.path, f), self.options).run_fatal()
 228         else:
 229             # revert all
 230             Command("svn revert %s -R" % self.path, self.options).run_fatal()
 231             Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " %
 232                     self.path, self.options).run_silent()
 233
 234     def is_clean(self):
 235         command="svn status %s" % self.path
 236         return len(Command(command,self.options).output_of(True)) == 0
 237
 238     def is_valid(self):
 239         return os.path.exists(os.path.join(self.path, ".svn"))
 240
 241
 242 class GitRepository:
 243     type = "git"
 244
 245     def __init__(self, path, options):
 246         self.path = path
 247         self.options = options
 248
 249     def name(self):
 250         return os.path.basename(self.path)
 251
 252     def url(self):
 253         self.repo_root()
 254
 255     def repo_root(self):
 256         c = Command("git remote show origin", self.options)
 257         out = self.__run_in_repo(c.output_of)
 258         for line in out.split('\n'):
 259             if line.strip().startswith("Fetch URL:"):
 260                 repo = line.split()[2]
 261
 262     @classmethod
 263     def checkout(cls, remote, local, options, depth=1):
 264         Command("rm -rf %s" % local, options).run_silent()
 265         Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal()
 266         return GitRepository(local, options)
 267
 268     @classmethod
 269     def remote_exists(cls, remote):
 270         return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0
 271
 272     def tag_exists(self, tagname):
 273         command = 'git tag -l | grep "^%s$"' % tagname
 274         c = Command(command, self.options)
 275         out = self.__run_in_repo(c.output_of, with_stderr=True)
 276         return len(out) > 0
 277
 278     def __run_in_repo(self, fun, *args, **kwargs):
 279         cwd = os.getcwd()
 280         os.chdir(self.path)
 281         ret = fun(*args, **kwargs)
 282         os.chdir(cwd)
 283         return ret
 284
 285     def __run_command_in_repo(self, command, ignore_errors=False):
 286         c = Command(command, self.options)
 287         if ignore_errors:
 288             return self.__run_in_repo(c.output_of)
 289         else:
 290             return self.__run_in_repo(c.run_fatal)
 291
 292     def update(self, subdir=None, recursive=None):
 293         return self.__run_command_in_repo("git pull")
 294
 295     def to_branch(self, branch, remote=True):
 296         if remote:
 297             branch = "origin/%s" % branch
 298         return self.__run_command_in_repo("git checkout %s" % branch)
 299
 300     def to_tag(self, tag):
 301         return self.__run_command_in_repo("git checkout %s" % tag)
 302
 303     def tag(self, tagname, logfile):
 304         self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile))
 305         self.commit(logfile)
 306
 307     def diff(self, f=""):
 308         c = Command("git diff %s" % f, self.options)
 309         return self.__run_in_repo(c.output_of, with_stderr=True)
 310
 311     def diff_with_tag(self, tagname):
 312         c = Command("git diff %s" % tagname, self.options)
 313         return self.__run_in_repo(c.output_of, with_stderr=True)
 314
 315     def commit(self, logfile):
 316         self.__run_command_in_repo("git add -A", ignore_errors=True)
 317         self.__run_command_in_repo("git commit -F  %s" % logfile, ignore_errors=True)
 318         self.__run_command_in_repo("git push")
 319         self.__run_command_in_repo("git push --tags")
 320
 321     def revert(self, f=""):
 322         if f:
 323             self.__run_command_in_repo("git checkout %s" % f)
 324         else:
 325             # revert all
 326             self.__run_command_in_repo("git --no-pager reset --hard")
 327             self.__run_command_in_repo("git --no-pager clean -f")
 328
 329     def is_clean(self):
 330         def check_commit():
 331             command="git status"
 332             s="nothing to commit (working directory clean)"
 333             return Command(command, self.options).output_of(True).find(s) >= 0
 334         return self.__run_in_repo(check_commit)
 335
 336     def is_valid(self):
 337         return os.path.exists(os.path.join(self.path, ".git"))
 338
 339
 340 class Repository:
 341     """ Generic repository """
 342     supported_repo_types = [SvnRepository, GitRepository]
 343
 344     def __init__(self, path, options):
 345         self.path = path
 346         self.options = options
 347         for repo in self.supported_repo_types:
 348             self.repo = repo(self.path, self.options)
 349             if self.repo.is_valid():
 350                 break
 351
 352     @classmethod
 353     def has_moved_to_git(cls, module, svnpath):
 354         module = git_to_svn_name(module)
 355         return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (svnpath, module))
 356
 357     @classmethod
 358     def remote_exists(cls, remote):
 359         for repo in Repository.supported_repo_types:
 360             if repo.remote_exists(remote):
 361                 return True
 362         return False
 363
 364     def __getattr__(self, attr):
 365         return getattr(self.repo, attr)
 366
 367
 368
 369 # support for tagged module is minimal, and is for the Build class only
 370 class Module:
 371
 372     svn_magic_line="--This line, and those below, will be ignored--"
 373     setting_tag_format = "Setting tag %s"
 374
 375     redirectors=[ # ('module_name_varname','name'),
 376                   ('module_version_varname','version'),
 377                   ('module_taglevel_varname','taglevel'), ]
 378
 379     # where to store user's config
 380     config_storage="CONFIG"
 381     #
 382     config={}
 383
 384     import commands
 385     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
 386                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
 387                  ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
 388                  ('gituser', "Enter your user name (login name) on git server", os.getlogin()),
 389                  ("build", "Enter the name of your build module","build"),
 390                  ('username',"Enter your firstname and lastname for changelogs",""),
 391                  ("email","Enter your email address for changelogs",""),
 392                  ]
 393
 394     @classmethod
 395     def prompt_config_option(cls, key, message, default):
 396         cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
 397
 398     @classmethod
 399     def prompt_config (cls):
 400         for (key,message,default) in cls.configKeys:
 401             cls.config[key]=""
 402             while not cls.config[key]:
 403                 cls.prompt_config_option(key, message, default)
 404
 405
 406     # for parsing module spec name:branch
 407     matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")
 408     # special form for tagged module - for Build
 409     matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")
 410     # parsing specfiles
 411     matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
 412
 413     def __init__ (self,module_spec,options):
 414         # parse module spec
 415         attempt=Module.matcher_branch_spec.match(module_spec)
 416         if attempt:
 417             self.name=attempt.group('name')
 418             self.branch=attempt.group('branch')
 419         else:
 420             attempt=Module.matcher_tag_spec.match(module_spec)
 421             if attempt:
 422                 self.name=attempt.group('name')
 423                 self.tagname=attempt.group('tagname')
 424             else:
 425                 self.name=module_spec
 426
 427         # when available prefer to use git module name internally
 428         self.name = svn_to_git_name(self.name)
 429
 430         self.options=options
 431         self.module_dir="%s/%s"%(options.workdir,self.name)
 432         self.repository = None
 433         self.build = None
 434
 435     def run (self,command):
 436         return Command(command,self.options).run()
 437     def run_fatal (self,command):
 438         return Command(command,self.options).run_fatal()
 439     def run_prompt (self,message,fun, *args):
 440         fun_msg = "%s(%s)" % (fun.func_name, ",".join(args))
 441         if not self.options.verbose:
 442             while True:
 443                 choice=prompt(message,True,('s','how'))
 444                 if choice is True:
 445                     fun(*args)
 446                     return
 447                 elif choice is False:
 448                     print 'About to run function:', fun_msg
 449         else:
 450             question=message+" - want to run function: " + fun_msg
 451             if prompt(question,True):
 452                 fun(*args)
 453
 454     def friendly_name (self):
 455         if hasattr(self,'branch'):
 456             return "%s:%s"%(self.name,self.branch)
 457         elif hasattr(self,'tagname'):
 458             return "%s@%s"%(self.name,self.tagname)
 459         else:
 460             return self.name
 461
 462     @classmethod
 463     def git_remote_dir (cls, name):
 464         return "%s@%s:/git/%s.git" % (cls.config['gituser'], cls.config['gitserver'], name)
 465
 466     @classmethod
 467     def svn_remote_dir (cls, name):
 468         name = git_to_svn_name(name)
 469         svn = cls.config['svnpath']
 470         if svn.endswith('/'):
 471             return "%s%s" % (svn, name)
 472         return "%s/%s" % (svn, name)
 473
 474     def svn_selected_remote(self):
 475         svn_name = git_to_svn_name(self.name)
 476         remote = self.svn_remote_dir(svn_name)
 477         if hasattr(self,'branch'):
 478             remote = "%s/branches/%s" % (remote, self.branch)
 479         elif hasattr(self,'tagname'):
 480             remote = "%s/tags/%s" % (remote, self.tagname)
 481         else:
 482             remote = "%s/trunk" % remote
 483         return remote
 484
 485     ####################
 486     @classmethod
 487     def init_homedir (cls, options):
 488         if options.verbose and options.mode not in Main.silent_modes:
 489             print 'Checking for', options.workdir
 490         storage="%s/%s"%(options.workdir, cls.config_storage)
 491         # sanity check. Either the topdir exists AND we have a config/storage
 492         # or topdir does not exist and we create it
 493         # to avoid people use their own daily svn repo
 494         if os.path.isdir(options.workdir) and not os.path.isfile(storage):
 495             print """The directory %s exists and has no CONFIG file
 496 If this is your regular working directory, please provide another one as the
 497 module-* commands need a fresh working dir. Make sure that you do not use
 498 that for other purposes than tagging""" % options.workdir
 499             sys.exit(1)
 500
 501         def checkout_build():
 502             print "Checking out build module..."
 503             remote = cls.git_remote_dir(cls.config['build'])
 504             local = os.path.join(options.workdir, cls.config['build'])
 505             GitRepository.checkout(remote, local, options, depth=1)
 506             print "OK"
 507
 508         def store_config():
 509             f=file(storage,"w")
 510             for (key,message,default) in Module.configKeys:
 511                 f.write("%s=%s\n"%(key,Module.config[key]))
 512             f.close()
 513             if options.debug:
 514                 print 'Stored',storage
 515                 Command("cat %s"%storage,options).run()
 516
 517         def read_config():
 518             # read config
 519             f=open(storage)
 520             for line in f.readlines():
 521                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
 522                 Module.config[key]=value
 523             f.close()
 524
 525         if not os.path.isdir (options.workdir):
 526             print "Cannot find",options.workdir,"let's create it"
 527             Command("mkdir -p %s" % options.workdir, options).run_silent()
 528             cls.prompt_config()
 529             checkout_build()
 530             store_config()
 531         else:
 532             read_config()
 533             # check missing config options
 534             old_layout = False
 535             for (key,message,default) in cls.configKeys:
 536                 if not Module.config.has_key(key):
 537                     print "Configuration changed for module-tools"
 538                     cls.prompt_config_option(key, message, default)
 539                     old_layout = True
 540
 541             if old_layout:
 542                 Command("rm -rf %s" % options.workdir, options).run_silent()
 543                 Command("mkdir -p %s" % options.workdir, options).run_silent()
 544                 checkout_build()
 545                 store_config()
 546
 547             build_dir = os.path.join(options.workdir, cls.config['build'])
 548             if not os.path.isdir(build_dir):
 549                 checkout_build()
 550             else:
 551                 build = Repository(build_dir, options)
 552                 if not build.is_clean():
 553                     print "build module needs a revert"
 554                     build.revert()
 555                     print "OK"
 556                 build.update()
 557
 558         if options.verbose and options.mode not in Main.silent_modes:
 559             print '******** Using config'
 560             for (key,message,default) in Module.configKeys:
 561                 print '\t',key,'=',Module.config[key]
 562
 563     def init_module_dir (self):
 564         if self.options.verbose:
 565             print 'Checking for',self.module_dir
 566
 567         if not os.path.isdir (self.module_dir):
 568             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 569                 self.repository = GitRepository.checkout(self.git_remote_dir(self.name),
 570                                                          self.module_dir,
 571                                                          self.options)
 572             else:
 573                 remote = self.svn_selected_remote()
 574                 self.repository = SvnRepository.checkout(remote,
 575                                                          self.module_dir,
 576                                                          self.options, recursive=False)
 577
 578         self.repository = Repository(self.module_dir, self.options)
 579         if self.repository.type == "svn":
 580             # check if module has moved to git
 581             if Repository.has_moved_to_git(self.name, Module.config['svnpath']):
 582                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 583                 self.init_module_dir()
 584             # check if we have the required branch/tag
 585             if self.repository.url() != self.svn_selected_remote():
 586                 Command("rm -rf %s" % self.module_dir, self.options).run_silent()
 587                 self.init_module_dir()
 588
 589         elif self.repository.type == "git":
 590             if hasattr(self,'branch'):
 591                 self.repository.to_branch(self.branch)
 592             elif hasattr(self,'tagname'):
 593                 self.repository.to_tag(self.tagname)
 594
 595         else:
 596             raise Exception, 'Cannot find %s - check module name'%self.module_dir
 597
 598
 599     def revert_module_dir (self):
 600         if self.options.fast_checks:
 601             if self.options.verbose: print 'Skipping revert of %s' % self.module_dir
 602             return
 603         if self.options.verbose:
 604             print 'Checking whether', self.module_dir, 'needs being reverted'
 605
 606         if not self.repository.is_clean():
 607             self.repository.revert()
 608
 609     def update_module_dir (self):
 610         if self.options.fast_checks:
 611             if self.options.verbose: print 'Skipping update of %s' % self.module_dir
 612             return
 613         if self.options.verbose:
 614             print 'Updating', self.module_dir
 615         self.repository.update()
 616
 617     def main_specname (self):
 618         attempt="%s/%s.spec"%(self.module_dir,self.name)
 619         if os.path.isfile (attempt):
 620             return attempt
 621         pattern1="%s/*.spec"%self.module_dir
 622         level1=glob(pattern1)
 623         if level1:
 624             return level1[0]
 625         pattern2="%s/*/*.spec"%self.module_dir
 626         level2=glob(pattern2)
 627
 628         if level2:
 629             return level2[0]
 630         raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2)
 631
 632     def all_specnames (self):
 633         level1=glob("%s/*.spec" % self.module_dir)
 634         if level1: return level1
 635         level2=glob("%s/*/*.spec" % self.module_dir)
 636         return level2
 637
 638     def parse_spec (self, specfile, varnames):
 639         if self.options.verbose:
 640             print 'Parsing',specfile,
 641             for var in varnames:
 642                 print "[%s]"%var,
 643             print ""
 644         result={}
 645         f=open(specfile)
 646         for line in f.readlines():
 647             attempt=Module.matcher_rpm_define.match(line)
 648             if attempt:
 649                 (define,var,value)=attempt.groups()
 650                 if var in varnames:
 651                     result[var]=value
 652         f.close()
 653         if self.options.debug:
 654             print 'found',len(result),'keys'
 655             for (k,v) in result.iteritems():
 656                 print k,'=',v
 657         return result
 658
 659     # stores in self.module_name_varname the rpm variable to be used for the module's name
 660     # and the list of these names in self.varnames
 661     def spec_dict (self):
 662         specfile=self.main_specname()
 663         redirector_keys = [ varname for (varname,default) in Module.redirectors]
 664         redirect_dict = self.parse_spec(specfile,redirector_keys)
 665         if self.options.debug:
 666             print '1st pass parsing done, redirect_dict=',redirect_dict
 667         varnames=[]
 668         for (varname,default) in Module.redirectors:
 669             if redirect_dict.has_key(varname):
 670                 setattr(self,varname,redirect_dict[varname])
 671                 varnames += [redirect_dict[varname]]
 672             else:
 673                 setattr(self,varname,default)
 674                 varnames += [ default ]
 675         self.varnames = varnames
 676         result = self.parse_spec (specfile,self.varnames)
 677         if self.options.debug:
 678             print '2st pass parsing done, varnames=',varnames,'result=',result
 679         return result
 680
 681     def patch_spec_var (self, patch_dict,define_missing=False):
 682         for specfile in self.all_specnames():
 683             # record the keys that were changed
 684             changed = dict ( [ (x,False) for x in patch_dict.keys() ] )
 685             newspecfile=specfile+".new"
 686             if self.options.verbose:
 687                 print 'Patching',specfile,'for',patch_dict.keys()
 688             spec=open (specfile)
 689             new=open(newspecfile,"w")
 690
 691             for line in spec.readlines():
 692                 attempt=Module.matcher_rpm_define.match(line)
 693                 if attempt:
 694                     (define,var,value)=attempt.groups()
 695                     if var in patch_dict.keys():
 696                         if self.options.debug:
 697                             print 'rewriting %s as %s'%(var,patch_dict[var])
 698                         new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))
 699                         changed[var]=True
 700                         continue
 701                 new.write(line)
 702             if define_missing:
 703                 for (key,was_changed) in changed.iteritems():
 704                     if not was_changed:
 705                         if self.options.debug:
 706                             print 'rewriting missing %s as %s'%(key,patch_dict[key])
 707                         new.write('\n%%define %s %s\n'%(key,patch_dict[key]))
 708             spec.close()
 709             new.close()
 710             os.rename(newspecfile,specfile)
 711
 712     # returns all lines until the magic line
 713     def unignored_lines (self, logfile):
 714         result=[]
 715         white_line_matcher = re.compile("\A\s*\Z")
 716         for logline in file(logfile).readlines():
 717             if logline.strip() == Module.svn_magic_line:
 718                 break
 719             elif white_line_matcher.match(logline):
 720                 continue
 721             else:
 722                 result.append(logline.strip()+'\n')
 723         return result
 724
 725     # creates a copy of the input with only the unignored lines
 726     def stripped_magic_line_filename (self, filein, fileout ,new_tag_name):
 727        f=file(fileout,'w')
 728        f.write(self.setting_tag_format%new_tag_name + '\n')
 729        for line in self.unignored_lines(filein):
 730            f.write(line)
 731        f.close()
 732
 733     def insert_changelog (self, logfile, oldtag, newtag):
 734         for specfile in self.all_specnames():
 735             newspecfile=specfile+".new"
 736             if self.options.verbose:
 737                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
 738             spec=open (specfile)
 739             new=open(newspecfile,"w")
 740             for line in spec.readlines():
 741                 new.write(line)
 742                 if re.compile('%changelog').match(line):
 743                     dateformat="* %a %b %d %Y"
 744                     datepart=time.strftime(dateformat)
 745                     logpart="%s <%s> - %s"%(Module.config['username'],
 746                                                  Module.config['email'],
 747                                                  newtag)
 748                     new.write(datepart+" "+logpart+"\n")
 749                     for logline in self.unignored_lines(logfile):
 750                         new.write("- " + logline)
 751                     new.write("\n")
 752             spec.close()
 753             new.close()
 754             os.rename(newspecfile,specfile)
 755
 756     def show_dict (self, spec_dict):
 757         if self.options.verbose:
 758             for (k,v) in spec_dict.iteritems():
 759                 print k,'=',v
 760
 761     def last_tag (self, spec_dict):
 762         try:
 763             return "%s-%s" % (spec_dict[self.module_version_varname],
 764                               spec_dict[self.module_taglevel_varname])
 765         except KeyError,err:
 766             raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
 767
 768     def tag_name (self, spec_dict, old_svn_name=False):
 769         base_tag_name = self.name
 770         if old_svn_name:
 771             base_tag_name = git_to_svn_name(self.name)
 772         return "%s-%s" % (base_tag_name, self.last_tag(spec_dict))
 773
 774
 775 ##############################
 776     # using fine_grain means replacing only those instances that currently refer to this tag
 777     # otherwise, <module>-SVNPATH is replaced unconditionnally
 778     def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):
 779         newtagsfile=tagsfile+".new"
 780         tags=open (tagsfile)
 781         new=open(newtagsfile,"w")
 782
 783         matches=0
 784         # fine-grain : replace those lines that refer to oldname
 785         if fine_grain:
 786             if self.options.verbose:
 787                 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),
 788             matcher=re.compile("^(.*)%s(.*)"%oldname)
 789             for line in tags.readlines():
 790                 if not matcher.match(line):
 791                     new.write(line)
 792                 else:
 793                     (begin,end)=matcher.match(line).groups()
 794                     new.write(begin+newname+end+"\n")
 795                     matches += 1
 796         # brute-force : change uncommented lines that define <module>-SVNPATH
 797         else:
 798             if self.options.verbose:
 799                 print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile),
 800             pattern="\A\s*(?P<make_name>[^\s]+)-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\
 801                                           %(self.name)
 802             matcher_module=re.compile(pattern)
 803             for line in tags.readlines():
 804                 attempt=matcher_module.match(line)
 805                 if attempt:
 806                     svnpath="%s-SVNPATH"%(attempt.group('make_name'))
 807                     if self.options.verbose:
 808                         print ' '+svnpath,
 809                     replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)
 810                     new.write(replacement)
 811                     matches += 1
 812                 else:
 813                     new.write(line)
 814         tags.close()
 815         new.close()
 816         os.rename(newtagsfile,tagsfile)
 817         if self.options.verbose: print "%d changes"%matches
 818         return matches
 819
 820     def check_tag(self, tagname, need_it=False, old_svn_tag_name=None):
 821         if self.options.verbose:
 822             print "Checking %s repository tag: %s - " % (self.repository.type, tagname),
 823
 824         found_tagname = tagname
 825         found = self.repository.tag_exists(tagname)
 826         if not found and old_svn_tag_name:
 827             if self.options.verbose:
 828                 print "KO"
 829                 print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name),
 830             found = self.repository.tag_exists(old_svn_tag_name)
 831             if found:
 832                 found_tagname = old_svn_tag_name
 833
 834         if (found and need_it) or (not found and not need_it):
 835             print "OK "
 836         else:
 837             print "KO"
 838             if found:
 839                 raise Exception, "tag (%s) is already there" % tagname
 840             else:
 841                 raise Exception, "can not find required tag (%s)" % tagname
 842
 843         return found_tagname
 844
 845
 846 ##############################
 847     def do_tag (self):
 848         self.init_module_dir()
 849         self.revert_module_dir()
 850         self.update_module_dir()
 851         # parse specfile
 852         spec_dict = self.spec_dict()
 853         self.show_dict(spec_dict)
 854
 855         # side effects
 856         old_tag_name = self.tag_name(spec_dict)
 857         old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True)
 858
 859         if (self.options.new_version):
 860             # new version set on command line
 861             spec_dict[self.module_version_varname] = self.options.new_version
 862             spec_dict[self.module_taglevel_varname] = 0
 863         else:
 864             # increment taglevel
 865             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
 866             spec_dict[self.module_taglevel_varname] = new_taglevel
 867
 868         new_tag_name = self.tag_name(spec_dict)
 869
 870         # sanity check
 871         old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name)
 872         new_tag_name = self.check_tag(new_tag_name, need_it=False)
 873
 874         # checking for diffs
 875         diff_output = self.repository.diff_with_tag(old_tag_name)
 876         if len(diff_output) == 0:
 877             if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):
 878                 return
 879
 880         # side effect in trunk's specfile
 881         self.patch_spec_var(spec_dict)
 882
 883         # prepare changelog file
 884         # we use the standard subversion magic string (see svn_magic_line)
 885         # so we can provide useful information, such as version numbers and diff
 886         # in the same file
 887         changelog="/tmp/%s-%d.edit"%(self.name,os.getpid())
 888         changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid())
 889         setting_tag_line=Module.setting_tag_format%new_tag_name
 890         file(changelog,"w").write("""
 891 %s
 892 %s
 893 Please write a changelog for this new tag in the section above
 894 """%(Module.svn_magic_line,setting_tag_line))
 895
 896         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
 897             file(changelog,"a").write('DIFF=========\n' + diff_output)
 898
 899         if self.options.debug:
 900             prompt('Proceed ?')
 901
 902         # edit it
 903         self.run("%s %s"%(self.options.editor,changelog))
 904         # strip magic line in second file - looks like svn has changed its magic line with 1.6
 905         # so we do the job ourselves
 906         self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name)
 907         # insert changelog in spec
 908         if self.options.changelog:
 909             self.insert_changelog (changelog,old_tag_name,new_tag_name)
 910
 911         ## update build
 912         build_path = os.path.join(self.options.workdir,
 913                                   Module.config['build'])
 914         build = Repository(build_path, self.options)
 915         if self.options.build_branch:
 916             build.to_branch(self.options.build_branch)
 917         if not build.is_clean():
 918             build.revert()
 919
 920         tagsfiles=glob(build.path+"/*-tags*.mk")
 921         tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )
 922         default_answer = 'y'
 923         tagsfiles.sort()
 924         while True:
 925             for tagsfile in tagsfiles:
 926                 status=tagsdict[tagsfile]
 927                 basename=os.path.basename(tagsfile)
 928                 print ".................... Dealing with %s"%basename
 929                 while tagsdict[tagsfile] == 'todo' :
 930                     choice = prompt ("insert %s in %s    "%(new_tag_name,basename),default_answer,
 931                                      [ ('y','es'), ('n', 'ext'), ('f','orce'),
 932                                        ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
 933                                      allow_outside=True)
 934                     if choice == 'y':
 935                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)
 936                     elif choice == 'n':
 937                         print 'Done with %s'%os.path.basename(tagsfile)
 938                         tagsdict[tagsfile]='done'
 939                     elif choice == 'f':
 940                         self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)
 941                     elif choice == 'd':
 942                         print build.diff(f=tagsfile)
 943                     elif choice == 'r':
 944                         build.revert(f=tagsfile)
 945                     elif choice == 'c':
 946                         self.run("cat %s"%tagsfile)
 947                     else:
 948                         name=self.name
 949                         print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s
 950 f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s
 951 d: show current diff for this tag file
 952 r: revert that tag file
 953 c: cat the current tag file
 954 n: move to next file"""%locals()
 955
 956             if prompt("Want to review changes on tags files",False):
 957                 tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] )
 958                 default_answer='d'
 959             else:
 960                 break
 961
 962         def diff_all_changes():
 963             print build.diff()
 964             print self.repository.diff()
 965
 966         def commit_all_changes(log):
 967             self.repository.commit(log)
 968             build.commit(log)
 969
 970         self.run_prompt("Review module and build", diff_all_changes)
 971         self.run_prompt("Commit module and build", commit_all_changes, changelog_svn)
 972         self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn)
 973
 974         if self.options.debug:
 975             print 'Preserving',changelog,'and stripped',changelog_svn
 976         else:
 977             os.unlink(changelog)
 978             os.unlink(changelog_svn)
 979
 980
 981 ##############################
 982     def do_version (self):
 983         self.init_module_dir()
 984         self.revert_module_dir()
 985         self.update_module_dir()
 986         spec_dict = self.spec_dict()
 987         if self.options.www:
 988             self.html_store_title('Version for module %s (%s)' % (self.friendly_name(),
 989                                                                   self.last_tag(spec_dict)))
 990         for varname in self.varnames:
 991             if not spec_dict.has_key(varname):
 992                 self.html_print ('Could not find %%define for %s'%varname)
 993                 return
 994             else:
 995                 self.html_print ("%-16s %s"%(varname,spec_dict[varname]))
 996         if self.options.verbose:
 997             self.html_print ("%-16s %s"%('main specfile:',self.main_specname()))
 998             self.html_print ("%-16s %s"%('specfiles:',self.all_specnames()))
 999         self.html_print_end()
1000
1001
1002
1003 ##############################
1004     # store and restitute html fragments
1005     @staticmethod
1006     def html_href (url,text): return '<a href="%s">%s</a>'%(url,text)
1007
1008     @staticmethod
1009     def html_anchor (url,text): return '<a name="%s">%s</a>'%(url,text)
1010
1011     @staticmethod
1012     def html_quote (text):
1013         return text.replace('&','&#38;').replace('<','&lt;').replace('>','&gt;')
1014
1015     # only the fake error module has multiple titles
1016     def html_store_title (self, title):
1017         if not hasattr(self,'titles'): self.titles=[]
1018         self.titles.append(title)
1019
1020     def html_store_raw (self, html):
1021         if not hasattr(self,'body'): self.body=''
1022         self.body += html
1023
1024     def html_store_pre (self, text):
1025         if not hasattr(self,'body'): self.body=''
1026         self.body += '<pre>' + self.html_quote(text) + '</pre>'
1027
1028     def html_print (self, txt):
1029         if not self.options.www:
1030             print txt
1031         else:
1032             if not hasattr(self,'in_list') or not self.in_list:
1033                 self.html_store_raw('<ul>')
1034                 self.in_list=True
1035             self.html_store_raw('<li>'+txt+'</li>')
1036
1037     def html_print_end (self):
1038         if self.options.www:
1039             self.html_store_raw ('</ul>')
1040
1041     @staticmethod
1042     def html_dump_header(title):
1043         nowdate=time.strftime("%Y-%m-%d")
1044         nowtime=time.strftime("%H:%M (%Z)")
1045         print """
1046 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
1047 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
1048 <head>
1049 <title> %s </title>
1050 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
1051 <style type="text/css">
1052 body { font-family:georgia, serif; }
1053 h1 {font-size: large; }
1054 p.title {font-size: x-large; }
1055 span.error {text-weight:bold; color: red; }
1056 </style>
1057 </head>
1058 <body>
1059 <p class='title'> %s - status on %s at %s</p>
1060 <ul>
1061 """%(title,title,nowdate,nowtime)
1062
1063     @staticmethod
1064     def html_dump_middle():
1065         print "</ul>"
1066
1067     @staticmethod
1068     def html_dump_footer():
1069         print "</body></html"
1070
1071     def html_dump_toc(self):
1072         if hasattr(self,'titles'):
1073             for title in self.titles:
1074                 print '<li>',self.html_href ('#'+self.friendly_name(),title),'</li>'
1075
1076     def html_dump_body(self):
1077         if hasattr(self,'titles'):
1078             for title in self.titles:
1079                 print '<hr /><h1>',self.html_anchor(self.friendly_name(),title),'</h1>'
1080         if hasattr(self,'body'):
1081             print self.body
1082             print '<p class="top">',self.html_href('#','Back to top'),'</p>'
1083
1084
1085 ##############################
1086 class Main:
1087
1088     module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
1089
1090 module-tools : a set of tools to manage subversion tags and specfile
1091   requires the specfile to either
1092   * define *version* and *taglevel*
1093   OR alternatively
1094   * define redirection variables module_version_varname / module_taglevel_varname
1095 Trunk:
1096   by default, the trunk of modules is taken into account
1097   in this case, just mention the module name as <module_desc>
1098 Branches:
1099   if you wish to work on a branch rather than on the trunk,
1100   you can use something like e.g. Mom:2.1 as <module_desc>
1101 """
1102     release_usage="""Usage: %prog [options] tag1 .. tagn
1103   Extract release notes from the changes in specfiles between several build tags, latest first
1104   Examples:
1105       release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
1106   You can refer to a (build) branch by prepending a colon, like in
1107       release-changelog :4.2 4.2-rc25
1108   You can refer to the build trunk by just mentioning 'trunk', e.g.
1109       release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
1110 """
1111     common_usage="""More help:
1112   see http://svn.planet-lab.org/wiki/ModuleTools"""
1113
1114     modes={
1115         'list' : "displays a list of available tags or branches",
1116         'version' : "check latest specfile and print out details",
1117         'diff' : "show difference between module (trunk or branch) and latest tag",
1118         'tag'  : """increment taglevel in specfile, insert changelog in specfile,
1119                 create new tag and and monitor its adoption in build/*-tags*.mk""",
1120         'branch' : """create a branch for this module, from the latest tag on the trunk,
1121                   and change trunk's version number to reflect the new branch name;
1122                   you can specify the new branch name by using module:branch""",
1123         'sync' : """create a tag from the module
1124                 this is a last resort option, mostly for repairs""",
1125         'changelog' : """extract changelog between build tags
1126                 expected arguments are a list of tags""",
1127         }
1128
1129     silent_modes = ['list']
1130     release_modes = ['changelog']
1131
1132     @staticmethod
1133     def optparse_list (option, opt, value, parser):
1134         try:
1135             setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
1136         except:
1137             setattr(parser.values,option.dest,value.split())
1138
1139     def run(self):
1140
1141         mode=None
1142         for function in Main.modes.keys():
1143             if sys.argv[0].find(function) >= 0:
1144                 mode = function
1145                 break
1146         if not mode:
1147             print "Unsupported command",sys.argv[0]
1148             print "Supported commands:" + " ".join(Main.modes.keys())
1149             sys.exit(1)
1150
1151         if mode not in Main.release_modes:
1152             usage = Main.module_usage
1153             usage += Main.common_usage
1154             usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])
1155         else:
1156             usage = Main.release_usage
1157             usage += Main.common_usage
1158
1159         parser=OptionParser(usage=usage)
1160
1161         if mode == "tag" or mode == 'branch':
1162             parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1163                               help="set new version and reset taglevel to 0")
1164         if mode == "tag" :
1165             parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1166                               help="do not update changelog section in specfile when tagging")
1167             parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1168                               help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")
1169         if mode == "tag" or mode == "sync" :
1170             parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1171                               help="specify editor")
1172
1173         if mode in ["diff","version"] :
1174             parser.add_option("-W","--www", action="store", dest="www", default=False,
1175                               help="export diff in html format, e.g. -W trunk")
1176
1177         default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1178         parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1179                           help="dry run - shell commands are only displayed")
1180         parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1181                           default=[], nargs=1,type="string",
1182                           help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1183 -- can be set multiple times, or use quotes""")
1184
1185         parser.add_option("-w","--workdir", action="store", dest="workdir",
1186                           default="%s/%s"%(os.getenv("HOME"),"modules"),
1187                           help="""name for dedicated working dir - defaults to ~/modules
1188 ** THIS MUST NOT ** be your usual working directory""")
1189         parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1190                           help="skip safety checks, such as svn updates -- use with care")
1191
1192         # default verbosity depending on function - temp
1193         verbose_modes= ['tag', 'sync', 'branch']
1194
1195         if mode not in verbose_modes:
1196             parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1197                               help="run in verbose mode")
1198         else:
1199             parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1200                               help="run in quiet (non-verbose) mode")
1201         (options, args) = parser.parse_args()
1202         options.mode=mode
1203         if not hasattr(options,'dry_run'):
1204             options.dry_run=False
1205         if not hasattr(options,'www'):
1206             options.www=False
1207         options.debug=False
1208
1209         ########## module-*
1210         if len(args) == 0:
1211             parser.print_help()
1212             sys.exit(1)
1213         Module.init_homedir(options)
1214
1215         modules=[ Module(modname,options) for modname in args ]
1216         for module in modules:
1217             if len(args)>1 and mode not in Main.silent_modes:
1218                 print '========================================',module.friendly_name()
1219             # call the method called do_<mode>
1220             method=Module.__dict__["do_%s"%mode]
1221             try:
1222                 method(module)
1223             except Exception,e:
1224                 import traceback
1225                 traceback.print_exc()
1226                 print 'Skipping module %s: '%modname,e
1227
1228 ####################
1229 if __name__ == "__main__" :
1230     try:
1231         Main().run()
1232     except KeyboardInterrupt:
1233         print '\nBye'