X-Git-Url: http://git.onelab.eu/?p=build.git;a=blobdiff_plain;f=module-tools.py;h=e9f3096517c6e24c7198cde863ca3882cf7a01b3;hp=e7c17c169fb5ef16b2b5de1704538b75b7c1bc34;hb=HEAD;hpb=9360c88099f3d6bb1f9721ae94fa08c7b5ee494e diff --git a/module-tools.py b/module-tools.py index e7c17c16..7678bdce 100755 --- a/module-tools.py +++ b/module-tools.py @@ -1,79 +1,57 @@ -#!/usr/bin/python -u +#!/usr/bin/env python3 -import sys, os +import sys, os, os.path import re import time import tempfile from glob import glob from optparse import OptionParser -# HARDCODED NAME CHANGES -# -# Moving to git we decided to rename some of the repositories. Here is -# a map of name changes applied in git repositories. -RENAMED_SVN_MODULES = { - "PLEWWW": "plewww", - "PLCAPI": "plcapi", - "BootManager": "bootmanager", - "BootCD": "bootcd", - "VserverReference": "vserver-reference", - "BootstrapFS": "bootstrapfs", - "MyPLC": "myplc", - "CoDemux": "codemux", - "NodeManager": "nodemanager", - "NodeUpdate": "nodeupdate", - "Monitor": "monitor" - } - -def svn_to_git_name(module): - if RENAMED_SVN_MODULES.has_key(module): - return RENAMED_SVN_MODULES[module] - return module - -def git_to_svn_name(module): - for key in RENAMED_SVN_MODULES: - if module == RENAMED_SVN_MODULES[key]: - return key - return module - - -# e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase -def prompt (question,default=True,other_choices=[],allow_outside=False): - if not isinstance (other_choices,list): +# e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase +def prompt(question, default=True, other_choices=[], allow_outside=False): + if not isinstance(other_choices, list): other_choices = [ other_choices ] chars = [ c for (c,rest) in other_choices ] choices = [] if 'y' not in chars: - if default is True: choices.append('[y]') - else : choices.append('y') + if default is True: + choices.append('[y]') + else : + choices.append('y') if 'n' not in chars: - if default is False: choices.append('[n]') - else : choices.append('n') + if default is False: + choices.append('[n]') + else: + choices.append('n') - for (char,choice) in other_choices: + for char, choice in other_choices: if default == char: - choices.append("["+char+"]"+choice) + choices.append("[{}]{}".format(char, choice)) else: - choices.append("<"+char+">"+choice) + choices.append("<{}>{}>".format(char, choice)) try: - answer=raw_input(question + " " + "/".join(choices) + " ? ") + answer = input(question + " " + "/".join(choices) + " ? ") if not answer: return default - answer=answer[0].lower() + answer = answer[0].lower() if answer == 'y': - if 'y' in chars: return 'y' - else: return True + if 'y' in chars: + return 'y' + else: + return True elif answer == 'n': - if 'n' in chars: return 'n' - else: return False + if 'n' in chars: + return 'n' + else: + return False elif other_choices: for (char,choice) in other_choices: if answer == char: return char if allow_outside: return answer - return prompt(question,default,other_choices) + return prompt(question, default, other_choices) except: raise @@ -85,171 +63,75 @@ def default_editor(): return editor ### fold long lines -fold_length=132 +fold_length = 132 -def print_fold (line): +def print_fold(line): while len(line) >= fold_length: - print line[:fold_length],'\\' - line=line[fold_length:] - print line + print(line[:fold_length],'\\') + line = line[fold_length:] + print(line) class Command: - def __init__ (self,command,options): - self.command=command - self.options=options - self.tmp="/tmp/command-%d"%os.getpid() + def __init__(self, command, options): + self.command = command + self.options = options + self.tmp = "/tmp/command-{}".format(os.getpid()) - def run (self): + def run(self): if self.options.dry_run: - print 'dry_run',self.command + print('dry_run', self.command) return 0 if self.options.verbose and self.options.mode not in Main.silent_modes: - print '+',self.command + print('+', self.command) sys.stdout.flush() return os.system(self.command) - def run_silent (self): + def run_silent(self): if self.options.dry_run: - print 'dry_run',self.command + print('dry_run', self.command) return 0 if self.options.verbose: - print '+',self.command,' .. ', + print('>', os.getcwd()) + print('+', self.command, ' .. ', end=' ') sys.stdout.flush() - retcod=os.system(self.command + " &> " + self.tmp) + retcod = os.system("{} &> {}".format(self.command, self.tmp)) if retcod != 0: - print "FAILED ! -- out+err below (command was %s)"%self.command - os.system("cat " + self.tmp) - print "FAILED ! -- end of quoted output" + print("FAILED ! -- out+err below (command was {})".format(self.command)) + os.system("cat {}".format(self.tmp)) + print("FAILED ! -- end of quoted output") elif self.options.verbose: - print "OK" + print("OK") os.unlink(self.tmp) return retcod def run_fatal(self): - if self.run_silent() !=0: - raise Exception,"Command %s failed"%self.command + if self.run_silent() != 0: + raise Exception("Command {} failed".format(self.command)) # returns stdout, like bash's $(mycommand) - def output_of (self,with_stderr=False): + def output_of(self, with_stderr=False, binary=False): if self.options.dry_run: - print 'dry_run',self.command + print('dry_run', self.command) return 'dry_run output' - tmp="/tmp/status-%d"%os.getpid() + tmp="/tmp/status-{}".format(os.getpid()) if self.options.debug: - print '+',self.command,' .. ', + print('+',self.command,' .. ', end=' ') sys.stdout.flush() - command=self.command + command = self.command if with_stderr: command += " &> " else: command += " > " command += tmp os.system(command) - result=file(tmp).read() + mode = "r" if not binary else "rb" + with open(tmp, mode) as f: + result=f.read() os.unlink(tmp) if self.options.debug: - print 'Done', + print('Done', end=' ') return result - -class SvnRepository: - type = "svn" - - def __init__(self, path, options): - self.path = path - self.options = options - - def name(self): - return os.path.basename(self.path) - - def url(self): - out = Command("svn info %s" % self.path, self.options).output_of() - for line in out.split('\n'): - if line.startswith("URL:"): - return line.split()[1].strip() - - def repo_root(self): - out = Command("svn info %s" % self.path, self.options).output_of() - for line in out.split('\n'): - if line.startswith("Repository Root:"): - root = line.split()[2].strip() - return "%s/%s" % (root, self.name()) - - @classmethod - def checkout(cls, remote, local, options, recursive=False): - if recursive: - svncommand = "svn co %s %s" % (remote, local) - else: - svncommand = "svn co -N %s %s" % (remote, local) - Command("rm -rf %s" % local, options).run_silent() - Command(svncommand, options).run_fatal() - - return SvnRepository(local, options) - - @classmethod - def remote_exists(cls, remote): - return os.system("svn list %s &> /dev/null" % remote) == 0 - - def tag_exists(self, tagname): - url = "%s/tags/%s" % (self.repo_root(), tagname) - return SvnRepository.remote_exists(url) - - def update(self, subdir="", recursive=True, branch=None): - path = os.path.join(self.path, subdir) - if recursive: - svncommand = "svn up %s" % path - else: - svncommand = "svn up -N %s" % path - Command(svncommand, self.options).run_fatal() - - def commit(self, logfile): - # add all new files to the repository - Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" % - self.path, self.options).output_of() - Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal() - - def to_branch(self, branch): - remote = "%s/branches/%s" % (self.repo_root(), branch) - SvnRepository.checkout(remote, self.path, self.options, recursive=True) - - def to_tag(self, tag): - remote = "%s/tags/%s" % (self.repo_root(), branch) - SvnRepository.checkout(remote, self.path, self.options, recursive=True) - - def tag(self, tagname, logfile): - tag_url = "%s/tags/%s" % (self.repo_root(), tagname) - self_url = self.url() - Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal() - - def diff(self, f=""): - if f: - f = os.path.join(self.path, f) - else: - f = self.path - return Command("svn diff %s" % f, self.options).output_of(True) - - def diff_with_tag(self, tagname): - tag_url = "%s/tags/%s" % (self.repo_root(), tagname) - return Command("svn diff %s %s" % (tag_url, self.url()), - self.options).output_of(True) - - def revert(self, f=""): - if f: - Command("svn revert %s" % os.path.join(self.path, f), self.options).run_fatal() - else: - # revert all - Command("svn revert %s -R" % self.path, self.options).run_fatal() - Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " % - self.path, self.options).run_silent() - - def is_clean(self): - command="svn status %s" % self.path - return len(Command(command,self.options).output_of(True)) == 0 - - def is_valid(self): - return os.path.exists(os.path.join(self.path, ".svn")) - - class GitRepository: type = "git" @@ -264,9 +146,9 @@ class GitRepository: return self.repo_root() def gitweb(self): - c = Command("git show | grep commit | awk '{print $2;}'", self.options) + c = Command("git show | grep commit | awk '{{print $2;}}'", self.options) out = self.__run_in_repo(c.output_of).strip() - return "http://git.onelab.eu/?p=%s.git;a=commit;h=%s" % (self.name(), out) + return "http://git.onelab.eu/?p={}.git;a=commit;h={}".format(self.name(), out) def repo_root(self): c = Command("git remote show origin", self.options) @@ -276,17 +158,18 @@ class GitRepository: return line.split()[2] @classmethod - def checkout(cls, remote, local, options, depth=0): - Command("rm -rf %s" % local, options).run_silent() - Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal() + def clone(cls, remote, local, options, depth=None): + Command("rm -rf {}".format(local), options).run_silent() + depth_option = "" if depth is None else " --depth {}".format(depth) + Command("git clone{} {} {}".format(depth_option, remote, local), options).run_fatal() return GitRepository(local, options) @classmethod - def remote_exists(cls, remote): - return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0 + def remote_exists(cls, remote, options): + return Command("git --no-pager ls-remote {} &> /dev/null".format(remote), options).run()==0 def tag_exists(self, tagname): - command = 'git tag -l | grep "^%s$"' % tagname + command = 'git tag -l | grep "^{}$"'.format(tagname) c = Command(command, self.options) out = self.__run_in_repo(c.output_of, with_stderr=True) return len(out) > 0 @@ -306,7 +189,7 @@ class GitRepository: return self.__run_in_repo(c.run_fatal) def __is_commit_id(self, id): - c = Command("git show %s | grep commit | awk '{print $2;}'" % id, self.options) + c = Command("git show {} | grep commit | awk '{{print $2;}}'".format(id), self.options) ret = self.__run_in_repo(c.output_of, with_stderr=False) if ret.strip() == id: return True @@ -314,52 +197,58 @@ class GitRepository: def update(self, subdir=None, recursive=None, branch="master"): if branch == "master": - self.__run_command_in_repo("git checkout %s" % branch) + self.__run_command_in_repo("git checkout {}".format(branch)) else: self.to_branch(branch, remote=True) - self.__run_command_in_repo("git fetch origin --tags") + self.__run_command_in_repo("git fetch origin --tags", ignore_errors=True) self.__run_command_in_repo("git fetch origin") if not self.__is_commit_id(branch): - # we don't need to merge anythign for commit ids. - self.__run_command_in_repo("git merge --ff origin/%s" % branch) + # we don't need to merge anything for commit ids. + self.__run_command_in_repo("git merge --ff origin/{}".format(branch)) def to_branch(self, branch, remote=True): self.revert() if remote: - command = "git branch --track %s origin/%s" % (branch, branch) + command = "git branch --track {} origin/{}".format(branch, branch) c = Command(command, self.options) self.__run_in_repo(c.output_of, with_stderr=True) - return self.__run_command_in_repo("git checkout %s" % branch) + return self.__run_command_in_repo("git checkout {}".format(branch)) def to_tag(self, tag): self.revert() - return self.__run_command_in_repo("git checkout %s" % tag) + return self.__run_command_in_repo("git checkout {}".format(tag)) def tag(self, tagname, logfile): - self.__run_command_in_repo("git tag %s -F %s" % (tagname, logfile)) + self.__run_command_in_repo("git tag {} -F {}".format(tagname, logfile)) self.commit(logfile) def diff(self, f=""): - c = Command("git diff %s" % f, self.options) - return self.__run_in_repo(c.output_of, with_stderr=True) + c = Command("git diff {}".format(f), self.options) + try: + return self.__run_in_repo(c.output_of, with_stderr=True) + except: + return self.__run_in_repo(c.output_of, with_stderr=True, binary=True) def diff_with_tag(self, tagname): - c = Command("git diff %s" % tagname, self.options) - return self.__run_in_repo(c.output_of, with_stderr=True) + c = Command("git diff {}".format(tagname), self.options) + try: + return self.__run_in_repo(c.output_of, with_stderr=True) + except: + return self.__run_in_repo(c.output_of, with_stderr=True, binary=True) def commit(self, logfile, branch="master"): self.__run_command_in_repo("git add .", ignore_errors=True) self.__run_command_in_repo("git add -u", ignore_errors=True) - self.__run_command_in_repo("git commit -F %s" % logfile, ignore_errors=True) + self.__run_command_in_repo("git commit -F {}".format(logfile), ignore_errors=True) if branch == "master" or self.__is_commit_id(branch): self.__run_command_in_repo("git push") else: - self.__run_command_in_repo("git push origin %s:%s" % (branch, branch)) - self.__run_command_in_repo("git push --tags") + self.__run_command_in_repo("git push origin {}:{}".format(branch, branch)) + self.__run_command_in_repo("git push --tags", ignore_errors=True) def revert(self, f=""): if f: - self.__run_command_in_repo("git checkout %s" % f) + self.__run_command_in_repo("git checkout {}".format(f)) else: # revert all self.__run_command_in_repo("git --no-pager reset --hard") @@ -367,39 +256,34 @@ class GitRepository: def is_clean(self): def check_commit(): - command="git status" - s="nothing to commit (working directory clean)" + command = "git status" + s = "nothing to commit, working directory clean" return Command(command, self.options).output_of(True).find(s) >= 0 return self.__run_in_repo(check_commit) def is_valid(self): return os.path.exists(os.path.join(self.path, ".git")) - + class Repository: - """ Generic repository """ - supported_repo_types = [SvnRepository, GitRepository] + """ + Generic repository + From old times when we had svn and git + """ + supported_repo_types = [ GitRepository ] def __init__(self, path, options): self.path = path self.options = options - for repo in self.supported_repo_types: - self.repo = repo(self.path, self.options) + for repo_class in self.supported_repo_types: + self.repo = repo_class(self.path, self.options) if self.repo.is_valid(): break @classmethod - def has_moved_to_git(cls, module, config): - module = svn_to_git_name(module) - # check if the module is already in Git -# return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (config['svnpath'], module)) - return GitRepository.remote_exists(Module.git_remote_dir(module)) - - - @classmethod - def remote_exists(cls, remote): - for repo in Repository.supported_repo_types: - if repo.remote_exists(remote): + def remote_exists(cls, remote, options): + for repo_class in Repository.supported_repo_types: + if repo_class.remote_exists(remote, options): return True return False @@ -411,670 +295,650 @@ class Repository: # support for tagged module is minimal, and is for the Build class only class Module: - svn_magic_line="--This line, and those below, will be ignored--" - setting_tag_format = "Setting tag %s" - - redirectors=[ # ('module_name_varname','name'), - ('module_version_varname','version'), - ('module_taglevel_varname','taglevel'), ] + edit_magic_line = "--This line, and those below, will be ignored--" + setting_tag_format = "Setting tag {}" + + redirectors = [ + # ('module_name_varname', 'name'), + ('module_version_varname', 'version'), + ('module_taglevel_varname', 'taglevel'), + ] # where to store user's config - config_storage="CONFIG" - # - config={} - - import commands - configKeys=[ ('svnpath',"Enter your toplevel svnpath", - "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")), - ('gitserver', "Enter your git server's hostname", "git.onelab.eu"), - ('gituser', "Enter your user name (login name) on git server", commands.getoutput("id -un")), - ("build", "Enter the name of your build module","build"), - ('username',"Enter your firstname and lastname for changelogs",""), - ("email","Enter your email address for changelogs",""), - ] + config_storage = "CONFIG" + # + config = {} + + import subprocess + configKeys = [ + ('gitserver', "Enter your git server's hostname", "git.onelab.eu"), + ('gituser', "Enter your user name (login name) on git server", subprocess.getoutput("id -un")), + ("build", "Enter the name of your build module", "build"), + ('username', "Enter your firstname and lastname for changelogs", ""), + ("email", "Enter your email address for changelogs", ""), + ] @classmethod def prompt_config_option(cls, key, message, default): - cls.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default + cls.config[key]=input("{} [{}] : ".format(message,default)).strip() or default @classmethod - def prompt_config (cls): + def prompt_config(cls): for (key,message,default) in cls.configKeys: cls.config[key]="" while not cls.config[key]: cls.prompt_config_option(key, message, default) # for parsing module spec name:branch - matcher_branch_spec=re.compile("\A(?P[\w\.-]+):(?P[\w\.-]+)\Z") + matcher_branch_spec = re.compile("\A(?P[\w\.\-\/]+):(?P[\w\.\-]+)\Z") # special form for tagged module - for Build - matcher_tag_spec=re.compile("\A(?P[\w\.-]+)@(?P[\w\.-]+)\Z") + matcher_tag_spec = re.compile("\A(?P[\w\.\-\/]+)@(?P[\w\.\-]+)\Z") + # parsing specfiles - matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") + matcher_rpm_define = re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") @classmethod def parse_module_spec(cls, module_spec): name = branch_or_tagname = module_type = "" - attempt=Module.matcher_branch_spec.match(module_spec) + attempt = Module.matcher_branch_spec.match(module_spec) if attempt: module_type = "branch" name=attempt.group('name') branch_or_tagname=attempt.group('branch') else: - attempt=Module.matcher_tag_spec.match(module_spec) + attempt = Module.matcher_tag_spec.match(module_spec) if attempt: module_type = "tag" name=attempt.group('name') branch_or_tagname=attempt.group('tagname') else: - name=module_spec + name = module_spec return name, branch_or_tagname, module_type - def __init__ (self,module_spec,options): + def __init__(self, module_spec, options): # parse module spec - self.name, branch_or_tagname, module_type = self.parse_module_spec(module_spec) + self.pathname, branch_or_tagname, module_type = self.parse_module_spec(module_spec) + self.name = os.path.basename(self.pathname) if module_type == "branch": - self.branch=branch_or_tagname + self.branch = branch_or_tagname elif module_type == "tag": - self.tagname=branch_or_tagname - - # when available prefer to use git module name internally - self.name = svn_to_git_name(self.name) + self.tagname = branch_or_tagname self.options=options - self.module_dir="%s/%s"%(options.workdir,self.name) + self.module_dir="{}/{}".format(options.workdir,self.pathname) self.repository = None self.build = None - def run (self,command): + def run(self,command): return Command(command,self.options).run() - def run_fatal (self,command): + def run_fatal(self,command): return Command(command,self.options).run_fatal() - def run_prompt (self,message,fun, *args): - fun_msg = "%s(%s)" % (fun.func_name, ",".join(args)) + def run_prompt(self,message,fun, *args): + fun_msg = "{}({})".format(fun.__name__, ",".join(args)) if not self.options.verbose: while True: - choice=prompt(message,True,('s','how')) - if choice is True: + choice = prompt(message, True, ('s','how')) + if choice: fun(*args) return - elif choice is False: - print 'About to run function:', fun_msg + else: + print('About to run function:', fun_msg) else: - question=message+" - want to run function: " + fun_msg - if prompt(question,True): + question = "{} - want to run function: {}".format(message, fun_msg) + if prompt(question, True): fun(*args) - def friendly_name (self): - if hasattr(self,'branch'): - return "%s:%s"%(self.name,self.branch) - elif hasattr(self,'tagname'): - return "%s@%s"%(self.name,self.tagname) + def friendly_name(self): + if hasattr(self, 'branch'): + return "{}:{}".format(self.pathname, self.branch) + elif hasattr(self, 'tagname'): + return "{}@{}".format(self.pathname, self.tagname) else: - return self.name - - @classmethod - def git_remote_dir (cls, name): - return "%s@%s:/git/%s.git" % (cls.config['gituser'], cls.config['gitserver'], name) + return self.pathname @classmethod - def svn_remote_dir (cls, name): - name = git_to_svn_name(name) - svn = cls.config['svnpath'] - if svn.endswith('/'): - return "%s%s" % (svn, name) - return "%s/%s" % (svn, name) - - def svn_selected_remote(self): - svn_name = git_to_svn_name(self.name) - remote = self.svn_remote_dir(svn_name) - if hasattr(self,'branch'): - remote = "%s/branches/%s" % (remote, self.branch) - elif hasattr(self,'tagname'): - remote = "%s/tags/%s" % (remote, self.tagname) - else: - remote = "%s/trunk" % remote - return remote + def git_remote_dir(cls, name): + return "{}@{}:/git/{}.git".format(cls.config['gituser'], cls.config['gitserver'], name) #################### @classmethod - def init_homedir (cls, options): + def init_homedir(cls, options): if options.verbose and options.mode not in Main.silent_modes: - print 'Checking for', options.workdir - storage="%s/%s"%(options.workdir, cls.config_storage) + print('Checking for', options.workdir) + storage="{}/{}".format(options.workdir, cls.config_storage) # sanity check. Either the topdir exists AND we have a config/storage # or topdir does not exist and we create it - # to avoid people use their own daily svn repo + # to avoid people use their own daily work repo if os.path.isdir(options.workdir) and not os.path.isfile(storage): - print """The directory %s exists and has no CONFIG file + print("""The directory {} exists and has no CONFIG file If this is your regular working directory, please provide another one as the -module-* commands need a fresh working dir. Make sure that you do not use -that for other purposes than tagging""" % options.workdir +module-* commands need a fresh working dir. Make sure that you do not use +that for other purposes than tagging""".format(options.workdir)) sys.exit(1) - def checkout_build(): - print "Checking out build module..." + def clone_build(): + print("Checking out build module...") remote = cls.git_remote_dir(cls.config['build']) local = os.path.join(options.workdir, cls.config['build']) - GitRepository.checkout(remote, local, options, depth=1) - print "OK" + GitRepository.clone(remote, local, options, depth=1) + print("OK") def store_config(): - f=file(storage,"w") - for (key,message,default) in Module.configKeys: - f.write("%s=%s\n"%(key,Module.config[key])) - f.close() + with open(storage, 'w') as f: + for (key, message, default) in Module.configKeys: + f.write("{}={}\n".format(key, Module.config[key])) if options.debug: - print 'Stored',storage - Command("cat %s"%storage,options).run() + print('Stored', storage) + Command("cat {}".format(storage),options).run() def read_config(): # read config - f=open(storage) - for line in f.readlines(): - (key,value)=re.compile("^(.+)=(.+)$").match(line).groups() - Module.config[key]=value - f.close() - - if not os.path.isdir (options.workdir): - print "Cannot find",options.workdir,"let's create it" - Command("mkdir -p %s" % options.workdir, options).run_silent() + with open(storage) as f: + for line in f.readlines(): + key, value = re.compile("^(.+)=(.+)$").match(line).groups() + Module.config[key] = value + + # owerride config variables using options. + if options.build_module: + Module.config['build'] = options.build_module + + if not os.path.isdir(options.workdir): + print("Cannot find {}, let's create it".format(options.workdir)) + Command("mkdir -p {}".format(options.workdir), options).run_silent() cls.prompt_config() - checkout_build() + clone_build() store_config() else: read_config() # check missing config options old_layout = False - for (key,message,default) in cls.configKeys: - if not Module.config.has_key(key): - print "Configuration changed for module-tools" + for key, message, default in cls.configKeys: + if key not in Module.config: + print("Configuration changed for module-tools") cls.prompt_config_option(key, message, default) old_layout = True - + if old_layout: - Command("rm -rf %s" % options.workdir, options).run_silent() - Command("mkdir -p %s" % options.workdir, options).run_silent() - checkout_build() + Command("rm -rf {}".format(options.workdir), options).run_silent() + Command("mkdir -p {}".format(options.workdir), options).run_silent() + clone_build() store_config() build_dir = os.path.join(options.workdir, cls.config['build']) if not os.path.isdir(build_dir): - checkout_build() + clone_build() else: build = Repository(build_dir, options) if not build.is_clean(): - print "build module needs a revert" + print("build module needs a revert") build.revert() - print "OK" + print("OK") build.update() if options.verbose and options.mode not in Main.silent_modes: - print '******** Using config' + print('******** Using config') for (key,message,default) in Module.configKeys: - print '\t',key,'=',Module.config[key] + print('\t{} = {}'.format(key,Module.config[key])) - def init_module_dir (self): + def init_module_dir(self): if self.options.verbose: - print 'Checking for',self.module_dir + print('Checking for', self.module_dir) - if not os.path.isdir (self.module_dir): - if Repository.has_moved_to_git(self.name, Module.config): - self.repository = GitRepository.checkout(self.git_remote_dir(self.name), - self.module_dir, - self.options) - else: - remote = self.svn_selected_remote() - self.repository = SvnRepository.checkout(remote, - self.module_dir, - self.options, recursive=False) + if not os.path.isdir(self.module_dir): + self.repository = GitRepository.clone(self.git_remote_dir(self.pathname), + self.module_dir, + self.options) self.repository = Repository(self.module_dir, self.options) - if self.repository.type == "svn": - # check if module has moved to git - if Repository.has_moved_to_git(self.name, Module.config): - Command("rm -rf %s" % self.module_dir, self.options).run_silent() - self.init_module_dir() - # check if we have the required branch/tag - if self.repository.url() != self.svn_selected_remote(): - Command("rm -rf %s" % self.module_dir, self.options).run_silent() - self.init_module_dir() - - elif self.repository.type == "git": - if hasattr(self,'branch'): + + if self.repository.type == "git": + if hasattr(self, 'branch'): self.repository.to_branch(self.branch) - elif hasattr(self,'tagname'): + elif hasattr(self, 'tagname'): self.repository.to_tag(self.tagname) - else: - raise Exception, 'Cannot find %s - check module name'%self.module_dir + raise Exception('Cannot find {} - or not a git module'.format(self.module_dir)) - def revert_module_dir (self): + def revert_module_dir(self): if self.options.fast_checks: - if self.options.verbose: print 'Skipping revert of %s' % self.module_dir + if self.options.verbose: print('Skipping revert of {}'.format(self.module_dir)) return if self.options.verbose: - print 'Checking whether', self.module_dir, 'needs being reverted' - + print('Checking whether', self.module_dir, 'needs being reverted') + if not self.repository.is_clean(): self.repository.revert() - def update_module_dir (self): + def update_module_dir(self): if self.options.fast_checks: - if self.options.verbose: print 'Skipping update of %s' % self.module_dir + if self.options.verbose: print('Skipping update of {}'.format(self.module_dir)) return if self.options.verbose: - print 'Updating', self.module_dir + print('Updating', self.module_dir) - if hasattr(self,'branch'): + if hasattr(self, 'branch'): self.repository.update(branch=self.branch) - elif hasattr(self,'tagname'): + elif hasattr(self, 'tagname'): self.repository.update(branch=self.tagname) else: self.repository.update() - def main_specname (self): - attempt="%s/%s.spec"%(self.module_dir,self.name) - if os.path.isfile (attempt): + def main_specname(self): + attempt = "{}/{}.spec".format(self.module_dir, self.name) + if os.path.isfile(attempt): return attempt - pattern1="%s/*.spec"%self.module_dir - level1=glob(pattern1) + pattern1 = "{}/*.spec".format(self.module_dir) + level1 = glob(pattern1) if level1: return level1[0] - pattern2="%s/*/*.spec"%self.module_dir - level2=glob(pattern2) + pattern2 = "{}/*/*.spec".format(self.module_dir) + level2 = glob(pattern2) if level2: return level2[0] - raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2) + raise Exception('Cannot guess specfile for module {} -- patterns were {} or {}'\ + .format(self.pathname,pattern1,pattern2)) - def all_specnames (self): - level1=glob("%s/*.spec" % self.module_dir) - if level1: return level1 - level2=glob("%s/*/*.spec" % self.module_dir) + def all_specnames(self): + level1 = glob("{}/*.spec".format(self.module_dir)) + if level1: + return level1 + level2 = glob("{}/*/*.spec".format(self.module_dir)) return level2 - def parse_spec (self, specfile, varnames): + def parse_spec(self, specfile, varnames): if self.options.verbose: - print 'Parsing',specfile, + print('Parsing',specfile, end=' ') for var in varnames: - print "[%s]"%var, - print "" + print("[{}]".format(var), end=' ') + print("") result={} - f=open(specfile) - for line in f.readlines(): - attempt=Module.matcher_rpm_define.match(line) - if attempt: - (define,var,value)=attempt.groups() - if var in varnames: - result[var]=value - f.close() + with open(specfile, encoding='utf-8') as f: + for line in f.readlines(): + attempt = Module.matcher_rpm_define.match(line) + if attempt: + define, var, value = attempt.groups() + if var in varnames: + result[var] = value if self.options.debug: - print 'found',len(result),'keys' - for (k,v) in result.iteritems(): - print k,'=',v + print('found {} keys'.format(len(result))) + for k, v in result.items(): + print('{} = {}'.format(k, v)) return result - + # stores in self.module_name_varname the rpm variable to be used for the module's name # and the list of these names in self.varnames - def spec_dict (self): - specfile=self.main_specname() - redirector_keys = [ varname for (varname,default) in Module.redirectors] - redirect_dict = self.parse_spec(specfile,redirector_keys) + def spec_dict(self): + specfile = self.main_specname() + redirector_keys = [ varname for (varname, default) in Module.redirectors] + redirect_dict = self.parse_spec(specfile, redirector_keys) if self.options.debug: - print '1st pass parsing done, redirect_dict=',redirect_dict - varnames=[] - for (varname,default) in Module.redirectors: - if redirect_dict.has_key(varname): - setattr(self,varname,redirect_dict[varname]) + print('1st pass parsing done, redirect_dict=', redirect_dict) + varnames = [] + for varname, default in Module.redirectors: + if varname in redirect_dict: + setattr(self, varname, redirect_dict[varname]) varnames += [redirect_dict[varname]] else: - setattr(self,varname,default) - varnames += [ default ] + setattr(self, varname, default) + varnames += [ default ] self.varnames = varnames - result = self.parse_spec (specfile,self.varnames) + result = self.parse_spec(specfile, self.varnames) if self.options.debug: - print '2st pass parsing done, varnames=',varnames,'result=',result + print('2st pass parsing done, varnames={} result={}'.format(varnames, result)) return result - def patch_spec_var (self, patch_dict,define_missing=False): + def patch_spec_var(self, patch_dict,define_missing=False): for specfile in self.all_specnames(): # record the keys that were changed - changed = dict ( [ (x,False) for x in patch_dict.keys() ] ) - newspecfile=specfile+".new" + changed = {x : False for x in list(patch_dict.keys()) } + newspecfile = "{}.new".format(specfile) if self.options.verbose: - print 'Patching',specfile,'for',patch_dict.keys() - spec=open (specfile) - new=open(newspecfile,"w") - - for line in spec.readlines(): - attempt=Module.matcher_rpm_define.match(line) - if attempt: - (define,var,value)=attempt.groups() - if var in patch_dict.keys(): - if self.options.debug: - print 'rewriting %s as %s'%(var,patch_dict[var]) - new.write('%%%s %s %s\n'%(define,var,patch_dict[var])) - changed[var]=True - continue - new.write(line) - if define_missing: - for (key,was_changed) in changed.iteritems(): - if not was_changed: - if self.options.debug: - print 'rewriting missing %s as %s'%(key,patch_dict[key]) - new.write('\n%%define %s %s\n'%(key,patch_dict[key])) - spec.close() - new.close() - os.rename(newspecfile,specfile) + print('Patching', specfile, 'for', list(patch_dict.keys())) + + with open(specfile, encoding='utf-8') as spec: + with open(newspecfile, "w", encoding='utf-8') as new: + for line in spec.readlines(): + attempt = Module.matcher_rpm_define.match(line) + if attempt: + define, var, value = attempt.groups() + if var in list(patch_dict.keys()): + if self.options.debug: + print('rewriting {} as {}'.format(var, patch_dict[var])) + new.write('%{} {} {}\n'.format(define, var, patch_dict[var])) + changed[var] = True + continue + new.write(line) + if define_missing: + for key, was_changed in changed.items(): + if not was_changed: + if self.options.debug: + print('rewriting missing {} as {}'.format(key, patch_dict[key])) + new.write('\n%define {} {}\n'.format(key, patch_dict[key])) + os.rename(newspecfile, specfile) # returns all lines until the magic line - def unignored_lines (self, logfile): - result=[] + def unignored_lines(self, logfile): + result = [] white_line_matcher = re.compile("\A\s*\Z") - for logline in file(logfile).readlines(): - if logline.strip() == Module.svn_magic_line: - break - elif white_line_matcher.match(logline): - continue - else: - result.append(logline.strip()+'\n') + with open(logfile) as f: + for logline in f.readlines(): + if logline.strip() == Module.edit_magic_line: + break + elif white_line_matcher.match(logline): + continue + else: + result.append(logline.strip()+'\n') return result # creates a copy of the input with only the unignored lines - def stripped_magic_line_filename (self, filein, fileout ,new_tag_name): - f=file(fileout,'w') - f.write(self.setting_tag_format%new_tag_name + '\n') - for line in self.unignored_lines(filein): - f.write(line) - f.close() - - def insert_changelog (self, logfile, oldtag, newtag): + def strip_magic_line_filename(self, filein, fileout ,new_tag_name): + with open(fileout,'w') as f: + f.write(self.setting_tag_format.format(new_tag_name) + '\n') + for line in self.unignored_lines(filein): + f.write(line) + + def insert_changelog(self, logfile, newtag): for specfile in self.all_specnames(): - newspecfile=specfile+".new" + newspecfile = "{}.new".format(specfile) if self.options.verbose: - print 'Inserting changelog from %s into %s'%(logfile,specfile) - spec=open (specfile) - new=open(newspecfile,"w") - for line in spec.readlines(): - new.write(line) - if re.compile('%changelog').match(line): - dateformat="* %a %b %d %Y" - datepart=time.strftime(dateformat) - logpart="%s <%s> - %s"%(Module.config['username'], - Module.config['email'], - newtag) - new.write(datepart+" "+logpart+"\n") - for logline in self.unignored_lines(logfile): - new.write("- " + logline) - new.write("\n") - spec.close() - new.close() + print('Inserting changelog from {} into {}'.format(logfile, specfile)) + + with open(specfile) as spec: + with open(newspecfile,"w") as new: + for line in spec.readlines(): + new.write(line) + if re.compile('%changelog').match(line): + dateformat="* %a %b %d %Y" + datepart=time.strftime(dateformat) + logpart="{} <{}> - {}".format(Module.config['username'], + Module.config['email'], + newtag) + new.write("{} {}\n".format(datepart,logpart)) + for logline in self.unignored_lines(logfile): + new.write("- " + logline) + new.write("\n") os.rename(newspecfile,specfile) - - def show_dict (self, spec_dict): + + def show_dict(self, spec_dict): if self.options.verbose: - for (k,v) in spec_dict.iteritems(): - print k,'=',v + for k, v in spec_dict.items(): + print('{} = {}'.format(k, v)) - def last_tag (self, spec_dict): + def last_tag(self, spec_dict): try: - return "%s-%s" % (spec_dict[self.module_version_varname], - spec_dict[self.module_taglevel_varname]) - except KeyError,err: - raise Exception,'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err) - - def tag_name (self, spec_dict, old_svn_name=False): - base_tag_name = self.name - if old_svn_name: - base_tag_name = git_to_svn_name(self.name) - return "%s-%s" % (base_tag_name, self.last_tag(spec_dict)) - + return "{}-{}".format(spec_dict[self.module_version_varname], + spec_dict[self.module_taglevel_varname]) + except KeyError as err: + raise Exception('Something is wrong with module {}, cannot determine {} - exiting'\ + .format(self.name, err)) + + def tag_name(self, spec_dict): + return "{}-{}".format(self.name, self.last_tag(spec_dict)) + + + pattern_format="\A\s*{module}-(GITPATH)\s*(=|:=)\s*(?P[^\s]+)/{module}[^\s]+" + + def is_mentioned_in_tagsfile(self, tagsfile): + # so that {module} gets replaced from format + module = self.name + module_matcher = re.compile(Module.pattern_format.format(**locals())) + with open(tagsfile) as f: + for line in f.readlines(): + if module_matcher.match(line): + return True + return False ############################## # using fine_grain means replacing only those instances that currently refer to this tag - # otherwise, -{SVNPATH,GITPATH} is replaced unconditionnally - def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True): - newtagsfile=tagsfile+".new" - tags=open (tagsfile) - new=open(newtagsfile,"w") - - matches=0 - # fine-grain : replace those lines that refer to oldname - if fine_grain: - if self.options.verbose: - print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile), - matcher=re.compile("^(.*)%s(.*)"%oldname) - for line in tags.readlines(): - if not matcher.match(line): - new.write(line) - else: - (begin,end)=matcher.match(line).groups() - new.write(begin+newname+end+"\n") - matches += 1 - # brute-force : change uncommented lines that define -SVNPATH - else: - if self.options.verbose: - print 'Searching for -SVNPATH or -GITPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile), - pattern="\A\s*%s-(SVNPATH|GITPATH)\s*(=|:=)\s*(?P[^\s]+)/%s[^\s]+"\ - %(self.name,self.name) - matcher_module=re.compile(pattern) - for line in tags.readlines(): - attempt=matcher_module.match(line) - if attempt: - if line.find("-GITPATH") >= 0: - modulepath = "%s-GITPATH"%self.name - replacement = "%-32s:= %s/%s.git@%s\n"%(modulepath,attempt.group('url_main'),self.name,newname) - else: - modulepath = "%s-SVNPATH"%self.name - replacement = "%-32s:= %s/%s/tags/%s\n"%(modulepath,attempt.group('url_main'),self.name,newname) + # otherwise, -GITPATH is replaced unconditionnally + def patch_tags_file(self, tagsfile, oldname, newname, fine_grain=True): + newtagsfile = "{}.new".format(tagsfile) + + with open(tagsfile) as tags: + with open(newtagsfile,"w") as new: + matches = 0 + # fine-grain : replace those lines that refer to oldname + if fine_grain: if self.options.verbose: - print ' ' + modulepath, - new.write(replacement) - matches += 1 + print('Replacing {} into {}\n\tin {} .. '.format(oldname, newname, tagsfile), end=' ') + matcher = re.compile("^(.*){}(.*)".format(oldname)) + for line in tags.readlines(): + if not matcher.match(line): + new.write(line) + else: + begin, end = matcher.match(line).groups() + new.write(begin+newname+end+"\n") + matches += 1 + # brute-force : change uncommented lines that define -GITPATH else: - new.write(line) - tags.close() - new.close() + if self.options.verbose: + print('Searching for -GITPATH lines referring to /{}/\n\tin {} .. '\ + .format(self.pathname, tagsfile), end=' ') + # so that {module} gets replaced from format + module = self.name + module_matcher = re.compile(Module.pattern_format.format(**locals())) + for line in tags.readlines(): + attempt = module_matcher.match(line) + if attempt: + if line.find("-GITPATH") >= 0: + modulepath = "{}-GITPATH".format(self.name) + replacement = "{:<32}:= {}/{}.git@{}\n"\ + .format(modulepath, attempt.group('url_main'), self.pathname, newname) + else: + print("Could not locate {}-GITPATH (be aware that support for svn has been removed)"\ + .format(self.name)) + return + if self.options.verbose: + print(' ' + modulepath, end=' ') + new.write(replacement) + matches += 1 + else: + new.write(line) + os.rename(newtagsfile,tagsfile) - if self.options.verbose: print "%d changes"%matches + if self.options.verbose: + print("{} changes".format(matches)) return matches - def check_tag(self, tagname, need_it=False, old_svn_tag_name=None): + def check_tag(self, tagname, need_it=False): if self.options.verbose: - print "Checking %s repository tag: %s - " % (self.repository.type, tagname), + print("Checking {} repository tag: {} - ".format(self.repository.type, tagname), end=' ') found_tagname = tagname found = self.repository.tag_exists(tagname) - if not found and old_svn_tag_name: - if self.options.verbose: - print "KO" - print "Checking %s repository tag: %s - " % (self.repository.type, old_svn_tag_name), - found = self.repository.tag_exists(old_svn_tag_name) - if found: - found_tagname = old_svn_tag_name if (found and need_it) or (not found and not need_it): if self.options.verbose: - print "OK", - if found: print "- found" - else: print "- not found" + print("OK", end=' ') + print("- found" if found else "- not found") else: if self.options.verbose: - print "KO" - if found: - raise Exception, "tag (%s) is already there" % tagname - else: - raise Exception, "can not find required tag (%s)" % tagname + print("KO") + exception_format = "tag ({}) is already there" if found else "can not find required tag ({})" + raise Exception(exception_format.format(tagname)) return found_tagname ############################## - def do_tag (self): + def do_tag(self): self.init_module_dir() self.revert_module_dir() self.update_module_dir() # parse specfile spec_dict = self.spec_dict() self.show_dict(spec_dict) - - # side effects - old_tag_name = self.tag_name(spec_dict) - old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True) - if (self.options.new_version): + # compute previous tag - if not bypassed + if not self.options.bypass: + old_tag_name = self.tag_name(spec_dict) + # sanity check + old_tag_name = self.check_tag(old_tag_name, need_it=True) + + if self.options.new_version: # new version set on command line spec_dict[self.module_version_varname] = self.options.new_version spec_dict[self.module_taglevel_varname] = 0 else: # increment taglevel - new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1) + new_taglevel = str( int(spec_dict[self.module_taglevel_varname]) + 1) spec_dict[self.module_taglevel_varname] = new_taglevel new_tag_name = self.tag_name(spec_dict) - # sanity check - old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name) new_tag_name = self.check_tag(new_tag_name, need_it=False) # checking for diffs - diff_output = self.repository.diff_with_tag(old_tag_name) - if len(diff_output) == 0: - if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False): - return + if not self.options.bypass: + diff_output = self.repository.diff_with_tag(old_tag_name) + if len(diff_output) == 0: + if not prompt("No pending difference in module {}, want to tag anyway".format(self.pathname), False): + return - # side effect in trunk's specfile + # side effect in head's specfile self.patch_spec_var(spec_dict) - # prepare changelog file - # we use the standard subversion magic string (see svn_magic_line) + # prepare changelog file + # we use the standard subversion magic string (see edit_magic_line) # so we can provide useful information, such as version numbers and diff # in the same file - changelog="/tmp/%s-%d.edit"%(self.name,os.getpid()) - changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid()) - setting_tag_line=Module.setting_tag_format%new_tag_name - file(changelog,"w").write(""" -%s -%s + changelog_plain = "/tmp/{}-{}.edit".format(self.name, os.getpid()) + changelog_strip = "/tmp/{}-{}.strip".format(self.name, os.getpid()) + setting_tag_line = Module.setting_tag_format.format(new_tag_name) + with open(changelog_plain, 'w') as f: + f.write(""" +{} +{} Please write a changelog for this new tag in the section above -"""%(Module.svn_magic_line,setting_tag_line)) +""".format(Module.edit_magic_line, setting_tag_line)) + + if self.options.bypass: + pass + elif prompt('Want to see diffs while writing changelog', True): + with open(changelog_plain, "a") as f: + f.write('DIFF=========\n' + diff_output) - if not self.options.verbose or prompt('Want to see diffs while writing changelog',True): - file(changelog,"a").write('DIFF=========\n' + diff_output) - if self.options.debug: prompt('Proceed ?') - # edit it - self.run("%s %s"%(self.options.editor,changelog)) - # strip magic line in second file - looks like svn has changed its magic line with 1.6 - # so we do the job ourselves - self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name) + # edit it + self.run("{} {}".format(self.options.editor, changelog_plain)) + # strip magic line in second file + self.strip_magic_line_filename(changelog_plain, changelog_strip, new_tag_name) # insert changelog in spec if self.options.changelog: - self.insert_changelog (changelog,old_tag_name,new_tag_name) + self.insert_changelog(changelog_plain, new_tag_name) ## update build - build_path = os.path.join(self.options.workdir, - Module.config['build']) + build_path = os.path.join(self.options.workdir, Module.config['build']) build = Repository(build_path, self.options) if self.options.build_branch: build.to_branch(self.options.build_branch) if not build.is_clean(): build.revert() - tagsfiles=glob(build.path+"/*-tags*.mk") - tagsdict=dict( [ (x,'todo') for x in tagsfiles ] ) + tagsfiles = glob(build.path+"/*-tags.mk") + tagsdict = { x : 'todo' for x in tagsfiles } default_answer = 'y' tagsfiles.sort() while True: + # do not bother if in bypass mode + if self.options.bypass: + break for tagsfile in tagsfiles: - status=tagsdict[tagsfile] - basename=os.path.basename(tagsfile) - print ".................... Dealing with %s"%basename + if not self.is_mentioned_in_tagsfile(tagsfile): + if self.options.verbose: + print("tagsfile {} does not mention {} - skipped".format(tagsfile, self.name)) + continue + status = tagsdict[tagsfile] + basename = os.path.basename(tagsfile) + print(".................... Dealing with {}".format(basename)) while tagsdict[tagsfile] == 'todo' : - choice = prompt ("insert %s in %s "%(new_tag_name,basename),default_answer, - [ ('y','es'), ('n', 'ext'), ('f','orce'), + choice = prompt("insert {} in {} ".format(new_tag_name, basename), + default_answer, + [ ('y','es'), ('n', 'ext'), ('f','orce'), ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] , allow_outside=True) if choice == 'y': - self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True) + self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=True) elif choice == 'n': - print 'Done with %s'%os.path.basename(tagsfile) - tagsdict[tagsfile]='done' + print('Done with {}'.format(os.path.basename(tagsfile))) + tagsdict[tagsfile] = 'done' elif choice == 'f': - self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False) + self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=False) elif choice == 'd': - print build.diff(f=os.path.basename(tagsfile)) + print(build.diff(f=os.path.basename(tagsfile))) elif choice == 'r': build.revert(f=tagsfile) elif choice == 'c': - self.run("cat %s"%tagsfile) + self.run("cat {}".format(tagsfile)) else: - name=self.name - print """y: change %(name)s-{SVNPATH,GITPATH} only if it currently refers to %(old_tag_name)s -f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s + name = self.name + print( +"""y: change {name}-GITPATH only if it currently refers to {old_tag_name} +f: unconditionnally change any line that assigns {name}-GITPATH to using {new_tag_name} d: show current diff for this tag file r: revert that tag file c: cat the current tag file -n: move to next file"""%locals() +n: move to next file""".format(**locals())) - if prompt("Want to review changes on tags files",False): - tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] ) - default_answer='d' + if prompt("Want to review changes on tags files", False): + tagsdict = {x : 'todo' for x in tagsfiles } + default_answer = 'd' else: break def diff_all_changes(): - print build.diff() - print self.repository.diff() + print(build.diff()) + print(self.repository.diff()) def commit_all_changes(log): - if hasattr(self,'branch'): + if hasattr(self, 'branch'): self.repository.commit(log, branch=self.branch) else: self.repository.commit(log) build.commit(log) self.run_prompt("Review module and build", diff_all_changes) - self.run_prompt("Commit module and build", commit_all_changes, changelog_svn) - self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn) + self.run_prompt("Commit module and build", commit_all_changes, changelog_strip) + self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_strip) if self.options.debug: - print 'Preserving',changelog,'and stripped',changelog_svn + print('Preserving {} and stripped {}', changelog_plain, changelog_strip) else: - os.unlink(changelog) - os.unlink(changelog_svn) + os.unlink(changelog_plain) + os.unlink(changelog_strip) ############################## - def do_version (self): + def do_version(self): self.init_module_dir() self.revert_module_dir() self.update_module_dir() spec_dict = self.spec_dict() if self.options.www: - self.html_store_title('Version for module %s (%s)' % (self.friendly_name(), - self.last_tag(spec_dict))) + self.html_store_title('Version for module {} ({})'\ + .format(self.friendly_name(), self.last_tag(spec_dict))) for varname in self.varnames: - if not spec_dict.has_key(varname): - self.html_print ('Could not find %%define for %s'%varname) + if varname not in spec_dict: + self.html_print('Could not find %define for {}'.format(varname)) return else: - self.html_print ("%-16s %s"%(varname,spec_dict[varname])) - self.html_print ("%-16s %s"%('url',self.repository.url())) + self.html_print("{:<16} {}".format(varname, spec_dict[varname])) + self.html_print("{:<16} {}".format('url', self.repository.url())) if self.options.verbose: - self.html_print ("%-16s %s"%('main specfile:',self.main_specname())) - self.html_print ("%-16s %s"%('specfiles:',self.all_specnames())) + self.html_print("{:<16} {}".format('main specfile:', self.main_specname())) + self.html_print("{:<16} {}".format('specfiles:', self.all_specnames())) self.html_print_end() ############################## - def do_diff (self): + def do_diff(self): self.init_module_dir() self.revert_module_dir() self.update_module_dir() @@ -1083,118 +947,122 @@ n: move to next file"""%locals() # side effects tag_name = self.tag_name(spec_dict) - old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True) # sanity check - tag_name = self.check_tag(tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name) + tag_name = self.check_tag(tag_name, need_it=True) if self.options.verbose: - print 'Getting diff' + print('Getting diff') diff_output = self.repository.diff_with_tag(tag_name) if self.options.list: if diff_output: - print self.name + print(self.pathname) else: - thename=self.friendly_name() - do_print=False + thename = self.friendly_name() + do_print = False if self.options.www and diff_output: - self.html_store_title("Diffs in module %s (%s) : %d chars"%(\ - thename,self.last_tag(spec_dict),len(diff_output))) + self.html_store_title("Diffs in module {} ({}) : {} chars"\ + .format(thename, self.last_tag(spec_dict), len(diff_output))) - self.html_store_raw ('

< (left) %s

' % tag_name) - self.html_store_raw ('

> (right) %s

' % thename) - self.html_store_pre (diff_output) + self.html_store_raw('

< (left) {}

'.format(tag_name)) + self.html_store_raw('

> (right) {}

'.format(thename)) + self.html_store_pre(diff_output) elif not self.options.www: - print 'x'*30,'module',thename - print 'x'*20,'<',tag_name - print 'x'*20,'>',thename - print diff_output + print('x'*30, 'module', thename) + print('x'*20, '<', tag_name) + print('x'*20, '>', thename) + print(diff_output) ############################## # store and restitute html fragments - @staticmethod - def html_href (url,text): return '%s'%(url,text) + @staticmethod + def html_href(url,text): + return '{}'.format(url, text) - @staticmethod - def html_anchor (url,text): return '%s'%(url,text) + @staticmethod + def html_anchor(url,text): + return '{}'.format(url,text) @staticmethod - def html_quote (text): - return text.replace('&','&').replace('<','<').replace('>','>') + def html_quote(text): + return text.replace('&', '&').replace('<', '<').replace('>', '>') # only the fake error module has multiple titles - def html_store_title (self, title): - if not hasattr(self,'titles'): self.titles=[] + def html_store_title(self, title): + if not hasattr(self,'titles'): + self.titles=[] self.titles.append(title) - def html_store_raw (self, html): - if not hasattr(self,'body'): self.body='' + def html_store_raw(self, html): + if not hasattr(self,'body'): + self.body='' self.body += html - def html_store_pre (self, text): - if not hasattr(self,'body'): self.body='' - self.body += '
' + self.html_quote(text) + '
' + def html_store_pre(self, text): + if not hasattr(self,'body'): + self.body='' + self.body += '
{}
'.format(self.html_quote(text)) - def html_print (self, txt): + def html_print(self, txt): if not self.options.www: - print txt + print(txt) else: - if not hasattr(self,'in_list') or not self.in_list: + if not hasattr(self, 'in_list') or not self.in_list: self.html_store_raw('
    ') - self.in_list=True - self.html_store_raw('
  • '+txt+'
    • ') + self.in_list = True + self.html_store_raw('
  • {}
    • '.format(txt)) - def html_print_end (self): + def html_print_end(self): if self.options.www: - self.html_store_raw ('
') + self.html_store_raw('') @staticmethod def html_dump_header(title): - nowdate=time.strftime("%Y-%m-%d") - nowtime=time.strftime("%H:%M (%Z)") - print """ + nowdate = time.strftime("%Y-%m-%d") + nowtime = time.strftime("%H:%M (%Z)") + print(""" - %s + {} -

%s - status on %s at %s

+

{} - status on {} at {}

    -"""%(title,title,nowdate,nowtime) +""".format(title, title, nowdate, nowtime)) @staticmethod def html_dump_middle(): - print "
" + print("") @staticmethod def html_dump_footer(): - print "',self.html_href ('#'+self.friendly_name(),title),'' + print('
  • ', self.html_href('#'+self.friendly_name(),title), '
    • ') def html_dump_body(self): if hasattr(self,'titles'): for title in self.titles: - print '

    ',self.html_anchor(self.friendly_name(),title),'

    ' + print('

    ', self.html_anchor(self.friendly_name(),title), '

    ') if hasattr(self,'body'): - print self.body - print '

    ',self.html_href('#','Back to top'),'

    ' + print(self.body) + print('

    ', self.html_href('#','Back to top'), '

    ') class Build(Module): - + def __get_modules(self, tagfile): self.init_module_dir() modules = {} @@ -1203,9 +1071,8 @@ class Build(Module): for line in open(tagfile): try: name, url = line.split(':=') - name, git_or_svn_path = name.rsplit('-', 1) - name = svn_to_git_name(name.strip()) - modules[name] = (git_or_svn_path.strip(), url.strip()) + name, git_path = name.rsplit('-', 1) + modules[name] = (git_path.strip(), url.strip()) except: pass return modules @@ -1225,21 +1092,21 @@ class Build(Module): module_type = "tag" elif url.find('/branches/') >= 0: module_type = "branch" - + modules[module] = {"module_type" : module_type, "path_type": path_type, "tag_or_branch": tag_or_branch, "url":url} return modules - - + + def modules_diff(first, second): diff = {} for module in first: - if module not in second: - print "=== module %s missing in right-hand side ==="%module + if module not in second: + print("=== module {} missing in right-hand side ===".format(module)) continue if first[module]['tag_or_branch'] != second[module]['tag_or_branch']: diff[module] = (first[module]['tag_or_branch'], second[module]['tag_or_branch']) @@ -1254,91 +1121,104 @@ def modules_diff(first, second): def release_changelog(options, buildtag_old, buildtag_new): - tagfile = options.distrotags[0] - if not tagfile: - print "ERROR: provide a tagfile name (eg. onelab, onelab-k27, planetlab)" + # the command line expects new old, so we treat the tagfiles in the same order + nb_tags = len(options.distrotags) + if nb_tags == 1: + tagfile_new = tagfile_old = options.distrotags[0] + elif nb_tags == 2: + tagfile_new, tagfile_old = options.distrotags + else: + print("ERROR: provide one or two tagfile name (eg. onelab-k32-tags.mk)") + print("two tagfiles can be mentioned when a tagfile has been renamed") + return + + if options.dry_run: + print("------------------------------ Computing Changelog from") + print("buildtag_old", buildtag_old, "tagfile_old", tagfile_old) + print("buildtag_new", buildtag_new, "tagfile_new", tagfile_new) return - tagfile = "%s-tags.mk" % tagfile - - print '----' - print '----' - print '----' - print '= build tag %s to %s =' % (buildtag_old, buildtag_new) - print '== distro %s (%s to %s) ==' % (tagfile, buildtag_old, buildtag_new) - - build = Build("build@%s" % buildtag_old, options) + + print('----') + print('----') + print('----') + print('= build tag {} to {} ='.format(buildtag_old, buildtag_new)) + print('== distro {} ({} to {}) =='.format(tagfile_new, buildtag_old, buildtag_new)) + + build = Build("build@{}".format(buildtag_old), options) build.init_module_dir() - first = build.get_modules(tagfile) + first = build.get_modules(tagfile_old) - print ' * from', buildtag_old, build.repository.gitweb() + print(' * from', buildtag_old, build.repository.gitweb()) - build = Build("build@%s" % buildtag_new, options) + build = Build("build@{}".format(buildtag_new), options) build.init_module_dir() - second = build.get_modules(tagfile) + second = build.get_modules(tagfile_new) - print ' * to', buildtag_new, build.repository.gitweb() + print(' * to', buildtag_new, build.repository.gitweb()) diff, new_modules, removed_modules = modules_diff(first, second) def get_module(name, tag): if not tag or tag == "trunk": - return Module("%s" % (module), options) + return Module("{}".format(module), options) else: - return Module("%s@%s" % (module, tag), options) + return Module("{}@{}".format(module, tag), options) for module in diff: - print '=== %s - %s to %s : package %s ===' % (tagfile, buildtag_old, buildtag_new, module) + print('=== {} - {} to {} : package {} ==='.format(tagfile_new, buildtag_old, buildtag_new, module)) first, second = diff[module] m = get_module(module, first) - os.system('rm -rf %s' % m.module_dir) # cleanup module dir + os.system('rm -rf {}'.format(m.module_dir)) # cleanup module dir m.init_module_dir() - if m.repository.type == "svn": - print ' * from', first, m.repository.url() - else: - print ' * from', first, m.repository.gitweb() + print(' * from', first, m.repository.gitweb()) specfile = m.main_specname() (tmpfd, tmpfile) = tempfile.mkstemp() - os.system("cp -f /%s %s" % (specfile, tmpfile)) - + os.system("cp -f /{} {}".format(specfile, tmpfile)) + m = get_module(module, second) - m.init_module_dir() + # patch for ipfw that, being managed in a separate repo, won't work for now + try: + m.init_module_dir() + except Exception as e: + print("""Could not retrieve module {} - skipped +{{{{{{ {} }}}}}} +""".format( m.friendly_name(), e)) + continue specfile = m.main_specname() - if m.repository.type == "svn": - print ' * to', second, m.repository.url() - else: - print ' * to', second, m.repository.gitweb() + print(' * to', second, m.repository.gitweb()) - print '{{{' - os.system("diff -u %s %s" % (tmpfile, specfile)) - print '}}}' + print('{{{') + os.system("diff -u {} {} | sed -e 's,{},[[previous version]],'"\ + .format(tmpfile, specfile, tmpfile)) + print('}}}') os.unlink(tmpfile) for module in new_modules: - print '=== %s : new package in build %s ===' % (tagfile, module) + print('=== {} : new package in build {} ==='.format(tagfile_new, module)) for module in removed_modules: - print '=== %s : removed package from build %s ===' % (tagfile, module) + print('=== {} : removed package from build {} ==='.format(tagfile_new, module)) -def adopt_master (options, args): +def adopt_tag(options, args): modules=[] for module in options.modules: modules += module.split() - for module in modules: + for module in modules: modobj=Module(module,options) for tags_file in args: if options.verbose: - print 'adopting master for',module,'in',tags_file - modobj.patch_tags_file(tags_file,'_unused_','master',fine_grain=False) + print('adopting tag {} for {} in {}'.format(options.tag, module, tags_file)) + modobj.patch_tags_file(tags_file, '_unused_', options.tag, fine_grain=False) if options.verbose: - Command("git diff %s"%" ".join(args),options).run() + Command("git diff {}".format(" ".join(args)), options).run() ############################## class Main: @@ -1348,13 +1228,13 @@ class Main: module-tools : a set of tools to manage subversion tags and specfile requires the specfile to either * define *version* and *taglevel* - OR alternatively + OR alternatively * define redirection variables module_version_varname / module_taglevel_varname -Trunk: - by default, the trunk of modules is taken into account +Master: + by default, the 'master' branch of modules is the target in this case, just mention the module name as Branches: - if you wish to work on a branch rather than on the trunk, + if you wish to work on another branch, you can use something like e.g. Mom:2.1 as """ release_usage="""Usage: %prog [options] tag1 .. tagn @@ -1365,39 +1245,42 @@ Branches: release-changelog :4.2 4.2-rc25 You can refer to the build trunk by just mentioning 'trunk', e.g. release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk + You can use 2 different tagfile names if that was renamed meanwhile + release-changelog -t onelab-tags.mk 5.0-rc29 -t onelab-k32-tags.mk 5.0-rc28 """ - master_usage="""Usage: %prog [options] tag-file[s] - With this command you can adopt one or several masters in your tag files - This should be run in your daily build workdir; no call of git nor svn is done + adopt_usage="""Usage: %prog [options] tag-file[s] + With this command you can adopt a specifi tag or branch in your tag files + This should be run in your daily build workdir; no call of git is done Examples: - module-master -m "plewww plcapi" -m Monitor onelab*tags.mk + adopt-tag -m "plewww plcapi" -m Monitor onelab*tags.mk + adopt-tag -m sfa -t sfa-1.0-33 *tags.mk """ common_usage="""More help: see http://svn.planet-lab.org/wiki/ModuleTools""" - modes={ + modes = { 'list' : "displays a list of available tags or branches", 'version' : "check latest specfile and print out details", 'diff' : "show difference between module (trunk or branch) and latest tag", 'tag' : """increment taglevel in specfile, insert changelog in specfile, - create new tag and and monitor its adoption in build/*-tags*.mk""", - 'branch' : """create a branch for this module, from the latest tag on the trunk, + create new tag and and monitor its adoption in build/*-tags.mk""", + 'branch' : """create a branch for this module, from the latest tag on the trunk, and change trunk's version number to reflect the new branch name; you can specify the new branch name by using module:branch""", 'sync' : """create a tag from the module this is a last resort option, mostly for repairs""", 'changelog' : """extract changelog between build tags expected arguments are a list of tags""", - 'master' : """locally adopt master or trunk for some modules""", + 'adopt' : """locally adopt a specific tag""", } silent_modes = ['list'] # 'changelog' is for release-changelog - # 'master' is for 'adopt-master' - regular_modes = set(modes.keys()).difference(set(['changelog','master'])) + # 'adopt' is for 'adopt-tag' + regular_modes = set(modes.keys()).difference(set(['changelog','adopt'])) @staticmethod - def optparse_list (option, opt, value, parser): + def optparse_list(option, opt, value, parser): try: setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split()) except: @@ -1406,69 +1289,74 @@ Branches: def run(self): mode=None - for function in Main.modes.keys(): + # hack - need to check for adopt first as 'adopt-tag' contains tag.. + for function in [ 'adopt' ] + list(Main.modes.keys()): if sys.argv[0].find(function) >= 0: mode = function break if not mode: - print "Unsupported command",sys.argv[0] - print "Supported commands:" + " ".join(Main.modes.keys()) + print("Unsupported command",sys.argv[0]) + print("Supported commands:" + " ".join(list(Main.modes.keys()))) sys.exit(1) - usage='undefined usage, mode=%s'%mode + usage='undefined usage, mode={}'.format(mode) if mode in Main.regular_modes: usage = Main.module_usage usage += Main.common_usage - usage += "\nmodule-%s : %s"%(mode,Main.modes[mode]) - elif mode=='changelog': + usage += "\nmodule-{} : {}".format(mode, Main.modes[mode]) + elif mode == 'changelog': usage = Main.release_usage usage += Main.common_usage - elif mode=='master': - usage = Main.master_usage + elif mode == 'adopt': + usage = Main.adopt_usage usage += Main.common_usage parser=OptionParser(usage=usage) - - # the 'master' mode is really special and doesn't share any option - if mode=='master': + + # the 'adopt' mode is really special and doesn't share any option + if mode == 'adopt': parser.add_option("-m","--module",action="append",dest="modules",default=[], help="modules, can be used several times or with quotes") - parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + parser.add_option("-t","--tag",action="store", dest="tag", default='master', + help="specify the tag to adopt, default is 'master'") + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, help="run in verbose mode") (options, args) = parser.parse_args() options.workdir='unused' options.dry_run=False - options.mode='master' + options.mode='adopt' if len(args)==0 or len(options.modules)==0: parser.print_help() sys.exit(1) - adopt_master (options,args) - return + adopt_tag(options,args) + return # the other commands (module-* and release-changelog) share the same skeleton - if mode == "tag" or mode == 'branch': + if mode in [ 'tag', 'branch'] : parser.add_option("-s","--set-version",action="store",dest="new_version",default=None, help="set new version and reset taglevel to 0") - if mode == "tag" : + parser.add_option("-0","--bypass",action="store_true",dest="bypass",default=False, + help="skip checks on existence of the previous tag") + if mode == 'tag' : parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True, help="do not update changelog section in specfile when tagging") parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None, - help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place") - if mode == "tag" or mode == "sync" : + help="specify a build branch; used for locating the *tags.mk files where adoption is to take place") + if mode in [ 'tag', 'sync' ] : parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(), help="specify editor") - if mode in ["diff","version"] : + if mode in ['diff','version'] : parser.add_option("-W","--www", action="store", dest="www", default=False, help="export diff in html format, e.g. -W trunk") - if mode == "diff" : + if mode == 'diff' : parser.add_option("-l","--list", action="store_true", dest="list", default=False, help="just list modules that exhibit differences") - + default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list" parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, - help="run on all modules as found in %s"%default_modules_list) + help="run on all modules as found in {}".format(default_modules_list)) parser.add_option("-f","--file",action="store",dest="modules_list",default=None, help="run on all modules found in specified file") parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False, @@ -1478,23 +1366,25 @@ Branches: help="""specify distro-tags files, e.g. onelab-tags-4.2.mk -- can be set multiple times, or use quotes""") - parser.add_option("-w","--workdir", action="store", dest="workdir", - default="%s/%s"%(os.getenv("HOME"),"modules"), + parser.add_option("-w","--workdir", action="store", dest="workdir", + default="{}/{}".format(os.getenv("HOME"),"modules"), help="""name for dedicated working dir - defaults to ~/modules ** THIS MUST NOT ** be your usual working directory""") parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False, - help="skip safety checks, such as svn updates -- use with care") + help="skip safety checks, such as git pulls -- use with care") + parser.add_option("-B","--build-module",action="store",dest="build_module",default=None, + help="specify a build module to owerride the one in the CONFIG") # default verbosity depending on function - temp verbose_modes= ['tag', 'sync', 'branch'] - + if mode not in verbose_modes: - parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, help="run in verbose mode") else: parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True, help="run in quiet (non-verbose) mode") - (options, args) = parser.parse_args() + options, args = parser.parse_args() options.mode=mode if not hasattr(options,'dry_run'): options.dry_run=False @@ -1502,48 +1392,48 @@ Branches: options.www=False options.debug=False - + ########## module-* if len(args) == 0: if options.all_modules: options.modules_list=default_modules_list if options.modules_list: - args=Command("grep -v '#' %s"%options.modules_list,options).output_of().split() + args=Command("grep -v '#' {}".format(options.modules_list), options).output_of().split() else: parser.print_help() sys.exit(1) Module.init_homedir(options) - + if mode in Main.regular_modes: - modules=[ Module(modname,options) for modname in args ] + modules = [ Module(modname, options) for modname in args ] # hack: create a dummy Module to store errors/warnings error_module = Module('__errors__',options) for module in modules: if len(args)>1 and mode not in Main.silent_modes: if not options.www: - print '========================================',module.friendly_name() + print('========================================', module.friendly_name()) # call the method called do_ - method=Module.__dict__["do_%s"%mode] + method = Module.__dict__["do_{}".format(mode)] try: method(module) - except Exception,e: + except Exception as e: if options.www: - title=' Skipping module %s - failure: %s '%\ - (module.friendly_name(), str(e)) + title=' Skipping module {} - failure: {} '\ + .format(module.friendly_name(), str(e)) error_module.html_store_title(title) else: import traceback traceback.print_exc() - print 'Skipping module %s: '%modname,e - + print('Skipping module {}: {}'.format(module.name,e)) + if options.www: if mode == "diff": - modetitle="Changes to tag in %s"%options.www + modetitle="Changes to tag in {}".format(options.www) elif mode == "version": - modetitle="Latest tags in %s"%options.www + modetitle="Latest tags in {}".format(options.www) modules.append(error_module) error_module.html_dump_header(modetitle) for module in modules: @@ -1553,12 +1443,16 @@ Branches: module.html_dump_body() Module.html_dump_footer() else: - release_changelog(options, *args) - - + # if we provide, say a b c d, we want to build (a,b) (b,c) and (c,d) + # remember that the changelog in the twiki comes latest first, so + # we typically have here latest latest-1 latest-2 + for (tag_new,tag_old) in zip ( args[:-1], args [1:]): + release_changelog(options, tag_old, tag_new) + + #################### if __name__ == "__main__" : try: Main().run() except KeyboardInterrupt: - print '\nBye' + print('\nBye')