X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=module-tools.py;h=e9f3096517c6e24c7198cde863ca3882cf7a01b3;hb=9770ae10ecdfae9bdcaa1232ac47c94d4b1891ee;hp=787d757a559e5dafdf01faf7d21ae6c6b718386a;hpb=338106480fd7a529f93cc2f973840258c6d8f29f;p=build.git diff --git a/module-tools.py b/module-tools.py index 787d757a..e9f30965 100755 --- a/module-tools.py +++ b/module-tools.py @@ -1,50 +1,57 @@ -#!/usr/bin/python -u - -subversion_id = "$Id$" +#!/usr/bin/env python3 import sys, os, os.path import re import time +import tempfile from glob import glob from optparse import OptionParser # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase -def prompt (question,default=True,other_choices=[],allow_outside=False): - if not isinstance (other_choices,list): +def prompt(question, default=True, other_choices=[], allow_outside=False): + if not isinstance(other_choices, list): other_choices = [ other_choices ] chars = [ c for (c,rest) in other_choices ] choices = [] if 'y' not in chars: - if default is True: choices.append('[y]') - else : choices.append('y') + if default is True: + choices.append('[y]') + else : + choices.append('y') if 'n' not in chars: - if default is False: choices.append('[n]') - else : choices.append('n') + if default is False: + choices.append('[n]') + else: + choices.append('n') - for (char,choice) in other_choices: + for char, choice in other_choices: if default == char: - choices.append("["+char+"]"+choice) + choices.append("[{}]{}".format(char, choice)) else: - choices.append("<"+char+">"+choice) + choices.append("<{}>{}>".format(char, choice)) try: - answer=raw_input(question + " " + "/".join(choices) + " ? ") + answer = input(question + " " + "/".join(choices) + " ? ") if not answer: return default - answer=answer[0].lower() + answer = answer[0].lower() if answer == 'y': - if 'y' in chars: return 'y' - else: return True + if 'y' in chars: + return 'y' + else: + return True elif answer == 'n': - if 'n' in chars: return 'n' - else: return False + if 'n' in chars: + return 'n' + else: + return False elif other_choices: for (char,choice) in other_choices: if answer == char: return char if allow_outside: return answer - return prompt(question,default,other_choices) + return prompt(question, default, other_choices) except: raise @@ -56,1220 +63,1171 @@ def default_editor(): return editor ### fold long lines -fold_length=132 +fold_length = 132 -def print_fold (line): +def print_fold(line): while len(line) >= fold_length: - print line[:fold_length],'\\' - line=line[fold_length:] - print line + print(line[:fold_length],'\\') + line = line[fold_length:] + print(line) class Command: - def __init__ (self,command,options): - self.command=command - self.options=options - self.tmp="/tmp/command-%d"%os.getpid() + def __init__(self, command, options): + self.command = command + self.options = options + self.tmp = "/tmp/command-{}".format(os.getpid()) - def run (self): + def run(self): if self.options.dry_run: - print 'dry_run',self.command + print('dry_run', self.command) return 0 if self.options.verbose and self.options.mode not in Main.silent_modes: - print '+',self.command + print('+', self.command) sys.stdout.flush() return os.system(self.command) - def run_silent (self): + def run_silent(self): if self.options.dry_run: - print 'dry_run',self.command + print('dry_run', self.command) return 0 if self.options.verbose: - print '+',self.command,' .. ', + print('>', os.getcwd()) + print('+', self.command, ' .. ', end=' ') sys.stdout.flush() - retcod=os.system(self.command + " &> " + self.tmp) + retcod = os.system("{} &> {}".format(self.command, self.tmp)) if retcod != 0: - print "FAILED ! -- out+err below (command was %s)"%self.command - os.system("cat " + self.tmp) - print "FAILED ! -- end of quoted output" + print("FAILED ! -- out+err below (command was {})".format(self.command)) + os.system("cat {}".format(self.tmp)) + print("FAILED ! -- end of quoted output") elif self.options.verbose: - print "OK" + print("OK") os.unlink(self.tmp) return retcod def run_fatal(self): - if self.run_silent() !=0: - raise Exception,"Command %s failed"%self.command + if self.run_silent() != 0: + raise Exception("Command {} failed".format(self.command)) # returns stdout, like bash's $(mycommand) - def output_of (self,with_stderr=False): + def output_of(self, with_stderr=False): if self.options.dry_run: - print 'dry_run',self.command + print('dry_run', self.command) return 'dry_run output' - tmp="/tmp/status-%d"%os.getpid() + tmp="/tmp/status-{}".format(os.getpid()) if self.options.debug: - print '+',self.command,' .. ', + print('+',self.command,' .. ', end=' ') sys.stdout.flush() - command=self.command + command = self.command if with_stderr: command += " &> " else: command += " > " command += tmp os.system(command) - result=file(tmp).read() + with open(tmp) as f: + result=f.read() os.unlink(tmp) if self.options.debug: - print 'Done', + print('Done', end=' ') return result + +class GitRepository: + type = "git" + + def __init__(self, path, options): + self.path = path + self.options = options + + def name(self): + return os.path.basename(self.path) + + def url(self): + return self.repo_root() + + def gitweb(self): + c = Command("git show | grep commit | awk '{{print $2;}}'", self.options) + out = self.__run_in_repo(c.output_of).strip() + return "http://git.onelab.eu/?p={}.git;a=commit;h={}".format(self.name(), out) + + def repo_root(self): + c = Command("git remote show origin", self.options) + out = self.__run_in_repo(c.output_of) + for line in out.split('\n'): + if line.strip().startswith("Fetch URL:"): + return line.split()[2] + + @classmethod + def clone(cls, remote, local, options, depth=None): + Command("rm -rf {}".format(local), options).run_silent() + depth_option = "" if depth is None else " --depth {}".format(depth) + Command("git clone{} {} {}".format(depth_option, remote, local), options).run_fatal() + return GitRepository(local, options) + + @classmethod + def remote_exists(cls, remote, options): + return Command("git --no-pager ls-remote {} &> /dev/null".format(remote), options).run()==0 + + def tag_exists(self, tagname): + command = 'git tag -l | grep "^{}$"'.format(tagname) + c = Command(command, self.options) + out = self.__run_in_repo(c.output_of, with_stderr=True) + return len(out) > 0 + + def __run_in_repo(self, fun, *args, **kwargs): + cwd = os.getcwd() + os.chdir(self.path) + ret = fun(*args, **kwargs) + os.chdir(cwd) + return ret + + def __run_command_in_repo(self, command, ignore_errors=False): + c = Command(command, self.options) + if ignore_errors: + return self.__run_in_repo(c.output_of) + else: + return self.__run_in_repo(c.run_fatal) + + def __is_commit_id(self, id): + c = Command("git show {} | grep commit | awk '{{print $2;}}'".format(id), self.options) + ret = self.__run_in_repo(c.output_of, with_stderr=False) + if ret.strip() == id: + return True + return False + + def update(self, subdir=None, recursive=None, branch="master"): + if branch == "master": + self.__run_command_in_repo("git checkout {}".format(branch)) + else: + self.to_branch(branch, remote=True) + self.__run_command_in_repo("git fetch origin --tags") + self.__run_command_in_repo("git fetch origin") + if not self.__is_commit_id(branch): + # we don't need to merge anything for commit ids. + self.__run_command_in_repo("git merge --ff origin/{}".format(branch)) + + def to_branch(self, branch, remote=True): + self.revert() + if remote: + command = "git branch --track {} origin/{}".format(branch, branch) + c = Command(command, self.options) + self.__run_in_repo(c.output_of, with_stderr=True) + return self.__run_command_in_repo("git checkout {}".format(branch)) + + def to_tag(self, tag): + self.revert() + return self.__run_command_in_repo("git checkout {}".format(tag)) + + def tag(self, tagname, logfile): + self.__run_command_in_repo("git tag {} -F {}".format(tagname, logfile)) + self.commit(logfile) + + def diff(self, f=""): + c = Command("git diff {}".format(f), self.options) + return self.__run_in_repo(c.output_of, with_stderr=True) + + def diff_with_tag(self, tagname): + c = Command("git diff {}".format(tagname), self.options) + return self.__run_in_repo(c.output_of, with_stderr=True) + + def commit(self, logfile, branch="master"): + self.__run_command_in_repo("git add .", ignore_errors=True) + self.__run_command_in_repo("git add -u", ignore_errors=True) + self.__run_command_in_repo("git commit -F {}".format(logfile), ignore_errors=True) + if branch == "master" or self.__is_commit_id(branch): + self.__run_command_in_repo("git push") + else: + self.__run_command_in_repo("git push origin {}:{}".format(branch, branch)) + self.__run_command_in_repo("git push --tags") -class Svnpath: - def __init__(self,path,options): - self.path=path - self.options=options + def revert(self, f=""): + if f: + self.__run_command_in_repo("git checkout {}".format(f)) + else: + # revert all + self.__run_command_in_repo("git --no-pager reset --hard") + self.__run_command_in_repo("git --no-pager clean -f") + + def is_clean(self): + def check_commit(): + command = "git status" + s = "nothing to commit, working directory clean" + return Command(command, self.options).output_of(True).find(s) >= 0 + return self.__run_in_repo(check_commit) + + def is_valid(self): + return os.path.exists(os.path.join(self.path, ".git")) + + +class Repository: + """ + Generic repository + From old times when we had svn and git + """ + supported_repo_types = [ GitRepository ] + + def __init__(self, path, options): + self.path = path + self.options = options + for repo_class in self.supported_repo_types: + self.repo = repo_class(self.path, self.options) + if self.repo.is_valid(): + break + + @classmethod + def remote_exists(cls, remote, options): + for repo_class in Repository.supported_repo_types: + if repo_class.remote_exists(remote, options): + return True + return False + + def __getattr__(self, attr): + return getattr(self.repo, attr) - def url_exists (self): - return os.system("svn list %s &> /dev/null"%self.path) == 0 - def dir_needs_revert (self): - command="svn status %s"%self.path - return len(Command(command,self.options).output_of(True)) != 0 - # turns out it's the same implem. - def file_needs_commit (self): - command="svn status %s"%self.path - return len(Command(command,self.options).output_of(True)) != 0 # support for tagged module is minimal, and is for the Build class only class Module: - svn_magic_line="--This line, and those below, will be ignored--" - setting_tag_format = "Setting tag %s" + edit_magic_line = "--This line, and those below, will be ignored--" + setting_tag_format = "Setting tag {}" - redirectors=[ # ('module_name_varname','name'), - ('module_version_varname','version'), - ('module_taglevel_varname','taglevel'), ] + redirectors = [ + # ('module_name_varname', 'name'), + ('module_version_varname', 'version'), + ('module_taglevel_varname', 'taglevel'), + ] # where to store user's config - config_storage="CONFIG" + config_storage = "CONFIG" # - config={} - - import commands - configKeys=[ ('svnpath',"Enter your toplevel svnpath", - "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")), - ("build", "Enter the name of your build module","build"), - ('username',"Enter your firstname and lastname for changelogs",""), - ("email","Enter your email address for changelogs",""), - ] - - @staticmethod - def prompt_config (): - for (key,message,default) in Module.configKeys: - Module.config[key]="" - while not Module.config[key]: - Module.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default - + config = {} + + import subprocess + configKeys = [ + ('gitserver', "Enter your git server's hostname", "git.onelab.eu"), + ('gituser', "Enter your user name (login name) on git server", subprocess.getoutput("id -un")), + ("build", "Enter the name of your build module", "build"), + ('username', "Enter your firstname and lastname for changelogs", ""), + ("email", "Enter your email address for changelogs", ""), + ] + + @classmethod + def prompt_config_option(cls, key, message, default): + cls.config[key]=input("{} [{}] : ".format(message,default)).strip() or default + + @classmethod + def prompt_config(cls): + for (key,message,default) in cls.configKeys: + cls.config[key]="" + while not cls.config[key]: + cls.prompt_config_option(key, message, default) # for parsing module spec name:branch - matcher_branch_spec=re.compile("\A(?P[\w\.-]+):(?P[\w\.-]+)\Z") + matcher_branch_spec = re.compile("\A(?P[\w\.\-\/]+):(?P[\w\.\-]+)\Z") # special form for tagged module - for Build - matcher_tag_spec=re.compile("\A(?P[\w-]+)@(?P[\w\.-]+)\Z") + matcher_tag_spec = re.compile("\A(?P[\w\.\-\/]+)@(?P[\w\.\-]+)\Z") + # parsing specfiles - matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") + matcher_rpm_define = re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") - def __init__ (self,module_spec,options): - # parse module spec - attempt=Module.matcher_branch_spec.match(module_spec) + @classmethod + def parse_module_spec(cls, module_spec): + name = branch_or_tagname = module_type = "" + + attempt = Module.matcher_branch_spec.match(module_spec) if attempt: - self.name=attempt.group('name') - self.branch=attempt.group('branch') + module_type = "branch" + name=attempt.group('name') + branch_or_tagname=attempt.group('branch') else: - attempt=Module.matcher_tag_spec.match(module_spec) + attempt = Module.matcher_tag_spec.match(module_spec) if attempt: - self.name=attempt.group('name') - self.tagname=attempt.group('tagname') + module_type = "tag" + name=attempt.group('name') + branch_or_tagname=attempt.group('tagname') else: - self.name=module_spec + name = module_spec + return name, branch_or_tagname, module_type - self.options=options - self.module_dir="%s/%s"%(options.workdir,self.name) - def friendly_name (self): - if hasattr(self,'branch'): - return "%s:%s"%(self.name,self.branch) - elif hasattr(self,'tagname'): - return "%s@%s"%(self.name,self.tagname) - else: - return self.name + def __init__(self, module_spec, options): + # parse module spec + self.pathname, branch_or_tagname, module_type = self.parse_module_spec(module_spec) + self.name = os.path.basename(self.pathname) - def edge_dir (self): - if hasattr(self,'branch'): - return "%s/branches/%s"%(self.module_dir,self.branch) - elif hasattr(self,'tagname'): - return "%s/tags/%s"%(self.module_dir,self.tagname) - else: - return "%s/trunk"%(self.module_dir) + if module_type == "branch": + self.branch = branch_or_tagname + elif module_type == "tag": + self.tagname = branch_or_tagname - def tags_dir (self): - return "%s/tags"%(self.module_dir) + self.options=options + self.module_dir="{}/{}".format(options.workdir,self.pathname) + self.repository = None + self.build = None - def run (self,command): + def run(self,command): return Command(command,self.options).run() - def run_fatal (self,command): + def run_fatal(self,command): return Command(command,self.options).run_fatal() - def run_prompt (self,message,command): + def run_prompt(self,message,fun, *args): + fun_msg = "{}({})".format(fun.__name__, ",".join(args)) if not self.options.verbose: while True: - choice=prompt(message,True,('s','how')) - if choice is True: - self.run(command) - return - elif choice is False: + choice = prompt(message, True, ('s','how')) + if choice: + fun(*args) return else: - print 'About to run:',command + print('About to run function:', fun_msg) else: - question=message+" - want to run " + command - if prompt(question,True): - self.run(command) - - #################### - # store and restitute html fragments - @staticmethod - def html_href (url,text): return '%s'%(url,text) - @staticmethod - def html_anchor (url,text): return '%s'%(url,text) - # there must be some smarter means to do that - dirty for now - @staticmethod - def html_quote (text): - return text.replace('&','&').replace('<','<').replace('>','>') - - # only the fake error module has multiple titles - def html_store_title (self, title): - if not hasattr(self,'titles'): self.titles=[] - self.titles.append(title) - def html_store_raw (self, html): - if not hasattr(self,'body'): self.body='' - self.body += html - def html_store_pre (self, text): - if not hasattr(self,'body'): self.body='' - self.body += '
' + self.html_quote(text) + '
' - - def html_print (self, txt): - if not self.options.www: - print txt + question = "{} - want to run function: {}".format(message, fun_msg) + if prompt(question, True): + fun(*args) + + def friendly_name(self): + if hasattr(self, 'branch'): + return "{}:{}".format(self.pathname, self.branch) + elif hasattr(self, 'tagname'): + return "{}@{}".format(self.pathname, self.tagname) else: - if not hasattr(self,'in_list') or not self.in_list: - self.html_store_raw('
    ') - self.in_list=True - self.html_store_raw('
  • '+txt+'
    • ') - def html_print_end (self): - if self.options.www: - self.html_store_raw ('
') - - @staticmethod - def html_dump_header(title): - nowdate=time.strftime("%Y-%m-%d") - nowtime=time.strftime("%H:%M (%Z)") - print """ - - - - %s - - - - -

%s - status on %s at %s

-
    -"""%(title,title,nowdate,nowtime) - - @staticmethod - def html_dump_middle(): - print "
" - - @staticmethod - def html_dump_footer(): - print "',self.html_href ('#'+self.friendly_name(),title),'' + return self.pathname - def html_dump_body(self): - if hasattr(self,'titles'): - for title in self.titles: - print '

',self.html_anchor(self.friendly_name(),title),'

' - if hasattr(self,'body'): - print self.body - print '

',self.html_href('#','Back to top'),'

' + @classmethod + def git_remote_dir(cls, name): + return "{}@{}:/git/{}.git".format(cls.config['gituser'], cls.config['gitserver'], name) #################### - @staticmethod - def init_homedir (options): - topdir=options.workdir + @classmethod + def init_homedir(cls, options): if options.verbose and options.mode not in Main.silent_modes: - print 'Checking for',topdir - storage="%s/%s"%(topdir,Module.config_storage) + print('Checking for', options.workdir) + storage="{}/{}".format(options.workdir, cls.config_storage) # sanity check. Either the topdir exists AND we have a config/storage # or topdir does not exist and we create it - # to avoid people use their own daily svn repo - if os.path.isdir(topdir) and not os.path.isfile(storage): - print """The directory %s exists and has no CONFIG file + # to avoid people use their own daily work repo + if os.path.isdir(options.workdir) and not os.path.isfile(storage): + print("""The directory {} exists and has no CONFIG file If this is your regular working directory, please provide another one as the module-* commands need a fresh working dir. Make sure that you do not use -that for other purposes than tagging"""%topdir +that for other purposes than tagging""".format(options.workdir)) sys.exit(1) - if not os.path.isdir (topdir): - print "Cannot find",topdir,"let's create it" - Module.prompt_config() - print "Checking ...", - Command("svn co -N %s %s"%(Module.config['svnpath'],topdir),options).run_fatal() - Command("svn co %s/%s %s/%s"%(Module.config['svnpath'], - Module.config['build'], - topdir, - Module.config['build']),options).run_fatal() - print "OK" - - # store config - f=file(storage,"w") - for (key,message,default) in Module.configKeys: - f.write("%s=%s\n"%(key,Module.config[key])) - f.close() + + def clone_build(): + print("Checking out build module...") + remote = cls.git_remote_dir(cls.config['build']) + local = os.path.join(options.workdir, cls.config['build']) + GitRepository.clone(remote, local, options, depth=1) + print("OK") + + def store_config(): + with open(storage, 'w') as f: + for (key, message, default) in Module.configKeys: + f.write("{}={}\n".format(key, Module.config[key])) if options.debug: - print 'Stored',storage - Command("cat %s"%storage,options).run() - else: + print('Stored', storage) + Command("cat {}".format(storage),options).run() + + def read_config(): # read config - f=open(storage) - for line in f.readlines(): - (key,value)=re.compile("^(.+)=(.+)$").match(line).groups() - Module.config[key]=value - f.close() + with open(storage) as f: + for line in f.readlines(): + key, value = re.compile("^(.+)=(.+)$").match(line).groups() + Module.config[key] = value + + # owerride config variables using options. + if options.build_module: + Module.config['build'] = options.build_module + + if not os.path.isdir(options.workdir): + print("Cannot find {}, let's create it".format(options.workdir)) + Command("mkdir -p {}".format(options.workdir), options).run_silent() + cls.prompt_config() + clone_build() + store_config() + else: + read_config() + # check missing config options + old_layout = False + for key, message, default in cls.configKeys: + if key not in Module.config: + print("Configuration changed for module-tools") + cls.prompt_config_option(key, message, default) + old_layout = True + + if old_layout: + Command("rm -rf {}".format(options.workdir), options).run_silent() + Command("mkdir -p {}".format(options.workdir), options).run_silent() + clone_build() + store_config() + + build_dir = os.path.join(options.workdir, cls.config['build']) + if not os.path.isdir(build_dir): + clone_build() + else: + build = Repository(build_dir, options) + if not build.is_clean(): + print("build module needs a revert") + build.revert() + print("OK") + build.update() + if options.verbose and options.mode not in Main.silent_modes: - print '******** Using config' + print('******** Using config') for (key,message,default) in Module.configKeys: - print '\t',key,'=',Module.config[key] + print('\t{} = {}'.format(key,Module.config[key])) - def init_module_dir (self): + def init_module_dir(self): if self.options.verbose: - print 'Checking for',self.module_dir - if not os.path.isdir (self.module_dir): - self.run_fatal("svn update -N %s"%self.module_dir) - if not os.path.isdir (self.module_dir): - raise Exception, 'Cannot find %s - check module name'%self.module_dir + print('Checking for', self.module_dir) - def init_subdir (self,fullpath, deep=False): - if self.options.verbose: - print 'Checking for',fullpath - opt="" - if not deep: opt="-N" - if not os.path.isdir (fullpath): - self.run_fatal("svn update %s %s"%(opt,fullpath)) + if not os.path.isdir(self.module_dir): + self.repository = GitRepository.clone(self.git_remote_dir(self.pathname), + self.module_dir, + self.options) - def revert_subdir (self,fullpath): + self.repository = Repository(self.module_dir, self.options) + + if self.repository.type == "git": + if hasattr(self, 'branch'): + self.repository.to_branch(self.branch) + elif hasattr(self, 'tagname'): + self.repository.to_tag(self.tagname) + else: + raise Exception('Cannot find {} - or not a git module'.format(self.module_dir)) + + + def revert_module_dir(self): if self.options.fast_checks: - if self.options.verbose: print 'Skipping revert of %s'%fullpath + if self.options.verbose: print('Skipping revert of {}'.format(self.module_dir)) return if self.options.verbose: - print 'Checking whether',fullpath,'needs being reverted' - if Svnpath(fullpath,self.options).dir_needs_revert(): - self.run_fatal("svn revert -R %s"%fullpath) + print('Checking whether', self.module_dir, 'needs being reverted') + + if not self.repository.is_clean(): + self.repository.revert() - def update_subdir (self,fullpath): + def update_module_dir(self): if self.options.fast_checks: - if self.options.verbose: print 'Skipping update of %s'%fullpath + if self.options.verbose: print('Skipping update of {}'.format(self.module_dir)) return if self.options.verbose: - print 'Updating',fullpath - self.run_fatal("svn update %s"%fullpath) - - def init_edge_dir (self): - # if branch, edge_dir is two steps down - if hasattr(self,'branch'): - self.init_subdir("%s/branches"%self.module_dir,deep=False) - elif hasattr(self,'tagname'): - self.init_subdir("%s/tags"%self.module_dir,deep=False) - self.init_subdir(self.edge_dir(),deep=True) - - def revert_edge_dir (self): - self.revert_subdir(self.edge_dir()) - - def update_edge_dir (self): - self.update_subdir(self.edge_dir()) - - def main_specname (self): - attempt="%s/%s.spec"%(self.edge_dir(),self.name) - if os.path.isfile (attempt): + print('Updating', self.module_dir) + + if hasattr(self, 'branch'): + self.repository.update(branch=self.branch) + elif hasattr(self, 'tagname'): + self.repository.update(branch=self.tagname) + else: + self.repository.update() + + def main_specname(self): + attempt = "{}/{}.spec".format(self.module_dir, self.name) + if os.path.isfile(attempt): return attempt - pattern1="%s/*.spec"%self.edge_dir() - level1=glob(pattern1) + pattern1 = "{}/*.spec".format(self.module_dir) + level1 = glob(pattern1) if level1: return level1[0] - pattern2="%s/*/*.spec"%self.edge_dir() - level2=glob(pattern2) + pattern2 = "{}/*/*.spec".format(self.module_dir) + level2 = glob(pattern2) + if level2: return level2[0] - raise Exception, 'Cannot guess specfile for module %s -- patterns were %s or %s'%(self.name,pattern1,pattern2) + raise Exception('Cannot guess specfile for module {} -- patterns were {} or {}'\ + .format(self.pathname,pattern1,pattern2)) - def all_specnames (self): - level1=glob("%s/*.spec"%self.edge_dir()) - if level1: return level1 - level2=glob("%s/*/*.spec"%self.edge_dir()) + def all_specnames(self): + level1 = glob("{}/*.spec".format(self.module_dir)) + if level1: + return level1 + level2 = glob("{}/*/*.spec".format(self.module_dir)) return level2 - def parse_spec (self, specfile, varnames): + def parse_spec(self, specfile, varnames): if self.options.verbose: - print 'Parsing',specfile, + print('Parsing',specfile, end=' ') for var in varnames: - print "[%s]"%var, - print "" + print("[{}]".format(var), end=' ') + print("") result={} - f=open(specfile) - for line in f.readlines(): - attempt=Module.matcher_rpm_define.match(line) - if attempt: - (define,var,value)=attempt.groups() - if var in varnames: - result[var]=value - f.close() + with open(specfile, encoding='utf-8') as f: + for line in f.readlines(): + attempt = Module.matcher_rpm_define.match(line) + if attempt: + define, var, value = attempt.groups() + if var in varnames: + result[var] = value if self.options.debug: - print 'found',len(result),'keys' - for (k,v) in result.iteritems(): - print k,'=',v + print('found {} keys'.format(len(result))) + for k, v in result.items(): + print('{} = {}'.format(k, v)) return result # stores in self.module_name_varname the rpm variable to be used for the module's name # and the list of these names in self.varnames - def spec_dict (self): - specfile=self.main_specname() - redirector_keys = [ varname for (varname,default) in Module.redirectors] - redirect_dict = self.parse_spec(specfile,redirector_keys) + def spec_dict(self): + specfile = self.main_specname() + redirector_keys = [ varname for (varname, default) in Module.redirectors] + redirect_dict = self.parse_spec(specfile, redirector_keys) if self.options.debug: - print '1st pass parsing done, redirect_dict=',redirect_dict - varnames=[] - for (varname,default) in Module.redirectors: - if redirect_dict.has_key(varname): - setattr(self,varname,redirect_dict[varname]) + print('1st pass parsing done, redirect_dict=', redirect_dict) + varnames = [] + for varname, default in Module.redirectors: + if varname in redirect_dict: + setattr(self, varname, redirect_dict[varname]) varnames += [redirect_dict[varname]] else: - setattr(self,varname,default) + setattr(self, varname, default) varnames += [ default ] self.varnames = varnames - result = self.parse_spec (specfile,self.varnames) + result = self.parse_spec(specfile, self.varnames) if self.options.debug: - print '2st pass parsing done, varnames=',varnames,'result=',result + print('2st pass parsing done, varnames={} result={}'.format(varnames, result)) return result - def patch_spec_var (self, patch_dict,define_missing=False): + def patch_spec_var(self, patch_dict,define_missing=False): for specfile in self.all_specnames(): # record the keys that were changed - changed = dict ( [ (x,False) for x in patch_dict.keys() ] ) - newspecfile=specfile+".new" + changed = {x : False for x in list(patch_dict.keys()) } + newspecfile = "{}.new".format(specfile) if self.options.verbose: - print 'Patching',specfile,'for',patch_dict.keys() - spec=open (specfile) - new=open(newspecfile,"w") - - for line in spec.readlines(): - attempt=Module.matcher_rpm_define.match(line) - if attempt: - (define,var,value)=attempt.groups() - if var in patch_dict.keys(): - if self.options.debug: - print 'rewriting %s as %s'%(var,patch_dict[var]) - new.write('%%%s %s %s\n'%(define,var,patch_dict[var])) - changed[var]=True - continue - new.write(line) - if define_missing: - for (key,was_changed) in changed.iteritems(): - if not was_changed: - if self.options.debug: - print 'rewriting missing %s as %s'%(key,patch_dict[key]) - new.write('\n%%define %s %s\n'%(key,patch_dict[key])) - spec.close() - new.close() - os.rename(newspecfile,specfile) + print('Patching', specfile, 'for', list(patch_dict.keys())) + + with open(specfile, encoding='utf-8') as spec: + with open(newspecfile, "w", encoding='utf-8') as new: + for line in spec.readlines(): + attempt = Module.matcher_rpm_define.match(line) + if attempt: + define, var, value = attempt.groups() + if var in list(patch_dict.keys()): + if self.options.debug: + print('rewriting {} as {}'.format(var, patch_dict[var])) + new.write('%{} {} {}\n'.format(define, var, patch_dict[var])) + changed[var] = True + continue + new.write(line) + if define_missing: + for key, was_changed in changed.items(): + if not was_changed: + if self.options.debug: + print('rewriting missing {} as {}'.format(key, patch_dict[key])) + new.write('\n%define {} {}\n'.format(key, patch_dict[key])) + os.rename(newspecfile, specfile) # returns all lines until the magic line - def unignored_lines (self, logfile): - result=[] + def unignored_lines(self, logfile): + result = [] white_line_matcher = re.compile("\A\s*\Z") - for logline in file(logfile).readlines(): - if logline.strip() == Module.svn_magic_line: - break - elif white_line_matcher.match(logline): - continue - else: - result.append(logline.strip()+'\n') + with open(logfile) as f: + for logline in f.readlines(): + if logline.strip() == Module.edit_magic_line: + break + elif white_line_matcher.match(logline): + continue + else: + result.append(logline.strip()+'\n') return result # creates a copy of the input with only the unignored lines - def stripped_magic_line_filename (self, filein, fileout ,new_tag_name): - f=file(fileout,'w') - f.write(self.setting_tag_format%new_tag_name + '\n') - for line in self.unignored_lines(filein): - f.write(line) - f.close() - - def insert_changelog (self, logfile, oldtag, newtag): + def strip_magic_line_filename(self, filein, fileout ,new_tag_name): + with open(fileout,'w') as f: + f.write(self.setting_tag_format.format(new_tag_name) + '\n') + for line in self.unignored_lines(filein): + f.write(line) + + def insert_changelog(self, logfile, newtag): for specfile in self.all_specnames(): - newspecfile=specfile+".new" + newspecfile = "{}.new".format(specfile) if self.options.verbose: - print 'Inserting changelog from %s into %s'%(logfile,specfile) - spec=open (specfile) - new=open(newspecfile,"w") - for line in spec.readlines(): - new.write(line) - if re.compile('%changelog').match(line): - dateformat="* %a %b %d %Y" - datepart=time.strftime(dateformat) - logpart="%s <%s> - %s"%(Module.config['username'], - Module.config['email'], - newtag) - new.write(datepart+" "+logpart+"\n") - for logline in self.unignored_lines(logfile): - new.write("- " + logline) - new.write("\n") - spec.close() - new.close() + print('Inserting changelog from {} into {}'.format(logfile, specfile)) + + with open(specfile) as spec: + with open(newspecfile,"w") as new: + for line in spec.readlines(): + new.write(line) + if re.compile('%changelog').match(line): + dateformat="* %a %b %d %Y" + datepart=time.strftime(dateformat) + logpart="{} <{}> - {}".format(Module.config['username'], + Module.config['email'], + newtag) + new.write("{} {}\n".format(datepart,logpart)) + for logline in self.unignored_lines(logfile): + new.write("- " + logline) + new.write("\n") os.rename(newspecfile,specfile) - def show_dict (self, spec_dict): + def show_dict(self, spec_dict): if self.options.verbose: - for (k,v) in spec_dict.iteritems(): - print k,'=',v - - def mod_url (self): - return "%s/%s"%(Module.config['svnpath'],self.name) - - def edge_url (self): - if hasattr(self,'branch'): - return "%s/branches/%s"%(self.mod_url(),self.branch) - elif hasattr(self,'tagname'): - return "%s/tags/%s"%(self.mod_url(),self.tagname) - else: - return "%s/trunk"%(self.mod_url()) + for k, v in spec_dict.items(): + print('{} = {}'.format(k, v)) - def last_tag (self, spec_dict): - return "%s-%s"%(spec_dict[self.module_version_varname],spec_dict[self.module_taglevel_varname]) - - def tag_name (self, spec_dict): + def last_tag(self, spec_dict): try: - return "%s-%s"%(self.name, - self.last_tag(spec_dict)) - except KeyError,err: - raise Exception, 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err) - - def tag_url (self, spec_dict): - return "%s/tags/%s"%(self.mod_url(),self.tag_name(spec_dict)) - - def check_svnpath_exists (self, url, message): - if self.options.fast_checks: - return - if self.options.verbose: - print 'Checking url (%s) %s'%(url,message), - ok=Svnpath(url,self.options).url_exists() - if ok: - if self.options.verbose: print 'exists - OK' - else: - if self.options.verbose: print 'KO' - raise Exception, 'Could not find %s URL %s'%(message,url) + return "{}-{}".format(spec_dict[self.module_version_varname], + spec_dict[self.module_taglevel_varname]) + except KeyError as err: + raise Exception('Something is wrong with module {}, cannot determine {} - exiting'\ + .format(self.name, err)) + + def tag_name(self, spec_dict): + return "{}-{}".format(self.name, self.last_tag(spec_dict)) + - def check_svnpath_not_exists (self, url, message): - if self.options.fast_checks: - return - if self.options.verbose: - print 'Checking url (%s) %s'%(url,message), - ok=not Svnpath(url,self.options).url_exists() - if ok: - if self.options.verbose: print 'does not exist - OK' - else: - if self.options.verbose: print 'KO' - raise Exception, '%s URL %s already exists - exiting'%(message,url) + pattern_format="\A\s*{module}-(GITPATH)\s*(=|:=)\s*(?P[^\s]+)/{module}[^\s]+" - # locate specfile, parse it, check it and show values + def is_mentioned_in_tagsfile(self, tagsfile): + # so that {module} gets replaced from format + module = self.name + module_matcher = re.compile(Module.pattern_format.format(**locals())) + with open(tagsfile) as f: + for line in f.readlines(): + if module_matcher.match(line): + return True + return False ############################## - def do_version (self): - self.init_module_dir() - self.init_edge_dir() - self.revert_edge_dir() - self.update_edge_dir() - spec_dict = self.spec_dict() - if self.options.www: - self.html_store_title('Version for module %s (%s)' % (self.friendly_name(),self.last_tag(spec_dict))) - for varname in self.varnames: - if not spec_dict.has_key(varname): - self.html_print ('Could not find %%define for %s'%varname) - return - else: - self.html_print ("%-16s %s"%(varname,spec_dict[varname])) - if self.options.show_urls: - self.html_print ("%-16s %s"%('edge url',self.edge_url())) - self.html_print ("%-16s %s"%('latest tag url',self.tag_url(spec_dict))) + # using fine_grain means replacing only those instances that currently refer to this tag + # otherwise, -GITPATH is replaced unconditionnally + def patch_tags_file(self, tagsfile, oldname, newname, fine_grain=True): + newtagsfile = "{}.new".format(tagsfile) + + with open(tagsfile) as tags: + with open(newtagsfile,"w") as new: + matches = 0 + # fine-grain : replace those lines that refer to oldname + if fine_grain: + if self.options.verbose: + print('Replacing {} into {}\n\tin {} .. '.format(oldname, newname, tagsfile), end=' ') + matcher = re.compile("^(.*){}(.*)".format(oldname)) + for line in tags.readlines(): + if not matcher.match(line): + new.write(line) + else: + begin, end = matcher.match(line).groups() + new.write(begin+newname+end+"\n") + matches += 1 + # brute-force : change uncommented lines that define -GITPATH + else: + if self.options.verbose: + print('Searching for -GITPATH lines referring to /{}/\n\tin {} .. '\ + .format(self.pathname, tagsfile), end=' ') + # so that {module} gets replaced from format + module = self.name + module_matcher = re.compile(Module.pattern_format.format(**locals())) + for line in tags.readlines(): + attempt = module_matcher.match(line) + if attempt: + if line.find("-GITPATH") >= 0: + modulepath = "{}-GITPATH".format(self.name) + replacement = "{:<32}:= {}/{}.git@{}\n"\ + .format(modulepath, attempt.group('url_main'), self.pathname, newname) + else: + print("Could not locate {}-GITPATH (be aware that support for svn has been removed)"\ + .format(self.name)) + return + if self.options.verbose: + print(' ' + modulepath, end=' ') + new.write(replacement) + matches += 1 + else: + new.write(line) + + os.rename(newtagsfile,tagsfile) if self.options.verbose: - self.html_print ("%-16s %s"%('main specfile:',self.main_specname())) - self.html_print ("%-16s %s"%('specfiles:',self.all_specnames())) - self.html_print_end() - -############################## - def do_list (self): -# print 'verbose',self.options.verbose -# print 'list_tags',self.options.list_tags -# print 'list_branches',self.options.list_branches -# print 'all_modules',self.options.all_modules - - (verbose,branches,pattern,exact) = (self.options.verbose,self.options.list_branches, - self.options.list_pattern,self.options.list_exact) - - extra_command="" - extra_message="" - if hasattr(self,'branch'): - pattern=self.branch - if pattern or exact: - if exact: - if verbose: grep="%s/$"%exact - else: grep="^%s$"%exact - else: - grep=pattern - extra_command=" | grep %s"%grep - extra_message=" matching %s"%grep - - if not branches: - message="==================== tags for %s"%self.friendly_name() - command="svn list " - if verbose: command+="--verbose " - command += "%s/tags"%self.mod_url() - command += extra_command - message += extra_message - if verbose: print message - self.run(command) - - else: - message="==================== branches for %s"%self.friendly_name() - command="svn list " - if verbose: command+="--verbose " - command += "%s/branches"%self.mod_url() - command += extra_command - message += extra_message - if verbose: print message - self.run(command) - -############################## - sync_warning="""*** WARNING -The module-sync function has the following limitations -* it does not handle changelogs -* it does not scan the -tags*.mk files to adopt the new tags""" + print("{} changes".format(matches)) + return matches - def do_sync(self): + def check_tag(self, tagname, need_it=False): if self.options.verbose: - print Module.sync_warning - if not prompt('Want to proceed anyway'): - return + print("Checking {} repository tag: {} - ".format(self.repository.type, tagname), end=' ') - self.init_module_dir() - self.init_edge_dir() - self.revert_edge_dir() - self.update_edge_dir() - spec_dict = self.spec_dict() - - edge_url=self.edge_url() - tag_name=self.tag_name(spec_dict) - tag_url=self.tag_url(spec_dict) - # check the tag does not exist yet - self.check_svnpath_not_exists(tag_url,"new tag") + found_tagname = tagname + found = self.repository.tag_exists(tagname) - if self.options.message: - svnopt='--message "%s"'%self.options.message + if (found and need_it) or (not found and not need_it): + if self.options.verbose: + print("OK", end=' ') + print("- found" if found else "- not found") else: - svnopt='--editor-cmd=%s'%self.options.editor - self.run_prompt("Create initial tag", - "svn copy %s %s %s"%(svnopt,edge_url,tag_url)) + if self.options.verbose: + print("KO") + exception_format = "tag ({}) is already there" if found else "can not find required tag ({})" + raise Exception(exception_format.format(tagname)) -############################## - def do_diff (self,compute_only=False): - self.init_module_dir() - self.init_edge_dir() - self.revert_edge_dir() - self.update_edge_dir() - spec_dict = self.spec_dict() - self.show_dict(spec_dict) + return found_tagname - edge_url=self.edge_url() - tag_url=self.tag_url(spec_dict) - self.check_svnpath_exists(edge_url,"edge track") - self.check_svnpath_exists(tag_url,"latest tag") - command="svn diff %s %s"%(tag_url,edge_url) - if compute_only: - if self.options.verbose: - print 'Getting diff with %s'%command - diff_output = Command(command,self.options).output_of() - # if used as a utility - if compute_only: - return (spec_dict,edge_url,tag_url,diff_output) - # otherwise print the result - if self.options.list: - if diff_output: - print self.name - else: - thename=self.friendly_name() - do_print=False - if self.options.www and diff_output: - self.html_store_title("Diffs in module %s (%s) : %d chars"%(\ - thename,self.last_tag(spec_dict),len(diff_output))) - link=self.html_href(tag_url,tag_url) - self.html_store_raw ('

< (left) %s

'%link) - link=self.html_href(edge_url,edge_url) - self.html_store_raw ('

> (right) %s

'%link) - self.html_store_pre (diff_output) - elif not self.options.www: - print 'x'*30,'module',thename - print 'x'*20,'<',tag_url - print 'x'*20,'>',edge_url - print diff_output ############################## - # using fine_grain means replacing only those instances that currently refer to this tag - # otherwise, -SVNPATH is replaced unconditionnally - def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True): - newtagsfile=tagsfile+".new" - tags=open (tagsfile) - new=open(newtagsfile,"w") - - matches=0 - # fine-grain : replace those lines that refer to oldname - if fine_grain: - if self.options.verbose: - print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile), - matcher=re.compile("^(.*)%s(.*)"%oldname) - for line in tags.readlines(): - if not matcher.match(line): - new.write(line) - else: - (begin,end)=matcher.match(line).groups() - new.write(begin+newname+end+"\n") - matches += 1 - # brute-force : change uncommented lines that define -SVNPATH - else: - if self.options.verbose: - print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile), - pattern="\A\s*(?P[^\s]+)-SVNPATH\s*(=|:=)\s*(?P[^\s]+)/%s/[^\s]+"\ - %(self.name) - matcher_module=re.compile(pattern) - for line in tags.readlines(): - attempt=matcher_module.match(line) - if attempt: - svnpath="%s-SVNPATH"%(attempt.group('make_name')) - if self.options.verbose: - print ' '+svnpath, - replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname) - new.write(replacement) - matches += 1 - else: - new.write(line) - tags.close() - new.close() - os.rename(newtagsfile,tagsfile) - if self.options.verbose: print "%d changes"%matches - return matches - - def do_tag (self): + def do_tag(self): self.init_module_dir() - self.init_edge_dir() - self.revert_edge_dir() - self.update_edge_dir() + self.revert_module_dir() + self.update_module_dir() # parse specfile spec_dict = self.spec_dict() self.show_dict(spec_dict) - # side effects - edge_url=self.edge_url() - old_tag_name = self.tag_name(spec_dict) - old_tag_url=self.tag_url(spec_dict) - if (self.options.new_version): + # compute previous tag - if not bypassed + if not self.options.bypass: + old_tag_name = self.tag_name(spec_dict) + # sanity check + old_tag_name = self.check_tag(old_tag_name, need_it=True) + + if self.options.new_version: # new version set on command line spec_dict[self.module_version_varname] = self.options.new_version spec_dict[self.module_taglevel_varname] = 0 else: # increment taglevel - new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1) + new_taglevel = str( int(spec_dict[self.module_taglevel_varname]) + 1) spec_dict[self.module_taglevel_varname] = new_taglevel - # sanity check new_tag_name = self.tag_name(spec_dict) - new_tag_url=self.tag_url(spec_dict) - self.check_svnpath_exists (edge_url,"edge track") - self.check_svnpath_exists (old_tag_url,"previous tag") - self.check_svnpath_not_exists (new_tag_url,"new tag") + # sanity check + new_tag_name = self.check_tag(new_tag_name, need_it=False) # checking for diffs - diff_output=Command("svn diff %s %s"%(old_tag_url,edge_url), - self.options).output_of() - if len(diff_output) == 0: - if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False): - return + if not self.options.bypass: + diff_output = self.repository.diff_with_tag(old_tag_name) + if len(diff_output) == 0: + if not prompt("No pending difference in module {}, want to tag anyway".format(self.pathname), False): + return - # side effect in trunk's specfile + # side effect in head's specfile self.patch_spec_var(spec_dict) # prepare changelog file - # we use the standard subversion magic string (see svn_magic_line) + # we use the standard subversion magic string (see edit_magic_line) # so we can provide useful information, such as version numbers and diff # in the same file - changelog="/tmp/%s-%d.edit"%(self.name,os.getpid()) - changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid()) - setting_tag_line=Module.setting_tag_format%new_tag_name - file(changelog,"w").write(""" -%s -%s + changelog_plain = "/tmp/{}-{}.edit".format(self.name, os.getpid()) + changelog_strip = "/tmp/{}-{}.strip".format(self.name, os.getpid()) + setting_tag_line = Module.setting_tag_format.format(new_tag_name) + with open(changelog_plain, 'w') as f: + f.write(""" +{} +{} Please write a changelog for this new tag in the section above -"""%(Module.svn_magic_line,setting_tag_line)) +""".format(Module.edit_magic_line, setting_tag_line)) - if not self.options.verbose or prompt('Want to see diffs while writing changelog',True): - file(changelog,"a").write('DIFF=========\n' + diff_output) + if self.options.bypass: + pass + elif prompt('Want to see diffs while writing changelog', True): + with open(changelog_plain, "a") as f: + f.write('DIFF=========\n' + diff_output) if self.options.debug: prompt('Proceed ?') # edit it - self.run("%s %s"%(self.options.editor,changelog)) - # strip magic line in second file - looks like svn has changed its magic line with 1.6 - # so we do the job ourselves - self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name) + self.run("{} {}".format(self.options.editor, changelog_plain)) + # strip magic line in second file + self.strip_magic_line_filename(changelog_plain, changelog_strip, new_tag_name) # insert changelog in spec if self.options.changelog: - self.insert_changelog (changelog,old_tag_name,new_tag_name) + self.insert_changelog(changelog_plain, new_tag_name) ## update build - try: - buildname=Module.config['build'] - except: - buildname="build" + build_path = os.path.join(self.options.workdir, Module.config['build']) + build = Repository(build_path, self.options) if self.options.build_branch: - buildname+=":"+self.options.build_branch - build = Module(buildname,self.options) - build.init_module_dir() - build.init_edge_dir() - build.revert_edge_dir() - build.update_edge_dir() - - tagsfiles=glob(build.edge_dir()+"/*-tags*.mk") - tagsdict=dict( [ (x,'todo') for x in tagsfiles ] ) + build.to_branch(self.options.build_branch) + if not build.is_clean(): + build.revert() + + tagsfiles = glob(build.path+"/*-tags.mk") + tagsdict = { x : 'todo' for x in tagsfiles } default_answer = 'y' tagsfiles.sort() while True: + # do not bother if in bypass mode + if self.options.bypass: + break for tagsfile in tagsfiles: - status=tagsdict[tagsfile] - basename=os.path.basename(tagsfile) - print ".................... Dealing with %s"%basename + if not self.is_mentioned_in_tagsfile(tagsfile): + if self.options.verbose: + print("tagsfile {} does not mention {} - skipped".format(tagsfile, self.name)) + continue + status = tagsdict[tagsfile] + basename = os.path.basename(tagsfile) + print(".................... Dealing with {}".format(basename)) while tagsdict[tagsfile] == 'todo' : - choice = prompt ("insert %s in %s "%(new_tag_name,basename),default_answer, + choice = prompt("insert {} in {} ".format(new_tag_name, basename), + default_answer, [ ('y','es'), ('n', 'ext'), ('f','orce'), ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] , allow_outside=True) if choice == 'y': - self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True) + self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=True) elif choice == 'n': - print 'Done with %s'%os.path.basename(tagsfile) - tagsdict[tagsfile]='done' + print('Done with {}'.format(os.path.basename(tagsfile))) + tagsdict[tagsfile] = 'done' elif choice == 'f': - self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False) + self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=False) elif choice == 'd': - self.run("svn diff %s"%tagsfile) + print(build.diff(f=os.path.basename(tagsfile))) elif choice == 'r': - self.run("svn revert %s"%tagsfile) + build.revert(f=tagsfile) elif choice == 'c': - self.run("cat %s"%tagsfile) + self.run("cat {}".format(tagsfile)) else: - name=self.name - print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s -f: unconditionnally change any line that assigns %(name)s-SVNPATH to using %(new_tag_name)s + name = self.name + print( +"""y: change {name}-GITPATH only if it currently refers to {old_tag_name} +f: unconditionnally change any line that assigns {name}-GITPATH to using {new_tag_name} d: show current diff for this tag file r: revert that tag file c: cat the current tag file -n: move to next file"""%locals() +n: move to next file""".format(**locals())) - if prompt("Want to review changes on tags files",False): - tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] ) - default_answer='d' + if prompt("Want to review changes on tags files", False): + tagsdict = {x : 'todo' for x in tagsfiles } + default_answer = 'd' else: break - paths="" - paths += self.edge_dir() + " " - paths += build.edge_dir() + " " - self.run_prompt("Review module and build","svn diff " + paths) - self.run_prompt("Commit module and build","svn commit --file %s %s"%(changelog_svn,paths)) - self.run_prompt("Create tag","svn copy --file %s %s %s"%(changelog_svn,edge_url,new_tag_url)) + def diff_all_changes(): + print(build.diff()) + print(self.repository.diff()) + + def commit_all_changes(log): + if hasattr(self, 'branch'): + self.repository.commit(log, branch=self.branch) + else: + self.repository.commit(log) + build.commit(log) + + self.run_prompt("Review module and build", diff_all_changes) + self.run_prompt("Commit module and build", commit_all_changes, changelog_strip) + self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_strip) if self.options.debug: - print 'Preserving',changelog,'and stripped',changelog_svn + print('Preserving {} and stripped {}', changelog_plain, changelog_strip) else: - os.unlink(changelog) - os.unlink(changelog_svn) - -############################## - def do_branch (self): - - # save self.branch if any, as a hint for the new branch - # do this before anything else and restore .branch to None, - # as this is part of the class's logic - new_trunk_name=None - if hasattr(self,'branch'): - new_trunk_name=self.branch - del self.branch - elif self.options.new_version: - new_trunk_name = self.options.new_version - - # compute diff - a way to initialize the whole stuff - # do_diff already does edge_dir initialization - # and it checks that edge_url and tag_url exist as well - (spec_dict,edge_url,tag_url,diff_listing) = self.do_diff(compute_only=True) - - # the version name in the trunk becomes the new branch name - branch_name = spec_dict[self.module_version_varname] - - # figure new branch name (the one for the trunk) if not provided on the command line - if not new_trunk_name: - # heuristic is to assume 'version' is a dot-separated name - # we isolate the rightmost part and try incrementing it by 1 - version=spec_dict[self.module_version_varname] - try: - m=re.compile("\A(?P.+)\.(?P[^\.]+)\Z") - (leftpart,rightmost)=m.match(version).groups() - incremented = int(rightmost)+1 - new_trunk_name="%s.%d"%(leftpart,incremented) - except: - raise Exception, 'Cannot figure next branch name from %s - exiting'%version + os.unlink(changelog_plain) + os.unlink(changelog_strip) - # record starting point tagname - latest_tag_name = self.tag_name(spec_dict) - print "**********" - print "Using starting point %s (%s)"%(tag_url,latest_tag_name) - print "Creating branch %s & moving trunk to %s"%(branch_name,new_trunk_name) - print "**********" +############################## + def do_version(self): + self.init_module_dir() + self.revert_module_dir() + self.update_module_dir() + spec_dict = self.spec_dict() + if self.options.www: + self.html_store_title('Version for module {} ({})'\ + .format(self.friendly_name(), self.last_tag(spec_dict))) + for varname in self.varnames: + if varname not in spec_dict: + self.html_print('Could not find %define for {}'.format(varname)) + return + else: + self.html_print("{:<16} {}".format(varname, spec_dict[varname])) + self.html_print("{:<16} {}".format('url', self.repository.url())) + if self.options.verbose: + self.html_print("{:<16} {}".format('main specfile:', self.main_specname())) + self.html_print("{:<16} {}".format('specfiles:', self.all_specnames())) + self.html_print_end() - # print warning if pending diffs - if diff_listing: - print """*** WARNING : Module %s has pending diffs on its trunk -It is safe to proceed, but please note that branch %s -will be based on latest tag %s and *not* on the current trunk"""%(self.name,branch_name,latest_tag_name) - while True: - answer = prompt ('Are you sure you want to proceed with branching',True,('d','iff')) - if answer is True: - break - elif answer is False: - raise Exception,"User quit" - elif answer == 'd': - print '<<<< %s'%tag_url - print '>>>> %s'%edge_url - print diff_listing - - branch_url = "%s/%s/branches/%s"%(Module.config['svnpath'],self.name,branch_name) - self.check_svnpath_not_exists (branch_url,"new branch") - - # patching trunk - spec_dict[self.module_version_varname]=new_trunk_name - spec_dict[self.module_taglevel_varname]='0' - # remember this in the trunk for easy location of the current branch - spec_dict['module_current_branch']=branch_name - self.patch_spec_var(spec_dict,True) - - # create commit log file - tmp="/tmp/branching-%d"%os.getpid() - f=open(tmp,"w") - f.write("Branch %s for module %s created (as new trunk) from tag %s\n"%(branch_name,self.name,latest_tag_name)) - f.close() - - # review the renumbering changes in trunk - command="svn diff %s"%self.edge_dir() - self.run_prompt("Review (renumbering) changes in trunk",command) - # create branch - command="svn copy --file %s %s %s"%(tmp,tag_url,branch_url) - self.run_prompt("Create branch",command) - # commit trunk - command="svn commit --file %s %s"%(tmp,self.edge_dir()) - self.run_prompt("Commit trunk",command) - # create initial tag for the new trunk - new_tag_url=self.tag_url(spec_dict) - command="svn copy --file %s %s %s"%(tmp,tag_url,new_tag_url) - self.run_prompt("Create initial tag in trunk",command) - os.unlink(tmp) - # print message about BRANCH - print """You might now wish to review your tags files -Please make sure you mention as appropriate -%s-BRANCH := %s""" %(self.name,branch_name) ############################## -class Package: - - def __init__(self, package, module, svnpath, spec,options): - self.package=package - self.module=module - self.svnrev = None - self.svnpath=svnpath - if svnpath.rfind('@') > 0: - self.svnpath, self.svnrev = svnpath.split('@') - self.spec=spec - self.specpath="%s/%s"%(self.svnpath,self.spec) - if self.svnrev: - self.specpath += "@%s" % self.svnrev - self.basename=os.path.basename(svnpath) - self.options=options + def do_diff(self): + self.init_module_dir() + self.revert_module_dir() + self.update_module_dir() + spec_dict = self.spec_dict() + self.show_dict(spec_dict) + + # side effects + tag_name = self.tag_name(spec_dict) + + # sanity check + tag_name = self.check_tag(tag_name, need_it=True) + + if self.options.verbose: + print('Getting diff') + diff_output = self.repository.diff_with_tag(tag_name) - # returns a http URL to the trac path where full diff can be viewed (between self and pkg) - # typically http://svn.planet-lab.org/changeset?old_path=Monitor%2Ftags%2FMonitor-1.0-7&new_path=Monitor%2Ftags%2FMonitor-1.0-13 - # xxx quick & dirty: rough url parsing - def trac_full_diff (self, pkg): - matcher=re.compile("\A(?P.*)://(?P[^/]+)/(svn/)?(?P.*)\Z") - self_match=matcher.match(self.svnpath) - pkg_match=matcher.match(pkg.svnpath) - if self_match and pkg_match: - (method,hostname,svn,path)=self_match.groups() - self_path=path.replace("/","%2F") - pkg_path=pkg_match.group('path').replace("/","%2F") - return "%s://%s/changeset?old_path=%s&new_path=%s"%(method,hostname,self_path,pkg_path) + if self.options.list: + if diff_output: + print(self.pathname) else: - return None - - def inline_full_diff (self, pkg): - print '{{{' - command='svn diff %s %s'%(self.svnpath,pkg.svnpath) - Command(command,self.options).run() - print '}}}' + thename = self.friendly_name() + do_print = False + if self.options.www and diff_output: + self.html_store_title("Diffs in module {} ({}) : {} chars"\ + .format(thename, self.last_tag(spec_dict), len(diff_output))) - def details (self): - return "[%s %s] [%s (spec)]"%(self.svnpath,self.basename,self.specpath) + self.html_store_raw('

< (left) {}

'.format(tag_name)) + self.html_store_raw('

> (right) {}

'.format(thename)) + self.html_store_pre(diff_output) + elif not self.options.www: + print('x'*30, 'module', thename) + print('x'*20, '<', tag_name) + print('x'*20, '>', thename) + print(diff_output) -class Build (Module): - - # we cannot get build's svnpath as for other packages as we'd get something in svn+ssh - # xxx quick & dirty - def __init__ (self, buildtag, options): - self.buildtag=buildtag - if buildtag == "trunk": - module_name="build" - self.display="trunk" - self.svnpath="http://svn.planet-lab.org/svn/build/trunk" - # if the buildtag start with a : (to use a branch rather than a tag) - elif buildtag.find(':') == 0 : - module_name="build%(buildtag)s"%locals() - self.display=buildtag[1:] - self.svnpath="http://svn.planet-lab.org/svn/build/branches/%s"%self.display - else : - module_name="build@%(buildtag)s"%locals() - self.display=buildtag - self.svnpath="http://svn.planet-lab.org/svn/build/tags/%s"%self.buildtag - Module.__init__(self,module_name,options) +############################## + # store and restitute html fragments + @staticmethod + def html_href(url,text): + return '{}'.format(url, text) + + @staticmethod + def html_anchor(url,text): + return '{}'.format(url,text) @staticmethod - def get_distro_from_distrotag (distrotag): - # mhh: remove -tag* from distrotags to get distro - n=distrotag.find('-tag') - if n>0: - return distrotag[:n] + def html_quote(text): + return text.replace('&', '&').replace('<', '<').replace('>', '>') + + # only the fake error module has multiple titles + def html_store_title(self, title): + if not hasattr(self,'titles'): + self.titles=[] + self.titles.append(title) + + def html_store_raw(self, html): + if not hasattr(self,'body'): + self.body='' + self.body += html + + def html_store_pre(self, text): + if not hasattr(self,'body'): + self.body='' + self.body += '
{}
'.format(self.html_quote(text)) + + def html_print(self, txt): + if not self.options.www: + print(txt) else: - return None + if not hasattr(self, 'in_list') or not self.in_list: + self.html_store_raw('
    ') + self.in_list = True + self.html_store_raw('
  • {}
    • '.format(txt)) - def get_packages (self,distrotag): - result={} - distro=Build.get_distro_from_distrotag(distrotag) - if not distro: - return result - make_options="--no-print-directory -C %s stage1=true PLDISTRO=%s PLDISTROTAGS=%s 2> /dev/null"%(self.edge_dir(),distro,distrotag) - command="make %s packages"%make_options - make_packages=Command(command,self.options).output_of() - if self.options.verbose: - print 'obtaining packages information with command:' - print command - pkg_line=re.compile("\Apackage=(?P[^\s]+)\s+ref_module=(?P[^\s]+)\s.*\Z") - for line in make_packages.split("\n"): - if not line: - continue - attempt=pkg_line.match(line) - if line and not attempt: - print "=====" - print "WARNING: line not understood from make packages" - print "in dir %s"%self.edge_dir - print "with options",make_options - print 'line=',line - print "=====" + def html_print_end(self): + if self.options.www: + self.html_store_raw('
') + + @staticmethod + def html_dump_header(title): + nowdate = time.strftime("%Y-%m-%d") + nowtime = time.strftime("%H:%M (%Z)") + print(""" + + + {} + + + + +

{} - status on {} at {}

+
    +""".format(title, title, nowdate, nowtime)) + + @staticmethod + def html_dump_middle(): + print("
") + + @staticmethod + def html_dump_footer(): + print("', self.html_href('#'+self.friendly_name(),title), '') + + def html_dump_body(self): + if hasattr(self,'titles'): + for title in self.titles: + print('

', self.html_anchor(self.friendly_name(),title), '

') + if hasattr(self,'body'): + print(self.body) + print('

', self.html_href('#','Back to top'), '

') + + + +class Build(Module): + + def __get_modules(self, tagfile): + self.init_module_dir() + modules = {} + + tagfile = os.path.join(self.module_dir, tagfile) + for line in open(tagfile): + try: + name, url = line.split(':=') + name, git_path = name.rsplit('-', 1) + modules[name] = (git_path.strip(), url.strip()) + except: + pass + return modules + + def get_modules(self, tagfile): + modules = self.__get_modules(tagfile) + for module in modules: + module_type = tag_or_branch = "" + + path_type, url = modules[module] + if path_type == "GITPATH": + module_spec = os.path.split(url)[-1].replace(".git","") + name, tag_or_branch, module_type = self.parse_module_spec(module_spec) else: - (package,module) = (attempt.group('package'),attempt.group('module')) - command="make %s +%s-SVNPATH"%(make_options,module) - svnpath=Command(command,self.options).output_of().strip() - command="make %s +%s-SPEC"%(make_options,package) - spec=Command(command,self.options).output_of().strip() - result[module]=Package(package,module,svnpath,spec,self.options) - return result + tag_or_branch = os.path.split(url)[-1].strip() + if url.find('/tags/') >= 0: + module_type = "tag" + elif url.find('/branches/') >= 0: + module_type = "branch" + + modules[module] = {"module_type" : module_type, + "path_type": path_type, + "tag_or_branch": tag_or_branch, + "url":url} + return modules + + - def get_distrotags (self): - return [os.path.basename(p) for p in glob("%s/*tags*mk"%self.edge_dir())] +def modules_diff(first, second): + diff = {} -class DiffCache: + for module in first: + if module not in second: + print("=== module {} missing in right-hand side ===".format(module)) + continue + if first[module]['tag_or_branch'] != second[module]['tag_or_branch']: + diff[module] = (first[module]['tag_or_branch'], second[module]['tag_or_branch']) - def __init__ (self): - self._cache={} + first_set = set(first.keys()) + second_set = set(second.keys()) - def key(self, frompath,topath): - return frompath+'-to-'+topath + new_modules = list(second_set - first_set) + removed_modules = list(first_set - second_set) - def fetch (self, frompath, topath): - key=self.key(frompath,topath) - if not self._cache.has_key(key): - return None - return self._cache[key] + return diff, new_modules, removed_modules - def store (self, frompath, topath, diff): - key=self.key(frompath,topath) - self._cache[key]=diff +def release_changelog(options, buildtag_old, buildtag_new): -class Release: + # the command line expects new old, so we treat the tagfiles in the same order + nb_tags = len(options.distrotags) + if nb_tags == 1: + tagfile_new = tagfile_old = options.distrotags[0] + elif nb_tags == 2: + tagfile_new, tagfile_old = options.distrotags + else: + print("ERROR: provide one or two tagfile name (eg. onelab-k32-tags.mk)") + print("two tagfiles can be mentioned when a tagfile has been renamed") + return - # header in diff output - discard_matcher=re.compile("\A(\+\+\+|---).*") + if options.dry_run: + print("------------------------------ Computing Changelog from") + print("buildtag_old", buildtag_old, "tagfile_old", tagfile_old) + print("buildtag_new", buildtag_new, "tagfile_new", tagfile_new) + return - @staticmethod - def do_changelog (buildtag_new,buildtag_old,options): - print "----" - print "----" - print "----" - (build_new,build_old) = (Build (buildtag_new,options), Build (buildtag_old,options)) - print "= build tag %s to %s = #build-%s"%(build_old.display,build_new.display,build_new.display) - for b in (build_new,build_old): - b.init_module_dir() - b.init_edge_dir() - b.update_edge_dir() - # find out the tags files that are common, unless option was specified - if options.distrotags: - distrotags=options.distrotags + print('----') + print('----') + print('----') + print('= build tag {} to {} ='.format(buildtag_old, buildtag_new)) + print('== distro {} ({} to {}) =='.format(tagfile_new, buildtag_old, buildtag_new)) + + build = Build("build@{}".format(buildtag_old), options) + build.init_module_dir() + first = build.get_modules(tagfile_old) + + print(' * from', buildtag_old, build.repository.gitweb()) + + build = Build("build@{}".format(buildtag_new), options) + build.init_module_dir() + second = build.get_modules(tagfile_new) + + print(' * to', buildtag_new, build.repository.gitweb()) + + diff, new_modules, removed_modules = modules_diff(first, second) + + + def get_module(name, tag): + if not tag or tag == "trunk": + return Module("{}".format(module), options) else: - distrotags_new=build_new.get_distrotags() - distrotags_old=build_old.get_distrotags() - distrotags = list(set(distrotags_new).intersection(set(distrotags_old))) - distrotags.sort() - if options.verbose: print "Found distrotags",distrotags - first_distrotag=True - diffcache = DiffCache() - for distrotag in distrotags: - distro=Build.get_distro_from_distrotag(distrotag) - if not distro: - continue - if first_distrotag: - first_distrotag=False - else: - print '----' - print '== distro %s (%s to %s) == #distro-%s-%s'%(distrotag,build_old.display,build_new.display,distro,build_new.display) - print ' * from %s/%s'%(build_old.svnpath,distrotag) - print ' * to %s/%s'%(build_new.svnpath,distrotag) - - # parse make packages - packages_new=build_new.get_packages(distrotag) - pnames_new=set(packages_new.keys()) - packages_old=build_old.get_packages(distrotag) - pnames_old=set(packages_old.keys()) - - # get names of created, deprecated, and preserved modules - pnames_created = list(pnames_new-pnames_old) - pnames_deprecated = list(pnames_old-pnames_new) - pnames = list(pnames_new.intersection(pnames_old)) - - pnames_created.sort() - pnames_deprecated.sort() - pnames.sort() - - if options.verbose: - print '--------------------' - print 'got packages for ',build_new.display - print pnames_new - print '--------------------' - print 'got packages for ',build_old.display - print pnames_old - print '--------------------' - print "Found new modules",pnames_created - print '--------------------' - print "Found deprecated modules",pnames_deprecated - print '--------------------' - print "Found preserved modules",pnames - print '--------------------' - - # display created and deprecated - for name in pnames_created: - print '=== %s : new package %s -- appeared in %s === #package-%s-%s-%s'%( - distrotag,name,build_new.display,name,distro,build_new.display) - pobj=packages_new[name] - print ' * %s'%pobj.details() - for name in pnames_deprecated: - print '=== %s : package %s -- deprecated, last occurrence in %s === #package-%s-%s-%s'%( - distrotag,name,build_old.display,name,distro,build_new.display) - pobj=packages_old[name] - if not pobj.svnpath: - print ' * codebase stored in CVS, specfile is %s'%pobj.spec - else: - print ' * %s'%pobj.details() + return Module("{}@{}".format(module, tag), options) - # display other packages - for name in pnames: - (pobj_new,pobj_old)=(packages_new[name],packages_old[name]) - if options.verbose: print "Dealing with package",name - if pobj_old.specpath == pobj_new.specpath: - continue - specdiff = diffcache.fetch(pobj_old.specpath,pobj_new.specpath) - if specdiff is None: - command="svn diff %s %s"%(pobj_old.specpath,pobj_new.specpath) - specdiff=Command(command,options).output_of() - diffcache.store(pobj_old.specpath,pobj_new.specpath,specdiff) - else: - if options.verbose: print 'got diff from cache' - if not specdiff: - continue - print '=== %s - %s to %s : package %s === #package-%s-%s-%s'%( - distrotag,build_old.display,build_new.display,name,name,distro,build_new.display) - print ' * from %s to %s'%(pobj_old.details(),pobj_new.details()) - if options.inline_diff: - pobj_old.inline_full_diff(pobj_new) - else: - trac_diff_url=pobj_old.trac_full_diff(pobj_new) - if trac_diff_url: - print ' * [%s View full diff]'%trac_diff_url - else: - print ' * No full diff available' - print '{{{' - for line in specdiff.split('\n'): - if not line: - continue - if Release.discard_matcher.match(line): - continue - if line[0] in ['@']: - print '----------' - elif line[0] in ['+','-']: - print_fold(line) - print '}}}' + + for module in diff: + print('=== {} - {} to {} : package {} ==='.format(tagfile_new, buildtag_old, buildtag_new, module)) + + first, second = diff[module] + m = get_module(module, first) + os.system('rm -rf {}'.format(m.module_dir)) # cleanup module dir + m.init_module_dir() + + print(' * from', first, m.repository.gitweb()) + + specfile = m.main_specname() + (tmpfd, tmpfile) = tempfile.mkstemp() + os.system("cp -f /{} {}".format(specfile, tmpfile)) + + m = get_module(module, second) + # patch for ipfw that, being managed in a separate repo, won't work for now + try: + m.init_module_dir() + except Exception as e: + print("""Could not retrieve module {} - skipped +{{{{{{ {} }}}}}} +""".format( m.friendly_name(), e)) + continue + specfile = m.main_specname() + + print(' * to', second, m.repository.gitweb()) + + print('{{{') + os.system("diff -u {} {} | sed -e 's,{},[[previous version]],'"\ + .format(tmpfile, specfile, tmpfile)) + print('}}}') + + os.unlink(tmpfile) + + for module in new_modules: + print('=== {} : new package in build {} ==='.format(tagfile_new, module)) + + for module in removed_modules: + print('=== {} : removed package from build {} ==='.format(tagfile_new, module)) + + +def adopt_tag(options, args): + modules=[] + for module in options.modules: + modules += module.split() + for module in modules: + modobj=Module(module,options) + for tags_file in args: + if options.verbose: + print('adopting tag {} for {} in {}'.format(options.tag, module, tags_file)) + modobj.patch_tags_file(tags_file, '_unused_', options.tag, fine_grain=False) + if options.verbose: + Command("git diff {}".format(" ".join(args)), options).run() ############################## class Main: module_usage="""Usage: %prog [options] module_desc [ .. module_desc ] -Revision: $Revision$ module-tools : a set of tools to manage subversion tags and specfile requires the specfile to either * define *version* and *taglevel* OR alternatively * define redirection variables module_version_varname / module_taglevel_varname -Trunk: - by default, the trunk of modules is taken into account +Master: + by default, the 'master' branch of modules is the target in this case, just mention the module name as Branches: - if you wish to work on a branch rather than on the trunk, + if you wish to work on another branch, you can use something like e.g. Mom:2.1 as """ release_usage="""Usage: %prog [options] tag1 .. tagn @@ -1280,16 +1238,25 @@ Branches: release-changelog :4.2 4.2-rc25 You can refer to the build trunk by just mentioning 'trunk', e.g. release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk + You can use 2 different tagfile names if that was renamed meanwhile + release-changelog -t onelab-tags.mk 5.0-rc29 -t onelab-k32-tags.mk 5.0-rc28 +""" + adopt_usage="""Usage: %prog [options] tag-file[s] + With this command you can adopt a specifi tag or branch in your tag files + This should be run in your daily build workdir; no call of git is done + Examples: + adopt-tag -m "plewww plcapi" -m Monitor onelab*tags.mk + adopt-tag -m sfa -t sfa-1.0-33 *tags.mk """ common_usage="""More help: see http://svn.planet-lab.org/wiki/ModuleTools""" - modes={ + modes = { 'list' : "displays a list of available tags or branches", 'version' : "check latest specfile and print out details", 'diff' : "show difference between module (trunk or branch) and latest tag", 'tag' : """increment taglevel in specfile, insert changelog in specfile, - create new tag and and monitor its adoption in build/*-tags*.mk""", + create new tag and and monitor its adoption in build/*-tags.mk""", 'branch' : """create a branch for this module, from the latest tag on the trunk, and change trunk's version number to reflect the new branch name; you can specify the new branch name by using module:branch""", @@ -1297,13 +1264,16 @@ Branches: this is a last resort option, mostly for repairs""", 'changelog' : """extract changelog between build tags expected arguments are a list of tags""", + 'adopt' : """locally adopt a specific tag""", } silent_modes = ['list'] - release_modes = ['changelog'] + # 'changelog' is for release-changelog + # 'adopt' is for 'adopt-tag' + regular_modes = set(modes.keys()).difference(set(['changelog','adopt'])) @staticmethod - def optparse_list (option, opt, value, parser): + def optparse_list(option, opt, value, parser): try: setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split()) except: @@ -1312,81 +1282,91 @@ Branches: def run(self): mode=None - for function in Main.modes.keys(): + # hack - need to check for adopt first as 'adopt-tag' contains tag.. + for function in [ 'adopt' ] + list(Main.modes.keys()): if sys.argv[0].find(function) >= 0: mode = function break if not mode: - print "Unsupported command",sys.argv[0] - print "Supported commands:" + " ".join(Main.modes.keys()) + print("Unsupported command",sys.argv[0]) + print("Supported commands:" + " ".join(list(Main.modes.keys()))) sys.exit(1) - if mode not in Main.release_modes: + usage='undefined usage, mode={}'.format(mode) + if mode in Main.regular_modes: usage = Main.module_usage usage += Main.common_usage - usage += "\nmodule-%s : %s"%(mode,Main.modes[mode]) - else: + usage += "\nmodule-{} : {}".format(mode, Main.modes[mode]) + elif mode == 'changelog': usage = Main.release_usage usage += Main.common_usage + elif mode == 'adopt': + usage = Main.adopt_usage + usage += Main.common_usage - parser=OptionParser(usage=usage,version=subversion_id) + parser=OptionParser(usage=usage) - if mode == 'list': - parser.add_option("-b","--branches",action="store_true",dest="list_branches",default=False, - help="list branches") - parser.add_option("-t","--tags",action="store_false",dest="list_branches", - help="list tags") - parser.add_option("-m","--match",action="store",dest="list_pattern",default=None, - help="grep pattern for filtering output") - parser.add_option("-x","--exact-match",action="store",dest="list_exact",default=None, - help="exact grep pattern for filtering output") - if mode == "tag" or mode == 'branch': + # the 'adopt' mode is really special and doesn't share any option + if mode == 'adopt': + parser.add_option("-m","--module",action="append",dest="modules",default=[], + help="modules, can be used several times or with quotes") + parser.add_option("-t","--tag",action="store", dest="tag", default='master', + help="specify the tag to adopt, default is 'master'") + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + help="run in verbose mode") + (options, args) = parser.parse_args() + options.workdir='unused' + options.dry_run=False + options.mode='adopt' + if len(args)==0 or len(options.modules)==0: + parser.print_help() + sys.exit(1) + adopt_tag(options,args) + return + + # the other commands (module-* and release-changelog) share the same skeleton + if mode in [ 'tag', 'branch'] : parser.add_option("-s","--set-version",action="store",dest="new_version",default=None, help="set new version and reset taglevel to 0") - if mode == "tag" : + parser.add_option("-0","--bypass",action="store_true",dest="bypass",default=False, + help="skip checks on existence of the previous tag") + if mode == 'tag' : parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True, help="do not update changelog section in specfile when tagging") parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None, - help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place") - if mode == "tag" or mode == "sync" : + help="specify a build branch; used for locating the *tags.mk files where adoption is to take place") + if mode in [ 'tag', 'sync' ] : parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(), help="specify editor") - if mode == "sync" : - parser.add_option("-m","--message", action="store", dest="message", default=None, - help="specify log message") - if mode in ["diff","version"] : + + if mode in ['diff','version'] : parser.add_option("-W","--www", action="store", dest="www", default=False, help="export diff in html format, e.g. -W trunk") - if mode == "diff" : + + if mode == 'diff' : parser.add_option("-l","--list", action="store_true", dest="list", default=False, help="just list modules that exhibit differences") - - if mode == 'version': - parser.add_option("-u","--url", action="store_true", dest="show_urls", default=False, - help="display URLs") default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list" - if mode not in Main.release_modes: - parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, - help="run on all modules as found in %s"%default_modules_list) - parser.add_option("-f","--file",action="store",dest="modules_list",default=None, - help="run on all modules found in specified file") - else: - parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False, - help="dry run - shell commands are only displayed") - parser.add_option("-i","--inline-diff",action="store_true",dest="inline_diff",default=False, - help="calls svn diff on whole module, not just only the spec file") - parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags", - default=[], nargs=1,type="string", - help="""specify distro-tags files, e.g. onelab-tags-4.2.mk + parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, + help="run on all modules as found in {}".format(default_modules_list)) + parser.add_option("-f","--file",action="store",dest="modules_list",default=None, + help="run on all modules found in specified file") + parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False, + help="dry run - shell commands are only displayed") + parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags", + default=[], nargs=1,type="string", + help="""specify distro-tags files, e.g. onelab-tags-4.2.mk -- can be set multiple times, or use quotes""") parser.add_option("-w","--workdir", action="store", dest="workdir", - default="%s/%s"%(os.getenv("HOME"),"modules"), + default="{}/{}".format(os.getenv("HOME"),"modules"), help="""name for dedicated working dir - defaults to ~/modules ** THIS MUST NOT ** be your usual working directory""") parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False, - help="skip safety checks, such as svn updates -- use with care") + help="skip safety checks, such as git pulls -- use with care") + parser.add_option("-B","--build-module",action="store",dest="build_module",default=None, + help="specify a build module to owerride the one in the CONFIG") # default verbosity depending on function - temp verbose_modes= ['tag', 'sync', 'branch'] @@ -1397,9 +1377,7 @@ Branches: else: parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True, help="run in quiet (non-verbose) mode") -# parser.add_option("-d","--debug", action="store_true", dest="debug", default=False, -# help="debug mode - mostly more verbose") - (options, args) = parser.parse_args() + options, args = parser.parse_args() options.mode=mode if not hasattr(options,'dry_run'): options.dry_run=False @@ -1407,55 +1385,48 @@ Branches: options.www=False options.debug=False - ########## release-* - if mode in Main.release_modes : - ########## changelog - if len(args) <= 1: + + + ########## module-* + if len(args) == 0: + if options.all_modules: + options.modules_list=default_modules_list + if options.modules_list: + args=Command("grep -v '#' {}".format(options.modules_list), options).output_of().split() + else: parser.print_help() sys.exit(1) - Module.init_homedir(options) - for n in range(len(args)-1): - [t_new,t_old]=args[n:n+2] - Release.do_changelog (t_new,t_old,options) - else: - ########## module-* - if len(args) == 0: - if options.all_modules: - options.modules_list=default_modules_list - if options.modules_list: - args=Command("grep -v '#' %s"%options.modules_list,options).output_of().split() - else: - parser.print_help() - sys.exit(1) - Module.init_homedir(options) + Module.init_homedir(options) + - # 2 passes for www output - modules=[ Module(modname,options) for modname in args ] + if mode in Main.regular_modes: + modules = [ Module(modname, options) for modname in args ] # hack: create a dummy Module to store errors/warnings error_module = Module('__errors__',options) - # pass 1 : do it, except if options.www for module in modules: - if len(args)>1 and mode not in Main.silent_modes and not options.www: - print '========================================',module.friendly_name() + if len(args)>1 and mode not in Main.silent_modes: + if not options.www: + print('========================================', module.friendly_name()) # call the method called do_ - method=Module.__dict__["do_%s"%mode] + method = Module.__dict__["do_{}".format(mode)] try: method(module) - except Exception,e: + except Exception as e: if options.www: - title=' Skipping module %s - failure: %s '%\ - (module.friendly_name(), str(e)) + title=' Skipping module {} - failure: {} '\ + .format(module.friendly_name(), str(e)) error_module.html_store_title(title) else: - print 'Skipping module %s: '%modname,e - - # in which case we do the actual printing in the second pass + import traceback + traceback.print_exc() + print('Skipping module {}: {}'.format(module.name,e)) + if options.www: if mode == "diff": - modetitle="Changes to tag in %s"%options.www + modetitle="Changes to tag in {}".format(options.www) elif mode == "version": - modetitle="Latest tags in %s"%options.www + modetitle="Latest tags in {}".format(options.www) modules.append(error_module) error_module.html_dump_header(modetitle) for module in modules: @@ -1464,10 +1435,17 @@ Branches: for module in modules: module.html_dump_body() Module.html_dump_footer() - + else: + # if we provide, say a b c d, we want to build (a,b) (b,c) and (c,d) + # remember that the changelog in the twiki comes latest first, so + # we typically have here latest latest-1 latest-2 + for (tag_new,tag_old) in zip ( args[:-1], args [1:]): + release_changelog(options, tag_old, tag_new) + + #################### if __name__ == "__main__" : try: Main().run() except KeyboardInterrupt: - print '\nBye' + print('\nBye')