X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=module-tools.py;h=7678bdce1e45a5b700e8805805dc9ad6defd4f1a;hb=6cdd17193b49743e1d05804e8875f373a235f014;hp=b9a859ad901ebf1e8ec675775d7e8c09abab981e;hpb=cdadf110fb405d51dbd3448ac232422fd156b12d;p=build.git diff --git a/module-tools.py b/module-tools.py index b9a859ad..7678bdce 100755 --- a/module-tools.py +++ b/module-tools.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 -u +#!/usr/bin/env python3 import sys, os, os.path import re @@ -7,37 +7,9 @@ import tempfile from glob import glob from optparse import OptionParser -# HARDCODED NAME CHANGES -# -# Moving to git we decided to rename some of the repositories. Here is -# a map of name changes applied in git repositories. -RENAMED_SVN_MODULES = { - "PLEWWW" : "plewww", - "PLCAPI" : "plcapi", - "BootManager" : "bootmanager", - "BootCD": "bootcd", - "MyPLC": "myplc", - "CoDemux": "codemux", - "NodeManager": "nodemanager", - "NodeUpdate": "nodeupdate", - "Monitor": "monitor", - } - -def svn_to_git_name(module): - if module in RENAMED_SVN_MODULES: - return RENAMED_SVN_MODULES[module] - return module - -def git_to_svn_name(module): - for key in RENAMED_SVN_MODULES: - if module == RENAMED_SVN_MODULES[key]: - return key - return module - - -# e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase -def prompt (question, default=True, other_choices=[], allow_outside=False): - if not isinstance (other_choices, list): +# e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase +def prompt(question, default=True, other_choices=[], allow_outside=False): + if not isinstance(other_choices, list): other_choices = [ other_choices ] chars = [ c for (c,rest) in other_choices ] @@ -93,19 +65,19 @@ def default_editor(): ### fold long lines fold_length = 132 -def print_fold (line): +def print_fold(line): while len(line) >= fold_length: print(line[:fold_length],'\\') line = line[fold_length:] print(line) class Command: - def __init__ (self, command, options): + def __init__(self, command, options): self.command = command self.options = options - self.tmp = "/tmp/command-%d"%os.getpid() + self.tmp = "/tmp/command-{}".format(os.getpid()) - def run (self): + def run(self): if self.options.dry_run: print('dry_run', self.command) return 0 @@ -114,7 +86,7 @@ class Command: sys.stdout.flush() return os.system(self.command) - def run_silent (self): + def run_silent(self): if self.options.dry_run: print('dry_run', self.command) return 0 @@ -137,7 +109,7 @@ class Command: raise Exception("Command {} failed".format(self.command)) # returns stdout, like bash's $(mycommand) - def output_of (self, with_stderr=False): + def output_of(self, with_stderr=False, binary=False): if self.options.dry_run: print('dry_run', self.command) return 'dry_run output' @@ -152,115 +124,13 @@ class Command: command += " > " command += tmp os.system(command) - with open(tmp) as f: + mode = "r" if not binary else "rb" + with open(tmp, mode) as f: result=f.read() os.unlink(tmp) if self.options.debug: print('Done', end=' ') return result - -class SvnRepository: - type = "svn" - - def __init__(self, path, options): - self.path = path - self.options = options - - def name(self): - return os.path.basename(self.path) - - def pathname(self): - # for svn modules pathname is just the name of the module as - # all modules are at the root - return self.name() - - def url(self): - out = Command("svn info {}".format(self.path), self.options).output_of() - for line in out.split('\n'): - if line.startswith("URL:"): - return line.split()[1].strip() - - def repo_root(self): - out = Command("svn info {}".format(self.path), self.options).output_of() - for line in out.split('\n'): - if line.startswith("Repository Root:"): - root = line.split()[2].strip() - return "{}/{}".format(root, self.pathname()) - - @classmethod - def clone(cls, remote, local, options, recursive=False): - if recursive: - svncommand = "svn co %s %s" % (remote, local) - else: - svncommand = "svn co -N %s %s" % (remote, local) - Command("rm -rf %s" % local, options).run_silent() - Command(svncommand, options).run_fatal() - - return SvnRepository(local, options) - - @classmethod - def remote_exists(cls, remote, options): - return Command ("svn list %s &> /dev/null" % remote , options).run()==0 - - def tag_exists(self, tagname): - url = "%s/tags/%s" % (self.repo_root(), tagname) - return SvnRepository.remote_exists(url, self.options) - - def update(self, subdir="", recursive=True, branch=None): - path = os.path.join(self.path, subdir) - if recursive: - svncommand = "svn up %s" % path - else: - svncommand = "svn up -N %s" % path - Command(svncommand, self.options).run_fatal() - - def commit(self, logfile): - # add all new files to the repository - Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs svn add" % - self.path, self.options).output_of() - Command("svn commit -F %s %s" % (logfile, self.path), self.options).run_fatal() - - def to_branch(self, branch): - remote = "%s/branches/%s" % (self.repo_root(), branch) - SvnRepository.clone(remote, self.path, self.options, recursive=True) - - def to_tag(self, tag): - remote = "%s/tags/%s" % (self.repo_root(), branch) - SvnRepository.clone(remote, self.path, self.options, recursive=True) - - def tag(self, tagname, logfile): - tag_url = "%s/tags/%s" % (self.repo_root(), tagname) - self_url = self.url() - Command("svn copy -F %s %s %s" % (logfile, self_url, tag_url), self.options).run_fatal() - - def diff(self, f=""): - if f: - f = os.path.join(self.path, f) - else: - f = self.path - return Command("svn diff %s" % f, self.options).output_of(True) - - def diff_with_tag(self, tagname): - tag_url = "%s/tags/%s" % (self.repo_root(), tagname) - return Command("svn diff %s %s" % (tag_url, self.url()), - self.options).output_of(True) - - def revert(self, f=""): - if f: - Command("svn revert %s" % os.path.join(self.path, f), self.options).run_fatal() - else: - # revert all - Command("svn revert %s -R" % self.path, self.options).run_fatal() - Command("svn status %s | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\\ /g' | xargs rm -rf " % - self.path, self.options).run_silent() - - def is_clean(self): - command="svn status %s" % self.path - return len(Command(command,self.options).output_of(True)) == 0 - - def is_valid(self): - return os.path.exists(os.path.join(self.path, ".svn")) - class GitRepository: type = "git" @@ -296,7 +166,7 @@ class GitRepository: @classmethod def remote_exists(cls, remote, options): - return Command ("git --no-pager ls-remote {} &> /dev/null".format(remote), options).run()==0 + return Command("git --no-pager ls-remote {} &> /dev/null".format(remote), options).run()==0 def tag_exists(self, tagname): command = 'git tag -l | grep "^{}$"'.format(tagname) @@ -330,10 +200,10 @@ class GitRepository: self.__run_command_in_repo("git checkout {}".format(branch)) else: self.to_branch(branch, remote=True) - self.__run_command_in_repo("git fetch origin --tags") + self.__run_command_in_repo("git fetch origin --tags", ignore_errors=True) self.__run_command_in_repo("git fetch origin") if not self.__is_commit_id(branch): - # we don't need to merge anythign for commit ids. + # we don't need to merge anything for commit ids. self.__run_command_in_repo("git merge --ff origin/{}".format(branch)) def to_branch(self, branch, remote=True): @@ -354,11 +224,17 @@ class GitRepository: def diff(self, f=""): c = Command("git diff {}".format(f), self.options) - return self.__run_in_repo(c.output_of, with_stderr=True) + try: + return self.__run_in_repo(c.output_of, with_stderr=True) + except: + return self.__run_in_repo(c.output_of, with_stderr=True, binary=True) def diff_with_tag(self, tagname): c = Command("git diff {}".format(tagname), self.options) - return self.__run_in_repo(c.output_of, with_stderr=True) + try: + return self.__run_in_repo(c.output_of, with_stderr=True) + except: + return self.__run_in_repo(c.output_of, with_stderr=True, binary=True) def commit(self, logfile, branch="master"): self.__run_command_in_repo("git add .", ignore_errors=True) @@ -368,7 +244,7 @@ class GitRepository: self.__run_command_in_repo("git push") else: self.__run_command_in_repo("git push origin {}:{}".format(branch, branch)) - self.__run_command_in_repo("git push --tags") + self.__run_command_in_repo("git push --tags", ignore_errors=True) def revert(self, f=""): if f: @@ -380,38 +256,34 @@ class GitRepository: def is_clean(self): def check_commit(): - command="git status" - s="nothing to commit (working directory clean)" + command = "git status" + s = "nothing to commit, working directory clean" return Command(command, self.options).output_of(True).find(s) >= 0 return self.__run_in_repo(check_commit) def is_valid(self): return os.path.exists(os.path.join(self.path, ".git")) - + class Repository: - """ Generic repository """ - supported_repo_types = [SvnRepository, GitRepository] + """ + Generic repository + From old times when we had svn and git + """ + supported_repo_types = [ GitRepository ] def __init__(self, path, options): self.path = path self.options = options - for repo in self.supported_repo_types: - self.repo = repo(self.path, self.options) + for repo_class in self.supported_repo_types: + self.repo = repo_class(self.path, self.options) if self.repo.is_valid(): break - @classmethod - def has_moved_to_git(cls, module, config,options): - module = svn_to_git_name(module) - # check if the module is already in Git - return GitRepository.remote_exists(Module.git_remote_dir(module), options) - - @classmethod def remote_exists(cls, remote, options): - for repo in Repository.supported_repo_types: - if repo.remote_exists(remote, options): + for repo_class in Repository.supported_repo_types: + if repo_class.remote_exists(remote, options): return True return False @@ -423,104 +295,103 @@ class Repository: # support for tagged module is minimal, and is for the Build class only class Module: - edit_magic_line="--This line, and those below, will be ignored--" + edit_magic_line = "--This line, and those below, will be ignored--" setting_tag_format = "Setting tag {}" - - redirectors=[ # ('module_name_varname', 'name'), - ('module_version_varname', 'version'), - ('module_taglevel_varname', 'taglevel'), ] + + redirectors = [ + # ('module_name_varname', 'name'), + ('module_version_varname', 'version'), + ('module_taglevel_varname', 'taglevel'), + ] # where to store user's config config_storage = "CONFIG" - # + # config = {} import subprocess - configKeys=[ ('svnpath', "Enter your toplevel svnpath", - "svn+ssh://%s@svn.planet-lab.org/svn/"%subprocess.getoutput("id -un")), - ('gitserver', "Enter your git server's hostname", "git.onelab.eu"), - ('gituser', "Enter your user name (login name) on git server", subprocess.getoutput("id -un")), - ("build", "Enter the name of your build module", "build"), - ('username', "Enter your firstname and lastname for changelogs", ""), - ("email", "Enter your email address for changelogs", ""), - ] + configKeys = [ + ('gitserver', "Enter your git server's hostname", "git.onelab.eu"), + ('gituser', "Enter your user name (login name) on git server", subprocess.getoutput("id -un")), + ("build", "Enter the name of your build module", "build"), + ('username', "Enter your firstname and lastname for changelogs", ""), + ("email", "Enter your email address for changelogs", ""), + ] @classmethod def prompt_config_option(cls, key, message, default): cls.config[key]=input("{} [{}] : ".format(message,default)).strip() or default @classmethod - def prompt_config (cls): + def prompt_config(cls): for (key,message,default) in cls.configKeys: cls.config[key]="" while not cls.config[key]: cls.prompt_config_option(key, message, default) - # for parsing module spec name:branch - matcher_branch_spec=re.compile("\A(?P[\w\.\-\/]+):(?P[\w\.\-]+)\Z") - # special form for tagged module - for Build - matcher_tag_spec=re.compile("\A(?P[\w\.\-\/]+)@(?P[\w\.\-]+)\Z") + # for parsing module spec name:branch + matcher_branch_spec = re.compile("\A(?P[\w\.\-\/]+):(?P[\w\.\-]+)\Z") + # special form for tagged module - for Build + matcher_tag_spec = re.compile("\A(?P[\w\.\-\/]+)@(?P[\w\.\-]+)\Z") + # parsing specfiles - matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") + matcher_rpm_define = re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") @classmethod def parse_module_spec(cls, module_spec): name = branch_or_tagname = module_type = "" - attempt=Module.matcher_branch_spec.match(module_spec) + attempt = Module.matcher_branch_spec.match(module_spec) if attempt: module_type = "branch" name=attempt.group('name') branch_or_tagname=attempt.group('branch') else: - attempt=Module.matcher_tag_spec.match(module_spec) + attempt = Module.matcher_tag_spec.match(module_spec) if attempt: module_type = "tag" name=attempt.group('name') branch_or_tagname=attempt.group('tagname') else: - name=module_spec + name = module_spec return name, branch_or_tagname, module_type - def __init__ (self,module_spec,options): + def __init__(self, module_spec, options): # parse module spec self.pathname, branch_or_tagname, module_type = self.parse_module_spec(module_spec) self.name = os.path.basename(self.pathname) if module_type == "branch": - self.branch=branch_or_tagname + self.branch = branch_or_tagname elif module_type == "tag": - self.tagname=branch_or_tagname - - # when available prefer to use git module name internally - self.name = svn_to_git_name(self.name) + self.tagname = branch_or_tagname self.options=options self.module_dir="{}/{}".format(options.workdir,self.pathname) self.repository = None self.build = None - def run (self,command): + def run(self,command): return Command(command,self.options).run() - def run_fatal (self,command): + def run_fatal(self,command): return Command(command,self.options).run_fatal() - def run_prompt (self,message,fun, *args): + def run_prompt(self,message,fun, *args): fun_msg = "{}({})".format(fun.__name__, ",".join(args)) if not self.options.verbose: while True: choice = prompt(message, True, ('s','how')) - if choice is True: + if choice: fun(*args) return - elif choice is False: + else: print('About to run function:', fun_msg) else: question = "{} - want to run function: {}".format(message, fun_msg) if prompt(question, True): fun(*args) - def friendly_name (self): + def friendly_name(self): if hasattr(self, 'branch'): return "{}:{}".format(self.pathname, self.branch) elif hasattr(self, 'tagname'): @@ -529,41 +400,22 @@ class Module: return self.pathname @classmethod - def git_remote_dir (cls, name): + def git_remote_dir(cls, name): return "{}@{}:/git/{}.git".format(cls.config['gituser'], cls.config['gitserver'], name) - @classmethod - def svn_remote_dir (cls, name): - name = git_to_svn_name(name) - svn = cls.config['svnpath'] - if svn.endswith('/'): - return "%s%s" % (svn, name) - return "%s/%s" % (svn, name) - - def svn_selected_remote(self): - svn_name = git_to_svn_name(self.name) - remote = self.svn_remote_dir(svn_name) - if hasattr(self, 'branch'): - remote = "{}/branches/{}".format(remote, self.branch) - elif hasattr(self,'tagname'): - remote = "{}/tags/{}".format(remote, self.tagname) - else: - remote = "{}/trunk".format(remote) - return remote - #################### @classmethod - def init_homedir (cls, options): + def init_homedir(cls, options): if options.verbose and options.mode not in Main.silent_modes: print('Checking for', options.workdir) storage="{}/{}".format(options.workdir, cls.config_storage) # sanity check. Either the topdir exists AND we have a config/storage # or topdir does not exist and we create it - # to avoid people use their own daily svn repo + # to avoid people use their own daily work repo if os.path.isdir(options.workdir) and not os.path.isfile(storage): print("""The directory {} exists and has no CONFIG file If this is your regular working directory, please provide another one as the -module-* commands need a fresh working dir. Make sure that you do not use +module-* commands need a fresh working dir. Make sure that you do not use that for other purposes than tagging""".format(options.workdir)) sys.exit(1) @@ -587,14 +439,14 @@ that for other purposes than tagging""".format(options.workdir)) with open(storage) as f: for line in f.readlines(): key, value = re.compile("^(.+)=(.+)$").match(line).groups() - Module.config[key] = value + Module.config[key] = value # owerride config variables using options. if options.build_module: Module.config['build'] = options.build_module - if not os.path.isdir (options.workdir): - print("Cannot find",options.workdir,"let's create it") + if not os.path.isdir(options.workdir): + print("Cannot find {}, let's create it".format(options.workdir)) Command("mkdir -p {}".format(options.workdir), options).run_silent() cls.prompt_config() clone_build() @@ -608,7 +460,7 @@ that for other purposes than tagging""".format(options.workdir)) print("Configuration changed for module-tools") cls.prompt_config_option(key, message, default) old_layout = True - + if old_layout: Command("rm -rf {}".format(options.workdir), options).run_silent() Command("mkdir -p {}".format(options.workdir), options).run_silent() @@ -631,53 +483,37 @@ that for other purposes than tagging""".format(options.workdir)) for (key,message,default) in Module.configKeys: print('\t{} = {}'.format(key,Module.config[key])) - def init_module_dir (self): + def init_module_dir(self): if self.options.verbose: - print('Checking for',self.module_dir) + print('Checking for', self.module_dir) - if not os.path.isdir (self.module_dir): - if Repository.has_moved_to_git(self.pathname, Module.config, self.options): - self.repository = GitRepository.clone(self.git_remote_dir(self.pathname), - self.module_dir, - self.options) - else: - remote = self.svn_selected_remote() - self.repository = SvnRepository.clone(remote, - self.module_dir, - self.options, recursive=False) + if not os.path.isdir(self.module_dir): + self.repository = GitRepository.clone(self.git_remote_dir(self.pathname), + self.module_dir, + self.options) self.repository = Repository(self.module_dir, self.options) - if self.repository.type == "svn": - # check if module has moved to git - if Repository.has_moved_to_git(self.pathname, Module.config, self.options): - Command("rm -rf %s" % self.module_dir, self.options).run_silent() - self.init_module_dir() - # check if we have the required branch/tag - if self.repository.url() != self.svn_selected_remote(): - Command("rm -rf %s" % self.module_dir, self.options).run_silent() - self.init_module_dir() - - elif self.repository.type == "git": + + if self.repository.type == "git": if hasattr(self, 'branch'): self.repository.to_branch(self.branch) elif hasattr(self, 'tagname'): self.repository.to_tag(self.tagname) - else: - raise Exception('Cannot find {} - check module name'.format(self.module_dir)) + raise Exception('Cannot find {} - or not a git module'.format(self.module_dir)) - def revert_module_dir (self): + def revert_module_dir(self): if self.options.fast_checks: if self.options.verbose: print('Skipping revert of {}'.format(self.module_dir)) return if self.options.verbose: print('Checking whether', self.module_dir, 'needs being reverted') - + if not self.repository.is_clean(): self.repository.revert() - def update_module_dir (self): + def update_module_dir(self): if self.options.fast_checks: if self.options.verbose: print('Skipping update of {}'.format(self.module_dir)) return @@ -691,9 +527,9 @@ that for other purposes than tagging""".format(options.workdir)) else: self.repository.update() - def main_specname (self): + def main_specname(self): attempt = "{}/{}.spec".format(self.module_dir, self.name) - if os.path.isfile (attempt): + if os.path.isfile(attempt): return attempt pattern1 = "{}/*.spec".format(self.module_dir) level1 = glob(pattern1) @@ -707,21 +543,21 @@ that for other purposes than tagging""".format(options.workdir)) raise Exception('Cannot guess specfile for module {} -- patterns were {} or {}'\ .format(self.pathname,pattern1,pattern2)) - def all_specnames (self): + def all_specnames(self): level1 = glob("{}/*.spec".format(self.module_dir)) if level1: return level1 level2 = glob("{}/*/*.spec".format(self.module_dir)) return level2 - def parse_spec (self, specfile, varnames): + def parse_spec(self, specfile, varnames): if self.options.verbose: print('Parsing',specfile, end=' ') for var in varnames: print("[{}]".format(var), end=' ') print("") result={} - with open(specfile) as f: + with open(specfile, encoding='utf-8') as f: for line in f.readlines(): attempt = Module.matcher_rpm_define.match(line) if attempt: @@ -733,10 +569,10 @@ that for other purposes than tagging""".format(options.workdir)) for k, v in result.items(): print('{} = {}'.format(k, v)) return result - + # stores in self.module_name_varname the rpm variable to be used for the module's name # and the list of these names in self.varnames - def spec_dict (self): + def spec_dict(self): specfile = self.main_specname() redirector_keys = [ varname for (varname, default) in Module.redirectors] redirect_dict = self.parse_spec(specfile, redirector_keys) @@ -749,23 +585,23 @@ that for other purposes than tagging""".format(options.workdir)) varnames += [redirect_dict[varname]] else: setattr(self, varname, default) - varnames += [ default ] + varnames += [ default ] self.varnames = varnames - result = self.parse_spec (specfile, self.varnames) + result = self.parse_spec(specfile, self.varnames) if self.options.debug: print('2st pass parsing done, varnames={} result={}'.format(varnames, result)) return result - def patch_spec_var (self, patch_dict,define_missing=False): + def patch_spec_var(self, patch_dict,define_missing=False): for specfile in self.all_specnames(): # record the keys that were changed - changed = dict ( [ (x,False) for x in list(patch_dict.keys()) ] ) + changed = {x : False for x in list(patch_dict.keys()) } newspecfile = "{}.new".format(specfile) if self.options.verbose: print('Patching', specfile, 'for', list(patch_dict.keys())) - with open (specfile) as spec: - with open(newspecfile, "w") as new: + with open(specfile, encoding='utf-8') as spec: + with open(newspecfile, "w", encoding='utf-8') as new: for line in spec.readlines(): attempt = Module.matcher_rpm_define.match(line) if attempt: @@ -786,7 +622,7 @@ that for other purposes than tagging""".format(options.workdir)) os.rename(newspecfile, specfile) # returns all lines until the magic line - def unignored_lines (self, logfile): + def unignored_lines(self, logfile): result = [] white_line_matcher = re.compile("\A\s*\Z") with open(logfile) as f: @@ -800,19 +636,19 @@ that for other purposes than tagging""".format(options.workdir)) return result # creates a copy of the input with only the unignored lines - def strip_magic_line_filename (self, filein, fileout ,new_tag_name): + def strip_magic_line_filename(self, filein, fileout ,new_tag_name): with open(fileout,'w') as f: f.write(self.setting_tag_format.format(new_tag_name) + '\n') for line in self.unignored_lines(filein): f.write(line) - def insert_changelog (self, logfile, newtag): + def insert_changelog(self, logfile, newtag): for specfile in self.all_specnames(): newspecfile = "{}.new".format(specfile) if self.options.verbose: print('Inserting changelog from {} into {}'.format(logfile, specfile)) - with open (specfile) as spec: + with open(specfile) as spec: with open(newspecfile,"w") as new: for line in spec.readlines(): new.write(line) @@ -827,13 +663,13 @@ that for other purposes than tagging""".format(options.workdir)) new.write("- " + logline) new.write("\n") os.rename(newspecfile,specfile) - - def show_dict (self, spec_dict): + + def show_dict(self, spec_dict): if self.options.verbose: for k, v in spec_dict.items(): print('{} = {}'.format(k, v)) - def last_tag (self, spec_dict): + def last_tag(self, spec_dict): try: return "{}-{}".format(spec_dict[self.module_version_varname], spec_dict[self.module_taglevel_varname]) @@ -841,17 +677,14 @@ that for other purposes than tagging""".format(options.workdir)) raise Exception('Something is wrong with module {}, cannot determine {} - exiting'\ .format(self.name, err)) - def tag_name (self, spec_dict, old_svn_name=False): - base_tag_name = self.name - if old_svn_name: - base_tag_name = git_to_svn_name(self.name) - return "{}-{}".format(base_tag_name, self.last_tag(spec_dict)) - + def tag_name(self, spec_dict): + return "{}-{}".format(self.name, self.last_tag(spec_dict)) + - pattern_format="\A\s*{module}-(SVNPATH|GITPATH)\s*(=|:=)\s*(?P[^\s]+)/{module}[^\s]+" + pattern_format="\A\s*{module}-(GITPATH)\s*(=|:=)\s*(?P[^\s]+)/{module}[^\s]+" - def is_mentioned_in_tagsfile (self, tagsfile): - # so that %(module)s gets replaced from format + def is_mentioned_in_tagsfile(self, tagsfile): + # so that {module} gets replaced from format module = self.name module_matcher = re.compile(Module.pattern_format.format(**locals())) with open(tagsfile) as f: @@ -862,8 +695,8 @@ that for other purposes than tagging""".format(options.workdir)) ############################## # using fine_grain means replacing only those instances that currently refer to this tag - # otherwise, -{SVNPATH,GITPATH} is replaced unconditionnally - def patch_tags_file (self, tagsfile, oldname, newname, fine_grain=True): + # otherwise, -GITPATH is replaced unconditionnally + def patch_tags_file(self, tagsfile, oldname, newname, fine_grain=True): newtagsfile = "{}.new".format(tagsfile) with open(tagsfile) as tags: @@ -881,10 +714,10 @@ that for other purposes than tagging""".format(options.workdir)) begin, end = matcher.match(line).groups() new.write(begin+newname+end+"\n") matches += 1 - # brute-force : change uncommented lines that define -SVNPATH + # brute-force : change uncommented lines that define -GITPATH else: if self.options.verbose: - print('Searching for -SVNPATH or -GITPATH lines referring to /{}/\n\tin {} .. '\ + print('Searching for -GITPATH lines referring to /{}/\n\tin {} .. '\ .format(self.pathname, tagsfile), end=' ') # so that {module} gets replaced from format module = self.name @@ -897,34 +730,27 @@ that for other purposes than tagging""".format(options.workdir)) replacement = "{:<32}:= {}/{}.git@{}\n"\ .format(modulepath, attempt.group('url_main'), self.pathname, newname) else: - modulepath = "{}-SVNPATH".format(self.name) - replacement = "{:-32}:= {}/{}/tags/{}\n"\ - .format(modulepath, attempt.group('url_main'), self.name, newname) + print("Could not locate {}-GITPATH (be aware that support for svn has been removed)"\ + .format(self.name)) + return if self.options.verbose: - print(' ' + modulepath, end=' ') + print(' ' + modulepath, end=' ') new.write(replacement) matches += 1 else: new.write(line) - + os.rename(newtagsfile,tagsfile) if self.options.verbose: print("{} changes".format(matches)) return matches - def check_tag(self, tagname, need_it=False, old_svn_tag_name=None): + def check_tag(self, tagname, need_it=False): if self.options.verbose: print("Checking {} repository tag: {} - ".format(self.repository.type, tagname), end=' ') found_tagname = tagname found = self.repository.tag_exists(tagname) - if not found and old_svn_tag_name: - if self.options.verbose: - print("KO") - print("Checking {} repository tag: {} - ".format(self.repository.type, old_svn_tag_name), end=' ') - found = self.repository.tag_exists(old_svn_tag_name) - if found: - found_tagname = old_svn_tag_name if (found and need_it) or (not found and not need_it): if self.options.verbose: @@ -940,27 +766,27 @@ that for other purposes than tagging""".format(options.workdir)) ############################## - def do_tag (self): + def do_tag(self): self.init_module_dir() self.revert_module_dir() self.update_module_dir() # parse specfile spec_dict = self.spec_dict() self.show_dict(spec_dict) - + # compute previous tag - if not bypassed if not self.options.bypass: old_tag_name = self.tag_name(spec_dict) # sanity check old_tag_name = self.check_tag(old_tag_name, need_it=True) - if (self.options.new_version): + if self.options.new_version: # new version set on command line spec_dict[self.module_version_varname] = self.options.new_version spec_dict[self.module_taglevel_varname] = 0 else: # increment taglevel - new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1) + new_taglevel = str( int(spec_dict[self.module_taglevel_varname]) + 1) spec_dict[self.module_taglevel_varname] = new_taglevel new_tag_name = self.tag_name(spec_dict) @@ -971,13 +797,13 @@ that for other purposes than tagging""".format(options.workdir)) if not self.options.bypass: diff_output = self.repository.diff_with_tag(old_tag_name) if len(diff_output) == 0: - if not prompt ("No pending difference in module %s, want to tag anyway"%self.pathname,False): + if not prompt("No pending difference in module {}, want to tag anyway".format(self.pathname), False): return # side effect in head's specfile self.patch_spec_var(spec_dict) - # prepare changelog file + # prepare changelog file # we use the standard subversion magic string (see edit_magic_line) # so we can provide useful information, such as version numbers and diff # in the same file @@ -988,26 +814,25 @@ that for other purposes than tagging""".format(options.workdir)) f.write(""" {} {} -Please write a changelog for this new tag in the section below +Please write a changelog for this new tag in the section above """.format(Module.edit_magic_line, setting_tag_line)) - if self.options.bypass: + if self.options.bypass: pass elif prompt('Want to see diffs while writing changelog', True): with open(changelog_plain, "a") as f: f.write('DIFF=========\n' + diff_output) - + if self.options.debug: prompt('Proceed ?') - # edit it + # edit it self.run("{} {}".format(self.options.editor, changelog_plain)) - # strip magic line in second file - looks like svn has changed its magic line with 1.6 - # so we do the job ourselves + # strip magic line in second file self.strip_magic_line_filename(changelog_plain, changelog_strip, new_tag_name) # insert changelog in spec if self.options.changelog: - self.insert_changelog (changelog_plain, new_tag_name) + self.insert_changelog(changelog_plain, new_tag_name) ## update build build_path = os.path.join(self.options.workdir, Module.config['build']) @@ -1018,7 +843,7 @@ Please write a changelog for this new tag in the section below build.revert() tagsfiles = glob(build.path+"/*-tags.mk") - tagsdict = dict( [ (x,'todo') for x in tagsfiles ] ) + tagsdict = { x : 'todo' for x in tagsfiles } default_answer = 'y' tagsfiles.sort() while True: @@ -1026,7 +851,7 @@ Please write a changelog for this new tag in the section below if self.options.bypass: break for tagsfile in tagsfiles: - if not self.is_mentioned_in_tagsfile (tagsfile): + if not self.is_mentioned_in_tagsfile(tagsfile): if self.options.verbose: print("tagsfile {} does not mention {} - skipped".format(tagsfile, self.name)) continue @@ -1034,9 +859,9 @@ Please write a changelog for this new tag in the section below basename = os.path.basename(tagsfile) print(".................... Dealing with {}".format(basename)) while tagsdict[tagsfile] == 'todo' : - choice = prompt ("insert {} in {} ".format(new_tag_name, basename), + choice = prompt("insert {} in {} ".format(new_tag_name, basename), default_answer, - [ ('y','es'), ('n', 'ext'), ('f','orce'), + [ ('y','es'), ('n', 'ext'), ('f','orce'), ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] , allow_outside=True) if choice == 'y': @@ -1055,15 +880,15 @@ Please write a changelog for this new tag in the section below else: name = self.name print( -"""y: change {name}-{{SVNPATH,GITPATH}} only if it currently refers to {old_tag_name} -f: unconditionnally change any line that assigns {name}-SVNPATH to using {new_tag_name} +"""y: change {name}-GITPATH only if it currently refers to {old_tag_name} +f: unconditionnally change any line that assigns {name}-GITPATH to using {new_tag_name} d: show current diff for this tag file r: revert that tag file c: cat the current tag file n: move to next file""".format(**locals())) if prompt("Want to review changes on tags files", False): - tagsdict = dict ( [ (x, 'todo') for x in tagsfiles ] ) + tagsdict = {x : 'todo' for x in tagsfiles } default_answer = 'd' else: break @@ -1091,7 +916,7 @@ n: move to next file""".format(**locals())) ############################## - def do_version (self): + def do_version(self): self.init_module_dir() self.revert_module_dir() self.update_module_dir() @@ -1101,19 +926,19 @@ n: move to next file""".format(**locals())) .format(self.friendly_name(), self.last_tag(spec_dict))) for varname in self.varnames: if varname not in spec_dict: - self.html_print ('Could not find %define for {}'.format(varname)) + self.html_print('Could not find %define for {}'.format(varname)) return else: - self.html_print ("{:<16} {}".format(varname, spec_dict[varname])) - self.html_print ("{:<16} {}".format('url', self.repository.url())) + self.html_print("{:<16} {}".format(varname, spec_dict[varname])) + self.html_print("{:<16} {}".format('url', self.repository.url())) if self.options.verbose: - self.html_print ("{:<16} {}".format('main specfile:', self.main_specname())) - self.html_print ("{:<16} {}".format('specfiles:', self.all_specnames())) + self.html_print("{:<16} {}".format('main specfile:', self.main_specname())) + self.html_print("{:<16} {}".format('specfiles:', self.all_specnames())) self.html_print_end() ############################## - def do_diff (self): + def do_diff(self): self.init_module_dir() self.revert_module_dir() self.update_module_dir() @@ -1122,10 +947,9 @@ n: move to next file""".format(**locals())) # side effects tag_name = self.tag_name(spec_dict) - old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True) # sanity check - tag_name = self.check_tag(tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name) + tag_name = self.check_tag(tag_name, need_it=True) if self.options.verbose: print('Getting diff') @@ -1141,9 +965,9 @@ n: move to next file""".format(**locals())) self.html_store_title("Diffs in module {} ({}) : {} chars"\ .format(thename, self.last_tag(spec_dict), len(diff_output))) - self.html_store_raw ('

< (left) {}

'"{:<16} {}".format(tag_name)) - self.html_store_raw ('

> (right) {}

'"{:<16} {}".format(thename)) - self.html_store_pre (diff_output) + self.html_store_raw('

< (left) {}

'.format(tag_name)) + self.html_store_raw('

> (right) {}

'.format(thename)) + self.html_store_pre(diff_output) elif not self.options.www: print('x'*30, 'module', thename) print('x'*20, '<', tag_name) @@ -1152,35 +976,35 @@ n: move to next file""".format(**locals())) ############################## # store and restitute html fragments - @staticmethod - def html_href (url,text): + @staticmethod + def html_href(url,text): return '{}'.format(url, text) - @staticmethod - def html_anchor (url,text): + @staticmethod + def html_anchor(url,text): return '{}'.format(url,text) @staticmethod - def html_quote (text): + def html_quote(text): return text.replace('&', '&').replace('<', '<').replace('>', '>') # only the fake error module has multiple titles - def html_store_title (self, title): + def html_store_title(self, title): if not hasattr(self,'titles'): self.titles=[] self.titles.append(title) - def html_store_raw (self, html): + def html_store_raw(self, html): if not hasattr(self,'body'): self.body='' self.body += html - def html_store_pre (self, text): + def html_store_pre(self, text): if not hasattr(self,'body'): self.body='' self.body += '
{}
'.format(self.html_quote(text)) - def html_print (self, txt): + def html_print(self, txt): if not self.options.www: print(txt) else: @@ -1189,9 +1013,9 @@ n: move to next file""".format(**locals())) self.in_list = True self.html_store_raw('
  • {}
    • '.format(txt)) - def html_print_end (self): + def html_print_end(self): if self.options.www: - self.html_store_raw ('') + self.html_store_raw('') @staticmethod def html_dump_header(title): @@ -1233,12 +1057,12 @@ span.error {{text-weight:bold; color: red; }} print('

    ', self.html_anchor(self.friendly_name(),title), '

    ') if hasattr(self,'body'): print(self.body) - print('

    ', self.html_href('#','Back to top'), '

    ') + print('

    ', self.html_href('#','Back to top'), '

    ') class Build(Module): - + def __get_modules(self, tagfile): self.init_module_dir() modules = {} @@ -1247,9 +1071,8 @@ class Build(Module): for line in open(tagfile): try: name, url = line.split(':=') - name, git_or_svn_path = name.rsplit('-', 1) - name = svn_to_git_name(name.strip()) - modules[name] = (git_or_svn_path.strip(), url.strip()) + name, git_path = name.rsplit('-', 1) + modules[name] = (git_path.strip(), url.strip()) except: pass return modules @@ -1269,20 +1092,20 @@ class Build(Module): module_type = "tag" elif url.find('/branches/') >= 0: module_type = "branch" - + modules[module] = {"module_type" : module_type, "path_type": path_type, "tag_or_branch": tag_or_branch, "url":url} return modules - - + + def modules_diff(first, second): diff = {} for module in first: - if module not in second: + if module not in second: print("=== module {} missing in right-hand side ===".format(module)) continue if first[module]['tag_or_branch'] != second[module]['tag_or_branch']: @@ -1351,15 +1174,12 @@ def release_changelog(options, buildtag_old, buildtag_new): os.system('rm -rf {}'.format(m.module_dir)) # cleanup module dir m.init_module_dir() - if m.repository.type == "svn": - print(' * from', first, m.repository.url()) - else: - print(' * from', first, m.repository.gitweb()) + print(' * from', first, m.repository.gitweb()) specfile = m.main_specname() (tmpfd, tmpfile) = tempfile.mkstemp() os.system("cp -f /{} {}".format(specfile, tmpfile)) - + m = get_module(module, second) # patch for ipfw that, being managed in a separate repo, won't work for now try: @@ -1371,10 +1191,7 @@ def release_changelog(options, buildtag_old, buildtag_new): continue specfile = m.main_specname() - if m.repository.type == "svn": - print(' * to', second, m.repository.url()) - else: - print(' * to', second, m.repository.gitweb()) + print(' * to', second, m.repository.gitweb()) print('{{{') os.system("diff -u {} {} | sed -e 's,{},[[previous version]],'"\ @@ -1390,11 +1207,11 @@ def release_changelog(options, buildtag_old, buildtag_new): print('=== {} : removed package from build {} ==='.format(tagfile_new, module)) -def adopt_tag (options, args): +def adopt_tag(options, args): modules=[] for module in options.modules: modules += module.split() - for module in modules: + for module in modules: modobj=Module(module,options) for tags_file in args: if options.verbose: @@ -1411,13 +1228,13 @@ class Main: module-tools : a set of tools to manage subversion tags and specfile requires the specfile to either * define *version* and *taglevel* - OR alternatively + OR alternatively * define redirection variables module_version_varname / module_taglevel_varname Master: by default, the 'master' branch of modules is the target in this case, just mention the module name as Branches: - if you wish to work on another branch, + if you wish to work on another branch, you can use something like e.g. Mom:2.1 as """ release_usage="""Usage: %prog [options] tag1 .. tagn @@ -1433,7 +1250,7 @@ Branches: """ adopt_usage="""Usage: %prog [options] tag-file[s] With this command you can adopt a specifi tag or branch in your tag files - This should be run in your daily build workdir; no call of git nor svn is done + This should be run in your daily build workdir; no call of git is done Examples: adopt-tag -m "plewww plcapi" -m Monitor onelab*tags.mk adopt-tag -m sfa -t sfa-1.0-33 *tags.mk @@ -1441,13 +1258,13 @@ Branches: common_usage="""More help: see http://svn.planet-lab.org/wiki/ModuleTools""" - modes = { + modes = { 'list' : "displays a list of available tags or branches", 'version' : "check latest specfile and print out details", 'diff' : "show difference between module (trunk or branch) and latest tag", 'tag' : """increment taglevel in specfile, insert changelog in specfile, create new tag and and monitor its adoption in build/*-tags.mk""", - 'branch' : """create a branch for this module, from the latest tag on the trunk, + 'branch' : """create a branch for this module, from the latest tag on the trunk, and change trunk's version number to reflect the new branch name; you can specify the new branch name by using module:branch""", 'sync' : """create a tag from the module @@ -1463,7 +1280,7 @@ Branches: regular_modes = set(modes.keys()).difference(set(['changelog','adopt'])) @staticmethod - def optparse_list (option, opt, value, parser): + def optparse_list(option, opt, value, parser): try: setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split()) except: @@ -1495,14 +1312,14 @@ Branches: usage += Main.common_usage parser=OptionParser(usage=usage) - + # the 'adopt' mode is really special and doesn't share any option if mode == 'adopt': parser.add_option("-m","--module",action="append",dest="modules",default=[], help="modules, can be used several times or with quotes") parser.add_option("-t","--tag",action="store", dest="tag", default='master', help="specify the tag to adopt, default is 'master'") - parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, help="run in verbose mode") (options, args) = parser.parse_args() options.workdir='unused' @@ -1511,8 +1328,8 @@ Branches: if len(args)==0 or len(options.modules)==0: parser.print_help() sys.exit(1) - adopt_tag (options,args) - return + adopt_tag(options,args) + return # the other commands (module-* and release-changelog) share the same skeleton if mode in [ 'tag', 'branch'] : @@ -1536,7 +1353,7 @@ Branches: if mode == 'diff' : parser.add_option("-l","--list", action="store_true", dest="list", default=False, help="just list modules that exhibit differences") - + default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list" parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, help="run on all modules as found in {}".format(default_modules_list)) @@ -1549,25 +1366,25 @@ Branches: help="""specify distro-tags files, e.g. onelab-tags-4.2.mk -- can be set multiple times, or use quotes""") - parser.add_option("-w","--workdir", action="store", dest="workdir", + parser.add_option("-w","--workdir", action="store", dest="workdir", default="{}/{}".format(os.getenv("HOME"),"modules"), help="""name for dedicated working dir - defaults to ~/modules ** THIS MUST NOT ** be your usual working directory""") parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False, - help="skip safety checks, such as svn updates -- use with care") + help="skip safety checks, such as git pulls -- use with care") parser.add_option("-B","--build-module",action="store",dest="build_module",default=None, help="specify a build module to owerride the one in the CONFIG") # default verbosity depending on function - temp verbose_modes= ['tag', 'sync', 'branch'] - + if mode not in verbose_modes: - parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, help="run in verbose mode") else: parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True, help="run in quiet (non-verbose) mode") - (options, args) = parser.parse_args() + options, args = parser.parse_args() options.mode=mode if not hasattr(options,'dry_run'): options.dry_run=False @@ -1575,7 +1392,7 @@ Branches: options.www=False options.debug=False - + ########## module-* if len(args) == 0: @@ -1587,10 +1404,10 @@ Branches: parser.print_help() sys.exit(1) Module.init_homedir(options) - + if mode in Main.regular_modes: - modules = [ Module(modname,options) for modname in args ] + modules = [ Module(modname, options) for modname in args ] # hack: create a dummy Module to store errors/warnings error_module = Module('__errors__',options) @@ -1610,8 +1427,8 @@ Branches: else: import traceback traceback.print_exc() - print('Skipping module {}: {}'.format(modname,e)) - + print('Skipping module {}: {}'.format(module.name,e)) + if options.www: if mode == "diff": modetitle="Changes to tag in {}".format(options.www) @@ -1631,8 +1448,8 @@ Branches: # we typically have here latest latest-1 latest-2 for (tag_new,tag_old) in zip ( args[:-1], args [1:]): release_changelog(options, tag_old, tag_new) - - + + #################### if __name__ == "__main__" : try: