3 import sys, os, os.path
8 from optparse import OptionParser
10 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase
11 def prompt(question, default=True, other_choices=[], allow_outside=False):
12 if not isinstance(other_choices, list):
13 other_choices = [ other_choices ]
14 chars = [ c for (c,rest) in other_choices ]
28 for char, choice in other_choices:
30 choices.append("[{}]{}".format(char, choice))
32 choices.append("<{}>{}>".format(char, choice))
34 answer = input(question + " " + "/".join(choices) + " ? ")
37 answer = answer[0].lower()
49 for (char,choice) in other_choices:
54 return prompt(question, default, other_choices)
60 editor = os.environ['EDITOR']
69 while len(line) >= fold_length:
70 print(line[:fold_length],'\\')
71 line = line[fold_length:]
75 def __init__(self, command, options):
76 self.command = command
77 self.options = options
78 self.tmp = "/tmp/command-{}".format(os.getpid())
81 if self.options.dry_run:
82 print('dry_run', self.command)
84 if self.options.verbose and self.options.mode not in Main.silent_modes:
85 print('+', self.command)
87 return os.system(self.command)
90 if self.options.dry_run:
91 print('dry_run', self.command)
93 if self.options.verbose:
94 print('>', os.getcwd())
95 print('+', self.command, ' .. ', end=' ')
97 retcod = os.system("{} &> {}".format(self.command, self.tmp))
99 print("FAILED ! -- out+err below (command was {})".format(self.command))
100 os.system("cat {}".format(self.tmp))
101 print("FAILED ! -- end of quoted output")
102 elif self.options.verbose:
108 if self.run_silent() != 0:
109 raise Exception("Command {} failed".format(self.command))
111 # returns stdout, like bash's $(mycommand)
112 def output_of(self, with_stderr=False, binary=False):
113 if self.options.dry_run:
114 print('dry_run', self.command)
115 return 'dry_run output'
116 tmp="/tmp/status-{}".format(os.getpid())
117 if self.options.debug:
118 print('+',self.command,' .. ', end=' ')
120 command = self.command
127 mode = "r" if not binary else "rb"
128 with open(tmp, mode) as f:
131 if self.options.debug:
132 print('Done', end=' ')
138 def __init__(self, path, options):
140 self.options = options
143 return os.path.basename(self.path)
146 return self.repo_root()
149 c = Command("git show | grep commit | awk '{{print $2;}}'", self.options)
150 out = self.__run_in_repo(c.output_of).strip()
151 return "http://git.onelab.eu/?p={}.git;a=commit;h={}".format(self.name(), out)
154 c = Command("git remote show origin", self.options)
155 out = self.__run_in_repo(c.output_of)
156 for line in out.split('\n'):
157 if line.strip().startswith("Fetch URL:"):
158 return line.split()[2]
161 def clone(cls, remote, local, options, depth=None):
162 Command("rm -rf {}".format(local), options).run_silent()
163 depth_option = "" if depth is None else " --depth {}".format(depth)
164 Command("git clone{} {} {}".format(depth_option, remote, local), options).run_fatal()
165 return GitRepository(local, options)
168 def remote_exists(cls, remote, options):
169 return Command("git --no-pager ls-remote {} &> /dev/null".format(remote), options).run()==0
171 def tag_exists(self, tagname):
172 command = 'git tag -l | grep "^{}$"'.format(tagname)
173 c = Command(command, self.options)
174 out = self.__run_in_repo(c.output_of, with_stderr=True)
177 def __run_in_repo(self, fun, *args, **kwargs):
180 ret = fun(*args, **kwargs)
184 def __run_command_in_repo(self, command, ignore_errors=False):
185 c = Command(command, self.options)
187 return self.__run_in_repo(c.output_of)
189 return self.__run_in_repo(c.run_fatal)
191 def __is_commit_id(self, id):
192 c = Command("git show {} | grep commit | awk '{{print $2;}}'".format(id), self.options)
193 ret = self.__run_in_repo(c.output_of, with_stderr=False)
194 if ret.strip() == id:
198 def update(self, subdir=None, recursive=None, branch="master"):
199 if branch == "master":
200 self.__run_command_in_repo("git checkout {}".format(branch))
202 self.to_branch(branch, remote=True)
203 self.__run_command_in_repo("git fetch origin --tags", ignore_errors=True)
204 self.__run_command_in_repo("git fetch origin")
205 if not self.__is_commit_id(branch):
206 # we don't need to merge anything for commit ids.
207 self.__run_command_in_repo("git merge --ff origin/{}".format(branch))
209 def to_branch(self, branch, remote=True):
212 command = "git branch --track {} origin/{}".format(branch, branch)
213 c = Command(command, self.options)
214 self.__run_in_repo(c.output_of, with_stderr=True)
215 return self.__run_command_in_repo("git checkout {}".format(branch))
217 def to_tag(self, tag):
219 return self.__run_command_in_repo("git checkout {}".format(tag))
221 def tag(self, tagname, logfile):
222 self.__run_command_in_repo("git tag {} -F {}".format(tagname, logfile))
225 def diff(self, f=""):
226 c = Command("git diff {}".format(f), self.options)
228 return self.__run_in_repo(c.output_of, with_stderr=True)
230 return self.__run_in_repo(c.output_of, with_stderr=True, binary=True)
232 def diff_with_tag(self, tagname):
233 c = Command("git diff {}".format(tagname), self.options)
235 return self.__run_in_repo(c.output_of, with_stderr=True)
237 return self.__run_in_repo(c.output_of, with_stderr=True, binary=True)
239 def commit(self, logfile, branch="master"):
240 self.__run_command_in_repo("git add .", ignore_errors=True)
241 self.__run_command_in_repo("git add -u", ignore_errors=True)
242 self.__run_command_in_repo("git commit -F {}".format(logfile), ignore_errors=True)
243 if branch == "master" or self.__is_commit_id(branch):
244 self.__run_command_in_repo("git push")
246 self.__run_command_in_repo("git push origin {}:{}".format(branch, branch))
247 self.__run_command_in_repo("git push --tags", ignore_errors=True)
249 def revert(self, f=""):
251 self.__run_command_in_repo("git checkout {}".format(f))
254 self.__run_command_in_repo("git --no-pager reset --hard")
255 self.__run_command_in_repo("git --no-pager clean -f")
259 command = "git status"
260 s = "nothing to commit, working directory clean"
261 return Command(command, self.options).output_of(True).find(s) >= 0
262 return self.__run_in_repo(check_commit)
265 return os.path.exists(os.path.join(self.path, ".git"))
271 From old times when we had svn and git
273 supported_repo_types = [ GitRepository ]
275 def __init__(self, path, options):
277 self.options = options
278 for repo_class in self.supported_repo_types:
279 self.repo = repo_class(self.path, self.options)
280 if self.repo.is_valid():
284 def remote_exists(cls, remote, options):
285 for repo_class in Repository.supported_repo_types:
286 if repo_class.remote_exists(remote, options):
290 def __getattr__(self, attr):
291 return getattr(self.repo, attr)
295 # support for tagged module is minimal, and is for the Build class only
298 edit_magic_line = "--This line, and those below, will be ignored--"
299 setting_tag_format = "Setting tag {}"
302 # ('module_name_varname', 'name'),
303 ('module_version_varname', 'version'),
304 ('module_taglevel_varname', 'taglevel'),
307 # where to store user's config
308 config_storage = "CONFIG"
314 ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),
315 ('gituser', "Enter your user name (login name) on git server", subprocess.getoutput("id -un")),
316 ("build", "Enter the name of your build module", "build"),
317 ('username', "Enter your firstname and lastname for changelogs", ""),
318 ("email", "Enter your email address for changelogs", ""),
322 def prompt_config_option(cls, key, message, default):
323 cls.config[key]=input("{} [{}] : ".format(message,default)).strip() or default
326 def prompt_config(cls):
327 for (key,message,default) in cls.configKeys:
329 while not cls.config[key]:
330 cls.prompt_config_option(key, message, default)
332 # for parsing module spec name:branch
333 matcher_branch_spec = re.compile("\A(?P<name>[\w\.\-\/]+):(?P<branch>[\w\.\-]+)\Z")
334 # special form for tagged module - for Build
335 matcher_tag_spec = re.compile("\A(?P<name>[\w\.\-\/]+)@(?P<tagname>[\w\.\-]+)\Z")
338 matcher_rpm_define = re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")
341 def parse_module_spec(cls, module_spec):
342 name = branch_or_tagname = module_type = ""
344 attempt = Module.matcher_branch_spec.match(module_spec)
346 module_type = "branch"
347 name=attempt.group('name')
348 branch_or_tagname=attempt.group('branch')
350 attempt = Module.matcher_tag_spec.match(module_spec)
353 name=attempt.group('name')
354 branch_or_tagname=attempt.group('tagname')
357 return name, branch_or_tagname, module_type
360 def __init__(self, module_spec, options):
362 self.pathname, branch_or_tagname, module_type = self.parse_module_spec(module_spec)
363 self.name = os.path.basename(self.pathname)
365 if module_type == "branch":
366 self.branch = branch_or_tagname
367 elif module_type == "tag":
368 self.tagname = branch_or_tagname
371 self.module_dir="{}/{}".format(options.workdir,self.pathname)
372 self.repository = None
375 def run(self,command):
376 return Command(command,self.options).run()
377 def run_fatal(self,command):
378 return Command(command,self.options).run_fatal()
379 def run_prompt(self,message,fun, *args):
380 fun_msg = "{}({})".format(fun.__name__, ",".join(args))
381 if not self.options.verbose:
383 choice = prompt(message, True, ('s','how'))
388 print('About to run function:', fun_msg)
390 question = "{} - want to run function: {}".format(message, fun_msg)
391 if prompt(question, True):
394 def friendly_name(self):
395 if hasattr(self, 'branch'):
396 return "{}:{}".format(self.pathname, self.branch)
397 elif hasattr(self, 'tagname'):
398 return "{}@{}".format(self.pathname, self.tagname)
403 def git_remote_dir(cls, name):
404 return "{}@{}:/git/{}.git".format(cls.config['gituser'], cls.config['gitserver'], name)
408 def init_homedir(cls, options):
409 if options.verbose and options.mode not in Main.silent_modes:
410 print('Checking for', options.workdir)
411 storage="{}/{}".format(options.workdir, cls.config_storage)
412 # sanity check. Either the topdir exists AND we have a config/storage
413 # or topdir does not exist and we create it
414 # to avoid people use their own daily work repo
415 if os.path.isdir(options.workdir) and not os.path.isfile(storage):
416 print("""The directory {} exists and has no CONFIG file
417 If this is your regular working directory, please provide another one as the
418 module-* commands need a fresh working dir. Make sure that you do not use
419 that for other purposes than tagging""".format(options.workdir))
423 print("Checking out build module...")
424 remote = cls.git_remote_dir(cls.config['build'])
425 local = os.path.join(options.workdir, cls.config['build'])
426 GitRepository.clone(remote, local, options, depth=1)
430 with open(storage, 'w') as f:
431 for (key, message, default) in Module.configKeys:
432 f.write("{}={}\n".format(key, Module.config[key]))
434 print('Stored', storage)
435 Command("cat {}".format(storage),options).run()
439 with open(storage) as f:
440 for line in f.readlines():
441 key, value = re.compile("^(.+)=(.+)$").match(line).groups()
442 Module.config[key] = value
444 # owerride config variables using options.
445 if options.build_module:
446 Module.config['build'] = options.build_module
448 if not os.path.isdir(options.workdir):
449 print("Cannot find {}, let's create it".format(options.workdir))
450 Command("mkdir -p {}".format(options.workdir), options).run_silent()
456 # check missing config options
458 for key, message, default in cls.configKeys:
459 if key not in Module.config:
460 print("Configuration changed for module-tools")
461 cls.prompt_config_option(key, message, default)
465 Command("rm -rf {}".format(options.workdir), options).run_silent()
466 Command("mkdir -p {}".format(options.workdir), options).run_silent()
470 build_dir = os.path.join(options.workdir, cls.config['build'])
471 if not os.path.isdir(build_dir):
474 build = Repository(build_dir, options)
475 if not build.is_clean():
476 print("build module needs a revert")
481 if options.verbose and options.mode not in Main.silent_modes:
482 print('******** Using config')
483 for (key,message,default) in Module.configKeys:
484 print('\t{} = {}'.format(key,Module.config[key]))
486 def init_module_dir(self):
487 if self.options.verbose:
488 print('Checking for', self.module_dir)
490 if not os.path.isdir(self.module_dir):
491 self.repository = GitRepository.clone(self.git_remote_dir(self.pathname),
495 self.repository = Repository(self.module_dir, self.options)
497 if self.repository.type == "git":
498 if hasattr(self, 'branch'):
499 self.repository.to_branch(self.branch)
500 elif hasattr(self, 'tagname'):
501 self.repository.to_tag(self.tagname)
503 raise Exception('Cannot find {} - or not a git module'.format(self.module_dir))
506 def revert_module_dir(self):
507 if self.options.fast_checks:
508 if self.options.verbose: print('Skipping revert of {}'.format(self.module_dir))
510 if self.options.verbose:
511 print('Checking whether', self.module_dir, 'needs being reverted')
513 if not self.repository.is_clean():
514 self.repository.revert()
516 def update_module_dir(self):
517 if self.options.fast_checks:
518 if self.options.verbose: print('Skipping update of {}'.format(self.module_dir))
520 if self.options.verbose:
521 print('Updating', self.module_dir)
523 if hasattr(self, 'branch'):
524 self.repository.update(branch=self.branch)
525 elif hasattr(self, 'tagname'):
526 self.repository.update(branch=self.tagname)
528 self.repository.update()
530 def main_specname(self):
531 attempt = "{}/{}.spec".format(self.module_dir, self.name)
532 if os.path.isfile(attempt):
534 pattern1 = "{}/*.spec".format(self.module_dir)
535 level1 = glob(pattern1)
538 pattern2 = "{}/*/*.spec".format(self.module_dir)
539 level2 = glob(pattern2)
543 raise Exception('Cannot guess specfile for module {} -- patterns were {} or {}'\
544 .format(self.pathname,pattern1,pattern2))
546 def all_specnames(self):
547 level1 = glob("{}/*.spec".format(self.module_dir))
550 level2 = glob("{}/*/*.spec".format(self.module_dir))
553 def parse_spec(self, specfile, varnames):
554 if self.options.verbose:
555 print('Parsing',specfile, end=' ')
557 print("[{}]".format(var), end=' ')
560 with open(specfile, encoding='utf-8') as f:
561 for line in f.readlines():
562 attempt = Module.matcher_rpm_define.match(line)
564 define, var, value = attempt.groups()
567 if self.options.debug:
568 print('found {} keys'.format(len(result)))
569 for k, v in result.items():
570 print('{} = {}'.format(k, v))
573 # stores in self.module_name_varname the rpm variable to be used for the module's name
574 # and the list of these names in self.varnames
576 specfile = self.main_specname()
577 redirector_keys = [ varname for (varname, default) in Module.redirectors]
578 redirect_dict = self.parse_spec(specfile, redirector_keys)
579 if self.options.debug:
580 print('1st pass parsing done, redirect_dict=', redirect_dict)
582 for varname, default in Module.redirectors:
583 if varname in redirect_dict:
584 setattr(self, varname, redirect_dict[varname])
585 varnames += [redirect_dict[varname]]
587 setattr(self, varname, default)
588 varnames += [ default ]
589 self.varnames = varnames
590 result = self.parse_spec(specfile, self.varnames)
591 if self.options.debug:
592 print('2st pass parsing done, varnames={} result={}'.format(varnames, result))
595 def patch_spec_var(self, patch_dict,define_missing=False):
596 for specfile in self.all_specnames():
597 # record the keys that were changed
598 changed = {x : False for x in list(patch_dict.keys()) }
599 newspecfile = "{}.new".format(specfile)
600 if self.options.verbose:
601 print('Patching', specfile, 'for', list(patch_dict.keys()))
603 with open(specfile, encoding='utf-8') as spec:
604 with open(newspecfile, "w", encoding='utf-8') as new:
605 for line in spec.readlines():
606 attempt = Module.matcher_rpm_define.match(line)
608 define, var, value = attempt.groups()
609 if var in list(patch_dict.keys()):
610 if self.options.debug:
611 print('rewriting {} as {}'.format(var, patch_dict[var]))
612 new.write('%{} {} {}\n'.format(define, var, patch_dict[var]))
617 for key, was_changed in changed.items():
619 if self.options.debug:
620 print('rewriting missing {} as {}'.format(key, patch_dict[key]))
621 new.write('\n%define {} {}\n'.format(key, patch_dict[key]))
622 os.rename(newspecfile, specfile)
624 # returns all lines until the magic line
625 def unignored_lines(self, logfile):
627 white_line_matcher = re.compile("\A\s*\Z")
628 with open(logfile) as f:
629 for logline in f.readlines():
630 if logline.strip() == Module.edit_magic_line:
632 elif white_line_matcher.match(logline):
635 result.append(logline.strip()+'\n')
638 # creates a copy of the input with only the unignored lines
639 def strip_magic_line_filename(self, filein, fileout ,new_tag_name):
640 with open(fileout,'w') as f:
641 f.write(self.setting_tag_format.format(new_tag_name) + '\n')
642 for line in self.unignored_lines(filein):
645 def insert_changelog(self, logfile, newtag):
646 for specfile in self.all_specnames():
647 newspecfile = "{}.new".format(specfile)
648 if self.options.verbose:
649 print('Inserting changelog from {} into {}'.format(logfile, specfile))
651 with open(specfile) as spec:
652 with open(newspecfile,"w") as new:
653 for line in spec.readlines():
655 if re.compile('%changelog').match(line):
656 dateformat="* %a %b %d %Y"
657 datepart=time.strftime(dateformat)
658 logpart="{} <{}> - {}".format(Module.config['username'],
659 Module.config['email'],
661 new.write("{} {}\n".format(datepart,logpart))
662 for logline in self.unignored_lines(logfile):
663 new.write("- " + logline)
665 os.rename(newspecfile,specfile)
667 def show_dict(self, spec_dict):
668 if self.options.verbose:
669 for k, v in spec_dict.items():
670 print('{} = {}'.format(k, v))
672 def last_tag(self, spec_dict):
674 return "{}-{}".format(spec_dict[self.module_version_varname],
675 spec_dict[self.module_taglevel_varname])
676 except KeyError as err:
677 raise Exception('Something is wrong with module {}, cannot determine {} - exiting'\
678 .format(self.name, err))
680 def tag_name(self, spec_dict):
681 return "{}-{}".format(self.name, self.last_tag(spec_dict))
684 pattern_format="\A\s*{module}-(GITPATH)\s*(=|:=)\s*(?P<url_main>[^\s]+)/{module}[^\s]+"
686 def is_mentioned_in_tagsfile(self, tagsfile):
687 # so that {module} gets replaced from format
689 module_matcher = re.compile(Module.pattern_format.format(**locals()))
690 with open(tagsfile) as f:
691 for line in f.readlines():
692 if module_matcher.match(line):
696 ##############################
697 # using fine_grain means replacing only those instances that currently refer to this tag
698 # otherwise, <module>-GITPATH is replaced unconditionnally
699 def patch_tags_file(self, tagsfile, oldname, newname, fine_grain=True):
700 newtagsfile = "{}.new".format(tagsfile)
702 with open(tagsfile) as tags:
703 with open(newtagsfile,"w") as new:
705 # fine-grain : replace those lines that refer to oldname
707 if self.options.verbose:
708 print('Replacing {} into {}\n\tin {} .. '.format(oldname, newname, tagsfile), end=' ')
709 matcher = re.compile("^(.*){}(.*)".format(oldname))
710 for line in tags.readlines():
711 if not matcher.match(line):
714 begin, end = matcher.match(line).groups()
715 new.write(begin+newname+end+"\n")
717 # brute-force : change uncommented lines that define <module>-GITPATH
719 if self.options.verbose:
720 print('Searching for -GITPATH lines referring to /{}/\n\tin {} .. '\
721 .format(self.pathname, tagsfile), end=' ')
722 # so that {module} gets replaced from format
724 module_matcher = re.compile(Module.pattern_format.format(**locals()))
725 for line in tags.readlines():
726 attempt = module_matcher.match(line)
728 if line.find("-GITPATH") >= 0:
729 modulepath = "{}-GITPATH".format(self.name)
730 replacement = "{:<32}:= {}/{}.git@{}\n"\
731 .format(modulepath, attempt.group('url_main'), self.pathname, newname)
733 print("Could not locate {}-GITPATH (be aware that support for svn has been removed)"\
736 if self.options.verbose:
737 print(' ' + modulepath, end=' ')
738 new.write(replacement)
743 os.rename(newtagsfile,tagsfile)
744 if self.options.verbose:
745 print("{} changes".format(matches))
748 def check_tag(self, tagname, need_it=False):
749 if self.options.verbose:
750 print("Checking {} repository tag: {} - ".format(self.repository.type, tagname), end=' ')
752 found_tagname = tagname
753 found = self.repository.tag_exists(tagname)
755 if (found and need_it) or (not found and not need_it):
756 if self.options.verbose:
758 print("- found" if found else "- not found")
760 if self.options.verbose:
762 exception_format = "tag ({}) is already there" if found else "can not find required tag ({})"
763 raise Exception(exception_format.format(tagname))
768 ##############################
770 self.init_module_dir()
771 self.revert_module_dir()
772 self.update_module_dir()
774 spec_dict = self.spec_dict()
775 self.show_dict(spec_dict)
777 # compute previous tag - if not bypassed
778 if not self.options.bypass:
779 old_tag_name = self.tag_name(spec_dict)
781 old_tag_name = self.check_tag(old_tag_name, need_it=True)
783 if self.options.new_version:
784 # new version set on command line
785 spec_dict[self.module_version_varname] = self.options.new_version
786 spec_dict[self.module_taglevel_varname] = 0
789 new_taglevel = str( int(spec_dict[self.module_taglevel_varname]) + 1)
790 spec_dict[self.module_taglevel_varname] = new_taglevel
792 new_tag_name = self.tag_name(spec_dict)
794 new_tag_name = self.check_tag(new_tag_name, need_it=False)
797 if not self.options.bypass:
798 diff_output = self.repository.diff_with_tag(old_tag_name)
799 if len(diff_output) == 0:
800 if not prompt("No pending difference in module {}, want to tag anyway".format(self.pathname), False):
803 # side effect in head's specfile
804 self.patch_spec_var(spec_dict)
806 # prepare changelog file
807 # we use the standard subversion magic string (see edit_magic_line)
808 # so we can provide useful information, such as version numbers and diff
810 changelog_plain = "/tmp/{}-{}.edit".format(self.name, os.getpid())
811 changelog_strip = "/tmp/{}-{}.strip".format(self.name, os.getpid())
812 setting_tag_line = Module.setting_tag_format.format(new_tag_name)
813 with open(changelog_plain, 'w') as f:
817 Please write a changelog for this new tag in the section above
818 """.format(Module.edit_magic_line, setting_tag_line))
820 if self.options.bypass:
822 elif prompt('Want to see diffs while writing changelog', True):
823 with open(changelog_plain, "a") as f:
824 f.write('DIFF=========\n' + diff_output)
826 if self.options.debug:
830 self.run("{} {}".format(self.options.editor, changelog_plain))
831 # strip magic line in second file
832 self.strip_magic_line_filename(changelog_plain, changelog_strip, new_tag_name)
833 # insert changelog in spec
834 if self.options.changelog:
835 self.insert_changelog(changelog_plain, new_tag_name)
838 build_path = os.path.join(self.options.workdir, Module.config['build'])
839 build = Repository(build_path, self.options)
840 if self.options.build_branch:
841 build.to_branch(self.options.build_branch)
842 if not build.is_clean():
845 tagsfiles = glob(build.path+"/*-tags.mk")
846 tagsdict = { x : 'todo' for x in tagsfiles }
850 # do not bother if in bypass mode
851 if self.options.bypass:
853 for tagsfile in tagsfiles:
854 if not self.is_mentioned_in_tagsfile(tagsfile):
855 if self.options.verbose:
856 print("tagsfile {} does not mention {} - skipped".format(tagsfile, self.name))
858 status = tagsdict[tagsfile]
859 basename = os.path.basename(tagsfile)
860 print(".................... Dealing with {}".format(basename))
861 while tagsdict[tagsfile] == 'todo' :
862 choice = prompt("insert {} in {} ".format(new_tag_name, basename),
864 [ ('y','es'), ('n', 'ext'), ('f','orce'),
865 ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,
868 self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=True)
870 print('Done with {}'.format(os.path.basename(tagsfile)))
871 tagsdict[tagsfile] = 'done'
873 self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=False)
875 print(build.diff(f=os.path.basename(tagsfile)))
877 build.revert(f=tagsfile)
879 self.run("cat {}".format(tagsfile))
883 """y: change {name}-GITPATH only if it currently refers to {old_tag_name}
884 f: unconditionnally change any line that assigns {name}-GITPATH to using {new_tag_name}
885 d: show current diff for this tag file
886 r: revert that tag file
887 c: cat the current tag file
888 n: move to next file""".format(**locals()))
890 if prompt("Want to review changes on tags files", False):
891 tagsdict = {x : 'todo' for x in tagsfiles }
896 def diff_all_changes():
898 print(self.repository.diff())
900 def commit_all_changes(log):
901 if hasattr(self, 'branch'):
902 self.repository.commit(log, branch=self.branch)
904 self.repository.commit(log)
907 self.run_prompt("Review module and build", diff_all_changes)
908 self.run_prompt("Commit module and build", commit_all_changes, changelog_strip)
909 self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_strip)
911 if self.options.debug:
912 print('Preserving {} and stripped {}', changelog_plain, changelog_strip)
914 os.unlink(changelog_plain)
915 os.unlink(changelog_strip)
918 ##############################
919 def do_version(self):
920 self.init_module_dir()
921 self.revert_module_dir()
922 self.update_module_dir()
923 spec_dict = self.spec_dict()
925 self.html_store_title('Version for module {} ({})'\
926 .format(self.friendly_name(), self.last_tag(spec_dict)))
927 for varname in self.varnames:
928 if varname not in spec_dict:
929 self.html_print('Could not find %define for {}'.format(varname))
932 self.html_print("{:<16} {}".format(varname, spec_dict[varname]))
933 self.html_print("{:<16} {}".format('url', self.repository.url()))
934 if self.options.verbose:
935 self.html_print("{:<16} {}".format('main specfile:', self.main_specname()))
936 self.html_print("{:<16} {}".format('specfiles:', self.all_specnames()))
937 self.html_print_end()
940 ##############################
942 self.init_module_dir()
943 self.revert_module_dir()
944 self.update_module_dir()
945 spec_dict = self.spec_dict()
946 self.show_dict(spec_dict)
949 tag_name = self.tag_name(spec_dict)
952 tag_name = self.check_tag(tag_name, need_it=True)
954 if self.options.verbose:
955 print('Getting diff')
956 diff_output = self.repository.diff_with_tag(tag_name)
958 if self.options.list:
962 thename = self.friendly_name()
964 if self.options.www and diff_output:
965 self.html_store_title("Diffs in module {} ({}) : {} chars"\
966 .format(thename, self.last_tag(spec_dict), len(diff_output)))
968 self.html_store_raw('<p> < (left) {} </p>'.format(tag_name))
969 self.html_store_raw('<p> > (right) {} </p>'.format(thename))
970 self.html_store_pre(diff_output)
971 elif not self.options.www:
972 print('x'*30, 'module', thename)
973 print('x'*20, '<', tag_name)
974 print('x'*20, '>', thename)
977 ##############################
978 # store and restitute html fragments
980 def html_href(url,text):
981 return '<a href="{}">{}</a>'.format(url, text)
984 def html_anchor(url,text):
985 return '<a name="{}">{}</a>'.format(url,text)
988 def html_quote(text):
989 return text.replace('&', '&').replace('<', '<').replace('>', '>')
991 # only the fake error module has multiple titles
992 def html_store_title(self, title):
993 if not hasattr(self,'titles'):
995 self.titles.append(title)
997 def html_store_raw(self, html):
998 if not hasattr(self,'body'):
1002 def html_store_pre(self, text):
1003 if not hasattr(self,'body'):
1005 self.body += '<pre>{}</pre>'.format(self.html_quote(text))
1007 def html_print(self, txt):
1008 if not self.options.www:
1011 if not hasattr(self, 'in_list') or not self.in_list:
1012 self.html_store_raw('<ul>')
1014 self.html_store_raw('<li>{}</li>'.format(txt))
1016 def html_print_end(self):
1017 if self.options.www:
1018 self.html_store_raw('</ul>')
1021 def html_dump_header(title):
1022 nowdate = time.strftime("%Y-%m-%d")
1023 nowtime = time.strftime("%H:%M (%Z)")
1024 print("""<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
1025 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
1028 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
1029 <style type="text/css">
1030 body {{ font-family:georgia, serif; }}
1031 h1 {{font-size: large; }}
1032 p.title {{font-size: x-large; }}
1033 span.error {{text-weight:bold; color: red; }}
1037 <p class='title'> {} - status on {} at {}</p>
1039 """.format(title, title, nowdate, nowtime))
1042 def html_dump_middle():
1046 def html_dump_footer():
1047 print("</body></html")
1049 def html_dump_toc(self):
1050 if hasattr(self,'titles'):
1051 for title in self.titles:
1052 print('<li>', self.html_href('#'+self.friendly_name(),title), '</li>')
1054 def html_dump_body(self):
1055 if hasattr(self,'titles'):
1056 for title in self.titles:
1057 print('<hr /><h1>', self.html_anchor(self.friendly_name(),title), '</h1>')
1058 if hasattr(self,'body'):
1060 print('<p class="top">', self.html_href('#','Back to top'), '</p>')
1064 class Build(Module):
1066 def __get_modules(self, tagfile):
1067 self.init_module_dir()
1070 tagfile = os.path.join(self.module_dir, tagfile)
1071 for line in open(tagfile):
1073 name, url = line.split(':=')
1074 name, git_path = name.rsplit('-', 1)
1075 modules[name] = (git_path.strip(), url.strip())
1080 def get_modules(self, tagfile):
1081 modules = self.__get_modules(tagfile)
1082 for module in modules:
1083 module_type = tag_or_branch = ""
1085 path_type, url = modules[module]
1086 if path_type == "GITPATH":
1087 module_spec = os.path.split(url)[-1].replace(".git","")
1088 name, tag_or_branch, module_type = self.parse_module_spec(module_spec)
1090 tag_or_branch = os.path.split(url)[-1].strip()
1091 if url.find('/tags/') >= 0:
1093 elif url.find('/branches/') >= 0:
1094 module_type = "branch"
1096 modules[module] = {"module_type" : module_type,
1097 "path_type": path_type,
1098 "tag_or_branch": tag_or_branch,
1104 def modules_diff(first, second):
1107 for module in first:
1108 if module not in second:
1109 print("=== module {} missing in right-hand side ===".format(module))
1111 if first[module]['tag_or_branch'] != second[module]['tag_or_branch']:
1112 diff[module] = (first[module]['tag_or_branch'], second[module]['tag_or_branch'])
1114 first_set = set(first.keys())
1115 second_set = set(second.keys())
1117 new_modules = list(second_set - first_set)
1118 removed_modules = list(first_set - second_set)
1120 return diff, new_modules, removed_modules
1122 def release_changelog(options, buildtag_old, buildtag_new):
1124 # the command line expects new old, so we treat the tagfiles in the same order
1125 nb_tags = len(options.distrotags)
1127 tagfile_new = tagfile_old = options.distrotags[0]
1129 tagfile_new, tagfile_old = options.distrotags
1131 print("ERROR: provide one or two tagfile name (eg. onelab-k32-tags.mk)")
1132 print("two tagfiles can be mentioned when a tagfile has been renamed")
1136 print("------------------------------ Computing Changelog from")
1137 print("buildtag_old", buildtag_old, "tagfile_old", tagfile_old)
1138 print("buildtag_new", buildtag_new, "tagfile_new", tagfile_new)
1144 print('= build tag {} to {} ='.format(buildtag_old, buildtag_new))
1145 print('== distro {} ({} to {}) =='.format(tagfile_new, buildtag_old, buildtag_new))
1147 build = Build("build@{}".format(buildtag_old), options)
1148 build.init_module_dir()
1149 first = build.get_modules(tagfile_old)
1151 print(' * from', buildtag_old, build.repository.gitweb())
1153 build = Build("build@{}".format(buildtag_new), options)
1154 build.init_module_dir()
1155 second = build.get_modules(tagfile_new)
1157 print(' * to', buildtag_new, build.repository.gitweb())
1159 diff, new_modules, removed_modules = modules_diff(first, second)
1162 def get_module(name, tag):
1163 if not tag or tag == "trunk":
1164 return Module("{}".format(module), options)
1166 return Module("{}@{}".format(module, tag), options)
1170 print('=== {} - {} to {} : package {} ==='.format(tagfile_new, buildtag_old, buildtag_new, module))
1172 first, second = diff[module]
1173 m = get_module(module, first)
1174 os.system('rm -rf {}'.format(m.module_dir)) # cleanup module dir
1177 print(' * from', first, m.repository.gitweb())
1179 specfile = m.main_specname()
1180 (tmpfd, tmpfile) = tempfile.mkstemp()
1181 os.system("cp -f /{} {}".format(specfile, tmpfile))
1183 m = get_module(module, second)
1184 # patch for ipfw that, being managed in a separate repo, won't work for now
1187 except Exception as e:
1188 print("""Could not retrieve module {} - skipped
1190 """.format( m.friendly_name(), e))
1192 specfile = m.main_specname()
1194 print(' * to', second, m.repository.gitweb())
1197 os.system("diff -u {} {} | sed -e 's,{},[[previous version]],'"\
1198 .format(tmpfile, specfile, tmpfile))
1203 for module in new_modules:
1204 print('=== {} : new package in build {} ==='.format(tagfile_new, module))
1206 for module in removed_modules:
1207 print('=== {} : removed package from build {} ==='.format(tagfile_new, module))
1210 def adopt_tag(options, args):
1212 for module in options.modules:
1213 modules += module.split()
1214 for module in modules:
1215 modobj=Module(module,options)
1216 for tags_file in args:
1218 print('adopting tag {} for {} in {}'.format(options.tag, module, tags_file))
1219 modobj.patch_tags_file(tags_file, '_unused_', options.tag, fine_grain=False)
1221 Command("git diff {}".format(" ".join(args)), options).run()
1223 ##############################
1226 module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]
1228 module-tools : a set of tools to manage subversion tags and specfile
1229 requires the specfile to either
1230 * define *version* and *taglevel*
1232 * define redirection variables module_version_varname / module_taglevel_varname
1234 by default, the 'master' branch of modules is the target
1235 in this case, just mention the module name as <module_desc>
1237 if you wish to work on another branch,
1238 you can use something like e.g. Mom:2.1 as <module_desc>
1240 release_usage="""Usage: %prog [options] tag1 .. tagn
1241 Extract release notes from the changes in specfiles between several build tags, latest first
1243 release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22
1244 You can refer to a (build) branch by prepending a colon, like in
1245 release-changelog :4.2 4.2-rc25
1246 You can refer to the build trunk by just mentioning 'trunk', e.g.
1247 release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk
1248 You can use 2 different tagfile names if that was renamed meanwhile
1249 release-changelog -t onelab-tags.mk 5.0-rc29 -t onelab-k32-tags.mk 5.0-rc28
1251 adopt_usage="""Usage: %prog [options] tag-file[s]
1252 With this command you can adopt a specifi tag or branch in your tag files
1253 This should be run in your daily build workdir; no call of git is done
1255 adopt-tag -m "plewww plcapi" -m Monitor onelab*tags.mk
1256 adopt-tag -m sfa -t sfa-1.0-33 *tags.mk
1258 common_usage="""More help:
1259 see http://svn.planet-lab.org/wiki/ModuleTools"""
1262 'list' : "displays a list of available tags or branches",
1263 'version' : "check latest specfile and print out details",
1264 'diff' : "show difference between module (trunk or branch) and latest tag",
1265 'tag' : """increment taglevel in specfile, insert changelog in specfile,
1266 create new tag and and monitor its adoption in build/*-tags.mk""",
1267 'branch' : """create a branch for this module, from the latest tag on the trunk,
1268 and change trunk's version number to reflect the new branch name;
1269 you can specify the new branch name by using module:branch""",
1270 'sync' : """create a tag from the module
1271 this is a last resort option, mostly for repairs""",
1272 'changelog' : """extract changelog between build tags
1273 expected arguments are a list of tags""",
1274 'adopt' : """locally adopt a specific tag""",
1277 silent_modes = ['list']
1278 # 'changelog' is for release-changelog
1279 # 'adopt' is for 'adopt-tag'
1280 regular_modes = set(modes.keys()).difference(set(['changelog','adopt']))
1283 def optparse_list(option, opt, value, parser):
1285 setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())
1287 setattr(parser.values,option.dest,value.split())
1292 # hack - need to check for adopt first as 'adopt-tag' contains tag..
1293 for function in [ 'adopt' ] + list(Main.modes.keys()):
1294 if sys.argv[0].find(function) >= 0:
1298 print("Unsupported command",sys.argv[0])
1299 print("Supported commands:" + " ".join(list(Main.modes.keys())))
1302 usage='undefined usage, mode={}'.format(mode)
1303 if mode in Main.regular_modes:
1304 usage = Main.module_usage
1305 usage += Main.common_usage
1306 usage += "\nmodule-{} : {}".format(mode, Main.modes[mode])
1307 elif mode == 'changelog':
1308 usage = Main.release_usage
1309 usage += Main.common_usage
1310 elif mode == 'adopt':
1311 usage = Main.adopt_usage
1312 usage += Main.common_usage
1314 parser=OptionParser(usage=usage)
1316 # the 'adopt' mode is really special and doesn't share any option
1318 parser.add_option("-m","--module",action="append",dest="modules",default=[],
1319 help="modules, can be used several times or with quotes")
1320 parser.add_option("-t","--tag",action="store", dest="tag", default='master',
1321 help="specify the tag to adopt, default is 'master'")
1322 parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1323 help="run in verbose mode")
1324 (options, args) = parser.parse_args()
1325 options.workdir='unused'
1326 options.dry_run=False
1327 options.mode='adopt'
1328 if len(args)==0 or len(options.modules)==0:
1331 adopt_tag(options,args)
1334 # the other commands (module-* and release-changelog) share the same skeleton
1335 if mode in [ 'tag', 'branch'] :
1336 parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
1337 help="set new version and reset taglevel to 0")
1338 parser.add_option("-0","--bypass",action="store_true",dest="bypass",default=False,
1339 help="skip checks on existence of the previous tag")
1341 parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
1342 help="do not update changelog section in specfile when tagging")
1343 parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,
1344 help="specify a build branch; used for locating the *tags.mk files where adoption is to take place")
1345 if mode in [ 'tag', 'sync' ] :
1346 parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),
1347 help="specify editor")
1349 if mode in ['diff','version'] :
1350 parser.add_option("-W","--www", action="store", dest="www", default=False,
1351 help="export diff in html format, e.g. -W trunk")
1354 parser.add_option("-l","--list", action="store_true", dest="list", default=False,
1355 help="just list modules that exhibit differences")
1357 default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"
1358 parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False,
1359 help="run on all modules as found in {}".format(default_modules_list))
1360 parser.add_option("-f","--file",action="store",dest="modules_list",default=None,
1361 help="run on all modules found in specified file")
1362 parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,
1363 help="dry run - shell commands are only displayed")
1364 parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",
1365 default=[], nargs=1,type="string",
1366 help="""specify distro-tags files, e.g. onelab-tags-4.2.mk
1367 -- can be set multiple times, or use quotes""")
1369 parser.add_option("-w","--workdir", action="store", dest="workdir",
1370 default="{}/{}".format(os.getenv("HOME"),"modules"),
1371 help="""name for dedicated working dir - defaults to ~/modules
1372 ** THIS MUST NOT ** be your usual working directory""")
1373 parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,
1374 help="skip safety checks, such as git pulls -- use with care")
1375 parser.add_option("-B","--build-module",action="store",dest="build_module",default=None,
1376 help="specify a build module to owerride the one in the CONFIG")
1378 # default verbosity depending on function - temp
1379 verbose_modes= ['tag', 'sync', 'branch']
1381 if mode not in verbose_modes:
1382 parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
1383 help="run in verbose mode")
1385 parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,
1386 help="run in quiet (non-verbose) mode")
1387 options, args = parser.parse_args()
1389 if not hasattr(options,'dry_run'):
1390 options.dry_run=False
1391 if not hasattr(options,'www'):
1399 if options.all_modules:
1400 options.modules_list=default_modules_list
1401 if options.modules_list:
1402 args=Command("grep -v '#' {}".format(options.modules_list), options).output_of().split()
1406 Module.init_homedir(options)
1409 if mode in Main.regular_modes:
1410 modules = [ Module(modname, options) for modname in args ]
1411 # hack: create a dummy Module to store errors/warnings
1412 error_module = Module('__errors__',options)
1414 for module in modules:
1415 if len(args)>1 and mode not in Main.silent_modes:
1417 print('========================================', module.friendly_name())
1418 # call the method called do_<mode>
1419 method = Module.__dict__["do_{}".format(mode)]
1422 except Exception as e:
1424 title='<span class="error"> Skipping module {} - failure: {} </span>'\
1425 .format(module.friendly_name(), str(e))
1426 error_module.html_store_title(title)
1429 traceback.print_exc()
1430 print('Skipping module {}: {}'.format(module.name,e))
1434 modetitle="Changes to tag in {}".format(options.www)
1435 elif mode == "version":
1436 modetitle="Latest tags in {}".format(options.www)
1437 modules.append(error_module)
1438 error_module.html_dump_header(modetitle)
1439 for module in modules:
1440 module.html_dump_toc()
1441 Module.html_dump_middle()
1442 for module in modules:
1443 module.html_dump_body()
1444 Module.html_dump_footer()
1446 # if we provide, say a b c d, we want to build (a,b) (b,c) and (c,d)
1447 # remember that the changelog in the twiki comes latest first, so
1448 # we typically have here latest latest-1 latest-2
1449 for (tag_new,tag_old) in zip ( args[:-1], args [1:]):
1450 release_changelog(options, tag_old, tag_new)
1453 ####################
1454 if __name__ == "__main__" :
1457 except KeyboardInterrupt: