X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=module-tools.py;h=aed29e34835ce02f1ea6b4c6672e35d392422468;hb=refs%2Ftags%2F5.3.8;hp=b4f9d254ec127fd1028fdb71dc64e262ef6529ac;hpb=67bfe0f64951be4a754ebf9b3ea02a03cae00f4f;p=build.git diff --git a/module-tools.py b/module-tools.py index b4f9d254..aed29e34 100755 --- a/module-tools.py +++ b/module-tools.py @@ -111,6 +111,7 @@ class Command: print 'dry_run',self.command return 0 if self.options.verbose: + print '>',os.getcwd() print '+',self.command,' .. ', sys.stdout.flush() retcod=os.system(self.command + " &> " + self.tmp) @@ -179,7 +180,7 @@ class SvnRepository: return "%s/%s" % (root, self.pathname()) @classmethod - def checkout(cls, remote, local, options, recursive=False): + def clone(cls, remote, local, options, recursive=False): if recursive: svncommand = "svn co %s %s" % (remote, local) else: @@ -190,12 +191,12 @@ class SvnRepository: return SvnRepository(local, options) @classmethod - def remote_exists(cls, remote): - return os.system("svn list %s &> /dev/null" % remote) == 0 + def remote_exists(cls, remote, options): + return Command ("svn list %s &> /dev/null" % remote , options).run()==0 def tag_exists(self, tagname): url = "%s/tags/%s" % (self.repo_root(), tagname) - return SvnRepository.remote_exists(url) + return SvnRepository.remote_exists(url, self.options) def update(self, subdir="", recursive=True, branch=None): path = os.path.join(self.path, subdir) @@ -213,11 +214,11 @@ class SvnRepository: def to_branch(self, branch): remote = "%s/branches/%s" % (self.repo_root(), branch) - SvnRepository.checkout(remote, self.path, self.options, recursive=True) + SvnRepository.clone(remote, self.path, self.options, recursive=True) def to_tag(self, tag): remote = "%s/tags/%s" % (self.repo_root(), branch) - SvnRepository.checkout(remote, self.path, self.options, recursive=True) + SvnRepository.clone(remote, self.path, self.options, recursive=True) def tag(self, tagname, logfile): tag_url = "%s/tags/%s" % (self.repo_root(), tagname) @@ -279,14 +280,15 @@ class GitRepository: return line.split()[2] @classmethod - def checkout(cls, remote, local, options, depth=0): + def clone(cls, remote, local, options, depth=None): Command("rm -rf %s" % local, options).run_silent() - Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal() + depth_option = "" if depth is None else " --depth {}".format(depth) + Command("git clone{} {} {}".format(depth_option, remote, local), options).run_fatal() return GitRepository(local, options) @classmethod - def remote_exists(cls, remote): - return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0 + def remote_exists(cls, remote, options): + return Command ("git --no-pager ls-remote %s &> /dev/null" % remote, options).run()==0 def tag_exists(self, tagname): command = 'git tag -l | grep "^%s$"' % tagname @@ -392,17 +394,16 @@ class Repository: break @classmethod - def has_moved_to_git(cls, module, config): + def has_moved_to_git(cls, module, config,options): module = svn_to_git_name(module) # check if the module is already in Git -# return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (config['svnpath'], module)) - return GitRepository.remote_exists(Module.git_remote_dir(module)) + return GitRepository.remote_exists(Module.git_remote_dir(module),options) @classmethod - def remote_exists(cls, remote): + def remote_exists(cls, remote, options): for repo in Repository.supported_repo_types: - if repo.remote_exists(remote): + if repo.remote_exists(remote, options): return True return False @@ -558,11 +559,11 @@ module-* commands need a fresh working dir. Make sure that you do not use that for other purposes than tagging""" % options.workdir sys.exit(1) - def checkout_build(): + def clone_build(): print "Checking out build module..." remote = cls.git_remote_dir(cls.config['build']) local = os.path.join(options.workdir, cls.config['build']) - GitRepository.checkout(remote, local, options, depth=1) + GitRepository.clone(remote, local, options, depth=1) print "OK" def store_config(): @@ -590,7 +591,7 @@ that for other purposes than tagging""" % options.workdir print "Cannot find",options.workdir,"let's create it" Command("mkdir -p %s" % options.workdir, options).run_silent() cls.prompt_config() - checkout_build() + clone_build() store_config() else: read_config() @@ -605,12 +606,12 @@ that for other purposes than tagging""" % options.workdir if old_layout: Command("rm -rf %s" % options.workdir, options).run_silent() Command("mkdir -p %s" % options.workdir, options).run_silent() - checkout_build() + clone_build() store_config() build_dir = os.path.join(options.workdir, cls.config['build']) if not os.path.isdir(build_dir): - checkout_build() + clone_build() else: build = Repository(build_dir, options) if not build.is_clean(): @@ -629,20 +630,20 @@ that for other purposes than tagging""" % options.workdir print 'Checking for',self.module_dir if not os.path.isdir (self.module_dir): - if Repository.has_moved_to_git(self.pathname, Module.config): - self.repository = GitRepository.checkout(self.git_remote_dir(self.pathname), - self.module_dir, - self.options) + if Repository.has_moved_to_git(self.pathname, Module.config, self.options): + self.repository = GitRepository.clone(self.git_remote_dir(self.pathname), + self.module_dir, + self.options) else: remote = self.svn_selected_remote() - self.repository = SvnRepository.checkout(remote, - self.module_dir, - self.options, recursive=False) + self.repository = SvnRepository.clone(remote, + self.module_dir, + self.options, recursive=False) self.repository = Repository(self.module_dir, self.options) if self.repository.type == "svn": # check if module has moved to git - if Repository.has_moved_to_git(self.pathname, Module.config): + if Repository.has_moved_to_git(self.pathname, Module.config, self.options): Command("rm -rf %s" % self.module_dir, self.options).run_silent() self.init_module_dir() # check if we have the required branch/tag @@ -800,7 +801,7 @@ that for other purposes than tagging""" % options.workdir f.write(line) f.close() - def insert_changelog (self, logfile, oldtag, newtag): + def insert_changelog (self, logfile, newtag): for specfile in self.all_specnames(): newspecfile=specfile+".new" if self.options.verbose: @@ -842,6 +843,17 @@ that for other purposes than tagging""" % options.workdir return "%s-%s" % (base_tag_name, self.last_tag(spec_dict)) + pattern_format="\A\s*%(module)s-(SVNPATH|GITPATH)\s*(=|:=)\s*(?P[^\s]+)/%(module)s[^\s]+" + + def is_mentioned_in_tagsfile (self, tagsfile): + # so that %(module)s gets replaced from format + module=self.name + module_matcher = re.compile(Module.pattern_format % locals()) + with open(tagsfile) as f: + for line in f.readlines(): + if module_matcher.match(line): return True + return False + ############################## # using fine_grain means replacing only those instances that currently refer to this tag # otherwise, -{SVNPATH,GITPATH} is replaced unconditionnally @@ -867,11 +879,11 @@ that for other purposes than tagging""" % options.workdir else: if self.options.verbose: print 'Searching for -SVNPATH or -GITPATH lines referring to /%s/\n\tin %s .. '%(self.pathname,tagsfile), - pattern="\A\s*%s-(SVNPATH|GITPATH)\s*(=|:=)\s*(?P[^\s]+)/%s[^\s]+"\ - %(self.name,self.name) - matcher_module=re.compile(pattern) + # so that %(module)s gets replaced from format + module=self.name + module_matcher=re.compile(Module.pattern_format % locals()) for line in tags.readlines(): - attempt=matcher_module.match(line) + attempt=module_matcher.match(line) if attempt: if line.find("-GITPATH") >= 0: modulepath = "%s-GITPATH"%self.name @@ -930,9 +942,11 @@ that for other purposes than tagging""" % options.workdir spec_dict = self.spec_dict() self.show_dict(spec_dict) - # side effects - old_tag_name = self.tag_name(spec_dict) - old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True) + # compute previous tag - if not bypassed + if not self.options.bypass: + old_tag_name = self.tag_name(spec_dict) + # sanity check + old_tag_name = self.check_tag(old_tag_name, need_it=True) if (self.options.new_version): # new version set on command line @@ -944,16 +958,15 @@ that for other purposes than tagging""" % options.workdir spec_dict[self.module_taglevel_varname] = new_taglevel new_tag_name = self.tag_name(spec_dict) - # sanity check - old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name) new_tag_name = self.check_tag(new_tag_name, need_it=False) # checking for diffs - diff_output = self.repository.diff_with_tag(old_tag_name) - if len(diff_output) == 0: - if not prompt ("No pending difference in module %s, want to tag anyway"%self.pathname,False): - return + if not self.options.bypass: + diff_output = self.repository.diff_with_tag(old_tag_name) + if len(diff_output) == 0: + if not prompt ("No pending difference in module %s, want to tag anyway"%self.pathname,False): + return # side effect in head's specfile self.patch_spec_var(spec_dict) @@ -968,10 +981,12 @@ that for other purposes than tagging""" % options.workdir file(changelog_plain,"w").write(""" %s %s -Please write a changelog for this new tag in the section above +Please write a changelog for this new tag in the section below """%(Module.edit_magic_line,setting_tag_line)) - if not self.options.verbose or prompt('Want to see diffs while writing changelog',True): + if self.options.bypass: + pass + elif prompt('Want to see diffs while writing changelog',True): file(changelog_plain,"a").write('DIFF=========\n' + diff_output) if self.options.debug: @@ -984,7 +999,7 @@ Please write a changelog for this new tag in the section above self.strip_magic_line_filename(changelog_plain,changelog_strip,new_tag_name) # insert changelog in spec if self.options.changelog: - self.insert_changelog (changelog_plain,old_tag_name,new_tag_name) + self.insert_changelog (changelog_plain,new_tag_name) ## update build build_path = os.path.join(self.options.workdir, @@ -1000,7 +1015,12 @@ Please write a changelog for this new tag in the section above default_answer = 'y' tagsfiles.sort() while True: + # do not bother if in bypass mode + if self.options.bypass: break for tagsfile in tagsfiles: + if not self.is_mentioned_in_tagsfile (tagsfile): + if self.options.verbose: print "tagsfile %s does not mention %s - skipped"%(tagsfile,self.name) + continue status=tagsdict[tagsfile] basename=os.path.basename(tagsfile) print ".................... Dealing with %s"%basename @@ -1262,36 +1282,38 @@ def modules_diff(first, second): def release_changelog(options, buildtag_old, buildtag_new): - try: - tagfile = options.distrotags[0] - if not tagfile: raise - except: - print "ERROR: provide a tagfile name (eg. onelab, onelab-k27, planetlab)" - return - # mmh, sounds wrong to blindly add the extension - # if in a build directory, guess from existing files - if os.path.isfile (tagfile): - pass - elif os.path.isfile ("%s-tags.mk" % tagfile): - tagfile="%s-tags.mk" % tagfile + # the command line expects new old, so we treat the tagfiles in the same order + nb_tags=len(options.distrotags) + if nb_tags==1: + tagfile_new=tagfile_old=options.distrotags[0] + elif nb_tags==2: + [tagfile_new,tagfile_old]=options.distrotags else: - tagfile = "%s-tags.mk" % tagfile - + print "ERROR: provide one or two tagfile name (eg. onelab-k32-tags.mk)" + print "two tagfiles can be mentioned when a tagfile has been renamed" + return + + if options.dry_run: + print "------------------------------ Computing Changelog from" + print "buildtag_old",buildtag_old,"tagfile_old",tagfile_old + print "buildtag_new",buildtag_new,"tagfile_new",tagfile_new + return + print '----' print '----' print '----' print '= build tag %s to %s =' % (buildtag_old, buildtag_new) - print '== distro %s (%s to %s) ==' % (tagfile, buildtag_old, buildtag_new) + print '== distro %s (%s to %s) ==' % (tagfile_new, buildtag_old, buildtag_new) build = Build("build@%s" % buildtag_old, options) build.init_module_dir() - first = build.get_modules(tagfile) + first = build.get_modules(tagfile_old) print ' * from', buildtag_old, build.repository.gitweb() build = Build("build@%s" % buildtag_new, options) build.init_module_dir() - second = build.get_modules(tagfile) + second = build.get_modules(tagfile_new) print ' * to', buildtag_new, build.repository.gitweb() @@ -1306,7 +1328,7 @@ def release_changelog(options, buildtag_old, buildtag_new): for module in diff: - print '=== %s - %s to %s : package %s ===' % (tagfile, buildtag_old, buildtag_new, module) + print '=== %s - %s to %s : package %s ===' % (tagfile_new, buildtag_old, buildtag_new, module) first, second = diff[module] m = get_module(module, first) @@ -1323,7 +1345,14 @@ def release_changelog(options, buildtag_old, buildtag_new): os.system("cp -f /%s %s" % (specfile, tmpfile)) m = get_module(module, second) - m.init_module_dir() + # patch for ipfw that, being managed in a separate repo, won't work for now + try: + m.init_module_dir() + except Exception,e: + print """Could not retrieve module %s - skipped +{{{ %s }}} +""" %( m.friendly_name(), e) + continue specfile = m.main_specname() if m.repository.type == "svn": @@ -1338,10 +1367,10 @@ def release_changelog(options, buildtag_old, buildtag_new): os.unlink(tmpfile) for module in new_modules: - print '=== %s : new package in build %s ===' % (tagfile, module) + print '=== %s : new package in build %s ===' % (tagfile_new, module) for module in removed_modules: - print '=== %s : removed package from build %s ===' % (tagfile, module) + print '=== %s : removed package from build %s ===' % (tagfile_new, module) def adopt_tag (options, args): @@ -1382,6 +1411,8 @@ Branches: release-changelog :4.2 4.2-rc25 You can refer to the build trunk by just mentioning 'trunk', e.g. release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk + You can use 2 different tagfile names if that was renamed meanwhile + release-changelog -t onelab-tags.mk 5.0-rc29 -t onelab-k32-tags.mk 5.0-rc28 """ adopt_usage="""Usage: %prog [options] tag-file[s] With this command you can adopt a specifi tag or branch in your tag files @@ -1449,7 +1480,7 @@ Branches: parser=OptionParser(usage=usage) # the 'adopt' mode is really special and doesn't share any option - if mode=='adopt': + if mode == 'adopt': parser.add_option("-m","--module",action="append",dest="modules",default=[], help="modules, can be used several times or with quotes") parser.add_option("-t","--tag",action="store", dest="tag", default='master', @@ -1467,23 +1498,25 @@ Branches: return # the other commands (module-* and release-changelog) share the same skeleton - if mode == "tag" or mode == 'branch': + if mode in [ 'tag', 'branch'] : parser.add_option("-s","--set-version",action="store",dest="new_version",default=None, help="set new version and reset taglevel to 0") - if mode == "tag" : + parser.add_option("-0","--bypass",action="store_true",dest="bypass",default=False, + help="skip checks on existence of the previous tag") + if mode == 'tag' : parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True, help="do not update changelog section in specfile when tagging") parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None, help="specify a build branch; used for locating the *tags.mk files where adoption is to take place") - if mode == "tag" or mode == "sync" : + if mode in [ 'tag', 'sync' ] : parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(), help="specify editor") - if mode in ["diff","version"] : + if mode in ['diff','version'] : parser.add_option("-W","--www", action="store", dest="www", default=False, help="export diff in html format, e.g. -W trunk") - if mode == "diff" : + if mode == 'diff' : parser.add_option("-l","--list", action="store_true", dest="list", default=False, help="just list modules that exhibit differences") @@ -1579,8 +1612,8 @@ Branches: # if we provide, say a b c d, we want to build (a,b) (b,c) and (c,d) # remember that the changelog in the twiki comes latest first, so # we typically have here latest latest-1 latest-2 - for (tag_to,tag_from) in zip ( args[:-1], args [1:]): - release_changelog(options, tag_from,tag_to) + for (tag_new,tag_old) in zip ( args[:-1], args [1:]): + release_changelog(options, tag_old, tag_new) ####################