X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=module-tools.py;h=58f7b85ffedefeb64e20cb9b644573ac86c2978c;hb=refs%2Fheads%2Fnm-one-spec;hp=2a10dd30fca79cacbbd40929e52cf73690be1768;hpb=1da274c54037898e191f9b8dd8dcfeb303f98154;p=build.git diff --git a/module-tools.py b/module-tools.py index 2a10dd30..58f7b85f 100755 --- a/module-tools.py +++ b/module-tools.py @@ -111,6 +111,7 @@ class Command: print 'dry_run',self.command return 0 if self.options.verbose: + print '>',os.getcwd() print '+',self.command,' .. ', sys.stdout.flush() retcod=os.system(self.command + " &> " + self.tmp) @@ -179,7 +180,7 @@ class SvnRepository: return "%s/%s" % (root, self.pathname()) @classmethod - def checkout(cls, remote, local, options, recursive=False): + def clone(cls, remote, local, options, recursive=False): if recursive: svncommand = "svn co %s %s" % (remote, local) else: @@ -190,12 +191,12 @@ class SvnRepository: return SvnRepository(local, options) @classmethod - def remote_exists(cls, remote): - return os.system("svn list %s &> /dev/null" % remote) == 0 + def remote_exists(cls, remote, options): + return Command ("svn list %s &> /dev/null" % remote , options).run()==0 def tag_exists(self, tagname): url = "%s/tags/%s" % (self.repo_root(), tagname) - return SvnRepository.remote_exists(url) + return SvnRepository.remote_exists(url, self.options) def update(self, subdir="", recursive=True, branch=None): path = os.path.join(self.path, subdir) @@ -213,11 +214,11 @@ class SvnRepository: def to_branch(self, branch): remote = "%s/branches/%s" % (self.repo_root(), branch) - SvnRepository.checkout(remote, self.path, self.options, recursive=True) + SvnRepository.clone(remote, self.path, self.options, recursive=True) def to_tag(self, tag): remote = "%s/tags/%s" % (self.repo_root(), branch) - SvnRepository.checkout(remote, self.path, self.options, recursive=True) + SvnRepository.clone(remote, self.path, self.options, recursive=True) def tag(self, tagname, logfile): tag_url = "%s/tags/%s" % (self.repo_root(), tagname) @@ -279,14 +280,14 @@ class GitRepository: return line.split()[2] @classmethod - def checkout(cls, remote, local, options, depth=0): + def clone(cls, remote, local, options, depth=0): Command("rm -rf %s" % local, options).run_silent() Command("git clone --depth %d %s %s" % (depth, remote, local), options).run_fatal() return GitRepository(local, options) @classmethod - def remote_exists(cls, remote): - return os.system("git --no-pager ls-remote %s &> /dev/null" % remote) == 0 + def remote_exists(cls, remote, options): + return Command ("git --no-pager ls-remote %s &> /dev/null" % remote, options).run()==0 def tag_exists(self, tagname): command = 'git tag -l | grep "^%s$"' % tagname @@ -392,17 +393,16 @@ class Repository: break @classmethod - def has_moved_to_git(cls, module, config): + def has_moved_to_git(cls, module, config,options): module = svn_to_git_name(module) # check if the module is already in Git -# return SvnRepository.remote_exists("%s/%s/aaaa-has-moved-to-git" % (config['svnpath'], module)) - return GitRepository.remote_exists(Module.git_remote_dir(module)) + return GitRepository.remote_exists(Module.git_remote_dir(module),options) @classmethod - def remote_exists(cls, remote): + def remote_exists(cls, remote, options): for repo in Repository.supported_repo_types: - if repo.remote_exists(remote): + if repo.remote_exists(remote, options): return True return False @@ -414,7 +414,7 @@ class Repository: # support for tagged module is minimal, and is for the Build class only class Module: - svn_magic_line="--This line, and those below, will be ignored--" + edit_magic_line="--This line, and those below, will be ignored--" setting_tag_format = "Setting tag %s" redirectors=[ # ('module_name_varname','name'), @@ -558,11 +558,11 @@ module-* commands need a fresh working dir. Make sure that you do not use that for other purposes than tagging""" % options.workdir sys.exit(1) - def checkout_build(): + def clone_build(): print "Checking out build module..." remote = cls.git_remote_dir(cls.config['build']) local = os.path.join(options.workdir, cls.config['build']) - GitRepository.checkout(remote, local, options, depth=1) + GitRepository.clone(remote, local, options, depth=1) print "OK" def store_config(): @@ -590,7 +590,7 @@ that for other purposes than tagging""" % options.workdir print "Cannot find",options.workdir,"let's create it" Command("mkdir -p %s" % options.workdir, options).run_silent() cls.prompt_config() - checkout_build() + clone_build() store_config() else: read_config() @@ -605,12 +605,12 @@ that for other purposes than tagging""" % options.workdir if old_layout: Command("rm -rf %s" % options.workdir, options).run_silent() Command("mkdir -p %s" % options.workdir, options).run_silent() - checkout_build() + clone_build() store_config() build_dir = os.path.join(options.workdir, cls.config['build']) if not os.path.isdir(build_dir): - checkout_build() + clone_build() else: build = Repository(build_dir, options) if not build.is_clean(): @@ -629,20 +629,20 @@ that for other purposes than tagging""" % options.workdir print 'Checking for',self.module_dir if not os.path.isdir (self.module_dir): - if Repository.has_moved_to_git(self.pathname, Module.config): - self.repository = GitRepository.checkout(self.git_remote_dir(self.pathname), - self.module_dir, - self.options) + if Repository.has_moved_to_git(self.pathname, Module.config, self.options): + self.repository = GitRepository.clone(self.git_remote_dir(self.pathname), + self.module_dir, + self.options) else: remote = self.svn_selected_remote() - self.repository = SvnRepository.checkout(remote, - self.module_dir, - self.options, recursive=False) + self.repository = SvnRepository.clone(remote, + self.module_dir, + self.options, recursive=False) self.repository = Repository(self.module_dir, self.options) if self.repository.type == "svn": # check if module has moved to git - if Repository.has_moved_to_git(self.pathname, Module.config): + if Repository.has_moved_to_git(self.pathname, Module.config, self.options): Command("rm -rf %s" % self.module_dir, self.options).run_silent() self.init_module_dir() # check if we have the required branch/tag @@ -784,7 +784,7 @@ that for other purposes than tagging""" % options.workdir result=[] white_line_matcher = re.compile("\A\s*\Z") for logline in file(logfile).readlines(): - if logline.strip() == Module.svn_magic_line: + if logline.strip() == Module.edit_magic_line: break elif white_line_matcher.match(logline): continue @@ -793,14 +793,14 @@ that for other purposes than tagging""" % options.workdir return result # creates a copy of the input with only the unignored lines - def stripped_magic_line_filename (self, filein, fileout ,new_tag_name): + def strip_magic_line_filename (self, filein, fileout ,new_tag_name): f=file(fileout,'w') f.write(self.setting_tag_format%new_tag_name + '\n') for line in self.unignored_lines(filein): f.write(line) f.close() - def insert_changelog (self, logfile, oldtag, newtag): + def insert_changelog (self, logfile, newtag): for specfile in self.all_specnames(): newspecfile=specfile+".new" if self.options.verbose: @@ -842,6 +842,17 @@ that for other purposes than tagging""" % options.workdir return "%s-%s" % (base_tag_name, self.last_tag(spec_dict)) + pattern_format="\A\s*%(module)s-(SVNPATH|GITPATH)\s*(=|:=)\s*(?P[^\s]+)/%(module)s[^\s]+" + + def is_mentioned_in_tagsfile (self, tagsfile): + # so that %(module)s gets replaced from format + module=self.name + module_matcher = re.compile(Module.pattern_format % locals()) + with open(tagsfile) as f: + for line in f.readlines(): + if module_matcher.match(line): return True + return False + ############################## # using fine_grain means replacing only those instances that currently refer to this tag # otherwise, -{SVNPATH,GITPATH} is replaced unconditionnally @@ -867,11 +878,11 @@ that for other purposes than tagging""" % options.workdir else: if self.options.verbose: print 'Searching for -SVNPATH or -GITPATH lines referring to /%s/\n\tin %s .. '%(self.pathname,tagsfile), - pattern="\A\s*%s-(SVNPATH|GITPATH)\s*(=|:=)\s*(?P[^\s]+)/%s[^\s]+"\ - %(self.name,self.name) - matcher_module=re.compile(pattern) + # so that %(module)s gets replaced from format + module=self.name + module_matcher=re.compile(Module.pattern_format % locals()) for line in tags.readlines(): - attempt=matcher_module.match(line) + attempt=module_matcher.match(line) if attempt: if line.find("-GITPATH") >= 0: modulepath = "%s-GITPATH"%self.name @@ -930,9 +941,11 @@ that for other purposes than tagging""" % options.workdir spec_dict = self.spec_dict() self.show_dict(spec_dict) - # side effects - old_tag_name = self.tag_name(spec_dict) - old_svn_tag_name = self.tag_name(spec_dict, old_svn_name=True) + # compute previous tag - if not bypassed + if not self.options.bypass: + old_tag_name = self.tag_name(spec_dict) + # sanity check + old_tag_name = self.check_tag(old_tag_name, need_it=True) if (self.options.new_version): # new version set on command line @@ -944,47 +957,48 @@ that for other purposes than tagging""" % options.workdir spec_dict[self.module_taglevel_varname] = new_taglevel new_tag_name = self.tag_name(spec_dict) - # sanity check - old_tag_name = self.check_tag(old_tag_name, need_it=True, old_svn_tag_name=old_svn_tag_name) new_tag_name = self.check_tag(new_tag_name, need_it=False) # checking for diffs - diff_output = self.repository.diff_with_tag(old_tag_name) - if len(diff_output) == 0: - if not prompt ("No pending difference in module %s, want to tag anyway"%self.pathname,False): - return + if not self.options.bypass: + diff_output = self.repository.diff_with_tag(old_tag_name) + if len(diff_output) == 0: + if not prompt ("No pending difference in module %s, want to tag anyway"%self.pathname,False): + return # side effect in head's specfile self.patch_spec_var(spec_dict) # prepare changelog file - # we use the standard subversion magic string (see svn_magic_line) + # we use the standard subversion magic string (see edit_magic_line) # so we can provide useful information, such as version numbers and diff # in the same file - changelog="/tmp/%s-%d.edit"%(self.name,os.getpid()) - changelog_svn="/tmp/%s-%d.svn"%(self.name,os.getpid()) + changelog_plain="/tmp/%s-%d.edit"%(self.name,os.getpid()) + changelog_strip="/tmp/%s-%d.strip"%(self.name,os.getpid()) setting_tag_line=Module.setting_tag_format%new_tag_name - file(changelog,"w").write(""" + file(changelog_plain,"w").write(""" %s %s -Please write a changelog for this new tag in the section above -"""%(Module.svn_magic_line,setting_tag_line)) +Please write a changelog for this new tag in the section below +"""%(Module.edit_magic_line,setting_tag_line)) - if not self.options.verbose or prompt('Want to see diffs while writing changelog',True): - file(changelog,"a").write('DIFF=========\n' + diff_output) + if self.options.bypass: + pass + elif prompt('Want to see diffs while writing changelog',True): + file(changelog_plain,"a").write('DIFF=========\n' + diff_output) if self.options.debug: prompt('Proceed ?') # edit it - self.run("%s %s"%(self.options.editor,changelog)) + self.run("%s %s"%(self.options.editor,changelog_plain)) # strip magic line in second file - looks like svn has changed its magic line with 1.6 # so we do the job ourselves - self.stripped_magic_line_filename(changelog,changelog_svn,new_tag_name) + self.strip_magic_line_filename(changelog_plain,changelog_strip,new_tag_name) # insert changelog in spec if self.options.changelog: - self.insert_changelog (changelog,old_tag_name,new_tag_name) + self.insert_changelog (changelog_plain,new_tag_name) ## update build build_path = os.path.join(self.options.workdir, @@ -1000,7 +1014,12 @@ Please write a changelog for this new tag in the section above default_answer = 'y' tagsfiles.sort() while True: + # do not bother if in bypass mode + if self.options.bypass: break for tagsfile in tagsfiles: + if not self.is_mentioned_in_tagsfile (tagsfile): + if self.options.verbose: print "tagsfile %s does not mention %s - skipped"%(tagsfile,self.name) + continue status=tagsdict[tagsfile] basename=os.path.basename(tagsfile) print ".................... Dealing with %s"%basename @@ -1049,14 +1068,14 @@ n: move to next file"""%locals() build.commit(log) self.run_prompt("Review module and build", diff_all_changes) - self.run_prompt("Commit module and build", commit_all_changes, changelog_svn) - self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_svn) + self.run_prompt("Commit module and build", commit_all_changes, changelog_strip) + self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_strip) if self.options.debug: - print 'Preserving',changelog,'and stripped',changelog_svn + print 'Preserving',changelog_plain,'and stripped',changelog_strip else: - os.unlink(changelog) - os.unlink(changelog_svn) + os.unlink(changelog_plain) + os.unlink(changelog_strip) ############################## @@ -1262,36 +1281,38 @@ def modules_diff(first, second): def release_changelog(options, buildtag_old, buildtag_new): - try: - tagfile = options.distrotags[0] - if not tagfile: raise - except: - print "ERROR: provide a tagfile name (eg. onelab, onelab-k27, planetlab)" - return - # mmh, sounds wrong to blindly add the extension - # if in a build directory, guess from existing files - if os.path.isfile (tagfile): - pass - elif os.path.isfile ("%s-tags.mk" % tagfile): - tagfile="%s-tags.mk" % tagfile + # the command line expects new old, so we treat the tagfiles in the same order + nb_tags=len(options.distrotags) + if nb_tags==1: + tagfile_new=tagfile_old=options.distrotags[0] + elif nb_tags==2: + [tagfile_new,tagfile_old]=options.distrotags else: - tagfile = "%s-tags.mk" % tagfile - + print "ERROR: provide one or two tagfile name (eg. onelab-k32-tags.mk)" + print "two tagfiles can be mentioned when a tagfile has been renamed" + return + + if options.dry_run: + print "------------------------------ Computing Changelog from" + print "buildtag_old",buildtag_old,"tagfile_old",tagfile_old + print "buildtag_new",buildtag_new,"tagfile_new",tagfile_new + return + print '----' print '----' print '----' print '= build tag %s to %s =' % (buildtag_old, buildtag_new) - print '== distro %s (%s to %s) ==' % (tagfile, buildtag_old, buildtag_new) + print '== distro %s (%s to %s) ==' % (tagfile_new, buildtag_old, buildtag_new) build = Build("build@%s" % buildtag_old, options) build.init_module_dir() - first = build.get_modules(tagfile) + first = build.get_modules(tagfile_old) print ' * from', buildtag_old, build.repository.gitweb() build = Build("build@%s" % buildtag_new, options) build.init_module_dir() - second = build.get_modules(tagfile) + second = build.get_modules(tagfile_new) print ' * to', buildtag_new, build.repository.gitweb() @@ -1306,7 +1327,7 @@ def release_changelog(options, buildtag_old, buildtag_new): for module in diff: - print '=== %s - %s to %s : package %s ===' % (tagfile, buildtag_old, buildtag_new, module) + print '=== %s - %s to %s : package %s ===' % (tagfile_new, buildtag_old, buildtag_new, module) first, second = diff[module] m = get_module(module, first) @@ -1323,7 +1344,14 @@ def release_changelog(options, buildtag_old, buildtag_new): os.system("cp -f /%s %s" % (specfile, tmpfile)) m = get_module(module, second) - m.init_module_dir() + # patch for ipfw that, being managed in a separate repo, won't work for now + try: + m.init_module_dir() + except Exception,e: + print """Could not retrieve module %s - skipped +{{{ %s }}} +""" %( m.friendly_name(), e) + continue specfile = m.main_specname() if m.repository.type == "svn": @@ -1338,10 +1366,10 @@ def release_changelog(options, buildtag_old, buildtag_new): os.unlink(tmpfile) for module in new_modules: - print '=== %s : new package in build %s ===' % (tagfile, module) + print '=== %s : new package in build %s ===' % (tagfile_new, module) for module in removed_modules: - print '=== %s : removed package from build %s ===' % (tagfile, module) + print '=== %s : removed package from build %s ===' % (tagfile_new, module) def adopt_tag (options, args): @@ -1382,6 +1410,8 @@ Branches: release-changelog :4.2 4.2-rc25 You can refer to the build trunk by just mentioning 'trunk', e.g. release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk + You can use 2 different tagfile names if that was renamed meanwhile + release-changelog -t onelab-tags.mk 5.0-rc29 -t onelab-k32-tags.mk 5.0-rc28 """ adopt_usage="""Usage: %prog [options] tag-file[s] With this command you can adopt a specifi tag or branch in your tag files @@ -1449,7 +1479,7 @@ Branches: parser=OptionParser(usage=usage) # the 'adopt' mode is really special and doesn't share any option - if mode=='adopt': + if mode == 'adopt': parser.add_option("-m","--module",action="append",dest="modules",default=[], help="modules, can be used several times or with quotes") parser.add_option("-t","--tag",action="store", dest="tag", default='master', @@ -1467,23 +1497,25 @@ Branches: return # the other commands (module-* and release-changelog) share the same skeleton - if mode == "tag" or mode == 'branch': + if mode in [ 'tag', 'branch'] : parser.add_option("-s","--set-version",action="store",dest="new_version",default=None, help="set new version and reset taglevel to 0") - if mode == "tag" : + parser.add_option("-0","--bypass",action="store_true",dest="bypass",default=False, + help="skip checks on existence of the previous tag") + if mode == 'tag' : parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True, help="do not update changelog section in specfile when tagging") parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None, help="specify a build branch; used for locating the *tags.mk files where adoption is to take place") - if mode == "tag" or mode == "sync" : + if mode in [ 'tag', 'sync' ] : parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(), help="specify editor") - if mode in ["diff","version"] : + if mode in ['diff','version'] : parser.add_option("-W","--www", action="store", dest="www", default=False, help="export diff in html format, e.g. -W trunk") - if mode == "diff" : + if mode == 'diff' : parser.add_option("-l","--list", action="store_true", dest="list", default=False, help="just list modules that exhibit differences") @@ -1579,8 +1611,8 @@ Branches: # if we provide, say a b c d, we want to build (a,b) (b,c) and (c,d) # remember that the changelog in the twiki comes latest first, so # we typically have here latest latest-1 latest-2 - for (tag_to,tag_from) in zip ( args[:-1], args [1:]): - release_changelog(options, tag_from,tag_to) + for (tag_new,tag_old) in zip ( args[:-1], args [1:]): + release_changelog(options, tag_old, tag_new) ####################