X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=module-tools.py;h=b9d0cb74bc1c8bf9af896f0067aef6626d318aeb;hb=1225c75ecc69c125ab8e68ba0365ff57a4959c98;hp=298461750224d67b0d90c4653cf919531f8d9c5c;hpb=a561b72a1419b73ebc507e598be101ef8f211b79;p=build.git diff --git a/module-tools.py b/module-tools.py index 29846175..b9d0cb74 100755 --- a/module-tools.py +++ b/module-tools.py @@ -55,6 +55,15 @@ def default_editor(): editor = "emacs" return editor +### fold long lines +fold_length=132 + +def print_fold (line): + while len(line) >= fold_length: + print line[:fold_length],'\\' + line=line[fold_length:] + print line + class Command: def __init__ (self,command,options): self.command=command @@ -160,7 +169,7 @@ class Module: # for parsing module spec name:branch - matcher_branch_spec=re.compile("\A(?P[\w-]+):(?P[\w\.-]+)\Z") + matcher_branch_spec=re.compile("\A(?P[\w\.-]+):(?P[\w\.-]+)\Z") # special form for tagged module - for Build matcher_tag_spec=re.compile("\A(?P[\w-]+)@(?P[\w\.-]+)\Z") # parsing specfiles @@ -318,10 +327,11 @@ that for other purposes than tagging"""%topdir if os.path.isfile (attempt): return attempt else: + pattern="%s/*.spec"%self.edge_dir() try: - return glob("%s/*.spec"%self.edge_dir())[0] + return glob(pattern)[0] except: - raise Exception, 'Cannot guess specfile for module %s'%self.name + raise Exception, 'Cannot guess specfile for module %s -- pattern was %s'%(self.name,pattern) def all_specnames (self): return glob("%s/*.spec"%self.edge_dir()) @@ -402,7 +412,7 @@ that for other purposes than tagging"""%topdir def unignored_lines (self, logfile): result=[] - exclude="Tagging module %s"%self.name + exclude="Setting tag %s"%self.name white_line_matcher = re.compile("\A\s*\Z") for logline in file(logfile).readlines(): if logline.strip() == Module.svn_magic_line: @@ -642,14 +652,16 @@ The module-sync function has the following limitations # brute-force : change uncommented lines that define -SVNPATH else: if self.options.verbose: - print 'Setting %s-SVNPATH for using %s\n\tin %s .. '%(self.name,newname,tagsfile), - pattern="\A\s*%s-SVNPATH\s*(=|:=)\s*(?P[^\s]+)/%s/[^\s]+"\ - %(self.name,self.name) + print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile), + pattern="\A\s*(?P[^\s]+)-SVNPATH\s*(=|:=)\s*(?P[^\s]+)/%s/[^\s]+"\ + %(self.name) matcher_module=re.compile(pattern) for line in tags.readlines(): attempt=matcher_module.match(line) if attempt: - svnpath="%s-SVNPATH"%self.name + svnpath="%s-SVNPATH"%(attempt.group('make_name')) + if self.options.verbose: + print ' '+svnpath, replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname) new.write(replacement) matches += 1 @@ -705,11 +717,11 @@ The module-sync function has the following limitations # so we can provide useful information, such as version numbers and diff # in the same file changelog="/tmp/%s-%d.txt"%(self.name,os.getpid()) - file(changelog,"w").write("""Tagging module %s - %s + file(changelog,"w").write("""Setting tag %s %s Please write a changelog for this new tag in the section above -"""%(self.name,new_tag_name,Module.svn_magic_line)) +"""%(new_tag_name,Module.svn_magic_line)) if not self.options.verbose or prompt('Want to see diffs while writing changelog',True): file(changelog,"a").write('DIFF=========\n' + diff_output) @@ -792,9 +804,11 @@ n: move to next file"""%locals() # do this before anything else and restore .branch to None, # as this is part of the class's logic new_trunk_name=None - if self.branch: + if hasattr(self,'branch'): new_trunk_name=self.branch - self.branch=None + del self.branch + elif self.options.new_version: + new_trunk_name = self.options.new_version # compute diff - a way to initialize the whole stuff # do_diff already does edge_dir initialization @@ -895,26 +909,41 @@ class Package: else: return None + def details (self): + return "[%s %s] [%s (spec)]"%(self.svnpath,self.basename,self.specpath) + class Build (Module): - def __init__ (self, buildtag,options): - self.buildtag=buildtag - Module.__init__(self,"build@%s"%buildtag,options) - # we cannot get build's svnpath as for other packages as we'd get something in svn+ssh # xxx quick & dirty - def get_svnpath (self): - self.svnpath="http://svn.planet-lab.org/svn/build/tags/%s"%self.buildtag + def __init__ (self, buildtag,options): + self.buildtag=buildtag + # if the buildtag start with a : (to use a branch rather than a tag) + if buildtag.find(':') == 0 : + module_name="build%(buildtag)s"%locals() + self.display=buildtag[1:] + self.svnpath="http://svn.planet-lab.org/svn/build/branches/%s"%self.display + else : + module_name="build@%(buildtag)s"%locals() + self.display=buildtag + self.svnpath="http://svn.planet-lab.org/svn/build/tags/%s"%self.buildtag + Module.__init__(self,module_name,options) - def get_packages (self,distrotag): + @staticmethod + def get_distro_from_distrotag (distrotag): # mhh: remove -tag* from distrotags to get distro n=distrotag.find('-tag') if n>0: - distro=distrotag[:n] + return distrotag[:n] else: - distro='planetlab' + return None + + def get_packages (self,distrotag): result={} - make_options="-C %s stage1=true PLDISTRO=%s PLDISTROTAGS=%s 2> /dev/null"%(self.edge_dir(),distro,distrotag) + distro=Build.get_distro_from_distrotag(distrotag) + if not distro: + return result + make_options="--no-print-directory -C %s stage1=true PLDISTRO=%s PLDISTROTAGS=%s 2> /dev/null"%(self.edge_dir(),distro,distrotag) command="make %s packages"%make_options make_packages=Command(command,self.options).output_of() pkg_line=re.compile("\Apackage=(?P[^\s]+)\s+ref_module=(?P[^\s]+)\s.*\Z") @@ -941,6 +970,23 @@ class Build (Module): def get_distrotags (self): return [os.path.basename(p) for p in glob("%s/*tags*mk"%self.edge_dir())] +class DiffCache: + + def __init__ (self): + self._cache={} + + def key(self, frompath,topath): + return frompath+'-to-'+topath + + def fetch (self, frompath, topath): + key=self.key(frompath,topath) + if not self._cache.has_key(key): + return None + return self._cache[key] + + def store (self, frompath, topath, diff): + key=self.key(frompath,topath) + self._cache[key]=diff class Release: @@ -952,38 +998,42 @@ class Release: print "----" print "----" print "----" - print "= build tag %s to %s = #tag-%s"%(buildtag_old,buildtag_new,buildtag_new) (build_new,build_old) = (Build (buildtag_new,options), Build (buildtag_old,options)) + print "= build tag %s to %s = #build-%s"%(build_old.display,build_new.display,build_new.display) for b in (build_new,build_old): b.init_module_dir() b.init_edge_dir() b.update_edge_dir() - b.get_svnpath() # find out the tags files that are common, unless option was specified if options.distrotags: - (distrotags_new,distrotags_old)=([options.distrotags],[options.distrotags]) + distrotags=options.distrotags else: distrotags_new=build_new.get_distrotags() distrotags_old=build_old.get_distrotags() - distrotags = list(set(distrotags_new).intersection(set(distrotags_old))) - distrotags.sort() - if options.verbose: - print "Found distrotags",distrotags + distrotags = list(set(distrotags_new).intersection(set(distrotags_old))) + distrotags.sort() + if options.verbose: print "Found distrotags",distrotags first_distrotag=True + diffcache = DiffCache() for distrotag in distrotags: + distro=Build.get_distro_from_distrotag(distrotag) + if not distro: + continue if first_distrotag: first_distrotag=False else: print '----' - print '== distro %s (%s to %s) == #distro-%s-%s'%(distrotag,buildtag_old,buildtag_new,distrotag,buildtag_new) + print '== distro %s (%s to %s) == #distro-%s-%s'%(distrotag,build_old.display,build_new.display,distro,build_new.display) print ' * from %s/%s'%(build_old.svnpath,distrotag) print ' * to %s/%s'%(build_new.svnpath,distrotag) # parse make packages packages_new=build_new.get_packages(distrotag) pnames_new=set(packages_new.keys()) + if options.verbose: print 'got packages for ',build_new.display packages_old=build_old.get_packages(distrotag) pnames_old=set(packages_old.keys()) + if options.verbose: print 'got packages for ',build_old.display # get created, deprecated, and preserved package names pnames_created = list(pnames_new-pnames_old) @@ -993,38 +1043,41 @@ class Release: pnames = list(pnames_new.intersection(pnames_old)) pnames.sort() - if options.verbose: - print "Found new/deprecated/preserved pnames",pnames_new,pnames_deprecated,pnames + if options.verbose: print "Found new/deprecated/preserved pnames",pnames_new,pnames_deprecated,pnames # display created and deprecated for name in pnames_created: - print '=== %s : new package %s -- appeared in %s === #pkg-%s-%s'%(distrotag,name,buildtag_new,name,buildtag_new) - obj=packages_new[name] - print ' * svn link: %s'%obj.svnpath - print ' * spec: %s'%obj.specpath + print '=== %s : new package %s -- appeared in %s === #package-%s-%s-%s'%( + distrotag,name,build_new.display,name,distro,build_new.display) + pobj=packages_new[name] + print ' * %s'%pobj.details() for name in pnames_deprecated: - print '=== %s : package %s -- deprecated, last occurrence in %s === #pkg-%s-%s'%(distrotag,name,buildtag_old,name,buildtag_new) - obj=packages_old[name] - if not obj.svnpath: - print ' * codebase stored in CVS, specfile is %s'%obj.spec + print '=== %s : package %s -- deprecated, last occurrence in %s === #package-%s-%s-%s'%( + distrotag,name,build_old.display,name,distro,build_new.display) + pobj=packages_old[name] + if not pobj.svnpath: + print ' * codebase stored in CVS, specfile is %s'%pobj.spec else: - print ' * svn link: %s'%obj.svnpath - print ' * spec: %s'%obj.specpath + print ' * %s'%pobj.details() # display other packages for name in pnames: (pobj_new,pobj_old)=(packages_new[name],packages_old[name]) - if options.verbose: - print "Dealing with package",name - if pobj_new.specpath == pobj_old.specpath: + if options.verbose: print "Dealing with package",name + if pobj_old.specpath == pobj_new.specpath: continue - command="svn diff %s %s"%(pobj_old.specpath,pobj_new.specpath) - specdiff=Command(command,options).output_of() + specdiff = diffcache.fetch(pobj_old.specpath,pobj_new.specpath) + if specdiff is None: + command="svn diff %s %s"%(pobj_old.specpath,pobj_new.specpath) + specdiff=Command(command,options).output_of() + diffcache.store(pobj_old.specpath,pobj_new.specpath,specdiff) + else: + if options.verbose: print 'got diff from cache' if not specdiff: continue - print '=== %s - %s to %s : package %s === #pkg-%s-%s'%(distrotag,buildtag_old,buildtag_new,name,name,buildtag_new) - print ' * from [%s %s] ([%s spec])'%(pobj_old.svnpath,pobj_old.basename,pobj_old.specpath), - print ' to [%s %s] ([%s spec])'%(pobj_new.svnpath,pobj_new.basename,pobj_new.specpath) + print '=== %s - %s to %s : package %s === #package-%s-%s-%s'%( + distrotag,build_old.display,build_new.display,name,name,distro,build_new.display) + print ' * from %s to %s'%(pobj_old.details(),pobj_new.details()) trac_diff_url=pobj_old.trac_full_diff(pobj_new) if trac_diff_url: print ' * [%s View full diff]'%trac_diff_url @@ -1037,7 +1090,7 @@ class Release: if line[0] in ['@']: print '----------' elif line[0] in ['+','-']: - print line + print_fold(line) print '}}}' ############################## @@ -1058,6 +1111,10 @@ Branches: """ release_usage="""Usage: %prog [options] tag1 .. tagn Extract release notes from the changes in specfiles between several build tags, latest first + Examples: + release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22 + You can refer to a (build) branch by prepending a colon, like in + release-changelog :4.2 4.2-rc25 """ common_usage="""More help: see http://svn.planet-lab.org/wiki/ModuleTools""" @@ -1080,6 +1137,13 @@ Branches: silent_modes = ['list'] release_modes = ['changelog'] + @staticmethod + def optparse_list (option, opt, value, parser): + try: + setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split()) + except: + setattr(parser.values,option.dest,value.split()) + def run(self): mode=None @@ -1136,21 +1200,25 @@ Branches: parser.add_option("-u","--url", action="store_true", dest="show_urls", default=False, help="display URLs") + default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list" if mode not in Main.release_modes: - all_modules=os.path.dirname(sys.argv[0])+"/modules.list" parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, - help="run on all modules as found in %s"%all_modules) + help="run on all modules as found in %s"%default_modules_list) + parser.add_option("-f","--file",action="store",dest="modules_list",default=None, + help="run on all modules found in specified file") else: parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False, help="dry run - shell commands are only displayed") - parser.add_option("-t","--distrotags",action="store",dest="distrotags",default=None, - help="specify a distro-tags file, e.g. onelab-tags-4.2.mk") + parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags", + default=[], nargs=1,type="string", + help="""specify distro-tags files, e.g. onelab-tags-4.2.mk +-- can be set multiple times, or use quotes""") parser.add_option("-w","--workdir", action="store", dest="workdir", default="%s/%s"%(os.getenv("HOME"),"modules"), help="""name for dedicated working dir - defaults to ~/modules ** THIS MUST NOT ** be your usual working directory""") - parser.add_option("-f","--fast-checks",action="store_true",dest="fast_checks",default=False, + parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False, help="skip safety checks, such as svn updates -- use with care") # default verbosity depending on function - temp @@ -1160,7 +1228,7 @@ Branches: parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, help="run in verbose mode") else: - parser.add_parser("-q","--quiet", action="store_false", dest="verbose", default=True, + parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True, help="run in quiet (non-verbose) mode") # parser.add_option("-d","--debug", action="store_true", dest="debug", default=False, # help="debug mode - mostly more verbose") @@ -1184,7 +1252,9 @@ Branches: ########## module-* if len(args) == 0: if options.all_modules: - args=Command("grep -v '#' %s"%all_modules,options).output_of().split() + options.modules_list=default_modules_list + if options.modules_list: + args=Command("grep -v '#' %s"%options.modules_list,options).output_of().split() else: parser.print_help() sys.exit(1)