X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=module-tools.py;h=b9d0cb74bc1c8bf9af896f0067aef6626d318aeb;hb=4bfad922a62b5782224598da4a17bd9419b744b1;hp=ecf635ad2c91fc72bb4efcb4f0b7ba7aaec95d73;hpb=12814b7ad55bf0bf100107a1c47d1a9f05aab15c;p=build.git diff --git a/module-tools.py b/module-tools.py index ecf635ad..b9d0cb74 100755 --- a/module-tools.py +++ b/module-tools.py @@ -9,26 +9,26 @@ from glob import glob from optparse import OptionParser # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase -def prompt (question,default=True,other_choices=[]): +def prompt (question,default=True,other_choices=[],allow_outside=False): if not isinstance (other_choices,list): other_choices = [ other_choices ] chars = [ c for (c,rest) in other_choices ] - the_prompt = [] + choices = [] if 'y' not in chars: - if default is True: the_prompt.append('[y]') - else : the_prompt.append('y') + if default is True: choices.append('[y]') + else : choices.append('y') if 'n' not in chars: - if default is False: the_prompt.append('[n]') - else : the_prompt.append('n') + if default is False: choices.append('[n]') + else : choices.append('n') for (char,choice) in other_choices: if default == char: - the_prompt.append("["+char+"]"+choice) + choices.append("["+char+"]"+choice) else: - the_prompt.append("<"+char+">"+choice) + choices.append("<"+char+">"+choice) try: - answer=raw_input(question + " " + "/".join(the_prompt) + " ? ") + answer=raw_input(question + " " + "/".join(choices) + " ? ") if not answer: return default answer=answer[0].lower() @@ -42,10 +42,28 @@ def prompt (question,default=True,other_choices=[]): for (char,choice) in other_choices: if answer == char: return char + if allow_outside: + return answer return prompt(question,default,other_choices) except: raise +def default_editor(): + try: + editor = os.environ['EDITOR'] + except: + editor = "emacs" + return editor + +### fold long lines +fold_length=132 + +def print_fold (line): + while len(line) >= fold_length: + print line[:fold_length],'\\' + line=line[fold_length:] + print line + class Command: def __init__ (self,command,options): self.command=command @@ -53,18 +71,24 @@ class Command: self.tmp="/tmp/command-%d"%os.getpid() def run (self): - if self.options.verbose: + if self.options.dry_run: + print 'dry_run',self.command + return 0 + if self.options.verbose and self.options.mode not in Main.silent_modes: print '+',self.command sys.stdout.flush() return os.system(self.command) def run_silent (self): + if self.options.dry_run: + print 'dry_run',self.command + return 0 if self.options.verbose: print '+',self.command,' .. ', sys.stdout.flush() retcod=os.system(self.command + " &> " + self.tmp) if retcod != 0: - print "FAILED ! -- output quoted below " + print "FAILED ! -- out+err below (command was %s)"%self.command os.system("cat " + self.tmp) print "FAILED ! -- end of quoted output" elif self.options.verbose: @@ -78,6 +102,9 @@ class Command: # returns stdout, like bash's $(mycommand) def output_of (self,with_stderr=False): + if self.options.dry_run: + print 'dry_run',self.command + return 'dry_run output' tmp="/tmp/status-%d"%os.getpid() if self.options.debug: print '+',self.command,' .. ', @@ -111,6 +138,7 @@ class Svnpath: command="svn status %s"%self.path return len(Command(command,self.options).output_of(True)) != 0 +# support for tagged module is minimal, and is for the Build class only class Module: svn_magic_line="--This line, and those below, will be ignored--" @@ -141,7 +169,10 @@ class Module: # for parsing module spec name:branch - matcher_branch_spec=re.compile("\A(?P[\w-]+):(?P[\w\.]+)\Z") + matcher_branch_spec=re.compile("\A(?P[\w\.-]+):(?P[\w\.-]+)\Z") + # special form for tagged module - for Build + matcher_tag_spec=re.compile("\A(?P[\w-]+)@(?P[\w\.-]+)\Z") + # parsing specfiles matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*") def __init__ (self,module_spec,options): @@ -151,26 +182,34 @@ class Module: self.name=attempt.group('name') self.branch=attempt.group('branch') else: - self.name=module_spec - self.branch=None + attempt=Module.matcher_tag_spec.match(module_spec) + if attempt: + self.name=attempt.group('name') + self.tagname=attempt.group('tagname') + else: + self.name=module_spec self.options=options - self.moddir="%s/%s"%(options.workdir,self.name) + self.module_dir="%s/%s"%(options.workdir,self.name) def friendly_name (self): - if not self.branch: - return self.name - else: + if hasattr(self,'branch'): return "%s:%s"%(self.name,self.branch) + elif hasattr(self,'tagname'): + return "%s@%s"%(self.name,self.tagname) + else: + return self.name def edge_dir (self): - if not self.branch: - return "%s/trunk"%(self.moddir) + if hasattr(self,'branch'): + return "%s/branches/%s"%(self.module_dir,self.branch) + elif hasattr(self,'tagname'): + return "%s/tags/%s"%(self.module_dir,self.tagname) else: - return "%s/branches/%s"%(self.moddir,self.branch) + return "%s/trunk"%(self.module_dir) def tags_dir (self): - return "%s/tags"%(self.moddir) + return "%s/tags"%(self.module_dir) def run (self,command): return Command(command,self.options).run() @@ -178,16 +217,24 @@ class Module: return Command(command,self.options).run_fatal() def run_prompt (self,message,command): if not self.options.verbose: - question=message + while True: + choice=prompt(message,True,('s','how')) + if choice is True: + self.run(command) + return + elif choice is False: + return + else: + print 'About to run:',command else: question=message+" - want to run " + command - if prompt(question,True): - self.run(command) + if prompt(question,True): + self.run(command) @staticmethod def init_homedir (options): topdir=options.workdir - if options.verbose: + if options.verbose and options.mode not in Main.silent_modes: print 'Checking for',topdir storage="%s/%s"%(topdir,Module.config_storage) # sanity check. Either the topdir exists AND we have a config/storage @@ -225,25 +272,24 @@ that for other purposes than tagging"""%topdir (key,value)=re.compile("^(.+)=(.+)$").match(line).groups() Module.config[key]=value f.close() - if options.verbose: + if options.verbose and options.mode not in Main.silent_modes: print '******** Using config' for (key,message,default) in Module.configKeys: print '\t',key,'=',Module.config[key] - def init_moddir (self): + def init_module_dir (self): if self.options.verbose: - print 'Checking for',self.moddir - if not os.path.isdir (self.moddir): - self.run_fatal("svn up -N %s"%self.moddir) - if not os.path.isdir (self.moddir): - print 'Cannot find %s - check module name'%self.moddir - sys.exit(1) + print 'Checking for',self.module_dir + if not os.path.isdir (self.module_dir): + self.run_fatal("svn update -N %s"%self.module_dir) + if not os.path.isdir (self.module_dir): + raise Exception, 'Cannot find %s - check module name'%self.module_dir def init_subdir (self,fullpath): if self.options.verbose: print 'Checking for',fullpath if not os.path.isdir (fullpath): - self.run_fatal("svn up -N %s"%fullpath) + self.run_fatal("svn update -N %s"%fullpath) def revert_subdir (self,fullpath): if self.options.fast_checks: @@ -264,8 +310,10 @@ that for other purposes than tagging"""%topdir def init_edge_dir (self): # if branch, edge_dir is two steps down - if self.branch: - self.init_subdir("%s/branches"%self.moddir) + if hasattr(self,'branch'): + self.init_subdir("%s/branches"%self.module_dir) + elif hasattr(self,'tagname'): + self.init_subdir("%s/tags"%self.module_dir) self.init_subdir(self.edge_dir()) def revert_edge_dir (self): @@ -279,11 +327,11 @@ that for other purposes than tagging"""%topdir if os.path.isfile (attempt): return attempt else: + pattern="%s/*.spec"%self.edge_dir() try: - return glob("%s/*.spec"%self.edge_dir())[0] + return glob(pattern)[0] except: - print 'Cannot guess specfile for module %s'%self.name - sys.exit(1) + raise Exception, 'Cannot guess specfile for module %s -- pattern was %s'%(self.name,pattern) def all_specnames (self): return glob("%s/*.spec"%self.edge_dir()) @@ -331,8 +379,10 @@ that for other purposes than tagging"""%topdir print '2st pass parsing done, varnames=',varnames,'result=',result return result - def patch_spec_var (self, patch_dict): + def patch_spec_var (self, patch_dict,define_missing=False): for specfile in self.all_specnames(): + # record the keys that were changed + changed = dict ( [ (x,False) for x in patch_dict.keys() ] ) newspecfile=specfile+".new" if self.options.verbose: print 'Patching',specfile,'for',patch_dict.keys() @@ -344,21 +394,35 @@ that for other purposes than tagging"""%topdir if attempt: (define,var,value)=attempt.groups() if var in patch_dict.keys(): + if self.options.debug: + print 'rewriting %s as %s'%(var,patch_dict[var]) new.write('%%%s %s %s\n'%(define,var,patch_dict[var])) + changed[var]=True continue new.write(line) + if define_missing: + for (key,was_changed) in changed.iteritems(): + if not was_changed: + if self.options.debug: + print 'rewriting missing %s as %s'%(key,patch_dict[key]) + new.write('\n%%define %s %s\n'%(key,patch_dict[key])) spec.close() new.close() os.rename(newspecfile,specfile) def unignored_lines (self, logfile): result=[] - exclude="Tagging module %s"%self.name + exclude="Setting tag %s"%self.name + white_line_matcher = re.compile("\A\s*\Z") for logline in file(logfile).readlines(): if logline.strip() == Module.svn_magic_line: break - if logline.find(exclude) < 0: - result += [ logline ] + if logline.find(exclude) >= 0: + continue + elif white_line_matcher.match(logline): + continue + else: + result.append(logline.strip()+'\n') return result def insert_changelog (self, logfile, oldtag, newtag): @@ -373,9 +437,9 @@ that for other purposes than tagging"""%topdir if re.compile('%changelog').match(line): dateformat="* %a %b %d %Y" datepart=time.strftime(dateformat) - logpart="%s <%s> - %s %s"%(Module.config['username'], + logpart="%s <%s> - %s"%(Module.config['username'], Module.config['email'], - oldtag,newtag) + newtag) new.write(datepart+" "+logpart+"\n") for logline in self.unignored_lines(logfile): new.write("- " + logline) @@ -393,10 +457,12 @@ that for other purposes than tagging"""%topdir return "%s/%s"%(Module.config['svnpath'],self.name) def edge_url (self): - if not self.branch: - return "%s/trunk"%(self.mod_url()) - else: + if hasattr(self,'branch'): return "%s/branches/%s"%(self.mod_url(),self.branch) + elif hasattr(self,'tagname'): + return "%s/tags/%s"%(self.mod_url(),self.tagname) + else: + return "%s/trunk"%(self.mod_url()) def tag_name (self, spec_dict): try: @@ -405,8 +471,7 @@ that for other purposes than tagging"""%topdir spec_dict[self.module_version_varname], spec_dict[self.module_taglevel_varname]) except KeyError,err: - print 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err) - sys.exit(1) + raise Exception, 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err) def tag_url (self, spec_dict): return "%s/tags/%s"%(self.mod_url(),self.tag_name(spec_dict)) @@ -421,8 +486,8 @@ that for other purposes than tagging"""%topdir if self.options.verbose: print 'exists - OK' else: if self.options.verbose: print 'KO' - print 'Could not find %s URL %s'%(message,url) - sys.exit(1) + raise Exception, 'Could not find %s URL %s'%(message,url) + def check_svnpath_not_exists (self, url, message): if self.options.fast_checks: return @@ -433,13 +498,13 @@ that for other purposes than tagging"""%topdir if self.options.verbose: print 'does not exist - OK' else: if self.options.verbose: print 'KO' - print '%s URL %s already exists - exiting'%(message,url) - sys.exit(1) + raise Exception, '%s URL %s already exists - exiting'%(message,url) # locate specfile, parse it, check it and show values + ############################## def do_version (self): - self.init_moddir() + self.init_module_dir() self.init_edge_dir() self.revert_edge_dir() self.update_edge_dir() @@ -449,25 +514,70 @@ that for other purposes than tagging"""%topdir print 'Could not find %%define for %s'%varname return else: - print varname+":",spec_dict[varname] - print 'edge url',self.edge_url() - print 'latest tag url',self.tag_url(spec_dict) + print "%-16s %s"%(varname,spec_dict[varname]) + if self.options.show_urls: + print "%-16s %s"%('edge url',self.edge_url()) + print "%-16s %s"%('latest tag url',self.tag_url(spec_dict)) if self.options.verbose: - print 'main specfile:',self.main_specname() - print 'specfiles:',self.all_specnames() + print "%-16s %s"%('main specfile:',self.main_specname()) + print "%-16s %s"%('specfiles:',self.all_specnames()) - init_warning="""*** WARNING -The module-init function has the following limitations +############################## + def do_list (self): +# print 'verbose',self.options.verbose +# print 'list_tags',self.options.list_tags +# print 'list_branches',self.options.list_branches +# print 'all_modules',self.options.all_modules + + (verbose,branches,pattern,exact) = (self.options.verbose,self.options.list_branches, + self.options.list_pattern,self.options.list_exact) + + extra_command="" + extra_message="" + if hasattr(self,'branch'): + pattern=self.branch + if pattern or exact: + if exact: + if verbose: grep="%s/$"%exact + else: grep="^%s$"%exact + else: + grep=pattern + extra_command=" | grep %s"%grep + extra_message=" matching %s"%grep + + if not branches: + message="==================== tags for %s"%self.friendly_name() + command="svn list " + if verbose: command+="--verbose " + command += "%s/tags"%self.mod_url() + command += extra_command + message += extra_message + if verbose: print message + self.run(command) + + else: + message="==================== branches for %s"%self.friendly_name() + command="svn list " + if verbose: command+="--verbose " + command += "%s/branches"%self.mod_url() + command += extra_command + message += extra_message + if verbose: print message + self.run(command) + +############################## + sync_warning="""*** WARNING +The module-sync function has the following limitations * it does not handle changelogs * it does not scan the -tags*.mk files to adopt the new tags""" -############################## + def do_sync(self): if self.options.verbose: - print Module.init_warning + print Module.sync_warning if not prompt('Want to proceed anyway'): return - self.init_moddir() + self.init_module_dir() self.init_edge_dir() self.revert_edge_dir() self.update_edge_dir() @@ -488,7 +598,7 @@ The module-init function has the following limitations ############################## def do_diff (self,compute_only=False): - self.init_moddir() + self.init_module_dir() self.init_edge_dir() self.revert_edge_dir() self.update_edge_dir() @@ -501,7 +611,8 @@ The module-init function has the following limitations self.check_svnpath_exists(tag_url,"latest tag") command="svn diff %s %s"%(tag_url,edge_url) if compute_only: - print 'Getting diff with %s'%command + if self.options.verbose: + print 'Getting diff with %s'%command diff_output = Command(command,self.options).output_of() # if used as a utility if compute_only: @@ -541,14 +652,16 @@ The module-init function has the following limitations # brute-force : change uncommented lines that define -SVNPATH else: if self.options.verbose: - print 'Setting %s-SVNPATH for using %s\n\tin %s .. '%(self.name,newname,tagsfile), - pattern="\A\s*%s-SVNPATH\s*(=|:=)\s*(?P[^\s]+)/%s/[^\s]+"\ - %(self.name,self.name) + print 'Searching for -SVNPATH lines referring to /%s/\n\tin %s .. '%(self.name,tagsfile), + pattern="\A\s*(?P[^\s]+)-SVNPATH\s*(=|:=)\s*(?P[^\s]+)/%s/[^\s]+"\ + %(self.name) matcher_module=re.compile(pattern) for line in tags.readlines(): attempt=matcher_module.match(line) if attempt: - svnpath="%s-SVNPATH"%self.name + svnpath="%s-SVNPATH"%(attempt.group('make_name')) + if self.options.verbose: + print ' '+svnpath, replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname) new.write(replacement) matches += 1 @@ -561,7 +674,7 @@ The module-init function has the following limitations return matches def do_tag (self): - self.init_moddir() + self.init_module_dir() self.init_edge_dir() self.revert_edge_dir() self.update_edge_dir() @@ -593,7 +706,7 @@ The module-init function has the following limitations diff_output=Command("svn diff %s %s"%(old_tag_url,edge_url), self.options).output_of() if len(diff_output) == 0: - if not prompt ("No difference in trunk for module %s, want to tag anyway"%self.name,False): + if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False): return # side effect in trunk's specfile @@ -604,11 +717,11 @@ The module-init function has the following limitations # so we can provide useful information, such as version numbers and diff # in the same file changelog="/tmp/%s-%d.txt"%(self.name,os.getpid()) - file(changelog,"w").write("""Tagging module %s - %s + file(changelog,"w").write("""Setting tag %s %s Please write a changelog for this new tag in the section above -"""%(self.name,new_tag_name,Module.svn_magic_line)) +"""%(new_tag_name,Module.svn_magic_line)) if not self.options.verbose or prompt('Want to see diffs while writing changelog',True): file(changelog,"a").write('DIFF=========\n' + diff_output) @@ -627,49 +740,56 @@ Please write a changelog for this new tag in the section above buildname=Module.config['build'] except: buildname="build" + if self.options.build_branch: + buildname+=":"+self.options.build_branch build = Module(buildname,self.options) - build.init_moddir() + build.init_module_dir() build.init_edge_dir() build.revert_edge_dir() build.update_edge_dir() tagsfiles=glob(build.edge_dir()+"/*-tags*.mk") tagsdict=dict( [ (x,'todo') for x in tagsfiles ] ) + default_answer = 'y' while True: for (tagsfile,status) in tagsdict.iteritems(): + basename=os.path.basename(tagsfile) + print ".................... Dealing with %s"%basename while tagsdict[tagsfile] == 'todo' : - choice = prompt ("Want to adopt %s in %s "%(new_tag_name,os.path.basename(tagsfile)),'a', - [ ('n', 'ext'), ('a','uto'), ('d','iff'), ('r','evert'), ('h','elp') ] ) - if choice is True: - self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False) - elif choice is 'n': + choice = prompt ("insert %s in %s "%(new_tag_name,basename),default_answer, + [ ('y','es'), ('n', 'ext'), ('f','orce'), + ('d','iff'), ('r','evert'), ('h','elp') ] , + allow_outside=True) + if choice == 'y': + self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True) + elif choice == 'n': print 'Done with %s'%os.path.basename(tagsfile) tagsdict[tagsfile]='done' - elif choice == 'a': - self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True) + elif choice == 'f': + self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False) elif choice == 'd': self.run("svn diff %s"%tagsfile) elif choice == 'r': self.run("svn revert %s"%tagsfile) - elif choice == 'h': + else: name=self.name - print """y: unconditionnally changes any line setting %(name)s-SVNPATH to using %(new_tag_name)s -a: changes the definition of %(name)s only if it currently refers to %(old_tag_name)s -d: shows current diff for this tag file -r: reverts that tag file + print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s +f: unconditionnally change any line setting %(name)s-SVNPATH to using %(new_tag_name)s +d: show current diff for this tag file +r: revert that tag file n: move to next file"""%locals() - else: - print 'unexpected' + if prompt("Want to review changes on tags files",False): tagsdict = dict ( [ (x, 'todo') for tagsfile in tagsfiles ] ) + default_answer='d' else: break paths="" paths += self.edge_dir() + " " paths += build.edge_dir() + " " - self.run_prompt("Check","svn diff " + paths) - self.run_prompt("Commit","svn commit --file %s %s"%(changelog,paths)) + self.run_prompt("Review module and build","svn diff " + paths) + self.run_prompt("Commit module and build","svn commit --file %s %s"%(changelog,paths)) self.run_prompt("Create tag","svn copy --file %s %s %s"%(changelog,edge_url,new_tag_url)) if self.options.debug: @@ -683,170 +803,476 @@ n: move to next file"""%locals() # save self.branch if any, as a hint for the new branch # do this before anything else and restore .branch to None, # as this is part of the class's logic - new_branch_name=None - if self.branch: - new_branch_name=self.branch - self.branch=None - - # check module-diff is empty - # used in do_diff, and should be options for diff only - (spec_dict,edge_url,tag_url,diff_listing) = self.do_diff(compute_only=True) - if diff_listing: - print '*** WARNING : Module %s has pending diffs on its trunk'%self.name - while True: - answer = prompt ('Are you sure you want to proceed with branching',False,('d','iff')) - if answer is True: - break - elif answer is False: - sys.exit(1) - elif answer == 'd': - print '<<<< %s'%tag_url - print '>>>> %s'%edge_url - print diff_listing - + new_trunk_name=None + if hasattr(self,'branch'): + new_trunk_name=self.branch + del self.branch + elif self.options.new_version: + new_trunk_name = self.options.new_version + + # compute diff - a way to initialize the whole stuff # do_diff already does edge_dir initialization # and it checks that edge_url and tag_url exist as well - print "Using starting point %s"%tag_url + (spec_dict,edge_url,tag_url,diff_listing) = self.do_diff(compute_only=True) - # figure new branch name - if not new_branch_name: + # the version name in the trunk becomes the new branch name + branch_name = spec_dict[self.module_version_varname] + + # figure new branch name (the one for the trunk) if not provided on the command line + if not new_trunk_name: # heuristic is to assume 'version' is a dot-separated name # we isolate the rightmost part and try incrementing it by 1 - print 'Trying to guess a new branch name for the trunk' version=spec_dict[self.module_version_varname] try: m=re.compile("\A(?P.+)\.(?P[^\.]+)\Z") (leftpart,rightmost)=m.match(version).groups() incremented = int(rightmost)+1 - new_branch_name="%s.%d"%(leftpart,incremented) + new_trunk_name="%s.%d"%(leftpart,incremented) except: - print 'Cannot figure next branch name from %s - exiting'%version - sys.exit(1) + raise Exception, 'Cannot figure next branch name from %s - exiting'%version - branch_url = "%s/%s/branches/%s"%(Module.config['svnpath'],self.name,new_branch_name) - self.check_svnpath_not_exists (branch_url,"new branch") - # record starting point tagname - old_tag_name = self.tag_name(spec_dict) + latest_tag_name = self.tag_name(spec_dict) + + print "**********" + print "Using starting point %s (%s)"%(tag_url,latest_tag_name) + print "Creating branch %s & moving trunk to %s"%(branch_name,new_trunk_name) + print "**********" + # print warning if pending diffs + if diff_listing: + print """*** WARNING : Module %s has pending diffs on its trunk +It is safe to proceed, but please note that branch %s +will be based on latest tag %s and *not* on the current trunk"""%(self.name,branch_name,latest_tag_name) + while True: + answer = prompt ('Are you sure you want to proceed with branching',True,('d','iff')) + if answer is True: + break + elif answer is False: + raise Exception,"User quit" + elif answer == 'd': + print '<<<< %s'%tag_url + print '>>>> %s'%edge_url + print diff_listing + + branch_url = "%s/%s/branches/%s"%(Module.config['svnpath'],self.name,branch_name) + self.check_svnpath_not_exists (branch_url,"new branch") + # patching trunk - spec_dict[self.module_version_varname]=new_branch_name + spec_dict[self.module_version_varname]=new_trunk_name spec_dict[self.module_taglevel_varname]='0' - - self.patch_spec_var(spec_dict) + # remember this in the trunk for easy location of the current branch + spec_dict['module_current_branch']=branch_name + self.patch_spec_var(spec_dict,True) # create commit log file tmp="/tmp/branching-%d"%os.getpid() f=open(tmp,"w") - f.write("Branch %s for module %s created from tag %s\n"%(new_branch_name,self.name,old_tag_name)) + f.write("Branch %s for module %s created (as new trunk) from tag %s\n"%(new_trunk_name,self.name,latest_tag_name)) f.close() # we're done, let's commit the stuff command="svn diff %s"%self.edge_dir() - self.run_prompt("Check trunk",command) - command="svn copy %s %s"%(self.edge_dir(),branch_url) + self.run_prompt("Review changes in trunk",command) + command="svn copy --file %s %s %s"%(tmp,self.edge_url(),branch_url) self.run_prompt("Create branch",command) command="svn commit --file %s %s"%(tmp,self.edge_dir()) self.run_prompt("Commit trunk",command) + new_tag_url=self.tag_url(spec_dict) + command="svn copy --file %s %s %s"%(tmp,self.edge_url(),new_tag_url) + self.run_prompt("Create initial tag in trunk",command) os.unlink(tmp) ############################## -usage="""Usage: %prog options module_desc [ .. module_desc ] -Purpose: - manage subversion tags and specfile - requires the specfile to define *version* and *taglevel* +class Package: + + def __init__(self, package, module, svnpath, spec): + self.package=package + self.module=module + self.svnpath=svnpath + self.spec=spec + self.specpath="%s/%s"%(svnpath,spec) + self.basename=os.path.basename(svnpath) + + # returns a http URL to the trac path where full diff can be viewed (between self and pkg) + # typically http://svn.planet-lab.org/changeset?old_path=Monitor%2Ftags%2FMonitor-1.0-7&new_path=Monitor%2Ftags%2FMonitor-1.0-13 + # xxx quick & dirty: rough url parsing + def trac_full_diff (self, pkg): + matcher=re.compile("\A(?P.*)://(?P[^/]+)/(svn/)?(?P.*)\Z") + self_match=matcher.match(self.svnpath) + pkg_match=matcher.match(pkg.svnpath) + if self_match and pkg_match: + (method,hostname,svn,path)=self_match.groups() + self_path=path.replace("/","%2F") + pkg_path=pkg_match.group('path').replace("/","%2F") + return "%s://%s/changeset?old_path=%s&new_path=%s"%(method,hostname,self_path,pkg_path) + else: + return None + + def details (self): + return "[%s %s] [%s (spec)]"%(self.svnpath,self.basename,self.specpath) + +class Build (Module): + + # we cannot get build's svnpath as for other packages as we'd get something in svn+ssh + # xxx quick & dirty + def __init__ (self, buildtag,options): + self.buildtag=buildtag + # if the buildtag start with a : (to use a branch rather than a tag) + if buildtag.find(':') == 0 : + module_name="build%(buildtag)s"%locals() + self.display=buildtag[1:] + self.svnpath="http://svn.planet-lab.org/svn/build/branches/%s"%self.display + else : + module_name="build@%(buildtag)s"%locals() + self.display=buildtag + self.svnpath="http://svn.planet-lab.org/svn/build/tags/%s"%self.buildtag + Module.__init__(self,module_name,options) + + @staticmethod + def get_distro_from_distrotag (distrotag): + # mhh: remove -tag* from distrotags to get distro + n=distrotag.find('-tag') + if n>0: + return distrotag[:n] + else: + return None + + def get_packages (self,distrotag): + result={} + distro=Build.get_distro_from_distrotag(distrotag) + if not distro: + return result + make_options="--no-print-directory -C %s stage1=true PLDISTRO=%s PLDISTROTAGS=%s 2> /dev/null"%(self.edge_dir(),distro,distrotag) + command="make %s packages"%make_options + make_packages=Command(command,self.options).output_of() + pkg_line=re.compile("\Apackage=(?P[^\s]+)\s+ref_module=(?P[^\s]+)\s.*\Z") + for line in make_packages.split("\n"): + if not line: + continue + attempt=pkg_line.match(line) + if line and not attempt: + print "=====" + print "WARNING: line not understood from make packages" + print "in dir %s"%self.edge_dir + print "with options",make_options + print 'line=',line + print "=====" + else: + (package,module) = (attempt.group('package'),attempt.group('module')) + command="make %s +%s-SVNPATH"%(make_options,module) + svnpath=Command(command,self.options).output_of().strip() + command="make %s +%s-SPEC"%(make_options,package) + spec=Command(command,self.options).output_of().strip() + result[package]=Package(package,module,svnpath,spec) + return result + + def get_distrotags (self): + return [os.path.basename(p) for p in glob("%s/*tags*mk"%self.edge_dir())] + +class DiffCache: + + def __init__ (self): + self._cache={} + + def key(self, frompath,topath): + return frompath+'-to-'+topath + + def fetch (self, frompath, topath): + key=self.key(frompath,topath) + if not self._cache.has_key(key): + return None + return self._cache[key] + + def store (self, frompath, topath, diff): + key=self.key(frompath,topath) + self._cache[key]=diff + +class Release: + + # header in diff output + discard_matcher=re.compile("\A(\+\+\+|---).*") + + @staticmethod + def do_changelog (buildtag_new,buildtag_old,options): + print "----" + print "----" + print "----" + (build_new,build_old) = (Build (buildtag_new,options), Build (buildtag_old,options)) + print "= build tag %s to %s = #build-%s"%(build_old.display,build_new.display,build_new.display) + for b in (build_new,build_old): + b.init_module_dir() + b.init_edge_dir() + b.update_edge_dir() + # find out the tags files that are common, unless option was specified + if options.distrotags: + distrotags=options.distrotags + else: + distrotags_new=build_new.get_distrotags() + distrotags_old=build_old.get_distrotags() + distrotags = list(set(distrotags_new).intersection(set(distrotags_old))) + distrotags.sort() + if options.verbose: print "Found distrotags",distrotags + first_distrotag=True + diffcache = DiffCache() + for distrotag in distrotags: + distro=Build.get_distro_from_distrotag(distrotag) + if not distro: + continue + if first_distrotag: + first_distrotag=False + else: + print '----' + print '== distro %s (%s to %s) == #distro-%s-%s'%(distrotag,build_old.display,build_new.display,distro,build_new.display) + print ' * from %s/%s'%(build_old.svnpath,distrotag) + print ' * to %s/%s'%(build_new.svnpath,distrotag) + + # parse make packages + packages_new=build_new.get_packages(distrotag) + pnames_new=set(packages_new.keys()) + if options.verbose: print 'got packages for ',build_new.display + packages_old=build_old.get_packages(distrotag) + pnames_old=set(packages_old.keys()) + if options.verbose: print 'got packages for ',build_old.display + + # get created, deprecated, and preserved package names + pnames_created = list(pnames_new-pnames_old) + pnames_created.sort() + pnames_deprecated = list(pnames_old-pnames_new) + pnames_deprecated.sort() + pnames = list(pnames_new.intersection(pnames_old)) + pnames.sort() + + if options.verbose: print "Found new/deprecated/preserved pnames",pnames_new,pnames_deprecated,pnames + + # display created and deprecated + for name in pnames_created: + print '=== %s : new package %s -- appeared in %s === #package-%s-%s-%s'%( + distrotag,name,build_new.display,name,distro,build_new.display) + pobj=packages_new[name] + print ' * %s'%pobj.details() + for name in pnames_deprecated: + print '=== %s : package %s -- deprecated, last occurrence in %s === #package-%s-%s-%s'%( + distrotag,name,build_old.display,name,distro,build_new.display) + pobj=packages_old[name] + if not pobj.svnpath: + print ' * codebase stored in CVS, specfile is %s'%pobj.spec + else: + print ' * %s'%pobj.details() + + # display other packages + for name in pnames: + (pobj_new,pobj_old)=(packages_new[name],packages_old[name]) + if options.verbose: print "Dealing with package",name + if pobj_old.specpath == pobj_new.specpath: + continue + specdiff = diffcache.fetch(pobj_old.specpath,pobj_new.specpath) + if specdiff is None: + command="svn diff %s %s"%(pobj_old.specpath,pobj_new.specpath) + specdiff=Command(command,options).output_of() + diffcache.store(pobj_old.specpath,pobj_new.specpath,specdiff) + else: + if options.verbose: print 'got diff from cache' + if not specdiff: + continue + print '=== %s - %s to %s : package %s === #package-%s-%s-%s'%( + distrotag,build_old.display,build_new.display,name,name,distro,build_new.display) + print ' * from %s to %s'%(pobj_old.details(),pobj_new.details()) + trac_diff_url=pobj_old.trac_full_diff(pobj_new) + if trac_diff_url: + print ' * [%s View full diff]'%trac_diff_url + print '{{{' + for line in specdiff.split('\n'): + if not line: + continue + if Release.discard_matcher.match(line): + continue + if line[0] in ['@']: + print '----------' + elif line[0] in ['+','-']: + print_fold(line) + print '}}}' + +############################## +class Main: + + module_usage="""Usage: %prog [options] module_desc [ .. module_desc ] +module-tools : a set of tools to manage subversion tags and specfile + requires the specfile to either + * define *version* and *taglevel* OR alternatively - redirection variables module_version_varname / module_taglevel_varname + * define redirection variables module_version_varname / module_taglevel_varname Trunk: by default, the trunk of modules is taken into account in this case, just mention the module name as Branches: if you wish to work on a branch rather than on the trunk, you can use something like e.g. Mom:2.1 as -More help: - see http://svn.planet-lab.org/wiki/ModuleTools """ - -functions={ - 'version' : "only check specfile and print out details", - 'diff' : "show difference between trunk and latest tag", - 'tag' : """increment taglevel in specfile, insert changelog in specfile, + release_usage="""Usage: %prog [options] tag1 .. tagn + Extract release notes from the changes in specfiles between several build tags, latest first + Examples: + release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22 + You can refer to a (build) branch by prepending a colon, like in + release-changelog :4.2 4.2-rc25 +""" + common_usage="""More help: + see http://svn.planet-lab.org/wiki/ModuleTools""" + + modes={ + 'list' : "displays a list of available tags or branches", + 'version' : "check latest specfile and print out details", + 'diff' : "show difference between module (trunk or branch) and latest tag", + 'tag' : """increment taglevel in specfile, insert changelog in specfile, create new tag and and monitor its adoption in build/*-tags*.mk""", - 'branch' : """create a branch for this module, from the latest tag on the trunk, + 'branch' : """create a branch for this module, from the latest tag on the trunk, and change trunk's version number to reflect the new branch name; you can specify the new branch name by using module:branch""", - 'sync' : """create a tag from the trunk + 'sync' : """create a tag from the module this is a last resort option, mostly for repairs""", -} - -def main(): - - mode=None - for function in functions.keys(): - if sys.argv[0].find(function) >= 0: - mode = function - break - if not mode: - print "Unsupported command",sys.argv[0] - sys.exit(1) - - global usage - usage += "\nmodule-%s : %s"%(mode,functions[mode]) - all_modules=os.path.dirname(sys.argv[0])+"/modules.list" - - parser=OptionParser(usage=usage,version=subversion_id) - parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, - help="run on all modules as found in %s"%all_modules) - parser.add_option("-f","--fast-checks",action="store_true",dest="fast_checks",default=False, - help="skip safety checks, such as svn updates -- use with care") - if mode == "tag" or mode == 'branch': - parser.add_option("-s","--set-version",action="store",dest="new_version",default=None, - help="set new version and reset taglevel to 0") - if mode == "tag" : - parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True, - help="do not update changelog section in specfile when tagging") - if mode == "tag" or mode == "sync" : - parser.add_option("-e","--editor", action="store", dest="editor", default="emacs", - help="specify editor") - if mode == "sync" : - parser.add_option("-m","--message", action="store", dest="message", default=None, - help="specify log message") - if mode == "diff" : - parser.add_option("-o","--only", action="store_true", dest="only", default=False, - help="report diff only for modules that exhibit differences") - if mode == "diff" : - parser.add_option("-l","--list", action="store_true", dest="list", default=False, - help="just list modules that exhibit differences") - parser.add_option("-w","--workdir", action="store", dest="workdir", - default="%s/%s"%(os.getenv("HOME"),"modules"), - help="""name for dedicated working dir - defaults to ~/modules + 'changelog' : """extract changelog between build tags + expected arguments are a list of tags""", + } + + silent_modes = ['list'] + release_modes = ['changelog'] + + @staticmethod + def optparse_list (option, opt, value, parser): + try: + setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split()) + except: + setattr(parser.values,option.dest,value.split()) + + def run(self): + + mode=None + for function in Main.modes.keys(): + if sys.argv[0].find(function) >= 0: + mode = function + break + if not mode: + print "Unsupported command",sys.argv[0] + print "Supported commands:" + Modes.modes.keys.join(" ") + sys.exit(1) + + if mode not in Main.release_modes: + usage = Main.module_usage + usage += Main.common_usage + usage += "\nmodule-%s : %s"%(mode,Main.modes[mode]) + else: + usage = Main.release_usage + usage += Main.common_usage + + parser=OptionParser(usage=usage,version=subversion_id) + + if mode == 'list': + parser.add_option("-b","--branches",action="store_true",dest="list_branches",default=False, + help="list branches") + parser.add_option("-t","--tags",action="store_false",dest="list_branches", + help="list tags") + parser.add_option("-m","--match",action="store",dest="list_pattern",default=None, + help="grep pattern for filtering output") + parser.add_option("-x","--exact-match",action="store",dest="list_exact",default=None, + help="exact grep pattern for filtering output") + if mode == "tag" or mode == 'branch': + parser.add_option("-s","--set-version",action="store",dest="new_version",default=None, + help="set new version and reset taglevel to 0") + if mode == "tag" : + parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True, + help="do not update changelog section in specfile when tagging") + parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None, + help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place") + if mode == "tag" or mode == "sync" : + parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(), + help="specify editor") + if mode == "sync" : + parser.add_option("-m","--message", action="store", dest="message", default=None, + help="specify log message") + if mode == "diff" : + parser.add_option("-o","--only", action="store_true", dest="only", default=False, + help="report diff only for modules that exhibit differences") + if mode == "diff" : + parser.add_option("-l","--list", action="store_true", dest="list", default=False, + help="just list modules that exhibit differences") + + if mode == 'version': + parser.add_option("-u","--url", action="store_true", dest="show_urls", default=False, + help="display URLs") + + default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list" + if mode not in Main.release_modes: + parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False, + help="run on all modules as found in %s"%default_modules_list) + parser.add_option("-f","--file",action="store",dest="modules_list",default=None, + help="run on all modules found in specified file") + else: + parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False, + help="dry run - shell commands are only displayed") + parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags", + default=[], nargs=1,type="string", + help="""specify distro-tags files, e.g. onelab-tags-4.2.mk +-- can be set multiple times, or use quotes""") + + parser.add_option("-w","--workdir", action="store", dest="workdir", + default="%s/%s"%(os.getenv("HOME"),"modules"), + help="""name for dedicated working dir - defaults to ~/modules ** THIS MUST NOT ** be your usual working directory""") - parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=True, - help="run in verbose mode") - parser.add_option("-q","--quiet", action="store_false", dest="verbose", - help="run in quiet (non-verbose) mode") - parser.add_option("-d","--debug", action="store_true", dest="debug", default=False, - help="debug mode - mostly more verbose") - (options, args) = parser.parse_args() - - if len(args) == 0: - if options.all_modules: - args=Command("grep -v '#' %s"%all_modules,options).output_of().split() + parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False, + help="skip safety checks, such as svn updates -- use with care") + + # default verbosity depending on function - temp + verbose_modes= ['tag','sync'] + + if mode not in verbose_modes: + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + help="run in verbose mode") else: - parser.print_help() - sys.exit(1) - Module.init_homedir(options) - for modname in args: - module=Module(modname,options) - print '========================================',module.friendly_name() - # call the method called do_ - method=Module.__dict__["do_%s"%mode] - method(module) - -# basically, we exit if anything goes wrong + parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True, + help="run in quiet (non-verbose) mode") +# parser.add_option("-d","--debug", action="store_true", dest="debug", default=False, +# help="debug mode - mostly more verbose") + (options, args) = parser.parse_args() + options.mode=mode + if not hasattr(options,'dry_run'): + options.dry_run=False + options.debug=False + + ########## release-* + if mode in Main.release_modes : + ########## changelog + if len(args) <= 1: + parser.print_help() + sys.exit(1) + Module.init_homedir(options) + for n in range(len(args)-1): + [t_new,t_old]=args[n:n+2] + Release.do_changelog (t_new,t_old,options) + else: + ########## module-* + if len(args) == 0: + if options.all_modules: + options.modules_list=default_modules_list + if options.modules_list: + args=Command("grep -v '#' %s"%options.modules_list,options).output_of().split() + else: + parser.print_help() + sys.exit(1) + Module.init_homedir(options) + for modname in args: + module=Module(modname,options) + if len(args)>1 and mode not in Main.silent_modes: + print '========================================',module.friendly_name() + # call the method called do_ + method=Module.__dict__["do_%s"%mode] + try: + method(module) + except Exception,e: + print 'Skipping failed %s: '%modname,e + +#################### if __name__ == "__main__" : try: - main() + Main().run() except KeyboardInterrupt: print '\nBye' -