first step towards supporting branches - disabled for now
[build.git] / module-tag.py
1 #!/usr/bin/python -u
2
3 subversion_id = "$Id$"
4
5 import sys, os, os.path
6 import re
7 import time
8 from glob import glob
9 from optparse import OptionParser
10
11 def prompt (question,default=True):
12     if default:
13         question += " [y]/n ? "
14     else:
15         question += " y/[n] ? "
16     try:
17         answer=raw_input(question)
18         if not answer:
19             return default
20         elif answer[0] in [ 'y','Y']:
21             return True
22         elif answer[0] in [ 'n','N']:
23             return False
24         else:
25             return prompt(question,default)
26     except KeyboardInterrupt:
27         print "Aborted"
28         return False
29     except:
30         raise
31
32 class Command:
33     def __init__ (self,command,options):
34         self.command=command
35         self.options=options
36         self.tmp="/tmp/command-%d"%os.getpid()
37
38     def run (self):
39         if self.options.verbose:
40             print '+',self.command
41             sys.stdout.flush()
42         return os.system(self.command)
43
44     def run_silent (self):
45         if self.options.verbose:
46             print '+',self.command,' .. ',
47             sys.stdout.flush()
48         retcod=os.system(self.command + " &> " + self.tmp)
49         if retcod != 0:
50             print "FAILED ! -- output quoted below "
51             os.system("cat " + self.tmp)
52             print "FAILED ! -- end of quoted output"
53         elif self.options.verbose:
54             print "OK"
55         os.unlink(self.tmp)
56         return retcod
57
58     def run_fatal(self):
59         if self.run_silent() !=0:
60             raise Exception,"Command %s failed"%self.command
61
62     # returns stdout, like bash's $(mycommand)
63     def output_of (self,with_stderr=False):
64         tmp="/tmp/status-%d"%os.getpid()
65         if self.options.debug:
66             print '+',self.command,' .. ',
67             sys.stdout.flush()
68         command=self.command
69         if with_stderr:
70             command += " &> "
71         else:
72             command += " > "
73         command += tmp
74         os.system(command)
75         result=file(tmp).read()
76         os.unlink(tmp)
77         if self.options.debug:
78             print 'Done',
79         return result
80
81 class Svnpath:
82     def __init__(self,path,options):
83         self.path=path
84         self.options=options
85
86     def url_exists (self):
87         return os.system("svn list %s &> /dev/null"%self.path) == 0
88
89     def dir_needs_revert (self):
90         command="svn status %s"%self.path
91         return len(Command(command,self.options).output_of(True)) != 0
92     # turns out it's the same implem.
93     def file_needs_commit (self):
94         command="svn status %s"%self.path
95         return len(Command(command,self.options).output_of(True)) != 0
96
97 class Module:
98
99     svn_magic_line="--This line, and those below, will be ignored--"
100     
101     redirectors=[ # ('module_name_varname','name'),
102                   ('module_version_varname','version'),
103                   ('module_taglevel_varname','taglevel'), ]
104
105     # where to store user's config
106     config_storage="CONFIG"
107     # 
108     config={}
109
110     import commands
111     configKeys=[ ('svnpath',"Enter your toplevel svnpath",
112                   "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),
113                  ("build", "Enter the name of your build module","build"),
114                  ('username',"Enter your firstname and lastname for changelogs",""),
115                  ("email","Enter your email address for changelogs",""),
116                  ]
117
118     @staticmethod
119     def prompt_config ():
120         for (key,message,default) in Module.configKeys:
121             Module.config[key]=""
122             while not Module.config[key]:
123                 Module.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default
124
125
126     # for parsing module spec name:branch
127     matcher_branch_spec=mbq=re.compile("\A(?P<name>\w+):(?P<branch>[\w\.]+)\Z")
128     matcher_rpm_define=re.compile("%define\s+(\S+)\s+(\S*)\s*")
129
130     def __init__ (self,module_spec,options):
131         # parse module spec
132         attempt=Module.matcher_branch_spec.match(module_spec)
133         if attempt:
134             self.name=attempt.group('name')
135             self.branch=attempt.group('branch')
136         else:
137             self.name=module_spec
138             self.branch=None
139
140         self.options=options
141         self.moddir="%s/%s"%(options.workdir,self.name)
142         ### xxx cut this
143         if self.branch:
144             print 'Support for branches is experimental & disabled'
145             sys.exit(1)
146
147     def edge_dir (self):
148         if not self.branch:
149             return "%s/trunk"%(self.moddir)
150         else:
151             return "%s/branches/%s"%(self.moddir,self.branch)
152
153     def run (self,command):
154         return Command(command,self.options).run()
155     def run_fatal (self,command):
156         return Command(command,self.options).run_fatal()
157     def run_prompt (self,message,command):
158         if not self.options.verbose:
159             question=message
160         else:
161             question="Want to run " + command
162         if prompt(question,True):
163             self.run(command)            
164
165     @staticmethod
166     def init_homedir (options):
167         topdir=options.workdir
168         if options.verbose:
169             print 'Checking for',topdir
170         storage="%s/%s"%(topdir,Module.config_storage)
171         # sanity check. Either the topdir exists AND we have a config/storage
172         # or topdir does not exist and we create it
173         # to avoid people use their own daily svn repo
174         if os.path.isdir(topdir) and not os.path.isfile(storage):
175             print """The directory %s exists and has no CONFIG file
176 If this is your regular working directory, please provide another one as the
177 module-* commands need a fresh working dir. Make sure that you do not use 
178 that for other purposes than tagging"""%topdir
179             sys.exit(1)
180         if not os.path.isdir (topdir):
181             print "Cannot find",topdir,"let's create it"
182             Module.prompt_config()
183             print "Checking ...",
184             Command("svn co -N %s %s"%(Module.config['svnpath'],topdir),options).run_fatal()
185             Command("svn co -N %s/%s %s/%s"%(Module.config['svnpath'],
186                                              Module.config['build'],
187                                              topdir,
188                                              Module.config['build']),options).run_fatal()
189             print "OK"
190             
191             # store config
192             f=file(storage,"w")
193             for (key,message,default) in Module.configKeys:
194                 f.write("%s=%s\n"%(key,Module.config[key]))
195             f.close()
196             if options.debug:
197                 print 'Stored',storage
198                 Command("cat %s"%storage,options).run()
199         else:
200             # read config
201             f=open(storage)
202             for line in f.readlines():
203                 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()
204                 Module.config[key]=value                
205             f.close()
206         if options.verbose:
207             print '******** Using config'
208             for (key,message,default) in Module.configKeys:
209                 print '\t',key,'=',Module.config[key]
210
211     def init_moddir (self):
212         if self.options.verbose:
213             print 'Checking for',self.moddir
214         if not os.path.isdir (self.moddir):
215             self.run_fatal("svn up -N %s"%self.moddir)
216         if not os.path.isdir (self.moddir):
217             print 'Cannot find %s - check module name'%self.moddir
218             sys.exit(1)
219
220     def init_edge_dir (self):
221         if self.options.verbose:
222             print 'Checking for',self.edge_dir()
223         # if branch, edge_dir is two steps down
224         if self.branch:
225             intermediate="%s/branches"%self.moddir
226             if not os.path.isdir (intermediate):
227                 self.run_fatal("svn up -N %s"%intermediate)
228         if not os.path.isdir (self.edge_dir()):
229             self.run_fatal("svn up -N %s"%self.edge_dir())
230
231     def revert_edge_dir (self):
232         if self.options.verbose:
233             print 'Checking whether',self.edge_dir(),'needs being reverted'
234         if Svnpath(self.edge_dir(),self.options).dir_needs_revert():
235             self.run_fatal("svn revert -R %s"%self.edge_dir())
236
237     def update_edge_dir (self):
238         if self.options.fast_checks:
239             return
240         if self.options.verbose:
241             print 'Updating',self.edge_dir()
242         self.run_fatal("svn update -N %s"%self.edge_dir())
243
244     def main_specname (self):
245         attempt="%s/%s.spec"%(self.edge_dir(),self.name)
246         if os.path.isfile (attempt):
247             return attempt
248         else:
249             try:
250                 return glob("%s/*.spec"%self.edge_dir())[0]
251             except:
252                 print 'Cannot guess specfile for module %s'%self.name
253                 sys.exit(1)
254
255     def all_specnames (self):
256         return glob("%s/*.spec"%self.edge_dir())
257
258     def parse_spec (self, specfile, varnames):
259         if self.options.debug:
260             print 'parse_spec',specfile,
261         result={}
262         f=open(specfile)
263         for line in f.readlines():
264             attempt=Module.matcher_rpm_define.match(line)
265             if attempt:
266                 (var,value)=attempt.groups()
267                 if var in varnames:
268                     result[var]=value
269         f.close()
270         if self.options.verbose:
271             print 'found',len(result),'keys'
272         if self.options.debug:
273             for (k,v) in result.iteritems():
274                 print k,'=',v
275         return result
276                 
277     # stores in self.module_name_varname the rpm variable to be used for the module's name
278     # and the list of these names in self.varnames
279     def spec_dict (self):
280         specfile=self.main_specname()
281         redirector_keys = [ varname for (varname,default) in Module.redirectors]
282         redirect_dict = self.parse_spec(specfile,redirector_keys)
283         if self.options.debug:
284             print '1st pass parsing done, redirect_dict=',redirect_dict
285         varnames=[]
286         for (varname,default) in Module.redirectors:
287             if redirect_dict.has_key(varname):
288                 setattr(self,varname,redirect_dict[varname])
289                 varnames += [redirect_dict[varname]]
290             else:
291                 setattr(self,varname,default)
292                 varnames += [ default ] 
293         self.varnames = varnames
294         result = self.parse_spec (specfile,self.varnames)
295         if self.options.debug:
296             print '2st pass parsing done, varnames=',varnames,'result=',result
297         return result
298
299     def patch_spec_var (self, patch_dict):
300         for specfile in self.all_specnames():
301             newspecfile=specfile+".new"
302             if self.options.verbose:
303                 print 'Patching',specfile,'for',patch_dict.keys()
304             spec=open (specfile)
305             new=open(newspecfile,"w")
306
307             for line in spec.readlines():
308                 attempt=Module.matcher_rpm_define.match(line)
309                 if attempt:
310                     (var,value)=attempt.groups()
311                     if var in patch_dict.keys():
312                         new.write('%%define %s %s\n'%(var,patch_dict[var]))
313                         continue
314                 new.write(line)
315             spec.close()
316             new.close()
317             os.rename(newspecfile,specfile)
318
319     def unignored_lines (self, logfile):
320         result=[]
321         exclude="Tagging module %s"%self.name
322         for logline in file(logfile).readlines():
323             if logline.strip() == Module.svn_magic_line:
324                 break
325             if logline.find(exclude) < 0:
326                 result += [ logline ]
327         return result
328
329     def insert_changelog (self, logfile, oldtag, newtag):
330         for specfile in self.all_specnames():
331             newspecfile=specfile+".new"
332             if self.options.verbose:
333                 print 'Inserting changelog from %s into %s'%(logfile,specfile)
334             spec=open (specfile)
335             new=open(newspecfile,"w")
336             for line in spec.readlines():
337                 new.write(line)
338                 if re.compile('%changelog').match(line):
339                     dateformat="* %a %b %d %Y"
340                     datepart=time.strftime(dateformat)
341                     logpart="%s <%s> - %s %s"%(Module.config['username'],
342                                                  Module.config['email'],
343                                                  oldtag,newtag)
344                     new.write(datepart+" "+logpart+"\n")
345                     for logline in self.unignored_lines(logfile):
346                         new.write("- " + logline)
347                     new.write("\n")
348             spec.close()
349             new.close()
350             os.rename(newspecfile,specfile)
351             
352     def show_dict (self, spec_dict):
353         if self.options.verbose:
354             for (k,v) in spec_dict.iteritems():
355                 print k,'=',v
356
357     def edge_url (self):
358         if not self.branch:
359             return "%s/%s/trunk"%(Module.config['svnpath'],self.name)
360         else:
361             return "%s/%s/branches/%s"%(Module.config['svnpath'],self.name,self.branch)
362
363     def tag_name (self, spec_dict):
364         try:
365             return "%s-%s-%s"%(#spec_dict[self.module_name_varname],
366                 self.name,
367                 spec_dict[self.module_version_varname],
368                 spec_dict[self.module_taglevel_varname])
369         except KeyError,err:
370             print 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)
371             sys.exit(1)
372
373     def tag_url (self, spec_dict):
374         return "%s/%s/tags/%s"%(Module.config['svnpath'],self.name,self.tag_name(spec_dict))
375
376     def check_svnpath_exists (self, url, message):
377         if self.options.fast_checks:
378             return
379         if self.options.verbose:
380             print 'Checking url (%s) %s'%(url,message),
381         ok=Svnpath(url,self.options).url_exists()
382         if ok:
383             if self.options.verbose: print 'exists - OK'
384         else:
385             if self.options.verbose: print 'KO'
386             print 'Could not find %s URL %s'%(message,url)
387             sys.exit(1)
388     def check_svnpath_not_exists (self, url, message):
389         if self.options.fast_checks:
390             return
391         if self.options.verbose:
392             print 'Checking url (%s) %s'%(url,message),
393         ok=not Svnpath(url,self.options).url_exists()
394         if ok:
395             if self.options.verbose: print 'does not exist - OK'
396         else:
397             if self.options.verbose: print 'KO'
398             print '%s URL %s already exists - exiting'%(message,url)
399             sys.exit(1)
400
401     # locate specfile, parse it, check it and show values
402     def do_version (self):
403         self.init_moddir()
404         self.init_edge_dir()
405         self.revert_edge_dir()
406         self.update_edge_dir()
407         print '==============================',self.name
408         spec_dict = self.spec_dict()
409         print 'edge url',self.edge_url()
410         print 'latest tag url',self.tag_url(spec_dict)
411         print 'main specfile:',self.main_specname()
412         print 'specfiles:',self.all_specnames()
413         for varname in self.varnames:
414             if not spec_dict.has_key(varname):
415                 print 'Could not find %%define for %s'%varname
416                 return
417             else:
418                 print varname+":",spec_dict[varname]
419
420     init_warning="""WARNING
421 The module-init function has the following limitations
422 * it does not handle changelogs
423 * it does not scan the -tags*.mk files to adopt the new tags"""
424     def do_init(self):
425         if self.options.verbose:
426             print Module.init_warning
427             if not prompt('Want to proceed anyway'):
428                 return
429
430         self.init_moddir()
431         self.init_edge_dir()
432         self.revert_edge_dir()
433         self.update_edge_dir()
434         spec_dict = self.spec_dict()
435
436         edge_url=self.edge_url()
437         tag_name=self.tag_name(spec_dict)
438         tag_url=self.tag_url(spec_dict)
439         # check the tag does not exist yet
440         self.check_svnpath_not_exists(tag_url,"new tag")
441
442         if self.options.message:
443             svnopt='--message "%s"'%self.options.message
444         else:
445             svnopt='--editor-cmd=%s'%self.options.editor
446         self.run_prompt("Create initial tag",
447                         "svn copy %s %s %s"%(svnopt,edge_url,tag_url))
448
449     def do_diff (self):
450         self.init_moddir()
451         self.init_edge_dir()
452         self.revert_edge_dir()
453         self.update_edge_dir()
454         spec_dict = self.spec_dict()
455         self.show_dict(spec_dict)
456
457         edge_url=self.edge_url()
458         tag_url=self.tag_url(spec_dict)
459         self.check_svnpath_exists(edge_url,"edge track")
460         self.check_svnpath_exists(tag_url,"latest tag")
461         diff_output = Command("svn diff %s %s"%(tag_url,edge_url),self.options).output_of()
462         if self.options.list:
463             if diff_output:
464                 print self.name
465         else:
466             if not self.options.only or diff_output:
467                 print 'x'*40,'module',self.name
468                 print 'x'*20,'<',tag_url
469                 print 'x'*20,'>',edge_url
470                 print diff_output
471
472     def patch_tags_file (self, tagsfile, oldname, newname):
473         newtagsfile=tagsfile+".new"
474         if self.options.verbose:
475             print 'Replacing %s into %s in %s'%(oldname,newname,tagsfile)
476         tags=open (tagsfile)
477         new=open(newtagsfile,"w")
478         matcher=re.compile("^(.*)%s(.*)"%oldname)
479         for line in tags.readlines():
480             if not matcher.match(line):
481                 new.write(line)
482             else:
483                 (begin,end)=matcher.match(line).groups()
484                 new.write(begin+newname+end+"\n")
485         tags.close()
486         new.close()
487         os.rename(newtagsfile,tagsfile)
488
489     def do_tag (self):
490         self.init_moddir()
491         self.init_edge_dir()
492         self.revert_edge_dir()
493         self.update_edge_dir()
494         # parse specfile
495         spec_dict = self.spec_dict()
496         self.show_dict(spec_dict)
497         
498         # side effects
499         edge_url=self.edge_url()
500         old_tag_name = self.tag_name(spec_dict)
501         old_tag_url=self.tag_url(spec_dict)
502         if (self.options.new_version):
503             # new version set on command line
504             spec_dict[self.module_version_varname] = self.options.new_version
505             spec_dict[self.module_taglevel_varname] = 0
506         else:
507             # increment taglevel
508             new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)
509             spec_dict[self.module_taglevel_varname] = new_taglevel
510
511         # sanity check
512         new_tag_name = self.tag_name(spec_dict)
513         new_tag_url=self.tag_url(spec_dict)
514         self.check_svnpath_exists (edge_url,"edge track")
515         self.check_svnpath_exists (old_tag_url,"previous tag")
516         self.check_svnpath_not_exists (new_tag_url,"new tag")
517
518         # checking for diffs
519         diff_output=Command("svn diff %s %s"%(old_tag_url,edge_url),
520                             self.options).output_of()
521         if len(diff_output) == 0:
522             if not prompt ("No difference in trunk for module %s, want to tag anyway"%self.name,False):
523                 return
524
525         # side effect in trunk's specfile
526         self.patch_spec_var(spec_dict)
527
528         # prepare changelog file 
529         # we use the standard subversion magic string (see svn_magic_line)
530         # so we can provide useful information, such as version numbers and diff
531         # in the same file
532         changelog="/tmp/%s-%d.txt"%(self.name,os.getpid())
533         file(changelog,"w").write("""Tagging module %s - %s
534
535 %s
536 Please write a changelog for this new tag in the section above
537 """%(self.name,new_tag_name,Module.svn_magic_line))
538
539         if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):
540             file(changelog,"a").write('DIFF=========\n' + diff_output)
541         
542         if self.options.debug:
543             prompt('Proceed ?')
544
545         # edit it        
546         self.run("%s %s"%(self.options.editor,changelog))
547         # insert changelog in spec
548         if self.options.changelog:
549             self.insert_changelog (changelog,old_tag_name,new_tag_name)
550
551         ## update build
552         try:
553             buildname=Module.config['build']
554         except:
555             buildname="build"
556         build = Module(buildname,self.options)
557         build.init_moddir()
558         build.init_edge_dir()
559         build.revert_edge_dir()
560         build.update_edge_dir()
561         
562         for tagsfile in glob(build.edge_dir()+"/*-tags*.mk"):
563             if prompt("Want to adopt new tag in %s"%tagsfile):
564                 self.patch_tags_file(tagsfile,old_tag_name,new_tag_name)
565
566         paths=""
567         paths += self.edge_dir() + " "
568         paths += build.edge_dir() + " "
569         self.run_prompt("Check","svn diff " + paths)
570         self.run_prompt("Commit","svn commit --file %s %s"%(changelog,paths))
571         self.run_prompt("Create tag","svn copy --file %s %s %s"%(changelog,edge_url,new_tag_url))
572
573         if self.options.debug:
574             print 'Preserving',changelog
575         else:
576             os.unlink(changelog)
577             
578 usage="""Usage: %prog options module_desc [ .. module_desc ]
579 Purpose:
580   manage subversion tags and specfile
581   requires the specfile to define *version* and *taglevel*
582   OR alternatively 
583   redirection variables module_version_varname / module_taglevel_varname
584 Trunk:
585   by default, the trunk of modules is taken into account
586   in this case, just mention the module name as <module_desc>
587 Branches:
588   if you wish to work on a branch rather than on the trunk, 
589   you can use the following syntax for <module_desc>
590   Mom:2.1
591       works on Mom/branches/2.1 
592 """
593 # unsupported yet
594 #"""
595 #  branch:Mom
596 #      the branch_id is deduced from the current *version* in the trunk's specfile
597 #      e.g. if Mom/trunk/Mom.spec specifies %define version 2.3, then this script
598 #      would use Mom/branches/2.2
599 #      if if stated %define version 3.0, then the script fails
600 #"""
601
602 functions={ 
603     'diff' : "show difference between trunk and latest tag",
604     'tag'  : """increment taglevel in specfile, insert changelog in specfile,
605                 create new tag and and adopt it in build/*-tags*.mk""",
606     'init' : "create initial tag",
607     'version' : "only check specfile and print out details"}
608
609 def main():
610
611     if sys.argv[0].find("diff") >= 0:
612         mode = "diff"
613     elif sys.argv[0].find("tag") >= 0:
614         mode = "tag"
615     elif sys.argv[0].find("init") >= 0:
616         mode = "init"
617     elif sys.argv[0].find("version") >= 0:
618         mode = "version"
619     else:
620         print "Unsupported command",sys.argv[0]
621         sys.exit(1)
622
623     global usage
624     usage += "module-%s.py : %s"%(mode,functions[mode])
625     all_modules=os.path.dirname(sys.argv[0])+"/modules.list"
626
627     parser=OptionParser(usage=usage,version=subversion_id)
628     parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False,
629                       help="run on all modules as found in %s"%all_modules)
630     parser.add_option("-f","--fast-checks",action="store_true",dest="fast_checks",default=False,
631                       help="skip safety checks, such as svn updates -- use with care")
632     if mode == "tag" :
633         parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,
634                           help="set new version and reset taglevel to 0")
635     if mode == "tag" :
636         parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,
637                           help="do not update changelog section in specfile when tagging")
638     if mode == "tag" or mode == "init" :
639         parser.add_option("-e","--editor", action="store", dest="editor", default="emacs",
640                           help="specify editor")
641     if mode == "init" :
642         parser.add_option("-m","--message", action="store", dest="message", default=None,
643                           help="specify log message")
644     if mode == "diff" :
645         parser.add_option("-o","--only", action="store_true", dest="only", default=False,
646                           help="report diff only for modules that exhibit differences")
647     if mode == "diff" :
648         parser.add_option("-l","--list", action="store_true", dest="list", default=False,
649                           help="just list modules that exhibit differences")
650     parser.add_option("-w","--workdir", action="store", dest="workdir", 
651                       default="%s/%s"%(os.getenv("HOME"),"modules"),
652                       help="""name for dedicated working dir - defaults to ~/modules
653 ** THIS MUST NOT ** be your usual working directory""")
654     parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=True, 
655                       help="run in verbose mode")
656     parser.add_option("-q","--quiet", action="store_false", dest="verbose", 
657                       help="run in quiet (non-verbose) mode")
658     parser.add_option("-d","--debug", action="store_true", dest="debug", default=False, 
659                       help="debug mode - mostly more verbose")
660     (options, args) = parser.parse_args()
661
662     if len(args) == 0:
663         if options.all_modules:
664             args=Command("grep -v '#' %s"%all_modules,options).output_of().split()
665         else:
666             parser.print_help()
667             sys.exit(1)
668     Module.init_homedir(options)
669     for modname in args:
670         module=Module(modname,options)
671         if sys.argv[0].find("diff") >= 0:
672             module.do_diff()
673         elif sys.argv[0].find("tag") >= 0:
674             module.do_tag()
675         elif sys.argv[0].find("init") >= 0:
676             module.do_init()
677         elif sys.argv[0].find("version") >= 0:
678             module.do_version()
679         else:
680             print "Unsupported command",sys.argv[0]
681             parser.print_help()
682             sys.exit(1)
683
684 # basically, we exit if anything goes wrong
685 if __name__ == "__main__" :
686     main()