git.onelab.eu Git - build.git/blob

1 #!/usr/bin/python -u

2

3 subversion_id = "$Id$"

4

5 import sys, os, os.path

6 import re

7 import time

8 from glob import glob

9 from optparse import OptionParser

10

11 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase

12 def prompt (question,default=True,other_choices=[],allow_outside=False):

13 if not isinstance (other_choices,list):

14 other_choices = [ other_choices ]

15 chars = [ c for (c,rest) in other_choices ]

16

17 choices = []

18 if 'y' not in chars:

19 if default is True: choices.append('[y]')

20 else : choices.append('y')

21 if 'n' not in chars:

22 if default is False: choices.append('[n]')

23 else : choices.append('n')

24

25 for (char,choice) in other_choices:

26 if default == char:

27 choices.append("["+char+"]"+choice)

28 else:

29 choices.append("<"+char+">"+choice)

30 try:

31 answer=raw_input(question + " " + "/".join(choices) + " ? ")

32 if not answer:

33 return default

34 answer=answer[0].lower()

35 if answer == 'y':

36 if 'y' in chars: return 'y'

37 else: return True

38 elif answer == 'n':

39 if 'n' in chars: return 'n'

40 else: return False

41 elif other_choices:

42 for (char,choice) in other_choices:

43 if answer == char:

44 return char

45 if allow_outside:

46 return answer

47 return prompt(question,default,other_choices)

48 except:

49 raise

50

51 def default_editor():

52 try:

53 editor = os.environ['EDITOR']

54 except:

55 editor = "emacs"

56 return editor

57

58 ### fold long lines

59 fold_length=132

60

61 def print_fold (line):

62 while len(line) >= fold_length:

63 print line[:fold_length],'\\'

64 line=line[fold_length:]

65 print line

66

67 class Command:

68 def __init__ (self,command,options):

69 self.command=command

70 self.options=options

71 self.tmp="/tmp/command-%d"%os.getpid()

72

73 def run (self):

74 if self.options.dry_run:

75 print 'dry_run',self.command

76 return 0

77 if self.options.verbose and self.options.mode not in Main.silent_modes:

78 print '+',self.command

79 sys.stdout.flush()

80 return os.system(self.command)

81

82 def run_silent (self):

83 if self.options.dry_run:

84 print 'dry_run',self.command

85 return 0

86 if self.options.verbose:

87 print '+',self.command,' .. ',

88 sys.stdout.flush()

89 retcod=os.system(self.command + " &> " + self.tmp)

90 if retcod != 0:

91 print "FAILED ! -- out+err below (command was %s)"%self.command

92 os.system("cat " + self.tmp)

93 print "FAILED ! -- end of quoted output"

94 elif self.options.verbose:

95 print "OK"

96 os.unlink(self.tmp)

97 return retcod

98

99 def run_fatal(self):

100 if self.run_silent() !=0:

101 raise Exception,"Command %s failed"%self.command

102

103 # returns stdout, like bash's $(mycommand)

104 def output_of (self,with_stderr=False):

105 if self.options.dry_run:

106 print 'dry_run',self.command

107 return 'dry_run output'

108 tmp="/tmp/status-%d"%os.getpid()

109 if self.options.debug:

110 print '+',self.command,' .. ',

111 sys.stdout.flush()

112 command=self.command

113 if with_stderr:

114 command += " &> "

115 else:

116 command += " > "

117 command += tmp

118 os.system(command)

119 result=file(tmp).read()

120 os.unlink(tmp)

121 if self.options.debug:

122 print 'Done',

123 return result

124

125 class Svnpath:

126 def __init__(self,path,options):

127 self.path=path

128 self.options=options

129

130 def url_exists (self):

131 return os.system("svn list %s &> /dev/null"%self.path) == 0

132

133 def dir_needs_revert (self):

134 command="svn status %s"%self.path

135 return len(Command(command,self.options).output_of(True)) != 0

136 # turns out it's the same implem.

137 def file_needs_commit (self):

138 command="svn status %s"%self.path

139 return len(Command(command,self.options).output_of(True)) != 0

140

141 # support for tagged module is minimal, and is for the Build class only

142 class Module:

143

144 svn_magic_line="--This line, and those below, will be ignored--"

145

146 redirectors=[ # ('module_name_varname','name'),

147 ('module_version_varname','version'),

148 ('module_taglevel_varname','taglevel'), ]

149

150 # where to store user's config

151 config_storage="CONFIG"

152 #

153 config={}

154

155 import commands

156 configKeys=[ ('svnpath',"Enter your toplevel svnpath",

157 "svn+ssh://%s@svn.planet-lab.org/svn/"%commands.getoutput("id -un")),

158 ("build", "Enter the name of your build module","build"),

159 ('username',"Enter your firstname and lastname for changelogs",""),

160 ("email","Enter your email address for changelogs",""),

161 ]

162

163 @staticmethod

164 def prompt_config ():

165 for (key,message,default) in Module.configKeys:

166 Module.config[key]=""

167 while not Module.config[key]:

168 Module.config[key]=raw_input("%s [%s] : "%(message,default)).strip() or default

169

170

171 # for parsing module spec name:branch

172 matcher_branch_spec=re.compile("\A(?P<name>[\w\.-]+):(?P<branch>[\w\.-]+)\Z")

173 # special form for tagged module - for Build

174 matcher_tag_spec=re.compile("\A(?P<name>[\w-]+)@(?P<tagname>[\w\.-]+)\Z")

175 # parsing specfiles

176 matcher_rpm_define=re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")

177

178 def __init__ (self,module_spec,options):

179 # parse module spec

180 attempt=Module.matcher_branch_spec.match(module_spec)

181 if attempt:

182 self.name=attempt.group('name')

183 self.branch=attempt.group('branch')

184 else:

185 attempt=Module.matcher_tag_spec.match(module_spec)

186 if attempt:

187 self.name=attempt.group('name')

188 self.tagname=attempt.group('tagname')

189 else:

190 self.name=module_spec

191

192 self.options=options

193 self.module_dir="%s/%s"%(options.workdir,self.name)

194

195 def friendly_name (self):

196 if hasattr(self,'branch'):

197 return "%s:%s"%(self.name,self.branch)

198 elif hasattr(self,'tagname'):

199 return "%s@%s"%(self.name,self.tagname)

200 else:

201 return self.name

202

203 def edge_dir (self):

204 if hasattr(self,'branch'):

205 return "%s/branches/%s"%(self.module_dir,self.branch)

206 elif hasattr(self,'tagname'):

207 return "%s/tags/%s"%(self.module_dir,self.tagname)

208 else:

209 return "%s/trunk"%(self.module_dir)

210

211 def tags_dir (self):

212 return "%s/tags"%(self.module_dir)

213

214 def run (self,command):

215 return Command(command,self.options).run()

216 def run_fatal (self,command):

217 return Command(command,self.options).run_fatal()

218 def run_prompt (self,message,command):

219 if not self.options.verbose:

220 while True:

221 choice=prompt(message,True,('s','how'))

222 if choice is True:

223 self.run(command)

224 return

225 elif choice is False:

226 return

227 else:

228 print 'About to run:',command

229 else:

230 question=message+" - want to run " + command

231 if prompt(question,True):

232 self.run(command)

233

234 @staticmethod

235 def init_homedir (options):

236 topdir=options.workdir

237 if options.verbose and options.mode not in Main.silent_modes:

238 print 'Checking for',topdir

239 storage="%s/%s"%(topdir,Module.config_storage)

240 # sanity check. Either the topdir exists AND we have a config/storage

241 # or topdir does not exist and we create it

242 # to avoid people use their own daily svn repo

243 if os.path.isdir(topdir) and not os.path.isfile(storage):

244 print """The directory %s exists and has no CONFIG file

245 If this is your regular working directory, please provide another one as the

246 module-* commands need a fresh working dir. Make sure that you do not use

247 that for other purposes than tagging"""%topdir

248 sys.exit(1)

249 if not os.path.isdir (topdir):

250 print "Cannot find",topdir,"let's create it"

251 Module.prompt_config()

252 print "Checking ...",

253 Command("svn co -N %s %s"%(Module.config['svnpath'],topdir),options).run_fatal()

254 Command("svn co -N %s/%s %s/%s"%(Module.config['svnpath'],

255 Module.config['build'],

256 topdir,

257 Module.config['build']),options).run_fatal()

258 print "OK"

259

260 # store config

261 f=file(storage,"w")

262 for (key,message,default) in Module.configKeys:

263 f.write("%s=%s\n"%(key,Module.config[key]))

264 f.close()

265 if options.debug:

266 print 'Stored',storage

267 Command("cat %s"%storage,options).run()

268 else:

269 # read config

270 f=open(storage)

271 for line in f.readlines():

272 (key,value)=re.compile("^(.+)=(.+)$").match(line).groups()

273 Module.config[key]=value

274 f.close()

275 if options.verbose and options.mode not in Main.silent_modes:

276 print '******** Using config'

277 for (key,message,default) in Module.configKeys:

278 print '\t',key,'=',Module.config[key]

279

280 def init_module_dir (self):

281 if self.options.verbose:

282 print 'Checking for',self.module_dir

283 if not os.path.isdir (self.module_dir):

284 self.run_fatal("svn update -N %s"%self.module_dir)

285 if not os.path.isdir (self.module_dir):

286 raise Exception, 'Cannot find %s - check module name'%self.module_dir

287

288 def init_subdir (self,fullpath):

289 if self.options.verbose:

290 print 'Checking for',fullpath

291 if not os.path.isdir (fullpath):

292 self.run_fatal("svn update -N %s"%fullpath)

293

294 def revert_subdir (self,fullpath):

295 if self.options.fast_checks:

296 if self.options.verbose: print 'Skipping revert of %s'%fullpath

297 return

298 if self.options.verbose:

299 print 'Checking whether',fullpath,'needs being reverted'

300 if Svnpath(fullpath,self.options).dir_needs_revert():

301 self.run_fatal("svn revert -R %s"%fullpath)

302

303 def update_subdir (self,fullpath):

304 if self.options.fast_checks:

305 if self.options.verbose: print 'Skipping update of %s'%fullpath

306 return

307 if self.options.verbose:

308 print 'Updating',fullpath

309 self.run_fatal("svn update -N %s"%fullpath)

310

311 def init_edge_dir (self):

312 # if branch, edge_dir is two steps down

313 if hasattr(self,'branch'):

314 self.init_subdir("%s/branches"%self.module_dir)

315 elif hasattr(self,'tagname'):

316 self.init_subdir("%s/tags"%self.module_dir)

317 self.init_subdir(self.edge_dir())

318

319 def revert_edge_dir (self):

320 self.revert_subdir(self.edge_dir())

321

322 def update_edge_dir (self):

323 self.update_subdir(self.edge_dir())

324

325 def main_specname (self):

326 attempt="%s/%s.spec"%(self.edge_dir(),self.name)

327 if os.path.isfile (attempt):

328 return attempt

329 else:

330 pattern="%s/*.spec"%self.edge_dir()

331 try:

332 return glob(pattern)[0]

333 except:

334 raise Exception, 'Cannot guess specfile for module %s -- pattern was %s'%(self.name,pattern)

335

336 def all_specnames (self):

337 return glob("%s/*.spec"%self.edge_dir())

338

339 def parse_spec (self, specfile, varnames):

340 if self.options.verbose:

341 print 'Parsing',specfile,

342 for var in varnames:

343 print "[%s]"%var,

344 print ""

345 result={}

346 f=open(specfile)

347 for line in f.readlines():

348 attempt=Module.matcher_rpm_define.match(line)

349 if attempt:

350 (define,var,value)=attempt.groups()

351 if var in varnames:

352 result[var]=value

353 f.close()

354 if self.options.debug:

355 print 'found',len(result),'keys'

356 for (k,v) in result.iteritems():

357 print k,'=',v

358 return result

359

360 # stores in self.module_name_varname the rpm variable to be used for the module's name

361 # and the list of these names in self.varnames

362 def spec_dict (self):

363 specfile=self.main_specname()

364 redirector_keys = [ varname for (varname,default) in Module.redirectors]

365 redirect_dict = self.parse_spec(specfile,redirector_keys)

366 if self.options.debug:

367 print '1st pass parsing done, redirect_dict=',redirect_dict

368 varnames=[]

369 for (varname,default) in Module.redirectors:

370 if redirect_dict.has_key(varname):

371 setattr(self,varname,redirect_dict[varname])

372 varnames += [redirect_dict[varname]]

373 else:

374 setattr(self,varname,default)

375 varnames += [ default ]

376 self.varnames = varnames

377 result = self.parse_spec (specfile,self.varnames)

378 if self.options.debug:

379 print '2st pass parsing done, varnames=',varnames,'result=',result

380 return result

381

382 def patch_spec_var (self, patch_dict,define_missing=False):

383 for specfile in self.all_specnames():

384 # record the keys that were changed

385 changed = dict ( [ (x,False) for x in patch_dict.keys() ] )

386 newspecfile=specfile+".new"

387 if self.options.verbose:

388 print 'Patching',specfile,'for',patch_dict.keys()

389 spec=open (specfile)

390 new=open(newspecfile,"w")

391

392 for line in spec.readlines():

393 attempt=Module.matcher_rpm_define.match(line)

394 if attempt:

395 (define,var,value)=attempt.groups()

396 if var in patch_dict.keys():

397 if self.options.debug:

398 print 'rewriting %s as %s'%(var,patch_dict[var])

399 new.write('%%%s %s %s\n'%(define,var,patch_dict[var]))

400 changed[var]=True

401 continue

402 new.write(line)

403 if define_missing:

404 for (key,was_changed) in changed.iteritems():

405 if not was_changed:

406 if self.options.debug:

407 print 'rewriting missing %s as %s'%(key,patch_dict[key])

408 new.write('\n%%define %s %s\n'%(key,patch_dict[key]))

409 spec.close()

410 new.close()

411 os.rename(newspecfile,specfile)

412

413 def unignored_lines (self, logfile):

414 result=[]

415 exclude="Tagging module %s"%self.name

416 white_line_matcher = re.compile("\A\s*\Z")

417 for logline in file(logfile).readlines():

418 if logline.strip() == Module.svn_magic_line:

419 break

420 if logline.find(exclude) >= 0:

421 continue

422 elif white_line_matcher.match(logline):

423 continue

424 else:

425 result.append(logline.strip()+'\n')

426 return result

427

428 def insert_changelog (self, logfile, oldtag, newtag):

429 for specfile in self.all_specnames():

430 newspecfile=specfile+".new"

431 if self.options.verbose:

432 print 'Inserting changelog from %s into %s'%(logfile,specfile)

433 spec=open (specfile)

434 new=open(newspecfile,"w")

435 for line in spec.readlines():

436 new.write(line)

437 if re.compile('%changelog').match(line):

438 dateformat="* %a %b %d %Y"

439 datepart=time.strftime(dateformat)

440 logpart="%s <%s> - %s"%(Module.config['username'],

441 Module.config['email'],

442 newtag)

443 new.write(datepart+" "+logpart+"\n")

444 for logline in self.unignored_lines(logfile):

445 new.write("- " + logline)

446 new.write("\n")

447 spec.close()

448 new.close()

449 os.rename(newspecfile,specfile)

450

451 def show_dict (self, spec_dict):

452 if self.options.verbose:

453 for (k,v) in spec_dict.iteritems():

454 print k,'=',v

455

456 def mod_url (self):

457 return "%s/%s"%(Module.config['svnpath'],self.name)

458

459 def edge_url (self):

460 if hasattr(self,'branch'):

461 return "%s/branches/%s"%(self.mod_url(),self.branch)

462 elif hasattr(self,'tagname'):

463 return "%s/tags/%s"%(self.mod_url(),self.tagname)

464 else:

465 return "%s/trunk"%(self.mod_url())

466

467 def tag_name (self, spec_dict):

468 try:

469 return "%s-%s-%s"%(#spec_dict[self.module_name_varname],

470 self.name,

471 spec_dict[self.module_version_varname],

472 spec_dict[self.module_taglevel_varname])

473 except KeyError,err:

474 raise Exception, 'Something is wrong with module %s, cannot determine %s - exiting'%(self.name,err)

475

476 def tag_url (self, spec_dict):

477 return "%s/tags/%s"%(self.mod_url(),self.tag_name(spec_dict))

478

479 def check_svnpath_exists (self, url, message):

480 if self.options.fast_checks:

481 return

482 if self.options.verbose:

483 print 'Checking url (%s) %s'%(url,message),

484 ok=Svnpath(url,self.options).url_exists()

485 if ok:

486 if self.options.verbose: print 'exists - OK'

487 else:

488 if self.options.verbose: print 'KO'

489 raise Exception, 'Could not find %s URL %s'%(message,url)

490

491 def check_svnpath_not_exists (self, url, message):

492 if self.options.fast_checks:

493 return

494 if self.options.verbose:

495 print 'Checking url (%s) %s'%(url,message),

496 ok=not Svnpath(url,self.options).url_exists()

497 if ok:

498 if self.options.verbose: print 'does not exist - OK'

499 else:

500 if self.options.verbose: print 'KO'

501 raise Exception, '%s URL %s already exists - exiting'%(message,url)

502

503 # locate specfile, parse it, check it and show values

504

505 ##############################

506 def do_version (self):

507 self.init_module_dir()

508 self.init_edge_dir()

509 self.revert_edge_dir()

510 self.update_edge_dir()

511 spec_dict = self.spec_dict()

512 for varname in self.varnames:

513 if not spec_dict.has_key(varname):

514 print 'Could not find %%define for %s'%varname

515 return

516 else:

517 print "%-16s %s"%(varname,spec_dict[varname])

518 if self.options.show_urls:

519 print "%-16s %s"%('edge url',self.edge_url())

520 print "%-16s %s"%('latest tag url',self.tag_url(spec_dict))

521 if self.options.verbose:

522 print "%-16s %s"%('main specfile:',self.main_specname())

523 print "%-16s %s"%('specfiles:',self.all_specnames())

524

525 ##############################

526 def do_list (self):

527 # print 'verbose',self.options.verbose

528 # print 'list_tags',self.options.list_tags

529 # print 'list_branches',self.options.list_branches

530 # print 'all_modules',self.options.all_modules

531

532 (verbose,branches,pattern,exact) = (self.options.verbose,self.options.list_branches,

533 self.options.list_pattern,self.options.list_exact)

534

535 extra_command=""

536 extra_message=""

537 if hasattr(self,'branch'):

538 pattern=self.branch

539 if pattern or exact:

540 if exact:

541 if verbose: grep="%s/$"%exact

542 else: grep="^%s$"%exact

543 else:

544 grep=pattern

545 extra_command=" | grep %s"%grep

546 extra_message=" matching %s"%grep

547

548 if not branches:

549 message="==================== tags for %s"%self.friendly_name()

550 command="svn list "

551 if verbose: command+="--verbose "

552 command += "%s/tags"%self.mod_url()

553 command += extra_command

554 message += extra_message

555 if verbose: print message

556 self.run(command)

557

558 else:

559 message="==================== branches for %s"%self.friendly_name()

560 command="svn list "

561 if verbose: command+="--verbose "

562 command += "%s/branches"%self.mod_url()

563 command += extra_command

564 message += extra_message

565 if verbose: print message

566 self.run(command)

567

568 ##############################

569 sync_warning="""*** WARNING

570 The module-sync function has the following limitations

571 * it does not handle changelogs

572 * it does not scan the -tags*.mk files to adopt the new tags"""

573

574 def do_sync(self):

575 if self.options.verbose:

576 print Module.sync_warning

577 if not prompt('Want to proceed anyway'):

578 return

579

580 self.init_module_dir()

581 self.init_edge_dir()

582 self.revert_edge_dir()

583 self.update_edge_dir()

584 spec_dict = self.spec_dict()

585

586 edge_url=self.edge_url()

587 tag_name=self.tag_name(spec_dict)

588 tag_url=self.tag_url(spec_dict)

589 # check the tag does not exist yet

590 self.check_svnpath_not_exists(tag_url,"new tag")

591

592 if self.options.message:

593 svnopt='--message "%s"'%self.options.message

594 else:

595 svnopt='--editor-cmd=%s'%self.options.editor

596 self.run_prompt("Create initial tag",

597 "svn copy %s %s %s"%(svnopt,edge_url,tag_url))

598

599 ##############################

600 def do_diff (self,compute_only=False):

601 self.init_module_dir()

602 self.init_edge_dir()

603 self.revert_edge_dir()

604 self.update_edge_dir()

605 spec_dict = self.spec_dict()

606 self.show_dict(spec_dict)

607

608 edge_url=self.edge_url()

609 tag_url=self.tag_url(spec_dict)

610 self.check_svnpath_exists(edge_url,"edge track")

611 self.check_svnpath_exists(tag_url,"latest tag")

612 command="svn diff %s %s"%(tag_url,edge_url)

613 if compute_only:

614 if self.options.verbose:

615 print 'Getting diff with %s'%command

616 diff_output = Command(command,self.options).output_of()

617 # if used as a utility

618 if compute_only:

619 return (spec_dict,edge_url,tag_url,diff_output)

620 # otherwise print the result

621 if self.options.list:

622 if diff_output:

623 print self.name

624 else:

625 if not self.options.only or diff_output:

626 print 'x'*30,'module',self.friendly_name()

627 print 'x'*20,'<',tag_url

628 print 'x'*20,'>',edge_url

629 print diff_output

630

631 ##############################

632 # using fine_grain means replacing only those instances that currently refer to this tag

633 # otherwise, <module>-SVNPATH is replaced unconditionnally

634 def patch_tags_file (self, tagsfile, oldname, newname,fine_grain=True):

635 newtagsfile=tagsfile+".new"

636 tags=open (tagsfile)

637 new=open(newtagsfile,"w")

638

639 matches=0

640 # fine-grain : replace those lines that refer to oldname

641 if fine_grain:

642 if self.options.verbose:

643 print 'Replacing %s into %s\n\tin %s .. '%(oldname,newname,tagsfile),

644 matcher=re.compile("^(.*)%s(.*)"%oldname)

645 for line in tags.readlines():

646 if not matcher.match(line):

647 new.write(line)

648 else:

649 (begin,end)=matcher.match(line).groups()

650 new.write(begin+newname+end+"\n")

651 matches += 1

652 # brute-force : change uncommented lines that define <module>-SVNPATH

653 else:

654 if self.options.verbose:

655 print 'Setting %s-SVNPATH for using %s\n\tin %s .. '%(self.name,newname,tagsfile),

656 pattern="\A\s*%s-SVNPATH\s*(=|:=)\s*(?P<url_main>[^\s]+)/%s/[^\s]+"\

657 %(self.name,self.name)

658 matcher_module=re.compile(pattern)

659 for line in tags.readlines():

660 attempt=matcher_module.match(line)

661 if attempt:

662 svnpath="%s-SVNPATH"%self.name

663 replacement = "%-32s:= %s/%s/tags/%s\n"%(svnpath,attempt.group('url_main'),self.name,newname)

664 new.write(replacement)

665 matches += 1

666 else:

667 new.write(line)

668 tags.close()

669 new.close()

670 os.rename(newtagsfile,tagsfile)

671 if self.options.verbose: print "%d changes"%matches

672 return matches

673

674 def do_tag (self):

675 self.init_module_dir()

676 self.init_edge_dir()

677 self.revert_edge_dir()

678 self.update_edge_dir()

679 # parse specfile

680 spec_dict = self.spec_dict()

681 self.show_dict(spec_dict)

682

683 # side effects

684 edge_url=self.edge_url()

685 old_tag_name = self.tag_name(spec_dict)

686 old_tag_url=self.tag_url(spec_dict)

687 if (self.options.new_version):

688 # new version set on command line

689 spec_dict[self.module_version_varname] = self.options.new_version

690 spec_dict[self.module_taglevel_varname] = 0

691 else:

692 # increment taglevel

693 new_taglevel = str ( int (spec_dict[self.module_taglevel_varname]) + 1)

694 spec_dict[self.module_taglevel_varname] = new_taglevel

695

696 # sanity check

697 new_tag_name = self.tag_name(spec_dict)

698 new_tag_url=self.tag_url(spec_dict)

699 self.check_svnpath_exists (edge_url,"edge track")

700 self.check_svnpath_exists (old_tag_url,"previous tag")

701 self.check_svnpath_not_exists (new_tag_url,"new tag")

702

703 # checking for diffs

704 diff_output=Command("svn diff %s %s"%(old_tag_url,edge_url),

705 self.options).output_of()

706 if len(diff_output) == 0:

707 if not prompt ("No pending difference in module %s, want to tag anyway"%self.name,False):

708 return

709

710 # side effect in trunk's specfile

711 self.patch_spec_var(spec_dict)

712

713 # prepare changelog file

714 # we use the standard subversion magic string (see svn_magic_line)

715 # so we can provide useful information, such as version numbers and diff

716 # in the same file

717 changelog="/tmp/%s-%d.txt"%(self.name,os.getpid())

718 file(changelog,"w").write("""Tagging module %s - %s

719

720 %s

721 Please write a changelog for this new tag in the section above

722 """%(self.name,new_tag_name,Module.svn_magic_line))

723

724 if not self.options.verbose or prompt('Want to see diffs while writing changelog',True):

725 file(changelog,"a").write('DIFF=========\n' + diff_output)

726

727 if self.options.debug:

728 prompt('Proceed ?')

729

730 # edit it

731 self.run("%s %s"%(self.options.editor,changelog))

732 # insert changelog in spec

733 if self.options.changelog:

734 self.insert_changelog (changelog,old_tag_name,new_tag_name)

735

736 ## update build

737 try:

738 buildname=Module.config['build']

739 except:

740 buildname="build"

741 if self.options.build_branch:

742 buildname+=":"+self.options.build_branch

743 build = Module(buildname,self.options)

744 build.init_module_dir()

745 build.init_edge_dir()

746 build.revert_edge_dir()

747 build.update_edge_dir()

748

749 tagsfiles=glob(build.edge_dir()+"/*-tags*.mk")

750 tagsdict=dict( [ (x,'todo') for x in tagsfiles ] )

751 default_answer = 'y'

752 while True:

753 for (tagsfile,status) in tagsdict.iteritems():

754 basename=os.path.basename(tagsfile)

755 print ".................... Dealing with %s"%basename

756 while tagsdict[tagsfile] == 'todo' :

757 choice = prompt ("insert %s in %s "%(new_tag_name,basename),default_answer,

758 [ ('y','es'), ('n', 'ext'), ('f','orce'),

759 ('d','iff'), ('r','evert'), ('h','elp') ] ,

760 allow_outside=True)

761 if choice == 'y':

762 self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=True)

763 elif choice == 'n':

764 print 'Done with %s'%os.path.basename(tagsfile)

765 tagsdict[tagsfile]='done'

766 elif choice == 'f':

767 self.patch_tags_file(tagsfile,old_tag_name,new_tag_name,fine_grain=False)

768 elif choice == 'd':

769 self.run("svn diff %s"%tagsfile)

770 elif choice == 'r':

771 self.run("svn revert %s"%tagsfile)

772 else:

773 name=self.name

774 print """y: change %(name)s-SVNPATH only if it currently refers to %(old_tag_name)s

775 f: unconditionnally change any line setting %(name)s-SVNPATH to using %(new_tag_name)s

776 d: show current diff for this tag file

777 r: revert that tag file

778 n: move to next file"""%locals()

779

780 if prompt("Want to review changes on tags files",False):

781 tagsdict = dict ( [ (x, 'todo') for tagsfile in tagsfiles ] )

782 default_answer='d'

783 else:

784 break

785

786 paths=""

787 paths += self.edge_dir() + " "

788 paths += build.edge_dir() + " "

789 self.run_prompt("Review module and build","svn diff " + paths)

790 self.run_prompt("Commit module and build","svn commit --file %s %s"%(changelog,paths))

791 self.run_prompt("Create tag","svn copy --file %s %s %s"%(changelog,edge_url,new_tag_url))

792

793 if self.options.debug:

794 print 'Preserving',changelog

795 else:

796 os.unlink(changelog)

797

798 ##############################

799 def do_branch (self):

800

801 # save self.branch if any, as a hint for the new branch

802 # do this before anything else and restore .branch to None,

803 # as this is part of the class's logic

804 new_trunk_name=None

805 if hasattr(self,'branch'):

806 new_trunk_name=self.branch

807 del self.branch

808 elif self.options.new_version:

809 new_trunk_name = self.options.new_version

810

811 # compute diff - a way to initialize the whole stuff

812 # do_diff already does edge_dir initialization

813 # and it checks that edge_url and tag_url exist as well

814 (spec_dict,edge_url,tag_url,diff_listing) = self.do_diff(compute_only=True)

815

816 # the version name in the trunk becomes the new branch name

817 branch_name = spec_dict[self.module_version_varname]

818

819 # figure new branch name (the one for the trunk) if not provided on the command line

820 if not new_trunk_name:

821 # heuristic is to assume 'version' is a dot-separated name

822 # we isolate the rightmost part and try incrementing it by 1

823 version=spec_dict[self.module_version_varname]

824 try:

825 m=re.compile("\A(?P<leftpart>.+)\.(?P<rightmost>[^\.]+)\Z")

826 (leftpart,rightmost)=m.match(version).groups()

827 incremented = int(rightmost)+1

828 new_trunk_name="%s.%d"%(leftpart,incremented)

829 except:

830 raise Exception, 'Cannot figure next branch name from %s - exiting'%version

831

832 # record starting point tagname

833 latest_tag_name = self.tag_name(spec_dict)

834

835 print "**********"

836 print "Using starting point %s (%s)"%(tag_url,latest_tag_name)

837 print "Creating branch %s & moving trunk to %s"%(branch_name,new_trunk_name)

838 print "**********"

839

840 # print warning if pending diffs

841 if diff_listing:

842 print """*** WARNING : Module %s has pending diffs on its trunk

843 It is safe to proceed, but please note that branch %s

844 will be based on latest tag %s and *not* on the current trunk"""%(self.name,branch_name,latest_tag_name)

845 while True:

846 answer = prompt ('Are you sure you want to proceed with branching',True,('d','iff'))

847 if answer is True:

848 break

849 elif answer is False:

850 raise Exception,"User quit"

851 elif answer == 'd':

852 print '<<<< %s'%tag_url

853 print '>>>> %s'%edge_url

854 print diff_listing

855

856 branch_url = "%s/%s/branches/%s"%(Module.config['svnpath'],self.name,branch_name)

857 self.check_svnpath_not_exists (branch_url,"new branch")

858

859 # patching trunk

860 spec_dict[self.module_version_varname]=new_trunk_name

861 spec_dict[self.module_taglevel_varname]='0'

862 # remember this in the trunk for easy location of the current branch

863 spec_dict['module_current_branch']=branch_name

864 self.patch_spec_var(spec_dict,True)

865

866 # create commit log file

867 tmp="/tmp/branching-%d"%os.getpid()

868 f=open(tmp,"w")

869 f.write("Branch %s for module %s created (as new trunk) from tag %s\n"%(new_trunk_name,self.name,latest_tag_name))

870 f.close()

871

872 # we're done, let's commit the stuff

873 command="svn diff %s"%self.edge_dir()

874 self.run_prompt("Review changes in trunk",command)

875 command="svn copy --file %s %s %s"%(tmp,self.edge_url(),branch_url)

876 self.run_prompt("Create branch",command)

877 command="svn commit --file %s %s"%(tmp,self.edge_dir())

878 self.run_prompt("Commit trunk",command)

879 new_tag_url=self.tag_url(spec_dict)

880 command="svn copy --file %s %s %s"%(tmp,self.edge_url(),new_tag_url)

881 self.run_prompt("Create initial tag in trunk",command)

882 os.unlink(tmp)

883

884 ##############################

885 class Package:

886

887 def __init__(self, package, module, svnpath, spec):

888 self.package=package

889 self.module=module

890 self.svnpath=svnpath

891 self.spec=spec

892 self.specpath="%s/%s"%(svnpath,spec)

893 self.basename=os.path.basename(svnpath)

894

895 # returns a http URL to the trac path where full diff can be viewed (between self and pkg)

896 # typically http://svn.planet-lab.org/changeset?old_path=Monitor%2Ftags%2FMonitor-1.0-7&new_path=Monitor%2Ftags%2FMonitor-1.0-13

897 # xxx quick & dirty: rough url parsing

898 def trac_full_diff (self, pkg):

899 matcher=re.compile("\A(?P<method>.*)://(?P<hostname>[^/]+)/(svn/)?(?P<path>.*)\Z")

900 self_match=matcher.match(self.svnpath)

901 pkg_match=matcher.match(pkg.svnpath)

902 if self_match and pkg_match:

903 (method,hostname,svn,path)=self_match.groups()

904 self_path=path.replace("/","%2F")

905 pkg_path=pkg_match.group('path').replace("/","%2F")

906 return "%s://%s/changeset?old_path=%s&new_path=%s"%(method,hostname,self_path,pkg_path)

907 else:

908 return None

909

910 def details (self):

911 return "[%s %s] [%s (spec)]"%(self.svnpath,self.basename,self.specpath)

912

913 class Build (Module):

914

915 # we cannot get build's svnpath as for other packages as we'd get something in svn+ssh

916 # xxx quick & dirty

917 def __init__ (self, buildtag,options):

918 self.buildtag=buildtag

919 # if the buildtag start with a : (to use a branch rather than a tag)

920 if buildtag.find(':') == 0 :

921 module_name="build%(buildtag)s"%locals()

922 self.display=buildtag[1:]

923 self.svnpath="http://svn.planet-lab.org/svn/build/branches/%s"%self.display

924 else :

925 module_name="build@%(buildtag)s"%locals()

926 self.display=buildtag

927 self.svnpath="http://svn.planet-lab.org/svn/build/tags/%s"%self.buildtag

928 Module.__init__(self,module_name,options)

929

930 @staticmethod

931 def get_distro_from_distrotag (distrotag):

932 # mhh: remove -tag* from distrotags to get distro

933 n=distrotag.find('-tag')

934 if n>0:

935 return distrotag[:n]

936 else:

937 return None

938

939 def get_packages (self,distrotag):

940 result={}

941 distro=Build.get_distro_from_distrotag(distrotag)

942 if not distro:

943 return result

944 make_options="--no-print-directory -C %s stage1=true PLDISTRO=%s PLDISTROTAGS=%s 2> /dev/null"%(self.edge_dir(),distro,distrotag)

945 command="make %s packages"%make_options

946 make_packages=Command(command,self.options).output_of()

947 pkg_line=re.compile("\Apackage=(?P<package>[^\s]+)\s+ref_module=(?P<module>[^\s]+)\s.*\Z")

948 for line in make_packages.split("\n"):

949 if not line:

950 continue

951 attempt=pkg_line.match(line)

952 if line and not attempt:

953 print "====="

954 print "WARNING: line not understood from make packages"

955 print "in dir %s"%self.edge_dir

956 print "with options",make_options

957 print 'line=',line

958 print "====="

959 else:

960 (package,module) = (attempt.group('package'),attempt.group('module'))

961 command="make %s +%s-SVNPATH"%(make_options,module)

962 svnpath=Command(command,self.options).output_of().strip()

963 command="make %s +%s-SPEC"%(make_options,package)

964 spec=Command(command,self.options).output_of().strip()

965 result[package]=Package(package,module,svnpath,spec)

966 return result

967

968 def get_distrotags (self):

969 return [os.path.basename(p) for p in glob("%s/*tags*mk"%self.edge_dir())]

970

971 class DiffCache:

972

973 def __init__ (self):

974 self._cache={}

975

976 def key(self, frompath,topath):

977 return frompath+'-to-'+topath

978

979 def fetch (self, frompath, topath):

980 key=self.key(frompath,topath)

981 if not self._cache.has_key(key):

982 return None

983 return self._cache[key]

984

985 def store (self, frompath, topath, diff):

986 key=self.key(frompath,topath)

987 self._cache[key]=diff

988

989 class Release:

990

991 # header in diff output

992 discard_matcher=re.compile("\A(\+\+\+|---).*")

993

994 @staticmethod

995 def do_changelog (buildtag_new,buildtag_old,options):

996 print "----"

997 print "----"

998 print "----"

999 (build_new,build_old) = (Build (buildtag_new,options), Build (buildtag_old,options))

1000 print "= build tag %s to %s = #build-%s"%(build_old.display,build_new.display,build_new.display)

1001 for b in (build_new,build_old):

1002 b.init_module_dir()

1003 b.init_edge_dir()

1004 b.update_edge_dir()

1005 # find out the tags files that are common, unless option was specified

1006 if options.distrotags:

1007 distrotags=options.distrotags

1008 else:

1009 distrotags_new=build_new.get_distrotags()

1010 distrotags_old=build_old.get_distrotags()

1011 distrotags = list(set(distrotags_new).intersection(set(distrotags_old)))

1012 distrotags.sort()

1013 if options.verbose: print "Found distrotags",distrotags

1014 first_distrotag=True

1015 diffcache = DiffCache()

1016 for distrotag in distrotags:

1017 distro=Build.get_distro_from_distrotag(distrotag)

1018 if not distro:

1019 continue

1020 if first_distrotag:

1021 first_distrotag=False

1022 else:

1023 print '----'

1024 print '== distro %s (%s to %s) == #distro-%s-%s'%(distrotag,build_old.display,build_new.display,distro,build_new.display)

1025 print ' * from %s/%s'%(build_old.svnpath,distrotag)

1026 print ' * to %s/%s'%(build_new.svnpath,distrotag)

1027

1028 # parse make packages

1029 packages_new=build_new.get_packages(distrotag)

1030 pnames_new=set(packages_new.keys())

1031 if options.verbose: print 'got packages for ',build_new.display

1032 packages_old=build_old.get_packages(distrotag)

1033 pnames_old=set(packages_old.keys())

1034 if options.verbose: print 'got packages for ',build_old.display

1035

1036 # get created, deprecated, and preserved package names

1037 pnames_created = list(pnames_new-pnames_old)

1038 pnames_created.sort()

1039 pnames_deprecated = list(pnames_old-pnames_new)

1040 pnames_deprecated.sort()

1041 pnames = list(pnames_new.intersection(pnames_old))

1042 pnames.sort()

1043

1044 if options.verbose: print "Found new/deprecated/preserved pnames",pnames_new,pnames_deprecated,pnames

1045

1046 # display created and deprecated

1047 for name in pnames_created:

1048 print '=== %s : new package %s -- appeared in %s === #package-%s-%s-%s'%(

1049 distrotag,name,build_new.display,name,distro,build_new.display)

1050 pobj=packages_new[name]

1051 print ' * %s'%pobj.details()

1052 for name in pnames_deprecated:

1053 print '=== %s : package %s -- deprecated, last occurrence in %s === #package-%s-%s-%s'%(

1054 distrotag,name,build_old.display,name,distro,build_new.display)

1055 pobj=packages_old[name]

1056 if not pobj.svnpath:

1057 print ' * codebase stored in CVS, specfile is %s'%pobj.spec

1058 else:

1059 print ' * %s'%pobj.details()

1060

1061 # display other packages

1062 for name in pnames:

1063 (pobj_new,pobj_old)=(packages_new[name],packages_old[name])

1064 if options.verbose: print "Dealing with package",name

1065 if pobj_old.specpath == pobj_new.specpath:

1066 continue

1067 specdiff = diffcache.fetch(pobj_old.specpath,pobj_new.specpath)

1068 if specdiff is None:

1069 command="svn diff %s %s"%(pobj_old.specpath,pobj_new.specpath)

1070 specdiff=Command(command,options).output_of()

1071 diffcache.store(pobj_old.specpath,pobj_new.specpath,specdiff)

1072 else:

1073 if options.verbose: print 'got diff from cache'

1074 if not specdiff:

1075 continue

1076 print '=== %s - %s to %s : package %s === #package-%s-%s-%s'%(

1077 distrotag,build_old.display,build_new.display,name,name,distro,build_new.display)

1078 print ' * from %s to %s'%(pobj_old.details(),pobj_new.details())

1079 trac_diff_url=pobj_old.trac_full_diff(pobj_new)

1080 if trac_diff_url:

1081 print ' * [%s View full diff]'%trac_diff_url

1082 print '{{{'

1083 for line in specdiff.split('\n'):

1084 if not line:

1085 continue

1086 if Release.discard_matcher.match(line):

1087 continue

1088 if line[0] in ['@']:

1089 print '----------'

1090 elif line[0] in ['+','-']:

1091 print_fold(line)

1092 print '}}}'

1093

1094 ##############################

1095 class Main:

1096

1097 module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]

1098 module-tools : a set of tools to manage subversion tags and specfile

1099 requires the specfile to either

1100 * define *version* and *taglevel*

1101 OR alternatively

1102 * define redirection variables module_version_varname / module_taglevel_varname

1103 Trunk:

1104 by default, the trunk of modules is taken into account

1105 in this case, just mention the module name as <module_desc>

1106 Branches:

1107 if you wish to work on a branch rather than on the trunk,

1108 you can use something like e.g. Mom:2.1 as <module_desc>

1109 """

1110 release_usage="""Usage: %prog [options] tag1 .. tagn

1111 Extract release notes from the changes in specfiles between several build tags, latest first

1112 Examples:

1113 release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22

1114 You can refer to a (build) branch by prepending a colon, like in

1115 release-changelog :4.2 4.2-rc25

1116 """

1117 common_usage="""More help:

1118 see http://svn.planet-lab.org/wiki/ModuleTools"""

1119

1120 modes={

1121 'list' : "displays a list of available tags or branches",

1122 'version' : "check latest specfile and print out details",

1123 'diff' : "show difference between module (trunk or branch) and latest tag",

1124 'tag' : """increment taglevel in specfile, insert changelog in specfile,

1125 create new tag and and monitor its adoption in build/*-tags*.mk""",

1126 'branch' : """create a branch for this module, from the latest tag on the trunk,

1127 and change trunk's version number to reflect the new branch name;

1128 you can specify the new branch name by using module:branch""",

1129 'sync' : """create a tag from the module

1130 this is a last resort option, mostly for repairs""",

1131 'changelog' : """extract changelog between build tags

1132 expected arguments are a list of tags""",

1133 }

1134

1135 silent_modes = ['list']

1136 release_modes = ['changelog']

1137

1138 @staticmethod

1139 def optparse_list (option, opt, value, parser):

1140 try:

1141 setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())

1142 except:

1143 setattr(parser.values,option.dest,value.split())

1144

1145 def run(self):

1146

1147 mode=None

1148 for function in Main.modes.keys():

1149 if sys.argv[0].find(function) >= 0:

1150 mode = function

1151 break

1152 if not mode:

1153 print "Unsupported command",sys.argv[0]

1154 print "Supported commands:" + Modes.modes.keys.join(" ")

1155 sys.exit(1)

1156

1157 if mode not in Main.release_modes:

1158 usage = Main.module_usage

1159 usage += Main.common_usage

1160 usage += "\nmodule-%s : %s"%(mode,Main.modes[mode])

1161 else:

1162 usage = Main.release_usage

1163 usage += Main.common_usage

1164

1165 parser=OptionParser(usage=usage,version=subversion_id)

1166

1167 if mode == 'list':

1168 parser.add_option("-b","--branches",action="store_true",dest="list_branches",default=False,

1169 help="list branches")

1170 parser.add_option("-t","--tags",action="store_false",dest="list_branches",

1171 help="list tags")

1172 parser.add_option("-m","--match",action="store",dest="list_pattern",default=None,

1173 help="grep pattern for filtering output")

1174 parser.add_option("-x","--exact-match",action="store",dest="list_exact",default=None,

1175 help="exact grep pattern for filtering output")

1176 if mode == "tag" or mode == 'branch':

1177 parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,

1178 help="set new version and reset taglevel to 0")

1179 if mode == "tag" :

1180 parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,

1181 help="do not update changelog section in specfile when tagging")

1182 parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,

1183 help="specify a build branch; used for locating the *tags*.mk files where adoption is to take place")

1184 if mode == "tag" or mode == "sync" :

1185 parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),

1186 help="specify editor")

1187 if mode == "sync" :

1188 parser.add_option("-m","--message", action="store", dest="message", default=None,

1189 help="specify log message")

1190 if mode == "diff" :

1191 parser.add_option("-o","--only", action="store_true", dest="only", default=False,

1192 help="report diff only for modules that exhibit differences")

1193 if mode == "diff" :

1194 parser.add_option("-l","--list", action="store_true", dest="list", default=False,

1195 help="just list modules that exhibit differences")

1196

1197 if mode == 'version':

1198 parser.add_option("-u","--url", action="store_true", dest="show_urls", default=False,

1199 help="display URLs")

1200

1201 default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"

1202 if mode not in Main.release_modes:

1203 parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False,

1204 help="run on all modules as found in %s"%default_modules_list)

1205 parser.add_option("-f","--file",action="store",dest="modules_list",default=None,

1206 help="run on all modules found in specified file")

1207 else:

1208 parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,

1209 help="dry run - shell commands are only displayed")

1210 parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",

1211 default=[], nargs=1,type="string",

1212 help="""specify distro-tags files, e.g. onelab-tags-4.2.mk

1213 -- can be set multiple times, or use quotes""")

1214

1215 parser.add_option("-w","--workdir", action="store", dest="workdir",

1216 default="%s/%s"%(os.getenv("HOME"),"modules"),

1217 help="""name for dedicated working dir - defaults to ~/modules

1218 ** THIS MUST NOT ** be your usual working directory""")

1219 parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,

1220 help="skip safety checks, such as svn updates -- use with care")

1221

1222 # default verbosity depending on function - temp

1223 verbose_modes= ['tag','sync']

1224

1225 if mode not in verbose_modes:

1226 parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,

1227 help="run in verbose mode")

1228 else:

1229 parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,

1230 help="run in quiet (non-verbose) mode")

1231 # parser.add_option("-d","--debug", action="store_true", dest="debug", default=False,

1232 # help="debug mode - mostly more verbose")

1233 (options, args) = parser.parse_args()

1234 options.mode=mode

1235 if not hasattr(options,'dry_run'):

1236 options.dry_run=False

1237 options.debug=False

1238

1239 ########## release-*

1240 if mode in Main.release_modes :

1241 ########## changelog

1242 if len(args) <= 1:

1243 parser.print_help()

1244 sys.exit(1)

1245 Module.init_homedir(options)

1246 for n in range(len(args)-1):

1247 [t_new,t_old]=args[n:n+2]

1248 Release.do_changelog (t_new,t_old,options)

1249 else:

1250 ########## module-*

1251 if len(args) == 0:

1252 if options.all_modules:

1253 options.modules_list=default_modules_list

1254 if options.modules_list:

1255 args=Command("grep -v '#' %s"%options.modules_list,options).output_of().split()

1256 else:

1257 parser.print_help()

1258 sys.exit(1)

1259 Module.init_homedir(options)

1260 for modname in args:

1261 module=Module(modname,options)

1262 if len(args)>1 and mode not in Main.silent_modes:

1263 print '========================================',module.friendly_name()

1264 # call the method called do_<mode>

1265 method=Module.__dict__["do_%s"%mode]

1266 try:

1267 method(module)

1268 except Exception,e:

1269 print 'Skipping failed %s: '%modname,e

1270

1271 ####################

1272 if __name__ == "__main__" :

1273 try:

1274 Main().run()

1275 except KeyboardInterrupt:

1276 print '\nBye'