git.onelab.eu Git - build.git/blob

1 #!/usr/bin/env python3

2

3 import sys, os, os.path

4 import re

5 import time

6 import tempfile

7 from glob import glob

8 from optparse import OptionParser

9

10 # e.g. other_choices = [ ('d','iff') , ('g','uess') ] - lowercase

11 def prompt(question, default=True, other_choices=[], allow_outside=False):

12 if not isinstance(other_choices, list):

13 other_choices = [ other_choices ]

14 chars = [ c for (c,rest) in other_choices ]

15

16 choices = []

17 if 'y' not in chars:

18 if default is True:

19 choices.append('[y]')

20 else :

21 choices.append('y')

22 if 'n' not in chars:

23 if default is False:

24 choices.append('[n]')

25 else:

26 choices.append('n')

27

28 for char, choice in other_choices:

29 if default == char:

30 choices.append("[{}]{}".format(char, choice))

31 else:

32 choices.append("<{}>{}>".format(char, choice))

33 try:

34 answer = input(question + " " + "/".join(choices) + " ? ")

35 if not answer:

36 return default

37 answer = answer[0].lower()

38 if answer == 'y':

39 if 'y' in chars:

40 return 'y'

41 else:

42 return True

43 elif answer == 'n':

44 if 'n' in chars:

45 return 'n'

46 else:

47 return False

48 elif other_choices:

49 for (char,choice) in other_choices:

50 if answer == char:

51 return char

52 if allow_outside:

53 return answer

54 return prompt(question, default, other_choices)

55 except:

56 raise

57

58 def default_editor():

59 try:

60 editor = os.environ['EDITOR']

61 except:

62 editor = "emacs"

63 return editor

64

65 ### fold long lines

66 fold_length = 132

67

68 def print_fold(line):

69 while len(line) >= fold_length:

70 print(line[:fold_length],'\\')

71 line = line[fold_length:]

72 print(line)

73

74 class Command:

75 def __init__(self, command, options):

76 self.command = command

77 self.options = options

78 self.tmp = "/tmp/command-{}".format(os.getpid())

79

80 def run(self):

81 if self.options.dry_run:

82 print('dry_run', self.command)

83 return 0

84 if self.options.verbose and self.options.mode not in Main.silent_modes:

85 print('+', self.command)

86 sys.stdout.flush()

87 return os.system(self.command)

88

89 def run_silent(self):

90 if self.options.dry_run:

91 print('dry_run', self.command)

92 return 0

93 if self.options.verbose:

94 print('>', os.getcwd())

95 print('+', self.command, ' .. ', end=' ')

96 sys.stdout.flush()

97 retcod = os.system("{} &> {}".format(self.command, self.tmp))

98 if retcod != 0:

99 print("FAILED ! -- out+err below (command was {})".format(self.command))

100 os.system("cat {}".format(self.tmp))

101 print("FAILED ! -- end of quoted output")

102 elif self.options.verbose:

103 print("OK")

104 os.unlink(self.tmp)

105 return retcod

106

107 def run_fatal(self):

108 if self.run_silent() != 0:

109 raise Exception("Command {} failed".format(self.command))

110

111 # returns stdout, like bash's $(mycommand)

112 def output_of(self, with_stderr=False, binary=False):

113 if self.options.dry_run:

114 print('dry_run', self.command)

115 return 'dry_run output'

116 tmp="/tmp/status-{}".format(os.getpid())

117 if self.options.debug:

118 print('+',self.command,' .. ', end=' ')

119 sys.stdout.flush()

120 command = self.command

121 if with_stderr:

122 command += " &> "

123 else:

124 command += " > "

125 command += tmp

126 os.system(command)

127 mode = "r" if not binary else "rb"

128 with open(tmp, mode) as f:

129 result=f.read()

130 os.unlink(tmp)

131 if self.options.debug:

132 print('Done', end=' ')

133 return result

134

135 class GitRepository:

136 type = "git"

137

138 def __init__(self, path, options):

139 self.path = path

140 self.options = options

141

142 def name(self):

143 return os.path.basename(self.path)

144

145 def url(self):

146 return self.repo_root()

147

148 def gitweb(self):

149 c = Command("git show | grep commit | awk '{{print $2;}}'", self.options)

150 out = self.__run_in_repo(c.output_of).strip()

151 return "http://git.onelab.eu/?p={}.git;a=commit;h={}".format(self.name(), out)

152

153 def repo_root(self):

154 c = Command("git remote show origin", self.options)

155 out = self.__run_in_repo(c.output_of)

156 for line in out.split('\n'):

157 if line.strip().startswith("Fetch URL:"):

158 return line.split()[2]

159

160 @classmethod

161 def clone(cls, remote, local, options, depth=None):

162 Command("rm -rf {}".format(local), options).run_silent()

163 depth_option = "" if depth is None else " --depth {}".format(depth)

164 Command("git clone{} {} {}".format(depth_option, remote, local), options).run_fatal()

165 return GitRepository(local, options)

166

167 @classmethod

168 def remote_exists(cls, remote, options):

169 return Command("git --no-pager ls-remote {} &> /dev/null".format(remote), options).run()==0

170

171 def tag_exists(self, tagname):

172 command = 'git tag -l | grep "^{}$"'.format(tagname)

173 c = Command(command, self.options)

174 out = self.__run_in_repo(c.output_of, with_stderr=True)

175 return len(out) > 0

176

177 def __run_in_repo(self, fun, *args, **kwargs):

178 cwd = os.getcwd()

179 os.chdir(self.path)

180 ret = fun(*args, **kwargs)

181 os.chdir(cwd)

182 return ret

183

184 def __run_command_in_repo(self, command, ignore_errors=False):

185 c = Command(command, self.options)

186 if ignore_errors:

187 return self.__run_in_repo(c.output_of)

188 else:

189 return self.__run_in_repo(c.run_fatal)

190

191 def __is_commit_id(self, id):

192 c = Command("git show {} | grep commit | awk '{{print $2;}}'".format(id), self.options)

193 ret = self.__run_in_repo(c.output_of, with_stderr=False)

194 if ret.strip() == id:

195 return True

196 return False

197

198 def update(self, subdir=None, recursive=None, branch="master"):

199 if branch == "master":

200 self.__run_command_in_repo("git checkout {}".format(branch))

201 else:

202 self.to_branch(branch, remote=True)

203 self.__run_command_in_repo("git fetch origin --tags", ignore_errors=True)

204 self.__run_command_in_repo("git fetch origin")

205 if not self.__is_commit_id(branch):

206 # we don't need to merge anything for commit ids.

207 self.__run_command_in_repo("git merge --ff origin/{}".format(branch))

208

209 def to_branch(self, branch, remote=True):

210 self.revert()

211 if remote:

212 command = "git branch --track {} origin/{}".format(branch, branch)

213 c = Command(command, self.options)

214 self.__run_in_repo(c.output_of, with_stderr=True)

215 return self.__run_command_in_repo("git checkout {}".format(branch))

216

217 def to_tag(self, tag):

218 self.revert()

219 return self.__run_command_in_repo("git checkout {}".format(tag))

220

221 def tag(self, tagname, logfile):

222 self.__run_command_in_repo("git tag {} -F {}".format(tagname, logfile))

223 self.commit(logfile)

224

225 def diff(self, f=""):

226 c = Command("git diff {}".format(f), self.options)

227 try:

228 return self.__run_in_repo(c.output_of, with_stderr=True)

229 except:

230 return self.__run_in_repo(c.output_of, with_stderr=True, binary=True)

231

232 def diff_with_tag(self, tagname):

233 c = Command("git diff {}".format(tagname), self.options)

234 try:

235 return self.__run_in_repo(c.output_of, with_stderr=True)

236 except:

237 return self.__run_in_repo(c.output_of, with_stderr=True, binary=True)

238

239 def commit(self, logfile, branch="master"):

240 self.__run_command_in_repo("git add .", ignore_errors=True)

241 self.__run_command_in_repo("git add -u", ignore_errors=True)

242 self.__run_command_in_repo("git commit -F {}".format(logfile), ignore_errors=True)

243 if branch == "master" or self.__is_commit_id(branch):

244 self.__run_command_in_repo("git push")

245 else:

246 self.__run_command_in_repo("git push origin {}:{}".format(branch, branch))

247 self.__run_command_in_repo("git push --tags", ignore_errors=True)

248

249 def revert(self, f=""):

250 if f:

251 self.__run_command_in_repo("git checkout {}".format(f))

252 else:

253 # revert all

254 self.__run_command_in_repo("git --no-pager reset --hard")

255 self.__run_command_in_repo("git --no-pager clean -f")

256

257 def is_clean(self):

258 def check_commit():

259 command = "git status"

260 s = "nothing to commit, working directory clean"

261 return Command(command, self.options).output_of(True).find(s) >= 0

262 return self.__run_in_repo(check_commit)

263

264 def is_valid(self):

265 return os.path.exists(os.path.join(self.path, ".git"))

266

267

268 class Repository:

269 """

270 Generic repository

271 From old times when we had svn and git

272 """

273 supported_repo_types = [ GitRepository ]

274

275 def __init__(self, path, options):

276 self.path = path

277 self.options = options

278 for repo_class in self.supported_repo_types:

279 self.repo = repo_class(self.path, self.options)

280 if self.repo.is_valid():

281 break

282

283 @classmethod

284 def remote_exists(cls, remote, options):

285 for repo_class in Repository.supported_repo_types:

286 if repo_class.remote_exists(remote, options):

287 return True

288 return False

289

290 def __getattr__(self, attr):

291 return getattr(self.repo, attr)

292

293

294

295 # support for tagged module is minimal, and is for the Build class only

296 class Module:

297

298 edit_magic_line = "--This line, and those below, will be ignored--"

299 setting_tag_format = "Setting tag {}"

300

301 redirectors = [

302 # ('module_name_varname', 'name'),

303 ('module_version_varname', 'version'),

304 ('module_taglevel_varname', 'taglevel'),

305 ]

306

307 # where to store user's config

308 config_storage = "CONFIG"

309 #

310 config = {}

311

312 import subprocess

313 configKeys = [

314 ('gitserver', "Enter your git server's hostname", "git.onelab.eu"),

315 ('gituser', "Enter your user name (login name) on git server", subprocess.getoutput("id -un")),

316 ("build", "Enter the name of your build module", "build"),

317 ('username', "Enter your firstname and lastname for changelogs", ""),

318 ("email", "Enter your email address for changelogs", ""),

319 ]

320

321 @classmethod

322 def prompt_config_option(cls, key, message, default):

323 cls.config[key]=input("{} [{}] : ".format(message,default)).strip() or default

324

325 @classmethod

326 def prompt_config(cls):

327 for (key,message,default) in cls.configKeys:

328 cls.config[key]=""

329 while not cls.config[key]:

330 cls.prompt_config_option(key, message, default)

331

332 # for parsing module spec name:branch

333 matcher_branch_spec = re.compile("\A(?P<name>[\w\.\-\/]+):(?P<branch>[\w\.\-]+)\Z")

334 # special form for tagged module - for Build

335 matcher_tag_spec = re.compile("\A(?P<name>[\w\.\-\/]+)@(?P<tagname>[\w\.\-]+)\Z")

336

337 # parsing specfiles

338 matcher_rpm_define = re.compile("%(define|global)\s+(\S+)\s+(\S*)\s*")

339

340 @classmethod

341 def parse_module_spec(cls, module_spec):

342 name = branch_or_tagname = module_type = ""

343

344 attempt = Module.matcher_branch_spec.match(module_spec)

345 if attempt:

346 module_type = "branch"

347 name=attempt.group('name')

348 branch_or_tagname=attempt.group('branch')

349 else:

350 attempt = Module.matcher_tag_spec.match(module_spec)

351 if attempt:

352 module_type = "tag"

353 name=attempt.group('name')

354 branch_or_tagname=attempt.group('tagname')

355 else:

356 name = module_spec

357 return name, branch_or_tagname, module_type

358

359

360 def __init__(self, module_spec, options):

361 # parse module spec

362 self.pathname, branch_or_tagname, module_type = self.parse_module_spec(module_spec)

363 self.name = os.path.basename(self.pathname)

364

365 if module_type == "branch":

366 self.branch = branch_or_tagname

367 elif module_type == "tag":

368 self.tagname = branch_or_tagname

369

370 self.options=options

371 self.module_dir="{}/{}".format(options.workdir,self.pathname)

372 self.repository = None

373 self.build = None

374

375 def run(self,command):

376 return Command(command,self.options).run()

377 def run_fatal(self,command):

378 return Command(command,self.options).run_fatal()

379 def run_prompt(self,message,fun, *args):

380 fun_msg = "{}({})".format(fun.__name__, ",".join(args))

381 if not self.options.verbose:

382 while True:

383 choice = prompt(message, True, ('s','how'))

384 if choice:

385 fun(*args)

386 return

387 else:

388 print('About to run function:', fun_msg)

389 else:

390 question = "{} - want to run function: {}".format(message, fun_msg)

391 if prompt(question, True):

392 fun(*args)

393

394 def friendly_name(self):

395 if hasattr(self, 'branch'):

396 return "{}:{}".format(self.pathname, self.branch)

397 elif hasattr(self, 'tagname'):

398 return "{}@{}".format(self.pathname, self.tagname)

399 else:

400 return self.pathname

401

402 @classmethod

403 def git_remote_dir(cls, name):

404 return "{}@{}:/git/{}.git".format(cls.config['gituser'], cls.config['gitserver'], name)

405

406 ####################

407 @classmethod

408 def init_homedir(cls, options):

409 if options.verbose and options.mode not in Main.silent_modes:

410 print('Checking for', options.workdir)

411 storage="{}/{}".format(options.workdir, cls.config_storage)

412 # sanity check. Either the topdir exists AND we have a config/storage

413 # or topdir does not exist and we create it

414 # to avoid people use their own daily work repo

415 if os.path.isdir(options.workdir) and not os.path.isfile(storage):

416 print("""The directory {} exists and has no CONFIG file

417 If this is your regular working directory, please provide another one as the

418 module-* commands need a fresh working dir. Make sure that you do not use

419 that for other purposes than tagging""".format(options.workdir))

420 sys.exit(1)

421

422 def clone_build():

423 print("Checking out build module...")

424 remote = cls.git_remote_dir(cls.config['build'])

425 local = os.path.join(options.workdir, cls.config['build'])

426 GitRepository.clone(remote, local, options, depth=1)

427 print("OK")

428

429 def store_config():

430 with open(storage, 'w') as f:

431 for (key, message, default) in Module.configKeys:

432 f.write("{}={}\n".format(key, Module.config[key]))

433 if options.debug:

434 print('Stored', storage)

435 Command("cat {}".format(storage),options).run()

436

437 def read_config():

438 # read config

439 with open(storage) as f:

440 for line in f.readlines():

441 key, value = re.compile("^(.+)=(.+)$").match(line).groups()

442 Module.config[key] = value

443

444 # owerride config variables using options.

445 if options.build_module:

446 Module.config['build'] = options.build_module

447

448 if not os.path.isdir(options.workdir):

449 print("Cannot find {}, let's create it".format(options.workdir))

450 Command("mkdir -p {}".format(options.workdir), options).run_silent()

451 cls.prompt_config()

452 clone_build()

453 store_config()

454 else:

455 read_config()

456 # check missing config options

457 old_layout = False

458 for key, message, default in cls.configKeys:

459 if key not in Module.config:

460 print("Configuration changed for module-tools")

461 cls.prompt_config_option(key, message, default)

462 old_layout = True

463

464 if old_layout:

465 Command("rm -rf {}".format(options.workdir), options).run_silent()

466 Command("mkdir -p {}".format(options.workdir), options).run_silent()

467 clone_build()

468 store_config()

469

470 build_dir = os.path.join(options.workdir, cls.config['build'])

471 if not os.path.isdir(build_dir):

472 clone_build()

473 else:

474 build = Repository(build_dir, options)

475 if not build.is_clean():

476 print("build module needs a revert")

477 build.revert()

478 print("OK")

479 build.update()

480

481 if options.verbose and options.mode not in Main.silent_modes:

482 print('******** Using config')

483 for (key,message,default) in Module.configKeys:

484 print('\t{} = {}'.format(key,Module.config[key]))

485

486 def init_module_dir(self):

487 if self.options.verbose:

488 print('Checking for', self.module_dir)

489

490 if not os.path.isdir(self.module_dir):

491 self.repository = GitRepository.clone(self.git_remote_dir(self.pathname),

492 self.module_dir,

493 self.options)

494

495 self.repository = Repository(self.module_dir, self.options)

496

497 if self.repository.type == "git":

498 if hasattr(self, 'branch'):

499 self.repository.to_branch(self.branch)

500 elif hasattr(self, 'tagname'):

501 self.repository.to_tag(self.tagname)

502 else:

503 raise Exception('Cannot find {} - or not a git module'.format(self.module_dir))

504

505

506 def revert_module_dir(self):

507 if self.options.fast_checks:

508 if self.options.verbose: print('Skipping revert of {}'.format(self.module_dir))

509 return

510 if self.options.verbose:

511 print('Checking whether', self.module_dir, 'needs being reverted')

512

513 if not self.repository.is_clean():

514 self.repository.revert()

515

516 def update_module_dir(self):

517 if self.options.fast_checks:

518 if self.options.verbose: print('Skipping update of {}'.format(self.module_dir))

519 return

520 if self.options.verbose:

521 print('Updating', self.module_dir)

522

523 if hasattr(self, 'branch'):

524 self.repository.update(branch=self.branch)

525 elif hasattr(self, 'tagname'):

526 self.repository.update(branch=self.tagname)

527 else:

528 self.repository.update()

529

530 def main_specname(self):

531 attempt = "{}/{}.spec".format(self.module_dir, self.name)

532 if os.path.isfile(attempt):

533 return attempt

534 pattern1 = "{}/*.spec".format(self.module_dir)

535 level1 = glob(pattern1)

536 if level1:

537 return level1[0]

538 pattern2 = "{}/*/*.spec".format(self.module_dir)

539 level2 = glob(pattern2)

540

541 if level2:

542 return level2[0]

543 raise Exception('Cannot guess specfile for module {} -- patterns were {} or {}'\

544 .format(self.pathname,pattern1,pattern2))

545

546 def all_specnames(self):

547 level1 = glob("{}/*.spec".format(self.module_dir))

548 if level1:

549 return level1

550 level2 = glob("{}/*/*.spec".format(self.module_dir))

551 return level2

552

553 def parse_spec(self, specfile, varnames):

554 if self.options.verbose:

555 print('Parsing',specfile, end=' ')

556 for var in varnames:

557 print("[{}]".format(var), end=' ')

558 print("")

559 result={}

560 with open(specfile, encoding='utf-8') as f:

561 for line in f.readlines():

562 attempt = Module.matcher_rpm_define.match(line)

563 if attempt:

564 define, var, value = attempt.groups()

565 if var in varnames:

566 result[var] = value

567 if self.options.debug:

568 print('found {} keys'.format(len(result)))

569 for k, v in result.items():

570 print('{} = {}'.format(k, v))

571 return result

572

573 # stores in self.module_name_varname the rpm variable to be used for the module's name

574 # and the list of these names in self.varnames

575 def spec_dict(self):

576 specfile = self.main_specname()

577 redirector_keys = [ varname for (varname, default) in Module.redirectors]

578 redirect_dict = self.parse_spec(specfile, redirector_keys)

579 if self.options.debug:

580 print('1st pass parsing done, redirect_dict=', redirect_dict)

581 varnames = []

582 for varname, default in Module.redirectors:

583 if varname in redirect_dict:

584 setattr(self, varname, redirect_dict[varname])

585 varnames += [redirect_dict[varname]]

586 else:

587 setattr(self, varname, default)

588 varnames += [ default ]

589 self.varnames = varnames

590 result = self.parse_spec(specfile, self.varnames)

591 if self.options.debug:

592 print('2st pass parsing done, varnames={} result={}'.format(varnames, result))

593 return result

594

595 def patch_spec_var(self, patch_dict,define_missing=False):

596 for specfile in self.all_specnames():

597 # record the keys that were changed

598 changed = {x : False for x in list(patch_dict.keys()) }

599 newspecfile = "{}.new".format(specfile)

600 if self.options.verbose:

601 print('Patching', specfile, 'for', list(patch_dict.keys()))

602

603 with open(specfile, encoding='utf-8') as spec:

604 with open(newspecfile, "w", encoding='utf-8') as new:

605 for line in spec.readlines():

606 attempt = Module.matcher_rpm_define.match(line)

607 if attempt:

608 define, var, value = attempt.groups()

609 if var in list(patch_dict.keys()):

610 if self.options.debug:

611 print('rewriting {} as {}'.format(var, patch_dict[var]))

612 new.write('%{} {} {}\n'.format(define, var, patch_dict[var]))

613 changed[var] = True

614 continue

615 new.write(line)

616 if define_missing:

617 for key, was_changed in changed.items():

618 if not was_changed:

619 if self.options.debug:

620 print('rewriting missing {} as {}'.format(key, patch_dict[key]))

621 new.write('\n%define {} {}\n'.format(key, patch_dict[key]))

622 os.rename(newspecfile, specfile)

623

624 # returns all lines until the magic line

625 def unignored_lines(self, logfile):

626 result = []

627 white_line_matcher = re.compile("\A\s*\Z")

628 with open(logfile) as f:

629 for logline in f.readlines():

630 if logline.strip() == Module.edit_magic_line:

631 break

632 elif white_line_matcher.match(logline):

633 continue

634 else:

635 result.append(logline.strip()+'\n')

636 return result

637

638 # creates a copy of the input with only the unignored lines

639 def strip_magic_line_filename(self, filein, fileout ,new_tag_name):

640 with open(fileout,'w') as f:

641 f.write(self.setting_tag_format.format(new_tag_name) + '\n')

642 for line in self.unignored_lines(filein):

643 f.write(line)

644

645 def insert_changelog(self, logfile, newtag):

646 for specfile in self.all_specnames():

647 newspecfile = "{}.new".format(specfile)

648 if self.options.verbose:

649 print('Inserting changelog from {} into {}'.format(logfile, specfile))

650

651 with open(specfile) as spec:

652 with open(newspecfile,"w") as new:

653 for line in spec.readlines():

654 new.write(line)

655 if re.compile('%changelog').match(line):

656 dateformat="* %a %b %d %Y"

657 datepart=time.strftime(dateformat)

658 logpart="{} <{}> - {}".format(Module.config['username'],

659 Module.config['email'],

660 newtag)

661 new.write("{} {}\n".format(datepart,logpart))

662 for logline in self.unignored_lines(logfile):

663 new.write("- " + logline)

664 new.write("\n")

665 os.rename(newspecfile,specfile)

666

667 def show_dict(self, spec_dict):

668 if self.options.verbose:

669 for k, v in spec_dict.items():

670 print('{} = {}'.format(k, v))

671

672 def last_tag(self, spec_dict):

673 try:

674 return "{}-{}".format(spec_dict[self.module_version_varname],

675 spec_dict[self.module_taglevel_varname])

676 except KeyError as err:

677 raise Exception('Something is wrong with module {}, cannot determine {} - exiting'\

678 .format(self.name, err))

679

680 def tag_name(self, spec_dict):

681 return "{}-{}".format(self.name, self.last_tag(spec_dict))

682

683

684 pattern_format="\A\s*{module}-(GITPATH)\s*(=|:=)\s*(?P<url_main>[^\s]+)/{module}[^\s]+"

685

686 def is_mentioned_in_tagsfile(self, tagsfile):

687 # so that {module} gets replaced from format

688 module = self.name

689 module_matcher = re.compile(Module.pattern_format.format(**locals()))

690 with open(tagsfile) as f:

691 for line in f.readlines():

692 if module_matcher.match(line):

693 return True

694 return False

695

696 ##############################

697 # using fine_grain means replacing only those instances that currently refer to this tag

698 # otherwise, <module>-GITPATH is replaced unconditionnally

699 def patch_tags_file(self, tagsfile, oldname, newname, fine_grain=True):

700 newtagsfile = "{}.new".format(tagsfile)

701

702 with open(tagsfile) as tags:

703 with open(newtagsfile,"w") as new:

704 matches = 0

705 # fine-grain : replace those lines that refer to oldname

706 if fine_grain:

707 if self.options.verbose:

708 print('Replacing {} into {}\n\tin {} .. '.format(oldname, newname, tagsfile), end=' ')

709 matcher = re.compile("^(.*){}(.*)".format(oldname))

710 for line in tags.readlines():

711 if not matcher.match(line):

712 new.write(line)

713 else:

714 begin, end = matcher.match(line).groups()

715 new.write(begin+newname+end+"\n")

716 matches += 1

717 # brute-force : change uncommented lines that define <module>-GITPATH

718 else:

719 if self.options.verbose:

720 print('Searching for -GITPATH lines referring to /{}/\n\tin {} .. '\

721 .format(self.pathname, tagsfile), end=' ')

722 # so that {module} gets replaced from format

723 module = self.name

724 module_matcher = re.compile(Module.pattern_format.format(**locals()))

725 for line in tags.readlines():

726 attempt = module_matcher.match(line)

727 if attempt:

728 if line.find("-GITPATH") >= 0:

729 modulepath = "{}-GITPATH".format(self.name)

730 replacement = "{:<32}:= {}/{}.git@{}\n"\

731 .format(modulepath, attempt.group('url_main'), self.pathname, newname)

732 else:

733 print("Could not locate {}-GITPATH (be aware that support for svn has been removed)"\

734 .format(self.name))

735 return

736 if self.options.verbose:

737 print(' ' + modulepath, end=' ')

738 new.write(replacement)

739 matches += 1

740 else:

741 new.write(line)

742

743 os.rename(newtagsfile,tagsfile)

744 if self.options.verbose:

745 print("{} changes".format(matches))

746 return matches

747

748 def check_tag(self, tagname, need_it=False):

749 if self.options.verbose:

750 print("Checking {} repository tag: {} - ".format(self.repository.type, tagname), end=' ')

751

752 found_tagname = tagname

753 found = self.repository.tag_exists(tagname)

754

755 if (found and need_it) or (not found and not need_it):

756 if self.options.verbose:

757 print("OK", end=' ')

758 print("- found" if found else "- not found")

759 else:

760 if self.options.verbose:

761 print("KO")

762 exception_format = "tag ({}) is already there" if found else "can not find required tag ({})"

763 raise Exception(exception_format.format(tagname))

764

765 return found_tagname

766

767

768 ##############################

769 def do_tag(self):

770 self.init_module_dir()

771 self.revert_module_dir()

772 self.update_module_dir()

773 # parse specfile

774 spec_dict = self.spec_dict()

775 self.show_dict(spec_dict)

776

777 # compute previous tag - if not bypassed

778 if not self.options.bypass:

779 old_tag_name = self.tag_name(spec_dict)

780 # sanity check

781 old_tag_name = self.check_tag(old_tag_name, need_it=True)

782

783 if self.options.new_version:

784 # new version set on command line

785 spec_dict[self.module_version_varname] = self.options.new_version

786 spec_dict[self.module_taglevel_varname] = 0

787 else:

788 # increment taglevel

789 new_taglevel = str( int(spec_dict[self.module_taglevel_varname]) + 1)

790 spec_dict[self.module_taglevel_varname] = new_taglevel

791

792 new_tag_name = self.tag_name(spec_dict)

793 # sanity check

794 new_tag_name = self.check_tag(new_tag_name, need_it=False)

795

796 # checking for diffs

797 if not self.options.bypass:

798 diff_output = self.repository.diff_with_tag(old_tag_name)

799 if len(diff_output) == 0:

800 if not prompt("No pending difference in module {}, want to tag anyway".format(self.pathname), False):

801 return

802

803 # side effect in head's specfile

804 self.patch_spec_var(spec_dict)

805

806 # prepare changelog file

807 # we use the standard subversion magic string (see edit_magic_line)

808 # so we can provide useful information, such as version numbers and diff

809 # in the same file

810 changelog_plain = "/tmp/{}-{}.edit".format(self.name, os.getpid())

811 changelog_strip = "/tmp/{}-{}.strip".format(self.name, os.getpid())

812 setting_tag_line = Module.setting_tag_format.format(new_tag_name)

813 with open(changelog_plain, 'w') as f:

814 f.write("""

815 {}

816 {}

817 Please write a changelog for this new tag in the section above

818 """.format(Module.edit_magic_line, setting_tag_line))

819

820 if self.options.bypass:

821 pass

822 elif prompt('Want to see diffs while writing changelog', True):

823 with open(changelog_plain, "a") as f:

824 f.write('DIFF=========\n' + diff_output)

825

826 if self.options.debug:

827 prompt('Proceed ?')

828

829 # edit it

830 self.run("{} {}".format(self.options.editor, changelog_plain))

831 # strip magic line in second file

832 self.strip_magic_line_filename(changelog_plain, changelog_strip, new_tag_name)

833 # insert changelog in spec

834 if self.options.changelog:

835 self.insert_changelog(changelog_plain, new_tag_name)

836

837 ## update build

838 build_path = os.path.join(self.options.workdir, Module.config['build'])

839 build = Repository(build_path, self.options)

840 if self.options.build_branch:

841 build.to_branch(self.options.build_branch)

842 if not build.is_clean():

843 build.revert()

844

845 tagsfiles = glob(build.path+"/*-tags.mk")

846 tagsdict = { x : 'todo' for x in tagsfiles }

847 default_answer = 'y'

848 tagsfiles.sort()

849 while True:

850 # do not bother if in bypass mode

851 if self.options.bypass:

852 break

853 for tagsfile in tagsfiles:

854 if not self.is_mentioned_in_tagsfile(tagsfile):

855 if self.options.verbose:

856 print("tagsfile {} does not mention {} - skipped".format(tagsfile, self.name))

857 continue

858 status = tagsdict[tagsfile]

859 basename = os.path.basename(tagsfile)

860 print(".................... Dealing with {}".format(basename))

861 while tagsdict[tagsfile] == 'todo' :

862 choice = prompt("insert {} in {} ".format(new_tag_name, basename),

863 default_answer,

864 [ ('y','es'), ('n', 'ext'), ('f','orce'),

865 ('d','iff'), ('r','evert'), ('c', 'at'), ('h','elp') ] ,

866 allow_outside=True)

867 if choice == 'y':

868 self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=True)

869 elif choice == 'n':

870 print('Done with {}'.format(os.path.basename(tagsfile)))

871 tagsdict[tagsfile] = 'done'

872 elif choice == 'f':

873 self.patch_tags_file(tagsfile, old_tag_name, new_tag_name, fine_grain=False)

874 elif choice == 'd':

875 print(build.diff(f=os.path.basename(tagsfile)))

876 elif choice == 'r':

877 build.revert(f=tagsfile)

878 elif choice == 'c':

879 self.run("cat {}".format(tagsfile))

880 else:

881 name = self.name

882 print(

883 """y: change {name}-GITPATH only if it currently refers to {old_tag_name}

884 f: unconditionnally change any line that assigns {name}-GITPATH to using {new_tag_name}

885 d: show current diff for this tag file

886 r: revert that tag file

887 c: cat the current tag file

888 n: move to next file""".format(**locals()))

889

890 if prompt("Want to review changes on tags files", False):

891 tagsdict = {x : 'todo' for x in tagsfiles }

892 default_answer = 'd'

893 else:

894 break

895

896 def diff_all_changes():

897 print(build.diff())

898 print(self.repository.diff())

899

900 def commit_all_changes(log):

901 if hasattr(self, 'branch'):

902 self.repository.commit(log, branch=self.branch)

903 else:

904 self.repository.commit(log)

905 build.commit(log)

906

907 self.run_prompt("Review module and build", diff_all_changes)

908 self.run_prompt("Commit module and build", commit_all_changes, changelog_strip)

909 self.run_prompt("Create tag", self.repository.tag, new_tag_name, changelog_strip)

910

911 if self.options.debug:

912 print('Preserving {} and stripped {}', changelog_plain, changelog_strip)

913 else:

914 os.unlink(changelog_plain)

915 os.unlink(changelog_strip)

916

917

918 ##############################

919 def do_version(self):

920 self.init_module_dir()

921 self.revert_module_dir()

922 self.update_module_dir()

923 spec_dict = self.spec_dict()

924 if self.options.www:

925 self.html_store_title('Version for module {} ({})'\

926 .format(self.friendly_name(), self.last_tag(spec_dict)))

927 for varname in self.varnames:

928 if varname not in spec_dict:

929 self.html_print('Could not find %define for {}'.format(varname))

930 return

931 else:

932 self.html_print("{:<16} {}".format(varname, spec_dict[varname]))

933 self.html_print("{:<16} {}".format('url', self.repository.url()))

934 if self.options.verbose:

935 self.html_print("{:<16} {}".format('main specfile:', self.main_specname()))

936 self.html_print("{:<16} {}".format('specfiles:', self.all_specnames()))

937 self.html_print_end()

938

939

940 ##############################

941 def do_diff(self):

942 self.init_module_dir()

943 self.revert_module_dir()

944 self.update_module_dir()

945 spec_dict = self.spec_dict()

946 self.show_dict(spec_dict)

947

948 # side effects

949 tag_name = self.tag_name(spec_dict)

950

951 # sanity check

952 tag_name = self.check_tag(tag_name, need_it=True)

953

954 if self.options.verbose:

955 print('Getting diff')

956 diff_output = self.repository.diff_with_tag(tag_name)

957

958 if self.options.list:

959 if diff_output:

960 print(self.pathname)

961 else:

962 thename = self.friendly_name()

963 do_print = False

964 if self.options.www and diff_output:

965 self.html_store_title("Diffs in module {} ({}) : {} chars"\

966 .format(thename, self.last_tag(spec_dict), len(diff_output)))

967

968 self.html_store_raw('<p> < (left) {} </p>'.format(tag_name))

969 self.html_store_raw('<p> > (right) {} </p>'.format(thename))

970 self.html_store_pre(diff_output)

971 elif not self.options.www:

972 print('x'*30, 'module', thename)

973 print('x'*20, '<', tag_name)

974 print('x'*20, '>', thename)

975 print(diff_output)

976

977 ##############################

978 # store and restitute html fragments

979 @staticmethod

980 def html_href(url,text):

981 return '<a href="{}">{}</a>'.format(url, text)

982

983 @staticmethod

984 def html_anchor(url,text):

985 return '<a name="{}">{}</a>'.format(url,text)

986

987 @staticmethod

988 def html_quote(text):

989 return text.replace('&', '&').replace('<', '<').replace('>', '>')

990

991 # only the fake error module has multiple titles

992 def html_store_title(self, title):

993 if not hasattr(self,'titles'):

994 self.titles=[]

995 self.titles.append(title)

996

997 def html_store_raw(self, html):

998 if not hasattr(self,'body'):

999 self.body=''

1000 self.body += html

1001

1002 def html_store_pre(self, text):

1003 if not hasattr(self,'body'):

1004 self.body=''

1005 self.body += '<pre>{}</pre>'.format(self.html_quote(text))

1006

1007 def html_print(self, txt):

1008 if not self.options.www:

1009 print(txt)

1010 else:

1011 if not hasattr(self, 'in_list') or not self.in_list:

1012 self.html_store_raw('<ul>')

1013 self.in_list = True

1014 self.html_store_raw('<li>{}</li>'.format(txt))

1015

1016 def html_print_end(self):

1017 if self.options.www:

1018 self.html_store_raw('</ul>')

1019

1020 @staticmethod

1021 def html_dump_header(title):

1022 nowdate = time.strftime("%Y-%m-%d")

1023 nowtime = time.strftime("%H:%M (%Z)")

1024 print("""<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

1025 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">

1026 <head>

1027 <title> {} </title>

1028 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

1029 <style type="text/css">

1030 body {{ font-family:georgia, serif; }}

1031 h1 {{font-size: large; }}

1032 p.title {{font-size: x-large; }}

1033 span.error {{text-weight:bold; color: red; }}

1034 </style>

1035 </head>

1036 <body>

1037 <p class='title'> {} - status on {} at {}</p>

1038 <ul>

1039 """.format(title, title, nowdate, nowtime))

1040

1041 @staticmethod

1042 def html_dump_middle():

1043 print("</ul>")

1044

1045 @staticmethod

1046 def html_dump_footer():

1047 print("</body></html")

1048

1049 def html_dump_toc(self):

1050 if hasattr(self,'titles'):

1051 for title in self.titles:

1052 print('<li>', self.html_href('#'+self.friendly_name(),title), '</li>')

1053

1054 def html_dump_body(self):

1055 if hasattr(self,'titles'):

1056 for title in self.titles:

1057 print('<hr /><h1>', self.html_anchor(self.friendly_name(),title), '</h1>')

1058 if hasattr(self,'body'):

1059 print(self.body)

1060 print('<p class="top">', self.html_href('#','Back to top'), '</p>')

1061

1062

1063

1064 class Build(Module):

1065

1066 def __get_modules(self, tagfile):

1067 self.init_module_dir()

1068 modules = {}

1069

1070 tagfile = os.path.join(self.module_dir, tagfile)

1071 for line in open(tagfile):

1072 try:

1073 name, url = line.split(':=')

1074 name, git_path = name.rsplit('-', 1)

1075 modules[name] = (git_path.strip(), url.strip())

1076 except:

1077 pass

1078 return modules

1079

1080 def get_modules(self, tagfile):

1081 modules = self.__get_modules(tagfile)

1082 for module in modules:

1083 module_type = tag_or_branch = ""

1084

1085 path_type, url = modules[module]

1086 if path_type == "GITPATH":

1087 module_spec = os.path.split(url)[-1].replace(".git","")

1088 name, tag_or_branch, module_type = self.parse_module_spec(module_spec)

1089 else:

1090 tag_or_branch = os.path.split(url)[-1].strip()

1091 if url.find('/tags/') >= 0:

1092 module_type = "tag"

1093 elif url.find('/branches/') >= 0:

1094 module_type = "branch"

1095

1096 modules[module] = {"module_type" : module_type,

1097 "path_type": path_type,

1098 "tag_or_branch": tag_or_branch,

1099 "url":url}

1100 return modules

1101

1102

1103

1104 def modules_diff(first, second):

1105 diff = {}

1106

1107 for module in first:

1108 if module not in second:

1109 print("=== module {} missing in right-hand side ===".format(module))

1110 continue

1111 if first[module]['tag_or_branch'] != second[module]['tag_or_branch']:

1112 diff[module] = (first[module]['tag_or_branch'], second[module]['tag_or_branch'])

1113

1114 first_set = set(first.keys())

1115 second_set = set(second.keys())

1116

1117 new_modules = list(second_set - first_set)

1118 removed_modules = list(first_set - second_set)

1119

1120 return diff, new_modules, removed_modules

1121

1122 def release_changelog(options, buildtag_old, buildtag_new):

1123

1124 # the command line expects new old, so we treat the tagfiles in the same order

1125 nb_tags = len(options.distrotags)

1126 if nb_tags == 1:

1127 tagfile_new = tagfile_old = options.distrotags[0]

1128 elif nb_tags == 2:

1129 tagfile_new, tagfile_old = options.distrotags

1130 else:

1131 print("ERROR: provide one or two tagfile name (eg. onelab-k32-tags.mk)")

1132 print("two tagfiles can be mentioned when a tagfile has been renamed")

1133 return

1134

1135 if options.dry_run:

1136 print("------------------------------ Computing Changelog from")

1137 print("buildtag_old", buildtag_old, "tagfile_old", tagfile_old)

1138 print("buildtag_new", buildtag_new, "tagfile_new", tagfile_new)

1139 return

1140

1141 print('----')

1142 print('----')

1143 print('----')

1144 print('= build tag {} to {} ='.format(buildtag_old, buildtag_new))

1145 print('== distro {} ({} to {}) =='.format(tagfile_new, buildtag_old, buildtag_new))

1146

1147 build = Build("build@{}".format(buildtag_old), options)

1148 build.init_module_dir()

1149 first = build.get_modules(tagfile_old)

1150

1151 print(' * from', buildtag_old, build.repository.gitweb())

1152

1153 build = Build("build@{}".format(buildtag_new), options)

1154 build.init_module_dir()

1155 second = build.get_modules(tagfile_new)

1156

1157 print(' * to', buildtag_new, build.repository.gitweb())

1158

1159 diff, new_modules, removed_modules = modules_diff(first, second)

1160

1161

1162 def get_module(name, tag):

1163 if not tag or tag == "trunk":

1164 return Module("{}".format(module), options)

1165 else:

1166 return Module("{}@{}".format(module, tag), options)

1167

1168

1169 for module in diff:

1170 print('=== {} - {} to {} : package {} ==='.format(tagfile_new, buildtag_old, buildtag_new, module))

1171

1172 first, second = diff[module]

1173 m = get_module(module, first)

1174 os.system('rm -rf {}'.format(m.module_dir)) # cleanup module dir

1175 m.init_module_dir()

1176

1177 print(' * from', first, m.repository.gitweb())

1178

1179 specfile = m.main_specname()

1180 (tmpfd, tmpfile) = tempfile.mkstemp()

1181 os.system("cp -f /{} {}".format(specfile, tmpfile))

1182

1183 m = get_module(module, second)

1184 # patch for ipfw that, being managed in a separate repo, won't work for now

1185 try:

1186 m.init_module_dir()

1187 except Exception as e:

1188 print("""Could not retrieve module {} - skipped

1189 {{{{{{ {} }}}}}}

1190 """.format( m.friendly_name(), e))

1191 continue

1192 specfile = m.main_specname()

1193

1194 print(' * to', second, m.repository.gitweb())

1195

1196 print('{{{')

1197 os.system("diff -u {} {} | sed -e 's,{},[[previous version]],'"\

1198 .format(tmpfile, specfile, tmpfile))

1199 print('}}}')

1200

1201 os.unlink(tmpfile)

1202

1203 for module in new_modules:

1204 print('=== {} : new package in build {} ==='.format(tagfile_new, module))

1205

1206 for module in removed_modules:

1207 print('=== {} : removed package from build {} ==='.format(tagfile_new, module))

1208

1209

1210 def adopt_tag(options, args):

1211 modules=[]

1212 for module in options.modules:

1213 modules += module.split()

1214 for module in modules:

1215 modobj=Module(module,options)

1216 for tags_file in args:

1217 if options.verbose:

1218 print('adopting tag {} for {} in {}'.format(options.tag, module, tags_file))

1219 modobj.patch_tags_file(tags_file, '_unused_', options.tag, fine_grain=False)

1220 if options.verbose:

1221 Command("git diff {}".format(" ".join(args)), options).run()

1222

1223 ##############################

1224 class Main:

1225

1226 module_usage="""Usage: %prog [options] module_desc [ .. module_desc ]

1227

1228 module-tools : a set of tools to manage subversion tags and specfile

1229 requires the specfile to either

1230 * define *version* and *taglevel*

1231 OR alternatively

1232 * define redirection variables module_version_varname / module_taglevel_varname

1233 Master:

1234 by default, the 'master' branch of modules is the target

1235 in this case, just mention the module name as <module_desc>

1236 Branches:

1237 if you wish to work on another branch,

1238 you can use something like e.g. Mom:2.1 as <module_desc>

1239 """

1240 release_usage="""Usage: %prog [options] tag1 .. tagn

1241 Extract release notes from the changes in specfiles between several build tags, latest first

1242 Examples:

1243 release-changelog 4.2-rc25 4.2-rc24 4.2-rc23 4.2-rc22

1244 You can refer to a (build) branch by prepending a colon, like in

1245 release-changelog :4.2 4.2-rc25

1246 You can refer to the build trunk by just mentioning 'trunk', e.g.

1247 release-changelog -t coblitz-tags.mk coblitz-2.01-rc6 trunk

1248 You can use 2 different tagfile names if that was renamed meanwhile

1249 release-changelog -t onelab-tags.mk 5.0-rc29 -t onelab-k32-tags.mk 5.0-rc28

1250 """

1251 adopt_usage="""Usage: %prog [options] tag-file[s]

1252 With this command you can adopt a specifi tag or branch in your tag files

1253 This should be run in your daily build workdir; no call of git is done

1254 Examples:

1255 adopt-tag -m "plewww plcapi" -m Monitor onelab*tags.mk

1256 adopt-tag -m sfa -t sfa-1.0-33 *tags.mk

1257 """

1258 common_usage="""More help:

1259 see http://svn.planet-lab.org/wiki/ModuleTools"""

1260

1261 modes = {

1262 'list' : "displays a list of available tags or branches",

1263 'version' : "check latest specfile and print out details",

1264 'diff' : "show difference between module (trunk or branch) and latest tag",

1265 'tag' : """increment taglevel in specfile, insert changelog in specfile,

1266 create new tag and and monitor its adoption in build/*-tags.mk""",

1267 'branch' : """create a branch for this module, from the latest tag on the trunk,

1268 and change trunk's version number to reflect the new branch name;

1269 you can specify the new branch name by using module:branch""",

1270 'sync' : """create a tag from the module

1271 this is a last resort option, mostly for repairs""",

1272 'changelog' : """extract changelog between build tags

1273 expected arguments are a list of tags""",

1274 'adopt' : """locally adopt a specific tag""",

1275 }

1276

1277 silent_modes = ['list']

1278 # 'changelog' is for release-changelog

1279 # 'adopt' is for 'adopt-tag'

1280 regular_modes = set(modes.keys()).difference(set(['changelog','adopt']))

1281

1282 @staticmethod

1283 def optparse_list(option, opt, value, parser):

1284 try:

1285 setattr(parser.values,option.dest,getattr(parser.values,option.dest)+value.split())

1286 except:

1287 setattr(parser.values,option.dest,value.split())

1288

1289 def run(self):

1290

1291 mode=None

1292 # hack - need to check for adopt first as 'adopt-tag' contains tag..

1293 for function in [ 'adopt' ] + list(Main.modes.keys()):

1294 if sys.argv[0].find(function) >= 0:

1295 mode = function

1296 break

1297 if not mode:

1298 print("Unsupported command",sys.argv[0])

1299 print("Supported commands:" + " ".join(list(Main.modes.keys())))

1300 sys.exit(1)

1301

1302 usage='undefined usage, mode={}'.format(mode)

1303 if mode in Main.regular_modes:

1304 usage = Main.module_usage

1305 usage += Main.common_usage

1306 usage += "\nmodule-{} : {}".format(mode, Main.modes[mode])

1307 elif mode == 'changelog':

1308 usage = Main.release_usage

1309 usage += Main.common_usage

1310 elif mode == 'adopt':

1311 usage = Main.adopt_usage

1312 usage += Main.common_usage

1313

1314 parser=OptionParser(usage=usage)

1315

1316 # the 'adopt' mode is really special and doesn't share any option

1317 if mode == 'adopt':

1318 parser.add_option("-m","--module",action="append",dest="modules",default=[],

1319 help="modules, can be used several times or with quotes")

1320 parser.add_option("-t","--tag",action="store", dest="tag", default='master',

1321 help="specify the tag to adopt, default is 'master'")

1322 parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,

1323 help="run in verbose mode")

1324 (options, args) = parser.parse_args()

1325 options.workdir='unused'

1326 options.dry_run=False

1327 options.mode='adopt'

1328 if len(args)==0 or len(options.modules)==0:

1329 parser.print_help()

1330 sys.exit(1)

1331 adopt_tag(options,args)

1332 return

1333

1334 # the other commands (module-* and release-changelog) share the same skeleton

1335 if mode in [ 'tag', 'branch'] :

1336 parser.add_option("-s","--set-version",action="store",dest="new_version",default=None,

1337 help="set new version and reset taglevel to 0")

1338 parser.add_option("-0","--bypass",action="store_true",dest="bypass",default=False,

1339 help="skip checks on existence of the previous tag")

1340 if mode == 'tag' :

1341 parser.add_option("-c","--no-changelog", action="store_false", dest="changelog", default=True,

1342 help="do not update changelog section in specfile when tagging")

1343 parser.add_option("-b","--build-branch", action="store", dest="build_branch", default=None,

1344 help="specify a build branch; used for locating the *tags.mk files where adoption is to take place")

1345 if mode in [ 'tag', 'sync' ] :

1346 parser.add_option("-e","--editor", action="store", dest="editor", default=default_editor(),

1347 help="specify editor")

1348

1349 if mode in ['diff','version'] :

1350 parser.add_option("-W","--www", action="store", dest="www", default=False,

1351 help="export diff in html format, e.g. -W trunk")

1352

1353 if mode == 'diff' :

1354 parser.add_option("-l","--list", action="store_true", dest="list", default=False,

1355 help="just list modules that exhibit differences")

1356

1357 default_modules_list=os.path.dirname(sys.argv[0])+"/modules.list"

1358 parser.add_option("-a","--all",action="store_true",dest="all_modules",default=False,

1359 help="run on all modules as found in {}".format(default_modules_list))

1360 parser.add_option("-f","--file",action="store",dest="modules_list",default=None,

1361 help="run on all modules found in specified file")

1362 parser.add_option("-n","--dry-run",action="store_true",dest="dry_run",default=False,

1363 help="dry run - shell commands are only displayed")

1364 parser.add_option("-t","--distrotags",action="callback",callback=Main.optparse_list, dest="distrotags",

1365 default=[], nargs=1,type="string",

1366 help="""specify distro-tags files, e.g. onelab-tags-4.2.mk

1367 -- can be set multiple times, or use quotes""")

1368

1369 parser.add_option("-w","--workdir", action="store", dest="workdir",

1370 default="{}/{}".format(os.getenv("HOME"),"modules"),

1371 help="""name for dedicated working dir - defaults to ~/modules

1372 ** THIS MUST NOT ** be your usual working directory""")

1373 parser.add_option("-F","--fast-checks",action="store_true",dest="fast_checks",default=False,

1374 help="skip safety checks, such as git pulls -- use with care")

1375 parser.add_option("-B","--build-module",action="store",dest="build_module",default=None,

1376 help="specify a build module to owerride the one in the CONFIG")

1377

1378 # default verbosity depending on function - temp

1379 verbose_modes= ['tag', 'sync', 'branch']

1380

1381 if mode not in verbose_modes:

1382 parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,

1383 help="run in verbose mode")

1384 else:

1385 parser.add_option("-q","--quiet", action="store_false", dest="verbose", default=True,

1386 help="run in quiet (non-verbose) mode")

1387 options, args = parser.parse_args()

1388 options.mode=mode

1389 if not hasattr(options,'dry_run'):

1390 options.dry_run=False

1391 if not hasattr(options,'www'):

1392 options.www=False

1393 options.debug=False

1394

1395

1396

1397 ########## module-*

1398 if len(args) == 0:

1399 if options.all_modules:

1400 options.modules_list=default_modules_list

1401 if options.modules_list:

1402 args=Command("grep -v '#' {}".format(options.modules_list), options).output_of().split()

1403 else:

1404 parser.print_help()

1405 sys.exit(1)

1406 Module.init_homedir(options)

1407

1408

1409 if mode in Main.regular_modes:

1410 modules = [ Module(modname, options) for modname in args ]

1411 # hack: create a dummy Module to store errors/warnings

1412 error_module = Module('__errors__',options)

1413

1414 for module in modules:

1415 if len(args)>1 and mode not in Main.silent_modes:

1416 if not options.www:

1417 print('========================================', module.friendly_name())

1418 # call the method called do_<mode>

1419 method = Module.__dict__["do_{}".format(mode)]

1420 try:

1421 method(module)

1422 except Exception as e:

1423 if options.www:

1424 title='<span class="error"> Skipping module {} - failure: {} </span>'\

1425 .format(module.friendly_name(), str(e))

1426 error_module.html_store_title(title)

1427 else:

1428 import traceback

1429 traceback.print_exc()

1430 print('Skipping module {}: {}'.format(module.name,e))

1431

1432 if options.www:

1433 if mode == "diff":

1434 modetitle="Changes to tag in {}".format(options.www)

1435 elif mode == "version":

1436 modetitle="Latest tags in {}".format(options.www)

1437 modules.append(error_module)

1438 error_module.html_dump_header(modetitle)

1439 for module in modules:

1440 module.html_dump_toc()

1441 Module.html_dump_middle()

1442 for module in modules:

1443 module.html_dump_body()

1444 Module.html_dump_footer()

1445 else:

1446 # if we provide, say a b c d, we want to build (a,b) (b,c) and (c,d)

1447 # remember that the changelog in the twiki comes latest first, so

1448 # we typically have here latest latest-1 latest-2

1449 for (tag_new,tag_old) in zip ( args[:-1], args [1:]):

1450 release_changelog(options, tag_old, tag_new)

1451

1452

1453 ####################

1454 if __name__ == "__main__" :

1455 try:

1456 Main().run()

1457 except KeyboardInterrupt:

1458 print('\nBye')