support registering and updating records without using files
[sfa.git] / sfa / client / sfi.py
1 #
2 # sfi.py - basic SFA command-line client
3 # the actual binary in sfa/clientbin essentially runs main()
4 # this module is used in sfascan
5 #
6
7 import sys
8 sys.path.append('.')
9
10 import os, os.path
11 import socket
12 import datetime
13 import codecs
14 import pickle
15 import json
16 from lxml import etree
17 from StringIO import StringIO
18 from optparse import OptionParser
19 from pprint import PrettyPrinter
20
21 from sfa.trust.certificate import Keypair, Certificate
22 from sfa.trust.gid import GID
23 from sfa.trust.credential import Credential
24 from sfa.trust.sfaticket import SfaTicket
25
26 from sfa.util.sfalogging import sfi_logger
27 from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn, Xrn
28 from sfa.util.config import Config
29 from sfa.util.version import version_core
30 from sfa.util.cache import Cache
31
32 from sfa.storage.record import Record
33
34 from sfa.rspecs.rspec import RSpec
35 from sfa.rspecs.rspec_converter import RSpecConverter
36 from sfa.rspecs.version_manager import VersionManager
37
38 from sfa.client.sfaclientlib import SfaClientBootstrap
39 from sfa.client.sfaserverproxy import SfaServerProxy, ServerException
40 from sfa.client.client_helper import pg_users_arg, sfa_users_arg
41 from sfa.client.return_value import ReturnValue
42
43 CM_PORT=12346
44
45 # utility methods here
46 def optparse_listvalue_callback(option, opt, value, parser):
47     setattr(parser.values, option.dest, value.split(','))
48
49 # display methods
50 def display_rspec(rspec, format='rspec'):
51     if format in ['dns']:
52         tree = etree.parse(StringIO(rspec))
53         root = tree.getroot()
54         result = root.xpath("./network/site/node/hostname/text()")
55     elif format in ['ip']:
56         # The IP address is not yet part of the new RSpec
57         # so this doesn't do anything yet.
58         tree = etree.parse(StringIO(rspec))
59         root = tree.getroot()
60         result = root.xpath("./network/site/node/ipv4/text()")
61     else:
62         result = rspec
63
64     print result
65     return
66
67 def display_list(results):
68     for result in results:
69         print result
70
71 def display_records(recordList, dump=False):
72     ''' Print all fields in the record'''
73     for record in recordList:
74         display_record(record, dump)
75
76 def display_record(record, dump=False):
77     if dump:
78         record.dump()
79     else:
80         info = record.getdict()
81         print "%s (%s)" % (info['hrn'], info['type'])
82     return
83
84
85 def filter_records(type, records):
86     filtered_records = []
87     for record in records:
88         if (record['type'] == type) or (type == "all"):
89             filtered_records.append(record)
90     return filtered_records
91
92
93 # save methods
94 def save_raw_to_file(var, filename, format="text", banner=None):
95     if filename == "-":
96         # if filename is "-", send it to stdout
97         f = sys.stdout
98     else:
99         f = open(filename, "w")
100     if banner:
101         f.write(banner+"\n")
102     if format == "text":
103         f.write(str(var))
104     elif format == "pickled":
105         f.write(pickle.dumps(var))
106     elif format == "json":
107         if hasattr(json, "dumps"):
108             f.write(json.dumps(var))   # python 2.6
109         else:
110             f.write(json.write(var))   # python 2.5
111     else:
112         # this should never happen
113         print "unknown output format", format
114     if banner:
115         f.write('\n'+banner+"\n")
116
117 def save_rspec_to_file(rspec, filename):
118     if not filename.endswith(".rspec"):
119         filename = filename + ".rspec"
120     f = open(filename, 'w')
121     f.write(rspec)
122     f.close()
123     return
124
125 def save_records_to_file(filename, record_dicts, format="xml"):
126     if format == "xml":
127         index = 0
128         for record_dict in record_dicts:
129             if index > 0:
130                 save_record_to_file(filename + "." + str(index), record_dict)
131             else:
132                 save_record_to_file(filename, record_dict)
133             index = index + 1
134     elif format == "xmllist":
135         f = open(filename, "w")
136         f.write("<recordlist>\n")
137         for record_dict in record_dicts:
138             record_obj=Record(dict=record_dict)
139             f.write('<record hrn="' + record_obj.hrn + '" type="' + record_obj.type + '" />\n')
140         f.write("</recordlist>\n")
141         f.close()
142     elif format == "hrnlist":
143         f = open(filename, "w")
144         for record_dict in record_dicts:
145             record_obj=Record(dict=record_dict)
146             f.write(record_obj.hrn + "\n")
147         f.close()
148     else:
149         # this should never happen
150         print "unknown output format", format
151
152 def save_record_to_file(filename, record_dict):
153     record = Record(dict=record_dict)
154     xml = record.save_as_xml()
155     f=codecs.open(filename, encoding='utf-8',mode="w")
156     f.write(xml)
157     f.close()
158     return
159
160
161 # load methods
162 def load_record_from_opts(options):
163     record_dict = {}
164     if hasattr(options, 'xrn') and options.xrn:
165         if hasattr(options, 'type') and options.type:
166             xrn = Xrn(options.xrn, options.type)
167         else:
168             xrn = Xrn(options.xrn)
169         record_dict['urn'] = xrn.get_urn()
170         record_dict['hrn'] = xrn.get_hrn()
171         record_dict['type'] = xrn.get_type()
172     if hasattr(options, 'url') and options.url:
173         record_dict['url'] = options.url
174     if hasattr(options, 'description') and options.description:
175         record_dict['description'] = options.description
176     if hasattr(options, 'key') and options.key:
177         try:
178             pubkey = open(options.key, 'r').read()
179         except IOError:
180             pubkey = options.key
181         record_dict['keys'] = [pubkey]
182     if hasattr(options, 'slices') and options.slices:
183         record_dict['slices'] = options.slices
184     if hasattr(options, 'researchers') and options.researchers:
185         record_dict['researcher'] = options.researchers
186     if hasattr(options, 'email') and options.email:
187         record_dict['email'] = options.email
188     if hasattr(options, 'pis') and options.pis:
189         record_dict['pi'] = options.pis
190
191     # fill in the blanks
192     if record_dict['type'] == 'user':
193         if not 'first_name' in record_dict:
194             record_dict['first_name'] = record_dict['hrn']
195         if 'last_name' not in record_dict:
196             record_dict['last_name'] = record_dict['hrn'] 
197
198     return Record(dict=record_dict)
199
200 def load_record_from_file(filename):
201     f=codecs.open(filename, encoding="utf-8", mode="r")
202     xml_string = f.read()
203     f.close()
204     return Record(xml=xml_string)
205
206
207 import uuid
208 def unique_call_id(): return uuid.uuid4().urn
209
210 class Sfi:
211     
212     # dirty hack to make this class usable from the outside
213     required_options=['verbose',  'debug',  'registry',  'sm',  'auth',  'user', 'user_private_key']
214
215     @staticmethod
216     def default_sfi_dir ():
217         if os.path.isfile("./sfi_config"): 
218             return os.getcwd()
219         else:
220             return os.path.expanduser("~/.sfi/")
221
222     # dummy to meet Sfi's expectations for its 'options' field
223     # i.e. s/t we can do setattr on
224     class DummyOptions:
225         pass
226
227     def __init__ (self,options=None):
228         if options is None: options=Sfi.DummyOptions()
229         for opt in Sfi.required_options:
230             if not hasattr(options,opt): setattr(options,opt,None)
231         if not hasattr(options,'sfi_dir'): options.sfi_dir=Sfi.default_sfi_dir()
232         self.options = options
233         self.user = None
234         self.authority = None
235         self.logger = sfi_logger
236         self.logger.enable_console()
237         self.available_names = [ tuple[0] for tuple in Sfi.available ]
238         self.available_dict = dict (Sfi.available)
239    
240     # tuples command-name expected-args in the order in which they should appear in the help
241     available = [ 
242         ("version", ""),  
243         ("list", "authority"),
244         ("show", "name"),
245         ("add", "record"),
246         ("update", "record"),
247         ("remove", "name"),
248         ("slices", ""),
249         ("resources", "[slice_hrn]"),
250         ("create", "slice_hrn rspec"),
251         ("delete", "slice_hrn"),
252         ("status", "slice_hrn"),
253         ("start", "slice_hrn"),
254         ("stop", "slice_hrn"),
255         ("reset", "slice_hrn"),
256         ("renew", "slice_hrn time"),
257         ("shutdown", "slice_hrn"),
258         ("get_ticket", "slice_hrn rspec"),
259         ("redeem_ticket", "ticket"),
260         ("delegate", "name"),
261         ("create_gid", "[name]"),
262         ("get_trusted_certs", "cred"),
263         ("config", ""),
264         ]
265
266     def print_command_help (self, options):
267         verbose=getattr(options,'verbose')
268         format3="%18s %-15s %s"
269         line=80*'-'
270         if not verbose:
271             print format3%("command","cmd_args","description")
272             print line
273         else:
274             print line
275             self.create_parser().print_help()
276         for command in self.available_names:
277             args=self.available_dict[command]
278             method=getattr(self,command,None)
279             doc=""
280             if method: doc=getattr(method,'__doc__',"")
281             if not doc: doc="*** no doc found ***"
282             doc=doc.strip(" \t\n")
283             doc=doc.replace("\n","\n"+35*' ')
284             if verbose:
285                 print line
286             print format3%(command,args,doc)
287             if verbose:
288                 self.create_command_parser(command).print_help()
289
290     def create_command_parser(self, command):
291         if command not in self.available_dict:
292             msg="Invalid command\n"
293             msg+="Commands: "
294             msg += ','.join(self.available_names)            
295             self.logger.critical(msg)
296             sys.exit(2)
297
298         parser = OptionParser(usage="sfi [sfi_options] %s [cmd_options] %s" \
299                                      % (command, self.available_dict[command]))
300
301         if command in ("add", "update"):
302             parser.add_option('-x', '--xrn', dest='xrn', metavar='<xrn>', help='object hrn/urn (mandatory)')
303             parser.add_option('-t', '--type', dest='type', metavar='<type>', help='object type', default=None)
304             parser.add_option('-e', '--email', dest='email', default="",  help="email (mandatory for users)") 
305             parser.add_option('-u', '--url', dest='url', metavar='<url>', default=None, help="URL, useful for slices") 
306             parser.add_option('-d', '--description', dest='description', metavar='<description>', 
307                               help='Description, useful for slices', default=None)
308             parser.add_option('-k', '--key', dest='key', metavar='<key>', help='public key string or file', 
309                               default=None)
310             parser.add_option('-s', '--slices', dest='slices', metavar='<slices>', help='slice xrns',
311                               default='', type="str", action='callback', callback=optparse_listvalue_callback)
312             parser.add_option('-r', '--researchers', dest='researchers', metavar='<researchers>', 
313                               help='slice researchers', default='', type="str", action='callback', 
314                               callback=optparse_listvalue_callback)
315             parser.add_option('-p', '--pis', dest='pis', metavar='<PIs>', help='Principal Investigators/Project Managers',
316                               default='', type="str", action='callback', callback=optparse_listvalue_callback)
317             parser.add_option('-f', '--firstname', dest='firstname', metavar='<firstname>', help='user first name')
318             parser.add_option('-l', '--lastname', dest='lastname', metavar='<firstname>', help='user last name')
319
320         # user specifies remote aggregate/sm/component                          
321         if command in ("resources", "slices", "create", "delete", "start", "stop", 
322                        "restart", "shutdown",  "get_ticket", "renew", "status"):
323             parser.add_option("-d", "--delegate", dest="delegate", default=None, 
324                              action="store_true",
325                              help="Include a credential delegated to the user's root"+\
326                                   "authority in set of credentials for this call")
327
328         # registy filter option
329         if command in ("list", "show", "remove"):
330             parser.add_option("-t", "--type", dest="type", type="choice",
331                             help="type filter ([all]|user|slice|authority|node|aggregate)",
332                             choices=("all", "user", "slice", "authority", "node", "aggregate"),
333                             default="all")
334         if command in ("resources"):
335             # rspec version
336             parser.add_option("-r", "--rspec-version", dest="rspec_version", default="SFA 1",
337                               help="schema type and version of resulting RSpec")
338             # disable/enable cached rspecs
339             parser.add_option("-c", "--current", dest="current", default=False,
340                               action="store_true",  
341                               help="Request the current rspec bypassing the cache. Cached rspecs are returned by default")
342             # display formats
343             parser.add_option("-f", "--format", dest="format", type="choice",
344                              help="display format ([xml]|dns|ip)", default="xml",
345                              choices=("xml", "dns", "ip"))
346             #panos: a new option to define the type of information about resources a user is interested in
347             parser.add_option("-i", "--info", dest="info",
348                                 help="optional component information", default=None)
349
350
351         # 'create' does return the new rspec, makes sense to save that too
352         if command in ("resources", "show", "list", "create_gid", 'create'):
353            parser.add_option("-o", "--output", dest="file",
354                             help="output XML to file", metavar="FILE", default=None)
355
356         if command in ("show", "list"):
357            parser.add_option("-f", "--format", dest="format", type="choice",
358                              help="display format ([text]|xml)", default="text",
359                              choices=("text", "xml"))
360
361            parser.add_option("-F", "--fileformat", dest="fileformat", type="choice",
362                              help="output file format ([xml]|xmllist|hrnlist)", default="xml",
363                              choices=("xml", "xmllist", "hrnlist"))
364         if command == 'list':
365            parser.add_option("-r", "--recursive", dest="recursive", action='store_true',
366                              help="list all child records", default=False)
367         if command in ("delegate"):
368            parser.add_option("-u", "--user",
369                             action="store_true", dest="delegate_user", default=False,
370                             help="delegate user credential")
371            parser.add_option("-s", "--slice", dest="delegate_slice",
372                             help="delegate slice credential", metavar="HRN", default=None)
373         
374         if command in ("version"):
375             parser.add_option("-R","--registry-version",
376                               action="store_true", dest="version_registry", default=False,
377                               help="probe registry version instead of sliceapi")
378             parser.add_option("-l","--local",
379                               action="store_true", dest="version_local", default=False,
380                               help="display version of the local client")
381
382         return parser
383
384         
385     def create_parser(self):
386
387         # Generate command line parser
388         parser = OptionParser(usage="sfi [sfi_options] command [cmd_options] [cmd_args]",
389                              description="Commands: %s"%(" ".join(self.available_names)))
390         parser.add_option("-r", "--registry", dest="registry",
391                          help="root registry", metavar="URL", default=None)
392         parser.add_option("-s", "--sliceapi", dest="sm", default=None, metavar="URL",
393                          help="slice API - in general a SM URL, but can be used to talk to an aggregate")
394         parser.add_option("-R", "--raw", dest="raw", default=None,
395                           help="Save raw, unparsed server response to a file")
396         parser.add_option("", "--rawformat", dest="rawformat", type="choice",
397                           help="raw file format ([text]|pickled|json)", default="text",
398                           choices=("text","pickled","json"))
399         parser.add_option("", "--rawbanner", dest="rawbanner", default=None,
400                           help="text string to write before and after raw output")
401         parser.add_option("-d", "--dir", dest="sfi_dir",
402                          help="config & working directory - default is %default",
403                          metavar="PATH", default=Sfi.default_sfi_dir())
404         parser.add_option("-u", "--user", dest="user",
405                          help="user name", metavar="HRN", default=None)
406         parser.add_option("-a", "--auth", dest="auth",
407                          help="authority name", metavar="HRN", default=None)
408         parser.add_option("-v", "--verbose", action="count", dest="verbose", default=0,
409                          help="verbose mode - cumulative")
410         parser.add_option("-D", "--debug",
411                           action="store_true", dest="debug", default=False,
412                           help="Debug (xml-rpc) protocol messages")
413         # would it make sense to use ~/.ssh/id_rsa as a default here ?
414         parser.add_option("-k", "--private-key",
415                          action="store", dest="user_private_key", default=None,
416                          help="point to the private key file to use if not yet installed in sfi_dir")
417         parser.add_option("-t", "--timeout", dest="timeout", default=None,
418                          help="Amout of time to wait before timing out the request")
419         parser.add_option("-?", "--commands", 
420                          action="store_true", dest="command_help", default=False,
421                          help="one page summary on commands & exit")
422         parser.disable_interspersed_args()
423
424         return parser
425         
426
427     def print_help (self):
428         self.sfi_parser.print_help()
429         self.command_parser.print_help()
430
431     #
432     # Main: parse arguments and dispatch to command
433     #
434     def dispatch(self, command, command_options, command_args):
435         return getattr(self, command)(command_options, command_args)
436
437     def main(self):
438         self.sfi_parser = self.create_parser()
439         (options, args) = self.sfi_parser.parse_args()
440         if options.command_help: 
441             self.print_command_help(options)
442             sys.exit(1)
443         self.options = options
444
445         self.logger.setLevelFromOptVerbose(self.options.verbose)
446
447         if len(args) <= 0:
448             self.logger.critical("No command given. Use -h for help.")
449             self.print_command_help(options)
450             return -1
451     
452         command = args[0]
453         self.command_parser = self.create_command_parser(command)
454         (command_options, command_args) = self.command_parser.parse_args(args[1:])
455         self.command_options = command_options
456
457         self.read_config () 
458         self.bootstrap ()
459         self.logger.info("Command=%s" % command)
460
461         try:
462             self.dispatch(command, command_options, command_args)
463         except KeyError:
464             self.logger.critical ("Unknown command %s"%command)
465             raise
466             sys.exit(1)
467     
468         return
469     
470     ####################
471     def read_config(self):
472         config_file = os.path.join(self.options.sfi_dir,"sfi_config")
473         try:
474            config = Config (config_file)
475         except:
476            self.logger.critical("Failed to read configuration file %s"%config_file)
477            self.logger.info("Make sure to remove the export clauses and to add quotes")
478            if self.options.verbose==0:
479                self.logger.info("Re-run with -v for more details")
480            else:
481                self.logger.log_exc("Could not read config file %s"%config_file)
482            sys.exit(1)
483      
484         errors = 0
485         # Set SliceMgr URL
486         if (self.options.sm is not None):
487            self.sm_url = self.options.sm
488         elif hasattr(config, "SFI_SM"):
489            self.sm_url = config.SFI_SM
490         else:
491            self.logger.error("You need to set e.g. SFI_SM='http://your.slicemanager.url:12347/' in %s" % config_file)
492            errors += 1 
493
494         # Set Registry URL
495         if (self.options.registry is not None):
496            self.reg_url = self.options.registry
497         elif hasattr(config, "SFI_REGISTRY"):
498            self.reg_url = config.SFI_REGISTRY
499         else:
500            self.logger.errors("You need to set e.g. SFI_REGISTRY='http://your.registry.url:12345/' in %s" % config_file)
501            errors += 1 
502
503         # Set user HRN
504         if (self.options.user is not None):
505            self.user = self.options.user
506         elif hasattr(config, "SFI_USER"):
507            self.user = config.SFI_USER
508         else:
509            self.logger.errors("You need to set e.g. SFI_USER='plc.princeton.username' in %s" % config_file)
510            errors += 1 
511
512         # Set authority HRN
513         if (self.options.auth is not None):
514            self.authority = self.options.auth
515         elif hasattr(config, "SFI_AUTH"):
516            self.authority = config.SFI_AUTH
517         else:
518            self.logger.error("You need to set e.g. SFI_AUTH='plc.princeton' in %s" % config_file)
519            errors += 1 
520
521         self.config_file=config_file
522         if errors:
523            sys.exit(1)
524
525     def show_config (self):
526         print "From configuration file %s"%self.config_file
527         flags=[ 
528             ('SFI_USER','user'),
529             ('SFI_AUTH','authority'),
530             ('SFI_SM','sm_url'),
531             ('SFI_REGISTRY','reg_url'),
532             ]
533         for (external_name, internal_name) in flags:
534             print "%s='%s'"%(external_name,getattr(self,internal_name))
535
536     #
537     # Get various credential and spec files
538     #
539     # Establishes limiting conventions
540     #   - conflates MAs and SAs
541     #   - assumes last token in slice name is unique
542     #
543     # Bootstraps credentials
544     #   - bootstrap user credential from self-signed certificate
545     #   - bootstrap authority credential from user credential
546     #   - bootstrap slice credential from user credential
547     #
548     
549     # init self-signed cert, user credentials and gid
550     def bootstrap (self):
551         client_bootstrap = SfaClientBootstrap (self.user, self.reg_url, self.options.sfi_dir)
552         # if -k is provided, use this to initialize private key
553         if self.options.user_private_key:
554             client_bootstrap.init_private_key_if_missing (self.options.user_private_key)
555         else:
556             # trigger legacy compat code if needed 
557             # the name has changed from just <leaf>.pkey to <hrn>.pkey
558             if not os.path.isfile(client_bootstrap.private_key_filename()):
559                 self.logger.info ("private key not found, trying legacy name")
560                 try:
561                     legacy_private_key = os.path.join (self.options.sfi_dir, "%s.pkey"%get_leaf(self.user))
562                     self.logger.debug("legacy_private_key=%s"%legacy_private_key)
563                     client_bootstrap.init_private_key_if_missing (legacy_private_key)
564                     self.logger.info("Copied private key from legacy location %s"%legacy_private_key)
565                 except:
566                     self.logger.log_exc("Can't find private key ")
567                     sys.exit(1)
568             
569         # make it bootstrap
570         client_bootstrap.bootstrap_my_gid()
571         # extract what's needed
572         self.private_key = client_bootstrap.private_key()
573         self.my_credential_string = client_bootstrap.my_credential_string ()
574         self.my_gid = client_bootstrap.my_gid ()
575         self.client_bootstrap = client_bootstrap
576
577
578     def my_authority_credential_string(self):
579         if not self.authority:
580             self.logger.critical("no authority specified. Use -a or set SF_AUTH")
581             sys.exit(-1)
582         return self.client_bootstrap.authority_credential_string (self.authority)
583
584     def slice_credential_string(self, name):
585         return self.client_bootstrap.slice_credential_string (name)
586
587     # xxx should be supported by sfaclientbootstrap as well
588     def delegate_cred(self, object_cred, hrn, type='authority'):
589         # the gid and hrn of the object we are delegating
590         if isinstance(object_cred, str):
591             object_cred = Credential(string=object_cred) 
592         object_gid = object_cred.get_gid_object()
593         object_hrn = object_gid.get_hrn()
594     
595         if not object_cred.get_privileges().get_all_delegate():
596             self.logger.error("Object credential %s does not have delegate bit set"%object_hrn)
597             return
598
599         # the delegating user's gid
600         caller_gidfile = self.my_gid()
601   
602         # the gid of the user who will be delegated to
603         delegee_gid = self.client_bootstrap.gid(hrn,type)
604         delegee_hrn = delegee_gid.get_hrn()
605         dcred = object_cred.delegate(delegee_gid, self.private_key, caller_gidfile)
606         return dcred.save_to_string(save_parents=True)
607      
608     #
609     # Management of the servers
610     # 
611
612     def registry (self):
613         # cache the result
614         if not hasattr (self, 'registry_proxy'):
615             self.logger.info("Contacting Registry at: %s"%self.reg_url)
616             self.registry_proxy = SfaServerProxy(self.reg_url, self.private_key, self.my_gid, 
617                                                  timeout=self.options.timeout, verbose=self.options.debug)  
618         return self.registry_proxy
619
620     def sliceapi (self):
621         # cache the result
622         if not hasattr (self, 'sliceapi_proxy'):
623             # if the command exposes the --component option, figure it's hostname and connect at CM_PORT
624             if hasattr(self.command_options,'component') and self.command_options.component:
625                 # resolve the hrn at the registry
626                 node_hrn = self.command_options.component
627                 records = self.registry().Resolve(node_hrn, self.my_credential_string)
628                 records = filter_records('node', records)
629                 if not records:
630                     self.logger.warning("No such component:%r"% opts.component)
631                 record = records[0]
632                 cm_url = "http://%s:%d/"%(record['hostname'],CM_PORT)
633                 self.sliceapi_proxy=SfaServerProxy(cm_url, self.private_key, self.my_gid)
634             else:
635                 # otherwise use what was provided as --sliceapi, or SFI_SM in the config
636                 if not self.sm_url.startswith('http://') or self.sm_url.startswith('https://'):
637                     self.sm_url = 'http://' + self.sm_url
638                 self.logger.info("Contacting Slice Manager at: %s"%self.sm_url)
639                 self.sliceapi_proxy = SfaServerProxy(self.sm_url, self.private_key, self.my_gid, 
640                                                      timeout=self.options.timeout, verbose=self.options.debug)  
641         return self.sliceapi_proxy
642
643     def get_cached_server_version(self, server):
644         # check local cache first
645         cache = None
646         version = None 
647         cache_file = os.path.join(self.options.sfi_dir,'sfi_cache.dat')
648         cache_key = server.url + "-version"
649         try:
650             cache = Cache(cache_file)
651         except IOError:
652             cache = Cache()
653             self.logger.info("Local cache not found at: %s" % cache_file)
654
655         if cache:
656             version = cache.get(cache_key)
657
658         if not version: 
659             result = server.GetVersion()
660             version= ReturnValue.get_value(result)
661             # cache version for 20 minutes
662             cache.add(cache_key, version, ttl= 60*20)
663             self.logger.info("Updating cache file %s" % cache_file)
664             cache.save_to_file(cache_file)
665
666         return version   
667         
668     ### resurrect this temporarily so we can support V1 aggregates for a while
669     def server_supports_options_arg(self, server):
670         """
671         Returns true if server support the optional call_id arg, false otherwise. 
672         """
673         server_version = self.get_cached_server_version(server)
674         result = False
675         # xxx need to rewrite this 
676         if int(server_version.get('geni_api')) >= 2:
677             result = True
678         return result
679
680     def server_supports_call_id_arg(self, server):
681         server_version = self.get_cached_server_version(server)
682         result = False      
683         if 'sfa' in server_version and 'code_tag' in server_version:
684             code_tag = server_version['code_tag']
685             code_tag_parts = code_tag.split("-")
686             version_parts = code_tag_parts[0].split(".")
687             major, minor = version_parts[0], version_parts[1]
688             rev = code_tag_parts[1]
689             if int(major) == 1 and minor == 0 and build >= 22:
690                 result = True
691         return result                 
692
693     ### ois = options if supported
694     # to be used in something like serverproxy.Method (arg1, arg2, *self.ois(api_options))
695     def ois (self, server, option_dict):
696         if self.server_supports_options_arg (server): 
697             return [option_dict]
698         elif self.server_supports_call_id_arg (server):
699             return [ unique_call_id () ]
700         else: 
701             return []
702
703     ### cis = call_id if supported - like ois
704     def cis (self, server):
705         if self.server_supports_call_id_arg (server):
706             return [ unique_call_id ]
707         else:
708             return []
709
710     ######################################## miscell utilities
711     def get_rspec_file(self, rspec):
712        if (os.path.isabs(rspec)):
713           file = rspec
714        else:
715           file = os.path.join(self.options.sfi_dir, rspec)
716        if (os.path.isfile(file)):
717           return file
718        else:
719           self.logger.critical("No such rspec file %s"%rspec)
720           sys.exit(1)
721     
722     def get_record_file(self, record):
723        if (os.path.isabs(record)):
724           file = record
725        else:
726           file = os.path.join(self.options.sfi_dir, record)
727        if (os.path.isfile(file)):
728           return file
729        else:
730           self.logger.critical("No such registry record file %s"%record)
731           sys.exit(1)
732
733
734     #==========================================================================
735     # Following functions implement the commands
736     #
737     # Registry-related commands
738     #==========================================================================
739
740     def version(self, options, args):
741         """
742         display an SFA server version (GetVersion)
743 or version information about sfi itself
744         """
745         if options.version_local:
746             version=version_core()
747         else:
748             if options.version_registry:
749                 server=self.registry()
750             else:
751                 server = self.sliceapi()
752             result = server.GetVersion()
753             version = ReturnValue.get_value(result)
754         if self.options.raw:
755             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
756         else:
757             pprinter = PrettyPrinter(indent=4)
758             pprinter.pprint(version)
759
760     def list(self, options, args):
761         """
762         list entries in named authority registry (List)
763         """
764         if len(args)!= 1:
765             self.print_help()
766             sys.exit(1)
767         hrn = args[0]
768         opts = {}
769         if options.recursive:
770             opts['recursive'] = options.recursive
771         
772         try:
773             list = self.registry().List(hrn, self.my_credential_string, options)
774         except IndexError:
775             raise Exception, "Not enough parameters for the 'list' command"
776
777         # filter on person, slice, site, node, etc.
778         # THis really should be in the self.filter_records funct def comment...
779         list = filter_records(options.type, list)
780         for record in list:
781             print "%s (%s)" % (record['hrn'], record['type'])
782         if options.file:
783             save_records_to_file(options.file, list, options.fileformat)
784         return
785     
786     def show(self, options, args):
787         """
788         show details about named registry record (Resolve)
789         """
790         if len(args)!= 1:
791             self.print_help()
792             sys.exit(1)
793         hrn = args[0]
794         record_dicts = self.registry().Resolve(hrn, self.my_credential_string)
795         record_dicts = filter_records(options.type, record_dicts)
796         if not record_dicts:
797             self.logger.error("No record of type %s"% options.type)
798         records = [ Record(dict=record_dict) for record_dict in record_dicts ]
799         for record in records:
800             if (options.format == "text"):      record.dump()  
801             else:                               print record.save_as_xml() 
802         if options.file:
803             save_records_to_file(options.file, record_dicts, options.fileformat)
804         return
805     
806     def add(self, options, args):
807         "add record into registry from xml file (Register)"
808         auth_cred = self.my_authority_credential_string()
809         record = {}
810         if len(args) > 0:
811             record_filepath = args[0]
812             rec_file = self.get_record_file(record_filepath)
813             record.update(load_record_from_file(rec_file).todict())
814         if options:
815             record.update(load_record_from_opts(options).todict())
816         if not record:
817             self.print_help()
818             sys.exit(1)
819         return self.registry().Register(record, auth_cred)
820     
821     def update(self, options, args):
822         "update record into registry from xml file (Update)"
823         record_dict = {}
824         if len(args) > 0:
825             record_filepath = args[0]
826             rec_file = self.get_record_file(record_filepath)
827             record_dict.update(load_record_from_file(rec_file).todict())
828         if options:
829             record_dict.update(load_record_from_opts(options).todict())
830         if not record_dict:
831             self.print_help()
832             sys.exit(1)
833
834         record = Record(dict=record_dict)        
835         if record.type == "user":
836             if record.hrn == self.user:
837                 cred = self.my_credential_string
838             else:
839                 cred = self.my_authority_credential_string()
840         elif record.type in ["slice"]:
841             try:
842                 cred = self.slice_credential_string(record.hrn)
843             except ServerException, e:
844                # XXX smbaker -- once we have better error return codes, update this
845                # to do something better than a string compare
846                if "Permission error" in e.args[0]:
847                    cred = self.my_authority_credential_string()
848                else:
849                    raise
850         elif record.type in ["authority"]:
851             cred = self.my_authority_credential_string()
852         elif record.type == 'node':
853             cred = self.my_authority_credential_string()
854         else:
855             raise "unknown record type" + record.type
856         record_dict = record.todict()
857         return self.registry().Update(record_dict, cred)
858   
859     def remove(self, options, args):
860         "remove registry record by name (Remove)"
861         auth_cred = self.my_authority_credential_string()
862         if len(args)!=1:
863             self.print_help()
864             sys.exit(1)
865         hrn = args[0]
866         type = options.type 
867         if type in ['all']:
868             type = '*'
869         return self.registry().Remove(hrn, auth_cred, type)
870     
871     # ==================================================================
872     # Slice-related commands
873     # ==================================================================
874
875     def slices(self, options, args):
876         "list instantiated slices (ListSlices) - returns urn's"
877         server = self.sliceapi()
878         # creds
879         creds = [self.my_credential_string]
880         if options.delegate:
881             delegated_cred = self.delegate_cred(self.my_credential_string, get_authority(self.authority))
882             creds.append(delegated_cred)  
883         # options and call_id when supported
884         api_options = {}
885         api_options['call_id']=unique_call_id()
886         result = server.ListSlices(creds, *self.ois(server,api_options))
887         value = ReturnValue.get_value(result)
888         if self.options.raw:
889             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
890         else:
891             display_list(value)
892         return
893
894     # show rspec for named slice
895     def resources(self, options, args):
896         """
897         with no arg, discover available resources, (ListResources)
898 or with an slice hrn, shows currently provisioned resources
899         """
900         server = self.sliceapi()
901
902         # set creds
903         creds = []
904         if args:
905             creds.append(self.slice_credential_string(args[0]))
906         else:
907             creds.append(self.my_credential_string)
908         if options.delegate:
909             creds.append(self.delegate_cred(cred, get_authority(self.authority)))
910
911         # no need to check if server accepts the options argument since the options has
912         # been a required argument since v1 API
913         api_options = {}
914         # always send call_id to v2 servers
915         api_options ['call_id'] = unique_call_id()
916         # ask for cached value if available
917         api_options ['cached'] = True
918         if args:
919             hrn = args[0]
920             api_options['geni_slice_urn'] = hrn_to_urn(hrn, 'slice')
921         if options.info:
922             api_options['info'] = options.info
923         if options.current:
924             if options.current == True:
925                 api_options['cached'] = False
926             else:
927                 api_options['cached'] = True
928         if options.rspec_version:
929             version_manager = VersionManager()
930             server_version = self.get_cached_server_version(server)
931             if 'sfa' in server_version:
932                 # just request the version the client wants
933                 api_options['geni_rspec_version'] = version_manager.get_version(options.rspec_version).to_dict()
934             else:
935                 api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'}
936         else:
937             api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'}
938         result = server.ListResources (creds, api_options)
939         value = ReturnValue.get_value(result)
940         if self.options.raw:
941             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
942         if options.file is not None:
943             save_rspec_to_file(value, options.file)
944         if (self.options.raw is None) and (options.file is None):
945             display_rspec(value, options.format)
946
947         return
948
949     def create(self, options, args):
950         """
951         create or update named slice with given rspec
952         """
953         server = self.sliceapi()
954
955         # xxx do we need to check usage (len(args)) ?
956         # slice urn
957         slice_hrn = args[0]
958         slice_urn = hrn_to_urn(slice_hrn, 'slice')
959
960         # credentials
961         creds = [self.slice_credential_string(slice_hrn)]
962         delegated_cred = None
963         server_version = self.get_cached_server_version(server)
964         if server_version.get('interface') == 'slicemgr':
965             # delegate our cred to the slice manager
966             # do not delegate cred to slicemgr...not working at the moment
967             pass
968             #if server_version.get('hrn'):
969             #    delegated_cred = self.delegate_cred(slice_cred, server_version['hrn'])
970             #elif server_version.get('urn'):
971             #    delegated_cred = self.delegate_cred(slice_cred, urn_to_hrn(server_version['urn']))
972
973         # rspec
974         rspec_file = self.get_rspec_file(args[1])
975         rspec = open(rspec_file).read()
976
977         # users
978         # need to pass along user keys to the aggregate.
979         # users = [
980         #  { urn: urn:publicid:IDN+emulab.net+user+alice
981         #    keys: [<ssh key A>, <ssh key B>]
982         #  }]
983         users = []
984         slice_records = self.registry().Resolve(slice_urn, [self.my_credential_string])
985         if slice_records and 'researcher' in slice_records[0] and slice_records[0]['researcher']!=[]:
986             slice_record = slice_records[0]
987             user_hrns = slice_record['researcher']
988             user_urns = [hrn_to_urn(hrn, 'user') for hrn in user_hrns]
989             user_records = self.registry().Resolve(user_urns, [self.my_credential_string])
990
991             if 'sfa' not in server_version:
992                 users = pg_users_arg(user_records)
993                 rspec = RSpec(rspec)
994                 rspec.filter({'component_manager_id': server_version['urn']})
995                 rspec = RSpecConverter.to_pg_rspec(rspec.toxml(), content_type='request')
996             else:
997                 users = sfa_users_arg(user_records, slice_record)
998
999         # do not append users, keys, or slice tags. Anything
1000         # not contained in this request will be removed from the slice
1001
1002         # CreateSliver has supported the options argument for a while now so it should
1003         # be safe to assume this server support it
1004         api_options = {}
1005         api_options ['append'] = False
1006         api_options ['call_id'] = unique_call_id()
1007         result = server.CreateSliver(slice_urn, creds, rspec, users, *self.ois(server, api_options))
1008         value = ReturnValue.get_value(result)
1009         if self.options.raw:
1010             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1011         if options.file is not None:
1012             save_rspec_to_file (value, options.file)
1013         if (self.options.raw is None) and (options.file is None):
1014             print value
1015
1016         return value
1017
1018     def delete(self, options, args):
1019         """
1020         delete named slice (DeleteSliver)
1021         """
1022         server = self.sliceapi()
1023
1024         # slice urn
1025         slice_hrn = args[0]
1026         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1027
1028         # creds
1029         slice_cred = self.slice_credential_string(slice_hrn)
1030         creds = [slice_cred]
1031         if options.delegate:
1032             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1033             creds.append(delegated_cred)
1034         
1035         # options and call_id when supported
1036         api_options = {}
1037         api_options ['call_id'] = unique_call_id()
1038         result = server.DeleteSliver(slice_urn, creds, *self.ois(server, api_options ) )
1039         value = ReturnValue.get_value(result)
1040         if self.options.raw:
1041             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1042         else:
1043             print value
1044         return value 
1045   
1046     def status(self, options, args):
1047         """
1048         retrieve slice status (SliverStatus)
1049         """
1050         server = self.sliceapi()
1051
1052         # slice urn
1053         slice_hrn = args[0]
1054         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1055
1056         # creds 
1057         slice_cred = self.slice_credential_string(slice_hrn)
1058         creds = [slice_cred]
1059         if options.delegate:
1060             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1061             creds.append(delegated_cred)
1062
1063         # options and call_id when supported
1064         api_options = {}
1065         api_options['call_id']=unique_call_id()
1066         result = server.SliverStatus(slice_urn, creds, *self.ois(server,api_options))
1067         value = ReturnValue.get_value(result)
1068         if self.options.raw:
1069             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1070         else:
1071             print value
1072
1073     def start(self, options, args):
1074         """
1075         start named slice (Start)
1076         """
1077         server = self.sliceapi()
1078
1079         # the slice urn
1080         slice_hrn = args[0]
1081         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1082         
1083         # cred
1084         slice_cred = self.slice_credential_string(args[0])
1085         creds = [slice_cred]
1086         if options.delegate:
1087             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1088             creds.append(delegated_cred)
1089         # xxx Thierry - does this not need an api_options as well ?
1090         result = server.Start(slice_urn, creds)
1091         value = ReturnValue.get_value(result)
1092         if self.options.raw:
1093             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1094         else:
1095             print value
1096         return value
1097     
1098     def stop(self, options, args):
1099         """
1100         stop named slice (Stop)
1101         """
1102         server = self.sliceapi()
1103         # slice urn
1104         slice_hrn = args[0]
1105         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1106         # cred
1107         slice_cred = self.slice_credential_string(args[0])
1108         creds = [slice_cred]
1109         if options.delegate:
1110             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1111             creds.append(delegated_cred)
1112         result =  server.Stop(slice_urn, creds)
1113         value = ReturnValue.get_value(result)
1114         if self.options.raw:
1115             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1116         else:
1117             print value
1118         return value
1119     
1120     # reset named slice
1121     def reset(self, options, args):
1122         """
1123         reset named slice (reset_slice)
1124         """
1125         server = self.sliceapi()
1126         # slice urn
1127         slice_hrn = args[0]
1128         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1129         # cred
1130         slice_cred = self.slice_credential_string(args[0])
1131         creds = [slice_cred]
1132         if options.delegate:
1133             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1134             creds.append(delegated_cred)
1135         result = server.reset_slice(creds, slice_urn)
1136         value = ReturnValue.get_value(result)
1137         if self.options.raw:
1138             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1139         else:
1140             print value
1141         return value
1142
1143     def renew(self, options, args):
1144         """
1145         renew slice (RenewSliver)
1146         """
1147         server = self.sliceapi()
1148         # slice urn    
1149         slice_hrn = args[0]
1150         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1151         # creds
1152         slice_cred = self.slice_credential_string(args[0])
1153         creds = [slice_cred]
1154         if options.delegate:
1155             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1156             creds.append(delegated_cred)
1157         # time
1158         time = args[1]
1159         # options and call_id when supported
1160         api_options = {}
1161         api_options['call_id']=unique_call_id()
1162         result =  server.RenewSliver(slice_urn, creds, time, *self.ois(server,api_options))
1163         value = ReturnValue.get_value(result)
1164         if self.options.raw:
1165             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1166         else:
1167             print value
1168         return value
1169
1170
1171     def shutdown(self, options, args):
1172         """
1173         shutdown named slice (Shutdown)
1174         """
1175         server = self.sliceapi()
1176         # slice urn
1177         slice_hrn = args[0]
1178         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1179         # creds
1180         slice_cred = self.slice_credential_string(slice_hrn)
1181         creds = [slice_cred]
1182         if options.delegate:
1183             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1184             creds.append(delegated_cred)
1185         result = server.Shutdown(slice_urn, creds)
1186         value = ReturnValue.get_value(result)
1187         if self.options.raw:
1188             save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner)
1189         else:
1190             print value
1191         return value         
1192     
1193
1194     def get_ticket(self, options, args):
1195         """
1196         get a ticket for the specified slice
1197         """
1198         server = self.sliceapi()
1199         # slice urn
1200         slice_hrn, rspec_path = args[0], args[1]
1201         slice_urn = hrn_to_urn(slice_hrn, 'slice')
1202         # creds
1203         slice_cred = self.slice_credential_string(slice_hrn)
1204         creds = [slice_cred]
1205         if options.delegate:
1206             delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
1207             creds.append(delegated_cred)
1208         # rspec
1209         rspec_file = self.get_rspec_file(rspec_path) 
1210         rspec = open(rspec_file).read()
1211         # options and call_id when supported
1212         api_options = {}
1213         api_options['call_id']=unique_call_id()
1214         # get ticket at the server
1215         ticket_string = server.GetTicket(slice_urn, creds, rspec, *self.ois(server,api_options))
1216         # save
1217         file = os.path.join(self.options.sfi_dir, get_leaf(slice_hrn) + ".ticket")
1218         self.logger.info("writing ticket to %s"%file)
1219         ticket = SfaTicket(string=ticket_string)
1220         ticket.save_to_file(filename=file, save_parents=True)
1221
1222     def redeem_ticket(self, options, args):
1223         """
1224         Connects to nodes in a slice and redeems a ticket
1225 (slice hrn is retrieved from the ticket)
1226         """
1227         ticket_file = args[0]
1228         
1229         # get slice hrn from the ticket
1230         # use this to get the right slice credential 
1231         ticket = SfaTicket(filename=ticket_file)
1232         ticket.decode()
1233         ticket_string = ticket.save_to_string(save_parents=True)
1234
1235         slice_hrn = ticket.gidObject.get_hrn()
1236         slice_urn = hrn_to_urn(slice_hrn, 'slice') 
1237         #slice_hrn = ticket.attributes['slivers'][0]['hrn']
1238         slice_cred = self.slice_credential_string(slice_hrn)
1239         
1240         # get a list of node hostnames from the RSpec 
1241         tree = etree.parse(StringIO(ticket.rspec))
1242         root = tree.getroot()
1243         hostnames = root.xpath("./network/site/node/hostname/text()")
1244         
1245         # create an xmlrpc connection to the component manager at each of these
1246         # components and gall redeem_ticket
1247         connections = {}
1248         for hostname in hostnames:
1249             try:
1250                 self.logger.info("Calling redeem_ticket at %(hostname)s " % locals())
1251                 cm_url="http://%s:%s/"%(hostname,CM_PORT)
1252                 server = SfaServerProxy(cm_url, self.private_key, self.my_gid)
1253                 server = self.server_proxy(hostname, CM_PORT, self.private_key, 
1254                                            timeout=self.options.timeout, verbose=self.options.debug)
1255                 server.RedeemTicket(ticket_string, slice_cred)
1256                 self.logger.info("Success")
1257             except socket.gaierror:
1258                 self.logger.error("redeem_ticket failed on %s: Component Manager not accepting requests"%hostname)
1259             except Exception, e:
1260                 self.logger.log_exc(e.message)
1261         return
1262
1263     def create_gid(self, options, args):
1264         """
1265         Create a GID (CreateGid)
1266         """
1267         if len(args) < 1:
1268             self.print_help()
1269             sys.exit(1)
1270         target_hrn = args[0]
1271         gid = self.registry().CreateGid(self.my_credential_string, target_hrn, self.client_bootstrap.my_gid_string())
1272         if options.file:
1273             filename = options.file
1274         else:
1275             filename = os.sep.join([self.options.sfi_dir, '%s.gid' % target_hrn])
1276         self.logger.info("writing %s gid to %s" % (target_hrn, filename))
1277         GID(string=gid).save_to_file(filename)
1278          
1279
1280     def delegate(self, options, args):
1281         """
1282         (locally) create delegate credential for use by given hrn
1283         """
1284         delegee_hrn = args[0]
1285         if options.delegate_user:
1286             cred = self.delegate_cred(self.my_credential_string, delegee_hrn, 'user')
1287         elif options.delegate_slice:
1288             slice_cred = self.slice_credential_string(options.delegate_slice)
1289             cred = self.delegate_cred(slice_cred, delegee_hrn, 'slice')
1290         else:
1291             self.logger.warning("Must specify either --user or --slice <hrn>")
1292             return
1293         delegated_cred = Credential(string=cred)
1294         object_hrn = delegated_cred.get_gid_object().get_hrn()
1295         if options.delegate_user:
1296             dest_fn = os.path.join(self.options.sfi_dir, get_leaf(delegee_hrn) + "_"
1297                                   + get_leaf(object_hrn) + ".cred")
1298         elif options.delegate_slice:
1299             dest_fn = os.path.join(self.options.sfi_dir, get_leaf(delegee_hrn) + "_slice_"
1300                                   + get_leaf(object_hrn) + ".cred")
1301
1302         delegated_cred.save_to_file(dest_fn, save_parents=True)
1303
1304         self.logger.info("delegated credential for %s to %s and wrote to %s"%(object_hrn, delegee_hrn,dest_fn))
1305     
1306     def get_trusted_certs(self, options, args):
1307         """
1308         return uhe trusted certs at this interface (get_trusted_certs)
1309         """ 
1310         trusted_certs = self.registry().get_trusted_certs()
1311         for trusted_cert in trusted_certs:
1312             gid = GID(string=trusted_cert)
1313             gid.dump()
1314             cert = Certificate(string=trusted_cert)
1315             self.logger.debug('Sfi.get_trusted_certs -> %r'%cert.get_subject())
1316         return 
1317
1318     def config (self, options, args):
1319         "Display contents of current config"
1320         self.show_config()