2 # SFA XML-RPC and SOAP interfaces
10 from sfa.util.faults import RecordNotFound, MissingSfaInfo
11 from sfa.util.api import BaseAPI
12 from sfa.util.config import Config
13 from sfa.util.sfalogging import logger
14 import sfa.util.xmlrpcprotocol as xmlrpcprotocol
15 from sfa.util.xrn import hrn_to_urn
16 from sfa.util.plxrn import hostname_to_hrn, hrn_to_pl_slicename, \
17 hrn_to_pl_slicename, slicename_to_hrn, hrn_to_pl_login_base
18 from sfa.util.nodemanager import NodeManager
20 from sfa.trust.auth import Auth
21 from sfa.trust.rights import determine_rights
22 from sfa.trust.credential import Credential
23 from sfa.trust.certificate import Certificate, Keypair
24 from sfa.trust.gid import GID
26 from collections import defaultdict
28 class defaultdict(dict):
29 def __init__(self, default_factory=None, *a, **kw):
30 if (default_factory is not None and
31 not hasattr(default_factory, '__call__')):
32 raise TypeError('first argument must be callable')
33 dict.__init__(self, *a, **kw)
34 self.default_factory = default_factory
35 def __getitem__(self, key):
37 return dict.__getitem__(self, key)
39 return self.__missing__(key)
40 def __missing__(self, key):
41 if self.default_factory is None:
43 self[key] = value = self.default_factory()
46 if self.default_factory is None:
49 args = self.default_factory,
50 return type(self), args, None, None, self.items()
52 return self.__copy__()
54 return type(self)(self.default_factory, self)
55 def __deepcopy__(self, memo):
57 return type(self)(self.default_factory,
58 copy.deepcopy(self.items()))
60 return 'defaultdict(%s, %s)' % (self.default_factory,
62 ## end of http://code.activestate.com/recipes/523034/ }}}
64 def list_to_dict(recs, key):
66 convert a list of dictionaries into a dictionary keyed on the
67 specified dictionary key
69 keys = [rec[key] for rec in recs]
70 return dict(zip(keys, recs))
72 class SfaAPI(BaseAPI):
74 # flat list of method names
76 methods = sfa.methods.all
78 def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8",
79 methods='sfa.methods', peer_cert = None, interface = None,
80 key_file = None, cert_file = None, cache = None):
81 BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, \
82 peer_cert=peer_cert, interface=interface, key_file=key_file, \
83 cert_file=cert_file, cache=cache)
85 self.encoding = encoding
86 from sfa.util.table import SfaTable
87 self.SfaTable = SfaTable
88 # Better just be documenting the API
93 self.config = Config(config)
94 self.auth = Auth(peer_cert)
95 self.interface = interface
96 self.key_file = key_file
97 self.key = Keypair(filename=self.key_file)
98 self.cert_file = cert_file
99 self.cert = Certificate(filename=self.cert_file)
100 self.credential = None
101 # Initialize the PLC shell only if SFA wraps a myPLC
102 rspec_type = self.config.get_aggregate_type()
103 if (rspec_type == 'pl' or rspec_type == 'vini' or \
104 rspec_type == 'eucalyptus' or rspec_type == 'max'):
105 self.plshell = self.getPLCShell()
106 self.plshell_version = "4.3"
108 self.hrn = self.config.SFA_INTERFACE_HRN
109 self.time_format = "%Y-%m-%d %H:%M:%S"
112 def getPLCShell(self):
113 self.plauth = {'Username': self.config.SFA_PLC_USER,
114 'AuthMethod': 'password',
115 'AuthString': self.config.SFA_PLC_PASSWORD}
117 # The native shell (PLC.Shell.Shell) is more efficient than xmlrpc,
118 # but it leaves idle db connections open. use xmlrpc until we can figure
119 # out why PLC.Shell.Shell doesn't close db connection properly
121 # sys.path.append(os.path.dirname(os.path.realpath("/usr/bin/plcsh")))
122 # self.plshell_type = 'direct'
124 # shell = PLC.Shell.Shell(globals = globals())
127 self.plshell_type = 'xmlrpc'
128 url = self.config.SFA_PLC_URL
129 shell = xmlrpclib.Server(url, verbose = 0, allow_none = True)
132 def get_server(self, interface, cred, timeout=30):
134 Returns a connection to the specified interface. Use the specified
135 credential to determine the caller and look for the caller's key/cert
136 in the registry hierarchy cache.
138 from sfa.trust.hierarchy import Hierarchy
139 if not isinstance(cred, Credential):
140 cred_obj = Credential(string=cred)
143 caller_gid = cred_obj.get_gid_caller()
144 hierarchy = Hierarchy()
145 auth_info = hierarchy.get_auth_info(caller_gid.get_hrn())
146 key_file = auth_info.get_privkey_filename()
147 cert_file = auth_info.get_gid_filename()
148 server = interface.get_server(key_file, cert_file, timeout)
152 def getCredential(self):
154 Return a valid credential for this interface.
157 path = self.config.SFA_DATA_DIR
158 filename = ".".join([self.interface, self.hrn, type, "cred"])
159 cred_filename = path + os.sep + filename
161 if os.path.isfile(cred_filename):
162 cred = Credential(filename = cred_filename)
163 # make sure cred isnt expired
164 if not cred.get_expiration or \
165 datetime.datetime.utcnow() < cred.get_expiration():
166 return cred.save_to_string(save_parents=True)
168 # get a new credential
169 if self.interface in ['registry']:
170 cred = self.__getCredentialRaw()
172 cred = self.__getCredential()
173 cred.save_to_file(cred_filename, save_parents=True)
175 return cred.save_to_string(save_parents=True)
178 def getDelegatedCredential(self, creds):
180 Attempt to find a credential delegated to us in
181 the specified list of creds.
183 from sfa.trust.hierarchy import Hierarchy
184 if creds and not isinstance(creds, list):
186 hierarchy = Hierarchy()
188 delegated_cred = None
190 if hierarchy.auth_exists(Credential(string=cred).get_gid_caller().get_hrn()):
191 delegated_cred = cred
193 return delegated_cred
195 def __getCredential(self):
197 Get our credential from a remote registry
199 from sfa.server.registry import Registries
200 registries = Registries()
201 registry = registries.get_server(self.hrn, self.key_file, self.cert_file)
202 cert_string=self.cert.save_to_string(save_parents=True)
203 # get self credential
204 self_cred = registry.GetSelfCredential(cert_string, self.hrn, 'authority')
206 cred = registry.GetCredential(self_cred, self.hrn, 'authority')
207 return Credential(string=cred)
209 def __getCredentialRaw(self):
211 Get our current credential directly from the local registry.
215 auth_hrn = self.auth.get_authority(hrn)
217 # is this a root or sub authority
218 if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
220 auth_info = self.auth.get_auth_info(auth_hrn)
221 table = self.SfaTable()
222 records = table.findObjects({'hrn': hrn, 'type': 'authority+sa'})
226 type = record['type']
227 object_gid = record.get_gid_object()
228 new_cred = Credential(subject = object_gid.get_subject())
229 new_cred.set_gid_caller(object_gid)
230 new_cred.set_gid_object(object_gid)
231 new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename())
233 r1 = determine_rights(type, hrn)
234 new_cred.set_privileges(r1)
241 def loadCredential (self):
243 Attempt to load credential from file if it exists. If it doesnt get
244 credential from registry.
247 # see if this file exists
248 # XX This is really the aggregate's credential. Using this is easier than getting
249 # the registry's credential from iteslf (ssl errors).
250 ma_cred_filename = self.config.SFA_DATA_DIR + os.sep + self.interface + self.hrn + ".ma.cred"
252 self.credential = Credential(filename = ma_cred_filename)
254 self.credential = self.getCredentialFromRegistry()
259 # Convert SFA fields to PLC fields for use when registering up updating
260 # registry record in the PLC database
262 # @param type type of record (user, slice, ...)
263 # @param hrn human readable name
264 # @param sfa_fields dictionary of SFA fields
265 # @param pl_fields dictionary of PLC fields (output)
267 def sfa_fields_to_pl_fields(self, type, hrn, record):
269 def convert_ints(tmpdict, int_fields):
270 for field in int_fields:
272 tmpdict[field] = int(tmpdict[field])
275 #for field in record:
276 # pl_record[field] = record[field]
279 if not "instantiation" in pl_record:
280 pl_record["instantiation"] = "plc-instantiated"
281 pl_record["name"] = hrn_to_pl_slicename(hrn)
283 pl_record["url"] = record["url"]
284 if "description" in record:
285 pl_record["description"] = record["description"]
286 if "expires" in record:
287 pl_record["expires"] = int(record["expires"])
290 if not "hostname" in pl_record:
291 if not "hostname" in record:
292 raise MissingSfaInfo("hostname")
293 pl_record["hostname"] = record["hostname"]
294 if not "model" in pl_record:
295 pl_record["model"] = "geni"
297 elif type == "authority":
298 pl_record["login_base"] = hrn_to_pl_login_base(hrn)
300 if not "name" in pl_record:
301 pl_record["name"] = hrn
303 if not "abbreviated_name" in pl_record:
304 pl_record["abbreviated_name"] = hrn
306 if not "enabled" in pl_record:
307 pl_record["enabled"] = True
309 if not "is_public" in pl_record:
310 pl_record["is_public"] = True
314 def fill_record_pl_info(self, records):
316 Fill in the planetlab specific fields of a SFA record. This
317 involves calling the appropriate PLC method to retrieve the
318 database record for the object.
320 PLC data is filled into the pl_info field of the record.
322 @param record: record to fill in field (in/out param)
325 node_ids, site_ids, slice_ids = [], [], []
326 person_ids, key_ids = [], []
327 type_map = {'node': node_ids, 'authority': site_ids,
328 'slice': slice_ids, 'user': person_ids}
330 for record in records:
331 for type in type_map:
332 if type == record['type']:
333 type_map[type].append(record['pointer'])
336 nodes, sites, slices, persons, keys = {}, {}, {}, {}, {}
338 node_list = self.plshell.GetNodes(self.plauth, node_ids)
339 nodes = list_to_dict(node_list, 'node_id')
341 site_list = self.plshell.GetSites(self.plauth, site_ids)
342 sites = list_to_dict(site_list, 'site_id')
344 slice_list = self.plshell.GetSlices(self.plauth, slice_ids)
345 slices = list_to_dict(slice_list, 'slice_id')
347 person_list = self.plshell.GetPersons(self.plauth, person_ids)
348 persons = list_to_dict(person_list, 'person_id')
349 for person in persons:
350 key_ids.extend(persons[person]['key_ids'])
352 pl_records = {'node': nodes, 'authority': sites,
353 'slice': slices, 'user': persons}
356 key_list = self.plshell.GetKeys(self.plauth, key_ids)
357 keys = list_to_dict(key_list, 'key_id')
360 for record in records:
361 # records with pointer==-1 do not have plc info.
362 # for example, the top level authority records which are
363 # authorities, but not PL "sites"
364 if record['pointer'] == -1:
367 for type in pl_records:
368 if record['type'] == type:
369 if record['pointer'] in pl_records[type]:
370 record.update(pl_records[type][record['pointer']])
373 if record['type'] == 'user':
374 if 'key_ids' not in record:
375 logger.info("user record has no 'key_ids' - need to import from myplc ?")
377 pubkeys = [keys[key_id]['key'] for key_id in record['key_ids'] if key_id in keys]
378 record['keys'] = pubkeys
380 # fill in record hrns
381 records = self.fill_record_hrns(records)
385 def fill_record_hrns(self, records):
387 convert pl ids to hrns
391 slice_ids, person_ids, site_ids, node_ids = [], [], [], []
392 for record in records:
393 if 'site_id' in record:
394 site_ids.append(record['site_id'])
395 if 'site_ids' in records:
396 site_ids.extend(record['site_ids'])
397 if 'person_ids' in record:
398 person_ids.extend(record['person_ids'])
399 if 'slice_ids' in record:
400 slice_ids.extend(record['slice_ids'])
401 if 'node_ids' in record:
402 node_ids.extend(record['node_ids'])
405 slices, persons, sites, nodes = {}, {}, {}, {}
407 site_list = self.plshell.GetSites(self.plauth, site_ids, ['site_id', 'login_base'])
408 sites = list_to_dict(site_list, 'site_id')
410 person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'email'])
411 persons = list_to_dict(person_list, 'person_id')
413 slice_list = self.plshell.GetSlices(self.plauth, slice_ids, ['slice_id', 'name'])
414 slices = list_to_dict(slice_list, 'slice_id')
416 node_list = self.plshell.GetNodes(self.plauth, node_ids, ['node_id', 'hostname'])
417 nodes = list_to_dict(node_list, 'node_id')
419 # convert ids to hrns
420 for record in records:
421 # get all relevant data
422 type = record['type']
423 pointer = record['pointer']
429 if 'site_id' in record:
430 site = sites[record['site_id']]
431 login_base = site['login_base']
432 record['site'] = ".".join([auth_hrn, login_base])
433 if 'person_ids' in record:
434 emails = [persons[person_id]['email'] for person_id in record['person_ids'] \
435 if person_id in persons]
436 usernames = [email.split('@')[0] for email in emails]
437 person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames]
438 record['persons'] = person_hrns
439 if 'slice_ids' in record:
440 slicenames = [slices[slice_id]['name'] for slice_id in record['slice_ids'] \
441 if slice_id in slices]
442 slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
443 record['slices'] = slice_hrns
444 if 'node_ids' in record:
445 hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
447 node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
448 record['nodes'] = node_hrns
449 if 'site_ids' in record:
450 login_bases = [sites[site_id]['login_base'] for site_id in record['site_ids'] \
452 site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases]
453 record['sites'] = site_hrns
457 def fill_record_sfa_info(self, records):
459 def startswith(prefix, values):
460 return [value for value in values if value.startswith(prefix)]
465 for record in records:
466 person_ids.extend(record.get("person_ids", []))
467 site_ids.extend(record.get("site_ids", []))
468 if 'site_id' in record:
469 site_ids.append(record['site_id'])
471 # get all pis from the sites we've encountered
472 # and store them in a dictionary keyed on site_id
475 pi_filter = {'|roles': ['pi'], '|site_ids': site_ids}
476 pi_list = self.plshell.GetPersons(self.plauth, pi_filter, ['person_id', 'site_ids'])
478 # we will need the pi's hrns also
479 person_ids.append(pi['person_id'])
481 # we also need to keep track of the sites these pis
483 for site_id in pi['site_ids']:
484 if site_id in site_pis:
485 site_pis[site_id].append(pi)
487 site_pis[site_id] = [pi]
489 # get sfa records for all records associated with these records.
490 # we'll replace pl ids (person_ids) with hrns from the sfa records
493 # get the sfa records
494 table = self.SfaTable()
495 person_list, persons = [], {}
496 person_list = table.find({'type': 'user', 'pointer': person_ids})
497 # create a hrns keyed on the sfa record's pointer.
498 # Its possible for multiple records to have the same pointer so
499 # the dict's value will be a list of hrns.
500 persons = defaultdict(list)
501 for person in person_list:
502 persons[person['pointer']].append(person)
505 pl_person_list, pl_persons = [], {}
506 pl_person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'roles'])
507 pl_persons = list_to_dict(pl_person_list, 'person_id')
510 for record in records:
511 # skip records with no pl info (top level authorities)
512 #if record['pointer'] == -1:
515 type = record['type']
516 if (type == "slice"):
517 # all slice users are researchers
518 record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
520 record['researcher'] = []
521 for person_id in record.get('person_ids', []):
522 hrns = [person['hrn'] for person in persons[person_id]]
523 record['researcher'].extend(hrns)
525 # pis at the slice's site
526 if 'site_id' in record and record['site_id'] in site_pis:
527 pl_pis = site_pis[record['site_id']]
528 pi_ids = [pi['person_id'] for pi in pl_pis]
529 for person_id in pi_ids:
530 hrns = [person['hrn'] for person in persons[person_id]]
531 record['PI'].extend(hrns)
532 record['geni_creator'] = record['PI']
534 elif (type.startswith("authority")):
536 if record['hrn'] in self.aggregates:
538 record['url'] = self.aggregates[record['hrn']].get_url()
540 if record['pointer'] != -1:
542 record['operator'] = []
544 for pointer in record.get('person_ids', []):
545 if pointer not in persons or pointer not in pl_persons:
546 # this means there is not sfa or pl record for this user
548 hrns = [person['hrn'] for person in persons[pointer]]
549 roles = pl_persons[pointer]['roles']
551 record['PI'].extend(hrns)
553 record['operator'].extend(hrns)
555 record['owner'].extend(hrns)
556 # xxx TODO: OrganizationName
557 elif (type == "node"):
558 sfa_info['dns'] = record.get("hostname", "")
559 # xxx TODO: URI, LatLong, IP, DNS
561 elif (type == "user"):
562 sfa_info['email'] = record.get("email", "")
563 sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
564 sfa_info['geni_certificate'] = record['gid']
565 # xxx TODO: PostalAddress, Phone
566 record.update(sfa_info)
568 def fill_record_info(self, records):
570 Given a SFA record, fill in the PLC specific and SFA specific
571 fields in the record.
573 if not isinstance(records, list):
576 self.fill_record_pl_info(records)
577 self.fill_record_sfa_info(records)
579 def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc):
580 # get a list of the HRNs that are members of the old and new records
582 oldList = oldRecord.get(listName, [])
585 newList = record.get(listName, [])
586 # xxx ugly hack - somehow we receive here a list of {'text':value}
587 # instead of an expected list of strings
588 # please remove once this is issue is cleanly fixed
589 def normalize (value):
590 from types import StringTypes
591 from sfa.util.sfalogging import logger
592 if isinstance(value,StringTypes): return value
593 elif isinstance(value,dict):
594 newvalue=value['text']
595 logger.info("Normalizing %s=>%s"%(value,newvalue))
597 newList=[normalize(v) for v in newList]
599 # if the lists are the same, then we don't have to update anything
600 if (oldList == newList):
603 # build a list of the new person ids, by looking up each person to get
606 table = self.SfaTable()
607 records = table.find({'type': 'user', 'hrn': newList})
609 newIdList.append(rec['pointer'])
611 # build a list of the old person ids from the person_ids field
613 oldIdList = oldRecord.get("person_ids", [])
614 containerId = oldRecord.get_pointer()
616 # if oldRecord==None, then we are doing a Register, instead of an
619 containerId = record.get_pointer()
621 # add people who are in the new list, but not the oldList
622 for personId in newIdList:
623 if not (personId in oldIdList):
624 addFunc(self.plauth, personId, containerId)
626 # remove people who are in the old list, but not the new list
627 for personId in oldIdList:
628 if not (personId in newIdList):
629 delFunc(self.plauth, personId, containerId)
631 def update_membership(self, oldRecord, record):
632 if record.type == "slice":
633 self.update_membership_list(oldRecord, record, 'researcher',
634 self.plshell.AddPersonToSlice,
635 self.plshell.DeletePersonFromSlice)
636 elif record.type == "authority":
642 class ComponentAPI(BaseAPI):
644 def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", methods='sfa.methods',
645 peer_cert = None, interface = None, key_file = None, cert_file = None):
647 BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert,
648 interface=interface, key_file=key_file, cert_file=cert_file)
649 self.encoding = encoding
651 # Better just be documenting the API
655 self.nodemanager = NodeManager(self.config)
657 def sliver_exists(self):
658 sliver_dict = self.nodemanager.GetXIDs()
659 ### xxx slicename is undefined
660 if slicename in sliver_dict.keys():
665 def get_registry(self):
666 addr, port = self.config.SFA_REGISTRY_HOST, self.config.SFA_REGISTRY_PORT
667 url = "http://%(addr)s:%(port)s" % locals()
668 server = xmlrpcprotocol.get_server(url, self.key_file, self.cert_file)
671 def get_node_key(self):
672 # this call requires no authentication,
673 # so we can generate a random keypair here
675 (kfd, keyfile) = tempfile.mkstemp()
676 (cfd, certfile) = tempfile.mkstemp()
677 key = Keypair(create=True)
678 key.save_to_file(keyfile)
679 cert = Certificate(subject=subject)
680 cert.set_issuer(key=key, subject=subject)
683 cert.save_to_file(certfile)
684 registry = self.get_registry()
685 # the registry will scp the key onto the node
688 def getCredential(self):
690 Get our credential from a remote registry
692 path = self.config.SFA_DATA_DIR
693 config_dir = self.config.config_path
694 cred_filename = path + os.sep + 'node.cred'
696 credential = Credential(filename = cred_filename)
697 return credential.save_to_string(save_parents=True)
699 node_pkey_file = config_dir + os.sep + "node.key"
700 node_gid_file = config_dir + os.sep + "node.gid"
701 cert_filename = path + os.sep + 'server.cert'
702 if not os.path.exists(node_pkey_file) or \
703 not os.path.exists(node_gid_file):
707 gid = GID(filename=node_gid_file)
709 # get credential from registry
710 cert_str = Certificate(filename=cert_filename).save_to_string(save_parents=True)
711 registry = self.get_registry()
712 cred = registry.GetSelfCredential(cert_str, hrn, 'node')
713 # xxx credfile is undefined
714 Credential(string=cred).save_to_file(credfile, save_parents=True)
718 def clean_key_cred(self):
720 remove the existing keypair and cred and generate new ones
722 files = ["server.key", "server.cert", "node.cred"]
724 # xxx KEYDIR is undefined, could be meant to be "/var/lib/sfa/" from sfa_component_setup.py
725 filepath = KEYDIR + os.sep + f
726 if os.path.isfile(filepath):
729 # install the new key pair
730 # GetCredential will take care of generating the new keypair