2 # SFA XML-RPC and SOAP interfaces
10 from sfa.util.faults import RecordNotFound, MissingSfaInfo
11 from sfa.util.api import BaseAPI
12 from sfa.util.config import Config
13 from sfa.util.sfalogging import logger
14 import sfa.util.xmlrpcprotocol as xmlrpcprotocol
15 from sfa.util.xrn import hrn_to_urn
16 from sfa.util.plxrn import hostname_to_hrn, hrn_to_pl_slicename, \
17 hrn_to_pl_slicename, slicename_to_hrn, hrn_to_pl_login_base
18 from sfa.util.nodemanager import NodeManager
20 from sfa.trust.auth import Auth
21 from sfa.trust.rights import determine_rights
22 from sfa.trust.credential import Credential
23 from sfa.trust.certificate import Certificate, Keypair
24 from sfa.trust.gid import GID
26 from collections import defaultdict
28 class defaultdict(dict):
29 def __init__(self, default_factory=None, *a, **kw):
30 if (default_factory is not None and
31 not hasattr(default_factory, '__call__')):
32 raise TypeError('first argument must be callable')
33 dict.__init__(self, *a, **kw)
34 self.default_factory = default_factory
35 def __getitem__(self, key):
37 return dict.__getitem__(self, key)
39 return self.__missing__(key)
40 def __missing__(self, key):
41 if self.default_factory is None:
43 self[key] = value = self.default_factory()
46 if self.default_factory is None:
49 args = self.default_factory,
50 return type(self), args, None, None, self.items()
52 return self.__copy__()
54 return type(self)(self.default_factory, self)
55 def __deepcopy__(self, memo):
57 return type(self)(self.default_factory,
58 copy.deepcopy(self.items()))
60 return 'defaultdict(%s, %s)' % (self.default_factory,
62 ## end of http://code.activestate.com/recipes/523034/ }}}
64 def list_to_dict(recs, key):
66 convert a list of dictionaries into a dictionary keyed on the
67 specified dictionary key
69 keys = [rec[key] for rec in recs]
70 return dict(zip(keys, recs))
72 class SfaAPI(BaseAPI):
74 # flat list of method names
76 methods = sfa.methods.all
78 def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8",
79 methods='sfa.methods', peer_cert = None, interface = None,
80 key_file = None, cert_file = None, cache = None):
81 BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, \
82 peer_cert=peer_cert, interface=interface, key_file=key_file, \
83 cert_file=cert_file, cache=cache)
85 self.encoding = encoding
86 from sfa.util.table import SfaTable
87 self.SfaTable = SfaTable
88 # Better just be documenting the API
93 self.config = Config(config)
94 self.auth = Auth(peer_cert)
95 self.interface = interface
96 self.key_file = key_file
97 self.key = Keypair(filename=self.key_file)
98 self.cert_file = cert_file
99 self.cert = Certificate(filename=self.cert_file)
100 self.credential = None
101 # Initialize the PLC shell only if SFA wraps a myPLC
102 rspec_type = self.config.get_aggregate_type()
103 if (rspec_type == 'pl' or rspec_type == 'vini' or \
104 rspec_type == 'eucalyptus' or rspec_type == 'max'):
105 self.plshell = self.getPLCShell()
106 self.plshell_version = "4.3"
108 self.hrn = self.config.SFA_INTERFACE_HRN
109 self.time_format = "%Y-%m-%d %H:%M:%S"
112 def getPLCShell(self):
113 self.plauth = {'Username': self.config.SFA_PLC_USER,
114 'AuthMethod': 'password',
115 'AuthString': self.config.SFA_PLC_PASSWORD}
117 # The native shell (PLC.Shell.Shell) is more efficient than xmlrpc,
118 # but it leaves idle db connections open. use xmlrpc until we can figure
119 # out why PLC.Shell.Shell doesn't close db connection properly
121 # sys.path.append(os.path.dirname(os.path.realpath("/usr/bin/plcsh")))
122 # self.plshell_type = 'direct'
124 # shell = PLC.Shell.Shell(globals = globals())
127 self.plshell_type = 'xmlrpc'
128 url = self.config.SFA_PLC_URL
129 shell = xmlrpclib.Server(url, verbose = 0, allow_none = True)
132 def get_server(self, interface, cred, timeout=30):
134 Returns a connection to the specified interface. Use the specified
135 credential to determine the caller and look for the caller's key/cert
136 in the registry hierarchy cache.
138 from sfa.trust.hierarchy import Hierarchy
139 if not isinstance(cred, Credential):
140 cred_obj = Credential(string=cred)
143 caller_gid = cred_obj.get_gid_caller()
144 hierarchy = Hierarchy()
145 auth_info = hierarchy.get_auth_info(caller_gid.get_hrn())
146 key_file = auth_info.get_privkey_filename()
147 cert_file = auth_info.get_gid_filename()
148 server = interface.get_server(key_file, cert_file, timeout)
152 def getCredential(self):
154 Return a valid credential for this interface.
157 path = self.config.SFA_DATA_DIR
158 filename = ".".join([self.interface, self.hrn, type, "cred"])
159 cred_filename = path + os.sep + filename
161 if os.path.isfile(cred_filename):
162 cred = Credential(filename = cred_filename)
163 # make sure cred isnt expired
164 if not cred.get_expiration or \
165 datetime.datetime.utcnow() < cred.get_expiration():
166 return cred.save_to_string(save_parents=True)
168 # get a new credential
169 if self.interface in ['registry']:
170 cred = self.__getCredentialRaw()
172 cred = self.__getCredential()
173 cred.save_to_file(cred_filename, save_parents=True)
175 return cred.save_to_string(save_parents=True)
178 def getDelegatedCredential(self, creds):
180 Attempt to find a credential delegated to us in
181 the specified list of creds.
183 from sfa.trust.hierarchy import Hierarchy
184 if creds and not isinstance(creds, list):
186 hierarchy = Hierarchy()
188 delegated_cred = None
190 if hierarchy.auth_exists(Credential(string=cred).get_gid_caller().get_hrn()):
191 delegated_cred = cred
193 return delegated_cred
195 def __getCredential(self):
197 Get our credential from a remote registry
199 from sfa.server.registry import Registries
200 registries = Registries()
201 registry = registries.get_server(self.hrn, self.key_file, self.cert_file)
202 cert_string=self.cert.save_to_string(save_parents=True)
203 # get self credential
204 self_cred = registry.GetSelfCredential(cert_string, self.hrn, 'authority')
206 cred = registry.GetCredential(self_cred, self.hrn, 'authority')
207 return Credential(string=cred)
209 def __getCredentialRaw(self):
211 Get our current credential directly from the local registry.
215 auth_hrn = self.auth.get_authority(hrn)
217 # is this a root or sub authority
218 if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
220 auth_info = self.auth.get_auth_info(auth_hrn)
221 table = self.SfaTable()
222 records = table.findObjects({'hrn': hrn, 'type': 'authority+sa'})
226 type = record['type']
227 object_gid = record.get_gid_object()
228 new_cred = Credential(subject = object_gid.get_subject())
229 new_cred.set_gid_caller(object_gid)
230 new_cred.set_gid_object(object_gid)
231 new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename())
233 r1 = determine_rights(type, hrn)
234 new_cred.set_privileges(r1)
241 def loadCredential (self):
243 Attempt to load credential from file if it exists. If it doesnt get
244 credential from registry.
247 # see if this file exists
248 # XX This is really the aggregate's credential. Using this is easier than getting
249 # the registry's credential from iteslf (ssl errors).
250 ma_cred_filename = self.config.SFA_DATA_DIR + os.sep + self.interface + self.hrn + ".ma.cred"
252 self.credential = Credential(filename = ma_cred_filename)
254 self.credential = self.getCredentialFromRegistry()
259 # Convert SFA fields to PLC fields for use when registering up updating
260 # registry record in the PLC database
262 # @param type type of record (user, slice, ...)
263 # @param hrn human readable name
264 # @param sfa_fields dictionary of SFA fields
265 # @param pl_fields dictionary of PLC fields (output)
267 def sfa_fields_to_pl_fields(self, type, hrn, record):
269 def convert_ints(tmpdict, int_fields):
270 for field in int_fields:
272 tmpdict[field] = int(tmpdict[field])
275 #for field in record:
276 # pl_record[field] = record[field]
279 if not "instantiation" in pl_record:
280 pl_record["instantiation"] = "plc-instantiated"
281 pl_record["name"] = hrn_to_pl_slicename(hrn)
283 pl_record["url"] = record["url"]
284 if "description" in record:
285 pl_record["description"] = record["description"]
286 if "expires" in record:
287 pl_record["expires"] = int(record["expires"])
290 if not "hostname" in pl_record:
291 if not "hostname" in record:
292 raise MissingSfaInfo("hostname")
293 pl_record["hostname"] = record["hostname"]
294 if not "model" in pl_record:
295 pl_record["model"] = "geni"
297 elif type == "authority":
298 pl_record["login_base"] = hrn_to_pl_login_base(hrn)
300 if not "name" in pl_record:
301 pl_record["name"] = hrn
303 if not "abbreviated_name" in pl_record:
304 pl_record["abbreviated_name"] = hrn
306 if not "enabled" in pl_record:
307 pl_record["enabled"] = True
309 if not "is_public" in pl_record:
310 pl_record["is_public"] = True
314 def fill_record_pl_info(self, records):
316 Fill in the planetlab specific fields of a SFA record. This
317 involves calling the appropriate PLC method to retrieve the
318 database record for the object.
320 PLC data is filled into the pl_info field of the record.
322 @param record: record to fill in field (in/out param)
325 node_ids, site_ids, slice_ids = [], [], []
326 person_ids, key_ids = [], []
327 type_map = {'node': node_ids, 'authority': site_ids,
328 'slice': slice_ids, 'user': person_ids}
330 for record in records:
331 for type in type_map:
332 if type == record['type']:
333 type_map[type].append(record['pointer'])
336 nodes, sites, slices, persons, keys = {}, {}, {}, {}, {}
338 node_list = self.plshell.GetNodes(self.plauth, node_ids)
339 nodes = list_to_dict(node_list, 'node_id')
341 site_list = self.plshell.GetSites(self.plauth, site_ids)
342 sites = list_to_dict(site_list, 'site_id')
344 slice_list = self.plshell.GetSlices(self.plauth, slice_ids)
345 slices = list_to_dict(slice_list, 'slice_id')
347 person_list = self.plshell.GetPersons(self.plauth, person_ids)
348 persons = list_to_dict(person_list, 'person_id')
349 for person in persons:
350 key_ids.extend(persons[person]['key_ids'])
352 pl_records = {'node': nodes, 'authority': sites,
353 'slice': slices, 'user': persons}
356 key_list = self.plshell.GetKeys(self.plauth, key_ids)
357 keys = list_to_dict(key_list, 'key_id')
360 for record in records:
361 # records with pointer==-1 do not have plc info.
362 # for example, the top level authority records which are
363 # authorities, but not PL "sites"
364 if record['pointer'] == -1:
367 for type in pl_records:
368 if record['type'] == type:
369 if record['pointer'] in pl_records[type]:
370 record.update(pl_records[type][record['pointer']])
373 if record['type'] == 'user':
374 if 'key_ids' not in record:
375 logger.info("user record has no 'key_ids' - need to import from myplc ?")
377 pubkeys = [keys[key_id]['key'] for key_id in record['key_ids'] if key_id in keys]
378 record['keys'] = pubkeys
380 # fill in record hrns
381 records = self.fill_record_hrns(records)
385 def fill_record_hrns(self, records):
387 convert pl ids to hrns
391 slice_ids, person_ids, site_ids, node_ids = [], [], [], []
392 for record in records:
393 if 'site_id' in record:
394 site_ids.append(record['site_id'])
395 if 'site_ids' in records:
396 site_ids.extend(record['site_ids'])
397 if 'person_ids' in record:
398 person_ids.extend(record['person_ids'])
399 if 'slice_ids' in record:
400 slice_ids.extend(record['slice_ids'])
401 if 'node_ids' in record:
402 node_ids.extend(record['node_ids'])
405 slices, persons, sites, nodes = {}, {}, {}, {}
407 site_list = self.plshell.GetSites(self.plauth, site_ids, ['site_id', 'login_base'])
408 sites = list_to_dict(site_list, 'site_id')
410 person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'email'])
411 persons = list_to_dict(person_list, 'person_id')
413 slice_list = self.plshell.GetSlices(self.plauth, slice_ids, ['slice_id', 'name'])
414 slices = list_to_dict(slice_list, 'slice_id')
416 node_list = self.plshell.GetNodes(self.plauth, node_ids, ['node_id', 'hostname'])
417 nodes = list_to_dict(node_list, 'node_id')
419 # convert ids to hrns
420 for record in records:
421 # get all relevant data
422 type = record['type']
423 pointer = record['pointer']
429 if 'site_id' in record:
430 site = sites[record['site_id']]
431 login_base = site['login_base']
432 record['site'] = ".".join([auth_hrn, login_base])
433 if 'person_ids' in record:
434 emails = [persons[person_id]['email'] for person_id in record['person_ids'] \
435 if person_id in persons]
436 usernames = [email.split('@')[0] for email in emails]
437 person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames]
438 record['persons'] = person_hrns
439 if 'slice_ids' in record:
440 slicenames = [slices[slice_id]['name'] for slice_id in record['slice_ids'] \
441 if slice_id in slices]
442 slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
443 record['slices'] = slice_hrns
444 if 'node_ids' in record:
445 hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
447 node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
448 record['nodes'] = node_hrns
449 if 'site_ids' in record:
450 login_bases = [sites[site_id]['login_base'] for site_id in record['site_ids'] \
452 site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases]
453 record['sites'] = site_hrns
457 def fill_record_sfa_info(self, records):
459 def startswith(prefix, values):
460 return [value for value in values if value.startswith(prefix)]
465 for record in records:
466 person_ids.extend(record.get("person_ids", []))
467 site_ids.extend(record.get("site_ids", []))
468 if 'site_id' in record:
469 site_ids.append(record['site_id'])
471 # get all pis from the sites we've encountered
472 # and store them in a dictionary keyed on site_id
475 pi_filter = {'|roles': ['pi'], '|site_ids': site_ids}
476 pi_list = self.plshell.GetPersons(self.plauth, pi_filter, ['person_id', 'site_ids'])
478 # we will need the pi's hrns also
479 person_ids.append(pi['person_id'])
481 # we also need to keep track of the sites these pis
483 for site_id in pi['site_ids']:
484 if site_id in site_pis:
485 site_pis[site_id].append(pi)
487 site_pis[site_id] = [pi]
489 # get sfa records for all records associated with these records.
490 # we'll replace pl ids (person_ids) with hrns from the sfa records
493 # get the sfa records
494 table = self.SfaTable()
495 person_list, persons = [], {}
496 person_list = table.find({'type': 'user', 'pointer': person_ids})
497 # create a hrns keyed on the sfa record's pointer.
498 # Its possible for multiple records to have the same pointer so
499 # the dict's value will be a list of hrns.
500 persons = defaultdict(list)
501 for person in person_list:
502 persons[person['pointer']].append(person)
505 pl_person_list, pl_persons = [], {}
506 pl_person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'roles'])
507 pl_persons = list_to_dict(pl_person_list, 'person_id')
510 for record in records:
511 # skip records with no pl info (top level authorities)
512 #if record['pointer'] == -1:
515 type = record['type']
516 if (type == "slice"):
517 # all slice users are researchers
518 record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
520 record['researcher'] = []
521 for person_id in record.get('person_ids', []):
522 hrns = [person['hrn'] for person in persons[person_id]]
523 record['researcher'].extend(hrns)
525 # pis at the slice's site
526 if 'site_id' in record and record['site_id'] in site_pis:
527 pl_pis = site_pis[record['site_id']]
528 pi_ids = [pi['person_id'] for pi in pl_pis]
529 for person_id in pi_ids:
530 hrns = [person['hrn'] for person in persons[person_id]]
531 record['PI'].extend(hrns)
532 record['geni_creator'] = record['PI']
534 elif (type.startswith("authority")):
536 if record['hrn'] in self.aggregates:
538 record['url'] = self.aggregates[record['hrn']].get_url()
540 if record['pointer'] != -1:
542 record['operator'] = []
544 for pointer in record.get('person_ids', []):
545 if pointer not in persons or pointer not in pl_persons:
546 # this means there is not sfa or pl record for this user
548 hrns = [person['hrn'] for person in persons[pointer]]
549 roles = pl_persons[pointer]['roles']
551 record['PI'].extend(hrns)
553 record['operator'].extend(hrns)
555 record['owner'].extend(hrns)
556 # xxx TODO: OrganizationName
557 elif (type == "node"):
558 sfa_info['dns'] = record.get("hostname", "")
559 # xxx TODO: URI, LatLong, IP, DNS
561 elif (type == "user"):
562 sfa_info['email'] = record.get("email", "")
563 sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
564 sfa_info['geni_certificate'] = record['gid']
565 # xxx TODO: PostalAddress, Phone
566 record.update(sfa_info)
568 def fill_record_info(self, records):
570 Given a SFA record, fill in the PLC specific and SFA specific
571 fields in the record.
573 if not isinstance(records, list):
576 self.fill_record_pl_info(records)
577 self.fill_record_sfa_info(records)
579 def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc):
580 # get a list of the HRNs that are members of the old and new records
582 oldList = oldRecord.get(listName, [])
585 newList = record.get(listName, [])
586 # ugly hack to see what's next
587 #def normalize (value):
588 # from types import StringTypes
589 # from sfa.util.sfalogging import logger
590 # if isinstance(value,StringTypes): return value
591 # elif isinstance(value,dict):
592 # newvalue=value['text']
593 # logger.info("Normalizing %s=>%s"%(value,newvalue))
595 #newList=[normalize(v) for v in newList]
597 # if the lists are the same, then we don't have to update anything
598 if (oldList == newList):
601 # build a list of the new person ids, by looking up each person to get
604 table = self.SfaTable()
605 records = table.find({'type': 'user', 'hrn': newList})
607 newIdList.append(rec['pointer'])
609 # build a list of the old person ids from the person_ids field
611 oldIdList = oldRecord.get("person_ids", [])
612 containerId = oldRecord.get_pointer()
614 # if oldRecord==None, then we are doing a Register, instead of an
617 containerId = record.get_pointer()
619 # add people who are in the new list, but not the oldList
620 for personId in newIdList:
621 if not (personId in oldIdList):
622 addFunc(self.plauth, personId, containerId)
624 # remove people who are in the old list, but not the new list
625 for personId in oldIdList:
626 if not (personId in newIdList):
627 delFunc(self.plauth, personId, containerId)
629 def update_membership(self, oldRecord, record):
630 if record.type == "slice":
631 self.update_membership_list(oldRecord, record, 'researcher',
632 self.plshell.AddPersonToSlice,
633 self.plshell.DeletePersonFromSlice)
634 elif record.type == "authority":
640 class ComponentAPI(BaseAPI):
642 def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", methods='sfa.methods',
643 peer_cert = None, interface = None, key_file = None, cert_file = None):
645 BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert,
646 interface=interface, key_file=key_file, cert_file=cert_file)
647 self.encoding = encoding
649 # Better just be documenting the API
653 self.nodemanager = NodeManager(self.config)
655 def sliver_exists(self):
656 sliver_dict = self.nodemanager.GetXIDs()
657 ### xxx slicename is undefined
658 if slicename in sliver_dict.keys():
663 def get_registry(self):
664 addr, port = self.config.SFA_REGISTRY_HOST, self.config.SFA_REGISTRY_PORT
665 url = "http://%(addr)s:%(port)s" % locals()
666 server = xmlrpcprotocol.get_server(url, self.key_file, self.cert_file)
669 def get_node_key(self):
670 # this call requires no authentication,
671 # so we can generate a random keypair here
673 (kfd, keyfile) = tempfile.mkstemp()
674 (cfd, certfile) = tempfile.mkstemp()
675 key = Keypair(create=True)
676 key.save_to_file(keyfile)
677 cert = Certificate(subject=subject)
678 cert.set_issuer(key=key, subject=subject)
681 cert.save_to_file(certfile)
682 registry = self.get_registry()
683 # the registry will scp the key onto the node
686 def getCredential(self):
688 Get our credential from a remote registry
690 path = self.config.SFA_DATA_DIR
691 config_dir = self.config.config_path
692 cred_filename = path + os.sep + 'node.cred'
694 credential = Credential(filename = cred_filename)
695 return credential.save_to_string(save_parents=True)
697 node_pkey_file = config_dir + os.sep + "node.key"
698 node_gid_file = config_dir + os.sep + "node.gid"
699 cert_filename = path + os.sep + 'server.cert'
700 if not os.path.exists(node_pkey_file) or \
701 not os.path.exists(node_gid_file):
705 gid = GID(filename=node_gid_file)
707 # get credential from registry
708 cert_str = Certificate(filename=cert_filename).save_to_string(save_parents=True)
709 registry = self.get_registry()
710 cred = registry.GetSelfCredential(cert_str, hrn, 'node')
711 # xxx credfile is undefined
712 Credential(string=cred).save_to_file(credfile, save_parents=True)
716 def clean_key_cred(self):
718 remove the existing keypair and cred and generate new ones
720 files = ["server.key", "server.cert", "node.cred"]
722 # xxx KEYDIR is undefined, could be meant to be "/var/lib/sfa/" from sfa_component_setup.py
723 filepath = KEYDIR + os.sep + f
724 if os.path.isfile(filepath):
727 # install the new key pair
728 # GetCredential will take care of generating the new keypair