6 from types import StringTypes
7 from sfa.util.xrn import Xrn, get_leaf, get_authority, hrn_to_urn, urn_to_hrn
8 from sfa.util.plxrn import hrn_to_pl_slicename, hrn_to_pl_login_base
9 from sfa.util.rspec import *
10 from sfa.util.specdict import *
11 from sfa.util.faults import *
12 from sfa.util.record import SfaRecord
13 from sfa.util.policy import Policy
14 from sfa.util.prefixTree import prefixTree
15 from collections import defaultdict
21 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
23 def __init__(self, api, ttl = .5, origin_hrn=None):
25 #filepath = path + os.sep + filename
26 self.policy = Policy(self.api)
27 self.origin_hrn = origin_hrn
28 self.registry = api.registries[api.hrn]
29 self.credential = api.getCredential()
31 def get_slivers(self, xrn, node=None):
32 hrn, type = urn_to_hrn(xrn)
34 slice_name = hrn_to_pl_slicename(hrn)
35 # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
36 # of doing all of this?
37 #return self.api.GetSliceTicket(self.auth, slice_name)
39 # from PLCAPI.GetSlivers.get_slivers()
40 slice_fields = ['slice_id', 'name', 'instantiation', 'expires', 'person_ids', 'slice_tag_ids']
41 slices = self.api.plshell.GetSlices(self.api.plauth, slice_name, slice_fields)
42 # Build up list of users and slice attributes
44 all_slice_tag_ids = set()
46 person_ids.update(slice['person_ids'])
47 all_slice_tag_ids.update(slice['slice_tag_ids'])
48 person_ids = list(person_ids)
49 all_slice_tag_ids = list(all_slice_tag_ids)
50 # Get user information
51 all_persons_list = self.api.plshell.GetPersons(self.api.plauth, {'person_id':person_ids,'enabled':True}, ['person_id', 'enabled', 'key_ids'])
53 for person in all_persons_list:
54 all_persons[person['person_id']] = person
56 # Build up list of keys
58 for person in all_persons.values():
59 key_ids.update(person['key_ids'])
60 key_ids = list(key_ids)
61 # Get user account keys
62 all_keys_list = self.api.plshell.GetKeys(self.api.plauth, key_ids, ['key_id', 'key', 'key_type'])
64 for key in all_keys_list:
65 all_keys[key['key_id']] = key
66 # Get slice attributes
67 all_slice_tags_list = self.api.plshell.GetSliceTags(self.api.plauth, all_slice_tag_ids)
69 for slice_tag in all_slice_tags_list:
70 all_slice_tags[slice_tag['slice_tag_id']] = slice_tag
75 for person_id in slice['person_ids']:
76 if person_id in all_persons:
77 person = all_persons[person_id]
78 if not person['enabled']:
80 for key_id in person['key_ids']:
81 if key_id in all_keys:
82 key = all_keys[key_id]
83 keys += [{'key_type': key['key_type'],
86 # All (per-node and global) attributes for this slice
88 for slice_tag_id in slice['slice_tag_ids']:
89 if slice_tag_id in all_slice_tags:
90 slice_tags.append(all_slice_tags[slice_tag_id])
91 # Per-node sliver attributes take precedence over global
92 # slice attributes, so set them first.
93 # Then comes nodegroup slice attributes
94 # Followed by global slice attributes
95 sliver_attributes = []
98 for sliver_attribute in filter(lambda a: a['node_id'] == node['node_id'], slice_tags):
99 sliver_attributes.append(sliver_attribute['tagname'])
100 attributes.append({'tagname': sliver_attribute['tagname'],
101 'value': sliver_attribute['value']})
103 # set nodegroup slice attributes
104 for slice_tag in filter(lambda a: a['nodegroup_id'] in node['nodegroup_ids'], slice_tags):
105 # Do not set any nodegroup slice attributes for
106 # which there is at least one sliver attribute
108 if slice_tag not in slice_tags:
109 attributes.append({'tagname': slice_tag['tagname'],
110 'value': slice_tag['value']})
112 for slice_tag in filter(lambda a: a['node_id'] is None, slice_tags):
113 # Do not set any global slice attributes for
114 # which there is at least one sliver attribute
116 if slice_tag['tagname'] not in sliver_attributes:
117 attributes.append({'tagname': slice_tag['tagname'],
118 'value': slice_tag['value']})
120 # XXX Sanity check; though technically this should be a system invariant
121 # checked with an assertion
122 if slice['expires'] > MAXINT: slice['expires']= MAXINT
126 'name': slice['name'],
127 'slice_id': slice['slice_id'],
128 'instantiation': slice['instantiation'],
129 'expires': slice['expires'],
131 'attributes': attributes
136 def get_peer(self, xrn):
137 hrn, type = urn_to_hrn(xrn)
138 # Becaues of myplc federation, we first need to determine if this
139 # slice belongs to out local plc or a myplc peer. We will assume it
140 # is a local site, unless we find out otherwise
143 # get this slice's authority (site)
144 slice_authority = get_authority(hrn)
146 # get this site's authority (sfa root authority or sub authority)
147 site_authority = get_authority(slice_authority).lower()
149 # check if we are already peered with this site_authority, if so
150 peers = self.api.plshell.GetPeers(self.api.plauth, {}, ['peer_id', 'peername', 'shortname', 'hrn_root'])
151 for peer_record in peers:
152 names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)]
153 if site_authority in names:
158 def get_sfa_peer(self, xrn):
159 hrn, type = urn_to_hrn(xrn)
161 # return the authority for this hrn or None if we are the authority
163 slice_authority = get_authority(hrn)
164 site_authority = get_authority(slice_authority)
166 if site_authority != self.api.hrn:
167 sfa_peer = site_authority
171 def verify_slice_nodes(self, slice, requested_slivers, peer):
173 nodes = self.api.plshell.GetNodes(self.api.plauth, slice['node_ids'], ['hostname'])
174 current_slivers = [node['hostname'] for node in nodes]
176 # remove nodes not in rspec
177 deleted_nodes = list(set(current_slivers).difference(requested_slivers))
179 # add nodes from rspec
180 added_nodes = list(set(requested_slivers).difference(current_slivers))
184 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', slice['slice_id'], peer['shortname'])
185 self.api.plshell.AddSliceToNodes(self.api.plauth, slice['name'], added_nodes)
186 self.api.plshell.DeleteSliceFromNodes(self.api.plauth, slice['name'], deleted_nodes)
189 self.api.logger.log_exc('Failed to add/remove slice from nodes')
191 def handle_peer(self, site, slice, persons, peer):
196 self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', \
197 site['site_id'], peer['shortname'], slice['site_id'])
199 self.api.plshell.DeleteSite(self.api.plauth, site['site_id'])
205 self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', \
206 slice['slice_id'], peer['shortname'], slice['slice_id'])
208 self.api.plshell.DeleteSlice(self.api.plauth, slice['slice_id'])
212 for person in persons:
214 self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', \
215 person['person_id'], peer['shortname'], person['peer_person_id'])
217 for (key, remote_key_id) in zip(person['keys'], person['key_ids']):
219 self.api.plshell.BindObjectToPeer(self.api.plauth, 'key',\
220 key['key_id'], peer['shortname'], remote_key_id)
222 self.api.plshell.DeleteKey(self.api.plauth, key['key_id'])
223 self.api.logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname']))
225 self.api.plshell.DeletePerson(self.api.plauth, person['person_id'])
230 def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None):
231 (slice_hrn, type) = urn_to_hrn(slice_xrn)
232 site_hrn = get_authority(slice_hrn)
233 # login base can't be longer than 20 characters
234 slicename = hrn_to_pl_slicename(slice_hrn)
235 authority_name = slicename.split('_')[0]
236 login_base = authority_name[:20]
237 sites = self.api.plshell.GetSites(self.api.plauth, login_base)
239 # create new site record
240 site = {'name': 'geni.%s' % authority_name,
241 'abbreviated_name': authority_name,
242 'login_base': login_base,
246 'peer_site_id': None}
248 site['peer_site_id'] = slice_record.get('site_id', None)
249 site['site_id'] = self.api.plshell.AddSite(self.api.plauth, site)
250 # exempt federated sites from monitor policies
251 self.api.plshell.AddSiteTag(self.api.plauth, site['site_id'], 'exempt_site_until', "20200101")
253 # is this still necessary?
254 # add record to the local registry
255 if sfa_peer and slice_record:
256 peer_dict = {'type': 'authority', 'hrn': site_hrn, \
257 'peer_authority': sfa_peer, 'pointer': site['site_id']}
258 self.registry.register_peer_object(self.credential, peer_dict)
262 # unbind from peer so we can modify if necessary. Will bind back later
263 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'site', site['site_id'], peer['shortname'])
267 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
268 slicename = hrn_to_pl_slicename(slice_hrn)
269 parts = slicename.split("_")
270 login_base = parts[0]
271 slices = self.api.plshell.GetSlices(self.api.plauth, [slicename])
273 slice = {'name': slicename,
274 'url': slice_record.get('url', slice_hrn),
275 'description': slice_record.get('description', slice_hrn)}
277 slice['slice_id'] = self.api.plshell.AddSlice(self.api.plauth, slice)
278 slice['node_ids'] = []
279 slice['person_ids'] = []
281 slice['peer_slice_id'] = slice_record.get('slice_id', None)
282 # mark this slice as an sfa peer record
284 peer_dict = {'type': 'slice', 'hrn': slice_hrn,
285 'peer_authority': sfa_peer, 'pointer': slice['slice_id']}
286 self.registry.register_peer_object(self.credential, peer_dict)
290 slice['peer_slice_id'] = slice_record.get('slice_id', None)
291 # unbind from peer so we can modify if necessary. Will bind back later
292 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice',\
293 slice['slice_id'], peer['shortname'])
294 #Update existing record (e.g. expires field) it with the latest info.
295 if slice_record and slice['expires'] != slice_record['expires']:
296 self.api.plshell.UpdateSlice(self.api.plauth, slice['slice_id'],\
297 {'expires' : slice_record['expires']})
301 #def get_existing_persons(self, users):
302 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, append=True):
304 users_by_site = defaultdict(list)
308 if 'append' in user and user['append'] == False:
311 users_by_email[user['email']] = user
312 users_dict[user['email']] = user
314 hrn, type = urn_to_hrn(user['urn'])
315 username = get_leaf(hrn)
316 login_base = get_leaf(get_authority(user['urn']))
317 user['username'] = username
318 users_by_site[login_base].append(user)
320 existing_user_ids = []
322 # get existing users by email
323 existing_users = self.api.plshell.GetPersons(self.api.plauth, \
324 {'email': users_by_email.keys()}, ['person_id', 'key_ids', 'email'])
325 existing_user_ids.extend([user['email'] for user in existing_users])
328 # get a list of user sites (based on requeste user urns
329 site_list = self.api.plshell.GetSites(self.api.plauth, users_by_site.keys(), \
330 ['site_id', 'login_base', 'person_ids'])
334 # get all existing users at these sites
335 for site in site_list:
336 sites[site['site_id']] = site
337 site_user_ids.extend(site['person_ids'])
339 existing_site_persons_list = self.api.plshell.GetPersons(self.api.plauth, \
340 site_user_ids, ['person_id', 'key_ids', 'email', 'site_ids'])
342 # all requested users are either existing users or new (added) users
343 for login_base in users_by_site:
344 requested_site_users = users_by_site[login_base]
345 for requested_user in requested_site_users:
347 for existing_user in existing_site_persons_list:
348 for site_id in existing_user['site_ids']:
349 site = sites[site_id]
350 if login_base == site['login_base'] and \
351 existing_user['email'].startswith(requested_user['username']):
352 existing_user_ids.append(existing_user['email'])
353 users_dict[existing_user['email']] = requested_user
359 if user_found == False:
360 fake_email = requested_user['username'] + '@geni.net'
361 users_dict[fake_email] = requested_user
364 # requested slice users
365 requested_user_ids = users_dict.keys()
366 # existing slice users
367 existing_slice_users_filter = {'person_id': slice_record.get('person_ids', [])}
368 existing_slice_users = self.api.plshell.GetPersons(self.api.plauth, \
369 existing_slice_users_filter, ['person_id', 'key_ids', 'email'])
370 existing_slice_user_ids = [user['email'] for user in existing_slice_users]
372 # users to be added, removed or updated
373 added_user_ids = set(requested_user_ids).difference(existing_user_ids)
374 added_slice_user_ids = set(requested_user_ids).difference(existing_slice_user_ids)
375 removed_user_ids = set(existing_slice_user_ids).difference(requested_user_ids)
376 updated_user_ids = set(existing_slice_user_ids).intersection(requested_user_ids)
378 # Remove stale users (only if we are not appending).
380 for removed_user_id in removed_user_ids:
381 self.api.plshell.DeletePersonFromSlice(self.api.plauth, removed_user_id, slice_record['name'])
382 # update_existing users
383 updated_users_list = [user for user in existing_slice_users if user['email'] in \
385 self.verify_keys(existing_slice_users, updated_users_list, peer, append)
389 for added_user_id in added_user_ids:
390 added_user = users_dict[added_user_id]
391 hrn, type = urn_to_hrn(added_user['urn'])
393 'first_name': added_user.get('first_name', hrn),
394 'last_name': added_user.get('last_name', hrn),
395 'email': added_user_id,
396 'peer_person_id': None,
398 'key_ids': added_user.get('key_ids', []),
400 person['person_id'] = self.api.plshell.AddPerson(self.api.plauth, person)
402 person['peer_person_id'] = added_user['person_id']
403 added_persons.append(person)
406 self.api.plshell.UpdatePerson(self.api.plauth, person['person_id'], {'enabled': True})
409 self.api.plshell.AddPersonToSite(self.api.plauth, added_user_id, login_base)
411 for key_string in added_user.get('keys', []):
412 key = {'key':key_string, 'key_type':'ssh'}
413 key['key_id'] = self.api.plshell.AddPersonKey(self.api.plauth, person['person_id'], key)
414 person['keys'].append(key)
416 # add the registry record
418 peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \
419 'pointer': person['person_id']}
420 self.registry.register_peer_object(self.credential, peer_dict)
422 for added_slice_user_id in added_slice_user_ids.union(added_user_ids):
423 # add person to the slice
424 self.api.plshell.AddPersonToSlice(self.api.plauth, added_slice_user_id, slice_record['name'])
425 # if this is a peer record then it should already be bound to a peer.
426 # no need to return worry about it getting bound later
431 def verify_keys(self, persons, users, peer, append=True):
434 for person in persons:
435 key_ids.extend(person['key_ids'])
436 keylist = self.api.plshell.GetKeys(self.api.plauth, key_ids, ['key_id', 'key'])
439 keydict[key['key']] = key['key_id']
440 existing_keys = keydict.keys()
442 for person in persons:
443 persondict[person['email']] = person
449 user_keys = user.get('keys', [])
450 updated_persons.append(user)
451 for key_string in user_keys:
452 requested_keys.append(key_string)
453 if key_string not in existing_keys:
454 key = {'key': key_string, 'key_type': 'ssh'}
457 person = persondict[user['email']]
458 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'person', person['person_id'], peer['shortname'])
459 key['key_id'] = self.api.plshell.AddPersonKey(self.api.plauth, user['email'], key)
461 key_index = user_keys.index(key['key'])
462 remote_key_id = user['key_ids'][key_index]
463 self.api.plshell.BindObjectToPeer(self.api.plauth, 'key', key['key_id'], peer['shortname'], remote_key_id)
467 self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person['person_id'], peer['shortname'], user['person_id'])
469 # remove old keys (only if we are not appending)
471 removed_keys = set(existing_keys).difference(requested_keys)
472 for existing_key_id in keydict:
473 if keydict[existing_key_id] in removed_keys:
476 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'key', existing_key_id, peer['shortname'])
477 self.api.plshell.DeleteKey(self.api.plauth, existing_key_id)
481 def verify_slice_attributes(self, slice, requested_slice_attributes):
482 # get list of attributes users ar able to manage
483 slice_attributes = self.api.plshell.GetTagTypes(self.api.plauth, {'category': '*slice*', '|roles': ['user']})
484 valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes]
486 # get sliver attributes
487 added_slice_attributes = []
488 removed_slice_attributes = []
489 ignored_slice_attribute_names = []
490 existing_slice_attributes = self.api.plshell.GetSliceTags(self.api.plauth, {'slice_id': slice['slice_id']})
492 # get attributes that should be removed
493 for slice_tag in existing_slice_attributes:
494 if slice_tag['tagname'] in ignored_slice_attribute_names:
495 # If a slice already has a admin only role it was probably given to them by an
496 # admin, so we should ignore it.
497 ignored_slice_attribute_names.append(slice_tag['tagname'])
499 # If an existing slice attribute was not found in the request it should
501 attribute_found=False
502 for requested_attribute in requested_slice_attributes:
503 if requested_attribute['name'] == slice_tag['tagname'] and \
504 requested_attribute['value'] == slice_tag['value']:
508 if not attribute_found:
509 removed_slice_attributes.append(slice_tag)
511 # get attributes that should be added:
512 for requested_attribute in requested_slice_attributes:
513 # if the requested attribute wasn't found we should add it
514 if requested_attribute['name'] in valid_slice_attribute_names:
515 attribute_found = False
516 for existing_attribute in existing_slice_attributes:
517 if requested_attribute['name'] == existing_attribute['tagname'] and \
518 requested_attribute['value'] == existing_attribute['value']:
521 if not attribute_found:
522 added_slice_attributes.append(requested_attribute)
525 # remove stale attributes
526 for attribute in removed_slice_attributes:
528 self.api.plshell.DeleteSliceTag(self.api.plauth, attribute['slice_tag_id'])
530 self.api.logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
531 % (name, value, node_id, str(e)))
533 # add requested_attributes
534 for attribute in added_slice_attributes:
536 name, value, node_id = attribute['name'], attribute['value'], attribute.get('node_id', None)
537 self.api.plshell.AddSliceTag(self.api.plauth, slice['name'], name, value, node_id)
539 self.api.logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
540 % (name, value, node_id, str(e)))
542 def create_slice_aggregate(self, xrn, rspec):
543 hrn, type = urn_to_hrn(xrn)
544 # Determine if this is a peer slice
545 peer = self.get_peer(hrn)
546 sfa_peer = self.get_sfa_peer(hrn)
549 # Get the slice record from sfa
550 slicename = hrn_to_pl_slicename(hrn)
553 registry = self.api.registries[self.api.hrn]
554 credential = self.api.getCredential()
556 site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer)
557 slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer)
559 # find out where this slice is currently running
560 nodelist = self.api.plshell.GetNodes(self.api.plauth, slice['node_ids'], ['hostname'])
561 hostnames = [node['hostname'] for node in nodelist]
563 # get netspec details
564 nodespecs = spec.getDictsByTagName('NodeSpec')
566 # dict in which to store slice attributes to set for the nodes
568 for nodespec in nodespecs:
569 if isinstance(nodespec['name'], list):
570 for nodename in nodespec['name']:
572 for k in nodespec.keys():
573 rspec_attribute_value = nodespec[k]
574 if (self.rspec_to_slice_tag.has_key(k)):
575 slice_tag_name = self.rspec_to_slice_tag[k]
576 nodes[nodename][slice_tag_name] = rspec_attribute_value
577 elif isinstance(nodespec['name'], StringTypes):
578 nodename = nodespec['name']
580 for k in nodespec.keys():
581 rspec_attribute_value = nodespec[k]
582 if (self.rspec_to_slice_tag.has_key(k)):
583 slice_tag_name = self.rspec_to_slice_tag[k]
584 nodes[nodename][slice_tag_name] = rspec_attribute_value
586 for k in nodespec.keys():
587 rspec_attribute_value = nodespec[k]
588 if (self.rspec_to_slice_tag.has_key(k)):
589 slice_tag_name = self.rspec_to_slice_tag[k]
590 nodes[nodename][slice_tag_name] = rspec_attribute_value
592 node_names = nodes.keys()
593 # remove nodes not in rspec
594 deleted_nodes = list(set(hostnames).difference(node_names))
595 # add nodes from rspec
596 added_nodes = list(set(node_names).difference(hostnames))
600 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', slice['slice_id'], peer)
602 self.api.plshell.AddSliceToNodes(self.api.plauth, slicename, added_nodes)
604 # Add recognized slice tags
605 for node_name in node_names:
606 node = nodes[node_name]
607 for slice_tag in node.keys():
608 value = node[slice_tag]
609 if (isinstance(value, list)):
612 self.api.plshell.AddSliceTag(self.api.plauth, slicename, slice_tag, value, node_name)
614 self.api.plshell.DeleteSliceFromNodes(self.api.plauth, slicename, deleted_nodes)
617 self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice['slice_id'], peer, slice['peer_slice_id'])