sfa/plc/slices.py

   1 import datetime
   2 import time
   3 import traceback
   4 import sys
   5
   6 from types import StringTypes
   7 from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn, urn_to_hrn
   8 from sfa.util.plxrn import hrn_to_pl_slicename
   9 from sfa.util.rspec import *
  10 from sfa.util.specdict import *
  11 from sfa.util.faults import *
  12 from sfa.util.record import SfaRecord
  13 from sfa.util.policy import Policy
  14 from sfa.util.prefixTree import prefixTree
  15
  16 MAXINT =  2L**31-1
  17
  18 class Slices:
  19
  20     rspec_to_slice_tag = {'max_rate':'net_max_rate'}
  21
  22     def __init__(self, api, ttl = .5, origin_hrn=None):
  23         self.api = api
  24         #filepath = path + os.sep + filename
  25         self.policy = Policy(self.api)
  26         self.origin_hrn = origin_hrn
  27
  28     def get_slivers(self, xrn, node=None):
  29         hrn, type = urn_to_hrn(xrn)
  30
  31         slice_name = hrn_to_pl_slicename(hrn)
  32         # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
  33         # of doing all of this?
  34         #return self.api.GetSliceTicket(self.auth, slice_name)
  35
  36         # from PLCAPI.GetSlivers.get_slivers()
  37         slice_fields = ['slice_id', 'name', 'instantiation', 'expires', 'person_ids', 'slice_tag_ids']
  38         slices = self.api.plshell.GetSlices(self.api.plauth, slice_name, slice_fields)
  39         # Build up list of users and slice attributes
  40         person_ids = set()
  41         all_slice_tag_ids = set()
  42         for slice in slices:
  43             person_ids.update(slice['person_ids'])
  44             all_slice_tag_ids.update(slice['slice_tag_ids'])
  45         person_ids = list(person_ids)
  46         all_slice_tag_ids = list(all_slice_tag_ids)
  47         # Get user information
  48         all_persons_list = self.api.plshell.GetPersons(self.api.plauth, {'person_id':person_ids,'enabled':True}, ['person_id', 'enabled', 'key_ids'])
  49         all_persons = {}
  50         for person in all_persons_list:
  51             all_persons[person['person_id']] = person
  52
  53         # Build up list of keys
  54         key_ids = set()
  55         for person in all_persons.values():
  56             key_ids.update(person['key_ids'])
  57         key_ids = list(key_ids)
  58         # Get user account keys
  59         all_keys_list = self.api.plshell.GetKeys(self.api.plauth, key_ids, ['key_id', 'key', 'key_type'])
  60         all_keys = {}
  61         for key in all_keys_list:
  62             all_keys[key['key_id']] = key
  63         # Get slice attributes
  64         all_slice_tags_list = self.api.plshell.GetSliceTags(self.api.plauth, all_slice_tag_ids)
  65         all_slice_tags = {}
  66         for slice_tag in all_slice_tags_list:
  67             all_slice_tags[slice_tag['slice_tag_id']] = slice_tag
  68
  69         slivers = []
  70         for slice in slices:
  71             keys = []
  72             for person_id in slice['person_ids']:
  73                 if person_id in all_persons:
  74                     person = all_persons[person_id]
  75                     if not person['enabled']:
  76                         continue
  77                     for key_id in person['key_ids']:
  78                         if key_id in all_keys:
  79                             key = all_keys[key_id]
  80                             keys += [{'key_type': key['key_type'],
  81                                     'key': key['key']}]
  82             attributes = []
  83             # All (per-node and global) attributes for this slice
  84             slice_tags = []
  85             for slice_tag_id in slice['slice_tag_ids']:
  86                 if slice_tag_id in all_slice_tags:
  87                     slice_tags.append(all_slice_tags[slice_tag_id])
  88             # Per-node sliver attributes take precedence over global
  89             # slice attributes, so set them first.
  90             # Then comes nodegroup slice attributes
  91             # Followed by global slice attributes
  92             sliver_attributes = []
  93
  94             if node is not None:
  95                 for sliver_attribute in filter(lambda a: a['node_id'] == node['node_id'], slice_tags):
  96                     sliver_attributes.append(sliver_attribute['tagname'])
  97                     attributes.append({'tagname': sliver_attribute['tagname'],
  98                                     'value': sliver_attribute['value']})
  99
 100             # set nodegroup slice attributes
 101             for slice_tag in filter(lambda a: a['nodegroup_id'] in node['nodegroup_ids'], slice_tags):
 102                 # Do not set any nodegroup slice attributes for
 103                 # which there is at least one sliver attribute
 104                 # already set.
 105                 if slice_tag not in slice_tags:
 106                     attributes.append({'tagname': slice_tag['tagname'],
 107                         'value': slice_tag['value']})
 108
 109             for slice_tag in filter(lambda a: a['node_id'] is None, slice_tags):
 110                 # Do not set any global slice attributes for
 111                 # which there is at least one sliver attribute
 112                 # already set.
 113                 if slice_tag['tagname'] not in sliver_attributes:
 114                     attributes.append({'tagname': slice_tag['tagname'],
 115                                    'value': slice_tag['value']})
 116
 117             # XXX Sanity check; though technically this should be a system invariant
 118             # checked with an assertion
 119             if slice['expires'] > MAXINT:  slice['expires']= MAXINT
 120
 121             slivers.append({
 122                 'hrn': hrn,
 123                 'name': slice['name'],
 124                 'slice_id': slice['slice_id'],
 125                 'instantiation': slice['instantiation'],
 126                 'expires': slice['expires'],
 127                 'keys': keys,
 128                 'attributes': attributes
 129             })
 130
 131         return slivers
 132
 133     def get_peer(self, xrn):
 134         hrn, type = urn_to_hrn(xrn)
 135         # Becaues of myplc federation,  we first need to determine if this
 136         # slice belongs to out local plc or a myplc peer. We will assume it
 137         # is a local site, unless we find out otherwise
 138         peer = None
 139
 140         # get this slice's authority (site)
 141         slice_authority = get_authority(hrn)
 142
 143         # get this site's authority (sfa root authority or sub authority)
 144         site_authority = get_authority(slice_authority).lower()
 145
 146         # check if we are already peered with this site_authority, if so
 147         peers = self.api.plshell.GetPeers(self.api.plauth, {}, ['peer_id', 'peername', 'shortname', 'hrn_root'])
 148         for peer_record in peers:
 149             names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)]
 150             if site_authority in names:
 151                 peer = peer_record['shortname']
 152
 153         return peer
 154
 155     def get_sfa_peer(self, xrn):
 156         hrn, type = urn_to_hrn(xrn)
 157
 158         # return the authority for this hrn or None if we are the authority
 159         sfa_peer = None
 160         slice_authority = get_authority(hrn)
 161         site_authority = get_authority(slice_authority)
 162
 163         if site_authority != self.api.hrn:
 164             sfa_peer = site_authority
 165
 166         return sfa_peer
 167
 168     def verify_site(self, registry, credential, slice_hrn, peer, sfa_peer, reg_objects=None):
 169         authority = get_authority(slice_hrn)
 170         authority_urn = hrn_to_urn(authority, 'authority')
 171         login_base = None
 172         if reg_objects:
 173             site = reg_objects['site']
 174             login_base = site['login_base']
 175         else:
 176             site_records = registry.Resolve(authority_urn, [credential])
 177             site = {}
 178             for site_record in site_records:
 179                 if site_record['type'] == 'authority':
 180                     site = site_record
 181             if not site:
 182                 raise RecordNotFound(authority)
 183
 184         remote_site_id = site.pop('site_id')
 185
 186         if login_base is None:
 187             login_base = get_leaf(authority)
 188         sites = self.api.plshell.GetSites(self.api.plauth, login_base)
 189
 190         if not sites:
 191             site_id = self.api.plshell.AddSite(self.api.plauth, site)
 192             if peer:
 193                 try:
 194                     self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', site_id, peer, remote_site_id)
 195                 except Exception,e:
 196                     self.api.plshell.DeleteSite(self.api.plauth, site_id)
 197                     raise e
 198             # mark this site as an sfa peer record
 199             if sfa_peer and not reg_objects:
 200                 peer_dict = {'type': 'authority', 'hrn': authority, 'peer_authority': sfa_peer, 'pointer': site_id}
 201                 registry.register_peer_object(credential, peer_dict)
 202         else:
 203             site_id = sites[0]['site_id']
 204             remote_site_id = sites[0]['peer_site_id']
 205
 206             old_site = sites[0]
 207             #the site is already on the remote agg. Let us update(e.g. max_slices field) it with the latest info.
 208             self.sync_site(old_site, site, peer)
 209
 210
 211         return (site_id, remote_site_id)
 212
 213     def verify_slice(self, registry, credential, slice_hrn, site_id, remote_site_id, peer, sfa_peer, reg_objects=None):
 214         slice = {}
 215         slice_record = None
 216         authority = get_authority(slice_hrn)
 217
 218         if reg_objects:
 219             slice_record = reg_objects['slice_record']
 220         else:
 221             slice_records = registry.Resolve(slice_hrn, [credential])
 222
 223             for record in slice_records:
 224                 if record['type'] in ['slice']:
 225                     slice_record = record
 226             if not slice_record:
 227                 raise RecordNotFound(hrn)
 228
 229
 230         slicename = hrn_to_pl_slicename(slice_hrn)
 231         parts = slicename.split("_")
 232         login_base = parts[0]
 233         slices = self.api.plshell.GetSlices(self.api.plauth, [slicename])
 234         if not slices:
 235             slice_fields = {}
 236             slice_keys = ['name', 'url', 'description']
 237             for key in slice_keys:
 238                 if key in slice_record and slice_record[key]:
 239                     slice_fields[key] = slice_record[key]
 240             # add the slice
 241             slice_id = self.api.plshell.AddSlice(self.api.plauth, slice_fields)
 242             slice = slice_fields
 243             slice['slice_id'] = slice_id
 244
 245             # mark this slice as an sfa peer record
 246             if sfa_peer:
 247                 peer_dict = {'type': 'slice', 'hrn': slice_hrn, 'peer_authority': sfa_peer, 'pointer': slice_id}
 248                 registry.register_peer_object(credential, peer_dict)
 249
 250             #this belongs to a peer
 251             if peer:
 252                 try:
 253                     self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice_id, peer, slice_record['pointer'])
 254                 except Exception,e:
 255                     self.api.plshell.DeleteSlice(self.api.plauth,slice_id)
 256                     raise e
 257             slice['node_ids'] = []
 258         else:
 259             slice = slices[0]
 260             slice_id = slice['slice_id']
 261             site_id = slice['site_id']
 262             #the slice is alredy on the remote agg. Let us update(e.g. expires field) it with the latest info.
 263             self.sync_slice(slice, slice_record, peer)
 264
 265         slice['peer_slice_id'] = slice_record['pointer']
 266         self.verify_persons(registry, credential, slice_record, site_id, remote_site_id, peer, sfa_peer, reg_objects)
 267
 268         return slice
 269
 270     def verify_persons(self, registry, credential, slice_record, site_id, remote_site_id, peer, sfa_peer, reg_objects=None):
 271         # get the list of valid slice users from the registry and make
 272         # sure they are added to the slice
 273         slicename = hrn_to_pl_slicename(slice_record['hrn'])
 274         if reg_objects:
 275             researchers = reg_objects['users'].keys()
 276         else:
 277             researchers = slice_record.get('researcher', [])
 278         for researcher in researchers:
 279             if reg_objects:
 280                 person_dict = reg_objects['users'][researcher]
 281             else:
 282                 person_records = registry.Resolve(researcher, [credential])
 283                 for record in person_records:
 284                     if record['type'] in ['user'] and record['enabled']:
 285                         person_record = record
 286                 if not person_record:
 287                     return 1
 288                 person_dict = person_record
 289
 290             local_person=False
 291             if peer:
 292                 peer_id = self.api.plshell.GetPeers(self.api.plauth, {'shortname': peer}, ['peer_id'])[0]['peer_id']
 293                 persons = self.api.plshell.GetPersons(self.api.plauth, {'email': [person_dict['email']], 'peer_id': peer_id}, ['person_id', 'key_ids'])
 294                 if not persons:
 295                     persons = self.api.plshell.GetPersons(self.api.plauth, [person_dict['email']], ['person_id', 'key_ids'])
 296                     if persons:
 297                         local_person=True
 298
 299             else:
 300                 persons = self.api.plshell.GetPersons(self.api.plauth, [person_dict['email']], ['person_id', 'key_ids'])
 301
 302             if not persons:
 303                 person_id=self.api.plshell.AddPerson(self.api.plauth, person_dict)
 304                 self.api.plshell.UpdatePerson(self.api.plauth, person_id, {'enabled' : True})
 305
 306                 # mark this person as an sfa peer record
 307                 if sfa_peer:
 308                     peer_dict = {'type': 'user', 'hrn': researcher, 'peer_authority': sfa_peer, 'pointer': person_id}
 309                     registry.register_peer_object(credential, peer_dict)
 310
 311                 if peer:
 312                     try:
 313                         self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
 314                     except Exception,e:
 315                         self.api.plshell.DeletePerson(self.api.plauth,person_id)
 316                         raise e
 317                 key_ids = []
 318             else:
 319                 person_id = persons[0]['person_id']
 320                 key_ids = persons[0]['key_ids']
 321
 322
 323             # if this is a peer person, we must unbind them from the peer or PLCAPI will throw
 324             # an error
 325             try:
 326                 if peer and not local_person:
 327                     self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'person', person_id, peer)
 328                 if peer:
 329                     self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'site', site_id,  peer)
 330
 331                 self.api.plshell.AddPersonToSlice(self.api.plauth, person_dict['email'], slicename)
 332                 self.api.plshell.AddPersonToSite(self.api.plauth, person_dict['email'], site_id)
 333             finally:
 334                 if peer:
 335                     try: self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', site_id, peer, remote_site_id)
 336                     except: pass
 337                 if peer and not local_person:
 338                     try: self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
 339                     except: pass
 340
 341             self.verify_keys(registry, credential, person_dict, key_ids, person_id, peer, local_person)
 342
 343     def verify_keys(self, registry, credential, person_dict, key_ids, person_id,  peer, local_person):
 344         keylist = self.api.plshell.GetKeys(self.api.plauth, key_ids, ['key'])
 345         keys = [key['key'] for key in keylist]
 346
 347         #add keys that arent already there
 348         key_ids = person_dict['key_ids']
 349         for personkey in person_dict['keys']:
 350             if personkey not in keys:
 351                 key = {'key_type': 'ssh', 'key': personkey}
 352                 try:
 353                     if peer:
 354                         self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'person', person_id, peer)
 355                     key_id = self.api.plshell.AddPersonKey(self.api.plauth, person_dict['email'], key)
 356                 finally:
 357                     if peer and not local_person:
 358                         self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
 359                     if peer:
 360                         # xxx - thierry how are we getting the peer_key_id in here ?
 361                         try: self.api.plshell.BindObjectToPeer(self.api.plauth, 'key', key_id, peer, key_ids.pop(0))
 362                         except: pass
 363
 364     def create_slice_aggregate(self, xrn, rspec):
 365         hrn, type = urn_to_hrn(xrn)
 366         # Determine if this is a peer slice
 367         peer = self.get_peer(hrn)
 368         sfa_peer = self.get_sfa_peer(hrn)
 369
 370         spec = RSpec(rspec)
 371         # Get the slice record from sfa
 372         slicename = hrn_to_pl_slicename(hrn)
 373         slice = {}
 374         slice_record = None
 375         registry = self.api.registries[self.api.hrn]
 376         credential = self.api.getCredential()
 377
 378         site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer)
 379         slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer)
 380
 381         # find out where this slice is currently running
 382         nodelist = self.api.plshell.GetNodes(self.api.plauth, slice['node_ids'], ['hostname'])
 383         hostnames = [node['hostname'] for node in nodelist]
 384
 385         # get netspec details
 386         nodespecs = spec.getDictsByTagName('NodeSpec')
 387
 388         # dict in which to store slice attributes to set for the nodes
 389         nodes = {}
 390         for nodespec in nodespecs:
 391             if isinstance(nodespec['name'], list):
 392                 for nodename in nodespec['name']:
 393                     nodes[nodename] = {}
 394                     for k in nodespec.keys():
 395                         rspec_attribute_value = nodespec[k]
 396                         if (self.rspec_to_slice_tag.has_key(k)):
 397                             slice_tag_name = self.rspec_to_slice_tag[k]
 398                             nodes[nodename][slice_tag_name] = rspec_attribute_value
 399             elif isinstance(nodespec['name'], StringTypes):
 400                 nodename = nodespec['name']
 401                 nodes[nodename] = {}
 402                 for k in nodespec.keys():
 403                     rspec_attribute_value = nodespec[k]
 404                     if (self.rspec_to_slice_tag.has_key(k)):
 405                         slice_tag_name = self.rspec_to_slice_tag[k]
 406                         nodes[nodename][slice_tag_name] = rspec_attribute_value
 407
 408                 for k in nodespec.keys():
 409                     rspec_attribute_value = nodespec[k]
 410                     if (self.rspec_to_slice_tag.has_key(k)):
 411                         slice_tag_name = self.rspec_to_slice_tag[k]
 412                         nodes[nodename][slice_tag_name] = rspec_attribute_value
 413
 414         node_names = nodes.keys()
 415         # remove nodes not in rspec
 416         deleted_nodes = list(set(hostnames).difference(node_names))
 417         # add nodes from rspec
 418         added_nodes = list(set(node_names).difference(hostnames))
 419
 420         try:
 421             if peer:
 422                 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', slice['slice_id'], peer)
 423
 424             self.api.plshell.AddSliceToNodes(self.api.plauth, slicename, added_nodes)
 425
 426             # Add recognized slice tags
 427             for node_name in node_names:
 428                 node = nodes[node_name]
 429                 for slice_tag in node.keys():
 430                     value = node[slice_tag]
 431                     if (isinstance(value, list)):
 432                         value = value[0]
 433
 434                     self.api.plshell.AddSliceTag(self.api.plauth, slicename, slice_tag, value, node_name)
 435
 436             self.api.plshell.DeleteSliceFromNodes(self.api.plauth, slicename, deleted_nodes)
 437         finally:
 438             if peer:
 439                 self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice['slice_id'], peer, slice['peer_slice_id'])
 440
 441         return 1
 442
 443     def sync_site(self, old_record, new_record, peer):
 444         if old_record['max_slices'] != new_record['max_slices'] or old_record['max_slivers'] != new_record['max_slivers']:
 445             try:
 446                 if peer:
 447                     self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'site', old_record['site_id'], peer)
 448                 if old_record['max_slices'] != new_record['max_slices']:
 449                     self.api.plshell.UpdateSite(self.api.plauth, old_record['site_id'], {'max_slices' : new_record['max_slices']})
 450                 if old_record['max_slivers'] != new_record['max_slivers']:
 451                     self.api.plshell.UpdateSite(self.api.plauth, old_record['site_id'], {'max_slivers' : new_record['max_slivers']})
 452             finally:
 453                 if peer:
 454                     self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', old_record['site_id'], peer, old_record['peer_site_id'])
 455         return 1
 456
 457     def sync_slice(self, old_record, new_record, peer):
 458         if old_record['expires'] != new_record['expires']:
 459             try:
 460                 if peer:
 461                     self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', old_record['slice_id'], peer)
 462                 self.api.plshell.UpdateSlice(self.api.plauth, old_record['slice_id'], {'expires' : new_record['expires']})
 463             finally:
 464                 if peer:
 465                     self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', old_record['slice_id'], peer, old_record['peer_slice_id'])
 466         return 1