sfa/nitos/nitosslices.py

   1 from types import StringTypes
   2 from collections import defaultdict
   3
   4 from sfa.util.sfatime import utcparse, datetime_to_epoch
   5 from sfa.util.sfalogging import logger
   6 from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn
   7
   8 from sfa.rspecs.rspec import RSpec
   9
  10 from sfa.planetlab.vlink import VLink
  11 from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename
  12
  13 MAXINT =  2L**31-1
  14
  15 class NitosSlices:
  16
  17     rspec_to_slice_tag = {'max_rate':'net_max_rate'}
  18
  19     def __init__(self, driver):
  20         self.driver = driver
  21
  22     def get_peer(self, xrn):
  23         hrn, type = urn_to_hrn(xrn)
  24         #Does this slice belong to a local site or a peer NITOS site?
  25         peer = None
  26
  27         # get this slice's authority (site)
  28         slice_authority = get_authority(hrn)
  29
  30         # get this site's authority (sfa root authority or sub authority)
  31         site_authority = get_authority(slice_authority).lower()
  32
  33         # check if we are already peered with this site_authority, if so
  34         peers = self.driver.shell.GetPeers({}, ['peer_id', 'peername', 'shortname', 'hrn_root'])
  35         for peer_record in peers:
  36             names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)]
  37             if site_authority in names:
  38                 peer = peer_record
  39
  40         return peer
  41
  42     def get_sfa_peer(self, xrn):
  43         hrn, type = urn_to_hrn(xrn)
  44
  45         # return the authority for this hrn or None if we are the authority
  46         sfa_peer = None
  47         slice_authority = get_authority(hrn)
  48         site_authority = get_authority(slice_authority)
  49
  50         if site_authority != self.driver.hrn:
  51             sfa_peer = site_authority
  52
  53         return sfa_peer
  54
  55     def verify_slice_leases(self, slice, requested_leases, kept_leases, peer):
  56
  57         leases = self.driver.shell.GetLeases({'name':slice['name']}, ['lease_id'])
  58         grain = self.driver.shell.GetLeaseGranularity()
  59         current_leases = [lease['lease_id'] for lease in leases]
  60         deleted_leases = list(set(current_leases).difference(kept_leases))
  61
  62         try:
  63             if peer:
  64                 self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
  65             deleted=self.driver.shell.DeleteLeases(deleted_leases)
  66             for lease in requested_leases:
  67                 added=self.driver.shell.AddLeases(lease['hostname'], slice['name'], int(lease['start_time']), int(lease['duration']) * grain + int(lease['start_time']))
  68
  69         except:
  70             logger.log_exc('Failed to add/remove slice leases')
  71
  72         return leases
  73
  74
  75     def verify_slice_nodes(self, slice, requested_slivers, peer):
  76
  77         nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids'])
  78         current_slivers = [node['hostname'] for node in nodes]
  79
  80         # remove nodes not in rspec
  81         deleted_nodes = list(set(current_slivers).difference(requested_slivers))
  82
  83         # add nodes from rspec
  84         added_nodes = list(set(requested_slivers).difference(current_slivers))
  85
  86         try:
  87             if peer:
  88                 self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
  89             self.driver.shell.AddSliceToNodes(slice['name'], added_nodes)
  90             self.driver.shell.DeleteSliceFromNodes(slice['name'], deleted_nodes)
  91
  92         except:
  93             logger.log_exc('Failed to add/remove slice from nodes')
  94         return nodes
  95
  96     def free_egre_key(self):
  97         used = set()
  98         for tag in self.driver.shell.GetSliceTags({'tagname': 'egre_key'}):
  99                 used.add(int(tag['value']))
 100
 101         for i in range(1, 256):
 102             if i not in used:
 103                 key = i
 104                 break
 105         else:
 106             raise KeyError("No more EGRE keys available")
 107
 108         return str(key)
 109
 110     def verify_slice_links(self, slice, requested_links, nodes):
 111         # nodes is undefined here
 112         if not requested_links:
 113             return
 114
 115         # build dict of nodes
 116         nodes_dict = {}
 117         interface_ids = []
 118         for node in nodes:
 119             nodes_dict[node['node_id']] = node
 120             interface_ids.extend(node['interface_ids'])
 121         # build dict of interfaces
 122         interfaces = self.driver.shell.GetInterfaces(interface_ids)
 123         interfaces_dict = {}
 124         for interface in interfaces:
 125             interfaces_dict[interface['interface_id']] = interface
 126
 127         slice_tags = []
 128
 129         # set egre key
 130         slice_tags.append({'name': 'egre_key', 'value': self.free_egre_key()})
 131
 132         # set netns
 133         slice_tags.append({'name': 'netns', 'value': '1'})
 134
 135         # set cap_net_admin
 136         # need to update the attribute string?
 137         slice_tags.append({'name': 'capabilities', 'value': 'CAP_NET_ADMIN'})
 138
 139         for link in requested_links:
 140             # get the ip address of the first node in the link
 141             ifname1 = Xrn(link['interface1']['component_id']).get_leaf()
 142             (node_raw, device) = ifname1.split(':')
 143             node_id = int(node_raw.replace('node', ''))
 144             node = nodes_dict[node_id]
 145             if1 = interfaces_dict[node['interface_ids'][0]]
 146             ipaddr = if1['ip']
 147             topo_rspec = VLink.get_topo_rspec(link, ipaddr)
 148             # set topo_rspec tag
 149             slice_tags.append({'name': 'topo_rspec', 'value': str([topo_rspec]), 'node_id': node_id})
 150             # set vini_topo tag
 151             slice_tags.append({'name': 'vini_topo', 'value': 'manual', 'node_id': node_id})
 152             #self.driver.shell.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id)
 153
 154         self.verify_slice_attributes(slice, slice_tags, {'append': True}, admin=True)
 155
 156
 157
 158     def handle_peer(self, site, slice, persons, peer):
 159         if peer:
 160             # bind site
 161             try:
 162                 if site:
 163                     self.driver.shell.BindObjectToPeer('site', site['site_id'], peer['shortname'], slice['site_id'])
 164             except Exception,e:
 165                 self.driver.shell.DeleteSite(site['site_id'])
 166                 raise e
 167
 168             # bind slice
 169             try:
 170                 if slice:
 171                     self.driver.shell.BindObjectToPeer('slice', slice['slice_id'], peer['shortname'], slice['slice_id'])
 172             except Exception,e:
 173                 self.driver.shell.DeleteSlice(slice['slice_id'])
 174                 raise e
 175
 176             # bind persons
 177             for person in persons:
 178                 try:
 179                     self.driver.shell.BindObjectToPeer('person',
 180                                                      person['person_id'], peer['shortname'], person['peer_person_id'])
 181
 182                     for (key, remote_key_id) in zip(person['keys'], person['key_ids']):
 183                         try:
 184                             self.driver.shell.BindObjectToPeer( 'key', key['key_id'], peer['shortname'], remote_key_id)
 185                         except:
 186                             self.driver.shell.DeleteKey(key['key_id'])
 187                             logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname']))
 188                 except Exception,e:
 189                     self.driver.shell.DeletePerson(person['person_id'])
 190                     raise e
 191
 192         return slice
 193
 194     def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}):
 195         (slice_hrn, type) = urn_to_hrn(slice_xrn)
 196         site_hrn = get_authority(slice_hrn)
 197         # login base can't be longer than 20 characters
 198         slicename = hrn_to_pl_slicename(slice_hrn)
 199         authority_name = slicename.split('_')[0]
 200         login_base = authority_name[:20]
 201         sites = self.driver.shell.GetSites(login_base)
 202         if not sites:
 203             # create new site record
 204             site = {'name': 'geni.%s' % authority_name,
 205                     'abbreviated_name': authority_name,
 206                     'login_base': login_base,
 207                     'max_slices': 100,
 208                     'max_slivers': 1000,
 209                     'enabled': True,
 210                     'peer_site_id': None}
 211             if peer:
 212                 site['peer_site_id'] = slice_record.get('site_id', None)
 213             site['site_id'] = self.driver.shell.AddSite(site)
 214             # exempt federated sites from monitor policies
 215             self.driver.shell.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101")
 216
 217 #            # is this still necessary?
 218 #            # add record to the local registry
 219 #            if sfa_peer and slice_record:
 220 #                peer_dict = {'type': 'authority', 'hrn': site_hrn, \
 221 #                             'peer_authority': sfa_peer, 'pointer': site['site_id']}
 222 #                self.registry.register_peer_object(self.credential, peer_dict)
 223         else:
 224             site =  sites[0]
 225             if peer:
 226                 # unbind from peer so we can modify if necessary. Will bind back later
 227                 self.driver.shell.UnBindObjectFromPeer('site', site['site_id'], peer['shortname'])
 228
 229         return site
 230
 231     def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, options={}):
 232         slicename = hrn_to_pl_slicename(slice_hrn)
 233         parts = slicename.split("_")
 234         login_base = parts[0]
 235         slices = self.driver.shell.GetSlices([slicename])
 236         if not slices:
 237             slice = {'name': slicename,
 238                      'url': slice_record.get('url', slice_hrn),
 239                      'description': slice_record.get('description', slice_hrn)}
 240             # add the slice
 241             slice['slice_id'] = self.driver.shell.AddSlice(slice)
 242             slice['node_ids'] = []
 243             slice['person_ids'] = []
 244             if peer:
 245                 slice['peer_slice_id'] = slice_record.get('slice_id', None)
 246             # mark this slice as an sfa peer record
 247 #            if sfa_peer:
 248 #                peer_dict = {'type': 'slice', 'hrn': slice_hrn,
 249 #                             'peer_authority': sfa_peer, 'pointer': slice['slice_id']}
 250 #                self.registry.register_peer_object(self.credential, peer_dict)
 251         else:
 252             slice = slices[0]
 253             if peer:
 254                 slice['peer_slice_id'] = slice_record.get('slice_id', None)
 255                 # unbind from peer so we can modify if necessary. Will bind back later
 256                 self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
 257                 #Update existing record (e.g. expires field) it with the latest info.
 258             if slice_record.get('expires'):
 259                 requested_expires = int(datetime_to_epoch(utcparse(slice_record['expires'])))
 260                 if requested_expires and slice['expires'] != requested_expires:
 261                     self.driver.shell.UpdateSlice( slice['slice_id'], {'expires' : requested_expires})
 262
 263         return slice
 264
 265     #def get_existing_persons(self, users):
 266     def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}):
 267         users_by_email = {}
 268         users_by_site = defaultdict(list)
 269         users_dict = {}
 270         for user in users:
 271             user['urn'] = user['urn'].lower()
 272             hrn, type = urn_to_hrn(user['urn'])
 273             username = get_leaf(hrn)
 274             login_base = PlXrn(xrn=user['urn']).pl_login_base()
 275             user['username'] = username
 276             user['site'] = login_base
 277
 278             if 'email' in user:
 279                 user['email'] = user['email'].lower()
 280                 users_by_email[user['email']] = user
 281                 users_dict[user['email']] = user
 282             else:
 283                 users_by_site[user['site']].append(user)
 284
 285         # start building a list of existing users
 286         existing_user_ids = []
 287         existing_user_ids_filter = []
 288         if users_by_email:
 289             existing_user_ids_filter.extend(users_by_email.keys())
 290         if users_by_site:
 291             for login_base in users_by_site:
 292                 users = users_by_site[login_base]
 293                 for user in users:
 294                     existing_user_ids_filter.append(user['username']+'@geni.net')
 295         if existing_user_ids_filter:
 296             # get existing users by email
 297             existing_users = self.driver.shell.GetPersons({'email': existing_user_ids_filter},
 298                                                         ['person_id', 'key_ids', 'email'])
 299             existing_user_ids.extend([user['email'] for user in existing_users])
 300
 301         if users_by_site:
 302             # get a list of user sites (based on requeste user urns
 303             site_list = self.driver.shell.GetSites(users_by_site.keys(), \
 304                 ['site_id', 'login_base', 'person_ids'])
 305             # get all existing users at these sites
 306             sites = {}
 307             site_user_ids = []
 308             for site in site_list:
 309                 sites[site['site_id']] = site
 310                 site_user_ids.extend(site['person_ids'])
 311
 312             existing_site_persons_list = self.driver.shell.GetPersons(site_user_ids,
 313                                                                     ['person_id', 'key_ids', 'email', 'site_ids'])
 314
 315             # all requested users are either existing users or new (added) users
 316             for login_base in users_by_site:
 317                 requested_site_users = users_by_site[login_base]
 318                 for requested_user in requested_site_users:
 319                     user_found = False
 320                     for existing_user in existing_site_persons_list:
 321                         for site_id in existing_user['site_ids']:
 322                             if site_id in sites:
 323                                 site = sites[site_id]
 324                                 if login_base == site['login_base'] and \
 325                                    existing_user['email'].startswith(requested_user['username']+'@'):
 326                                     existing_user_ids.append(existing_user['email'])
 327                                     requested_user['email'] = existing_user['email']
 328                                     users_dict[existing_user['email']] = requested_user
 329                                     user_found = True
 330                                     break
 331                         if user_found:
 332                             break
 333
 334                     if user_found == False:
 335                         fake_email = requested_user['username'] + '@geni.net'
 336                         requested_user['email'] = fake_email
 337                         users_dict[fake_email] = requested_user
 338
 339         # requested slice users
 340         requested_user_ids = users_dict.keys()
 341         # existing slice users
 342         existing_slice_users_filter = {'person_id': slice_record.get('person_ids', [])}
 343         existing_slice_users = self.driver.shell.GetPersons(existing_slice_users_filter,
 344                                                           ['person_id', 'key_ids', 'email'])
 345         existing_slice_user_ids = [user['email'] for user in existing_slice_users]
 346
 347         # users to be added, removed or updated
 348         added_user_ids = set(requested_user_ids).difference(existing_user_ids)
 349         added_slice_user_ids = set(requested_user_ids).difference(existing_slice_user_ids)
 350         removed_user_ids = set(existing_slice_user_ids).difference(requested_user_ids)
 351         updated_user_ids = set(existing_slice_user_ids).intersection(requested_user_ids)
 352
 353         # Remove stale users (only if we are not appending).
 354         # Append by default.
 355         append = options.get('append', True)
 356         if append == False:
 357             for removed_user_id in removed_user_ids:
 358                 self.driver.shell.DeletePersonFromSlice(removed_user_id, slice_record['name'])
 359         # update_existing users
 360         updated_users_list = [user for user in users_dict.values() if user['email'] in \
 361           updated_user_ids]
 362         self.verify_keys(existing_slice_users, updated_users_list, peer, options)
 363
 364         added_persons = []
 365         # add new users
 366         for added_user_id in added_user_ids:
 367             added_user = users_dict[added_user_id]
 368             hrn, type = urn_to_hrn(added_user['urn'])
 369             person = {
 370                 'first_name': added_user.get('first_name', hrn),
 371                 'last_name': added_user.get('last_name', hrn),
 372                 'email': added_user_id,
 373                 'peer_person_id': None,
 374                 'keys': [],
 375                 'key_ids': added_user.get('key_ids', []),
 376             }
 377             person['person_id'] = self.driver.shell.AddPerson(person)
 378             if peer:
 379                 person['peer_person_id'] = added_user['person_id']
 380             added_persons.append(person)
 381
 382             # enable the account
 383             self.driver.shell.UpdatePerson(person['person_id'], {'enabled': True})
 384
 385             # add person to site
 386             self.driver.shell.AddPersonToSite(added_user_id, added_user['site'])
 387
 388             for key_string in added_user.get('keys', []):
 389                 key = {'key':key_string, 'key_type':'ssh'}
 390                 key['key_id'] = self.driver.shell.AddPersonKey(person['person_id'], key)
 391                 person['keys'].append(key)
 392
 393             # add the registry record
 394 #            if sfa_peer:
 395 #                peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \
 396 #                    'pointer': person['person_id']}
 397 #                self.registry.register_peer_object(self.credential, peer_dict)
 398
 399         for added_slice_user_id in added_slice_user_ids.union(added_user_ids):
 400             # add person to the slice
 401             self.driver.shell.AddPersonToSlice(added_slice_user_id, slice_record['name'])
 402             # if this is a peer record then it should already be bound to a peer.
 403             # no need to return worry about it getting bound later
 404
 405         return added_persons
 406
 407
 408     def verify_keys(self, persons, users, peer, options={}):
 409         # existing keys
 410         key_ids = []
 411         for person in persons:
 412             key_ids.extend(person['key_ids'])
 413         keylist = self.driver.shell.GetKeys(key_ids, ['key_id', 'key'])
 414         keydict = {}
 415         for key in keylist:
 416             keydict[key['key']] = key['key_id']
 417         existing_keys = keydict.keys()
 418         persondict = {}
 419         for person in persons:
 420             persondict[person['email']] = person
 421
 422         # add new keys
 423         requested_keys = []
 424         updated_persons = []
 425         for user in users:
 426             user_keys = user.get('keys', [])
 427             updated_persons.append(user)
 428             for key_string in user_keys:
 429                 requested_keys.append(key_string)
 430                 if key_string not in existing_keys:
 431                     key = {'key': key_string, 'key_type': 'ssh'}
 432                     try:
 433                         if peer:
 434                             person = persondict[user['email']]
 435                             self.driver.shell.UnBindObjectFromPeer('person', person['person_id'], peer['shortname'])
 436                         key['key_id'] = self.driver.shell.AddPersonKey(user['email'], key)
 437                         if peer:
 438                             key_index = user_keys.index(key['key'])
 439                             remote_key_id = user['key_ids'][key_index]
 440                             self.driver.shell.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id)
 441
 442                     finally:
 443                         if peer:
 444                             self.driver.shell.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id'])
 445
 446         # remove old keys (only if we are not appending)
 447         append = options.get('append', True)
 448         if append == False:
 449             removed_keys = set(existing_keys).difference(requested_keys)
 450             for existing_key_id in keydict:
 451                 if keydict[existing_key_id] in removed_keys:
 452                     try:
 453                         if peer:
 454                             self.driver.shell.UnBindObjectFromPeer('key', existing_key_id, peer['shortname'])
 455                         self.driver.shell.DeleteKey(existing_key_id)
 456                     except:
 457                         pass
 458
 459     def verify_slice_attributes(self, slice, requested_slice_attributes, options={}, admin=False):
 460         append = options.get('append', True)
 461         # get list of attributes users ar able to manage
 462         filter = {'category': '*slice*'}
 463         if not admin:
 464             filter['|roles'] = ['user']
 465         slice_attributes = self.driver.shell.GetTagTypes(filter)
 466         valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes]
 467
 468         # get sliver attributes
 469         added_slice_attributes = []
 470         removed_slice_attributes = []
 471         ignored_slice_attribute_names = []
 472         existing_slice_attributes = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']})
 473
 474         # get attributes that should be removed
 475         for slice_tag in existing_slice_attributes:
 476             if slice_tag['tagname'] in ignored_slice_attribute_names:
 477                 # If a slice already has a admin only role it was probably given to them by an
 478                 # admin, so we should ignore it.
 479                 ignored_slice_attribute_names.append(slice_tag['tagname'])
 480             else:
 481                 # If an existing slice attribute was not found in the request it should
 482                 # be removed
 483                 attribute_found=False
 484                 for requested_attribute in requested_slice_attributes:
 485                     if requested_attribute['name'] == slice_tag['tagname'] and \
 486                        requested_attribute['value'] == slice_tag['value']:
 487                         attribute_found=True
 488                         break
 489
 490             if not attribute_found and not append:
 491                 removed_slice_attributes.append(slice_tag)
 492
 493         # get attributes that should be added:
 494         for requested_attribute in requested_slice_attributes:
 495             # if the requested attribute wasn't found  we should add it
 496             if requested_attribute['name'] in valid_slice_attribute_names:
 497                 attribute_found = False
 498                 for existing_attribute in existing_slice_attributes:
 499                     if requested_attribute['name'] == existing_attribute['tagname'] and \
 500                        requested_attribute['value'] == existing_attribute['value']:
 501                         attribute_found=True
 502                         break
 503                 if not attribute_found:
 504                     added_slice_attributes.append(requested_attribute)
 505
 506
 507         # remove stale attributes
 508         for attribute in removed_slice_attributes:
 509             try:
 510                 self.driver.shell.DeleteSliceTag(attribute['slice_tag_id'])
 511             except Exception, e:
 512                 logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
 513                                 % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))
 514
 515         # add requested_attributes
 516         for attribute in added_slice_attributes:
 517             try:
 518                 self.driver.shell.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None))
 519             except Exception, e:
 520                 logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
 521                                 % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))
 522