sfa/nitos/nitosslices.py

   1 from types import StringTypes
   2 from collections import defaultdict
   3
   4 from sfa.util.sfatime import utcparse, datetime_to_epoch
   5 from sfa.util.sfalogging import logger
   6 from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn
   7
   8 from sfa.rspecs.rspec import RSpec
   9
  10 from sfa.nitos.nitosxrn import NitosXrn, hrn_to_nitos_slicename, xrn_to_hostname
  11
  12 MAXINT =  2L**31-1
  13
  14 class NitosSlices:
  15
  16     def __init__(self, driver):
  17         self.driver = driver
  18
  19
  20     def get_sfa_peer(self, xrn):
  21         hrn, type = urn_to_hrn(xrn)
  22
  23         # return the authority for this hrn or None if we are the authority
  24         sfa_peer = None
  25         slice_authority = get_authority(hrn)
  26         site_authority = get_authority(slice_authority)
  27
  28         if site_authority != self.driver.hrn:
  29             sfa_peer = site_authority
  30
  31         return sfa_peer
  32
  33     def verify_slice_leases_nodes(self, slice, rspec_requested_nodes):
  34         nodes = self.driver.shell.getNodes({}, [])
  35
  36         requested_nodes = []
  37         for node in rspec_requested_nodes:
  38              requested_node = {}
  39              nitos_nodes = []
  40              nitos_nodes.extend(nodes)
  41              slice_name = hrn_to_nitos_slicename(node['slice_id'])
  42              if slice_name != slice['slice_name']:
  43                  continue
  44              hostname = xrn_to_hostname(node['component_id'])
  45              nitos_node = self.driver.filter_nitos_results(nitos_nodes, {'hostname': hostname})[0]
  46              # fill the requested node with nitos ids
  47              requested_node['slice_id'] = slice['slice_id']
  48              requested_node['node_id'] = nitos_node['node_id']
  49              requested_node['start_time'] = node['start_time']
  50              requested_node['end_time'] = str(int(node['duration']) * int(self.driver.testbedInfo['grain']) + int(node['start_time']))
  51              requested_nodes.append(requested_node)
  52
  53         # get actual nodes reservation data for the slice
  54         reserved_nodes = self.driver.filter_nitos_results(self.driver.shell.getReservedNodes({}, []), {'slice_id': slice['slice_id']})
  55
  56         reserved_nodes_by_id = {}
  57         for node in reserved_nodes:
  58              reserved_nodes_by_id[node['reservation_id']] = {'slice_id': node['slice_id'], \
  59                                       'node_id': node['node_id'], 'start_time': node['start_time'], \
  60                                       'end_time': node['end_time']}
  61
  62         added_nodes = []
  63         kept_nodes_id = []
  64         deleted_nodes_id = []
  65         for reservation_id in reserved_nodes_by_id:
  66              if reserved_nodes_by_id[reservation_id] not in requested_nodes:
  67                  deleted_nodes_id.append(reservation_id)
  68              else:
  69                  kept_nodes_id.append(reservation_id)
  70                  requested_nodes.remove(reserved_nodes_by_id[reservation_id])
  71         added_nodes = requested_nodes
  72
  73         print "NODES: \nAdded: %s \nDeleted: %s\nKept: %s" %(added_nodes,deleted_nodes_id,kept_nodes_id)
  74
  75         try:
  76             deleted=self.driver.shell.releaseNodes({'reservation_id': deleted_nodes_id})
  77             for node in added_nodes:
  78                 added=self.driver.shell.reserveNodes({'slice_id': slice['slice_id'], 'start_time': node['start_time'], 'end_time': node['end_time'], 'node_id': [node['node_id']]})
  79
  80         except:
  81             logger.log_exc('Failed to add/remove slice leases nodes')
  82
  83         return added_nodes
  84
  85
  86     def verify_slice_leases_channels(self, slice, rspec_requested_channels):
  87         channels = self.driver.shell.getChannels({}, [])
  88
  89         requested_channels = []
  90         for channel in rspec_requested_channels:
  91              requested_channel = {}
  92              nitos_channels = []
  93              nitos_channels.extend(channels)
  94              slice_name = hrn_to_nitos_slicename(channel['slice_id'])
  95              if slice_name != slice['slice_name']:
  96                  continue
  97              channel_num = channel['channel_num']
  98              nitos_channel = self.driver.filter_nitos_results(nitos_channels, {'channel': channel_num})[0]
  99              # fill the requested channel with nitos ids
 100              requested_channel['slice_id'] = slice['slice_id']
 101              requested_channel['channel_id'] = nitos_channel['channel_id']
 102              requested_channel['start_time'] = channel['start_time']
 103              requested_channel['end_time'] = str(int(channel['duration']) * int(self.driver.testbedInfo['grain']) + int(channel['start_time']))
 104              requested_channels.append(requested_channel)
 105
 106         # get actual channel reservation data for the slice
 107         reserved_channels = self.driver.filter_nitos_results(self.driver.shell.getReservedChannels(), {'slice_id': slice['slice_id']})
 108
 109         reserved_channels_by_id = {}
 110         for channel in reserved_channels:
 111              reserved_channels_by_id[channel['reservation_id']] = {'slice_id': channel['slice_id'], \
 112                                       'channel_id': channel['channel_id'], 'start_time': channel['start_time'], \
 113                                       'end_time': channel['end_time']}
 114
 115         added_channels = []
 116         kept_channels_id = []
 117         deleted_channels_id = []
 118         for reservation_id in reserved_channels_by_id:
 119              if reserved_channels_by_id[reservation_id] not in requested_channels:
 120                  deleted_channels_id.append(reservation_id)
 121              else:
 122                  kept_channels_id.append(reservation_id)
 123                  requested_channels.remove(reserved_channels_by_id[reservation_id])
 124         added_channels = requested_channels
 125
 126         print "CHANNELS: \nAdded: %s \nDeleted: %s\nKept: %s" %(added_channels,deleted_channels_id,kept_channels_id)
 127
 128         try:
 129             deleted=self.driver.shell.releaseChannels({'reservation_id': deleted_channels_id})
 130             for channel in added_channels:
 131                 added=self.driver.shell.reserveChannels({'slice_id': slice['slice_id'], 'start_time': channel['start_time'], 'end_time': channel['end_time'], 'channel_id': [channel['channel_id']]})
 132
 133         except:
 134             logger.log_exc('Failed to add/remove slice leases channels')
 135
 136         return added_channels
 137
 138     def verify_slice_leases(self, slice, requested_leases, kept_leases):
 139
 140         leases = self.driver.shell.getLeases({'name':slice['name']}, ['lease_id'])
 141         grain = self.driver.shell.GetLeaseGranularity()
 142         current_leases = [lease['lease_id'] for lease in leases]
 143         deleted_leases = list(set(current_leases).difference(kept_leases))
 144
 145         try:
 146             if peer:
 147                 self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
 148             deleted=self.driver.shell.DeleteLeases(deleted_leases)
 149             for lease in requested_leases:
 150                 added=self.driver.shell.AddLeases(lease['hostname'], slice['name'], int(lease['start_time']), int(lease['duration']) * grain + int(lease['start_time']))
 151
 152         except:
 153             logger.log_exc('Failed to add/remove slice leases')
 154
 155         return leases
 156
 157
 158     def verify_slice_nodes(self, slice, requested_slivers):
 159
 160         nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids'])
 161         current_slivers = [node['hostname'] for node in nodes]
 162
 163         # remove nodes not in rspec
 164         deleted_nodes = list(set(current_slivers).difference(requested_slivers))
 165
 166         # add nodes from rspec
 167         added_nodes = list(set(requested_slivers).difference(current_slivers))
 168
 169         try:
 170             if peer:
 171                 self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
 172             self.driver.shell.AddSliceToNodes(slice['name'], added_nodes)
 173             self.driver.shell.DeleteSliceFromNodes(slice['name'], deleted_nodes)
 174
 175         except:
 176             logger.log_exc('Failed to add/remove slice from nodes')
 177         return nodes
 178
 179     def free_egre_key(self):
 180         used = set()
 181         for tag in self.driver.shell.GetSliceTags({'tagname': 'egre_key'}):
 182                 used.add(int(tag['value']))
 183
 184         for i in range(1, 256):
 185             if i not in used:
 186                 key = i
 187                 break
 188         else:
 189             raise KeyError("No more EGRE keys available")
 190
 191         return str(key)
 192
 193
 194
 195     def verify_slice(self, slice_hrn, slice_record, sfa_peer, options={}):
 196         slicename = hrn_to_nitos_slicename(slice_hrn)
 197         slices = self.driver.shell.getSlices({}, [])
 198         slices = self.driver.filter_nitos_results(slices, {'slice_name': slicename})
 199         if not slices:
 200             slice = {'name': slicename}
 201             # add the slice
 202             slice['slice_id'] = self.driver.shell.addSlice(slice)
 203             slice['node_ids'] = []
 204             slice['user_ids'] = []
 205         else:
 206             slice = slices[0]
 207
 208         return slice
 209
 210     #def get_existing_persons(self, users):
 211     def verify_users(self, slice_hrn, slice_record, users, sfa_peer, options={}):
 212
 213         slice_user_ids = slice_record['user_ids']
 214         all_users = self.driver.shell.getUsers()
 215         # filter slice users
 216         slice_users = [user for user in all_users if user['user_id'] in slice_user_ids]
 217
 218         slicename = hrn_to_nitos_slicename(slice_hrn)
 219         slices = self.driver.shell.getSlices({}, [])
 220         slices = self.driver.filter_nitos_results(slices, {'slice_name': slicename})
 221
 222         slice_user
 223         users_by_email = {}
 224         users_by_site = defaultdict(list)
 225         users_dict = {}
 226         for user in users:
 227             user['urn'] = user['urn'].lower()
 228             hrn, type = urn_to_hrn(user['urn'])
 229             username = get_leaf(hrn)
 230             login_base = PlXrn(xrn=user['urn']).pl_login_base()
 231             user['username'] = username
 232             user['site'] = login_base
 233
 234             if 'email' in user:
 235                 user['email'] = user['email'].lower()
 236                 users_by_email[user['email']] = user
 237                 users_dict[user['email']] = user
 238             else:
 239                 users_by_site[user['site']].append(user)
 240
 241         # start building a list of existing users
 242         existing_user_ids = []
 243         existing_user_ids_filter = []
 244         if users_by_email:
 245             existing_user_ids_filter.extend(users_by_email.keys())
 246         if users_by_site:
 247             for login_base in users_by_site:
 248                 users = users_by_site[login_base]
 249                 for user in users:
 250                     existing_user_ids_filter.append(user['username']+'@geni.net')
 251         if existing_user_ids_filter:
 252             # get existing users by email
 253             existing_users = self.driver.shell.GetPersons({'email': existing_user_ids_filter},
 254                                                         ['person_id', 'key_ids', 'email'])
 255             existing_user_ids.extend([user['email'] for user in existing_users])
 256
 257         if users_by_site:
 258             # get a list of user sites (based on requeste user urns
 259             site_list = self.driver.shell.GetSites(users_by_site.keys(), \
 260                 ['site_id', 'login_base', 'person_ids'])
 261             # get all existing users at these sites
 262             sites = {}
 263             site_user_ids = []
 264             for site in site_list:
 265                 sites[site['site_id']] = site
 266                 site_user_ids.extend(site['person_ids'])
 267
 268             existing_site_persons_list = self.driver.shell.GetPersons(site_user_ids,
 269                                                                     ['person_id', 'key_ids', 'email', 'site_ids'])
 270
 271             # all requested users are either existing users or new (added) users
 272             for login_base in users_by_site:
 273                 requested_site_users = users_by_site[login_base]
 274                 for requested_user in requested_site_users:
 275                     user_found = False
 276                     for existing_user in existing_site_persons_list:
 277                         for site_id in existing_user['site_ids']:
 278                             if site_id in sites:
 279                                 site = sites[site_id]
 280                                 if login_base == site['login_base'] and \
 281                                    existing_user['email'].startswith(requested_user['username']+'@'):
 282                                     existing_user_ids.append(existing_user['email'])
 283                                     requested_user['email'] = existing_user['email']
 284                                     users_dict[existing_user['email']] = requested_user
 285                                     user_found = True
 286                                     break
 287                         if user_found:
 288                             break
 289
 290                     if user_found == False:
 291                         fake_email = requested_user['username'] + '@geni.net'
 292                         requested_user['email'] = fake_email
 293                         users_dict[fake_email] = requested_user
 294
 295         # requested slice users
 296         requested_user_ids = users_dict.keys()
 297         # existing slice users
 298         existing_slice_users_filter = {'person_id': slice_record.get('person_ids', [])}
 299         existing_slice_users = self.driver.shell.GetPersons(existing_slice_users_filter,
 300                                                           ['person_id', 'key_ids', 'email'])
 301         existing_slice_user_ids = [user['email'] for user in existing_slice_users]
 302
 303         # users to be added, removed or updated
 304         added_user_ids = set(requested_user_ids).difference(existing_user_ids)
 305         added_slice_user_ids = set(requested_user_ids).difference(existing_slice_user_ids)
 306         removed_user_ids = set(existing_slice_user_ids).difference(requested_user_ids)
 307         updated_user_ids = set(existing_slice_user_ids).intersection(requested_user_ids)
 308
 309         # Remove stale users (only if we are not appending).
 310         # Append by default.
 311         append = options.get('append', True)
 312         if append == False:
 313             for removed_user_id in removed_user_ids:
 314                 self.driver.shell.DeletePersonFromSlice(removed_user_id, slice_record['name'])
 315         # update_existing users
 316         updated_users_list = [user for user in users_dict.values() if user['email'] in \
 317           updated_user_ids]
 318         self.verify_keys(existing_slice_users, updated_users_list, peer, options)
 319
 320         added_persons = []
 321         # add new users
 322         for added_user_id in added_user_ids:
 323             added_user = users_dict[added_user_id]
 324             hrn, type = urn_to_hrn(added_user['urn'])
 325             person = {
 326                 'first_name': added_user.get('first_name', hrn),
 327                 'last_name': added_user.get('last_name', hrn),
 328                 'email': added_user_id,
 329                 'peer_person_id': None,
 330                 'keys': [],
 331                 'key_ids': added_user.get('key_ids', []),
 332             }
 333             person['person_id'] = self.driver.shell.AddPerson(person)
 334             if peer:
 335                 person['peer_person_id'] = added_user['person_id']
 336             added_persons.append(person)
 337
 338             # enable the account
 339             self.driver.shell.UpdatePerson(person['person_id'], {'enabled': True})
 340
 341             # add person to site
 342             self.driver.shell.AddPersonToSite(added_user_id, added_user['site'])
 343
 344             for key_string in added_user.get('keys', []):
 345                 key = {'key':key_string, 'key_type':'ssh'}
 346                 key['key_id'] = self.driver.shell.AddPersonKey(person['person_id'], key)
 347                 person['keys'].append(key)
 348
 349             # add the registry record
 350 #            if sfa_peer:
 351 #                peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \
 352 #                    'pointer': person['person_id']}
 353 #                self.registry.register_peer_object(self.credential, peer_dict)
 354
 355         for added_slice_user_id in added_slice_user_ids.union(added_user_ids):
 356             # add person to the slice
 357             self.driver.shell.AddPersonToSlice(added_slice_user_id, slice_record['name'])
 358             # if this is a peer record then it should already be bound to a peer.
 359             # no need to return worry about it getting bound later
 360
 361         return added_persons
 362
 363
 364     def verify_keys(self, persons, users, options={}):
 365         # existing keys
 366         key_ids = []
 367         for person in persons:
 368             key_ids.extend(person['key_ids'])
 369         keylist = self.driver.shell.GetKeys(key_ids, ['key_id', 'key'])
 370         keydict = {}
 371         for key in keylist:
 372             keydict[key['key']] = key['key_id']
 373         existing_keys = keydict.keys()
 374         persondict = {}
 375         for person in persons:
 376             persondict[person['email']] = person
 377
 378         # add new keys
 379         requested_keys = []
 380         updated_persons = []
 381         for user in users:
 382             user_keys = user.get('keys', [])
 383             updated_persons.append(user)
 384             for key_string in user_keys:
 385                 requested_keys.append(key_string)
 386                 if key_string not in existing_keys:
 387                     key = {'key': key_string, 'key_type': 'ssh'}
 388                     try:
 389                         if peer:
 390                             person = persondict[user['email']]
 391                             self.driver.shell.UnBindObjectFromPeer('person', person['person_id'], peer['shortname'])
 392                         key['key_id'] = self.driver.shell.AddPersonKey(user['email'], key)
 393                         if peer:
 394                             key_index = user_keys.index(key['key'])
 395                             remote_key_id = user['key_ids'][key_index]
 396                             self.driver.shell.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id)
 397
 398                     finally:
 399                         if peer:
 400                             self.driver.shell.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id'])
 401
 402         # remove old keys (only if we are not appending)
 403         append = options.get('append', True)
 404         if append == False:
 405             removed_keys = set(existing_keys).difference(requested_keys)
 406             for existing_key_id in keydict:
 407                 if keydict[existing_key_id] in removed_keys:
 408                     try:
 409                         if peer:
 410                             self.driver.shell.UnBindObjectFromPeer('key', existing_key_id, peer['shortname'])
 411                         self.driver.shell.DeleteKey(existing_key_id)
 412                     except:
 413                         pass
 414
 415