sfa/openstack/nova_driver.py

   1 import time
   2 import datetime
   3
   4 from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \
   5     RecordNotFound, SfaNotImplemented, SfaInvalidArgument, UnsupportedOperation
   6
   7 from sfa.util.sfalogging import logger
   8 from sfa.util.defaultdict import defaultdict
   9 from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch
  10 from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf
  11 from sfa.openstack.osxrn import OSXrn, hrn_to_os_slicename, hrn_to_os_tenant_name
  12 from sfa.util.cache import Cache
  13 from sfa.trust.credential import Credential
  14 # used to be used in get_ticket
  15 #from sfa.trust.sfaticket import SfaTicket
  16 from sfa.rspecs.version_manager import VersionManager
  17 from sfa.rspecs.rspec import RSpec
  18 from sfa.storage.model import RegRecord, SliverAllocation
  19
  20 # the driver interface, mostly provides default behaviours
  21 from sfa.managers.driver import Driver
  22 from sfa.openstack.shell import Shell
  23 from sfa.openstack.osaggregate import OSAggregate
  24 from sfa.planetlab.plslices import PlSlices
  25
  26 def list_to_dict(recs, key):
  27     """
  28     convert a list of dictionaries into a dictionary keyed on the
  29     specified dictionary key
  30     """
  31     return dict ( [ (rec[key],rec) for rec in recs ] )
  32
  33 #
  34 # PlShell is just an xmlrpc serverproxy where methods
  35 # can be sent as-is; it takes care of authentication
  36 # from the global config
  37 #
  38 class NovaDriver(Driver):
  39
  40     # the cache instance is a class member so it survives across incoming requests
  41     cache = None
  42
  43     def __init__ (self, api):
  44         Driver.__init__(self, api)
  45         config = api.config
  46         self.shell = Shell(config=config)
  47         self.cache=None
  48         if config.SFA_AGGREGATE_CACHING:
  49             if NovaDriver.cache is None:
  50                 NovaDriver.cache = Cache()
  51             self.cache = NovaDriver.cache
  52
  53     def sliver_to_slice_xrn(self, xrn):
  54         sliver_id_parts = Xrn(xrn).get_sliver_id_parts()
  55         slice = self.shell.auth_manager.tenants.find(id=sliver_id_parts[0])
  56         if not slice:
  57             raise Forbidden("Unable to locate slice record for sliver:  %s" % xrn)
  58         slice_xrn = OSXrn(name=slice.name, type='slice')
  59         return slice_xrn
  60
  61     def check_sliver_credentials(self, creds, urns):
  62         # build list of cred object hrns
  63         slice_cred_names = []
  64         for cred in creds:
  65             slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn()
  66             slice_cred_names.append(OSXrn(xrn=slice_cred_hrn).get_slicename())
  67
  68         # look up slice name of slivers listed in urns arg
  69         slice_ids = []
  70         for urn in urns:
  71             sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts()
  72             slice_ids.append(sliver_id_parts[0])
  73
  74         if not slice_ids:
  75              raise Forbidden("sliver urn not provided")
  76
  77         sliver_names = []
  78         for slice_id in slice_ids:
  79             slice = self.shell.auth_manager.tenants.find(slice_id)
  80             sliver_names.append(slice['name'])
  81
  82         # make sure we have a credential for every specified sliver ierd
  83         for sliver_name in sliver_names:
  84             if sliver_name not in slice_cred_names:
  85                 msg = "Valid credential not found for target: %s" % sliver_name
  86                 raise Forbidden(msg)
  87
  88     ########################################
  89     ########## registry oriented
  90     ########################################
  91
  92     ########## disabled users
  93     def is_enabled (self, record):
  94         # all records are enabled
  95         return True
  96
  97     def augment_records_with_testbed_info (self, sfa_records):
  98         return self.fill_record_info (sfa_records)
  99
 100     ##########
 101     def register (self, sfa_record, hrn, pub_key):
 102
 103         if sfa_record['type'] == 'slice':
 104             record = self.register_slice(sfa_record, hrn)
 105         elif sfa_record['type'] == 'user':
 106             record = self.register_user(sfa_record, hrn, pub_key)
 107         elif sfa_record['type'].startswith('authority'):
 108             record = self.register_authority(sfa_record, hrn)
 109         # We should be returning the records id as a pointer but
 110         # this is a string and the records table expects this to be an
 111         # int.
 112         #return record.id
 113         return -1
 114
 115     def register_slice(self, sfa_record, hrn):
 116         # add slice description, name, researchers, PI
 117         name = hrn_to_os_tenant_name(hrn)
 118         description = sfa_record.get('description', None)
 119         self.shell.auth_manager.tenants.create(name, description)
 120         tenant = self.shell.auth_manager.tenants.find(name=name)
 121         auth_hrn = OSXrn(xrn=hrn, type='slice').get_authority_hrn()
 122         parent_tenant_name = OSXrn(xrn=auth_hrn, type='slice').get_tenant_name()
 123         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
 124         researchers = sfa_record.get('researchers', [])
 125         for researcher in researchers:
 126             name = Xrn(researcher).get_leaf()
 127             user = self.shell.auth_manager.users.find(name=name)
 128             self.shell.auth_manager.roles.add_user_role(user, 'Member', tenant)
 129             self.shell.auth_manager.roles.add_user_role(user, 'user', tenant)
 130
 131
 132         pis = sfa_record.get('pis', [])
 133         for pi in pis:
 134             name = Xrn(pi).get_leaf()
 135             user = self.shell.auth_manager.users.find(name=name)
 136             self.shell.auth_manager.roles.add_user_role(user, 'pi', tenant)
 137             self.shell.auth_manager.roles.add_user_role(user, 'pi', parent_tenant)
 138
 139         return tenant
 140
 141     def register_user(self, sfa_record, hrn, pub_key):
 142         # add person roles, projects and keys
 143         email = sfa_record.get('email', None)
 144         xrn = Xrn(hrn)
 145         name = xrn.get_leaf()
 146         auth_hrn = xrn.get_authority_hrn()
 147         tenant_name = OSXrn(xrn=auth_hrn, type='authority').get_tenant_name()
 148         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 149         self.shell.auth_manager.users.create(name, email=email, tenant_id=tenant.id)
 150         user = self.shell.auth_manager.users.find(name=name)
 151         slices = sfa_records.get('slices', [])
 152         for slice in projects:
 153             slice_tenant_name = OSXrn(xrn=slice, type='slice').get_tenant_name()
 154             slice_tenant = self.shell.auth_manager.tenants.find(name=slice_tenant_name)
 155             self.shell.auth_manager.roles.add_user_role(user, slice_tenant, 'user')
 156         keys = sfa_records.get('keys', [])
 157         for key in keys:
 158             keyname = OSXrn(xrn=hrn, type='user').get_slicename()
 159             self.shell.nova_client.keypairs.create(keyname, key)
 160         return user
 161
 162     def register_authority(self, sfa_record, hrn):
 163         name = OSXrn(xrn=hrn, type='authority').get_tenant_name()
 164         self.shell.auth_manager.tenants.create(name, sfa_record.get('description', ''))
 165         tenant = self.shell.auth_manager.tenants.find(name=name)
 166         return tenant
 167
 168
 169     ##########
 170     # xxx actually old_sfa_record comes filled with plc stuff as well in the original code
 171     def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
 172         type = new_sfa_record['type']
 173
 174         # new_key implemented for users only
 175         if new_key and type not in [ 'user' ]:
 176             raise UnknownSfaType(type)
 177
 178         elif type == "slice":
 179             # can update project manager and description
 180             name = hrn_to_os_slicename(hrn)
 181             researchers = sfa_record.get('researchers', [])
 182             pis = sfa_record.get('pis', [])
 183             project_manager = None
 184             description = sfa_record.get('description', None)
 185             if pis:
 186                 project_manager = Xrn(pis[0], 'user').get_leaf()
 187             elif researchers:
 188                 project_manager = Xrn(researchers[0], 'user').get_leaf()
 189             self.shell.auth_manager.modify_project(name, project_manager, description)
 190
 191         elif type == "user":
 192             # can techinally update access_key and secret_key,
 193             # but that is not in our scope, so we do nothing.
 194             pass
 195         return True
 196
 197
 198     ##########
 199     def remove (self, sfa_record):
 200         type=sfa_record['type']
 201         if type == 'user':
 202             name = Xrn(sfa_record['hrn']).get_leaf()
 203             if self.shell.auth_manager.get_user(name):
 204                 self.shell.auth_manager.delete_user(name)
 205         elif type == 'slice':
 206             name = hrn_to_os_slicename(sfa_record['hrn'])
 207             if self.shell.auth_manager.get_project(name):
 208                 self.shell.auth_manager.delete_project(name)
 209         return True
 210
 211
 212     ####################
 213     def fill_record_info(self, records):
 214         """
 215         Given a (list of) SFA record, fill in the PLC specific
 216         and SFA specific fields in the record.
 217         """
 218         if not isinstance(records, list):
 219             records = [records]
 220
 221         for record in records:
 222             if record['type'] == 'user':
 223                 record = self.fill_user_record_info(record)
 224             elif record['type'] == 'slice':
 225                 record = self.fill_slice_record_info(record)
 226             elif record['type'].startswith('authority'):
 227                 record = self.fill_auth_record_info(record)
 228             else:
 229                 continue
 230             record['geni_urn'] = hrn_to_urn(record['hrn'], record['type'])
 231             record['geni_certificate'] = record['gid']
 232             #if os_record.created_at is not None:
 233             #    record['date_created'] = datetime_to_string(utcparse(os_record.created_at))
 234             #if os_record.updated_at is not None:
 235             #    record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at))
 236
 237         return records
 238
 239     def fill_user_record_info(self, record):
 240         xrn = Xrn(record['hrn'])
 241         name = xrn.get_leaf()
 242         record['name'] = name
 243         user = self.shell.auth_manager.users.find(name=name)
 244         record['email'] = user.email
 245         tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 246         slices = []
 247         all_tenants = self.shell.auth_manager.tenants.list()
 248         for tmp_tenant in all_tenants:
 249             if tmp_tenant.name.startswith(tenant.name +"."):
 250                 for tmp_user in tmp_tenant.list_users():
 251                     if tmp_user.name == user.name:
 252                         slice_hrn = ".".join([self.hrn, tmp_tenant.name])
 253                         slices.append(slice_hrn)
 254         record['slices'] = slices
 255         roles = self.shell.auth_manager.roles.roles_for_user(user, tenant)
 256         record['roles'] = [role.name for role in roles]
 257         keys = self.shell.nova_manager.keypairs.findall(name=record['hrn'])
 258         record['keys'] = [key.public_key for key in keys]
 259         return record
 260
 261     def fill_slice_record_info(self, record):
 262         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 263         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 264         parent_tenant_name = OSXrn(xrn=tenant_name).get_authority_hrn()
 265         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
 266         researchers = []
 267         pis = []
 268
 269         # look for users and pis in slice tenant
 270         for user in tenant.list_users():
 271             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 272                 if role.name.lower() == 'pi':
 273                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 274                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 275                     pis.append(hrn)
 276                 elif role.name.lower() in ['user', 'member']:
 277                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 278                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 279                     researchers.append(hrn)
 280
 281         # look for pis in the slice's parent (site/organization) tenant
 282         for user in parent_tenant.list_users():
 283             for role in self.shell.auth_manager.roles.roles_for_user(user, parent_tenant):
 284                 if role.name.lower() == 'pi':
 285                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 286                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 287                     pis.append(hrn)
 288         record['name'] = tenant_name
 289         record['description'] = tenant.description
 290         record['PI'] = pis
 291         if pis:
 292             record['geni_creator'] = pis[0]
 293         else:
 294             record['geni_creator'] = None
 295         record['researcher'] = researchers
 296         return record
 297
 298     def fill_auth_record_info(self, record):
 299         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 300         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 301         researchers = []
 302         pis = []
 303
 304         # look for users and pis in slice tenant
 305         for user in tenant.list_users():
 306             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 307                 hrn = ".".join([self.hrn, tenant.name, user.name])
 308                 if role.name.lower() == 'pi':
 309                     pis.append(hrn)
 310                 elif role.name.lower() in ['user', 'member']:
 311                     researchers.append(hrn)
 312
 313         # look for slices
 314         slices = []
 315         all_tenants = self.shell.auth_manager.tenants.list()
 316         for tmp_tenant in all_tenants:
 317             if tmp_tenant.name.startswith(tenant.name+"."):
 318                 slices.append(".".join([self.hrn, tmp_tenant.name]))
 319
 320         record['name'] = tenant_name
 321         record['description'] = tenant.description
 322         record['PI'] = pis
 323         record['enabled'] = tenant.enabled
 324         record['researchers'] = researchers
 325         record['slices'] = slices
 326         return record
 327
 328     ####################
 329     # plcapi works by changes, compute what needs to be added/deleted
 330     def update_relation (self, subject_type, target_type, subject_id, target_ids):
 331         # hard-wire the code for slice/user for now, could be smarter if needed
 332         if subject_type =='slice' and target_type == 'user':
 333             subject=self.shell.project_get(subject_id)[0]
 334             current_target_ids = [user.name for user in subject.members]
 335             add_target_ids = list ( set (target_ids).difference(current_target_ids))
 336             del_target_ids = list ( set (current_target_ids).difference(target_ids))
 337             logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id)))
 338             for target_id in add_target_ids:
 339                 self.shell.project_add_member(target_id,subject_id)
 340                 logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id)))
 341             for target_id in del_target_ids:
 342                 logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id)))
 343                 self.shell.project_remove_member(target_id, subject_id)
 344         else:
 345             logger.info('unexpected relation to maintain, %s -> %s'%(subject_type,target_type))
 346
 347
 348     ########################################
 349     ########## aggregate oriented
 350     ########################################
 351
 352     def testbed_name (self): return "openstack"
 353
 354     def aggregate_version (self):
 355         return {}
 356
 357     # first 2 args are None in case of resource discovery
 358     def list_resources (self, version=None, options=None):
 359         if options is None: options={}
 360         aggregate = OSAggregate(self)
 361         rspec =  aggregate.list_resources(version=version, options=options)
 362         return rspec
 363
 364     def describe(self, urns, version=None, options=None):
 365         if options is None: options={}
 366         aggregate = OSAggregate(self)
 367         return aggregate.describe(urns, version=version, options=options)
 368
 369     def status (self, urns, options=None):
 370         if options is None: options={}
 371         aggregate = OSAggregate(self)
 372         desc =  aggregate.describe(urns)
 373         status = {'geni_urn': desc['geni_urn'],
 374                   'geni_slivers': desc['geni_slivers']}
 375         return status
 376
 377     def allocate (self, urn, rspec_string, expiration, options=None):
 378         if options is None: options={}
 379         xrn = Xrn(urn)
 380         aggregate = OSAggregate(self)
 381
 382         # assume first user is the caller and use their context
 383         # for the ec2/euca api connection. Also, use the first users
 384         # key as the project key.
 385         key_name = None
 386         if len(users) > 1:
 387             key_name = aggregate.create_instance_key(xrn.get_hrn(), users[0])
 388
 389         # collect public keys
 390         users = options.get('geni_users', [])
 391         pubkeys = []
 392         for user in users:
 393             pubkeys.extend(user['keys'])
 394
 395         rspec = RSpec(rspec_string)
 396         instance_name = hrn_to_os_slicename(slice_hrn)
 397         tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
 398         slivers = aggregate.run_instances(instance_name, tenant_name, \
 399                                           rspec_string, key_name, pubkeys)
 400
 401         # update all sliver allocation states setting then to geni_allocated
 402         sliver_ids = [sliver.id for sliver in slivers]
 403         dbsession=self.api.dbsession()
 404         SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',dbsession)
 405
 406         return aggregate.describe(urns=[urn], version=rspec.version)
 407
 408     def provision(self, urns, options=None):
 409         if options is None: options={}
 410         # update sliver allocation states and set them to geni_provisioned
 411         aggregate = OSAggregate(self)
 412         instances = aggregate.get_instances(urns)
 413         sliver_ids = []
 414         for instance in instances:
 415             sliver_hrn = "%s.%s" % (self.driver.hrn, instance.id)
 416             sliver_ids.append(Xrn(sliver_hrn, type='sliver').urn)
 417         dbsession=self.api.dbsession()
 418         SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',dbsession)
 419         version_manager = VersionManager()
 420         rspec_version = version_manager.get_version(options['geni_rspec_version'])
 421         return self.describe(urns, rspec_version, options=options)
 422
 423     def delete (self, urns, options=None):
 424         if options is None: options={}
 425         # collect sliver ids so we can update sliver allocation states after
 426         # we remove the slivers.
 427         aggregate = OSAggregate(self)
 428         instances = aggregate.get_instances(urns)
 429         sliver_ids = []
 430         for instance in instances:
 431             sliver_hrn = "%s.%s" % (self.driver.hrn, instance.id)
 432             sliver_ids.append(Xrn(sliver_hrn, type='sliver').urn)
 433
 434             # delete the instance
 435             aggregate.delete_instance(instance)
 436
 437         # delete sliver allocation states
 438         dbsession=self.api.dbsession()
 439         SliverAllocation.delete_allocations(sliver_ids, dbsession)
 440
 441         # return geni_slivers
 442         geni_slivers = []
 443         for sliver_id in sliver_ids:
 444             geni_slivers.append(
 445                 {'geni_sliver_urn': sliver['sliver_id'],
 446                  'geni_allocation_status': 'geni_unallocated',
 447                  'geni_expires': None})
 448         return geni_slivers
 449
 450     def renew (self, urns, expiration_time, options=None):
 451         if options is None: options={}
 452         description = self.describe(urns, None, options)
 453         return description['geni_slivers']
 454
 455     def perform_operational_action  (self, urns, action, options=None):
 456         if options is None: options={}
 457         aggregate = OSAggregate(self)
 458         action = action.lower()
 459         if action == 'geni_start':
 460             action_method = aggregate.start_instances
 461         elif action == 'geni_stop':
 462             action_method = aggregate.stop_instances
 463         elif action == 'geni_restart':
 464             action_method = aggreate.restart_instances
 465         else:
 466             raise UnsupportedOperation(action)
 467
 468          # fault if sliver is not full allocated (operational status is geni_pending_allocation)
 469         description = self.describe(urns, None, options)
 470         for sliver in description['geni_slivers']:
 471             if sliver['geni_operational_status'] == 'geni_pending_allocation':
 472                 raise UnsupportedOperation(action, "Sliver must be fully allocated (operational status is not geni_pending_allocation)")
 473         #
 474         # Perform Operational Action Here
 475         #
 476
 477         instances = aggregate.get_instances(urns)
 478         for instance in instances:
 479             tenant_name = self.driver.shell.auth_manager.client.tenant_name
 480             action_method(tenant_name, instance.name, instance.id)
 481         description = self.describe(urns)
 482         geni_slivers = self.describe(urns, None, options)['geni_slivers']
 483         return geni_slivers
 484
 485     def shutdown(self, xrn, options=None):
 486         if options is None: options={}
 487         xrn = OSXrn(xrn=xrn, type='slice')
 488         tenant_name = xrn.get_tenant_name()
 489         name = xrn.get_slicename()
 490         self.driver.shell.nova_manager.connect(tenant=tenant_name)
 491         instances = self.driver.shell.nova_manager.servers.findall(name=name)
 492         for instance in instances:
 493             self.driver.shell.nova_manager.servers.shutdown(instance)
 494         return True