sfa/openstack/nova_driver.py

   1 import time
   2 import datetime
   3
   4 from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \
   5     RecordNotFound, SfaNotImplemented, SfaInvalidArgument, UnsupportedOperation
   6
   7 from sfa.util.sfalogging import logger
   8 from sfa.util.defaultdict import defaultdict
   9 from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch
  10 from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf
  11 from sfa.openstack.osxrn import OSXrn, hrn_to_os_slicename, hrn_to_os_tenant_name
  12 from sfa.util.cache import Cache
  13 from sfa.trust.credential import Credential
  14 # used to be used in get_ticket
  15 #from sfa.trust.sfaticket import SfaTicket
  16 from sfa.rspecs.version_manager import VersionManager
  17 from sfa.rspecs.rspec import RSpec
  18 from sfa.storage.model import RegRecord, SliverAllocation
  19
  20 # the driver interface, mostly provides default behaviours
  21 from sfa.managers.driver import Driver
  22 from sfa.openstack.shell import Shell
  23 from sfa.openstack.osaggregate import OSAggregate
  24 from sfa.planetlab.plslices import PlSlices
  25
  26 def list_to_dict(recs, key):
  27     """
  28     convert a list of dictionaries into a dictionary keyed on the
  29     specified dictionary key
  30     """
  31     return dict ( [ (rec[key],rec) for rec in recs ] )
  32
  33 #
  34 # PlShell is just an xmlrpc serverproxy where methods
  35 # can be sent as-is; it takes care of authentication
  36 # from the global config
  37 #
  38 class NovaDriver(Driver):
  39
  40     # the cache instance is a class member so it survives across incoming requests
  41     cache = None
  42
  43     def __init__ (self, api):
  44         Driver.__init__(self, api)
  45         config = api.config
  46         self.shell = Shell(config=config)
  47         self.cache=None
  48         if config.SFA_AGGREGATE_CACHING:
  49             if NovaDriver.cache is None:
  50                 NovaDriver.cache = Cache()
  51             self.cache = NovaDriver.cache
  52
  53     def sliver_to_slice_xrn(self, xrn):
  54         sliver_id_parts = Xrn(xrn).get_sliver_id_parts()
  55         slice = self.shell.auth_manager.tenants.find(id=sliver_id_parts[0])
  56         if not slice:
  57             raise Forbidden("Unable to locate slice record for sliver:  %s" % xrn)
  58         slice_xrn = OSXrn(name=slice.name, type='slice')
  59         return slice_xrn
  60
  61     def check_sliver_credentials(self, creds, urns):
  62         # build list of cred object hrns
  63         slice_cred_names = []
  64         for cred in creds:
  65             slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn()
  66             slice_cred_names.append(OSXrn(xrn=slice_cred_hrn).get_slicename())
  67
  68         # look up slice name of slivers listed in urns arg
  69         slice_ids = []
  70         for urn in urns:
  71             sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts()
  72             slice_ids.append(sliver_id_parts[0])
  73
  74         if not slice_ids:
  75              raise Forbidden("sliver urn not provided")
  76
  77         sliver_names = []
  78         for slice_id in slice_ids:
  79             slice = self.shell.auth_manager.tenants.find(slice_id)
  80             sliver_names.append(slice['name'])
  81
  82         # make sure we have a credential for every specified sliver ierd
  83         for sliver_name in sliver_names:
  84             if sliver_name not in slice_cred_names:
  85                 msg = "Valid credential not found for target: %s" % sliver_name
  86                 raise Forbidden(msg)
  87
  88     ########################################
  89     ########## registry oriented
  90     ########################################
  91
  92     ########## disabled users
  93     def is_enabled (self, record):
  94         # all records are enabled
  95         return True
  96
  97     def augment_records_with_testbed_info (self, sfa_records):
  98         return self.fill_record_info (sfa_records)
  99
 100     ##########
 101     def register (self, sfa_record, hrn, pub_key):
 102
 103         if sfa_record['type'] == 'slice':
 104             record = self.register_slice(sfa_record, hrn)
 105         elif sfa_record['type'] == 'user':
 106             record = self.register_user(sfa_record, hrn, pub_key)
 107         elif sfa_record['type'].startswith('authority'):
 108             record = self.register_authority(sfa_record, hrn)
 109         # We should be returning the records id as a pointer but
 110         # this is a string and the records table expects this to be an
 111         # int.
 112         #return record.id
 113         return -1
 114
 115     def register_slice(self, sfa_record, hrn):
 116         # add slice description, name, researchers, PI
 117         name = hrn_to_os_tenant_name(hrn)
 118         description = sfa_record.get('description', None)
 119         self.shell.auth_manager.tenants.create(name, description)
 120         tenant = self.shell.auth_manager.tenants.find(name=name)
 121         auth_hrn = OSXrn(xrn=hrn, type='slice').get_authority_hrn()
 122         parent_tenant_name = OSXrn(xrn=auth_hrn, type='slice').get_tenant_name()
 123         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
 124         researchers = sfa_record.get('researchers', [])
 125         for researcher in researchers:
 126             name = Xrn(researcher).get_leaf()
 127             user = self.shell.auth_manager.users.find(name=name)
 128             self.shell.auth_manager.roles.add_user_role(user, 'Member', tenant)
 129             self.shell.auth_manager.roles.add_user_role(user, 'user', tenant)
 130
 131
 132         pis = sfa_record.get('pis', [])
 133         for pi in pis:
 134             name = Xrn(pi).get_leaf()
 135             user = self.shell.auth_manager.users.find(name=name)
 136             self.shell.auth_manager.roles.add_user_role(user, 'pi', tenant)
 137             self.shell.auth_manager.roles.add_user_role(user, 'pi', parent_tenant)
 138
 139         return tenant
 140
 141     def register_user(self, sfa_record, hrn, pub_key):
 142         # add person roles, projects and keys
 143         email = sfa_record.get('email', None)
 144         xrn = Xrn(hrn)
 145         name = xrn.get_leaf()
 146         auth_hrn = xrn.get_authority_hrn()
 147         tenant_name = OSXrn(xrn=auth_hrn, type='authority').get_tenant_name()
 148         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 149         self.shell.auth_manager.users.create(name, email=email, tenant_id=tenant.id)
 150         user = self.shell.auth_manager.users.find(name=name)
 151         slices = sfa_records.get('slices', [])
 152         for slice in projects:
 153             slice_tenant_name = OSXrn(xrn=slice, type='slice').get_tenant_name()
 154             slice_tenant = self.shell.auth_manager.tenants.find(name=slice_tenant_name)
 155             self.shell.auth_manager.roles.add_user_role(user, slice_tenant, 'user')
 156         keys = sfa_records.get('keys', [])
 157         for key in keys:
 158             keyname = OSXrn(xrn=hrn, type='user').get_slicename()
 159             self.shell.nova_client.keypairs.create(keyname, key)
 160         return user
 161
 162     def register_authority(self, sfa_record, hrn):
 163         name = OSXrn(xrn=hrn, type='authority').get_tenant_name()
 164         self.shell.auth_manager.tenants.create(name, sfa_record.get('description', ''))
 165         tenant = self.shell.auth_manager.tenants.find(name=name)
 166         return tenant
 167
 168
 169     ##########
 170     # xxx actually old_sfa_record comes filled with plc stuff as well in the original code
 171     def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
 172         type = new_sfa_record['type']
 173
 174         # new_key implemented for users only
 175         if new_key and type not in [ 'user' ]:
 176             raise UnknownSfaType(type)
 177
 178         elif type == "slice":
 179             # can update project manager and description
 180             name = hrn_to_os_slicename(hrn)
 181             researchers = sfa_record.get('researchers', [])
 182             pis = sfa_record.get('pis', [])
 183             project_manager = None
 184             description = sfa_record.get('description', None)
 185             if pis:
 186                 project_manager = Xrn(pis[0], 'user').get_leaf()
 187             elif researchers:
 188                 project_manager = Xrn(researchers[0], 'user').get_leaf()
 189             self.shell.auth_manager.modify_project(name, project_manager, description)
 190
 191         elif type == "user":
 192             # can techinally update access_key and secret_key,
 193             # but that is not in our scope, so we do nothing.
 194             pass
 195         return True
 196
 197
 198     ##########
 199     def remove (self, sfa_record):
 200         type=sfa_record['type']
 201         if type == 'user':
 202             name = Xrn(sfa_record['hrn']).get_leaf()
 203             if self.shell.auth_manager.get_user(name):
 204                 self.shell.auth_manager.delete_user(name)
 205         elif type == 'slice':
 206             name = hrn_to_os_slicename(sfa_record['hrn'])
 207             if self.shell.auth_manager.get_project(name):
 208                 self.shell.auth_manager.delete_project(name)
 209         return True
 210
 211
 212     ####################
 213     def fill_record_info(self, records):
 214         """
 215         Given a (list of) SFA record, fill in the PLC specific
 216         and SFA specific fields in the record.
 217         """
 218         if not isinstance(records, list):
 219             records = [records]
 220
 221         for record in records:
 222             if record['type'] == 'user':
 223                 record = self.fill_user_record_info(record)
 224             elif record['type'] == 'slice':
 225                 record = self.fill_slice_record_info(record)
 226             elif record['type'].startswith('authority'):
 227                 record = self.fill_auth_record_info(record)
 228             else:
 229                 continue
 230             record['geni_urn'] = hrn_to_urn(record['hrn'], record['type'])
 231             record['geni_certificate'] = record['gid']
 232             #if os_record.created_at is not None:
 233             #    record['date_created'] = datetime_to_string(utcparse(os_record.created_at))
 234             #if os_record.updated_at is not None:
 235             #    record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at))
 236
 237         return records
 238
 239     def fill_user_record_info(self, record):
 240         xrn = Xrn(record['hrn'])
 241         name = xrn.get_leaf()
 242         record['name'] = name
 243         user = self.shell.auth_manager.users.find(name=name)
 244         record['email'] = user.email
 245         tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 246         slices = []
 247         all_tenants = self.shell.auth_manager.tenants.list()
 248         for tmp_tenant in all_tenants:
 249             if tmp_tenant.name.startswith(tenant.name +"."):
 250                 for tmp_user in tmp_tenant.list_users():
 251                     if tmp_user.name == user.name:
 252                         slice_hrn = ".".join([self.hrn, tmp_tenant.name])
 253                         slices.append(slice_hrn)
 254         record['slices'] = slices
 255         roles = self.shell.auth_manager.roles.roles_for_user(user, tenant)
 256         record['roles'] = [role.name for role in roles]
 257         keys = self.shell.nova_manager.keypairs.findall(name=record['hrn'])
 258         record['keys'] = [key.public_key for key in keys]
 259         return record
 260
 261     def fill_slice_record_info(self, record):
 262         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 263         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 264         parent_tenant_name = OSXrn(xrn=tenant_name).get_authority_hrn()
 265         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
 266         researchers = []
 267         pis = []
 268
 269         # look for users and pis in slice tenant
 270         for user in tenant.list_users():
 271             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 272                 if role.name.lower() == 'pi':
 273                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 274                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 275                     pis.append(hrn)
 276                 elif role.name.lower() in ['user', 'member']:
 277                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 278                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 279                     researchers.append(hrn)
 280
 281         # look for pis in the slice's parent (site/organization) tenant
 282         for user in parent_tenant.list_users():
 283             for role in self.shell.auth_manager.roles.roles_for_user(user, parent_tenant):
 284                 if role.name.lower() == 'pi':
 285                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 286                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 287                     pis.append(hrn)
 288         record['name'] = tenant_name
 289         record['description'] = tenant.description
 290         record['PI'] = pis
 291         if pis:
 292             record['geni_creator'] = pis[0]
 293         else:
 294             record['geni_creator'] = None
 295         record['researcher'] = researchers
 296         return record
 297
 298     def fill_auth_record_info(self, record):
 299         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 300         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 301         researchers = []
 302         pis = []
 303
 304         # look for users and pis in slice tenant
 305         for user in tenant.list_users():
 306             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 307                 hrn = ".".join([self.hrn, tenant.name, user.name])
 308                 if role.name.lower() == 'pi':
 309                     pis.append(hrn)
 310                 elif role.name.lower() in ['user', 'member']:
 311                     researchers.append(hrn)
 312
 313         # look for slices
 314         slices = []
 315         all_tenants = self.shell.auth_manager.tenants.list()
 316         for tmp_tenant in all_tenants:
 317             if tmp_tenant.name.startswith(tenant.name+"."):
 318                 slices.append(".".join([self.hrn, tmp_tenant.name]))
 319
 320         record['name'] = tenant_name
 321         record['description'] = tenant.description
 322         record['PI'] = pis
 323         record['enabled'] = tenant.enabled
 324         record['researchers'] = researchers
 325         record['slices'] = slices
 326         return record
 327
 328     ####################
 329     # plcapi works by changes, compute what needs to be added/deleted
 330     def update_relation (self, subject_type, target_type, subject_id, target_ids):
 331         # hard-wire the code for slice/user for now, could be smarter if needed
 332         if subject_type =='slice' and target_type == 'user':
 333             subject=self.shell.project_get(subject_id)[0]
 334             current_target_ids = [user.name for user in subject.members]
 335             add_target_ids = list ( set (target_ids).difference(current_target_ids))
 336             del_target_ids = list ( set (current_target_ids).difference(target_ids))
 337             logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id)))
 338             for target_id in add_target_ids:
 339                 self.shell.project_add_member(target_id,subject_id)
 340                 logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id)))
 341             for target_id in del_target_ids:
 342                 logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id)))
 343                 self.shell.project_remove_member(target_id, subject_id)
 344         else:
 345             logger.info('unexpected relation to maintain, %s -> %s'%(subject_type,target_type))
 346
 347
 348     ########################################
 349     ########## aggregate oriented
 350     ########################################
 351
 352     def testbed_name (self): return "openstack"
 353
 354     def aggregate_version (self):
 355         return {}
 356
 357     # first 2 args are None in case of resource discovery
 358     def list_resources (self, version=None, options={}):
 359         aggregate = OSAggregate(self)
 360         rspec =  aggregate.list_resources(version=version, options=options)
 361         return rspec
 362
 363     def describe(self, urns, version=None, options={}):
 364         aggregate = OSAggregate(self)
 365         return aggregate.describe(urns, version=version, options=options)
 366
 367     def status (self, urns, options={}):
 368         aggregate = OSAggregate(self)
 369         desc =  aggregate.describe(urns)
 370         status = {'geni_urn': desc['geni_urn'],
 371                   'geni_slivers': desc['geni_slivers']}
 372         return status
 373
 374     def allocate (self, urn, rspec_string, expiration, options={}):
 375         xrn = Xrn(urn)
 376         aggregate = OSAggregate(self)
 377
 378         # assume first user is the caller and use their context
 379         # for the ec2/euca api connection. Also, use the first users
 380         # key as the project key.
 381         key_name = None
 382         if len(users) > 1:
 383             key_name = aggregate.create_instance_key(xrn.get_hrn(), users[0])
 384
 385         # collect public keys
 386         users = options.get('geni_users', [])
 387         pubkeys = []
 388         for user in users:
 389             pubkeys.extend(user['keys'])
 390
 391         rspec = RSpec(rspec_string)
 392         instance_name = hrn_to_os_slicename(slice_hrn)
 393         tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
 394         slivers = aggregate.run_instances(instance_name, tenant_name, \
 395                                           rspec_string, key_name, pubkeys)
 396
 397         # update all sliver allocation states setting then to geni_allocated
 398         sliver_ids = [sliver.id for sliver in slivers]
 399         dbsession=self.api.dbsession()
 400         SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',dbsession)
 401
 402         return aggregate.describe(urns=[urn], version=rspec.version)
 403
 404     def provision(self, urns, options={}):
 405         # update sliver allocation states and set them to geni_provisioned
 406         aggregate = OSAggregate(self)
 407         instances = aggregate.get_instances(urns)
 408         sliver_ids = []
 409         for instance in instances:
 410             sliver_hrn = "%s.%s" % (self.driver.hrn, instance.id)
 411             sliver_ids.append(Xrn(sliver_hrn, type='sliver').urn)
 412         dbsession=self.api.dbsession()
 413         SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',dbsession)
 414         version_manager = VersionManager()
 415         rspec_version = version_manager.get_version(options['geni_rspec_version'])
 416         return self.describe(urns, rspec_version, options=options)
 417
 418     def delete (self, urns, options={}):
 419         # collect sliver ids so we can update sliver allocation states after
 420         # we remove the slivers.
 421         aggregate = OSAggregate(self)
 422         instances = aggregate.get_instances(urns)
 423         sliver_ids = []
 424         for instance in instances:
 425             sliver_hrn = "%s.%s" % (self.driver.hrn, instance.id)
 426             sliver_ids.append(Xrn(sliver_hrn, type='sliver').urn)
 427
 428             # delete the instance
 429             aggregate.delete_instance(instance)
 430
 431         # delete sliver allocation states
 432         dbsession=self.api.dbsession()
 433         SliverAllocation.delete_allocations(sliver_ids, dbsession)
 434
 435         # return geni_slivers
 436         geni_slivers = []
 437         for sliver_id in sliver_ids:
 438             geni_slivers.append(
 439                 {'geni_sliver_urn': sliver['sliver_id'],
 440                  'geni_allocation_status': 'geni_unallocated',
 441                  'geni_expires': None})
 442         return geni_slivers
 443
 444     def renew (self, urns, expiration_time, options={}):
 445         description = self.describe(urns, None, options)
 446         return description['geni_slivers']
 447
 448     def perform_operational_action  (self, urns, action, options={}):
 449         aggregate = OSAggregate(self)
 450         action = action.lower()
 451         if action == 'geni_start':
 452             action_method = aggregate.start_instances
 453         elif action == 'geni_stop':
 454             action_method = aggregate.stop_instances
 455         elif action == 'geni_restart':
 456             action_method = aggreate.restart_instances
 457         else:
 458             raise UnsupportedOperation(action)
 459
 460          # fault if sliver is not full allocated (operational status is geni_pending_allocation)
 461         description = self.describe(urns, None, options)
 462         for sliver in description['geni_slivers']:
 463             if sliver['geni_operational_status'] == 'geni_pending_allocation':
 464                 raise UnsupportedOperation(action, "Sliver must be fully allocated (operational status is not geni_pending_allocation)")
 465         #
 466         # Perform Operational Action Here
 467         #
 468
 469         instances = aggregate.get_instances(urns)
 470         for instance in instances:
 471             tenant_name = self.driver.shell.auth_manager.client.tenant_name
 472             action_method(tenant_name, instance.name, instance.id)
 473         description = self.describe(urns)
 474         geni_slivers = self.describe(urns, None, options)['geni_slivers']
 475         return geni_slivers
 476
 477     def shutdown(self, xrn, options={}):
 478         xrn = OSXrn(xrn=xrn, type='slice')
 479         tenant_name = xrn.get_tenant_name()
 480         name = xrn.get_slicename()
 481         self.driver.shell.nova_manager.connect(tenant=tenant_name)
 482         instances = self.driver.shell.nova_manager.servers.findall(name=name)
 483         for instance in instances:
 484             self.driver.shell.nova_manager.servers.shutdown(instance)
 485         return True