sfa/openstack/nova_driver.py

   1 import time
   2 import datetime
   3
   4 from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \
   5     RecordNotFound, SfaNotImplemented, SliverDoesNotExist, \
   6     SfaInvalidArgument
   7
   8 from sfa.util.sfalogging import logger
   9 from sfa.util.defaultdict import defaultdict
  10 from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch
  11 from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf, urn_to_sliver_id
  12 from sfa.openstack.osxrn import OSXrn, hrn_to_os_slicename, hrn_to_os_tenant_name
  13 from sfa.util.cache import Cache
  14 from sfa.trust.credential import Credential
  15 # used to be used in get_ticket
  16 #from sfa.trust.sfaticket import SfaTicket
  17
  18 from sfa.rspecs.version_manager import VersionManager
  19 from sfa.rspecs.rspec import RSpec
  20
  21 # the driver interface, mostly provides default behaviours
  22 from sfa.managers.driver import Driver
  23 from sfa.openstack.shell import Shell
  24 from sfa.openstack.osaggregate import OSAggregate
  25 from sfa.planetlab.plslices import PlSlices
  26
  27 def list_to_dict(recs, key):
  28     """
  29     convert a list of dictionaries into a dictionary keyed on the
  30     specified dictionary key
  31     """
  32     return dict ( [ (rec[key],rec) for rec in recs ] )
  33
  34 #
  35 # PlShell is just an xmlrpc serverproxy where methods
  36 # can be sent as-is; it takes care of authentication
  37 # from the global config
  38 #
  39 class NovaDriver(Driver):
  40
  41     # the cache instance is a class member so it survives across incoming requests
  42     cache = None
  43
  44     def __init__ (self, config):
  45         Driver.__init__(self, config)
  46         self.shell = Shell(config)
  47         self.cache=None
  48         if config.SFA_AGGREGATE_CACHING:
  49             if NovaDriver.cache is None:
  50                 NovaDriver.cache = Cache()
  51             self.cache = NovaDriver.cache
  52
  53     ########################################
  54     ########## registry oriented
  55     ########################################
  56
  57     ########## disabled users
  58     def is_enabled (self, record):
  59         # all records are enabled
  60         return True
  61
  62     def augment_records_with_testbed_info (self, sfa_records):
  63         return self.fill_record_info (sfa_records)
  64
  65     ##########
  66     def register (self, sfa_record, hrn, pub_key):
  67
  68         if sfa_record['type'] == 'slice':
  69             record = self.register_slice(sfa_record, hrn)
  70         elif sfa_record['type'] == 'user':
  71             record = self.register_user(sfa_record, hrn, pub_key)
  72         elif sfa_record['type'].startswith('authority'):
  73             record = self.register_authority(sfa_record, hrn)
  74         # We should be returning the records id as a pointer but
  75         # this is a string and the records table expects this to be an
  76         # int.
  77         #return record.id
  78         return -1
  79
  80     def register_slice(self, sfa_record, hrn):
  81         # add slice description, name, researchers, PI
  82         name = hrn_to_os_tenant_name(hrn)
  83         description = sfa_record.get('description', None)
  84         self.shell.auth_manager.tenants.create(name, description)
  85         tenant = self.shell.auth_manager.tenants.find(name=name)
  86         auth_hrn = OSXrn(xrn=hrn, type='slice').get_authority_hrn()
  87         parent_tenant_name = OSXrn(xrn=auth_hrn, type='slice').get_tenant_name()
  88         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
  89         researchers = sfa_record.get('researchers', [])
  90         for researcher in researchers:
  91             name = Xrn(researcher).get_leaf()
  92             user = self.shell.auth_manager.users.find(name=name)
  93             self.shell.auth_manager.roles.add_user_role(user, 'user', tenant)
  94
  95         pis = sfa_record.get('pis', [])
  96         for pi in pis:
  97             name = Xrn(pi).get_leaf()
  98             user = self.shell.auth_manager.users.find(name=name)
  99             self.shell.auth_manager.roles.add_user_role(user, 'pi', tenant)
 100             self.shell.auth_manager.roles.add_user_role(user, 'pi', parent_tenant)
 101
 102         return tenant
 103
 104     def register_user(self, sfa_record, hrn, pub_key):
 105         # add person roles, projects and keys
 106         email = sfa_record.get('email', None)
 107         xrn = Xrn(hrn)
 108         name = xrn.get_leaf()
 109         auth_hrn = xrn.get_authority_hrn()
 110         tenant_name = OSXrn(xrn=auth_hrn, type='authority').get_tenant_name()
 111         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 112         self.shell.auth_manager.users.create(name, email=email, tenant_id=tenant.id)
 113         user = self.shell.auth_manager.users.find(name=name)
 114         slices = sfa_records.get('slices', [])
 115         for slice in projects:
 116             slice_tenant_name = OSXrn(xrn=slice, type='slice').get_tenant_name()
 117             slice_tenant = self.shell.auth_manager.tenants.find(name=slice_tenant_name)
 118             self.shell.auth_manager.roles.add_user_role(user, slice_tenant, 'user')
 119         keys = sfa_records.get('keys', [])
 120         for key in keys:
 121             keyname = OSXrn(xrn=hrn, type='user').get_slicename()
 122             self.shell.nova_client.keypairs.create(keyname, key)
 123         return user
 124
 125     def register_authority(self, sfa_record, hrn):
 126         name = OSXrn(xrn=hrn, type='authority').get_tenant_name()
 127         self.shell.auth_manager.tenants.create(name, sfa_record.get('description', ''))
 128         tenant = self.shell.auth_manager.tenants.find(name=name)
 129         return tenant
 130
 131
 132     ##########
 133     # xxx actually old_sfa_record comes filled with plc stuff as well in the original code
 134     def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
 135         type = new_sfa_record['type']
 136
 137         # new_key implemented for users only
 138         if new_key and type not in [ 'user' ]:
 139             raise UnknownSfaType(type)
 140
 141         elif type == "slice":
 142             # can update project manager and description
 143             name = hrn_to_os_slicename(hrn)
 144             researchers = sfa_record.get('researchers', [])
 145             pis = sfa_record.get('pis', [])
 146             project_manager = None
 147             description = sfa_record.get('description', None)
 148             if pis:
 149                 project_manager = Xrn(pis[0], 'user').get_leaf()
 150             elif researchers:
 151                 project_manager = Xrn(researchers[0], 'user').get_leaf()
 152             self.shell.auth_manager.modify_project(name, project_manager, description)
 153
 154         elif type == "user":
 155             # can techinally update access_key and secret_key,
 156             # but that is not in our scope, so we do nothing.
 157             pass
 158         return True
 159
 160
 161     ##########
 162     def remove (self, sfa_record):
 163         type=sfa_record['type']
 164         if type == 'user':
 165             name = Xrn(sfa_record['hrn']).get_leaf()
 166             if self.shell.auth_manager.get_user(name):
 167                 self.shell.auth_manager.delete_user(name)
 168         elif type == 'slice':
 169             name = hrn_to_os_slicename(sfa_record['hrn'])
 170             if self.shell.auth_manager.get_project(name):
 171                 self.shell.auth_manager.delete_project(name)
 172         return True
 173
 174
 175     ####################
 176     def fill_record_info(self, records):
 177         """
 178         Given a (list of) SFA record, fill in the PLC specific
 179         and SFA specific fields in the record.
 180         """
 181         if not isinstance(records, list):
 182             records = [records]
 183
 184         for record in records:
 185             if record['type'] == 'user':
 186                 record = self.fill_user_record_info(record)
 187             elif record['type'] == 'slice':
 188                 record = self.fill_slice_record_info(record)
 189             elif record['type'].startswith('authority'):
 190                 record = self.fill_auth_record_info(record)
 191             else:
 192                 continue
 193             record['geni_urn'] = hrn_to_urn(record['hrn'], record['type'])
 194             record['geni_certificate'] = record['gid']
 195             #if os_record.created_at is not None:
 196             #    record['date_created'] = datetime_to_string(utcparse(os_record.created_at))
 197             #if os_record.updated_at is not None:
 198             #    record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at))
 199
 200         return records
 201
 202     def fill_user_record_info(self, record):
 203         xrn = Xrn(record['hrn'])
 204         name = xrn.get_leaf()
 205         record['name'] = name
 206         user = self.shell.auth_manager.users.find(name=name)
 207         record['email'] = user.email
 208         tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 209         slices = []
 210         all_tenants = self.shell.auth_manager.tenants.list()
 211         for tmp_tenant in all_tenants:
 212             if tmp_tenant.name.startswith(tenant.name +"."):
 213                 for tmp_user in tmp_tenant.list_users():
 214                     if tmp_user.name == user.name:
 215                         slice_hrn = ".".join([self.hrn, tmp_tenant.name])
 216                         slices.append(slice_hrn)
 217         record['slices'] = slices
 218         roles = self.shell.auth_manager.roles.roles_for_user(user, tenant)
 219         record['roles'] = [role.name for role in roles]
 220         keys = self.shell.nova_manager.keypairs.findall(name=record['hrn'])
 221         record['keys'] = [key.public_key for key in keys]
 222         return record
 223
 224     def fill_slice_record_info(self, record):
 225         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 226         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 227         parent_tenant_name = OSXrn(xrn=tenant_name).get_authority_hrn()
 228         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
 229         researchers = []
 230         pis = []
 231
 232         # look for users and pis in slice tenant
 233         for user in tenant.list_users():
 234             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 235                 if role.name.lower() == 'pi':
 236                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 237                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 238                     pis.append(hrn)
 239                 elif role.name.lower() in ['user', 'member']:
 240                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 241                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 242                     researchers.append(hrn)
 243
 244         # look for pis in the slice's parent (site/organization) tenant
 245         for user in parent_tenant.list_users():
 246             for role in self.shell.auth_manager.roles.roles_for_user(user, parent_tenant):
 247                 if role.name.lower() == 'pi':
 248                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 249                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 250                     pis.append(hrn)
 251         record['name'] = tenant_name
 252         record['description'] = tenant.description
 253         record['PI'] = pis
 254         if pis:
 255             record['geni_creator'] = pis[0]
 256         else:
 257             record['geni_creator'] = None
 258         record['researcher'] = researchers
 259         return record
 260
 261     def fill_auth_record_info(self, record):
 262         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 263         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 264         researchers = []
 265         pis = []
 266
 267         # look for users and pis in slice tenant
 268         for user in tenant.list_users():
 269             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 270                 hrn = ".".join([self.hrn, tenant.name, user.name])
 271                 if role.name.lower() == 'pi':
 272                     pis.append(hrn)
 273                 elif role.name.lower() in ['user', 'member']:
 274                     researchers.append(hrn)
 275
 276         # look for slices
 277         slices = []
 278         all_tenants = self.shell.auth_manager.tenants.list()
 279         for tmp_tenant in all_tenants:
 280             if tmp_tenant.name.startswith(tenant.name+"."):
 281                 slices.append(".".join([self.hrn, tmp_tenant.name]))
 282
 283         record['name'] = tenant_name
 284         record['description'] = tenant.description
 285         record['PI'] = pis
 286         record['enabled'] = tenant.enabled
 287         record['researchers'] = researchers
 288         record['slices'] = slices
 289         return record
 290
 291     ####################
 292     # plcapi works by changes, compute what needs to be added/deleted
 293     def update_relation (self, subject_type, target_type, subject_id, target_ids):
 294         # hard-wire the code for slice/user for now, could be smarter if needed
 295         if subject_type =='slice' and target_type == 'user':
 296             subject=self.shell.project_get(subject_id)[0]
 297             current_target_ids = [user.name for user in subject.members]
 298             add_target_ids = list ( set (target_ids).difference(current_target_ids))
 299             del_target_ids = list ( set (current_target_ids).difference(target_ids))
 300             logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id)))
 301             for target_id in add_target_ids:
 302                 self.shell.project_add_member(target_id,subject_id)
 303                 logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id)))
 304             for target_id in del_target_ids:
 305                 logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id)))
 306                 self.shell.project_remove_member(target_id, subject_id)
 307         else:
 308             logger.info('unexpected relation to maintain, %s -> %s'%(subject_type,target_type))
 309
 310
 311     ########################################
 312     ########## aggregate oriented
 313     ########################################
 314
 315     def testbed_name (self): return "openstack"
 316
 317     # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
 318     def aggregate_version (self):
 319         version_manager = VersionManager()
 320         ad_rspec_versions = []
 321         request_rspec_versions = []
 322         for rspec_version in version_manager.versions:
 323             if rspec_version.content_type in ['*', 'ad']:
 324                 ad_rspec_versions.append(rspec_version.to_dict())
 325             if rspec_version.content_type in ['*', 'request']:
 326                 request_rspec_versions.append(rspec_version.to_dict())
 327         return {
 328             'testbed':self.testbed_name(),
 329             'geni_request_rspec_versions': request_rspec_versions,
 330             'geni_ad_rspec_versions': ad_rspec_versions,
 331             'geni_single_allocation': False, # Accept operations that act on as subset of slivers in a given state
 332             'geni_allocate': 'geni_many', # Multiple slivers can exist and be incrementally added, including those which connect or overlap in some way.
 333             'geni_best_effort': 'true',
 334             }
 335
 336     def list_slices (self, creds, options):
 337         # look in cache first
 338         if self.cache:
 339             slices = self.cache.get('slices')
 340             if slices:
 341                 logger.debug("OpenStackDriver.list_slices returns from cache")
 342                 return slices
 343
 344         # get data from db
 345         instance_urns = []
 346         instances = self.shell.nova_manager.servers.findall()
 347         for instance in instances:
 348             if instance.name not in instance_urns:
 349                 instance_urns.append(OSXrn(instance.name, type='slice').urn)
 350
 351         # cache the result
 352         if self.cache:
 353             logger.debug ("OpenStackDriver.list_slices stores value in cache")
 354             self.cache.add('slices', instance_urns)
 355
 356         return instance_urns
 357
 358     # first 2 args are None in case of resource discovery
 359     def list_resources (self, slice_urn, slice_hrn, creds, options):
 360         cached_requested = options.get('cached', True)
 361
 362         version_manager = VersionManager()
 363         # get the rspec's return format from options
 364         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
 365         version_string = "rspec_%s" % (rspec_version)
 366
 367         #panos adding the info option to the caching key (can be improved)
 368         if options.get('info'):
 369             version_string = version_string + "_"+options.get('info', 'default')
 370
 371         # look in cache first
 372         if cached_requested and self.cache and not slice_hrn:
 373             rspec = self.cache.get(version_string)
 374             if rspec:
 375                 logger.debug("OpenStackDriver.ListResources: returning cached advertisement")
 376                 return rspec
 377
 378         #panos: passing user-defined options
 379         #print "manager options = ",options
 380         aggregate = OSAggregate(self)
 381         rspec =  aggregate.get_rspec(slice_xrn=slice_urn, version=rspec_version,
 382                                      options=options)
 383
 384         # cache the result
 385         if self.cache and not slice_hrn:
 386             logger.debug("OpenStackDriver.ListResources: stores advertisement in cache")
 387             self.cache.add(version_string, rspec)
 388
 389         return rspec
 390
 391     def sliver_status (self, slice_urn, slice_hrn):
 392         # find out where this slice is currently running
 393         project_name = hrn_to_os_slicename(slice_hrn)
 394         project = self.shell.auth_manager.get_project(project_name)
 395         instances = self.shell.db.instance_get_all_by_project(project_name)
 396         if len(instances) == 0:
 397             raise SliverDoesNotExist("You have not allocated any slivers here")
 398
 399         result = {}
 400         result['geni_urn'] = slice_urn
 401         result['plos_login'] = 'root'
 402         # do we need real dates here?
 403         result['plos_expires'] = None
 404         result['geni_expires'] = None
 405
 406         resources = []
 407         for instance in instances:
 408             res = {}
 409             # instances are accessed by ip, not hostname. We need to report the ip
 410             # somewhere so users know where to ssh to.
 411             res['geni_expires'] = None
 412             res['plos_hostname'] = instance.hostname
 413             res['plos_created_at'] = datetime_to_string(utcparse(instance.created_at))
 414             res['plos_boot_state'] = instance.vm_state
 415             res['plos_sliver_type'] = instance.instance_type.name
 416             sliver_id =  Xrn(slice_urn).get_sliver_id(instance.project_id, \
 417                                                       instance.hostname, instance.id)
 418             res['geni_urn'] = sliver_id
 419
 420             if instance.vm_state == 'running':
 421                 res['boot_state'] = 'ready'
 422                 res['geni_status'] = 'ready'
 423             else:
 424                 res['boot_state'] = 'unknown'
 425                 res['geni_status'] = 'unknown'
 426             res['geni_allocation_status'] = 'geni_provisioned'
 427             resources.append(res)
 428
 429         result['geni_resources'] = resources
 430         return result
 431
 432     def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options):
 433
 434         aggregate = OSAggregate(self)
 435         rspec = RSpec(rspec_string)
 436         instance_name = hrn_to_os_slicename(slice_hrn)
 437
 438         # assume first user is the caller and use their context
 439         # for the ec2/euca api connection. Also, use the first users
 440         # key as the project key.
 441         key_name = None
 442         if len(users) > 1:
 443             key_name = aggregate.create_instance_key(slice_hrn, users[0])
 444
 445         # collect public keys
 446         pubkeys = []
 447         for user in users:
 448             pubkeys.extend(user['keys'])
 449
 450         aggregate.run_instances(instance_name, rspec_string, key_name, pubkeys)
 451
 452         return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version)
 453
 454     def delete_sliver (self, slice_urn, slice_hrn, creds, options):
 455         aggregate = OSAggregate(self)
 456         project_name = hrn_to_os_slicename(slice_hrn)
 457         return aggregate.delete_instances(project_name)
 458
 459     def update_sliver(self, slice_urn, slice_hrn, rspec, creds, options):
 460         name = hrn_to_os_slicename(slice_hrn)
 461         aggregate = OSAggregate(self)
 462         return aggregate.update_instances(name)
 463
 464     def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options):
 465         return True
 466
 467     def start_slice (self, slice_urn, slice_hrn, creds):
 468         return 1
 469
 470     def stop_slice (self, slice_urn, slice_hrn, creds):
 471         name = OSXrn(xrn=slice_urn).name
 472         aggregate = OSAggregate(self)
 473         return aggregate.stop_instances(name)
 474
 475     def reset_slice (self, slice_urn, slice_hrn, creds):
 476         raise SfaNotImplemented ("reset_slice not available at this interface")
 477
 478     # xxx this code is quite old and has not run for ages
 479     # it is obviously totally broken and needs a rewrite
 480     def get_ticket (self, slice_urn, slice_hrn, creds, rspec_string, options):
 481         raise SfaNotImplemented,"OpenStackDriver.get_ticket needs a rewrite"
 482 # please keep this code for future reference
 483 #        slices = PlSlices(self)
 484 #        peer = slices.get_peer(slice_hrn)
 485 #        sfa_peer = slices.get_sfa_peer(slice_hrn)
 486 #
 487 #        # get the slice record
 488 #        credential = api.getCredential()
 489 #        interface = api.registries[api.hrn]
 490 #        registry = api.server_proxy(interface, credential)
 491 #        records = registry.Resolve(xrn, credential)
 492 #
 493 #        # make sure we get a local slice record
 494 #        record = None
 495 #        for tmp_record in records:
 496 #            if tmp_record['type'] == 'slice' and \
 497 #               not tmp_record['peer_authority']:
 498 #    #Error (E0602, GetTicket): Undefined variable 'SliceRecord'
 499 #                slice_record = SliceRecord(dict=tmp_record)
 500 #        if not record:
 501 #            raise RecordNotFound(slice_hrn)
 502 #
 503 #        # similar to CreateSliver, we must verify that the required records exist
 504 #        # at this aggregate before we can issue a ticket
 505 #        # parse rspec
 506 #        rspec = RSpec(rspec_string)
 507 #        requested_attributes = rspec.version.get_slice_attributes()
 508 #
 509 #        # ensure site record exists
 510 #        site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer)
 511 #        # ensure slice record exists
 512 #        slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer)
 513 #        # ensure person records exists
 514 #    # xxx users is undefined in this context
 515 #        persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer)
 516 #        # ensure slice attributes exists
 517 #        slices.verify_slice_attributes(slice, requested_attributes)
 518 #
 519 #        # get sliver info
 520 #        slivers = slices.get_slivers(slice_hrn)
 521 #
 522 #        if not slivers:
 523 #            raise SliverDoesNotExist(slice_hrn)
 524 #
 525 #        # get initscripts
 526 #        initscripts = []
 527 #        data = {
 528 #            'timestamp': int(time.time()),
 529 #            'initscripts': initscripts,
 530 #            'slivers': slivers
 531 #        }
 532 #
 533 #        # create the ticket
 534 #        object_gid = record.get_gid_object()
 535 #        new_ticket = SfaTicket(subject = object_gid.get_subject())
 536 #        new_ticket.set_gid_caller(api.auth.client_gid)
 537 #        new_ticket.set_gid_object(object_gid)
 538 #        new_ticket.set_issuer(key=api.key, subject=self.hrn)
 539 #        new_ticket.set_pubkey(object_gid.get_pubkey())
 540 #        new_ticket.set_attributes(data)
 541 #        new_ticket.set_rspec(rspec)
 542 #        #new_ticket.set_parent(api.auth.hierarchy.get_auth_ticket(auth_hrn))
 543 #        new_ticket.encode()
 544 #        new_ticket.sign()
 545 #
 546 #        return new_ticket.save_to_string(save_parents=True)