sfa/openstack/nova_driver.py

   1 import time
   2 import datetime
   3
   4 from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \
   5     RecordNotFound, SfaNotImplemented, SliverDoesNotExist, \
   6     SfaInvalidArgument
   7
   8 from sfa.util.sfalogging import logger
   9 from sfa.util.defaultdict import defaultdict
  10 from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch
  11 from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf, urn_to_sliver_id
  12 from sfa.openstack.osxrn import OSXrn, hrn_to_os_slicename, hrn_to_os_tenant_name
  13 from sfa.util.cache import Cache
  14 from sfa.trust.credential import Credential
  15 # used to be used in get_ticket
  16 #from sfa.trust.sfaticket import SfaTicket
  17
  18 from sfa.rspecs.version_manager import VersionManager
  19 from sfa.rspecs.rspec import RSpec
  20
  21 # the driver interface, mostly provides default behaviours
  22 from sfa.managers.driver import Driver
  23 from sfa.openstack.shell import Shell
  24 from sfa.openstack.osaggregate import OSAggregate
  25 from sfa.planetlab.plslices import PlSlices
  26
  27 def list_to_dict(recs, key):
  28     """
  29     convert a list of dictionaries into a dictionary keyed on the
  30     specified dictionary key
  31     """
  32     return dict ( [ (rec[key],rec) for rec in recs ] )
  33
  34 #
  35 # PlShell is just an xmlrpc serverproxy where methods
  36 # can be sent as-is; it takes care of authentication
  37 # from the global config
  38 #
  39 class NovaDriver(Driver):
  40
  41     # the cache instance is a class member so it survives across incoming requests
  42     cache = None
  43
  44     def __init__ (self, config):
  45         Driver.__init__(self, config)
  46         self.shell = Shell(config)
  47         self.cache=None
  48         if config.SFA_AGGREGATE_CACHING:
  49             if NovaDriver.cache is None:
  50                 NovaDriver.cache = Cache()
  51             self.cache = NovaDriver.cache
  52
  53     ########################################
  54     ########## registry oriented
  55     ########################################
  56
  57     ########## disabled users
  58     def is_enabled (self, record):
  59         # all records are enabled
  60         return True
  61
  62     def augment_records_with_testbed_info (self, sfa_records):
  63         return self.fill_record_info (sfa_records)
  64
  65     ##########
  66     def register (self, sfa_record, hrn, pub_key):
  67
  68         if sfa_record['type'] == 'slice':
  69             record = self.register_slice(sfa_record, hrn)
  70         elif sfa_record['type'] == 'user':
  71             record = self.register_user(sfa_record, hrn, pub_key)
  72         elif sfa_record['type'].startswith('authority'):
  73             record = self.register_authority(sfa_record, hrn)
  74         # We should be returning the records id as a pointer but
  75         # this is a string and the records table expects this to be an
  76         # int.
  77         #return record.id
  78         return -1
  79
  80     def register_slice(self, sfa_record, hrn):
  81         # add slice description, name, researchers, PI
  82         name = hrn_to_os_tenant_name(hrn)
  83         description = sfa_record.get('description', None)
  84         self.shell.auth_manager.tenants.create(name, description)
  85         tenant = self.shell.auth_manager.tenants.find(name=name)
  86         auth_hrn = OSXrn(xrn=hrn, type='slice').get_authority_hrn()
  87         parent_tenant_name = OSXrn(xrn=auth_hrn, type='slice').get_tenant_name()
  88         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
  89         researchers = sfa_record.get('researchers', [])
  90         for researcher in researchers:
  91             name = Xrn(researcher).get_leaf()
  92             user = self.shell.auth_manager.users.find(name=name)
  93             self.shell.auth_manager.roles.add_user_role(user, 'user', tenant)
  94
  95         pis = sfa_record.get('pis', [])
  96         for pi in pis:
  97             name = Xrn(pi).get_leaf()
  98             user = self.shell.auth_manager.users.find(name=name)
  99             self.shell.auth_manager.roles.add_user_role(user, 'pi', tenant)
 100             self.shell.auth_manager.roles.add_user_role(user, 'pi', parent_tenant)
 101
 102         return tenant
 103
 104     def register_user(self, sfa_record, hrn, pub_key):
 105         # add person roles, projects and keys
 106         email = sfa_record.get('email', None)
 107         xrn = Xrn(hrn)
 108         name = xrn.get_leaf()
 109         auth_hrn = xrn.get_authority_hrn()
 110         tenant_name = OSXrn(xrn=auth_hrn, type='authority').get_tenant_name()
 111         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 112         self.shell.auth_manager.users.create(name, email=email, tenant_id=tenant.id)
 113         user = self.shell.auth_manager.users.find(name=name)
 114         slices = sfa_records.get('slices', [])
 115         for slice in projects:
 116             slice_tenant_name = OSXrn(xrn=slice, type='slice').get_tenant_name()
 117             slice_tenant = self.shell.auth_manager.tenants.find(name=slice_tenant_name)
 118             self.shell.auth_manager.roles.add_user_role(user, slice_tenant, 'user')
 119         keys = sfa_records.get('keys', [])
 120         for key in keys:
 121             self.shell.nova_client.keypairs.create(name, key)
 122         return user
 123
 124     def register_authority(self, sfa_record, hrn):
 125         name = OSXrn(xrn=hrn, type='authority').get_tenant_name()
 126         self.shell.auth_manager.tenants.create(name, sfa_record.get('description', ''))
 127         tenant = self.shell.auth_manager.tenants.find(name=name)
 128         return tenant
 129
 130
 131     ##########
 132     # xxx actually old_sfa_record comes filled with plc stuff as well in the original code
 133     def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
 134         type = new_sfa_record['type']
 135
 136         # new_key implemented for users only
 137         if new_key and type not in [ 'user' ]:
 138             raise UnknownSfaType(type)
 139
 140         elif type == "slice":
 141             # can update project manager and description
 142             name = hrn_to_os_slicename(hrn)
 143             researchers = sfa_record.get('researchers', [])
 144             pis = sfa_record.get('pis', [])
 145             project_manager = None
 146             description = sfa_record.get('description', None)
 147             if pis:
 148                 project_manager = Xrn(pis[0], 'user').get_leaf()
 149             elif researchers:
 150                 project_manager = Xrn(researchers[0], 'user').get_leaf()
 151             self.shell.auth_manager.modify_project(name, project_manager, description)
 152
 153         elif type == "user":
 154             # can techinally update access_key and secret_key,
 155             # but that is not in our scope, so we do nothing.
 156             pass
 157         return True
 158
 159
 160     ##########
 161     def remove (self, sfa_record):
 162         type=sfa_record['type']
 163         if type == 'user':
 164             name = Xrn(sfa_record['hrn']).get_leaf()
 165             if self.shell.auth_manager.get_user(name):
 166                 self.shell.auth_manager.delete_user(name)
 167         elif type == 'slice':
 168             name = hrn_to_os_slicename(sfa_record['hrn'])
 169             if self.shell.auth_manager.get_project(name):
 170                 self.shell.auth_manager.delete_project(name)
 171         return True
 172
 173
 174     ####################
 175     def fill_record_info(self, records):
 176         """
 177         Given a (list of) SFA record, fill in the PLC specific
 178         and SFA specific fields in the record.
 179         """
 180         if not isinstance(records, list):
 181             records = [records]
 182
 183         for record in records:
 184             if record['type'] == 'user':
 185                 record = self.fill_user_record_info(record)
 186             elif record['type'] == 'slice':
 187                 record = self.fill_slice_record_info(record)
 188             elif record['type'].startswith('authority'):
 189                 record = self.fill_auth_record_info(record)
 190             else:
 191                 continue
 192             record['geni_urn'] = hrn_to_urn(record['hrn'], record['type'])
 193             record['geni_certificate'] = record['gid']
 194             #if os_record.created_at is not None:
 195             #    record['date_created'] = datetime_to_string(utcparse(os_record.created_at))
 196             #if os_record.updated_at is not None:
 197             #    record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at))
 198
 199         return records
 200
 201     def fill_user_record_info(self, record):
 202         xrn = Xrn(record['hrn'])
 203         name = xrn.get_leaf()
 204         record['name'] = name
 205         user = self.shell.auth_manager.users.find(name=name)
 206         record['email'] = user.email
 207         tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 208         slices = []
 209         all_tenants = self.shell.auth_manager.tenants.list()
 210         for tmp_tenant in all_tenants:
 211             if tmp_tenant.name.startswith(tenant.name +"."):
 212                 for tmp_user in tmp_tenant.list_users():
 213                     if tmp_user.name == user.name:
 214                         slice_hrn = ".".join([self.hrn, tmp_tenant.name])
 215                         slices.append(slice_hrn)
 216         record['slices'] = slices
 217         roles = self.shell.auth_manager.roles.roles_for_user(user, tenant)
 218         record['roles'] = [role.name for role in roles]
 219         keys = self.shell.nova_manager.keypairs.findall(name=record['hrn'])
 220         record['keys'] = [key.public_key for key in keys]
 221         return record
 222
 223     def fill_slice_record_info(self, record):
 224         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 225         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 226         parent_tenant_name = OSXrn(xrn=tenant_name).get_authority_hrn()
 227         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
 228         researchers = []
 229         pis = []
 230
 231         # look for users and pis in slice tenant
 232         for user in tenant.list_users():
 233             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 234                 if role.name.lower() == 'pi':
 235                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 236                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 237                     pis.append(hrn)
 238                 elif role.name.lower() in ['user', 'member']:
 239                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 240                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 241                     researchers.append(hrn)
 242
 243         # look for pis in the slice's parent (site/organization) tenant
 244         for user in parent_tenant.list_users():
 245             for role in self.shell.auth_manager.roles.roles_for_user(user, parent_tenant):
 246                 if role.name.lower() == 'pi':
 247                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 248                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 249                     pis.append(hrn)
 250         record['name'] = tenant_name
 251         record['description'] = tenant.description
 252         record['PI'] = pis
 253         if pis:
 254             record['geni_creator'] = pis[0]
 255         else:
 256             record['geni_creator'] = None
 257         record['researcher'] = researchers
 258         return record
 259
 260     def fill_auth_record_info(self, record):
 261         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 262         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 263         researchers = []
 264         pis = []
 265
 266         # look for users and pis in slice tenant
 267         for user in tenant.list_users():
 268             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 269                 hrn = ".".join([self.hrn, tenant.name, user.name])
 270                 if role.name.lower() == 'pi':
 271                     pis.append(hrn)
 272                 elif role.name.lower() in ['user', 'member']:
 273                     researchers.append(hrn)
 274
 275         # look for slices
 276         slices = []
 277         all_tenants = self.shell.auth_manager.tenants.list()
 278         for tmp_tenant in all_tenants:
 279             if tmp_tenant.name.startswith(tenant.name+"."):
 280                 slices.append(".".join([self.hrn, tmp_tenant.name]))
 281
 282         record['name'] = tenant_name
 283         record['description'] = tenant.description
 284         record['PI'] = pis
 285         record['enabled'] = tenant.enabled
 286         record['researchers'] = researchers
 287         record['slices'] = slices
 288         return record
 289
 290     ####################
 291     # plcapi works by changes, compute what needs to be added/deleted
 292     def update_relation (self, subject_type, target_type, subject_id, target_ids):
 293         # hard-wire the code for slice/user for now, could be smarter if needed
 294         if subject_type =='slice' and target_type == 'user':
 295             subject=self.shell.project_get(subject_id)[0]
 296             current_target_ids = [user.name for user in subject.members]
 297             add_target_ids = list ( set (target_ids).difference(current_target_ids))
 298             del_target_ids = list ( set (current_target_ids).difference(target_ids))
 299             logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id)))
 300             for target_id in add_target_ids:
 301                 self.shell.project_add_member(target_id,subject_id)
 302                 logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id)))
 303             for target_id in del_target_ids:
 304                 logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id)))
 305                 self.shell.project_remove_member(target_id, subject_id)
 306         else:
 307             logger.info('unexpected relation to maintain, %s -> %s'%(subject_type,target_type))
 308
 309
 310     ########################################
 311     ########## aggregate oriented
 312     ########################################
 313
 314     def testbed_name (self): return "openstack"
 315
 316     # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
 317     def aggregate_version (self):
 318         version_manager = VersionManager()
 319         ad_rspec_versions = []
 320         request_rspec_versions = []
 321         for rspec_version in version_manager.versions:
 322             if rspec_version.content_type in ['*', 'ad']:
 323                 ad_rspec_versions.append(rspec_version.to_dict())
 324             if rspec_version.content_type in ['*', 'request']:
 325                 request_rspec_versions.append(rspec_version.to_dict())
 326         return {
 327             'testbed':self.testbed_name(),
 328             'geni_request_rspec_versions': request_rspec_versions,
 329             'geni_ad_rspec_versions': ad_rspec_versions,
 330             }
 331
 332     def list_slices (self, creds, options):
 333         # look in cache first
 334         if self.cache:
 335             slices = self.cache.get('slices')
 336             if slices:
 337                 logger.debug("OpenStackDriver.list_slices returns from cache")
 338                 return slices
 339
 340         # get data from db
 341         projs = self.shell.auth_manager.get_projects()
 342         slice_urns = [OSXrn(proj.name, 'slice').urn for proj in projs]
 343
 344         # cache the result
 345         if self.cache:
 346             logger.debug ("OpenStackDriver.list_slices stores value in cache")
 347             self.cache.add('slices', slice_urns)
 348
 349         return slice_urns
 350
 351     # first 2 args are None in case of resource discovery
 352     def list_resources (self, slice_urn, slice_hrn, creds, options):
 353         cached_requested = options.get('cached', True)
 354
 355         version_manager = VersionManager()
 356         # get the rspec's return format from options
 357         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
 358         version_string = "rspec_%s" % (rspec_version)
 359
 360         #panos adding the info option to the caching key (can be improved)
 361         if options.get('info'):
 362             version_string = version_string + "_"+options.get('info', 'default')
 363
 364         # look in cache first
 365         if cached_requested and self.cache and not slice_hrn:
 366             rspec = self.cache.get(version_string)
 367             if rspec:
 368                 logger.debug("OpenStackDriver.ListResources: returning cached advertisement")
 369                 return rspec
 370
 371         #panos: passing user-defined options
 372         #print "manager options = ",options
 373         aggregate = OSAggregate(self)
 374         rspec =  aggregate.get_rspec(slice_xrn=slice_urn, version=rspec_version,
 375                                      options=options)
 376
 377         # cache the result
 378         if self.cache and not slice_hrn:
 379             logger.debug("OpenStackDriver.ListResources: stores advertisement in cache")
 380             self.cache.add(version_string, rspec)
 381
 382         return rspec
 383
 384     def sliver_status (self, slice_urn, slice_hrn):
 385         # find out where this slice is currently running
 386         project_name = hrn_to_os_slicename(slice_hrn)
 387         project = self.shell.auth_manager.get_project(project_name)
 388         instances = self.shell.db.instance_get_all_by_project(project_name)
 389         if len(instances) == 0:
 390             raise SliverDoesNotExist("You have not allocated any slivers here")
 391
 392         result = {}
 393         top_level_status = 'unknown'
 394         if instances:
 395             top_level_status = 'ready'
 396         result['geni_urn'] = slice_urn
 397         result['plos_login'] = 'root'
 398         result['plos_expires'] = None
 399
 400         resources = []
 401         for instance in instances:
 402             res = {}
 403             # instances are accessed by ip, not hostname. We need to report the ip
 404             # somewhere so users know where to ssh to.
 405             res['plos_hostname'] = instance.hostname
 406             res['plos_created_at'] = datetime_to_string(utcparse(instance.created_at))
 407             res['plos_boot_state'] = instance.vm_state
 408             res['plos_sliver_type'] = instance.instance_type.name
 409             sliver_id =  Xrn(slice_urn).get_sliver_id(instance.project_id, \
 410                                                       instance.hostname, instance.id)
 411             res['geni_urn'] = sliver_id
 412
 413             if instance.vm_state == 'running':
 414                 res['boot_state'] = 'ready';
 415             else:
 416                 res['boot_state'] = 'unknown'
 417             resources.append(res)
 418
 419         result['geni_status'] = top_level_status
 420         result['geni_resources'] = resources
 421         return result
 422
 423     def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options):
 424
 425         aggregate = OSAggregate(self)
 426         rspec = RSpec(rspec_string)
 427         instance_name = hrn_to_os_slicename(slice_hrn)
 428
 429         # assume first user is the caller and use their context
 430         # for the ec2/euca api connection. Also, use the first users
 431         # key as the project key.
 432         key_name = None
 433         if len(users) > 1:
 434             key_name = aggregate.create_instance_key(slice_hrn, users[0])
 435
 436         # collect public keys
 437         pubkeys = []
 438         for user in users:
 439             pubkeys.extend(user['keys'])
 440
 441         aggregate.run_instances(instance_name, rspec_string, key_name, pubkeys)
 442
 443         return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version)
 444
 445     def delete_sliver (self, slice_urn, slice_hrn, creds, options):
 446         aggregate = OSAggregate(self)
 447         project_name = hrn_to_os_slicename(slice_hrn)
 448         return aggregate.delete_instances(project_name)
 449
 450     def update_sliver(self, slice_urn, slice_hrn, rspec, creds, options):
 451         name = hrn_to_os_slicename(slice_hrn)
 452         aggregate = OSAggregate(self)
 453         return aggregate.update_instances(name)
 454
 455     def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options):
 456         return True
 457
 458     def start_slice (self, slice_urn, slice_hrn, creds):
 459         return 1
 460
 461     def stop_slice (self, slice_urn, slice_hrn, creds):
 462         name = OSXrn(xrn=slice_urn).name
 463         aggregate = OSAggregate(self)
 464         return aggregate.stop_instances(name)
 465
 466     def reset_slice (self, slice_urn, slice_hrn, creds):
 467         raise SfaNotImplemented ("reset_slice not available at this interface")
 468
 469     # xxx this code is quite old and has not run for ages
 470     # it is obviously totally broken and needs a rewrite
 471     def get_ticket (self, slice_urn, slice_hrn, creds, rspec_string, options):
 472         raise SfaNotImplemented,"OpenStackDriver.get_ticket needs a rewrite"
 473 # please keep this code for future reference
 474 #        slices = PlSlices(self)
 475 #        peer = slices.get_peer(slice_hrn)
 476 #        sfa_peer = slices.get_sfa_peer(slice_hrn)
 477 #
 478 #        # get the slice record
 479 #        credential = api.getCredential()
 480 #        interface = api.registries[api.hrn]
 481 #        registry = api.server_proxy(interface, credential)
 482 #        records = registry.Resolve(xrn, credential)
 483 #
 484 #        # make sure we get a local slice record
 485 #        record = None
 486 #        for tmp_record in records:
 487 #            if tmp_record['type'] == 'slice' and \
 488 #               not tmp_record['peer_authority']:
 489 #    #Error (E0602, GetTicket): Undefined variable 'SliceRecord'
 490 #                slice_record = SliceRecord(dict=tmp_record)
 491 #        if not record:
 492 #            raise RecordNotFound(slice_hrn)
 493 #
 494 #        # similar to CreateSliver, we must verify that the required records exist
 495 #        # at this aggregate before we can issue a ticket
 496 #        # parse rspec
 497 #        rspec = RSpec(rspec_string)
 498 #        requested_attributes = rspec.version.get_slice_attributes()
 499 #
 500 #        # ensure site record exists
 501 #        site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer)
 502 #        # ensure slice record exists
 503 #        slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer)
 504 #        # ensure person records exists
 505 #    # xxx users is undefined in this context
 506 #        persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer)
 507 #        # ensure slice attributes exists
 508 #        slices.verify_slice_attributes(slice, requested_attributes)
 509 #
 510 #        # get sliver info
 511 #        slivers = slices.get_slivers(slice_hrn)
 512 #
 513 #        if not slivers:
 514 #            raise SliverDoesNotExist(slice_hrn)
 515 #
 516 #        # get initscripts
 517 #        initscripts = []
 518 #        data = {
 519 #            'timestamp': int(time.time()),
 520 #            'initscripts': initscripts,
 521 #            'slivers': slivers
 522 #        }
 523 #
 524 #        # create the ticket
 525 #        object_gid = record.get_gid_object()
 526 #        new_ticket = SfaTicket(subject = object_gid.get_subject())
 527 #        new_ticket.set_gid_caller(api.auth.client_gid)
 528 #        new_ticket.set_gid_object(object_gid)
 529 #        new_ticket.set_issuer(key=api.key, subject=self.hrn)
 530 #        new_ticket.set_pubkey(object_gid.get_pubkey())
 531 #        new_ticket.set_attributes(data)
 532 #        new_ticket.set_rspec(rspec)
 533 #        #new_ticket.set_parent(api.auth.hierarchy.get_auth_ticket(auth_hrn))
 534 #        new_ticket.encode()
 535 #        new_ticket.sign()
 536 #
 537 #        return new_ticket.save_to_string(save_parents=True)