sfa/openstack/nova_driver.py

   1 import time
   2 import datetime
   3
   4 from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \
   5     RecordNotFound, SfaNotImplemented, SliverDoesNotExist, \
   6     SfaInvalidArgument
   7
   8 from sfa.util.sfalogging import logger
   9 from sfa.util.defaultdict import defaultdict
  10 from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch
  11 from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf, urn_to_sliver_id
  12 from sfa.openstack.osxrn import OSXrn, hrn_to_os_slicename, hrn_to_os_tenant_name
  13 from sfa.util.cache import Cache
  14 from sfa.trust.credential import Credential
  15 # used to be used in get_ticket
  16 #from sfa.trust.sfaticket import SfaTicket
  17
  18 from sfa.rspecs.version_manager import VersionManager
  19 from sfa.rspecs.rspec import RSpec
  20
  21 # the driver interface, mostly provides default behaviours
  22 from sfa.managers.driver import Driver
  23 from sfa.openstack.shell import Shell
  24 from sfa.openstack.osaggregate import OSAggregate
  25 from sfa.planetlab.plslices import PlSlices
  26
  27 def list_to_dict(recs, key):
  28     """
  29     convert a list of dictionaries into a dictionary keyed on the
  30     specified dictionary key
  31     """
  32     return dict ( [ (rec[key],rec) for rec in recs ] )
  33
  34 #
  35 # PlShell is just an xmlrpc serverproxy where methods
  36 # can be sent as-is; it takes care of authentication
  37 # from the global config
  38 #
  39 class NovaDriver(Driver):
  40
  41     # the cache instance is a class member so it survives across incoming requests
  42     cache = None
  43
  44     def __init__ (self, config):
  45         Driver.__init__(self, config)
  46         self.shell = Shell(config=config)
  47         self.cache=None
  48         if config.SFA_AGGREGATE_CACHING:
  49             if NovaDriver.cache is None:
  50                 NovaDriver.cache = Cache()
  51             self.cache = NovaDriver.cache
  52
  53     ########################################
  54     ########## registry oriented
  55     ########################################
  56
  57     ########## disabled users
  58     def is_enabled (self, record):
  59         # all records are enabled
  60         return True
  61
  62     def augment_records_with_testbed_info (self, sfa_records):
  63         return self.fill_record_info (sfa_records)
  64
  65     ##########
  66     def register (self, sfa_record, hrn, pub_key):
  67
  68         if sfa_record['type'] == 'slice':
  69             record = self.register_slice(sfa_record, hrn)
  70         elif sfa_record['type'] == 'user':
  71             record = self.register_user(sfa_record, hrn, pub_key)
  72         elif sfa_record['type'].startswith('authority'):
  73             record = self.register_authority(sfa_record, hrn)
  74         # We should be returning the records id as a pointer but
  75         # this is a string and the records table expects this to be an
  76         # int.
  77         #return record.id
  78         return -1
  79
  80     def register_slice(self, sfa_record, hrn):
  81         # add slice description, name, researchers, PI
  82         name = hrn_to_os_tenant_name(hrn)
  83         description = sfa_record.get('description', None)
  84         self.shell.auth_manager.tenants.create(name, description)
  85         tenant = self.shell.auth_manager.tenants.find(name=name)
  86         auth_hrn = OSXrn(xrn=hrn, type='slice').get_authority_hrn()
  87         parent_tenant_name = OSXrn(xrn=auth_hrn, type='slice').get_tenant_name()
  88         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
  89         researchers = sfa_record.get('researchers', [])
  90         for researcher in researchers:
  91             name = Xrn(researcher).get_leaf()
  92             user = self.shell.auth_manager.users.find(name=name)
  93             self.shell.auth_manager.roles.add_user_role(user, 'Member', tenant)
  94             self.shell.auth_manager.roles.add_user_role(user, 'user', tenant)
  95
  96
  97         pis = sfa_record.get('pis', [])
  98         for pi in pis:
  99             name = Xrn(pi).get_leaf()
 100             user = self.shell.auth_manager.users.find(name=name)
 101             self.shell.auth_manager.roles.add_user_role(user, 'pi', tenant)
 102             self.shell.auth_manager.roles.add_user_role(user, 'pi', parent_tenant)
 103
 104         return tenant
 105
 106     def register_user(self, sfa_record, hrn, pub_key):
 107         # add person roles, projects and keys
 108         email = sfa_record.get('email', None)
 109         xrn = Xrn(hrn)
 110         name = xrn.get_leaf()
 111         auth_hrn = xrn.get_authority_hrn()
 112         tenant_name = OSXrn(xrn=auth_hrn, type='authority').get_tenant_name()
 113         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 114         self.shell.auth_manager.users.create(name, email=email, tenant_id=tenant.id)
 115         user = self.shell.auth_manager.users.find(name=name)
 116         slices = sfa_records.get('slices', [])
 117         for slice in projects:
 118             slice_tenant_name = OSXrn(xrn=slice, type='slice').get_tenant_name()
 119             slice_tenant = self.shell.auth_manager.tenants.find(name=slice_tenant_name)
 120             self.shell.auth_manager.roles.add_user_role(user, slice_tenant, 'user')
 121         keys = sfa_records.get('keys', [])
 122         for key in keys:
 123             keyname = OSXrn(xrn=hrn, type='user').get_slicename()
 124             self.shell.nova_client.keypairs.create(keyname, key)
 125         return user
 126
 127     def register_authority(self, sfa_record, hrn):
 128         name = OSXrn(xrn=hrn, type='authority').get_tenant_name()
 129         self.shell.auth_manager.tenants.create(name, sfa_record.get('description', ''))
 130         tenant = self.shell.auth_manager.tenants.find(name=name)
 131         return tenant
 132
 133
 134     ##########
 135     # xxx actually old_sfa_record comes filled with plc stuff as well in the original code
 136     def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
 137         type = new_sfa_record['type']
 138
 139         # new_key implemented for users only
 140         if new_key and type not in [ 'user' ]:
 141             raise UnknownSfaType(type)
 142
 143         elif type == "slice":
 144             # can update project manager and description
 145             name = hrn_to_os_slicename(hrn)
 146             researchers = sfa_record.get('researchers', [])
 147             pis = sfa_record.get('pis', [])
 148             project_manager = None
 149             description = sfa_record.get('description', None)
 150             if pis:
 151                 project_manager = Xrn(pis[0], 'user').get_leaf()
 152             elif researchers:
 153                 project_manager = Xrn(researchers[0], 'user').get_leaf()
 154             self.shell.auth_manager.modify_project(name, project_manager, description)
 155
 156         elif type == "user":
 157             # can techinally update access_key and secret_key,
 158             # but that is not in our scope, so we do nothing.
 159             pass
 160         return True
 161
 162
 163     ##########
 164     def remove (self, sfa_record):
 165         type=sfa_record['type']
 166         if type == 'user':
 167             name = Xrn(sfa_record['hrn']).get_leaf()
 168             if self.shell.auth_manager.get_user(name):
 169                 self.shell.auth_manager.delete_user(name)
 170         elif type == 'slice':
 171             name = hrn_to_os_slicename(sfa_record['hrn'])
 172             if self.shell.auth_manager.get_project(name):
 173                 self.shell.auth_manager.delete_project(name)
 174         return True
 175
 176
 177     ####################
 178     def fill_record_info(self, records):
 179         """
 180         Given a (list of) SFA record, fill in the PLC specific
 181         and SFA specific fields in the record.
 182         """
 183         if not isinstance(records, list):
 184             records = [records]
 185
 186         for record in records:
 187             if record['type'] == 'user':
 188                 record = self.fill_user_record_info(record)
 189             elif record['type'] == 'slice':
 190                 record = self.fill_slice_record_info(record)
 191             elif record['type'].startswith('authority'):
 192                 record = self.fill_auth_record_info(record)
 193             else:
 194                 continue
 195             record['geni_urn'] = hrn_to_urn(record['hrn'], record['type'])
 196             record['geni_certificate'] = record['gid']
 197             #if os_record.created_at is not None:
 198             #    record['date_created'] = datetime_to_string(utcparse(os_record.created_at))
 199             #if os_record.updated_at is not None:
 200             #    record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at))
 201
 202         return records
 203
 204     def fill_user_record_info(self, record):
 205         xrn = Xrn(record['hrn'])
 206         name = xrn.get_leaf()
 207         record['name'] = name
 208         user = self.shell.auth_manager.users.find(name=name)
 209         record['email'] = user.email
 210         tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 211         slices = []
 212         all_tenants = self.shell.auth_manager.tenants.list()
 213         for tmp_tenant in all_tenants:
 214             if tmp_tenant.name.startswith(tenant.name +"."):
 215                 for tmp_user in tmp_tenant.list_users():
 216                     if tmp_user.name == user.name:
 217                         slice_hrn = ".".join([self.hrn, tmp_tenant.name])
 218                         slices.append(slice_hrn)
 219         record['slices'] = slices
 220         roles = self.shell.auth_manager.roles.roles_for_user(user, tenant)
 221         record['roles'] = [role.name for role in roles]
 222         keys = self.shell.nova_manager.keypairs.findall(name=record['hrn'])
 223         record['keys'] = [key.public_key for key in keys]
 224         return record
 225
 226     def fill_slice_record_info(self, record):
 227         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 228         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 229         parent_tenant_name = OSXrn(xrn=tenant_name).get_authority_hrn()
 230         parent_tenant = self.shell.auth_manager.tenants.find(name=parent_tenant_name)
 231         researchers = []
 232         pis = []
 233
 234         # look for users and pis in slice tenant
 235         for user in tenant.list_users():
 236             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 237                 if role.name.lower() == 'pi':
 238                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 239                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 240                     pis.append(hrn)
 241                 elif role.name.lower() in ['user', 'member']:
 242                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 243                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 244                     researchers.append(hrn)
 245
 246         # look for pis in the slice's parent (site/organization) tenant
 247         for user in parent_tenant.list_users():
 248             for role in self.shell.auth_manager.roles.roles_for_user(user, parent_tenant):
 249                 if role.name.lower() == 'pi':
 250                     user_tenant = self.shell.auth_manager.tenants.find(id=user.tenantId)
 251                     hrn = ".".join([self.hrn, user_tenant.name, user.name])
 252                     pis.append(hrn)
 253         record['name'] = tenant_name
 254         record['description'] = tenant.description
 255         record['PI'] = pis
 256         if pis:
 257             record['geni_creator'] = pis[0]
 258         else:
 259             record['geni_creator'] = None
 260         record['researcher'] = researchers
 261         return record
 262
 263     def fill_auth_record_info(self, record):
 264         tenant_name = hrn_to_os_tenant_name(record['hrn'])
 265         tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
 266         researchers = []
 267         pis = []
 268
 269         # look for users and pis in slice tenant
 270         for user in tenant.list_users():
 271             for role in self.shell.auth_manager.roles.roles_for_user(user, tenant):
 272                 hrn = ".".join([self.hrn, tenant.name, user.name])
 273                 if role.name.lower() == 'pi':
 274                     pis.append(hrn)
 275                 elif role.name.lower() in ['user', 'member']:
 276                     researchers.append(hrn)
 277
 278         # look for slices
 279         slices = []
 280         all_tenants = self.shell.auth_manager.tenants.list()
 281         for tmp_tenant in all_tenants:
 282             if tmp_tenant.name.startswith(tenant.name+"."):
 283                 slices.append(".".join([self.hrn, tmp_tenant.name]))
 284
 285         record['name'] = tenant_name
 286         record['description'] = tenant.description
 287         record['PI'] = pis
 288         record['enabled'] = tenant.enabled
 289         record['researchers'] = researchers
 290         record['slices'] = slices
 291         return record
 292
 293     ####################
 294     # plcapi works by changes, compute what needs to be added/deleted
 295     def update_relation (self, subject_type, target_type, subject_id, target_ids):
 296         # hard-wire the code for slice/user for now, could be smarter if needed
 297         if subject_type =='slice' and target_type == 'user':
 298             subject=self.shell.project_get(subject_id)[0]
 299             current_target_ids = [user.name for user in subject.members]
 300             add_target_ids = list ( set (target_ids).difference(current_target_ids))
 301             del_target_ids = list ( set (current_target_ids).difference(target_ids))
 302             logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id)))
 303             for target_id in add_target_ids:
 304                 self.shell.project_add_member(target_id,subject_id)
 305                 logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id)))
 306             for target_id in del_target_ids:
 307                 logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id)))
 308                 self.shell.project_remove_member(target_id, subject_id)
 309         else:
 310             logger.info('unexpected relation to maintain, %s -> %s'%(subject_type,target_type))
 311
 312
 313     ########################################
 314     ########## aggregate oriented
 315     ########################################
 316
 317     def testbed_name (self): return "openstack"
 318
 319     def aggregate_version (self):
 320         return {}
 321
 322     def list_slices (self, creds, options):
 323         # get data from db
 324         instance_urns = []
 325         instances = self.shell.nova_manager.servers.findall()
 326         for instance in instances:
 327             if instance.name not in instance_urns:
 328                 instance_urns.append(OSXrn(instance.name, type='slice').urn)
 329         return instance_urns
 330
 331     # first 2 args are None in case of resource discovery
 332     def list_resources (self, creds, version, options):
 333         aggregate = OSAggregate(self)
 334         rspec =  aggregate.list_resources(version=version, options=options)
 335         return rspec
 336
 337     def describe(self, creds, urns, version, options):
 338         aggregate = OSAggregate(self)
 339         return aggregate.describe(urns, version=version, options=options)
 340
 341     def sliver_status (self, slice_urn, slice_hrn):
 342         # update nova connection
 343         tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
 344         self.shell.nova_manager.connect(tenant=tenant_name)
 345
 346         # find out where this slice is currently running
 347         project_name = hrn_to_os_slicename(slice_hrn)
 348         instances = self.shell.nova_manager.servers.findall(name=project_name)
 349         if len(instances) == 0:
 350             raise SliverDoesNotExist("You have not allocated any slivers here")
 351
 352         result = {}
 353         result['geni_urn'] = slice_urn
 354         result['plos_login'] = 'root'
 355         # do we need real dates here?
 356         result['plos_expires'] = None
 357         result['geni_expires'] = None
 358
 359         resources = []
 360         for instance in instances:
 361             res = {}
 362             # instances are accessed by ip, not hostname. We need to report the ip
 363             # somewhere so users know where to ssh to.
 364             res['geni_expires'] = None
 365             #res['plos_hostname'] = instance.hostname
 366             res['plos_created_at'] = datetime_to_string(utcparse(instance.created))
 367             res['plos_boot_state'] = instance.status
 368             res['plos_sliver_type'] = self.shell.nova_manager.flavors.find(id=instance.flavor['id']).name
 369             sliver_id =  Xrn(slice_urn).get_sliver_id(instance.id)
 370             res['geni_urn'] = sliver_id
 371
 372             if instance.status.lower() == 'active':
 373                 res['boot_state'] = 'ready'
 374                 res['geni_status'] = 'ready'
 375             else:
 376                 res['boot_state'] = 'unknown'
 377                 res['geni_status'] = 'unknown'
 378             res['geni_allocation_status'] = 'geni_provisioned'
 379             resources.append(res)
 380
 381         result['geni_resources'] = resources
 382         return result
 383
 384     def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options):
 385
 386         aggregate = OSAggregate(self)
 387
 388         # assume first user is the caller and use their context
 389         # for the ec2/euca api connection. Also, use the first users
 390         # key as the project key.
 391         key_name = None
 392         if len(users) > 1:
 393             key_name = aggregate.create_instance_key(slice_hrn, users[0])
 394
 395         # collect public keys
 396         pubkeys = []
 397         for user in users:
 398             pubkeys.extend(user['keys'])
 399
 400         rspec = RSpec(rspec_string)
 401         instance_name = hrn_to_os_slicename(slice_hrn)
 402         tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
 403         aggregate.run_instances(instance_name, tenant_name, rspec_string, key_name, pubkeys)
 404
 405         return aggregate.describe(slice_xrn=slice_urn, version=rspec.version)
 406
 407     def delete_sliver (self, slice_urn, slice_hrn, creds, options):
 408         aggregate = OSAggregate(self)
 409         tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
 410         project_name = hrn_to_os_slicename(slice_hrn)
 411         return aggregate.delete_instances(project_name, tenant_name)
 412
 413     def update_sliver(self, slice_urn, slice_hrn, rspec, creds, options):
 414         name = hrn_to_os_slicename(slice_hrn)
 415         tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
 416         aggregate = OSAggregate(self)
 417         return aggregate.update_instances(name)
 418
 419     def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options):
 420         return True
 421
 422     def start_slice (self, slice_urn, slice_hrn, creds):
 423         return 1
 424
 425     def stop_slice (self, slice_urn, slice_hrn, creds):
 426         tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
 427         name = OSXrn(xrn=slice_urn).name
 428         aggregate = OSAggregate(self)
 429         return aggregate.stop_instances(name, tenant_name)
 430
 431     def reset_slice (self, slice_urn, slice_hrn, creds):
 432         raise SfaNotImplemented ("reset_slice not available at this interface")
 433
 434     # xxx this code is quite old and has not run for ages
 435     # it is obviously totally broken and needs a rewrite
 436     def get_ticket (self, slice_urn, slice_hrn, creds, rspec_string, options):
 437         raise SfaNotImplemented,"OpenStackDriver.get_ticket needs a rewrite"
 438 # please keep this code for future reference
 439 #        slices = PlSlices(self)
 440 #        peer = slices.get_peer(slice_hrn)
 441 #        sfa_peer = slices.get_sfa_peer(slice_hrn)
 442 #
 443 #        # get the slice record
 444 #        credential = api.getCredential()
 445 #        interface = api.registries[api.hrn]
 446 #        registry = api.server_proxy(interface, credential)
 447 #        records = registry.Resolve(xrn, credential)
 448 #
 449 #        # make sure we get a local slice record
 450 #        record = None
 451 #        for tmp_record in records:
 452 #            if tmp_record['type'] == 'slice' and \
 453 #               not tmp_record['peer_authority']:
 454 #    #Error (E0602, GetTicket): Undefined variable 'SliceRecord'
 455 #                slice_record = SliceRecord(dict=tmp_record)
 456 #        if not record:
 457 #            raise RecordNotFound(slice_hrn)
 458 #
 459 #        # similar to CreateSliver, we must verify that the required records exist
 460 #        # at this aggregate before we can issue a ticket
 461 #        # parse rspec
 462 #        rspec = RSpec(rspec_string)
 463 #        requested_attributes = rspec.version.get_slice_attributes()
 464 #
 465 #        # ensure site record exists
 466 #        site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer)
 467 #        # ensure slice record exists
 468 #        slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer)
 469 #        # ensure person records exists
 470 #    # xxx users is undefined in this context
 471 #        persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer)
 472 #        # ensure slice attributes exists
 473 #        slices.verify_slice_attributes(slice, requested_attributes)
 474 #
 475 #        # get sliver info
 476 #        slivers = slices.get_slivers(slice_hrn)
 477 #
 478 #        if not slivers:
 479 #            raise SliverDoesNotExist(slice_hrn)
 480 #
 481 #        # get initscripts
 482 #        initscripts = []
 483 #        data = {
 484 #            'timestamp': int(time.time()),
 485 #            'initscripts': initscripts,
 486 #            'slivers': slivers
 487 #        }
 488 #
 489 #        # create the ticket
 490 #        object_gid = record.get_gid_object()
 491 #        new_ticket = SfaTicket(subject = object_gid.get_subject())
 492 #        new_ticket.set_gid_caller(api.auth.client_gid)
 493 #        new_ticket.set_gid_object(object_gid)
 494 #        new_ticket.set_issuer(key=api.key, subject=self.hrn)
 495 #        new_ticket.set_pubkey(object_gid.get_pubkey())
 496 #        new_ticket.set_attributes(data)
 497 #        new_ticket.set_rspec(rspec)
 498 #        #new_ticket.set_parent(api.auth.hierarchy.get_auth_ticket(auth_hrn))
 499 #        new_ticket.encode()
 500 #        new_ticket.sign()
 501 #
 502 #        return new_ticket.save_to_string(save_parents=True)