sfa/dummy/dummydriver.py

   1 import time
   2 import datetime
   3 #
   4 from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \
   5     RecordNotFound, SfaNotImplemented, SliverDoesNotExist, SearchFailed, \
   6     UnsupportedOperation, Forbidden
   7 from sfa.util.sfalogging import logger
   8 from sfa.util.defaultdict import defaultdict
   9 from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch
  10 from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf
  11 from sfa.util.cache import Cache
  12
  13 # one would think the driver should not need to mess with the SFA db, but..
  14 from sfa.storage.model import RegRecord, SliverAllocation
  15 from sfa.trust.credential import Credential
  16
  17 # used to be used in get_ticket
  18 #from sfa.trust.sfaticket import SfaTicket
  19 from sfa.rspecs.version_manager import VersionManager
  20 from sfa.rspecs.rspec import RSpec
  21
  22 # the driver interface, mostly provides default behaviours
  23 from sfa.managers.driver import Driver
  24
  25 from sfa.dummy.dummyshell import DummyShell
  26 from sfa.dummy.dummyaggregate import DummyAggregate
  27 from sfa.dummy.dummyslices import DummySlices
  28 from sfa.dummy.dummyxrn import DummyXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_dummy_slicename, xrn_to_hostname
  29
  30
  31 def list_to_dict(recs, key):
  32     """
  33     convert a list of dictionaries into a dictionary keyed on the
  34     specified dictionary key
  35     """
  36     return dict([(rec[key], rec) for rec in recs])
  37
  38 #
  39 # DummyShell is just an xmlrpc serverproxy where methods can be sent as-is;
  40 #
  41
  42
  43 class DummyDriver (Driver):
  44
  45     # the cache instance is a class member so it survives across incoming
  46     # requests
  47     cache = None
  48
  49     def __init__(self, api):
  50         Driver.__init__(self, api)
  51         config = api.config
  52         self.hrn = config.SFA_INTERFACE_HRN
  53         self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
  54         self.shell = DummyShell(config)
  55         self.testbedInfo = self.shell.GetTestbedInfo()
  56
  57     def check_sliver_credentials(self, creds, urns):
  58         # build list of cred object hrns
  59         slice_cred_names = []
  60         for cred in creds:
  61             slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn()
  62             slice_cred_names.append(
  63                 DummyXrn(xrn=slice_cred_hrn).dummy_slicename())
  64
  65         # look up slice name of slivers listed in urns arg
  66         slice_ids = []
  67         for urn in urns:
  68             sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts()
  69             try:
  70                 slice_ids.append(int(sliver_id_parts[0]))
  71             except ValueError:
  72                 pass
  73
  74         if not slice_ids:
  75             raise Forbidden("sliver urn not provided")
  76
  77         slices = self.shell.GetSlices({'slice_ids': slice_ids})
  78         sliver_names = [slice['slice_name'] for slice in slices]
  79
  80         # make sure we have a credential for every specified sliver ierd
  81         for sliver_name in sliver_names:
  82             if sliver_name not in slice_cred_names:
  83                 msg = "Valid credential not found for target: %s" % sliver_name
  84                 raise Forbidden(msg)
  85
  86     ########################################
  87     # registry oriented
  88     ########################################
  89
  90     def augment_records_with_testbed_info(self, sfa_records):
  91         return self.fill_record_info(sfa_records)
  92
  93     ##########
  94     def register(self, sfa_record, hrn, pub_key):
  95         type = sfa_record['type']
  96         dummy_record = self.sfa_fields_to_dummy_fields(type, hrn, sfa_record)
  97
  98         if type == 'authority':
  99             pointer = -1
 100
 101         elif type == 'slice':
 102             slices = self.shell.GetSlices(
 103                 {'slice_name': dummy_record['slice_name']})
 104             if not slices:
 105                 pointer = self.shell.AddSlice(dummy_record)
 106             else:
 107                 pointer = slices[0]['slice_id']
 108
 109         elif type == 'user':
 110             users = self.shell.GetUsers({'email': sfa_record['email']})
 111             if not users:
 112                 pointer = self.shell.AddUser(dummy_record)
 113             else:
 114                 pointer = users[0]['user_id']
 115
 116             # Add the user's key
 117             if pub_key:
 118                 self.shell.AddUserKey({'user_id': pointer, 'key': pub_key})
 119
 120         elif type == 'node':
 121             nodes = self.shell.GetNodes(dummy_record['hostname'])
 122             if not nodes:
 123                 pointer = self.shell.AddNode(dummy_record)
 124             else:
 125                 pointer = users[0]['node_id']
 126
 127         return pointer
 128
 129     ##########
 130     def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
 131         pointer = old_sfa_record['pointer']
 132         type = old_sfa_record['type']
 133         dummy_record = self.sfa_fields_to_dummy_fields(
 134             type, hrn, new_sfa_record)
 135
 136         # new_key implemented for users only
 137         if new_key and type not in ['user']:
 138             raise UnknownSfaType(type)
 139
 140         if type == "slice":
 141             self.shell.UpdateSlice(
 142                 {'slice_id': pointer, 'fields': dummy_record})
 143
 144         elif type == "user":
 145             self.shell.UpdateUser({'user_id': pointer, 'fields': dummy_record})
 146
 147             if new_key:
 148                 self.shell.AddUserKey({'user_id': pointer, 'key': new_key})
 149
 150         elif type == "node":
 151             self.shell.UpdateNode({'node_id': pointer, 'fields': dummy_record})
 152
 153         return True
 154
 155     ##########
 156     def remove(self, sfa_record):
 157         type = sfa_record['type']
 158         pointer = sfa_record['pointer']
 159         if type == 'user':
 160             self.shell.DeleteUser({'user_id': pointer})
 161         elif type == 'slice':
 162             self.shell.DeleteSlice({'slice_id': pointer})
 163         elif type == 'node':
 164             self.shell.DeleteNode({'node_id': pointer})
 165
 166         return True
 167
 168     ##
 169     # Convert SFA fields to Dummy testbed fields for use when registering or updating
 170     # registry record in the dummy testbed
 171     #
 172
 173     def sfa_fields_to_dummy_fields(self, type, hrn, sfa_record):
 174
 175         dummy_record = {}
 176
 177         if type == "slice":
 178             dummy_record["slice_name"] = hrn_to_dummy_slicename(hrn)
 179
 180         elif type == "node":
 181             if "hostname" not in sfa_record:
 182                 raise MissingSfaInfo("hostname")
 183             dummy_record["hostname"] = sfa_record["hostname"]
 184             if "type" in sfa_record:
 185                 dummy_record["type"] = sfa_record["type"]
 186             else:
 187                 dummy_record["type"] = "dummy_type"
 188
 189         elif type == "authority":
 190             dummy_record["name"] = hrn
 191
 192         elif type == "user":
 193             dummy_record["user_name"] = sfa_record["email"].split('@')[0]
 194             dummy_record["email"] = sfa_record["email"]
 195
 196         return dummy_record
 197
 198     ####################
 199     def fill_record_info(self, records):
 200         """
 201         Given a (list of) SFA record, fill in the DUMMY TESTBED specific
 202         and SFA specific fields in the record.
 203         """
 204         if not isinstance(records, list):
 205             records = [records]
 206
 207         self.fill_record_dummy_info(records)
 208         self.fill_record_hrns(records)
 209         self.fill_record_sfa_info(records)
 210         return records
 211
 212     def fill_record_dummy_info(self, records):
 213         """
 214         Fill in the DUMMY specific fields of a SFA record. This
 215         involves calling the appropriate DUMMY method to retrieve the
 216         database record for the object.
 217
 218         @param record: record to fill in field (in/out param)
 219         """
 220         # get ids by type
 221         node_ids, slice_ids, user_ids = [], [], []
 222         type_map = {'node': node_ids, 'slice': slice_ids, 'user': user_ids}
 223
 224         for record in records:
 225             for type in type_map:
 226                 if type == record['type']:
 227                     type_map[type].append(record['pointer'])
 228
 229         # get dummy records
 230         nodes, slices, users = {}, {}, {}
 231         if node_ids:
 232             node_list = self.shell.GetNodes({'node_ids': node_ids})
 233             nodes = list_to_dict(node_list, 'node_id')
 234         if slice_ids:
 235             slice_list = self.shell.GetSlices({'slice_ids': slice_ids})
 236             slices = list_to_dict(slice_list, 'slice_id')
 237         if user_ids:
 238             user_list = self.shell.GetUsers({'user_ids': user_ids})
 239             users = list_to_dict(user_list, 'user_id')
 240
 241         dummy_records = {'node': nodes, 'slice': slices, 'user': users}
 242
 243         # fill record info
 244         for record in records:
 245             # records with pointer==-1 do not have dummy info.
 246             if record['pointer'] == -1:
 247                 continue
 248
 249             for type in dummy_records:
 250                 if record['type'] == type:
 251                     if record['pointer'] in dummy_records[type]:
 252                         record.update(dummy_records[type][record['pointer']])
 253                         break
 254             # fill in key info
 255             if record['type'] == 'user':
 256                 record['key_ids'] = []
 257                 record['keys'] = []
 258                 for key in dummy_records['user'][record['pointer']]['keys']:
 259                     record['key_ids'].append(-1)
 260                     record['keys'].append(key)
 261
 262         return records
 263
 264     def fill_record_hrns(self, records):
 265         """
 266         convert dummy ids to hrns
 267         """
 268
 269         # get ids
 270         slice_ids, user_ids, node_ids = [], [], []
 271         for record in records:
 272             if 'user_ids' in record:
 273                 user_ids.extend(record['user_ids'])
 274             if 'slice_ids' in record:
 275                 slice_ids.extend(record['slice_ids'])
 276             if 'node_ids' in record:
 277                 node_ids.extend(record['node_ids'])
 278
 279         # get dummy records
 280         slices, users, nodes = {}, {}, {}
 281         if user_ids:
 282             user_list = self.shell.GetUsers({'user_ids': user_ids})
 283             users = list_to_dict(user_list, 'user_id')
 284         if slice_ids:
 285             slice_list = self.shell.GetSlices({'slice_ids': slice_ids})
 286             slices = list_to_dict(slice_list, 'slice_id')
 287         if node_ids:
 288             node_list = self.shell.GetNodes({'node_ids': node_ids})
 289             nodes = list_to_dict(node_list, 'node_id')
 290
 291         # convert ids to hrns
 292         for record in records:
 293             # get all relevant data
 294             type = record['type']
 295             pointer = record['pointer']
 296             testbed_name = self.testbed_name()
 297             auth_hrn = self.hrn
 298             if pointer == -1:
 299                 continue
 300
 301             if 'user_ids' in record:
 302                 emails = [users[user_id]['email'] for user_id in record['user_ids']
 303                           if user_id in users]
 304                 usernames = [email.split('@')[0] for email in emails]
 305                 user_hrns = [".".join([auth_hrn, testbed_name, username])
 306                              for username in usernames]
 307                 record['users'] = user_hrns
 308             if 'slice_ids' in record:
 309                 slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids']
 310                               if slice_id in slices]
 311                 slice_hrns = [slicename_to_hrn(
 312                     auth_hrn, slicename) for slicename in slicenames]
 313                 record['slices'] = slice_hrns
 314             if 'node_ids' in record:
 315                 hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids']
 316                              if node_id in nodes]
 317                 node_hrns = [hostname_to_hrn(
 318                     auth_hrn, login_base, hostname) for hostname in hostnames]
 319                 record['nodes'] = node_hrns
 320
 321         return records
 322
 323     def fill_record_sfa_info(self, records):
 324
 325         def startswith(prefix, values):
 326             return [value for value in values if value.startswith(prefix)]
 327
 328         # get user ids
 329         user_ids = []
 330         for record in records:
 331             user_ids.extend(record.get("user_ids", []))
 332
 333         # get sfa records for all records associated with these records.
 334         # we'll replace pl ids (person_ids) with hrns from the sfa records
 335         # we obtain
 336
 337         # get the registry records
 338         user_list, users = [], {}
 339         user_list = self.api.dbsession().query(
 340             RegRecord).filter(RegRecord.pointer.in_(user_ids))
 341         # create a hrns keyed on the sfa record's pointer.
 342         # Its possible for multiple records to have the same pointer so
 343         # the dict's value will be a list of hrns.
 344         users = defaultdict(list)
 345         for user in user_list:
 346             users[user.pointer].append(user)
 347
 348         # get the dummy records
 349         dummy_user_list, dummy_users = [], {}
 350         dummy_user_list = self.shell.GetUsers({'user_ids': user_ids})
 351         dummy_users = list_to_dict(dummy_user_list, 'user_id')
 352
 353         # fill sfa info
 354         for record in records:
 355             # skip records with no pl info (top level authorities)
 356             # if record['pointer'] == -1:
 357             #    continue
 358             sfa_info = {}
 359             type = record['type']
 360             logger.info(
 361                 "fill_record_sfa_info - incoming record typed %s" % type)
 362             if (type == "slice"):
 363                 # all slice users are researchers
 364                 record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
 365                 record['PI'] = []
 366                 record['researcher'] = []
 367                 for user_id in record.get('user_ids', []):
 368                     hrns = [user.hrn for user in users[user_id]]
 369                     record['researcher'].extend(hrns)
 370
 371             elif (type.startswith("authority")):
 372                 record['url'] = None
 373                 logger.info("fill_record_sfa_info - authority xherex")
 374
 375             elif (type == "node"):
 376                 sfa_info['dns'] = record.get("hostname", "")
 377                 # xxx TODO: URI, LatLong, IP, DNS
 378
 379             elif (type == "user"):
 380                 logger.info('setting user.email')
 381                 sfa_info['email'] = record.get("email", "")
 382                 sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
 383                 sfa_info['geni_certificate'] = record['gid']
 384                 # xxx TODO: PostalAddress, Phone
 385             record.update(sfa_info)
 386
 387     ####################
 388     def update_relation(self, subject_type, target_type, relation_name, subject_id, target_ids):
 389         # hard-wire the code for slice/user for now, could be smarter if needed
 390         if subject_type == 'slice' and target_type == 'user' and relation_name == 'researcher':
 391             subject = self.shell.GetSlices({'slice_id': subject_id})[0]
 392             if 'user_ids' not in list(subject.keys()):
 393                 subject['user_ids'] = []
 394             current_target_ids = subject['user_ids']
 395             add_target_ids = list(
 396                 set(target_ids).difference(current_target_ids))
 397             del_target_ids = list(
 398                 set(current_target_ids).difference(target_ids))
 399             logger.debug("subject_id = %s (type=%s)" %
 400                          (subject_id, type(subject_id)))
 401             for target_id in add_target_ids:
 402                 self.shell.AddUserToSlice(
 403                     {'user_id': target_id, 'slice_id': subject_id})
 404                 logger.debug("add_target_id = %s (type=%s)" %
 405                              (target_id, type(target_id)))
 406             for target_id in del_target_ids:
 407                 logger.debug("del_target_id = %s (type=%s)" %
 408                              (target_id, type(target_id)))
 409                 self.shell.DeleteUserFromSlice(
 410                     {'user_id': target_id, 'slice_id': subject_id})
 411         else:
 412             logger.info('unexpected relation %s to maintain, %s -> %s' %
 413                         (relation_name, subject_type, target_type))
 414
 415     ########################################
 416     # aggregate oriented
 417     ########################################
 418
 419     def testbed_name(self): return "dummy"
 420
 421     def aggregate_version(self):
 422         return {}
 423
 424     def list_resources(self, version=None, options=None):
 425         if options is None:
 426             options = {}
 427         aggregate = DummyAggregate(self)
 428         rspec = aggregate.list_resources(version=version, options=options)
 429         return rspec
 430
 431     def describe(self, urns, version, options=None):
 432         if options is None:
 433             options = {}
 434         aggregate = DummyAggregate(self)
 435         return aggregate.describe(urns, version=version, options=options)
 436
 437     def status(self, urns, options=None):
 438         if options is None:
 439             options = {}
 440         aggregate = DummyAggregate(self)
 441         desc = aggregate.describe(urns, version='GENI 3')
 442         status = {'geni_urn': desc['geni_urn'],
 443                   'geni_slivers': desc['geni_slivers']}
 444         return status
 445
 446     def allocate(self, urn, rspec_string, expiration, options=None):
 447         if options is None:
 448             options = {}
 449         xrn = Xrn(urn)
 450         aggregate = DummyAggregate(self)
 451         slices = DummySlices(self)
 452         slice_record = None
 453         users = options.get('geni_users', [])
 454         if users:
 455             slice_record = users[0].get('slice_record', {})
 456
 457         # parse rspec
 458         rspec = RSpec(rspec_string)
 459         requested_attributes = rspec.version.get_slice_attributes()
 460
 461         # ensure slice record exists
 462         slice = slices.verify_slice(
 463             xrn.hrn, slice_record, expiration=expiration, options=options)
 464         # ensure person records exists
 465         #persons = slices.verify_persons(xrn.hrn, slice, users, peer, sfa_peer, options=options)
 466
 467         # add/remove slice from nodes
 468         request_nodes = rspec.version.get_nodes_with_slivers()
 469         nodes = slices.verify_slice_nodes(urn, slice, request_nodes)
 470
 471         return aggregate.describe([xrn.get_urn()], version=rspec.version)
 472
 473     def provision(self, urns, options=None):
 474         if options is None:
 475             options = {}
 476         # update users
 477         slices = DummySlices(self)
 478         aggregate = DummyAggregate(self)
 479         slivers = aggregate.get_slivers(urns)
 480         slice = slivers[0]
 481         geni_users = options.get('geni_users', [])
 482         #users = slices.verify_users(None, slice, geni_users, options=options)
 483         # update sliver allocation states and set them to geni_provisioned
 484         sliver_ids = [sliver['sliver_id'] for sliver in slivers]
 485         dbsession = self.api.dbsession()
 486         SliverAllocation.set_allocations(
 487             sliver_ids, 'geni_provisioned', dbsession)
 488         version_manager = VersionManager()
 489         rspec_version = version_manager.get_version(
 490             options['geni_rspec_version'])
 491         return self.describe(urns, rspec_version, options=options)
 492
 493     def delete(self, urns, options=None):
 494         if options is None:
 495             options = {}
 496         # collect sliver ids so we can update sliver allocation states after
 497         # we remove the slivers.
 498         aggregate = DummyAggregate(self)
 499         slivers = aggregate.get_slivers(urns)
 500         if slivers:
 501             slice_id = slivers[0]['slice_id']
 502             node_ids = []
 503             sliver_ids = []
 504             for sliver in slivers:
 505                 node_ids.append(sliver['node_id'])
 506                 sliver_ids.append(sliver['sliver_id'])
 507
 508             # determine if this is a peer slice
 509             # xxx I wonder if this would not need to use PlSlices.get_peer instead
 510             # in which case plc.peers could be deprecated as this here
 511             # is the only/last call to this last method in plc.peers
 512             slice_hrn = DummyXrn(auth=self.hrn, slicename=slivers[
 513                                  0]['slice_name']).get_hrn()
 514             try:
 515                 self.shell.DeleteSliceFromNodes(
 516                     {'slice_id': slice_id, 'node_ids': node_ids})
 517                 # delete sliver allocation states
 518                 dbsession = self.api.dbsession()
 519                 SliverAllocation.delete_allocations(sliver_ids, dbsession)
 520             finally:
 521                 pass
 522
 523         # prepare return struct
 524         geni_slivers = []
 525         for sliver in slivers:
 526             geni_slivers.append(
 527                 {'geni_sliver_urn': sliver['sliver_id'],
 528                  'geni_allocation_status': 'geni_unallocated',
 529                  'geni_expires': datetime_to_string(utcparse(sliver['expires']))})
 530         return geni_slivers
 531
 532     def renew(self, urns, expiration_time, options=None):
 533         if options is None:
 534             options = {}
 535         aggregate = DummyAggregate(self)
 536         slivers = aggregate.get_slivers(urns)
 537         if not slivers:
 538             raise SearchFailed(urns)
 539         slice = slivers[0]
 540         requested_time = utcparse(expiration_time)
 541         record = {'expires': int(datetime_to_epoch(requested_time))}
 542         self.shell.UpdateSlice(
 543             {'slice_id': slice['slice_id'], 'fileds': record})
 544         description = self.describe(urns, 'GENI 3', options)
 545         return description['geni_slivers']
 546
 547     def perform_operational_action(self, urns, action, options=None):
 548         if options is None:
 549             options = {}
 550         # Dummy doesn't support operational actions. Lets pretend like it
 551         # supports start, but reject everything else.
 552         action = action.lower()
 553         if action not in ['geni_start']:
 554             raise UnsupportedOperation(action)
 555
 556         # fault if sliver is not full allocated (operational status is
 557         # geni_pending_allocation)
 558         description = self.describe(urns, 'GENI 3', options)
 559         for sliver in description['geni_slivers']:
 560             if sliver['geni_operational_status'] == 'geni_pending_allocation':
 561                 raise UnsupportedOperation(
 562                     action, "Sliver must be fully allocated (operational status is not geni_pending_allocation)")
 563         #
 564         # Perform Operational Action Here
 565         #
 566
 567         geni_slivers = self.describe(urns, 'GENI 3', options)['geni_slivers']
 568         return geni_slivers
 569
 570     def shutdown(self, xrn, options=None):
 571         if options is None:
 572             options = {}
 573         xrn = DummyXrn(xrn=xrn, type='slice')
 574         slicename = xrn.pl_slicename()
 575         slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
 576         if not slices:
 577             raise RecordNotFound(slice_hrn)
 578         slice_id = slices[0]['slice_id']
 579         slice_tags = self.shell.GetSliceTags(
 580             {'slice_id': slice_id, 'tagname': 'enabled'})
 581         if not slice_tags:
 582             self.shell.AddSliceTag(slice_id, 'enabled', '0')
 583         elif slice_tags[0]['value'] != "0":
 584             tag_id = slice_tags[0]['slice_tag_id']
 585             self.shell.UpdateSliceTag(tag_id, '0')
 586         return 1