sfa/managers/slice_manager.py

   1 import sys
   2 import time
   3 import traceback
   4 from copy import copy
   5 from lxml import etree
   6
   7 from sfa.trust.sfaticket import SfaTicket
   8 from sfa.trust.credential import Credential
   9
  10 from sfa.util.sfalogging import logger
  11 from sfa.util.xrn import Xrn, urn_to_hrn
  12 from sfa.util.version import version_core
  13 from sfa.util.callids import Callids
  14 from sfa.util.cache import Cache
  15
  16 from sfa.client.multiclient import MultiClient
  17
  18 from sfa.rspecs.rspec_converter import RSpecConverter
  19 from sfa.rspecs.version_manager import VersionManager
  20 from sfa.rspecs.rspec import RSpec
  21
  22 from sfa.client.client_helper import sfa_to_pg_users_arg
  23 from sfa.client.return_value import ReturnValue
  24
  25 class SliceManager:
  26
  27     # the cache instance is a class member so it survives across incoming requests
  28     cache = None
  29
  30     def __init__ (self, config):
  31         self.cache=None
  32         if config.SFA_SM_CACHING:
  33             if SliceManager.cache is None:
  34                 SliceManager.cache = Cache()
  35             self.cache = SliceManager.cache
  36
  37     def GetVersion(self, api, options):
  38         # peers explicitly in aggregates.xml
  39         peers =dict ([ (peername,interface.get_url()) for (peername,interface) in api.aggregates.iteritems()
  40                        if peername != api.hrn])
  41         version_manager = VersionManager()
  42         ad_rspec_versions = []
  43         request_rspec_versions = []
  44         cred_types = [{'geni_type': 'geni_sfa', 'geni_version': str(i)} for i in range(4)[-2:]]
  45         for rspec_version in version_manager.versions:
  46             if rspec_version.content_type in ['*', 'ad']:
  47                 ad_rspec_versions.append(rspec_version.to_dict())
  48             if rspec_version.content_type in ['*', 'request']:
  49                 request_rspec_versions.append(rspec_version.to_dict())
  50         xrn=Xrn(api.hrn, 'authority+sm')
  51         version_more = {
  52             'interface':'slicemgr',
  53             'sfa': 2,
  54             'geni_api': 3,
  55             'geni_api_versions': {'3': 'http://%s:%s' % (api.config.SFA_SM_HOST, api.config.SFA_SM_PORT)},
  56             'hrn' : xrn.get_hrn(),
  57             'urn' : xrn.get_urn(),
  58             'peers': peers,
  59             'geni_single_allocation': 0, # Accept operations that act on as subset of slivers in a given state.
  60             'geni_allocate': 'geni_many',# Multiple slivers can exist and be incrementally added, including those which connect or overlap in some way.
  61             'geni_credential_types': cred_types,
  62             }
  63         sm_version=version_core(version_more)
  64         # local aggregate if present needs to have localhost resolved
  65         if api.hrn in api.aggregates:
  66             local_am_url=api.aggregates[api.hrn].get_url()
  67             sm_version['peers'][api.hrn]=local_am_url.replace('localhost',sm_version['hostname'])
  68         return sm_version
  69
  70     def drop_slicemgr_stats(self, rspec):
  71         try:
  72             stats_elements = rspec.xml.xpath('//statistics')
  73             for node in stats_elements:
  74                 node.getparent().remove(node)
  75         except Exception as e:
  76             logger.warn("drop_slicemgr_stats failed: %s " % (str(e)))
  77
  78     def add_slicemgr_stat(self, rspec, callname, aggname, elapsed, status, exc_info=None):
  79         try:
  80             stats_tags = rspec.xml.xpath('//statistics[@call="%s"]' % callname)
  81             if stats_tags:
  82                 stats_tag = stats_tags[0]
  83             else:
  84                 stats_tag = rspec.xml.root.add_element("statistics", call=callname)
  85
  86             stat_tag = stats_tag.add_element("aggregate", name=str(aggname),
  87                                              elapsed=str(elapsed), status=str(status))
  88
  89             if exc_info:
  90                 exc_tag = stat_tag.add_element("exc_info", name=str(exc_info[1]))
  91
  92                 # formats the traceback as one big text blob
  93                 #exc_tag.text = "\n".join(traceback.format_exception(exc_info[0], exc_info[1], exc_info[2]))
  94
  95                 # formats the traceback as a set of xml elements
  96                 tb = traceback.extract_tb(exc_info[2])
  97                 for item in tb:
  98                     exc_frame = exc_tag.add_element("tb_frame", filename=str(item[0]),
  99                                                     line=str(item[1]), func=str(item[2]), code=str(item[3]))
 100
 101         except Exception as e:
 102             logger.warn("add_slicemgr_stat failed on  %s: %s" %(aggname, str(e)))
 103
 104     def ListResources(self, api, creds, options):
 105         call_id = options.get('call_id')
 106         if Callids().already_handled(call_id): return ""
 107
 108         version_manager = VersionManager()
 109
 110         def _ListResources(aggregate, server, credential, options):
 111             forward_options = copy(options)
 112             tStart = time.time()
 113             try:
 114                 version = api.get_cached_server_version(server)
 115                 # force ProtoGENI aggregates to give us a v2 RSpec
 116                 forward_options['geni_rspec_version'] = options.get('geni_rspec_version')
 117                 result = server.ListResources(credential, forward_options)
 118                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
 119             except Exception as e:
 120                 api.logger.log_exc("ListResources failed at %s" %(server.url))
 121                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
 122
 123         # get slice's hrn from options
 124         xrn = options.get('geni_slice_urn', '')
 125         (hrn, type) = urn_to_hrn(xrn)
 126         if 'geni_compressed' in options:
 127             del(options['geni_compressed'])
 128
 129         # get the rspec's return format from options
 130         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
 131         version_string = "rspec_%s" % (rspec_version)
 132
 133         # look in cache first
 134         cached_requested = options.get('cached', True)
 135         if not xrn and self.cache and cached_requested:
 136             rspec =  self.cache.get(version_string)
 137             if rspec:
 138                 api.logger.debug("SliceManager.ListResources returns cached advertisement")
 139                 return rspec
 140
 141         # get the callers hrn
 142         valid_cred = api.auth.checkCredentials(creds, 'listnodes', hrn)[0]
 143         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 144
 145         # attempt to use delegated credential first
 146         cred = api.getDelegatedCredential(creds)
 147         if not cred:
 148             cred = api.getCredential()
 149         multiclient = MultiClient()
 150         for aggregate in api.aggregates:
 151             # prevent infinite loop. Dont send request back to caller
 152             # unless the caller is the aggregate's SM
 153             if caller_hrn == aggregate and aggregate != api.hrn:
 154                 continue
 155
 156             # get the rspec from the aggregate
 157             interface = api.aggregates[aggregate]
 158             server = api.server_proxy(interface, cred)
 159             multiclient.run(_ListResources, aggregate, server, [cred], options)
 160
 161
 162         results = multiclient.get_results()
 163         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
 164         if xrn:
 165             result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'manifest')
 166         else:
 167             result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'ad')
 168         rspec = RSpec(version=result_version)
 169         for result in results:
 170             self.add_slicemgr_stat(rspec, "ListResources", result["aggregate"], result["elapsed"],
 171                                    result["status"], result.get("exc_info",None))
 172             if result["status"]=="success":
 173                 res = result['result']['value']
 174                 try:
 175                     rspec.version.merge(ReturnValue.get_value(res))
 176                 except:
 177                     api.logger.log_exc("SM.ListResources: Failed to merge aggregate rspec")
 178
 179         # cache the result
 180         if self.cache and not xrn:
 181             api.logger.debug("SliceManager.ListResources caches advertisement")
 182             self.cache.add(version_string, rspec.toxml())
 183
 184         return rspec.toxml()
 185
 186
 187     def Allocate(self, api, xrn, creds, rspec_str, expiration, options):
 188         call_id = options.get('call_id')
 189         if Callids().already_handled(call_id): return ""
 190
 191         version_manager = VersionManager()
 192         def _Allocate(aggregate, server, xrn, credential, rspec, options):
 193             tStart = time.time()
 194             try:
 195                 # Need to call GetVersion at an aggregate to determine the supported
 196                 # rspec type/format beofre calling CreateSliver at an Aggregate.
 197                 #server_version = api.get_cached_server_version(server)
 198                 #if 'sfa' not in server_version and 'geni_api' in server_version:
 199                     # sfa aggregtes support both sfa and pg rspecs, no need to convert
 200                     # if aggregate supports sfa rspecs. otherwise convert to pg rspec
 201                     #rspec = RSpec(RSpecConverter.to_pg_rspec(rspec, 'request'))
 202                     #filter = {'component_manager_id': server_version['urn']}
 203                     #rspec.filter(filter)
 204                     #rspec = rspec.toxml()
 205                 result = server.Allocate(xrn, credential, rspec, options)
 206                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
 207             except:
 208                 logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
 209                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
 210
 211         # Validate the RSpec against PlanetLab's schema --disabled for now
 212         # The schema used here needs to aggregate the PL and VINI schemas
 213         # schema = "/var/www/html/schemas/pl.rng"
 214         rspec = RSpec(rspec_str)
 215     #    schema = None
 216     #    if schema:
 217     #        rspec.validate(schema)
 218
 219         # if there is a <statistics> section, the aggregates don't care about it,
 220         # so delete it.
 221         self.drop_slicemgr_stats(rspec)
 222
 223         # attempt to use delegated credential first
 224         cred = api.getDelegatedCredential(creds)
 225         if not cred:
 226             cred = api.getCredential()
 227
 228         # get the callers hrn
 229         hrn, type = urn_to_hrn(xrn)
 230         valid_cred = api.auth.checkCredentials(creds, 'createsliver', hrn)[0]
 231         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 232         multiclient = MultiClient()
 233         for aggregate in api.aggregates:
 234             # prevent infinite loop. Dont send request back to caller
 235             # unless the caller is the aggregate's SM
 236             if caller_hrn == aggregate and aggregate != api.hrn:
 237                 continue
 238             interface = api.aggregates[aggregate]
 239             server = api.server_proxy(interface, cred)
 240             # Just send entire RSpec to each aggregate
 241             multiclient.run(_Allocate, aggregate, server, xrn, [cred], rspec.toxml(), options)
 242
 243         results = multiclient.get_results()
 244         manifest_version = version_manager._get_version(rspec.version.type, rspec.version.version, 'manifest')
 245         result_rspec = RSpec(version=manifest_version)
 246         geni_urn = None
 247         geni_slivers = []
 248
 249         for result in results:
 250             self.add_slicemgr_stat(result_rspec, "Allocate", result["aggregate"], result["elapsed"],
 251                                    result["status"], result.get("exc_info",None))
 252             if result["status"]=="success":
 253                 try:
 254                     res = result['result']['value']
 255                     geni_urn = res['geni_urn']
 256                     result_rspec.version.merge(ReturnValue.get_value(res['geni_rspec']))
 257                     geni_slivers.extend(res['geni_slivers'])
 258                 except:
 259                     api.logger.log_exc("SM.Allocate: Failed to merge aggregate rspec")
 260         return {
 261             'geni_urn': geni_urn,
 262             'geni_rspec': result_rspec.toxml(),
 263             'geni_slivers': geni_slivers
 264         }
 265
 266
 267     def Provision(self, api, xrn, creds, options):
 268         call_id = options.get('call_id')
 269         if Callids().already_handled(call_id): return ""
 270
 271         version_manager = VersionManager()
 272         def _Provision(aggregate, server, xrn, credential, options):
 273             tStart = time.time()
 274             try:
 275                 # Need to call GetVersion at an aggregate to determine the supported
 276                 # rspec type/format beofre calling CreateSliver at an Aggregate.
 277                 server_version = api.get_cached_server_version(server)
 278                 result = server.Provision(xrn, credential, options)
 279                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
 280             except:
 281                 logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
 282                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
 283
 284         # attempt to use delegated credential first
 285         cred = api.getDelegatedCredential(creds)
 286         if not cred:
 287             cred = api.getCredential()
 288
 289         # get the callers hrn
 290         valid_cred = api.auth.checkCredentials(creds, 'createsliver', xrn)[0]
 291         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 292         multiclient = MultiClient()
 293         for aggregate in api.aggregates:
 294             # prevent infinite loop. Dont send request back to caller
 295             # unless the caller is the aggregate's SM
 296             if caller_hrn == aggregate and aggregate != api.hrn:
 297                 continue
 298             interface = api.aggregates[aggregate]
 299             server = api.server_proxy(interface, cred)
 300             # Just send entire RSpec to each aggregate
 301             multiclient.run(_Provision, aggregate, server, xrn, [cred], options)
 302
 303         results = multiclient.get_results()
 304         manifest_version = version_manager._get_version('GENI', '3', 'manifest')
 305         result_rspec = RSpec(version=manifest_version)
 306         geni_slivers = []
 307         geni_urn  = None
 308         for result in results:
 309             self.add_slicemgr_stat(result_rspec, "Provision", result["aggregate"], result["elapsed"],
 310                                    result["status"], result.get("exc_info",None))
 311             if result["status"]=="success":
 312                 try:
 313                     res = result['result']['value']
 314                     geni_urn = res['geni_urn']
 315                     result_rspec.version.merge(ReturnValue.get_value(res['geni_rspec']))
 316                     geni_slivers.extend(res['geni_slivers'])
 317                 except:
 318                     api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
 319         return {
 320             'geni_urn': geni_urn,
 321             'geni_rspec': result_rspec.toxml(),
 322             'geni_slivers': geni_slivers
 323         }
 324
 325
 326
 327     def Renew(self, api, xrn, creds, expiration_time, options):
 328         call_id = options.get('call_id')
 329         if Callids().already_handled(call_id): return True
 330
 331         def _Renew(aggregate, server, xrn, creds, expiration_time, options):
 332             try:
 333                 result=server.Renew(xrn, creds, expiration_time, options)
 334                 if type(result)!=dict:
 335                     result = {'code': {'geni_code': 0}, 'value': result}
 336                 result['aggregate'] = aggregate
 337                 return result
 338             except:
 339                 logger.log_exc('Something wrong in _Renew with URL %s'%server.url)
 340                 return {'aggregate': aggregate, 'exc_info': traceback.format_exc(),
 341                         'code': {'geni_code': -1},
 342                         'value': False, 'output': ""}
 343
 344         # get the callers hrn
 345         valid_cred = api.auth.checkCredentials(creds, 'renewsliver', xrn)[0]
 346         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 347
 348         # attempt to use delegated credential first
 349         cred = api.getDelegatedCredential(creds)
 350         if not cred:
 351             cred = api.getCredential(minimumExpiration=31*86400)
 352         multiclient = MultiClient()
 353         for aggregate in api.aggregates:
 354             # prevent infinite loop. Dont send request back to caller
 355             # unless the caller is the aggregate's SM
 356             if caller_hrn == aggregate and aggregate != api.hrn:
 357                 continue
 358             interface = api.aggregates[aggregate]
 359             server = api.server_proxy(interface, cred)
 360             multiclient.run(_Renew, aggregate, server, xrn, [cred], expiration_time, options)
 361
 362         results = multiclient.get_results()
 363
 364         geni_code = 0
 365         geni_output = ",".join([x.get('output',"") for x in results])
 366         geni_value = reduce (lambda x,y: x and y, [result.get('value',False) for result in results], True)
 367         for agg_result in results:
 368             agg_geni_code = agg_result['code'].get('geni_code',0)
 369             if agg_geni_code:
 370                 geni_code = agg_geni_code
 371
 372         results = {'aggregates': results, 'code': {'geni_code': geni_code}, 'value': geni_value, 'output': geni_output}
 373
 374         return results
 375
 376     def Delete(self, api, xrn, creds, options):
 377         call_id = options.get('call_id')
 378         if Callids().already_handled(call_id): return ""
 379
 380         def _Delete(server, xrn, creds, options):
 381             return server.Delete(xrn, creds, options)
 382
 383         (hrn, type) = urn_to_hrn(xrn[0])
 384         # get the callers hrn
 385         valid_cred = api.auth.checkCredentials(creds, 'deletesliver', hrn)[0]
 386         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 387
 388         # attempt to use delegated credential first
 389         cred = api.getDelegatedCredential(creds)
 390         if not cred:
 391             cred = api.getCredential()
 392         multiclient = MultiClient()
 393         for aggregate in api.aggregates:
 394             # prevent infinite loop. Dont send request back to caller
 395             # unless the caller is the aggregate's SM
 396             if caller_hrn == aggregate and aggregate != api.hrn:
 397                 continue
 398             interface = api.aggregates[aggregate]
 399             server = api.server_proxy(interface, cred)
 400             multiclient.run(_Delete, server, xrn, [cred], options)
 401
 402         results = []
 403         for result in multiclient.get_results():
 404             results += ReturnValue.get_value(result)
 405         return results
 406
 407
 408     # first draft at a merging SliverStatus
 409     def Status(self, api, slice_xrn, creds, options):
 410         def _Status(server, xrn, creds, options):
 411             return server.Status(xrn, creds, options)
 412
 413         call_id = options.get('call_id')
 414         if Callids().already_handled(call_id): return {}
 415         # attempt to use delegated credential first
 416         cred = api.getDelegatedCredential(creds)
 417         if not cred:
 418             cred = api.getCredential()
 419         multiclient = MultiClient()
 420         for aggregate in api.aggregates:
 421             interface = api.aggregates[aggregate]
 422             server = api.server_proxy(interface, cred)
 423             multiclient.run (_Status, server, slice_xrn, [cred], options)
 424         results = [ReturnValue.get_value(result) for result in multiclient.get_results()]
 425
 426         # get rid of any void result - e.g. when call_id was hit, where by convention we return {}
 427         results = [ result for result in results if result and result['geni_slivers']]
 428
 429         # do not try to combine if there's no result
 430         if not results : return {}
 431
 432         # otherwise let's merge stuff
 433         geni_slivers = []
 434         geni_urn  = None
 435         for result in results:
 436             try:
 437                 geni_urn = result['geni_urn']
 438                 geni_slivers.extend(result['geni_slivers'])
 439             except:
 440                 api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
 441         return {
 442             'geni_urn': geni_urn,
 443             'geni_slivers': geni_slivers
 444         }
 445
 446
 447     def Describe(self, api, creds, xrns, options):
 448         def _Describe(server, xrn, creds, options):
 449             return server.Describe(xrn, creds, options)
 450
 451         call_id = options.get('call_id')
 452         if Callids().already_handled(call_id): return {}
 453         # attempt to use delegated credential first
 454         cred = api.getDelegatedCredential(creds)
 455         if not cred:
 456             cred = api.getCredential()
 457         multiclient = MultiClient()
 458         for aggregate in api.aggregates:
 459             interface = api.aggregates[aggregate]
 460             server = api.server_proxy(interface, cred)
 461             multiclient.run (_Describe, server, xrns, [cred], options)
 462         results = [ReturnValue.get_value(result) for result in multiclient.get_results()]
 463
 464         # get rid of any void result - e.g. when call_id was hit, where by convention we return {}
 465         results = [ result for result in results if result and result.get('geni_urn')]
 466
 467         # do not try to combine if there's no result
 468         if not results : return {}
 469
 470         # otherwise let's merge stuff
 471         version_manager = VersionManager()
 472         manifest_version = version_manager._get_version('GENI', '3', 'manifest')
 473         result_rspec = RSpec(version=manifest_version)
 474         geni_slivers = []
 475         geni_urn  = None
 476         for result in results:
 477             try:
 478                 geni_urn = result['geni_urn']
 479                 result_rspec.version.merge(ReturnValue.get_value(result['geni_rspec']))
 480                 geni_slivers.extend(result['geni_slivers'])
 481             except:
 482                 api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
 483         return {
 484             'geni_urn': geni_urn,
 485             'geni_rspec': result_rspec.toxml(),
 486             'geni_slivers': geni_slivers
 487         }
 488
 489     def PerformOperationalAction(self, api, xrn, creds, action, options):
 490         # get the callers hrn
 491         valid_cred = api.auth.checkCredentials(creds, 'createsliver', xrn)[0]
 492         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 493
 494         # attempt to use delegated credential first
 495         cred = api.getDelegatedCredential(creds)
 496         if not cred:
 497             cred = api.getCredential()
 498         multiclient = MultiClient()
 499         for aggregate in api.aggregates:
 500             # prevent infinite loop. Dont send request back to caller
 501             # unless the caller is the aggregate's SM
 502             if caller_hrn == aggregate and aggregate != api.hrn:
 503                 continue
 504             interface = api.aggregates[aggregate]
 505             server = api.server_proxy(interface, cred)
 506             multiclient.run(server.PerformOperationalAction, xrn, [cred], action, options)
 507         multiclient.get_results()
 508         return 1
 509
 510     def Shutdown(self, api, xrn, creds, options=None):
 511         if options is None: options={}
 512         xrn = Xrn(xrn)
 513         # get the callers hrn
 514         valid_cred = api.auth.checkCredentials(creds, 'stopslice', xrn.hrn)[0]
 515         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 516
 517         # attempt to use delegated credential first
 518         cred = api.getDelegatedCredential(creds)
 519         if not cred:
 520             cred = api.getCredential()
 521         multiclient = MultiClient()
 522         for aggregate in api.aggregates:
 523             # prevent infinite loop. Dont send request back to caller
 524             # unless the caller is the aggregate's SM
 525             if caller_hrn == aggregate and aggregate != api.hrn:
 526                 continue
 527             interface = api.aggregates[aggregate]
 528             server = api.server_proxy(interface, cred)
 529             multiclient.run(server.Shutdown, xrn.urn, cred)
 530         multiclient.get_results()
 531         return 1
 532