sfa/managers/slice_manager.py

   1 import sys
   2 import time
   3 import traceback
   4 from StringIO import StringIO
   5 from copy import copy
   6 from lxml import etree
   7
   8 from sfa.trust.sfaticket import SfaTicket
   9 from sfa.trust.credential import Credential
  10
  11 from sfa.util.sfalogging import logger
  12 from sfa.util.xrn import Xrn, urn_to_hrn
  13 from sfa.util.version import version_core
  14 from sfa.util.callids import Callids
  15 from sfa.util.cache import Cache
  16
  17 from sfa.client.multiclient import MultiClient
  18
  19 from sfa.rspecs.rspec_converter import RSpecConverter
  20 from sfa.rspecs.version_manager import VersionManager
  21 from sfa.rspecs.rspec import RSpec
  22
  23 from sfa.client.client_helper import sfa_to_pg_users_arg
  24 from sfa.client.return_value import ReturnValue
  25
  26 class SliceManager:
  27
  28     # the cache instance is a class member so it survives across incoming requests
  29     cache = None
  30
  31     def __init__ (self, config):
  32         self.cache=None
  33         if config.SFA_SM_CACHING:
  34             if SliceManager.cache is None:
  35                 SliceManager.cache = Cache()
  36             self.cache = SliceManager.cache
  37
  38     def GetVersion(self, api, options):
  39         # peers explicitly in aggregates.xml
  40         peers =dict ([ (peername,interface.get_url()) for (peername,interface) in api.aggregates.iteritems()
  41                        if peername != api.hrn])
  42         version_manager = VersionManager()
  43         ad_rspec_versions = []
  44         request_rspec_versions = []
  45         cred_types = [{'geni_type': 'geni_sfa', 'geni_version': str(i)} for i in range(4)[-2:]]
  46         for rspec_version in version_manager.versions:
  47             if rspec_version.content_type in ['*', 'ad']:
  48                 ad_rspec_versions.append(rspec_version.to_dict())
  49             if rspec_version.content_type in ['*', 'request']:
  50                 request_rspec_versions.append(rspec_version.to_dict())
  51         xrn=Xrn(api.hrn, 'authority+sm')
  52         version_more = {
  53             'interface':'slicemgr',
  54             'sfa': 2,
  55             'geni_api': 3,
  56             'geni_api_versions': {'3': 'http://%s:%s' % (api.config.SFA_SM_HOST, api.config.SFA_SM_PORT)},
  57             'hrn' : xrn.get_hrn(),
  58             'urn' : xrn.get_urn(),
  59             'peers': peers,
  60             'geni_single_allocation': 0, # Accept operations that act on as subset of slivers in a given state.
  61             'geni_allocate': 'geni_many',# Multiple slivers can exist and be incrementally added, including those which connect or overlap in some way.
  62             'geni_credential_types': cred_types,
  63             }
  64         sm_version=version_core(version_more)
  65         # local aggregate if present needs to have localhost resolved
  66         if api.hrn in api.aggregates:
  67             local_am_url=api.aggregates[api.hrn].get_url()
  68             sm_version['peers'][api.hrn]=local_am_url.replace('localhost',sm_version['hostname'])
  69         return sm_version
  70
  71     def drop_slicemgr_stats(self, rspec):
  72         try:
  73             stats_elements = rspec.xml.xpath('//statistics')
  74             for node in stats_elements:
  75                 node.getparent().remove(node)
  76         except Exception as e:
  77             logger.warn("drop_slicemgr_stats failed: %s " % (str(e)))
  78
  79     def add_slicemgr_stat(self, rspec, callname, aggname, elapsed, status, exc_info=None):
  80         try:
  81             stats_tags = rspec.xml.xpath('//statistics[@call="%s"]' % callname)
  82             if stats_tags:
  83                 stats_tag = stats_tags[0]
  84             else:
  85                 stats_tag = rspec.xml.root.add_element("statistics", call=callname)
  86
  87             stat_tag = stats_tag.add_element("aggregate", name=str(aggname),
  88                                              elapsed=str(elapsed), status=str(status))
  89
  90             if exc_info:
  91                 exc_tag = stat_tag.add_element("exc_info", name=str(exc_info[1]))
  92
  93                 # formats the traceback as one big text blob
  94                 #exc_tag.text = "\n".join(traceback.format_exception(exc_info[0], exc_info[1], exc_info[2]))
  95
  96                 # formats the traceback as a set of xml elements
  97                 tb = traceback.extract_tb(exc_info[2])
  98                 for item in tb:
  99                     exc_frame = exc_tag.add_element("tb_frame", filename=str(item[0]),
 100                                                     line=str(item[1]), func=str(item[2]), code=str(item[3]))
 101
 102         except Exception as e:
 103             logger.warn("add_slicemgr_stat failed on  %s: %s" %(aggname, str(e)))
 104
 105     def ListResources(self, api, creds, options):
 106         call_id = options.get('call_id')
 107         if Callids().already_handled(call_id): return ""
 108
 109         version_manager = VersionManager()
 110
 111         def _ListResources(aggregate, server, credential, options):
 112             forward_options = copy(options)
 113             tStart = time.time()
 114             try:
 115                 version = api.get_cached_server_version(server)
 116                 # force ProtoGENI aggregates to give us a v2 RSpec
 117                 forward_options['geni_rspec_version'] = options.get('geni_rspec_version')
 118                 result = server.ListResources(credential, forward_options)
 119                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
 120             except Exception as e:
 121                 api.logger.log_exc("ListResources failed at %s" %(server.url))
 122                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
 123
 124         # get slice's hrn from options
 125         xrn = options.get('geni_slice_urn', '')
 126         (hrn, type) = urn_to_hrn(xrn)
 127         if 'geni_compressed' in options:
 128             del(options['geni_compressed'])
 129
 130         # get the rspec's return format from options
 131         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
 132         version_string = "rspec_%s" % (rspec_version)
 133
 134         # look in cache first
 135         cached_requested = options.get('cached', True)
 136         if not xrn and self.cache and cached_requested:
 137             rspec =  self.cache.get(version_string)
 138             if rspec:
 139                 api.logger.debug("SliceManager.ListResources returns cached advertisement")
 140                 return rspec
 141
 142         # get the callers hrn
 143         valid_cred = api.auth.checkCredentials(creds, 'listnodes', hrn)[0]
 144         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 145
 146         # attempt to use delegated credential first
 147         cred = api.getDelegatedCredential(creds)
 148         if not cred:
 149             cred = api.getCredential()
 150         multiclient = MultiClient()
 151         for aggregate in api.aggregates:
 152             # prevent infinite loop. Dont send request back to caller
 153             # unless the caller is the aggregate's SM
 154             if caller_hrn == aggregate and aggregate != api.hrn:
 155                 continue
 156
 157             # get the rspec from the aggregate
 158             interface = api.aggregates[aggregate]
 159             server = api.server_proxy(interface, cred)
 160             multiclient.run(_ListResources, aggregate, server, [cred], options)
 161
 162
 163         results = multiclient.get_results()
 164         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
 165         if xrn:
 166             result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'manifest')
 167         else:
 168             result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'ad')
 169         rspec = RSpec(version=result_version)
 170         for result in results:
 171             self.add_slicemgr_stat(rspec, "ListResources", result["aggregate"], result["elapsed"],
 172                                    result["status"], result.get("exc_info",None))
 173             if result["status"]=="success":
 174                 res = result['result']['value']
 175                 try:
 176                     rspec.version.merge(ReturnValue.get_value(res))
 177                 except:
 178                     api.logger.log_exc("SM.ListResources: Failed to merge aggregate rspec")
 179
 180         # cache the result
 181         if self.cache and not xrn:
 182             api.logger.debug("SliceManager.ListResources caches advertisement")
 183             self.cache.add(version_string, rspec.toxml())
 184
 185         return rspec.toxml()
 186
 187
 188     def Allocate(self, api, xrn, creds, rspec_str, expiration, options):
 189         call_id = options.get('call_id')
 190         if Callids().already_handled(call_id): return ""
 191
 192         version_manager = VersionManager()
 193         def _Allocate(aggregate, server, xrn, credential, rspec, options):
 194             tStart = time.time()
 195             try:
 196                 # Need to call GetVersion at an aggregate to determine the supported
 197                 # rspec type/format beofre calling CreateSliver at an Aggregate.
 198                 #server_version = api.get_cached_server_version(server)
 199                 #if 'sfa' not in server_version and 'geni_api' in server_version:
 200                     # sfa aggregtes support both sfa and pg rspecs, no need to convert
 201                     # if aggregate supports sfa rspecs. otherwise convert to pg rspec
 202                     #rspec = RSpec(RSpecConverter.to_pg_rspec(rspec, 'request'))
 203                     #filter = {'component_manager_id': server_version['urn']}
 204                     #rspec.filter(filter)
 205                     #rspec = rspec.toxml()
 206                 result = server.Allocate(xrn, credential, rspec, options)
 207                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
 208             except:
 209                 logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
 210                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
 211
 212         # Validate the RSpec against PlanetLab's schema --disabled for now
 213         # The schema used here needs to aggregate the PL and VINI schemas
 214         # schema = "/var/www/html/schemas/pl.rng"
 215         rspec = RSpec(rspec_str)
 216     #    schema = None
 217     #    if schema:
 218     #        rspec.validate(schema)
 219
 220         # if there is a <statistics> section, the aggregates don't care about it,
 221         # so delete it.
 222         self.drop_slicemgr_stats(rspec)
 223
 224         # attempt to use delegated credential first
 225         cred = api.getDelegatedCredential(creds)
 226         if not cred:
 227             cred = api.getCredential()
 228
 229         # get the callers hrn
 230         hrn, type = urn_to_hrn(xrn)
 231         valid_cred = api.auth.checkCredentials(creds, 'createsliver', hrn)[0]
 232         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 233         multiclient = MultiClient()
 234         for aggregate in api.aggregates:
 235             # prevent infinite loop. Dont send request back to caller
 236             # unless the caller is the aggregate's SM
 237             if caller_hrn == aggregate and aggregate != api.hrn:
 238                 continue
 239             interface = api.aggregates[aggregate]
 240             server = api.server_proxy(interface, cred)
 241             # Just send entire RSpec to each aggregate
 242             multiclient.run(_Allocate, aggregate, server, xrn, [cred], rspec.toxml(), options)
 243
 244         results = multiclient.get_results()
 245         manifest_version = version_manager._get_version(rspec.version.type, rspec.version.version, 'manifest')
 246         result_rspec = RSpec(version=manifest_version)
 247         geni_urn = None
 248         geni_slivers = []
 249
 250         for result in results:
 251             self.add_slicemgr_stat(result_rspec, "Allocate", result["aggregate"], result["elapsed"],
 252                                    result["status"], result.get("exc_info",None))
 253             if result["status"]=="success":
 254                 try:
 255                     res = result['result']['value']
 256                     geni_urn = res['geni_urn']
 257                     result_rspec.version.merge(ReturnValue.get_value(res['geni_rspec']))
 258                     geni_slivers.extend(res['geni_slivers'])
 259                 except:
 260                     api.logger.log_exc("SM.Allocate: Failed to merge aggregate rspec")
 261         return {
 262             'geni_urn': geni_urn,
 263             'geni_rspec': result_rspec.toxml(),
 264             'geni_slivers': geni_slivers
 265         }
 266
 267
 268     def Provision(self, api, xrn, creds, options):
 269         call_id = options.get('call_id')
 270         if Callids().already_handled(call_id): return ""
 271
 272         version_manager = VersionManager()
 273         def _Provision(aggregate, server, xrn, credential, options):
 274             tStart = time.time()
 275             try:
 276                 # Need to call GetVersion at an aggregate to determine the supported
 277                 # rspec type/format beofre calling CreateSliver at an Aggregate.
 278                 server_version = api.get_cached_server_version(server)
 279                 result = server.Provision(xrn, credential, options)
 280                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
 281             except:
 282                 logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
 283                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
 284
 285         # attempt to use delegated credential first
 286         cred = api.getDelegatedCredential(creds)
 287         if not cred:
 288             cred = api.getCredential()
 289
 290         # get the callers hrn
 291         valid_cred = api.auth.checkCredentials(creds, 'createsliver', xrn)[0]
 292         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 293         multiclient = MultiClient()
 294         for aggregate in api.aggregates:
 295             # prevent infinite loop. Dont send request back to caller
 296             # unless the caller is the aggregate's SM
 297             if caller_hrn == aggregate and aggregate != api.hrn:
 298                 continue
 299             interface = api.aggregates[aggregate]
 300             server = api.server_proxy(interface, cred)
 301             # Just send entire RSpec to each aggregate
 302             multiclient.run(_Provision, aggregate, server, xrn, [cred], options)
 303
 304         results = multiclient.get_results()
 305         manifest_version = version_manager._get_version('GENI', '3', 'manifest')
 306         result_rspec = RSpec(version=manifest_version)
 307         geni_slivers = []
 308         geni_urn  = None
 309         for result in results:
 310             self.add_slicemgr_stat(result_rspec, "Provision", result["aggregate"], result["elapsed"],
 311                                    result["status"], result.get("exc_info",None))
 312             if result["status"]=="success":
 313                 try:
 314                     res = result['result']['value']
 315                     geni_urn = res['geni_urn']
 316                     result_rspec.version.merge(ReturnValue.get_value(res['geni_rspec']))
 317                     geni_slivers.extend(res['geni_slivers'])
 318                 except:
 319                     api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
 320         return {
 321             'geni_urn': geni_urn,
 322             'geni_rspec': result_rspec.toxml(),
 323             'geni_slivers': geni_slivers
 324         }
 325
 326
 327
 328     def Renew(self, api, xrn, creds, expiration_time, options):
 329         call_id = options.get('call_id')
 330         if Callids().already_handled(call_id): return True
 331
 332         def _Renew(aggregate, server, xrn, creds, expiration_time, options):
 333             try:
 334                 result=server.Renew(xrn, creds, expiration_time, options)
 335                 if type(result)!=dict:
 336                     result = {'code': {'geni_code': 0}, 'value': result}
 337                 result['aggregate'] = aggregate
 338                 return result
 339             except:
 340                 logger.log_exc('Something wrong in _Renew with URL %s'%server.url)
 341                 return {'aggregate': aggregate, 'exc_info': traceback.format_exc(),
 342                         'code': {'geni_code': -1},
 343                         'value': False, 'output': ""}
 344
 345         # get the callers hrn
 346         valid_cred = api.auth.checkCredentials(creds, 'renewsliver', xrn)[0]
 347         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 348
 349         # attempt to use delegated credential first
 350         cred = api.getDelegatedCredential(creds)
 351         if not cred:
 352             cred = api.getCredential(minimumExpiration=31*86400)
 353         multiclient = MultiClient()
 354         for aggregate in api.aggregates:
 355             # prevent infinite loop. Dont send request back to caller
 356             # unless the caller is the aggregate's SM
 357             if caller_hrn == aggregate and aggregate != api.hrn:
 358                 continue
 359             interface = api.aggregates[aggregate]
 360             server = api.server_proxy(interface, cred)
 361             multiclient.run(_Renew, aggregate, server, xrn, [cred], expiration_time, options)
 362
 363         results = multiclient.get_results()
 364
 365         geni_code = 0
 366         geni_output = ",".join([x.get('output',"") for x in results])
 367         geni_value = reduce (lambda x,y: x and y, [result.get('value',False) for result in results], True)
 368         for agg_result in results:
 369             agg_geni_code = agg_result['code'].get('geni_code',0)
 370             if agg_geni_code:
 371                 geni_code = agg_geni_code
 372
 373         results = {'aggregates': results, 'code': {'geni_code': geni_code}, 'value': geni_value, 'output': geni_output}
 374
 375         return results
 376
 377     def Delete(self, api, xrn, creds, options):
 378         call_id = options.get('call_id')
 379         if Callids().already_handled(call_id): return ""
 380
 381         def _Delete(server, xrn, creds, options):
 382             return server.Delete(xrn, creds, options)
 383
 384         (hrn, type) = urn_to_hrn(xrn[0])
 385         # get the callers hrn
 386         valid_cred = api.auth.checkCredentials(creds, 'deletesliver', hrn)[0]
 387         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 388
 389         # attempt to use delegated credential first
 390         cred = api.getDelegatedCredential(creds)
 391         if not cred:
 392             cred = api.getCredential()
 393         multiclient = MultiClient()
 394         for aggregate in api.aggregates:
 395             # prevent infinite loop. Dont send request back to caller
 396             # unless the caller is the aggregate's SM
 397             if caller_hrn == aggregate and aggregate != api.hrn:
 398                 continue
 399             interface = api.aggregates[aggregate]
 400             server = api.server_proxy(interface, cred)
 401             multiclient.run(_Delete, server, xrn, [cred], options)
 402
 403         results = []
 404         for result in multiclient.get_results():
 405             results += ReturnValue.get_value(result)
 406         return results
 407
 408
 409     # first draft at a merging SliverStatus
 410     def Status(self, api, slice_xrn, creds, options):
 411         def _Status(server, xrn, creds, options):
 412             return server.Status(xrn, creds, options)
 413
 414         call_id = options.get('call_id')
 415         if Callids().already_handled(call_id): return {}
 416         # attempt to use delegated credential first
 417         cred = api.getDelegatedCredential(creds)
 418         if not cred:
 419             cred = api.getCredential()
 420         multiclient = MultiClient()
 421         for aggregate in api.aggregates:
 422             interface = api.aggregates[aggregate]
 423             server = api.server_proxy(interface, cred)
 424             multiclient.run (_Status, server, slice_xrn, [cred], options)
 425         results = [ReturnValue.get_value(result) for result in multiclient.get_results()]
 426
 427         # get rid of any void result - e.g. when call_id was hit, where by convention we return {}
 428         results = [ result for result in results if result and result['geni_slivers']]
 429
 430         # do not try to combine if there's no result
 431         if not results : return {}
 432
 433         # otherwise let's merge stuff
 434         geni_slivers = []
 435         geni_urn  = None
 436         for result in results:
 437             try:
 438                 geni_urn = result['geni_urn']
 439                 geni_slivers.extend(result['geni_slivers'])
 440             except:
 441                 api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
 442         return {
 443             'geni_urn': geni_urn,
 444             'geni_slivers': geni_slivers
 445         }
 446
 447
 448     def Describe(self, api, creds, xrns, options):
 449         def _Describe(server, xrn, creds, options):
 450             return server.Describe(xrn, creds, options)
 451
 452         call_id = options.get('call_id')
 453         if Callids().already_handled(call_id): return {}
 454         # attempt to use delegated credential first
 455         cred = api.getDelegatedCredential(creds)
 456         if not cred:
 457             cred = api.getCredential()
 458         multiclient = MultiClient()
 459         for aggregate in api.aggregates:
 460             interface = api.aggregates[aggregate]
 461             server = api.server_proxy(interface, cred)
 462             multiclient.run (_Describe, server, xrns, [cred], options)
 463         results = [ReturnValue.get_value(result) for result in multiclient.get_results()]
 464
 465         # get rid of any void result - e.g. when call_id was hit, where by convention we return {}
 466         results = [ result for result in results if result and result.get('geni_urn')]
 467
 468         # do not try to combine if there's no result
 469         if not results : return {}
 470
 471         # otherwise let's merge stuff
 472         version_manager = VersionManager()
 473         manifest_version = version_manager._get_version('GENI', '3', 'manifest')
 474         result_rspec = RSpec(version=manifest_version)
 475         geni_slivers = []
 476         geni_urn  = None
 477         for result in results:
 478             try:
 479                 geni_urn = result['geni_urn']
 480                 result_rspec.version.merge(ReturnValue.get_value(result['geni_rspec']))
 481                 geni_slivers.extend(result['geni_slivers'])
 482             except:
 483                 api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
 484         return {
 485             'geni_urn': geni_urn,
 486             'geni_rspec': result_rspec.toxml(),
 487             'geni_slivers': geni_slivers
 488         }
 489
 490     def PerformOperationalAction(self, api, xrn, creds, action, options):
 491         # get the callers hrn
 492         valid_cred = api.auth.checkCredentials(creds, 'createsliver', xrn)[0]
 493         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 494
 495         # attempt to use delegated credential first
 496         cred = api.getDelegatedCredential(creds)
 497         if not cred:
 498             cred = api.getCredential()
 499         multiclient = MultiClient()
 500         for aggregate in api.aggregates:
 501             # prevent infinite loop. Dont send request back to caller
 502             # unless the caller is the aggregate's SM
 503             if caller_hrn == aggregate and aggregate != api.hrn:
 504                 continue
 505             interface = api.aggregates[aggregate]
 506             server = api.server_proxy(interface, cred)
 507             multiclient.run(server.PerformOperationalAction, xrn, [cred], action, options)
 508         multiclient.get_results()
 509         return 1
 510
 511     def Shutdown(self, api, xrn, creds, options=None):
 512         if options is None: options={}
 513         xrn = Xrn(xrn)
 514         # get the callers hrn
 515         valid_cred = api.auth.checkCredentials(creds, 'stopslice', xrn.hrn)[0]
 516         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
 517
 518         # attempt to use delegated credential first
 519         cred = api.getDelegatedCredential(creds)
 520         if not cred:
 521             cred = api.getCredential()
 522         multiclient = MultiClient()
 523         for aggregate in api.aggregates:
 524             # prevent infinite loop. Dont send request back to caller
 525             # unless the caller is the aggregate's SM
 526             if caller_hrn == aggregate and aggregate != api.hrn:
 527                 continue
 528             interface = api.aggregates[aggregate]
 529             server = api.server_proxy(interface, cred)
 530             multiclient.run(server.Shutdown, xrn.urn, cred)
 531         multiclient.get_results()
 532         return 1
 533