b8aafcc2bfc6a4d7163811f0f690d8041c0b10f4
[sfa.git] / sfa / managers / slice_manager.py
1 import sys
2 import time
3 import traceback
4 from StringIO import StringIO
5 from copy import copy
6 from lxml import etree
7
8 from sfa.trust.sfaticket import SfaTicket
9 from sfa.trust.credential import Credential
10
11 from sfa.util.sfalogging import logger
12 from sfa.util.xrn import Xrn, urn_to_hrn
13 from sfa.util.version import version_core
14 from sfa.util.callids import Callids
15 from sfa.util.cache import Cache
16
17 from sfa.client.multiclient import MultiClient
18
19 from sfa.rspecs.rspec_converter import RSpecConverter
20 from sfa.rspecs.version_manager import VersionManager
21 from sfa.rspecs.rspec import RSpec 
22
23 from sfa.client.client_helper import sfa_to_pg_users_arg
24 from sfa.client.return_value import ReturnValue
25
26 class SliceManager:
27
28     # the cache instance is a class member so it survives across incoming requests
29     cache = None
30
31     def __init__ (self, config):
32         self.cache=None
33         if config.SFA_SM_CACHING:
34             if SliceManager.cache is None:
35                 SliceManager.cache = Cache()
36             self.cache = SliceManager.cache
37         
38     def GetVersion(self, api, options):
39         # peers explicitly in aggregates.xml
40         peers =dict ([ (peername,interface.get_url()) for (peername,interface) in api.aggregates.iteritems()
41                        if peername != api.hrn])
42         version_manager = VersionManager()
43         ad_rspec_versions = []
44         request_rspec_versions = []
45         cred_types = [{'geni_type': 'geni_sfa', 'geni_version': str(i)} for i in range(4)[-2:]]
46         for rspec_version in version_manager.versions:
47             if rspec_version.content_type in ['*', 'ad']:
48                 ad_rspec_versions.append(rspec_version.to_dict())
49             if rspec_version.content_type in ['*', 'request']:
50                 request_rspec_versions.append(rspec_version.to_dict())
51         xrn=Xrn(api.hrn, 'authority+sm')
52         version_more = {
53             'interface':'slicemgr',
54             'sfa': 2,
55             'geni_api': 3,
56             'geni_api_versions': {'3': 'http://%s:%s' % (api.config.SFA_SM_HOST, api.config.SFA_SM_PORT)},
57             'hrn' : xrn.get_hrn(),
58             'urn' : xrn.get_urn(),
59             'peers': peers,
60             'geni_single_allocation': 0, # Accept operations that act on as subset of slivers in a given state.
61             'geni_allocate': 'geni_many',# Multiple slivers can exist and be incrementally added, including those which connect or overlap in some way.
62             'geni_credential_types': cred_types,
63             }
64         sm_version=version_core(version_more)
65         # local aggregate if present needs to have localhost resolved
66         if api.hrn in api.aggregates:
67             local_am_url=api.aggregates[api.hrn].get_url()
68             sm_version['peers'][api.hrn]=local_am_url.replace('localhost',sm_version['hostname'])
69         return sm_version
70     
71     def drop_slicemgr_stats(self, rspec):
72         try:
73             stats_elements = rspec.xml.xpath('//statistics')
74             for node in stats_elements:
75                 node.getparent().remove(node)
76         except Exception as e:
77             logger.warn("drop_slicemgr_stats failed: %s " % (str(e)))
78     
79     def add_slicemgr_stat(self, rspec, callname, aggname, elapsed, status, exc_info=None):
80         try:
81             stats_tags = rspec.xml.xpath('//statistics[@call="%s"]' % callname)
82             if stats_tags:
83                 stats_tag = stats_tags[0]
84             else:
85                 stats_tag = rspec.xml.root.add_element("statistics", call=callname)
86
87             stat_tag = stats_tag.add_element("aggregate", name=str(aggname), 
88                                              elapsed=str(elapsed), status=str(status))
89
90             if exc_info:
91                 exc_tag = stat_tag.add_element("exc_info", name=str(exc_info[1]))
92
93                 # formats the traceback as one big text blob
94                 #exc_tag.text = "\n".join(traceback.format_exception(exc_info[0], exc_info[1], exc_info[2]))
95
96                 # formats the traceback as a set of xml elements
97                 tb = traceback.extract_tb(exc_info[2])
98                 for item in tb:
99                     exc_frame = exc_tag.add_element("tb_frame", filename=str(item[0]), 
100                                                     line=str(item[1]), func=str(item[2]), code=str(item[3]))
101
102         except Exception as e:
103             logger.warn("add_slicemgr_stat failed on  %s: %s" %(aggname, str(e)))
104     
105     def ListResources(self, api, creds, options):
106         call_id = options.get('call_id') 
107         if Callids().already_handled(call_id): return ""
108
109         version_manager = VersionManager()
110
111         def _ListResources(aggregate, server, credential, options):
112             forward_options = copy(options)
113             tStart = time.time()
114             try:
115                 version = api.get_cached_server_version(server)
116                 # force ProtoGENI aggregates to give us a v2 RSpec
117                 forward_options['geni_rspec_version'] = options.get('geni_rspec_version')
118                 result = server.ListResources(credential, forward_options)
119                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
120             except Exception as e:
121                 api.logger.log_exc("ListResources failed at %s" %(server.url))
122                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
123     
124         # get slice's hrn from options
125         xrn = options.get('geni_slice_urn', '')
126         (hrn, type) = urn_to_hrn(xrn)
127         if 'geni_compressed' in options:
128             del(options['geni_compressed'])
129     
130         # get the rspec's return format from options
131         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
132         version_string = "rspec_%s" % (rspec_version)
133     
134         # look in cache first
135         cached_requested = options.get('cached', True)
136         if not xrn and self.cache and cached_requested:
137             rspec =  self.cache.get(version_string)
138             if rspec:
139                 api.logger.debug("SliceManager.ListResources returns cached advertisement")
140                 return rspec
141     
142         # get the callers hrn
143         valid_cred = api.auth.checkCredentials(creds, 'listnodes', hrn)[0]
144         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
145     
146         # attempt to use delegated credential first
147         cred = api.getDelegatedCredential(creds)
148         if not cred:
149             cred = api.getCredential()
150         multiclient = MultiClient()
151         for aggregate in api.aggregates:
152             # prevent infinite loop. Dont send request back to caller
153             # unless the caller is the aggregate's SM
154             if caller_hrn == aggregate and aggregate != api.hrn:
155                 continue
156     
157             # get the rspec from the aggregate
158             interface = api.aggregates[aggregate]
159             server = api.server_proxy(interface, cred)
160             multiclient.run(_ListResources, aggregate, server, [cred], options)
161     
162     
163         results = multiclient.get_results()
164         rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
165         if xrn:    
166             result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'manifest')
167         else: 
168             result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'ad')
169         rspec = RSpec(version=result_version)
170         for result in results:
171             self.add_slicemgr_stat(rspec, "ListResources", result["aggregate"], result["elapsed"], 
172                                    result["status"], result.get("exc_info",None))
173             if result["status"]=="success":
174                 res = result['result']['value']
175                 try:
176                     rspec.version.merge(ReturnValue.get_value(res))
177                 except:
178                     api.logger.log_exc("SM.ListResources: Failed to merge aggregate rspec")
179     
180         # cache the result
181         if self.cache and not xrn:
182             api.logger.debug("SliceManager.ListResources caches advertisement")
183             self.cache.add(version_string, rspec.toxml())
184     
185         return rspec.toxml()
186
187
188     def Allocate(self, api, xrn, creds, rspec_str, expiration, options):
189         call_id = options.get('call_id')
190         if Callids().already_handled(call_id): return ""
191     
192         version_manager = VersionManager()
193         def _Allocate(aggregate, server, xrn, credential, rspec, options):
194             tStart = time.time()
195             try:
196                 # Need to call GetVersion at an aggregate to determine the supported
197                 # rspec type/format beofre calling CreateSliver at an Aggregate.
198                 #server_version = api.get_cached_server_version(server)
199                 #if 'sfa' not in server_version and 'geni_api' in server_version:
200                     # sfa aggregtes support both sfa and pg rspecs, no need to convert
201                     # if aggregate supports sfa rspecs. otherwise convert to pg rspec
202                     #rspec = RSpec(RSpecConverter.to_pg_rspec(rspec, 'request'))
203                     #filter = {'component_manager_id': server_version['urn']}
204                     #rspec.filter(filter)
205                     #rspec = rspec.toxml()
206                 result = server.Allocate(xrn, credential, rspec, options)
207                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
208             except:
209                 logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
210                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
211
212         # Validate the RSpec against PlanetLab's schema --disabled for now
213         # The schema used here needs to aggregate the PL and VINI schemas
214         # schema = "/var/www/html/schemas/pl.rng"
215         rspec = RSpec(rspec_str)
216     #    schema = None
217     #    if schema:
218     #        rspec.validate(schema)
219     
220         # if there is a <statistics> section, the aggregates don't care about it,
221         # so delete it.
222         self.drop_slicemgr_stats(rspec)
223     
224         # attempt to use delegated credential first
225         cred = api.getDelegatedCredential(creds)
226         if not cred:
227             cred = api.getCredential()
228     
229         # get the callers hrn
230         hrn, type = urn_to_hrn(xrn)
231         valid_cred = api.auth.checkCredentials(creds, 'createsliver', hrn)[0]
232         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
233         multiclient = MultiClient()
234         for aggregate in api.aggregates:
235             # prevent infinite loop. Dont send request back to caller
236             # unless the caller is the aggregate's SM 
237             if caller_hrn == aggregate and aggregate != api.hrn:
238                 continue
239             interface = api.aggregates[aggregate]
240             server = api.server_proxy(interface, cred)
241             # Just send entire RSpec to each aggregate
242             multiclient.run(_Allocate, aggregate, server, xrn, [cred], rspec.toxml(), options)
243                 
244         results = multiclient.get_results()
245         manifest_version = version_manager._get_version(rspec.version.type, rspec.version.version, 'manifest')
246         result_rspec = RSpec(version=manifest_version)
247         geni_urn = None
248         geni_slivers = []
249
250         for result in results:
251             self.add_slicemgr_stat(result_rspec, "Allocate", result["aggregate"], result["elapsed"], 
252                                    result["status"], result.get("exc_info",None))
253             if result["status"]=="success":
254                 try:
255                     res = result['result']['value']
256                     geni_urn = res['geni_urn']
257                     result_rspec.version.merge(ReturnValue.get_value(res['geni_rspec']))
258                     geni_slivers.extend(res['geni_slivers'])
259                 except:
260                     api.logger.log_exc("SM.Allocate: Failed to merge aggregate rspec")
261         return {
262             'geni_urn': geni_urn,
263             'geni_rspec': result_rspec.toxml(),
264             'geni_slivers': geni_slivers
265         }
266
267
268     def Provision(self, api, xrn, creds, options):
269         call_id = options.get('call_id')
270         if Callids().already_handled(call_id): return ""
271
272         version_manager = VersionManager()
273         def _Provision(aggregate, server, xrn, credential, options):
274             tStart = time.time()
275             try:
276                 # Need to call GetVersion at an aggregate to determine the supported
277                 # rspec type/format beofre calling CreateSliver at an Aggregate.
278                 server_version = api.get_cached_server_version(server)
279                 result = server.Provision(xrn, credential, options)
280                 return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
281             except:
282                 logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
283                 return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
284
285         # attempt to use delegated credential first
286         cred = api.getDelegatedCredential(creds)
287         if not cred:
288             cred = api.getCredential()
289
290         # get the callers hrn
291         valid_cred = api.auth.checkCredentials(creds, 'createsliver', xrn)[0]
292         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
293         multiclient = MultiClient()
294         for aggregate in api.aggregates:
295             # prevent infinite loop. Dont send request back to caller
296             # unless the caller is the aggregate's SM
297             if caller_hrn == aggregate and aggregate != api.hrn:
298                 continue
299             interface = api.aggregates[aggregate]
300             server = api.server_proxy(interface, cred)
301             # Just send entire RSpec to each aggregate
302             multiclient.run(_Provision, aggregate, server, xrn, [cred], options)
303
304         results = multiclient.get_results()
305         manifest_version = version_manager._get_version('GENI', '3', 'manifest')
306         result_rspec = RSpec(version=manifest_version)
307         geni_slivers = []
308         geni_urn  = None  
309         for result in results:
310             self.add_slicemgr_stat(result_rspec, "Provision", result["aggregate"], result["elapsed"],
311                                    result["status"], result.get("exc_info",None))
312             if result["status"]=="success":
313                 try:
314                     res = result['result']['value']
315                     geni_urn = res['geni_urn']
316                     result_rspec.version.merge(ReturnValue.get_value(res['geni_rspec']))
317                     geni_slivers.extend(res['geni_slivers'])
318                 except:
319                     api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
320         return {
321             'geni_urn': geni_urn,
322             'geni_rspec': result_rspec.toxml(),
323             'geni_slivers': geni_slivers
324         }            
325
326
327     
328     def Renew(self, api, xrn, creds, expiration_time, options):
329         call_id = options.get('call_id')
330         if Callids().already_handled(call_id): return True
331
332         def _Renew(aggregate, server, xrn, creds, expiration_time, options):
333             try:
334                 result=server.Renew(xrn, creds, expiration_time, options)
335                 if type(result)!=dict:
336                     result = {'code': {'geni_code': 0}, 'value': result}
337                 result['aggregate'] = aggregate
338                 return result
339             except:
340                 logger.log_exc('Something wrong in _Renew with URL %s'%server.url)
341                 return {'aggregate': aggregate, 'exc_info': traceback.format_exc(),
342                         'code': {'geni_code': -1},
343                         'value': False, 'output': ""}
344
345         # get the callers hrn
346         valid_cred = api.auth.checkCredentials(creds, 'renewsliver', xrn)[0]
347         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
348
349         # attempt to use delegated credential first
350         cred = api.getDelegatedCredential(creds)
351         if not cred:
352             cred = api.getCredential(minimumExpiration=31*86400)
353         multiclient = MultiClient()
354         for aggregate in api.aggregates:
355             # prevent infinite loop. Dont send request back to caller
356             # unless the caller is the aggregate's SM
357             if caller_hrn == aggregate and aggregate != api.hrn:
358                 continue
359             interface = api.aggregates[aggregate]
360             server = api.server_proxy(interface, cred)
361             multiclient.run(_Renew, aggregate, server, xrn, [cred], expiration_time, options)
362
363         results = multiclient.get_results()
364
365         geni_code = 0
366         geni_output = ",".join([x.get('output',"") for x in results])
367         geni_value = reduce (lambda x,y: x and y, [result.get('value',False) for result in results], True)
368         for agg_result in results:
369             agg_geni_code = agg_result['code'].get('geni_code',0)
370             if agg_geni_code:
371                 geni_code = agg_geni_code
372
373         results = {'aggregates': results, 'code': {'geni_code': geni_code}, 'value': geni_value, 'output': geni_output}
374
375         return results
376
377     def Delete(self, api, xrn, creds, options):
378         call_id = options.get('call_id')
379         if Callids().already_handled(call_id): return ""
380
381         def _Delete(server, xrn, creds, options):
382             return server.Delete(xrn, creds, options)
383
384         (hrn, type) = urn_to_hrn(xrn[0])
385         # get the callers hrn
386         valid_cred = api.auth.checkCredentials(creds, 'deletesliver', hrn)[0]
387         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
388
389         # attempt to use delegated credential first
390         cred = api.getDelegatedCredential(creds)
391         if not cred:
392             cred = api.getCredential()
393         multiclient = MultiClient()
394         for aggregate in api.aggregates:
395             # prevent infinite loop. Dont send request back to caller
396             # unless the caller is the aggregate's SM
397             if caller_hrn == aggregate and aggregate != api.hrn:
398                 continue
399             interface = api.aggregates[aggregate]
400             server = api.server_proxy(interface, cred)
401             multiclient.run(_Delete, server, xrn, [cred], options)
402         
403         results = []
404         for result in multiclient.get_results():
405             results += ReturnValue.get_value(result)
406         return results
407     
408     
409     # first draft at a merging SliverStatus
410     def Status(self, api, slice_xrn, creds, options):
411         def _Status(server, xrn, creds, options):
412             return server.Status(xrn, creds, options)
413
414         call_id = options.get('call_id') 
415         if Callids().already_handled(call_id): return {}
416         # attempt to use delegated credential first
417         cred = api.getDelegatedCredential(creds)
418         if not cred:
419             cred = api.getCredential()
420         multiclient = MultiClient()
421         for aggregate in api.aggregates:
422             interface = api.aggregates[aggregate]
423             server = api.server_proxy(interface, cred)
424             multiclient.run (_Status, server, slice_xrn, [cred], options)
425         results = [ReturnValue.get_value(result) for result in multiclient.get_results()]
426     
427         # get rid of any void result - e.g. when call_id was hit, where by convention we return {}
428         results = [ result for result in results if result and result['geni_slivers']]
429     
430         # do not try to combine if there's no result
431         if not results : return {}
432     
433         # otherwise let's merge stuff
434         geni_slivers = []
435         geni_urn  = None
436         for result in results:
437             try:
438                 geni_urn = result['geni_urn']
439                 geni_slivers.extend(result['geni_slivers'])
440             except:
441                 api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
442         return {
443             'geni_urn': geni_urn,
444             'geni_slivers': geni_slivers
445         }
446
447    
448     def Describe(self, api, creds, xrns, options):
449         def _Describe(server, xrn, creds, options):
450             return server.Describe(xrn, creds, options)
451
452         call_id = options.get('call_id')
453         if Callids().already_handled(call_id): return {}
454         # attempt to use delegated credential first
455         cred = api.getDelegatedCredential(creds)
456         if not cred:
457             cred = api.getCredential()
458         multiclient = MultiClient()
459         for aggregate in api.aggregates:
460             interface = api.aggregates[aggregate]
461             server = api.server_proxy(interface, cred)
462             multiclient.run (_Describe, server, xrns, [cred], options)
463         results = [ReturnValue.get_value(result) for result in multiclient.get_results()]
464
465         # get rid of any void result - e.g. when call_id was hit, where by convention we return {}
466         results = [ result for result in results if result and result.get('geni_urn')]
467
468         # do not try to combine if there's no result
469         if not results : return {}
470
471         # otherwise let's merge stuff
472         version_manager = VersionManager()
473         manifest_version = version_manager._get_version('GENI', '3', 'manifest')
474         result_rspec = RSpec(version=manifest_version)
475         geni_slivers = []
476         geni_urn  = None
477         for result in results:
478             try:
479                 geni_urn = result['geni_urn']
480                 result_rspec.version.merge(ReturnValue.get_value(result['geni_rspec']))
481                 geni_slivers.extend(result['geni_slivers'])
482             except:
483                 api.logger.log_exc("SM.Provision: Failed to merge aggregate rspec")
484         return {
485             'geni_urn': geni_urn,
486             'geni_rspec': result_rspec.toxml(),    
487             'geni_slivers': geni_slivers
488         }  
489     
490     def PerformOperationalAction(self, api, xrn, creds, action, options):
491         # get the callers hrn
492         valid_cred = api.auth.checkCredentials(creds, 'createsliver', xrn)[0]
493         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
494     
495         # attempt to use delegated credential first
496         cred = api.getDelegatedCredential(creds)
497         if not cred:
498             cred = api.getCredential()
499         multiclient = MultiClient()
500         for aggregate in api.aggregates:
501             # prevent infinite loop. Dont send request back to caller
502             # unless the caller is the aggregate's SM
503             if caller_hrn == aggregate and aggregate != api.hrn:
504                 continue
505             interface = api.aggregates[aggregate]
506             server = api.server_proxy(interface, cred)    
507             multiclient.run(server.PerformOperationalAction, xrn, [cred], action, options)
508         multiclient.get_results()    
509         return 1
510      
511     def Shutdown(self, api, xrn, creds, options=None):
512         if options is None: options={}
513         xrn = Xrn(xrn)  
514         # get the callers hrn
515         valid_cred = api.auth.checkCredentials(creds, 'stopslice', xrn.hrn)[0]
516         caller_hrn = Credential(cred=valid_cred).get_gid_caller().get_hrn()
517     
518         # attempt to use delegated credential first
519         cred = api.getDelegatedCredential(creds)
520         if not cred:
521             cred = api.getCredential()
522         multiclient = MultiClient()
523         for aggregate in api.aggregates:
524             # prevent infinite loop. Dont send request back to caller
525             # unless the caller is the aggregate's SM
526             if caller_hrn == aggregate and aggregate != api.hrn:
527                 continue
528             interface = api.aggregates[aggregate]
529             server = api.server_proxy(interface, cred)
530             multiclient.run(server.Shutdown, xrn.urn, cred)
531         multiclient.get_results()    
532         return 1
533