7a9e5cc003725ad07f725da65a79ab9d02c53089
[sfa.git] / sfa / plc / slices.py
1 ### $Id$
2 ### $URL$
3
4 import datetime
5 import time
6 import traceback
7 import sys
8
9 from types import StringTypes
10 from sfa.util.misc import *
11 from sfa.util.rspec import *
12 from sfa.util.specdict import *
13 from sfa.util.faults import *
14 from sfa.util.storage import *
15 from sfa.util.record import GeniRecord
16 from sfa.util.policy import Policy
17 from sfa.util.prefixTree import prefixTree
18 from sfa.util.debug import log
19 from sfa.server.aggregate import Aggregates
20 from sfa.server.registry import Registries
21
22 class Slices(SimpleStorage):
23
24     rspec_to_slice_tag = {'max_rate':'net_max_rate'}
25
26     def __init__(self, api, ttl = .5, caller_cred=None):
27         self.api = api
28         self.ttl = ttl
29         self.threshold = None
30         path = self.api.config.SFA_DATA_DIR
31         filename = ".".join([self.api.interface, self.api.hrn, "slices"])
32         filepath = path + os.sep + filename
33         self.slices_file = filepath
34         SimpleStorage.__init__(self, self.slices_file)
35         self.policy = Policy(self.api)    
36         self.load()
37         self.caller_cred=caller_cred
38
39
40     def get_peer(self, hrn):
41         # Becaues of myplc federation,  we first need to determine if this
42         # slice belongs to out local plc or a myplc peer. We will assume it 
43         # is a local site, unless we find out otherwise  
44         peer = None
45
46         # get this slice's authority (site)
47         slice_authority = get_authority(hrn)
48
49         # get this site's authority (sfa root authority or sub authority)
50         site_authority = get_authority(slice_authority).lower()
51
52         # check if we are already peered with this site_authority, if so
53         peers = self.api.plshell.GetPeers(self.api.plauth, {}, ['peer_id', 'peername', 'shortname', 'hrn_root'])
54         for peer_record in peers:
55             names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)]
56             if site_authority in names:
57                 peer = peer_record['shortname']
58
59         return peer
60
61     def get_sfa_peer(self, hrn):
62         # return the authority for this hrn or None if we are the authority
63         sfa_peer = None
64         slice_authority = get_authority(hrn)
65         site_authority = get_authority(slice_authority)
66
67         if site_authority != self.api.hrn:
68             sfa_peer = site_authority
69
70         return sfa_peer 
71
72     def refresh(self):
73         """
74         Update the cached list of slices
75         """
76         # Reload components list
77         now = datetime.datetime.now()
78         if not self.has_key('threshold') or not self.has_key('timestamp') or \
79            now > datetime.datetime.fromtimestamp(time.mktime(time.strptime(self['threshold'], self.api.time_format))):
80             if self.api.interface in ['aggregate']:
81                 self.refresh_slices_aggregate()
82             elif self.api.interface in ['slicemgr']:
83                 self.refresh_slices_smgr()
84
85     def refresh_slices_aggregate(self):
86         slices = self.api.plshell.GetSlices(self.api.plauth, {'peer_id': None}, ['name'])
87         slice_hrns = [slicename_to_hrn(self.api.hrn, slice['name']) for slice in slices]
88
89          # update timestamp and threshold
90         timestamp = datetime.datetime.now()
91         hr_timestamp = timestamp.strftime(self.api.time_format)
92         delta = datetime.timedelta(hours=self.ttl)
93         threshold = timestamp + delta
94         hr_threshold = threshold.strftime(self.api.time_format)
95         
96         slice_details = {'hrn': slice_hrns,
97                          'timestamp': hr_timestamp,
98                          'threshold': hr_threshold
99                         }
100         self.update(slice_details)
101         self.write()     
102         
103
104     def refresh_slices_smgr(self):
105         slice_hrns = []
106         aggregates = Aggregates(self.api)
107         credential = self.api.getCredential()
108         for aggregate in aggregates:
109             success = False
110             # request hash is optional so lets try the call without it 
111             try:
112                 slices = aggregates[aggregate].get_slices(credential)
113                 slice_hrns.extend(slices)
114                 success = True
115             except:
116                 print >> log, "%s" % (traceback.format_exc())
117                 print >> log, "Error calling slices at aggregate %(aggregate)s" % locals()
118
119             # try sending the request hash if the previous call failed 
120             if not success:
121                 arg_list = [credential]
122                 request_hash = self.api.key.compute_hash(arg_list)
123                 try:
124                     slices = aggregates[aggregate].get_slices(credential, request_hash)
125                     slice_hrns.extend(slices)
126                     success = True
127                 except:
128                     print >> log, "%s" % (traceback.format_exc())
129                     print >> log, "Error calling slices at aggregate %(aggregate)s" % locals()
130
131         # update timestamp and threshold
132         timestamp = datetime.datetime.now()
133         hr_timestamp = timestamp.strftime(self.api.time_format)
134         delta = datetime.timedelta(hours=self.ttl)
135         threshold = timestamp + delta
136         hr_threshold = threshold.strftime(self.api.time_format)
137
138         slice_details = {'hrn': slice_hrns,
139                          'timestamp': hr_timestamp,
140                          'threshold': hr_threshold
141                         }
142         self.update(slice_details)
143         self.write()
144
145
146     def delete_slice(self, hrn):
147         if self.api.interface in ['aggregate']:
148             self.delete_slice_aggregate(hrn)
149         elif self.api.interface in ['slicemgr']:
150             self.delete_slice_smgr(hrn)
151         
152     def delete_slice_aggregate(self, hrn):
153
154         slicename = hrn_to_pl_slicename(hrn)
155         slices = self.api.plshell.GetSlices(self.api.plauth, {'name': slicename})
156         if not slices:
157             return 1        
158         slice = slices[0]
159
160         # determine if this is a peer slice
161         peer = self.get_peer(hrn)
162         if peer:
163             self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', slice['slice_id'], peer)
164         self.api.plshell.DeleteSliceFromNodes(self.api.plauth, slicename, slice['node_ids'])
165         if peer:
166             self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice['slice_id'], peer, slice['peer_slice_id'])
167         return 1
168
169     def delete_slice_smgr(self, hrn):
170         credential = self.api.getCredential()
171         caller_cred = self.caller_cred
172         aggregates = Aggregates(self.api)
173         for aggregate in aggregates:
174             success = False
175             # request hash is optional so lets try the call without it
176             try:
177                 aggregates[aggregate].delete_slice(credential, hrn, caller_cred)
178                 success = True
179             except:
180                 print >> log, "%s" % (traceback.format_exc())
181                 print >> log, "Error calling list nodes at aggregate %s" % aggregate
182             
183             # try sending the request hash if the previous call failed 
184             if not success:
185                 try:
186                     arg_list = [credential, hrn]
187                     request_hash = self.api.key.compute_hash(arg_list)
188                     aggregates[aggregate].delete_slice(credential, hrn, request_hash, caller_cred)
189                     success = True
190                 except:
191                     print >> log, "%s" % (traceback.format_exc())
192                     print >> log, "Error calling list nodes at aggregate %s" % aggregate
193                         
194     def create_slice(self, hrn, rspec):
195         
196         # check our slice policy before we procede
197         whitelist = self.policy['slice_whitelist']     
198         blacklist = self.policy['slice_blacklist']
199        
200         if whitelist and hrn not in whitelist or \
201            blacklist and hrn in blacklist:
202             policy_file = self.policy.policy_file
203             print >> log, "Slice %(hrn)s not allowed by policy %(policy_file)s" % locals()
204             return 1
205
206         if self.api.interface in ['aggregate']:     
207             self.create_slice_aggregate(hrn, rspec)
208         elif self.api.interface in ['slicemgr']:
209             self.create_slice_smgr(hrn, rspec)
210
211     def verify_site(self, registry, credential, slice_hrn, peer, sfa_peer):
212         authority = get_authority(slice_hrn)
213         try:
214             site_records = registry.resolve(credential, authority)
215         except:
216             arg_list = [credential, authority]
217             request_hash = self.api.key.compute_hash(arg_list)
218             site_records = registry.resolve(credential, authority, request_hash)
219             
220         site = {}
221         for site_record in site_records:
222             if site_record['type'] == 'authority':
223                 site = site_record
224         if not site:
225             raise RecordNotFound(authority)
226         remote_site_id = site.pop('site_id')    
227                 
228         login_base = get_leaf(authority)
229         sites = self.api.plshell.GetSites(self.api.plauth, login_base)
230         if not sites:
231             site_id = self.api.plshell.AddSite(self.api.plauth, site)
232             if peer:
233                 self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', site_id, peer, remote_site_id)   
234             # mark this site as an sfa peer record
235             if sfa_peer:
236                 peer_dict = {'type': 'authority', 'hrn': authority, 'peer_authority': sfa_peer, 'pointer': site_id}
237                 try:
238                     registry.register_peer_object(credential, peer_dict)
239                 except:
240                     arg_list = [credential]
241                     request_hash = self.api.key.compute_hash(arg_list) 
242                     registry.register_peer_object(credential, peer_dict, request_hash)
243         else:
244             site_id = sites[0]['site_id']
245             remote_site_id = sites[0]['peer_site_id']
246
247
248         return (site_id, remote_site_id) 
249
250     def verify_slice(self, registry, credential, slice_hrn, site_id, remote_site_id, peer, sfa_peer):
251         slice = {}
252         slice_record = None
253         authority = get_authority(slice_hrn)
254         try:
255             slice_records = registry.resolve(credential, slice_hrn)
256         except:    
257             arg_list = [credential, slice_hrn]
258             request_hash = self.api.key.compute_hash(arg_list)
259             slice_records = registry.resolve(credential, slice_hrn, request_hash)
260
261         for record in slice_records:
262             if record['type'] in ['slice']:
263                 slice_record = record
264         if not slice_record:
265             raise RecordNotFound(hrn)
266         slicename = hrn_to_pl_slicename(slice_hrn)
267         parts = slicename.split("_")
268         login_base = parts[0]
269         slices = self.api.plshell.GetSlices(self.api.plauth, [slicename], ['slice_id', 'node_ids', 'site_id']) 
270         if not slices:
271             slice_fields = {}
272             slice_keys = ['name', 'url', 'description']
273             for key in slice_keys:
274                 if key in slice_record and slice_record[key]:
275                     slice_fields[key] = slice_record[key]
276
277             # add the slice  
278             slice_id = self.api.plshell.AddSlice(self.api.plauth, slice_fields)
279             slice = slice_fields
280             slice['slice_id'] = slice_id
281
282             # mark this slice as an sfa peer record
283             if sfa_peer:
284                 peer_dict = {'type': 'slice', 'hrn': slice_hrn, 'peer_authority': sfa_peer, 'pointer': slice_id}
285                 try:
286                     registry.register_peer_object(credential, peer_dict)
287                 except:
288                     arg_list = [credential]
289                     request_hash = self.api.key.compute_hash(arg_list) 
290                     registry.register_peer_object(credential, peer_dict, request_hash)
291
292             #this belongs to a peer
293             if peer:
294                 self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice_id, peer, slice_record['pointer'])
295             slice['node_ids'] = []
296         else:
297             slice = slices[0]
298             slice_id = slice['slice_id']
299             site_id = slice['site_id']
300
301         slice['peer_slice_id'] = slice_record['pointer']
302         self.verify_persons(registry, credential, slice_record, site_id, remote_site_id, peer, sfa_peer)
303     
304         return slice        
305
306     def verify_persons(self, registry, credential, slice_record, site_id, remote_site_id, peer, sfa_peer):
307         # get the list of valid slice users from the registry and make 
308         # sure they are added to the slice 
309         slicename = hrn_to_pl_slicename(slice_record['hrn'])
310         researchers = slice_record.get('researcher', [])
311         for researcher in researchers:
312             person_record = {}
313             try:
314                 person_records = registry.resolve(credential, researcher)
315             except:
316                 arg_list = [credential, researcher]
317                 request_hash = self.api.key.compute_hash(arg_list) 
318                 person_records = registry.resolve(credential, researcher, request_hash)
319             for record in person_records:
320                 if record['type'] in ['user']:
321                     person_record = record
322             if not person_record:
323                 pass
324             person_dict = person_record
325             local_person=False
326             if peer:
327                 peer_id = self.api.plshell.GetPeers(self.api.plauth, {'shortname': peer}, ['peer_id'])[0]['peer_id']
328                 persons = self.api.plshell.GetPersons(self.api.plauth, {'email': [person_dict['email']], 'peer_id': peer_id}, ['person_id', 'key_ids'])
329                 if not persons:
330                     persons = self.api.plshell.GetPersons(self.api.plauth, [person_dict['email']], ['person_id', 'key_ids'])
331                     if persons:
332                        local_person=True
333
334             else:
335                 persons = self.api.plshell.GetPersons(self.api.plauth, [person_dict['email']], ['person_id', 'key_ids'])   
336         
337             if not persons:
338                 person_id=self.api.plshell.AddPerson(self.api.plauth, person_dict)
339                 self.api.plshell.UpdatePerson(self.api.plauth, person_id, {'enabled' : True})
340                 
341                 # mark this person as an sfa peer record
342                 if sfa_peer:
343                     peer_dict = {'type': 'user', 'hrn': researcher, 'peer_authority': sfa_peer, 'pointer': person_id}
344                     try:
345                         registry.register_peer_object(credential, peer_dict)
346                     except:
347                         arg_list = [credential]
348                         request_hash = self.api.key.compute_hash(arg_list) 
349                         registry.register_peer_object(credential, peer_dict, request_hash)
350
351                 if peer:
352                     self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
353                 key_ids = []
354             else:
355                 person_id = persons[0]['person_id']
356                 key_ids = persons[0]['key_ids']
357
358
359             # if this is a peer person, we must unbind them from the peer or PLCAPI will throw
360             # an error
361             if peer:
362                 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'person', person_id, peer)
363                 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'site', site_id,  peer)
364
365             self.api.plshell.AddPersonToSlice(self.api.plauth, person_dict['email'], slicename)
366             self.api.plshell.AddPersonToSite(self.api.plauth, person_dict['email'], site_id)
367             if peer and not local_person:
368                 self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
369             if peer:
370                 self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', site_id, peer, remote_site_id)
371             
372             self.verify_keys(registry, credential, person_dict, key_ids, person_id, peer, local_person)
373
374     def verify_keys(self, registry, credential, person_dict, key_ids, person_id,  peer, local_person):
375         keylist = self.api.plshell.GetKeys(self.api.plauth, key_ids, ['key'])
376         keys = [key['key'] for key in keylist]
377         
378         #add keys that arent already there
379         key_ids = person_dict['key_ids']
380         for personkey in person_dict['keys']:
381             if personkey not in keys:
382                 key = {'key_type': 'ssh', 'key': personkey}
383                 if peer:
384                     self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'person', person_id, peer)
385                 key_id = self.api.plshell.AddPersonKey(self.api.plauth, person_dict['email'], key)
386                 if peer and not local_person:
387                     self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
388                 if peer:
389                     try: self.api.plshell.BindObjectToPeer(self.api.plauth, 'key', key_id, peer, key_ids.pop(0))
390
391                     except: pass   
392
393     def create_slice_aggregate(self, hrn, rspec):
394
395         # Determine if this is a peer slice
396         peer = self.get_peer(hrn)
397         sfa_peer = self.get_sfa_peer(hrn)
398
399         spec = RSpec(rspec)
400         # Get the slice record from sfa
401         slicename = hrn_to_pl_slicename(hrn) 
402         slice = {}
403         slice_record = None
404         registries = Registries(self.api)
405         registry = registries[self.api.hrn]
406         credential = self.api.getCredential()
407
408         site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer)
409         slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer)
410
411         # find out where this slice is currently running
412         nodelist = self.api.plshell.GetNodes(self.api.plauth, slice['node_ids'], ['hostname'])
413         hostnames = [node['hostname'] for node in nodelist]
414
415         # get netspec details
416         nodespecs = spec.getDictsByTagName('NodeSpec')
417
418         # dict in which to store slice attributes to set for the nodes
419         nodes = {}
420         for nodespec in nodespecs:
421             if isinstance(nodespec['name'], list):
422                 for nodename in nodespec['name']:
423                     nodes[nodename] = {}
424                     for k in nodespec.keys():
425                         rspec_attribute_value = nodespec[k]
426                         if (self.rspec_to_slice_tag.has_key(k)):
427                             slice_tag_name = self.rspec_to_slice_tag[k]
428                             nodes[nodename][slice_tag_name] = rspec_attribute_value
429             elif isinstance(nodespec['name'], StringTypes):
430                 nodename = nodespec['name']
431                 nodes[nodename] = {}
432                 for k in nodespec.keys():
433                     rspec_attribute_value = nodespec[k]
434                     if (self.rspec_to_slice_tag.has_key(k)):
435                         slice_tag_name = self.rspec_to_slice_tag[k]
436                         nodes[nodename][slice_tag_name] = rspec_attribute_value
437
438                 for k in nodespec.keys():
439                     rspec_attribute_value = nodespec[k]
440                     if (self.rspec_to_slice_tag.has_key(k)):
441                         slice_tag_name = self.rspec_to_slice_tag[k]
442                         nodes[nodename][slice_tag_name] = rspec_attribute_value
443
444         node_names = nodes.keys()
445         # remove nodes not in rspec
446         deleted_nodes = list(set(hostnames).difference(node_names))
447         # add nodes from rspec
448         added_nodes = list(set(node_names).difference(hostnames))
449
450         if peer:
451             self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', slice['slice_id'], peer)
452
453         self.api.plshell.AddSliceToNodes(self.api.plauth, slicename, added_nodes) 
454
455         # Add recognized slice tags
456         for node_name in node_names:
457             node = nodes[node_name]
458             for slice_tag in node.keys():
459                 value = node[slice_tag]
460                 if (isinstance(value, list)):
461                     value = value[0]
462
463                 self.api.plshell.AddSliceTag(self.api.plauth, slicename, slice_tag, value, node_name)
464
465         self.api.plshell.DeleteSliceFromNodes(self.api.plauth, slicename, deleted_nodes)
466         if peer:
467             self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice['slice_id'], peer, slice['peer_slice_id'])
468
469         return 1
470
471     def create_slice_smgr(self, hrn, rspec):
472         spec = RSpec()
473         tempspec = RSpec()
474         spec.parseString(rspec)
475         slicename = hrn_to_pl_slicename(hrn)
476         specDict = spec.toDict()
477         if specDict.has_key('RSpec'): specDict = specDict['RSpec']
478         if specDict.has_key('start_time'): start_time = specDict['start_time']
479         else: start_time = 0
480         if specDict.has_key('end_time'): end_time = specDict['end_time']
481         else: end_time = 0
482
483         rspecs = {}
484         aggregates = Aggregates(self.api)
485         credential = self.api.getCredential()
486
487         # split the netspecs into individual rspecs
488         netspecs = spec.getDictsByTagName('NetSpec')
489         for netspec in netspecs:
490             net_hrn = netspec['name']
491             resources = {'start_time': start_time, 'end_time': end_time, 'networks': netspec}
492             resourceDict = {'RSpec': resources}
493             tempspec.parseDict(resourceDict)
494             rspecs[net_hrn] = tempspec.toxml()
495
496         # send each rspec to the appropriate aggregate/sm
497         caller_cred = self.caller_cred 
498         for net_hrn in rspecs:
499             try:
500                 # if we are directly connected to the aggregate then we can just send them the rspec
501                 # if not, then we may be connected to an sm thats connected to the aggregate
502                 if net_hrn in aggregates:
503                     # send the whloe rspec to the local aggregate
504                     if net_hrn in [self.api.hrn]:
505                         try:
506                             aggregates[net_hrn].create_slice(credential, hrn, rspec, caller_cred)
507                         except:
508                             arg_list = [credential,hrn,rspec]
509                             request_hash = self.api.key.compute_hash(arg_list)
510                             aggregates[net_hrn].create_slice(credential, hrn, rspec, request_hash, caller_cred)
511                     else:
512                         try:
513                             aggregates[net_hrn].create_slice(credential, hrn, rspecs[net_hrn], caller_cred)
514                         except:
515                             arg_list = [credential,hrn,rspecs[net_hrn]]
516                             request_hash = self.api.key.compute_hash(arg_list)
517                             aggregates[net_hrn].create_slice(credential, hrn, rspecs[net_hrn], request_hash, caller_cred)
518                 else:
519                     # lets forward this rspec to a sm that knows about the network
520                     arg_list = [credential, net_hrn]
521                     request_hash = self.api.compute_hash(arg_list)    
522                     for aggregate in aggregates:
523                         try:
524                             network_found = aggregates[aggregate].get_aggregates(credential, net_hrn)
525                         except:
526                             network_found = aggregates[aggregate].get_aggregates(credential, net_hrn, request_hash)
527                         if network_networks:
528                             try:
529                                 aggregates[aggregate].create_slice(credential, hrn, rspecs[net_hrn], caller_cred)
530                             except:
531                                 arg_list = [credential, hrn, rspecs[net_hrn]]
532                                 request_hash = self.api.key.compute_hash(arg_list) 
533                                 aggregates[aggregate].create_slice(credential, hrn, rspecs[net_hrn], request_hash, caller_cred)
534                      
535             except:
536                 print >> log, "Error creating slice %(hrn)s at aggregate %(net_hrn)s" % locals()
537                 traceback.print_exc()
538         return 1
539
540
541     def start_slice(self, hrn):
542         if self.api.interface in ['aggregate']:
543             self.start_slice_aggregate(hrn)
544         elif self.api.interface in ['slicemgr']:
545             self.start_slice_smgr(hrn)
546
547     def start_slice_aggregate(self, hrn):
548         slicename = hrn_to_pl_slicename(hrn)
549         slices = self.api.plshell.GetSlices(self.api.plauth, {'name': slicename}, ['slice_id'])
550         if not slices:
551             raise RecordNotFound(hrn)
552         slice_id = slices[0]
553         attributes = self.api.plshell.GetSliceAttributes(self.api.plauth, {'slice_id': slice_id, 'name': 'enabled'}, ['slice_attribute_id'])
554         attribute_id = attreibutes[0]['slice_attribute_id']
555         self.api.plshell.UpdateSliceAttribute(self.api.plauth, attribute_id, "1" )
556         return 1
557
558     def start_slice_smgr(self, hrn):
559         credential = self.api.getCredential()
560         aggregates = Aggregates(self.api)
561         for aggregate in aggregates:
562             aggregates[aggregate].start_slice(credential, hrn)
563         return 1
564
565
566     def stop_slice(self, hrn):
567         if self.api.interface in ['aggregate']:
568             self.stop_slice_aggregate(hrn)
569         elif self.api.interface in ['slicemgr']:
570             self.stop_slice_smgr(hrn)
571
572     def stop_slice_aggregate(self, hrn):
573         slicename = hrn_to_pl_slicename(hrn)
574         slices = self.api.plshell.GetSlices(self.api.plauth, {'name': slicename}, ['slice_id'])
575         if not slices:
576             raise RecordNotFound(hrn)
577         slice_id = slices[0]['slice_id']
578         attributes = self.api.plshell.GetSliceAttributes(self.api.plauth, {'slice_id': slice_id, 'name': 'enabled'}, ['slice_attribute_id'])
579         attribute_id = attributes[0]['slice_attribute_id']
580         self.api.plshell.UpdateSliceAttribute(self.api.plauth, attribute_id, "0")
581         return 1
582
583     def stop_slice_smgr(self, hrn):
584         credential = self.api.getCredential()
585         aggregates = Aggregates(self.api)
586         arg_list = [credential, hrn]
587         request_hash = self.api.key.compute_hash(arg_list)
588         for aggregate in aggregates:
589             try:
590                 aggregates[aggregate].stop_slice(credential, hrn)
591             except:  
592                 aggregates[aggregate].stop_slice(credential, hrn, request_hash)
593