9 from types import StringTypes
10 from sfa.util.misc import *
11 from sfa.util.rspec import *
12 from sfa.util.specdict import *
13 from sfa.util.faults import *
14 from sfa.util.storage import *
15 from sfa.util.record import GeniRecord
16 from sfa.util.policy import Policy
17 from sfa.util.prefixTree import prefixTree
18 from sfa.util.debug import log
19 from sfa.server.aggregate import Aggregates
20 from sfa.server.registry import Registries
22 class Slices(SimpleStorage):
24 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
26 def __init__(self, api, ttl = .5, caller_cred=None):
30 path = self.api.config.SFA_DATA_DIR
31 filename = ".".join([self.api.interface, self.api.hrn, "slices"])
32 filepath = path + os.sep + filename
33 self.slices_file = filepath
34 SimpleStorage.__init__(self, self.slices_file)
35 self.policy = Policy(self.api)
37 self.caller_cred=caller_cred
39 def get_slivers(self, hrn):
41 slice_name = hrn_to_pl_slicename(hrn)
42 # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
43 # of doing all of this?
44 #return self.api.GetSliceTicket(self.auth, slice_name)
46 # from PLCAPI.GetSlivers.get_slivers()
47 slice_fields = ['slice_id', 'name', 'instantiation', 'expires', 'person_ids', 'slice_tag_ids'])
48 slices = self.api.GetSlices(self.auth, slice_name, slice_fields)
49 # Build up list of users and slice attributes
51 all_slice_tag_ids = set()
53 person_ids.update(slice['person_ids'])
54 slice_tag_ids.update(slice['slice_tag_ids'])
56 # Get user information
57 all_persons = Persons(api, {'person_id':person_ids,'enabled':True}, ['person_id', 'enabled', 'key_ids']).dict()
59 # Build up list of keys
61 for person in all_persons.values():
62 key_ids.update(person['key_ids'])
64 # Get user account keys
65 all_keys = Keys(api, key_ids, ['key_id', 'key', 'key_type']).dict()
67 # Get slice attributes
68 all_slice_tags = SliceTags(api, slice_tag_ids).dict()
73 for person_id in slice['person_ids']:
74 if person_id in all_persons:
75 person = all_persons[person_id]
76 if not person['enabled']:
78 for key_id in person['key_ids']:
79 if key_id in all_keys:
80 key = all_keys[key_id]
81 keys += [{'key_type': key['key_type'],
84 # All (per-node and global) attributes for this slice
86 for slice_tag_id in slice['slice_tag_ids']:
87 if slice_tag_id in all_slice_tags:
88 slice_tags.append(all_slice_tags[slice_tag_id])
89 # Per-node sliver attributes take precedence over global
90 # slice attributes, so set them first.
91 # Then comes nodegroup slice attributes
92 # Followed by global slice attributes
93 sliver_attributes = []
96 for sliver_attribute in filter(lambda a: a['node_id'] == node['node_id'], slice_tags):
97 sliver_attributes.append(sliver_attribute['tagname'])
98 attributes.append({'tagname': sliver_attribute['tagname'],
99 'value': sliver_attribute['value']})
101 # set nodegroup slice attributes
102 for slice_tag in filter(lambda a: a['nodegroup_id'] in node['nodegroup_ids'], slice_tags):
103 # Do not set any nodegroup slice attributes for
104 # which there is at least one sliver attribute
106 if slice_tag not in slice_tags:
107 attributes.append({'tagname': slice_tag['tagname'],
108 'value': slice_tag['value']})
110 for slice_tag in filter(lambda a: a['node_id'] is None, slice_tags):
111 # Do not set any global slice attributes for
112 # which there is at least one sliver attribute
114 if slice_tag['tagname'] not in sliver_attributes:
115 attributes.append({'tagname': slice_tag['tagname'],
116 'value': slice_tag['value']})
118 # XXX Sanity check; though technically this should be a system invariant
119 # checked with an assertion
120 if slice['expires'] > MAXINT: slice['expires']= MAXINT
124 'name': slice['name'],
125 'slice_id': slice['slice_id'],
126 'instantiation': slice['instantiation'],
127 'expires': slice['expires'],
129 'attributes': attributes
134 def get_peer(self, hrn):
135 # Becaues of myplc federation, we first need to determine if this
136 # slice belongs to out local plc or a myplc peer. We will assume it
137 # is a local site, unless we find out otherwise
140 # get this slice's authority (site)
141 slice_authority = get_authority(hrn)
143 # get this site's authority (sfa root authority or sub authority)
144 site_authority = get_authority(slice_authority).lower()
146 # check if we are already peered with this site_authority, if so
147 peers = self.api.plshell.GetPeers(self.api.plauth, {}, ['peer_id', 'peername', 'shortname', 'hrn_root'])
148 for peer_record in peers:
149 names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)]
150 if site_authority in names:
151 peer = peer_record['shortname']
155 def get_sfa_peer(self, hrn):
156 # return the authority for this hrn or None if we are the authority
158 slice_authority = get_authority(hrn)
159 site_authority = get_authority(slice_authority)
161 if site_authority != self.api.hrn:
162 sfa_peer = site_authority
168 Update the cached list of slices
170 # Reload components list
171 now = datetime.datetime.now()
172 if not self.has_key('threshold') or not self.has_key('timestamp') or \
173 now > datetime.datetime.fromtimestamp(time.mktime(time.strptime(self['threshold'], self.api.time_format))):
174 if self.api.interface in ['aggregate']:
175 self.refresh_slices_aggregate()
176 elif self.api.interface in ['slicemgr']:
177 self.refresh_slices_smgr()
179 def refresh_slices_aggregate(self):
180 slices = self.api.plshell.GetSlices(self.api.plauth, {'peer_id': None}, ['name'])
181 slice_hrns = [slicename_to_hrn(self.api.hrn, slice['name']) for slice in slices]
183 # update timestamp and threshold
184 timestamp = datetime.datetime.now()
185 hr_timestamp = timestamp.strftime(self.api.time_format)
186 delta = datetime.timedelta(hours=self.ttl)
187 threshold = timestamp + delta
188 hr_threshold = threshold.strftime(self.api.time_format)
190 slice_details = {'hrn': slice_hrns,
191 'timestamp': hr_timestamp,
192 'threshold': hr_threshold
194 self.update(slice_details)
198 def refresh_slices_smgr(self):
200 aggregates = Aggregates(self.api)
201 credential = self.api.getCredential()
202 for aggregate in aggregates:
204 # request hash is optional so lets try the call without it
206 slices = aggregates[aggregate].get_slices(credential)
207 slice_hrns.extend(slices)
210 print >> log, "%s" % (traceback.format_exc())
211 print >> log, "Error calling slices at aggregate %(aggregate)s" % locals()
213 # try sending the request hash if the previous call failed
215 arg_list = [credential]
216 request_hash = self.api.key.compute_hash(arg_list)
218 slices = aggregates[aggregate].get_slices(credential, request_hash)
219 slice_hrns.extend(slices)
222 print >> log, "%s" % (traceback.format_exc())
223 print >> log, "Error calling slices at aggregate %(aggregate)s" % locals()
225 # update timestamp and threshold
226 timestamp = datetime.datetime.now()
227 hr_timestamp = timestamp.strftime(self.api.time_format)
228 delta = datetime.timedelta(hours=self.ttl)
229 threshold = timestamp + delta
230 hr_threshold = threshold.strftime(self.api.time_format)
232 slice_details = {'hrn': slice_hrns,
233 'timestamp': hr_timestamp,
234 'threshold': hr_threshold
236 self.update(slice_details)
240 def delete_slice(self, hrn):
241 if self.api.interface in ['aggregate']:
242 self.delete_slice_aggregate(hrn)
243 elif self.api.interface in ['slicemgr']:
244 self.delete_slice_smgr(hrn)
246 def delete_slice_aggregate(self, hrn):
248 slicename = hrn_to_pl_slicename(hrn)
249 slices = self.api.plshell.GetSlices(self.api.plauth, {'name': slicename})
254 # determine if this is a peer slice
255 peer = self.get_peer(hrn)
257 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', slice['slice_id'], peer)
258 self.api.plshell.DeleteSliceFromNodes(self.api.plauth, slicename, slice['node_ids'])
260 self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice['slice_id'], peer, slice['peer_slice_id'])
263 def delete_slice_smgr(self, hrn):
264 credential = self.api.getCredential()
265 caller_cred = self.caller_cred
266 aggregates = Aggregates(self.api)
267 for aggregate in aggregates:
269 # request hash is optional so lets try the call without it
272 aggregates[aggregate].delete_slice(credential, hrn, request_hash, caller_cred)
275 print >> log, "%s" % (traceback.format_exc())
276 print >> log, "Error calling list nodes at aggregate %s" % aggregate
278 # try sending the request hash if the previous call failed
281 arg_list = [credential, hrn]
282 request_hash = self.api.key.compute_hash(arg_list)
283 aggregates[aggregate].delete_slice(credential, hrn, request_hash, caller_cred)
286 print >> log, "%s" % (traceback.format_exc())
287 print >> log, "Error calling list nodes at aggregate %s" % aggregate
289 def create_slice(self, hrn, rspec):
291 # check our slice policy before we procede
292 whitelist = self.policy['slice_whitelist']
293 blacklist = self.policy['slice_blacklist']
295 if whitelist and hrn not in whitelist or \
296 blacklist and hrn in blacklist:
297 policy_file = self.policy.policy_file
298 print >> log, "Slice %(hrn)s not allowed by policy %(policy_file)s" % locals()
301 if self.api.interface in ['aggregate']:
302 self.create_slice_aggregate(hrn, rspec)
303 elif self.api.interface in ['slicemgr']:
304 self.create_slice_smgr(hrn, rspec)
306 def verify_site(self, registry, credential, slice_hrn, peer, sfa_peer):
307 authority = get_authority(slice_hrn)
309 site_records = registry.resolve(credential, authority)
311 arg_list = [credential, authority]
312 request_hash = self.api.key.compute_hash(arg_list)
313 site_records = registry.resolve(credential, authority, request_hash)
316 for site_record in site_records:
317 if site_record['type'] == 'authority':
320 raise RecordNotFound(authority)
321 remote_site_id = site.pop('site_id')
323 login_base = get_leaf(authority)
324 sites = self.api.plshell.GetSites(self.api.plauth, login_base)
326 site_id = self.api.plshell.AddSite(self.api.plauth, site)
328 self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', site_id, peer, remote_site_id)
329 # mark this site as an sfa peer record
331 peer_dict = {'type': 'authority', 'hrn': authority, 'peer_authority': sfa_peer, 'pointer': site_id}
333 registry.register_peer_object(credential, peer_dict)
335 arg_list = [credential]
336 request_hash = self.api.key.compute_hash(arg_list)
337 registry.register_peer_object(credential, peer_dict, request_hash)
339 site_id = sites[0]['site_id']
340 remote_site_id = sites[0]['peer_site_id']
343 return (site_id, remote_site_id)
345 def verify_slice(self, registry, credential, slice_hrn, site_id, remote_site_id, peer, sfa_peer):
348 authority = get_authority(slice_hrn)
350 slice_records = registry.resolve(credential, slice_hrn)
352 arg_list = [credential, slice_hrn]
353 request_hash = self.api.key.compute_hash(arg_list)
354 slice_records = registry.resolve(credential, slice_hrn, request_hash)
356 for record in slice_records:
357 if record['type'] in ['slice']:
358 slice_record = record
360 raise RecordNotFound(hrn)
361 slicename = hrn_to_pl_slicename(slice_hrn)
362 parts = slicename.split("_")
363 login_base = parts[0]
364 slices = self.api.plshell.GetSlices(self.api.plauth, [slicename], ['slice_id', 'node_ids', 'site_id'])
367 slice_keys = ['name', 'url', 'description']
368 for key in slice_keys:
369 if key in slice_record and slice_record[key]:
370 slice_fields[key] = slice_record[key]
373 slice_id = self.api.plshell.AddSlice(self.api.plauth, slice_fields)
375 slice['slice_id'] = slice_id
377 # mark this slice as an sfa peer record
379 peer_dict = {'type': 'slice', 'hrn': slice_hrn, 'peer_authority': sfa_peer, 'pointer': slice_id}
381 registry.register_peer_object(credential, peer_dict)
383 arg_list = [credential]
384 request_hash = self.api.key.compute_hash(arg_list)
385 registry.register_peer_object(credential, peer_dict, request_hash)
387 #this belongs to a peer
389 self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice_id, peer, slice_record['pointer'])
390 slice['node_ids'] = []
393 slice_id = slice['slice_id']
394 site_id = slice['site_id']
396 slice['peer_slice_id'] = slice_record['pointer']
397 self.verify_persons(registry, credential, slice_record, site_id, remote_site_id, peer, sfa_peer)
401 def verify_persons(self, registry, credential, slice_record, site_id, remote_site_id, peer, sfa_peer):
402 # get the list of valid slice users from the registry and make
403 # sure they are added to the slice
404 slicename = hrn_to_pl_slicename(slice_record['hrn'])
405 researchers = slice_record.get('researcher', [])
406 for researcher in researchers:
409 person_records = registry.resolve(credential, researcher)
411 arg_list = [credential, researcher]
412 request_hash = self.api.key.compute_hash(arg_list)
413 person_records = registry.resolve(credential, researcher, request_hash)
414 for record in person_records:
415 if record['type'] in ['user']:
416 person_record = record
417 if not person_record:
419 person_dict = person_record
422 peer_id = self.api.plshell.GetPeers(self.api.plauth, {'shortname': peer}, ['peer_id'])[0]['peer_id']
423 persons = self.api.plshell.GetPersons(self.api.plauth, {'email': [person_dict['email']], 'peer_id': peer_id}, ['person_id', 'key_ids'])
425 persons = self.api.plshell.GetPersons(self.api.plauth, [person_dict['email']], ['person_id', 'key_ids'])
430 persons = self.api.plshell.GetPersons(self.api.plauth, [person_dict['email']], ['person_id', 'key_ids'])
433 person_id=self.api.plshell.AddPerson(self.api.plauth, person_dict)
434 self.api.plshell.UpdatePerson(self.api.plauth, person_id, {'enabled' : True})
436 # mark this person as an sfa peer record
438 peer_dict = {'type': 'user', 'hrn': researcher, 'peer_authority': sfa_peer, 'pointer': person_id}
440 registry.register_peer_object(credential, peer_dict)
442 arg_list = [credential]
443 request_hash = self.api.key.compute_hash(arg_list)
444 registry.register_peer_object(credential, peer_dict, request_hash)
447 self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
450 person_id = persons[0]['person_id']
451 key_ids = persons[0]['key_ids']
454 # if this is a peer person, we must unbind them from the peer or PLCAPI will throw
457 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'person', person_id, peer)
458 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'site', site_id, peer)
460 self.api.plshell.AddPersonToSlice(self.api.plauth, person_dict['email'], slicename)
461 self.api.plshell.AddPersonToSite(self.api.plauth, person_dict['email'], site_id)
462 if peer and not local_person:
463 self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
465 self.api.plshell.BindObjectToPeer(self.api.plauth, 'site', site_id, peer, remote_site_id)
467 self.verify_keys(registry, credential, person_dict, key_ids, person_id, peer, local_person)
469 def verify_keys(self, registry, credential, person_dict, key_ids, person_id, peer, local_person):
470 keylist = self.api.plshell.GetKeys(self.api.plauth, key_ids, ['key'])
471 keys = [key['key'] for key in keylist]
473 #add keys that arent already there
474 key_ids = person_dict['key_ids']
475 for personkey in person_dict['keys']:
476 if personkey not in keys:
477 key = {'key_type': 'ssh', 'key': personkey}
479 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'person', person_id, peer)
480 key_id = self.api.plshell.AddPersonKey(self.api.plauth, person_dict['email'], key)
481 if peer and not local_person:
482 self.api.plshell.BindObjectToPeer(self.api.plauth, 'person', person_id, peer, person_dict['pointer'])
484 try: self.api.plshell.BindObjectToPeer(self.api.plauth, 'key', key_id, peer, key_ids.pop(0))
488 def create_slice_aggregate(self, hrn, rspec):
490 # Determine if this is a peer slice
491 peer = self.get_peer(hrn)
492 sfa_peer = self.get_sfa_peer(hrn)
495 # Get the slice record from sfa
496 slicename = hrn_to_pl_slicename(hrn)
499 registries = Registries(self.api)
500 registry = registries[self.api.hrn]
501 credential = self.api.getCredential()
503 site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer)
504 slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer)
506 # find out where this slice is currently running
507 nodelist = self.api.plshell.GetNodes(self.api.plauth, slice['node_ids'], ['hostname'])
508 hostnames = [node['hostname'] for node in nodelist]
510 # get netspec details
511 nodespecs = spec.getDictsByTagName('NodeSpec')
513 # dict in which to store slice attributes to set for the nodes
515 for nodespec in nodespecs:
516 if isinstance(nodespec['name'], list):
517 for nodename in nodespec['name']:
519 for k in nodespec.keys():
520 rspec_attribute_value = nodespec[k]
521 if (self.rspec_to_slice_tag.has_key(k)):
522 slice_tag_name = self.rspec_to_slice_tag[k]
523 nodes[nodename][slice_tag_name] = rspec_attribute_value
524 elif isinstance(nodespec['name'], StringTypes):
525 nodename = nodespec['name']
527 for k in nodespec.keys():
528 rspec_attribute_value = nodespec[k]
529 if (self.rspec_to_slice_tag.has_key(k)):
530 slice_tag_name = self.rspec_to_slice_tag[k]
531 nodes[nodename][slice_tag_name] = rspec_attribute_value
533 for k in nodespec.keys():
534 rspec_attribute_value = nodespec[k]
535 if (self.rspec_to_slice_tag.has_key(k)):
536 slice_tag_name = self.rspec_to_slice_tag[k]
537 nodes[nodename][slice_tag_name] = rspec_attribute_value
539 node_names = nodes.keys()
540 # remove nodes not in rspec
541 deleted_nodes = list(set(hostnames).difference(node_names))
542 # add nodes from rspec
543 added_nodes = list(set(node_names).difference(hostnames))
546 self.api.plshell.UnBindObjectFromPeer(self.api.plauth, 'slice', slice['slice_id'], peer)
548 self.api.plshell.AddSliceToNodes(self.api.plauth, slicename, added_nodes)
550 # Add recognized slice tags
551 for node_name in node_names:
552 node = nodes[node_name]
553 for slice_tag in node.keys():
554 value = node[slice_tag]
555 if (isinstance(value, list)):
558 self.api.plshell.AddSliceTag(self.api.plauth, slicename, slice_tag, value, node_name)
560 self.api.plshell.DeleteSliceFromNodes(self.api.plauth, slicename, deleted_nodes)
562 self.api.plshell.BindObjectToPeer(self.api.plauth, 'slice', slice['slice_id'], peer, slice['peer_slice_id'])
566 def create_slice_smgr(self, hrn, rspec):
569 spec.parseString(rspec)
570 slicename = hrn_to_pl_slicename(hrn)
571 specDict = spec.toDict()
572 if specDict.has_key('RSpec'): specDict = specDict['RSpec']
573 if specDict.has_key('start_time'): start_time = specDict['start_time']
575 if specDict.has_key('end_time'): end_time = specDict['end_time']
579 aggregates = Aggregates(self.api)
580 credential = self.api.getCredential()
582 # split the netspecs into individual rspecs
583 netspecs = spec.getDictsByTagName('NetSpec')
584 for netspec in netspecs:
585 net_hrn = netspec['name']
586 resources = {'start_time': start_time, 'end_time': end_time, 'networks': netspec}
587 resourceDict = {'RSpec': resources}
588 tempspec.parseDict(resourceDict)
589 rspecs[net_hrn] = tempspec.toxml()
591 # send each rspec to the appropriate aggregate/sm
592 caller_cred = self.caller_cred
593 for net_hrn in rspecs:
595 # if we are directly connected to the aggregate then we can just send them the rspec
596 # if not, then we may be connected to an sm thats connected to the aggregate
597 if net_hrn in aggregates:
598 # send the whloe rspec to the local aggregate
599 if net_hrn in [self.api.hrn]:
602 aggregates[net_hrn].create_slice(credential, hrn, rspec, request_hash, caller_cred)
604 arg_list = [credential,hrn,rspec]
605 request_hash = self.api.key.compute_hash(arg_list)
606 aggregates[net_hrn].create_slice(credential, hrn, rspec, request_hash, caller_cred)
610 aggregates[net_hrn].create_slice(credential, hrn, rspecs[net_hrn], request_hash, caller_cred)
612 arg_list = [credential,hrn,rspecs[net_hrn]]
613 request_hash = self.api.key.compute_hash(arg_list)
614 aggregates[net_hrn].create_slice(credential, hrn, rspecs[net_hrn], request_hash, caller_cred)
616 # lets forward this rspec to a sm that knows about the network
617 arg_list = [credential, net_hrn]
618 request_hash = self.api.compute_hash(arg_list)
619 for aggregate in aggregates:
621 network_found = aggregates[aggregate].get_aggregates(credential, net_hrn)
623 network_found = aggregates[aggregate].get_aggregates(credential, net_hrn, request_hash)
627 aggregates[aggregate].create_slice(credential, hrn, rspecs[net_hrn], request_hash, caller_cred)
629 arg_list = [credential, hrn, rspecs[net_hrn]]
630 request_hash = self.api.key.compute_hash(arg_list)
631 aggregates[aggregate].create_slice(credential, hrn, rspecs[net_hrn], request_hash, caller_cred)
634 print >> log, "Error creating slice %(hrn)s at aggregate %(net_hrn)s" % locals()
635 traceback.print_exc()
639 def start_slice(self, hrn):
640 if self.api.interface in ['aggregate']:
641 self.start_slice_aggregate(hrn)
642 elif self.api.interface in ['slicemgr']:
643 self.start_slice_smgr(hrn)
645 def start_slice_aggregate(self, hrn):
646 slicename = hrn_to_pl_slicename(hrn)
647 slices = self.api.plshell.GetSlices(self.api.plauth, {'name': slicename}, ['slice_id'])
649 raise RecordNotFound(hrn)
651 attributes = self.api.plshell.GetSliceAttributes(self.api.plauth, {'slice_id': slice_id, 'name': 'enabled'}, ['slice_attribute_id'])
652 attribute_id = attreibutes[0]['slice_attribute_id']
653 self.api.plshell.UpdateSliceAttribute(self.api.plauth, attribute_id, "1" )
656 def start_slice_smgr(self, hrn):
657 credential = self.api.getCredential()
658 aggregates = Aggregates(self.api)
659 for aggregate in aggregates:
660 aggregates[aggregate].start_slice(credential, hrn)
664 def stop_slice(self, hrn):
665 if self.api.interface in ['aggregate']:
666 self.stop_slice_aggregate(hrn)
667 elif self.api.interface in ['slicemgr']:
668 self.stop_slice_smgr(hrn)
670 def stop_slice_aggregate(self, hrn):
671 slicename = hrn_to_pl_slicename(hrn)
672 slices = self.api.plshell.GetSlices(self.api.plauth, {'name': slicename}, ['slice_id'])
674 raise RecordNotFound(hrn)
675 slice_id = slices[0]['slice_id']
676 attributes = self.api.plshell.GetSliceAttributes(self.api.plauth, {'slice_id': slice_id, 'name': 'enabled'}, ['slice_attribute_id'])
677 attribute_id = attributes[0]['slice_attribute_id']
678 self.api.plshell.UpdateSliceAttribute(self.api.plauth, attribute_id, "0")
681 def stop_slice_smgr(self, hrn):
682 credential = self.api.getCredential()
683 aggregates = Aggregates(self.api)
684 arg_list = [credential, hrn]
685 request_hash = self.api.key.compute_hash(arg_list)
686 for aggregate in aggregates:
688 aggregates[aggregate].stop_slice(credential, hrn)
690 aggregates[aggregate].stop_slice(credential, hrn, request_hash)