1 from types import StringTypes
2 from collections import defaultdict
4 from sfa.util.xrn import get_leaf, get_authority, urn_to_hrn
5 from sfa.util.plxrn import hrn_to_pl_slicename
6 from sfa.util.policy import Policy
7 from sfa.rspecs.rspec import RSpec
8 from sfa.plc.vlink import VLink
9 from sfa.util.xrn import Xrn
10 from sfa.util.sfalogging import logger
12 from sqlalchemy import Column, Integer, String, DateTime
13 from sqlalchemy import Table, Column, MetaData, join, ForeignKey
14 from sfa.storage.model import RegRecord
15 from sfa.storage.alchemy import dbsession,engine
21 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
23 #def __init__(self, api, ttl = .5, origin_hrn=None):
25 ##filepath = path + os.sep + filename
26 #self.policy = Policy(self.api)
27 #self.origin_hrn = origin_hrn
28 #self.registry = api.registries[api.hrn]
29 #self.credential = api.getCredential()
34 def __init__(self, driver):
38 def get_slivers(self, xrn, node=None):
39 hrn, type = urn_to_hrn(xrn)
41 slice_name = hrn_to_pl_slicename(hrn)
42 # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
43 # of doing all of this?
44 #return self.api.driver.GetSliceTicket(self.auth, slice_name)
48 slice = self.driver.GetSlices(slice_filter = slice_name, slice_filter_type = 'slice_hrn')
51 # Get user information
52 alchemy_person = dbsession.query(RegRecord).filter_by(record_id = slice['record_id_user']).first()
55 sliver_attributes = []
57 if slice['oar_job_id'] is not -1:
58 nodes_all = self.GetNodes({'hostname':slice['node_ids']},
59 ['node_id', 'hostname','site','boot_state'])
60 nodeall_byhostname = dict([(n['hostname'], n) for n in nodes_all])
61 nodes = slice['node_ids']
64 #for sliver_attribute in filter(lambda a: a['node_id'] == node['node_id'], slice_tags):
65 sliver_attribute['tagname'] = 'slab-tag'
66 sliver_attribute['value'] = 'slab-value'
67 sliver_attributes.append(sliver_attribute['tagname'])
68 attributes.append({'tagname': sliver_attribute['tagname'],
69 'value': sliver_attribute['value']})
71 # set nodegroup slice attributes
72 for slice_tag in filter(lambda a: a['nodegroup_id'] in node['nodegroup_ids'], slice_tags):
73 # Do not set any nodegroup slice attributes for
74 # which there is at least one sliver attribute
76 if slice_tag not in slice_tags:
77 attributes.append({'tagname': slice_tag['tagname'],
78 'value': slice_tag['value']})
80 for slice_tag in filter(lambda a: a['node_id'] is None, slice_tags):
81 # Do not set any global slice attributes for
82 # which there is at least one sliver attribute
84 if slice_tag['tagname'] not in sliver_attributes:
85 attributes.append({'tagname': slice_tag['tagname'],
86 'value': slice_tag['value']})
88 # XXX Sanity check; though technically this should be a system invariant
89 # checked with an assertion
90 if slice['expires'] > MAXINT: slice['expires']= MAXINT
94 'name': slice['name'],
95 'slice_id': slice['slice_id'],
96 'instantiation': slice['instantiation'],
97 'expires': slice['expires'],
99 'attributes': attributes
109 def get_peer(self, xrn):
110 hrn, type = urn_to_hrn(xrn)
111 #Does this slice belong to a local site or a peer senslab site?
114 # get this slice's authority (site)
115 slice_authority = get_authority(hrn)
116 site_authority = slice_authority
117 # get this site's authority (sfa root authority or sub authority)
118 #site_authority = get_authority(slice_authority).lower()
119 print>>sys.stderr, " \r\n \r\n \t slices.py get_peer slice_authority %s site_authority %s hrn %s" %(slice_authority, site_authority, hrn)
120 # check if we are already peered with this site_authority, if so
121 #peers = self.driver.GetPeers({})
122 peers = self.driver.GetPeers(peer_filter = slice_authority)
123 for peer_record in peers:
125 if site_authority == peer_record.hrn:
127 print>>sys.stderr, " \r\n \r\n \t slices.py get_peerAPRES Mpeer %s " %(peer)
130 def get_sfa_peer(self, xrn):
131 hrn, type = urn_to_hrn(xrn)
133 # return the authority for this hrn or None if we are the authority
135 slice_authority = get_authority(hrn)
136 site_authority = get_authority(slice_authority)
138 if site_authority != self.driver.hrn:
139 sfa_peer = site_authority
143 def verify_slice_nodes(self, slice, requested_slivers, peer):
147 if slice['node_ids']:
148 nodes = self.driver.GetNodes(slice['node_ids'], ['hostname'])
149 current_slivers = [node['hostname'] for node in nodes]
151 # remove nodes not in rspec
152 deleted_nodes = list(set(current_slivers).difference(requested_slivers))
154 # add nodes from rspec
155 added_nodes = list(set(requested_slivers).difference(current_slivers))
158 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
159 #PI is a list, get the only username in this list
160 #so that the OAR/LDAP knows the user: remove the authority from the name
161 tmp= slice['PI'][0].split(".")
162 username = tmp[(len(tmp)-1)]
163 #Update the table with the nodes that populate the slice
164 self.driver.db.update_job(slice['name'],nodes = added_nodes)
165 print>>sys.stderr, "\r\n \\r\n \r\n \t\t\t VERIFY_SLICE_NODES slice %s \r\n \r\n \r\n " %(slice)
166 #If there is a timeslot specified, then a job can be launched
168 slot = slice['timeslot']
169 self.driver.LaunchExperimentOnOAR(slice, added_nodes, username)
175 self.driver.DeleteSliceFromNodes(slice['name'], deleted_nodes)
178 logger.log_exc('Failed to add/remove slice from nodes')
181 def free_egre_key(self):
183 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
184 used.add(int(tag['value']))
186 for i in range(1, 256):
191 raise KeyError("No more EGRE keys available")
200 def handle_peer(self, site, slice, persons, peer):
205 self.driver.BindObjectToPeer('site', site['site_id'], peer['shortname'], slice['site_id'])
207 self.driver.DeleteSite(site['site_id'])
213 self.driver.BindObjectToPeer('slice', slice['slice_id'], peer['shortname'], slice['slice_id'])
215 self.driver.DeleteSlice(slice['slice_id'])
219 for person in persons:
221 self.driver.BindObjectToPeer('person',
222 person['person_id'], peer['shortname'], person['peer_person_id'])
224 for (key, remote_key_id) in zip(person['keys'], person['key_ids']):
226 self.driver.BindObjectToPeer( 'key', key['key_id'], peer['shortname'], remote_key_id)
228 self.driver.DeleteKey(key['key_id'])
229 logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname']))
231 self.driver.DeletePerson(person['person_id'])
236 #def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}):
237 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
238 #site_hrn = get_authority(slice_hrn)
239 ## login base can't be longer than 20 characters
240 ##slicename = hrn_to_pl_slicename(slice_hrn)
241 #authority_name = slice_hrn.split('.')[0]
242 #login_base = authority_name[:20]
243 #print >>sys.stderr, " \r\n \r\n \t\t SLABSLICES.PY verify_site authority_name %s login_base %s slice_hrn %s" %(authority_name,login_base,slice_hrn)
245 #sites = self.driver.GetSites(login_base)
247 ## create new site record
248 #site = {'name': 'geni.%s' % authority_name,
249 #'abbreviated_name': authority_name,
250 #'login_base': login_base,
252 #'max_slivers': 1000,
254 #'peer_site_id': None}
256 #site['peer_site_id'] = slice_record.get('site_id', None)
257 #site['site_id'] = self.driver.AddSite(site)
258 ## exempt federated sites from monitor policies
259 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101")
261 ### is this still necessary?
262 ### add record to the local registry
263 ##if sfa_peer and slice_record:
264 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
265 ##'peer_authority': sfa_peer, 'pointer': site['site_id']}
266 ##self.registry.register_peer_object(self.credential, peer_dict)
270 ## unbind from peer so we can modify if necessary. Will bind back later
271 #self.driver.UnBindObjectFromPeer('site', site['site_id'], peer['shortname'])
275 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, options={} ):
277 login_base = slice_hrn.split(".")[0]
278 slicename = slice_hrn
279 sl = self.driver.GetSlices(slice_filter=slicename, slice_filter_type = 'slice_hrn')
282 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice slicename %s sl %s slice_record %s"%(slicename ,sl, slice_record)
284 slice.update(slice_record)
285 #del slice['last_updated']
286 #del slice['date_created']
288 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
289 ## unbind from peer so we can modify if necessary. Will bind back later
290 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
291 #Update existing record (e.g. expires field) it with the latest info.
292 ##if slice_record and slice['expires'] != slice_record['expires']:
293 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : slice_record['expires']})
295 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice UH-Oh...slice_record %s peer %s sfa_peer %s "%(slice_record, peer,sfa_peer)
296 slice = {'slice_hrn': slicename,
297 #'url': slice_record.get('url', slice_hrn),
298 #'description': slice_record.get('description', slice_hrn)
300 'record_id_user' : slice_record['person_ids'][0],
301 'record_id_slice': slice_record['record_id'],
302 'peer_authority':str(peer.hrn)
306 self.driver.AddSlice(slice)
307 #slice['slice_id'] = self.driver.AddSlice(slice)
308 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice ADDSLICE OHYEEEEEEEEEEAH! "
309 #slice['node_ids']=[]
310 #slice['person_ids'] = []
312 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
313 # mark this slice as an sfa peer record
315 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
316 #'peer_authority': sfa_peer, 'pointer': slice['slice_id']}
317 #self.registry.register_peer_object(self.credential, peer_dict)
324 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}):
331 if 'urn' in user and (not 'hrn' in user ) :
332 user['hrn'],user['type'] = urn_to_hrn(user['urn'])
334 if 'person_id' in user and 'hrn' in user:
335 users_by_id[user['person_id']] = user
336 users_dict[user['person_id']] = {'person_id':user['person_id'], 'hrn':user['hrn']}
338 users_by_hrn[user['hrn']] = user
339 users_dict[user['hrn']] = {'person_id':user['person_id'], 'hrn':user['hrn']}
341 print>>sys.stderr, " \r\n \r\n \t slabslices.py verify_person users_dict %s \r\n user_by_hrn %s \r\n \tusers_by_id %s " %( users_dict,users_by_hrn, users_by_id)
343 existing_user_ids = []
344 existing_user_hrns = []
346 #Check if user is in LDAP using its hrn.
347 #Assuming Senslab is centralised : one LDAP for all sites, user_id unknown from LDAP
348 # LDAP does not provide users id, therfore we rely on hrns
350 #Construct the list of filters for GetPersons
352 for hrn in users_by_hrn:
353 filter_user.append ( {'hrn':hrn})
354 existing_users = self.driver.GetPersons(filter_user)
355 #existing_users = self.driver.GetPersons({'hrn': users_by_hrn.keys()})
356 #existing_users = self.driver.GetPersons({'hrn': users_by_hrn.keys()},
359 for user in existing_users :
360 #for k in users_dict[user['hrn']] :
361 existing_user_hrns.append (users_dict[user['hrn']]['hrn'])
362 existing_user_ids.append (users_dict[user['hrn']]['person_id'])
364 #User from another federated site , does not have a senslab account yet?
365 #or have multiple SFA accounts
366 #Check before adding them to LDAP
370 if isinstance(users,list):
371 ldap_reslt = self.driver.ldap.LdapSearch(users[0])
373 ldap_reslt = self.driver.ldap.LdapSearch(users)
375 existing_users = ldap_reslt[0]
376 existing_user_hrns.append (users_dict[user['hrn']]['hrn'])
377 existing_user_ids.append (users_dict[user['hrn']]['person_id'])
379 #User not existing in LDAP
381 print>>sys.stderr, " \r\n \r\n \t slabslices.py verify_person users HUMHUMHUMHUM ... %s \r\n \t ldap_reslt %s " %(users, ldap_reslt)
384 # requested slice users
385 requested_user_ids = users_by_id.keys()
386 requested_user_hrns = users_by_hrn.keys()
387 print>>sys.stderr, " \r\n \r\n \t slabslices.py verify_person requested_user_ids %s user_by_hrn %s " %( requested_user_ids,users_by_hrn)
388 # existing slice users
389 existing_slice_users_filter = {'hrn': slice_record['PI'][0]}
390 print>>sys.stderr, " \r\n \r\n slices.py verify_person requested_user_ids %s existing_slice_users_filter %s slice_record %s" %(requested_user_ids,existing_slice_users_filter,slice_record)
392 existing_slice_users = self.driver.GetPersons([existing_slice_users_filter])
393 #existing_slice_users = self.driver.GetPersons(existing_slice_users_filter,['hrn','pkey'])
394 print>>sys.stderr, " \r\n \r\n slices.py verify_person existing_slice_users %s " %(existing_slice_users)
396 existing_slice_user_hrns = [user['hrn'] for user in existing_slice_users]
398 # users to be added, removed or updated
400 added_user_hrns = set(requested_user_hrns).difference(set(existing_user_hrns))
402 added_slice_user_hrns = set(requested_user_hrns).difference(existing_slice_user_hrns)
404 removed_user_hrns = set(existing_slice_user_hrns).difference(requested_user_hrns)
407 updated_user_hrns = set(existing_slice_user_hrns).intersection(requested_user_hrns)
408 # Remove stale users (only if we are not appending)
409 append = options.get('append', True)
411 for removed_user_hrn in removed_user_hrns:
412 self.driver.DeletePersonFromSlice(removed_user_hrn, slice_record['name'])
413 # update_existing users
414 updated_users_list = [user for user in existing_slice_users if user['hrn'] in \
416 #self.verify_keys(existing_slice_users, updated_users_list, peer, append)
420 for added_user_hrn in added_user_hrns:
421 added_user = users_dict[added_user_hrn]
422 #hrn, type = urn_to_hrn(added_user['urn'])
424 #'first_name': added_user.get('first_name', hrn),
425 #'last_name': added_user.get('last_name', hrn),
426 'person_id': added_user['person_id'],
427 #'peer_person_id': None,
429 #'key_ids': added_user.get('key_ids', []),
432 person['person_id'] = self.driver.AddPerson(person)
434 person['peer_person_id'] = added_user['person_id']
435 added_persons.append(person)
438 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
441 #self.driver.AddPersonToSite(added_user_id, login_base)
443 #for key_string in added_user.get('keys', []):
444 #key = {'key':key_string, 'key_type':'ssh'}
445 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], key)
446 #person['keys'].append(key)
448 # add the registry record
450 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \
451 #'pointer': person['person_id']}
452 #self.registry.register_peer_object(self.credential, peer_dict)
453 for added_slice_user_hrn in added_slice_user_hrns.union(added_user_hrns):
454 self.driver.AddPersonToSlice(added_slice_user_hrn, slice_record['name'])
455 #for added_slice_user_id in added_slice_user_ids.union(added_user_ids):
456 # add person to the slice
457 #self.driver.AddPersonToSlice(added_slice_user_id, slice_record['name'])
458 # if this is a peer record then it should already be bound to a peer.
459 # no need to return worry about it getting bound later
464 def verify_keys(self, persons, users, peer, options={}):
467 for person in persons:
468 key_ids.extend(person['key_ids'])
469 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
472 keydict[key['key']] = key['key_id']
473 existing_keys = keydict.keys()
475 for person in persons:
476 persondict[person['email']] = person
482 user_keys = user.get('keys', [])
483 updated_persons.append(user)
484 for key_string in user_keys:
485 requested_keys.append(key_string)
486 if key_string not in existing_keys:
487 key = {'key': key_string, 'key_type': 'ssh'}
490 person = persondict[user['email']]
491 self.driver.UnBindObjectFromPeer('person', person['person_id'], peer['shortname'])
492 key['key_id'] = self.driver.AddPersonKey(user['email'], key)
494 key_index = user_keys.index(key['key'])
495 remote_key_id = user['key_ids'][key_index]
496 self.driver.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id)
500 self.driver.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id'])
502 # remove old keys (only if we are not appending)
504 removed_keys = set(existing_keys).difference(requested_keys)
505 for existing_key_id in keydict:
506 if keydict[existing_key_id] in removed_keys:
509 self.driver.UnBindObjectFromPeer('key', existing_key_id, peer['shortname'])
510 self.driver.DeleteKey(existing_key_id)
514 #def verify_slice_attributes(self, slice, requested_slice_attributes, append=False, admin=False):
515 ## get list of attributes users ar able to manage
516 #filter = {'category': '*slice*'}
518 #filter['|roles'] = ['user']
519 #slice_attributes = self.driver.GetTagTypes(filter)
520 #valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes]
522 ## get sliver attributes
523 #added_slice_attributes = []
524 #removed_slice_attributes = []
525 #ignored_slice_attribute_names = []
526 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': slice['slice_id']})
528 ## get attributes that should be removed
529 #for slice_tag in existing_slice_attributes:
530 #if slice_tag['tagname'] in ignored_slice_attribute_names:
531 ## If a slice already has a admin only role it was probably given to them by an
532 ## admin, so we should ignore it.
533 #ignored_slice_attribute_names.append(slice_tag['tagname'])
535 ## If an existing slice attribute was not found in the request it should
537 #attribute_found=False
538 #for requested_attribute in requested_slice_attributes:
539 #if requested_attribute['name'] == slice_tag['tagname'] and \
540 #requested_attribute['value'] == slice_tag['value']:
541 #attribute_found=True
544 #if not attribute_found and not append:
545 #removed_slice_attributes.append(slice_tag)
547 ## get attributes that should be added:
548 #for requested_attribute in requested_slice_attributes:
549 ## if the requested attribute wasn't found we should add it
550 #if requested_attribute['name'] in valid_slice_attribute_names:
551 #attribute_found = False
552 #for existing_attribute in existing_slice_attributes:
553 #if requested_attribute['name'] == existing_attribute['tagname'] and \
554 #requested_attribute['value'] == existing_attribute['value']:
555 #attribute_found=True
557 #if not attribute_found:
558 #added_slice_attributes.append(requested_attribute)
561 ## remove stale attributes
562 #for attribute in removed_slice_attributes:
564 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
565 #except Exception, e:
566 #self.logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
567 #% (name, value, node_id, str(e)))
569 ## add requested_attributes
570 #for attribute in added_slice_attributes:
572 #self.driver.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None))
573 #except Exception, e:
574 #self.logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
575 #% (name, value, node_id, str(e)))
577 #def create_slice_aggregate(self, xrn, rspec):
578 #hrn, type = urn_to_hrn(xrn)
579 ## Determine if this is a peer slice
580 #peer = self.get_peer(hrn)
581 #sfa_peer = self.get_sfa_peer(hrn)
584 ## Get the slice record from sfa
585 #slicename = hrn_to_pl_slicename(hrn)
588 #registry = self.api.registries[self.api.hrn]
589 #credential = self.api.getCredential()
591 #site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer)
592 #slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer)
594 ## find out where this slice is currently running
595 #nodelist = self.driver.GetNodes(slice['node_ids'], ['hostname'])
596 #hostnames = [node['hostname'] for node in nodelist]
598 ## get netspec details
599 #nodespecs = spec.getDictsByTagName('NodeSpec')
601 ## dict in which to store slice attributes to set for the nodes
603 #for nodespec in nodespecs:
604 #if isinstance(nodespec['name'], list):
605 #for nodename in nodespec['name']:
606 #nodes[nodename] = {}
607 #for k in nodespec.keys():
608 #rspec_attribute_value = nodespec[k]
609 #if (self.rspec_to_slice_tag.has_key(k)):
610 #slice_tag_name = self.rspec_to_slice_tag[k]
611 #nodes[nodename][slice_tag_name] = rspec_attribute_value
612 #elif isinstance(nodespec['name'], StringTypes):
613 #nodename = nodespec['name']
614 #nodes[nodename] = {}
615 #for k in nodespec.keys():
616 #rspec_attribute_value = nodespec[k]
617 #if (self.rspec_to_slice_tag.has_key(k)):
618 #slice_tag_name = self.rspec_to_slice_tag[k]
619 #nodes[nodename][slice_tag_name] = rspec_attribute_value
621 #for k in nodespec.keys():
622 #rspec_attribute_value = nodespec[k]
623 #if (self.rspec_to_slice_tag.has_key(k)):
624 #slice_tag_name = self.rspec_to_slice_tag[k]
625 #nodes[nodename][slice_tag_name] = rspec_attribute_value
627 #node_names = nodes.keys()
628 ## remove nodes not in rspec
629 #deleted_nodes = list(set(hostnames).difference(node_names))
630 ## add nodes from rspec
631 #added_nodes = list(set(node_names).difference(hostnames))
635 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer)
637 #self.driver.LaunchExperimentOnOAR(slicename, added_nodes)
639 ## Add recognized slice tags
640 #for node_name in node_names:
641 #node = nodes[node_name]
642 #for slice_tag in node.keys():
643 #value = node[slice_tag]
644 #if (isinstance(value, list)):
647 #self.driver.AddSliceTag(slicename, slice_tag, value, node_name)
649 #self.driver.DeleteSliceFromNodes(slicename, deleted_nodes)
652 #self.driver.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id'])