1 from types import StringTypes
2 from collections import defaultdict
4 from sfa.util.xrn import get_leaf, get_authority, urn_to_hrn
5 from sfa.util.plxrn import hrn_to_pl_slicename
6 from sfa.util.policy import Policy
7 from sfa.rspecs.rspec import RSpec
8 from sfa.plc.vlink import VLink
9 from sfa.util.xrn import Xrn
10 from sfa.util.sfalogging import logger
12 from sqlalchemy import Column, Integer, String, DateTime
13 from sqlalchemy import Table, Column, MetaData, join, ForeignKey
14 from sfa.storage.model import RegRecord
15 from sfa.storage.alchemy import dbsession,engine
21 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
23 #def __init__(self, api, ttl = .5, origin_hrn=None):
25 ##filepath = path + os.sep + filename
26 #self.policy = Policy(self.api)
27 #self.origin_hrn = origin_hrn
28 #self.registry = api.registries[api.hrn]
29 #self.credential = api.getCredential()
34 def __init__(self, driver):
38 def get_slivers(self, xrn, node=None):
39 hrn, type = urn_to_hrn(xrn)
41 slice_name = hrn_to_pl_slicename(hrn)
42 # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
43 # of doing all of this?
44 #return self.api.driver.GetSliceTicket(self.auth, slice_name)
48 slice = self.driver.GetSlices(slice_filter = slice_name, filter_type = 'slice_hrn')
51 # Get user information
52 alchemy_person = dbsession.query(RegRecord).filter_by(record_id = slice['record_id_user']).first()
55 sliver_attributes = []
57 if slice['oar_job_id'] is not -1:
58 nodes_all = self.GetNodes({'hostname':slice['node_ids']},
59 ['node_id', 'hostname','site','boot_state'])
60 nodeall_byhostname = dict([(n['hostname'], n) for n in nodes_all])
61 nodes = slice['node_ids']
64 #for sliver_attribute in filter(lambda a: a['node_id'] == node['node_id'], slice_tags):
65 sliver_attribute['tagname'] = 'slab-tag'
66 sliver_attribute['value'] = 'slab-value'
67 sliver_attributes.append(sliver_attribute['tagname'])
68 attributes.append({'tagname': sliver_attribute['tagname'],
69 'value': sliver_attribute['value']})
71 # set nodegroup slice attributes
72 for slice_tag in filter(lambda a: a['nodegroup_id'] in node['nodegroup_ids'], slice_tags):
73 # Do not set any nodegroup slice attributes for
74 # which there is at least one sliver attribute
76 if slice_tag not in slice_tags:
77 attributes.append({'tagname': slice_tag['tagname'],
78 'value': slice_tag['value']})
80 for slice_tag in filter(lambda a: a['node_id'] is None, slice_tags):
81 # Do not set any global slice attributes for
82 # which there is at least one sliver attribute
84 if slice_tag['tagname'] not in sliver_attributes:
85 attributes.append({'tagname': slice_tag['tagname'],
86 'value': slice_tag['value']})
88 # XXX Sanity check; though technically this should be a system invariant
89 # checked with an assertion
90 if slice['expires'] > MAXINT: slice['expires']= MAXINT
94 'name': slice['name'],
95 'slice_id': slice['slice_id'],
96 'instantiation': slice['instantiation'],
97 'expires': slice['expires'],
99 'attributes': attributes
109 def get_peer(self, xrn):
110 hrn, type = urn_to_hrn(xrn)
111 #Does this slice belong to a local site or a peer senslab site?
114 # get this slice's authority (site)
115 slice_authority = get_authority(hrn)
116 site_authority = slice_authority
117 # get this site's authority (sfa root authority or sub authority)
118 #site_authority = get_authority(slice_authority).lower()
119 print>>sys.stderr, " \r\n \r\n \t slices.py get_peer slice_authority %s site_authority %s hrn %s" %(slice_authority, site_authority, hrn)
120 # check if we are already peered with this site_authority, if so
121 #peers = self.driver.GetPeers({})
122 peers = self.driver.GetPeers(peer_filter = slice_authority)
123 for peer_record in peers:
125 if site_authority == peer_record.hrn:
127 print>>sys.stderr, " \r\n \r\n \t slices.py get_peerAPRES Mpeer %s " %(peer)
130 def get_sfa_peer(self, xrn):
131 hrn, type = urn_to_hrn(xrn)
133 # return the authority for this hrn or None if we are the authority
135 slice_authority = get_authority(hrn)
136 site_authority = get_authority(slice_authority)
138 if site_authority != self.driver.hrn:
139 sfa_peer = site_authority
143 def verify_slice_nodes(self, slice, requested_slivers, peer):
147 if slice['node_ids']:
148 nodes = self.driver.GetNodes(slice['node_ids'], ['hostname'])
149 current_slivers = [node['hostname'] for node in nodes]
151 # remove nodes not in rspec
152 deleted_nodes = list(set(current_slivers).difference(requested_slivers))
154 # add nodes from rspec
155 added_nodes = list(set(requested_slivers).difference(current_slivers))
158 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
159 #PI is a list, get the only username in this list
160 #so that the OAR/LDAP knows the user: remove the authority from the name
161 tmp= slice['PI'][0].split(".")
162 username = tmp[(len(tmp)-1)]
163 #Update the table with the nodes that populate the slice
164 self.driver.db.update_job(slice['name'],nodes = added_nodes)
165 print>>sys.stderr, "\r\n \\r\n \r\n \t\t\t VERIFY_SLICE_NODES slice %s \r\n \r\n \r\n " %(slice)
166 #If there is a timeslot specified, then a job can be launched
168 slot = slice['timeslot']
169 self.driver.LaunchExperimentOnOAR(slice, added_nodes, username)
175 self.driver.DeleteSliceFromNodes(slice['name'], deleted_nodes)
178 logger.log_exc('Failed to add/remove slice from nodes')
181 def free_egre_key(self):
183 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
184 used.add(int(tag['value']))
186 for i in range(1, 256):
191 raise KeyError("No more EGRE keys available")
200 def handle_peer(self, site, slice, persons, peer):
205 self.driver.BindObjectToPeer('site', site['site_id'], peer['shortname'], slice['site_id'])
207 self.driver.DeleteSite(site['site_id'])
213 self.driver.BindObjectToPeer('slice', slice['slice_id'], peer['shortname'], slice['slice_id'])
215 self.driver.DeleteSlice(slice['slice_id'])
219 for person in persons:
221 self.driver.BindObjectToPeer('person',
222 person['person_id'], peer['shortname'], person['peer_person_id'])
224 for (key, remote_key_id) in zip(person['keys'], person['key_ids']):
226 self.driver.BindObjectToPeer( 'key', key['key_id'], peer['shortname'], remote_key_id)
228 self.driver.DeleteKey(key['key_id'])
229 logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname']))
231 self.driver.DeletePerson(person['person_id'])
236 #def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}):
237 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
238 #site_hrn = get_authority(slice_hrn)
239 ## login base can't be longer than 20 characters
240 ##slicename = hrn_to_pl_slicename(slice_hrn)
241 #authority_name = slice_hrn.split('.')[0]
242 #login_base = authority_name[:20]
243 #print >>sys.stderr, " \r\n \r\n \t\t SLABSLICES.PY verify_site authority_name %s login_base %s slice_hrn %s" %(authority_name,login_base,slice_hrn)
245 #sites = self.driver.GetSites(login_base)
247 ## create new site record
248 #site = {'name': 'geni.%s' % authority_name,
249 #'abbreviated_name': authority_name,
250 #'login_base': login_base,
252 #'max_slivers': 1000,
254 #'peer_site_id': None}
256 #site['peer_site_id'] = slice_record.get('site_id', None)
257 #site['site_id'] = self.driver.AddSite(site)
258 ## exempt federated sites from monitor policies
259 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101")
261 ### is this still necessary?
262 ### add record to the local registry
263 ##if sfa_peer and slice_record:
264 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
265 ##'peer_authority': sfa_peer, 'pointer': site['site_id']}
266 ##self.registry.register_peer_object(self.credential, peer_dict)
270 ## unbind from peer so we can modify if necessary. Will bind back later
271 #self.driver.UnBindObjectFromPeer('site', site['site_id'], peer['shortname'])
275 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, options={} ):
277 login_base = slice_hrn.split(".")[0]
278 slicename = slice_hrn
279 sl = self.driver.GetSlices(slice_filter=slicename, filter_type = 'slice_hrn')
282 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice slicename %s sl %s slice_record %s"%(slicename ,sl, slice_record)
284 slice.update(slice_record)
285 #del slice['last_updated']
286 #del slice['date_created']
288 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
289 ## unbind from peer so we can modify if necessary. Will bind back later
290 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
291 #Update existing record (e.g. expires field) it with the latest info.
292 ##if slice_record and slice['expires'] != slice_record['expires']:
293 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : slice_record['expires']})
295 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice UH-Oh...slice_record %s peer %s sfa_peer %s "%(slice_record, peer,sfa_peer)
296 slice = {'slice_hrn': slicename,
297 #'url': slice_record.get('url', slice_hrn),
298 #'description': slice_record.get('description', slice_hrn)
300 'record_id_user' : slice_record['person_ids'][0],
301 'record_id_slice': slice_record['record_id'],
302 'peer_authority':str(peer.hrn)
306 self.driver.AddSlice(slice)
307 #slice['slice_id'] = self.driver.AddSlice(slice)
308 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice ADDSLICE OHYEEEEEEEEEEAH! "
309 #slice['node_ids']=[]
310 #slice['person_ids'] = []
312 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
313 # mark this slice as an sfa peer record
315 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
316 #'peer_authority': sfa_peer, 'pointer': slice['slice_id']}
317 #self.registry.register_peer_object(self.credential, peer_dict)
324 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}):
331 if 'urn' in user and (not 'hrn' in user ) :
332 user['hrn'],user['type'] = urn_to_hrn(user['urn'])
334 if 'person_id' in user and 'hrn' in user:
335 users_by_id[user['person_id']] = user
336 users_dict[user['person_id']] = {'person_id':user['person_id'], 'hrn':user['hrn']}
338 users_by_hrn[user['hrn']] = user
339 users_dict[user['hrn']] = {'person_id':user['person_id'], 'hrn':user['hrn']}
341 print>>sys.stderr, " \r\n \r\n \t slabslices.py verify_person users_dict %s \r\n user_by_hrn %s \r\n \tusers_by_id %s " %( users_dict,users_by_hrn, users_by_id)
343 existing_user_ids = []
344 existing_user_hrns = []
346 #Check if user is in LDAP using its hrn.
347 #Assuming Senslab is centralised : one LDAP for all sites, user_id unknown from LDAP
348 # LDAP does not provide users id, therfore we rely on hrns
350 existing_users = self.driver.GetPersons({'hrn': users_by_hrn.keys()})
351 #existing_users = self.driver.GetPersons({'hrn': users_by_hrn.keys()},
354 for user in existing_users :
355 #for k in users_dict[user['hrn']] :
356 existing_user_hrns.append (users_dict[user['hrn']]['hrn'])
357 existing_user_ids.append (users_dict[user['hrn']]['person_id'])
359 #User from another federated site , does not have a senslab account yet?
360 #or have multiple SFA accounts
361 #Check before adding them to LDAP
365 if isinstance(users,list):
366 ldap_reslt = self.driver.ldap.ldapSearch(users[0])
368 ldap_reslt = self.driver.ldap.ldapSearch(users)
370 existing_users = ldap_reslt[0]
371 existing_user_hrns.append (users_dict[user['hrn']]['hrn'])
372 existing_user_ids.append (users_dict[user['hrn']]['person_id'])
374 #User not existing in LDAP
376 print>>sys.stderr, " \r\n \r\n \t slabslices.py verify_person users HUMHUMHUMHUM ... %s \r\n \t ldap_reslt %s " %(users, ldap_reslt)
379 # requested slice users
380 requested_user_ids = users_by_id.keys()
381 requested_user_hrns = users_by_hrn.keys()
382 print>>sys.stderr, " \r\n \r\n \t slabslices.py verify_person requested_user_ids %s user_by_hrn %s " %( requested_user_ids,users_by_hrn)
383 # existing slice users
384 existing_slice_users_filter = {'hrn': slice_record.get('PI', [])}
385 print>>sys.stderr, " \r\n \r\n slices.py verify_person requested_user_ids %s existing_slice_users_filter %s slice_record %s" %(requested_user_ids,existing_slice_users_filter,slice_record)
387 existing_slice_users = self.driver.GetPersons(existing_slice_users_filter)
388 #existing_slice_users = self.driver.GetPersons(existing_slice_users_filter,['hrn','pkey'])
389 print>>sys.stderr, " \r\n \r\n slices.py verify_person existing_slice_users %s " %(existing_slice_users)
391 existing_slice_user_hrns = [user['hrn'] for user in existing_slice_users]
393 # users to be added, removed or updated
395 added_user_hrns = set(requested_user_hrns).difference(set(existing_user_hrns))
397 added_slice_user_hrns = set(requested_user_hrns).difference(existing_slice_user_hrns)
399 removed_user_hrns = set(existing_slice_user_hrns).difference(requested_user_hrns)
402 updated_user_hrns = set(existing_slice_user_hrns).intersection(requested_user_hrns)
403 # Remove stale users (only if we are not appending)
404 append = options.get('append', True)
406 for removed_user_hrn in removed_user_hrns:
407 self.driver.DeletePersonFromSlice(removed_user_hrn, slice_record['name'])
408 # update_existing users
409 updated_users_list = [user for user in existing_slice_users if user['hrn'] in \
411 #self.verify_keys(existing_slice_users, updated_users_list, peer, append)
415 for added_user_hrn in added_user_hrns:
416 added_user = users_dict[added_user_hrn]
417 #hrn, type = urn_to_hrn(added_user['urn'])
419 #'first_name': added_user.get('first_name', hrn),
420 #'last_name': added_user.get('last_name', hrn),
421 'person_id': added_user['person_id'],
422 #'peer_person_id': None,
424 #'key_ids': added_user.get('key_ids', []),
427 person['person_id'] = self.driver.AddPerson(person)
429 person['peer_person_id'] = added_user['person_id']
430 added_persons.append(person)
433 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
436 #self.driver.AddPersonToSite(added_user_id, login_base)
438 #for key_string in added_user.get('keys', []):
439 #key = {'key':key_string, 'key_type':'ssh'}
440 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], key)
441 #person['keys'].append(key)
443 # add the registry record
445 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \
446 #'pointer': person['person_id']}
447 #self.registry.register_peer_object(self.credential, peer_dict)
448 for added_slice_user_hrn in added_slice_user_hrns.union(added_user_hrns):
449 self.driver.AddPersonToSlice(added_slice_user_hrn, slice_record['name'])
450 #for added_slice_user_id in added_slice_user_ids.union(added_user_ids):
451 # add person to the slice
452 #self.driver.AddPersonToSlice(added_slice_user_id, slice_record['name'])
453 # if this is a peer record then it should already be bound to a peer.
454 # no need to return worry about it getting bound later
459 def verify_keys(self, persons, users, peer, options={}):
462 for person in persons:
463 key_ids.extend(person['key_ids'])
464 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
467 keydict[key['key']] = key['key_id']
468 existing_keys = keydict.keys()
470 for person in persons:
471 persondict[person['email']] = person
477 user_keys = user.get('keys', [])
478 updated_persons.append(user)
479 for key_string in user_keys:
480 requested_keys.append(key_string)
481 if key_string not in existing_keys:
482 key = {'key': key_string, 'key_type': 'ssh'}
485 person = persondict[user['email']]
486 self.driver.UnBindObjectFromPeer('person', person['person_id'], peer['shortname'])
487 key['key_id'] = self.driver.AddPersonKey(user['email'], key)
489 key_index = user_keys.index(key['key'])
490 remote_key_id = user['key_ids'][key_index]
491 self.driver.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id)
495 self.driver.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id'])
497 # remove old keys (only if we are not appending)
499 removed_keys = set(existing_keys).difference(requested_keys)
500 for existing_key_id in keydict:
501 if keydict[existing_key_id] in removed_keys:
504 self.driver.UnBindObjectFromPeer('key', existing_key_id, peer['shortname'])
505 self.driver.DeleteKey(existing_key_id)
509 #def verify_slice_attributes(self, slice, requested_slice_attributes, append=False, admin=False):
510 ## get list of attributes users ar able to manage
511 #filter = {'category': '*slice*'}
513 #filter['|roles'] = ['user']
514 #slice_attributes = self.driver.GetTagTypes(filter)
515 #valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes]
517 ## get sliver attributes
518 #added_slice_attributes = []
519 #removed_slice_attributes = []
520 #ignored_slice_attribute_names = []
521 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': slice['slice_id']})
523 ## get attributes that should be removed
524 #for slice_tag in existing_slice_attributes:
525 #if slice_tag['tagname'] in ignored_slice_attribute_names:
526 ## If a slice already has a admin only role it was probably given to them by an
527 ## admin, so we should ignore it.
528 #ignored_slice_attribute_names.append(slice_tag['tagname'])
530 ## If an existing slice attribute was not found in the request it should
532 #attribute_found=False
533 #for requested_attribute in requested_slice_attributes:
534 #if requested_attribute['name'] == slice_tag['tagname'] and \
535 #requested_attribute['value'] == slice_tag['value']:
536 #attribute_found=True
539 #if not attribute_found and not append:
540 #removed_slice_attributes.append(slice_tag)
542 ## get attributes that should be added:
543 #for requested_attribute in requested_slice_attributes:
544 ## if the requested attribute wasn't found we should add it
545 #if requested_attribute['name'] in valid_slice_attribute_names:
546 #attribute_found = False
547 #for existing_attribute in existing_slice_attributes:
548 #if requested_attribute['name'] == existing_attribute['tagname'] and \
549 #requested_attribute['value'] == existing_attribute['value']:
550 #attribute_found=True
552 #if not attribute_found:
553 #added_slice_attributes.append(requested_attribute)
556 ## remove stale attributes
557 #for attribute in removed_slice_attributes:
559 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
560 #except Exception, e:
561 #self.logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
562 #% (name, value, node_id, str(e)))
564 ## add requested_attributes
565 #for attribute in added_slice_attributes:
567 #self.driver.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None))
568 #except Exception, e:
569 #self.logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
570 #% (name, value, node_id, str(e)))
572 #def create_slice_aggregate(self, xrn, rspec):
573 #hrn, type = urn_to_hrn(xrn)
574 ## Determine if this is a peer slice
575 #peer = self.get_peer(hrn)
576 #sfa_peer = self.get_sfa_peer(hrn)
579 ## Get the slice record from sfa
580 #slicename = hrn_to_pl_slicename(hrn)
583 #registry = self.api.registries[self.api.hrn]
584 #credential = self.api.getCredential()
586 #site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer)
587 #slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer)
589 ## find out where this slice is currently running
590 #nodelist = self.driver.GetNodes(slice['node_ids'], ['hostname'])
591 #hostnames = [node['hostname'] for node in nodelist]
593 ## get netspec details
594 #nodespecs = spec.getDictsByTagName('NodeSpec')
596 ## dict in which to store slice attributes to set for the nodes
598 #for nodespec in nodespecs:
599 #if isinstance(nodespec['name'], list):
600 #for nodename in nodespec['name']:
601 #nodes[nodename] = {}
602 #for k in nodespec.keys():
603 #rspec_attribute_value = nodespec[k]
604 #if (self.rspec_to_slice_tag.has_key(k)):
605 #slice_tag_name = self.rspec_to_slice_tag[k]
606 #nodes[nodename][slice_tag_name] = rspec_attribute_value
607 #elif isinstance(nodespec['name'], StringTypes):
608 #nodename = nodespec['name']
609 #nodes[nodename] = {}
610 #for k in nodespec.keys():
611 #rspec_attribute_value = nodespec[k]
612 #if (self.rspec_to_slice_tag.has_key(k)):
613 #slice_tag_name = self.rspec_to_slice_tag[k]
614 #nodes[nodename][slice_tag_name] = rspec_attribute_value
616 #for k in nodespec.keys():
617 #rspec_attribute_value = nodespec[k]
618 #if (self.rspec_to_slice_tag.has_key(k)):
619 #slice_tag_name = self.rspec_to_slice_tag[k]
620 #nodes[nodename][slice_tag_name] = rspec_attribute_value
622 #node_names = nodes.keys()
623 ## remove nodes not in rspec
624 #deleted_nodes = list(set(hostnames).difference(node_names))
625 ## add nodes from rspec
626 #added_nodes = list(set(node_names).difference(hostnames))
630 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer)
632 #self.driver.LaunchExperimentOnOAR(slicename, added_nodes)
634 ## Add recognized slice tags
635 #for node_name in node_names:
636 #node = nodes[node_name]
637 #for slice_tag in node.keys():
638 #value = node[slice_tag]
639 #if (isinstance(value, list)):
642 #self.driver.AddSliceTag(slicename, slice_tag, value, node_name)
644 #self.driver.DeleteSliceFromNodes(slicename, deleted_nodes)
647 #self.driver.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id'])