1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 #def get_slivers(self, xrn, node=None):
17 #hrn, hrn_type = urn_to_hrn(xrn)
19 #slice_name = hrn_to_pl_slicename(hrn)
20 ## XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
21 ## of doing all of this?
22 ##return self.api.driver.GetSliceTicket(self.auth, slice_name)
26 #sfa_slice = self.driver.GetSlices(slice_filter = slice_name, \
27 # slice_filter_type = 'slice_hrn')
30 ## Get user information
32 #alchemy_person = dbsession.query(RegRecord).filter_by(record_id = \
33 #sfa_slice['record_id_user']).first()
36 #sliver_attributes = []
38 #if sfa_slice['oar_job_id'] is not -1:
39 #nodes_all = self.driver.GetNodes({'hostname': \
40 #sfa_slice['node_ids']},
41 #['node_id', 'hostname','site','boot_state'])
42 #nodeall_byhostname = dict([(n['hostname'], n) for n in nodes_all])
43 #nodes = sfa_slice['node_ids']
46 ##for sliver_attribute in filter(lambda a: a['node_id'] == \
47 #node['node_id'], slice_tags):
48 #sliver_attribute['tagname'] = 'slab-tag'
49 #sliver_attribute['value'] = 'slab-value'
50 #sliver_attributes.append(sliver_attribute['tagname'])
51 #attributes.append({'tagname': sliver_attribute['tagname'],
52 #'value': sliver_attribute['value']})
54 ## set nodegroup slice attributes
55 #for slice_tag in filter(lambda a: a['nodegroup_id'] \
56 #in node['nodegroup_ids'], slice_tags):
57 ## Do not set any nodegroup slice attributes for
58 ## which there is at least one sliver attribute
60 #if slice_tag not in slice_tags:
61 #attributes.append({'tagname': slice_tag['tagname'],
62 #'value': slice_tag['value']})
64 #for slice_tag in filter(lambda a: a['node_id'] is None, \
66 ## Do not set any global slice attributes for
67 ## which there is at least one sliver attribute
69 #if slice_tag['tagname'] not in sliver_attributes:
70 #attributes.append({'tagname': slice_tag['tagname'],
71 #'value': slice_tag['value']})
73 ## XXX Sanity check; though technically this should
74 ## be a system invariant
75 ## checked with an assertion
76 #if sfa_slice['expires'] > MAXINT: sfa_slice['expires']= MAXINT
80 #'name': sfa_slice['name'],
81 #'slice_id': sfa_slice['slice_id'],
82 #'instantiation': sfa_slice['instantiation'],
83 #'expires': sfa_slice['expires'],
85 #'attributes': attributes
95 def get_peer(self, xrn):
96 hrn, hrn_type = urn_to_hrn(xrn)
97 #Does this slice belong to a local site or a peer senslab site?
100 # get this slice's authority (site)
101 slice_authority = get_authority(hrn)
102 site_authority = slice_authority
103 # get this site's authority (sfa root authority or sub authority)
104 #site_authority = get_authority(slice_authority).lower()
105 logger.debug("SLABSLICES \ get_peer slice_authority %s \
106 site_authority %s hrn %s" %(slice_authority, \
107 site_authority, hrn))
108 #This slice belongs to the current site
109 if site_authority == self.driver.root_auth :
111 # check if we are already peered with this site_authority, if so
112 #peers = self.driver.GetPeers({})
113 peers = self.driver.GetPeers(peer_filter = slice_authority)
114 for peer_record in peers:
116 if site_authority == peer_record.hrn:
118 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
121 def get_sfa_peer(self, xrn):
122 hrn, hrn_type = urn_to_hrn(xrn)
124 # return the authority for this hrn or None if we are the authority
126 slice_authority = get_authority(hrn)
127 site_authority = get_authority(slice_authority)
129 if site_authority != self.driver.hrn:
130 sfa_peer = site_authority
135 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
138 #First get the list of current leases from OAR
139 leases = self.driver.GetLeases({'name':sfa_slice['slice_hrn']})
140 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
141 leases %s "%(requested_jobs_dict, leases ))
145 current_nodes_reserved_by_start_time = {}
146 requested_nodes_by_start_time = {}
147 leases_by_start_time = {}
148 #Create reduced dictionary with key start_time and value
150 #-for the leases already registered by OAR first
151 # then for the new leases requested by the user
153 #Leases already scheduled/running in OAR
154 for lease in leases :
155 current_nodes_reserved_by_start_time[lease['t_from']] = \
156 lease['reserved_nodes']
157 leases_by_start_time[lease['t_from']] = lease
160 for start_time in requested_jobs_dict:
161 requested_nodes_by_start_time[int(start_time)] = \
162 requested_jobs_dict[start_time]['hostname']
164 #Check if there is any difference between the leases already
165 #registered in OAR and the requested jobs.
166 #Difference could be:
167 #-Lease deleted in the requested jobs
168 #-Added/removed nodes
172 #Find all deleted leases
174 list(set(leases_by_start_time.keys()).\
175 difference(requested_nodes_by_start_time.keys()))
176 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
177 for start_time in start_time_list]
180 reschedule_jobs_dict = {}
181 #Find added or removed nodes in exisiting leases
182 for start_time in requested_nodes_by_start_time:
183 if start_time in current_nodes_reserved_by_start_time:
185 if requested_nodes_by_start_time[start_time] == \
186 current_nodes_reserved_by_start_time[start_time]:
191 set(requested_nodes_by_start_time[start_time])
193 update_node_set.difference(\
194 current_nodes_reserved_by_start_time[start_time])
196 update_node_set.intersection(\
197 current_nodes_reserved_by_start_time[start_time])
200 current_nodes_reserved_by_start_time[start_time])
202 old_nodes_set.difference(\
203 requested_nodes_by_start_time[start_time])
204 logger.debug("SLABSLICES verify_slice_leases \
205 shared_nodes %s added_nodes %s removed_nodes %s"\
206 %(shared_nodes, added_nodes,removed_nodes ))
207 #If the lease is modified, delete it before
209 #Add the deleted lease job id in the list
210 #WARNING :rescheduling does not work if there is already
211 # 2 running/scheduled jobs because deleting a job
212 #takes time SA 18/10/2012
213 if added_nodes or removed_nodes:
214 deleted_leases.append(\
215 leases_by_start_time[start_time]['lease_id'])
217 if added_nodes or shared_nodes:
218 reschedule_jobs_dict[str(start_time)] = \
219 requested_jobs_dict[str(start_time)]
223 logger.debug("SLABSLICES verify_slice_leases NEW LEASE")
224 job = requested_jobs_dict[str(start_time)]
225 self.driver.AddLeases(job['hostname'], \
226 sfa_slice, int(job['start_time']), \
227 int(job['duration']))
229 #Deleted leases are the ones with lease id not declared in the Rspec
231 self.driver.DeleteLeases(deleted_leases, sfa_slice['slice_hrn'])
232 logger.debug("SLABSLICES \
233 verify_slice_leases slice %s deleted_leases %s"\
234 %(sfa_slice, deleted_leases))
237 if reschedule_jobs_dict :
238 for start_time in reschedule :
239 job = reschedule_jobs_dict[start_time]
240 self.driver.AddLeases(job['hostname'], \
241 sfa_slice, int(job['start_time']), \
242 int(job['duration']))
245 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
249 if 'node_ids' in sfa_slice:
250 nodes = self.driver.GetNodes(sfa_slice['list_node_ids'], \
252 current_slivers = [node['hostname'] for node in nodes]
254 # remove nodes not in rspec
255 deleted_nodes = list(set(current_slivers).\
256 difference(requested_slivers))
257 # add nodes from rspec
258 #added_nodes = list(set(requested_slivers).\
259 #difference(current_slivers))
261 #Update the table with the nodes that populate the slice
262 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
263 \r\n \r\n deleted_nodes %s"\
264 %(sfa_slice, deleted_nodes))
267 #Delete the entire experience
268 self.driver.DeleteSliceFromNodes(sfa_slice)
269 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
275 def free_egre_key(self):
277 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
278 used.add(int(tag['value']))
280 for i in range(1, 256):
285 raise KeyError("No more EGRE keys available")
294 def handle_peer(self, site, sfa_slice, persons, peer):
299 self.driver.BindObjectToPeer('site', site['site_id'], \
300 peer['shortname'], sfa_slice['site_id'])
301 except Exception, error:
302 self.driver.DeleteSite(site['site_id'])
308 self.driver.BindObjectToPeer('slice', slice['slice_id'], \
309 peer['shortname'], sfa_slice['slice_id'])
310 except Exception, error:
311 self.driver.DeleteSlice(sfa_slice['slice_id'])
315 for person in persons:
317 self.driver.BindObjectToPeer('person', \
318 person['person_id'], peer['shortname'], \
319 person['peer_person_id'])
321 for (key, remote_key_id) in zip(person['keys'], \
324 self.driver.BindObjectToPeer( 'key', \
325 key['key_id'], peer['shortname'], \
328 self.driver.DeleteKey(key['key_id'])
329 logger.log_exc("failed to bind key: %s \
330 to peer: %s " % (key['key_id'], \
332 except Exception, error:
333 self.driver.DeletePerson(person['person_id'])
338 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
339 #sfa_peer=None, options={}):
340 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
341 #site_hrn = get_authority(slice_hrn)
342 ## login base can't be longer than 20 characters
343 ##slicename = hrn_to_pl_slicename(slice_hrn)
344 #authority_name = slice_hrn.split('.')[0]
345 #login_base = authority_name[:20]
346 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
347 #login_base %s slice_hrn %s" \
348 #%(authority_name,login_base,slice_hrn)
350 #sites = self.driver.GetSites(login_base)
352 ## create new site record
353 #site = {'name': 'geni.%s' % authority_name,
354 #'abbreviated_name': authority_name,
355 #'login_base': login_base,
357 #'max_slivers': 1000,
359 #'peer_site_id': None}
361 #site['peer_site_id'] = slice_record.get('site_id', None)
362 #site['site_id'] = self.driver.AddSite(site)
363 ## exempt federated sites from monitor policies
364 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', \
367 ### is this still necessary?
368 ### add record to the local registry
369 ##if sfa_peer and slice_record:
370 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
371 ##'peer_authority': sfa_peer, 'pointer': \
373 ##self.registry.register_peer_object(self.credential, peer_dict)
377 ## unbind from peer so we can modify if necessary.
378 ## Will bind back later
379 #self.driver.UnBindObjectFromPeer('site', site['site_id'], \
384 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
386 #login_base = slice_hrn.split(".")[0]
387 slicename = slice_hrn
388 slices_list = self.driver.GetSlices(slice_filter = slicename, \
389 slice_filter_type = 'slice_hrn')
391 for sl in slices_list:
393 logger.debug("SLABSLICE \tverify_slice slicename %s sl %s \
394 slice_record %s"%(slicename, sl, \
397 sfa_slice.update(slice_record)
398 #del slice['last_updated']
399 #del slice['date_created']
401 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
402 ## unbind from peer so we can modify if necessary.
403 ## Will bind back later
404 #self.driver.UnBindObjectFromPeer('slice', \
405 #slice['slice_id'], \
407 #Update existing record (e.g. expires field)
408 #it with the latest info.
409 ##if slice_record and slice['expires'] != slice_record['expires']:
410 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : \
411 #slice_record['expires']})
413 logger.debug(" SLABSLICES \tverify_slice Oups \
414 slice_record %s peer %s sfa_peer %s "\
415 %(slice_record, peer,sfa_peer))
416 sfa_slice = {'slice_hrn': slicename,
417 #'url': slice_record.get('url', slice_hrn),
418 #'description': slice_record.get('description', slice_hrn)
420 'record_id_user' : slice_record['person_ids'][0],
421 'record_id_slice': slice_record['record_id'],
422 'peer_authority':str(peer.hrn)
426 self.driver.AddSlice(sfa_slice)
427 #slice['slice_id'] = self.driver.AddSlice(slice)
428 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
429 #slice['node_ids']=[]
430 #slice['person_ids'] = []
432 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
433 # mark this slice as an sfa peer record
435 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
436 #'peer_authority': sfa_peer, 'pointer': \
438 #self.registry.register_peer_object(self.credential, peer_dict)
445 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
448 users is a record list. Records can either be local records
449 or users records from known and trusted federated sites.
450 If the user is from another site that senslab doesn't trust yet,
451 then Resolve will raise an error before getting to create_sliver.
453 #TODO SA 21/08/12 verify_persons Needs review
458 #users_dict : dict whose keys can either be the user's hrn or its id.
459 #Values contains only id and hrn
462 #First create dicts by hrn and id for each user in the user record list:
465 if 'urn' in user and (not 'hrn' in user ) :
466 user['hrn'], user['type'] = urn_to_hrn(user['urn'])
468 if 'person_id' in user and 'hrn' in user:
469 users_by_id[user['person_id']] = user
470 users_dict[user['person_id']] = {'person_id':\
471 user['person_id'], 'hrn':user['hrn']}
473 users_by_hrn[user['hrn']] = user
474 users_dict[user['hrn']] = {'person_id':user['person_id'], \
478 logger.debug( "SLABSLICE.PY \t verify_person \
479 users_dict %s \r\n user_by_hrn %s \r\n \
481 %(users_dict,users_by_hrn, users_by_id))
483 existing_user_ids = []
484 existing_user_hrns = []
486 # Check if user is in Senslab LDAP using its hrn.
487 # Assuming Senslab is centralised : one LDAP for all sites,
488 # user_id unknown from LDAP
489 # LDAP does not provide users id, therefore we rely on hrns containing
490 # the login of the user.
491 # If the hrn is not a senslab hrn, the user may not be in LDAP.
493 #Construct the list of filters (list of dicts) for GetPersons
495 for hrn in users_by_hrn:
496 filter_user.append (users_by_hrn[hrn])
497 logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " \
499 #Check user's in LDAP with GetPersons
500 #Needed because what if the user has been deleted in LDAP but
502 existing_users = self.driver.GetPersons(filter_user)
504 #User's in senslab LDAP
506 for user in existing_users :
507 existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
509 append(users_dict[user['hrn']]['person_id'])
511 # User from another known trusted federated site. Check
512 # if a senslab account matching the email has already been created.
515 if isinstance(users, list):
517 req += users[0]['email']
519 req += users['email']
521 ldap_reslt = self.driver.ldap.LdapSearch(req)
523 logger.debug(" SLABSLICE.PY \tverify_person users \
524 USER already in Senslab \t ldap_reslt %s \
526 existing_users.append(ldap_reslt[1])
529 #User not existing in LDAP
530 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
531 logger.debug(" SLABSLICE.PY \tverify_person users \
532 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
533 ldap_reslt %s " %(users, ldap_reslt))
535 requested_user_ids = users_by_id.keys()
536 requested_user_hrns = users_by_hrn.keys()
537 logger.debug("SLABSLICE.PY \tverify_person requested_user_ids %s \
538 user_by_hrn %s " %(requested_user_ids, users_by_hrn))
541 #Check that the user of the slice in the slice record
542 #matches the existing users
544 if slice_record['record_id_user'] in requested_user_ids and \
545 slice_record['PI'][0] in requested_user_hrns:
546 logger.debug(" SLABSLICE \tverify_person \
547 requested_user_ids %s = \
548 slice_record['record_id_user'] %s" \
549 %(requested_user_ids,slice_record['record_id_user']))
555 # users to be added, removed or updated
556 #One user in one senslab slice : there should be no need
557 #to remove/ add any user from/to a slice.
558 #However a user from SFA which is not registered in Senslab yet
559 #should be added to the LDAP.
561 added_user_hrns = set(requested_user_hrns).\
562 difference(set(existing_user_hrns))
564 #self.verify_keys(existing_slice_users, updated_users_list, \
569 for added_user_hrn in added_user_hrns:
570 added_user = users_dict[added_user_hrn]
571 #hrn, type = urn_to_hrn(added_user['urn'])
573 #'first_name': added_user.get('first_name', hrn),
574 #'last_name': added_user.get('last_name', hrn),
575 'first_name': added_user['first_name'],
576 'last_name': added_user['last_name'],
577 'person_id': added_user['person_id'],
578 'peer_person_id': None,
580 'key_ids': added_user.get('key_ids', []),
583 person['person_id'] = self.driver.AddPerson(person)
585 person['peer_person_id'] = added_user['person_id']
586 added_persons.append(person)
589 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
592 #self.driver.AddPersonToSite(added_user_id, login_base)
594 #for key_string in added_user.get('keys', []):
595 #key = {'key':key_string, 'key_type':'ssh'}
596 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], \
598 #person['keys'].append(key)
600 # add the registry record
602 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
604 #'pointer': person['person_id']}
605 #self.registry.register_peer_object(self.credential, peer_dict)
606 #for added_slice_user_hrn in \
607 #added_slice_user_hrns.union(added_user_hrns):
608 #self.driver.AddPersonToSlice(added_slice_user_hrn, \
609 #slice_record['name'])
610 #for added_slice_user_id in \
611 #added_slice_user_ids.union(added_user_ids):
612 # add person to the slice
613 #self.driver.AddPersonToSlice(added_slice_user_id, \
614 #slice_record['name'])
615 # if this is a peer record then it
616 # should already be bound to a peer.
617 # no need to return worry about it getting bound later
622 def verify_keys(self, persons, users, peer, options={}):
625 for person in persons:
626 key_ids.extend(person['key_ids'])
627 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
630 keydict[key['key']] = key['key_id']
631 existing_keys = keydict.keys()
633 for person in persons:
634 persondict[person['email']] = person
640 user_keys = user.get('keys', [])
641 updated_persons.append(user)
642 for key_string in user_keys:
643 requested_keys.append(key_string)
644 if key_string not in existing_keys:
645 key = {'key': key_string, 'key_type': 'ssh'}
648 person = persondict[user['email']]
649 self.driver.UnBindObjectFromPeer('person', \
650 person['person_id'], peer['shortname'])
652 self.driver.AddPersonKey(user['email'], key)
654 key_index = user_keys.index(key['key'])
655 remote_key_id = user['key_ids'][key_index]
656 self.driver.BindObjectToPeer('key', \
657 key['key_id'], peer['shortname'], \
662 self.driver.BindObjectToPeer('person', \
663 person['person_id'], peer['shortname'], \
666 # remove old keys (only if we are not appending)
667 append = options.get('append', True)
669 removed_keys = set(existing_keys).difference(requested_keys)
670 for existing_key_id in keydict:
671 if keydict[existing_key_id] in removed_keys:
674 self.driver.UnBindObjectFromPeer('key', \
675 existing_key_id, peer['shortname'])
676 self.driver.DeleteKey(existing_key_id)
679 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
680 #append=False, admin=False):
681 ## get list of attributes users ar able to manage
682 #filter = {'category': '*slice*'}
684 #filter['|roles'] = ['user']
685 #slice_attributes = self.driver.GetTagTypes(filter)
686 #valid_slice_attribute_names = [attribute['tagname'] \
687 #for attribute in slice_attributes]
689 ## get sliver attributes
690 #added_slice_attributes = []
691 #removed_slice_attributes = []
692 #ignored_slice_attribute_names = []
693 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': \
696 ## get attributes that should be removed
697 #for slice_tag in existing_slice_attributes:
698 #if slice_tag['tagname'] in ignored_slice_attribute_names:
699 ## If a slice already has a admin only role
700 ## it was probably given to them by an
701 ## admin, so we should ignore it.
702 #ignored_slice_attribute_names.append(slice_tag['tagname'])
704 ## If an existing slice attribute was not
705 ## found in the request it should
707 #attribute_found=False
708 #for requested_attribute in requested_slice_attributes:
709 #if requested_attribute['name'] == slice_tag['tagname'] \
710 #and requested_attribute['value'] == slice_tag['value']:
711 #attribute_found=True
714 #if not attribute_found and not append:
715 #removed_slice_attributes.append(slice_tag)
717 ## get attributes that should be added:
718 #for requested_attribute in requested_slice_attributes:
719 ## if the requested attribute wasn't found we should add it
720 #if requested_attribute['name'] in valid_slice_attribute_names:
721 #attribute_found = False
722 #for existing_attribute in existing_slice_attributes:
723 #if requested_attribute['name'] == \
724 #existing_attribute['tagname'] and \
725 #requested_attribute['value'] == \
726 #existing_attribute['value']:
727 #attribute_found=True
729 #if not attribute_found:
730 #added_slice_attributes.append(requested_attribute)
733 ## remove stale attributes
734 #for attribute in removed_slice_attributes:
736 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
737 #except Exception, error:
738 #self.logger.warn('Failed to remove sliver attribute. name: \
739 #%s, value: %s, node_id: %s\nCause:%s'\
740 #% (name, value, node_id, str(error)))
742 ## add requested_attributes
743 #for attribute in added_slice_attributes:
745 #self.driver.AddSliceTag(slice['name'], attribute['name'], \
746 #attribute['value'], attribute.get('node_id', None))
747 #except Exception, error:
748 #self.logger.warn('Failed to add sliver attribute. name: %s, \
749 #value: %s, node_id: %s\nCause:%s'\
750 #% (name, value, node_id, str(error)))