1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 #def get_slivers(self, xrn, node=None):
17 #hrn, hrn_type = urn_to_hrn(xrn)
19 #slice_name = hrn_to_pl_slicename(hrn)
20 ## XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
21 ## of doing all of this?
22 ##return self.api.driver.GetSliceTicket(self.auth, slice_name)
26 #sfa_slice = self.driver.GetSlices(slice_filter = slice_name, \
27 # slice_filter_type = 'slice_hrn')
30 ## Get user information
32 #alchemy_person = dbsession.query(RegRecord).filter_by(record_id = \
33 #sfa_slice['record_id_user']).first()
36 #sliver_attributes = []
38 #if sfa_slice['oar_job_id'] is not -1:
39 #nodes_all = self.driver.GetNodes({'hostname': \
40 #sfa_slice['node_ids']},
41 #['node_id', 'hostname','site','boot_state'])
42 #nodeall_byhostname = dict([(n['hostname'], n) for n in nodes_all])
43 #nodes = sfa_slice['node_ids']
46 ##for sliver_attribute in filter(lambda a: a['node_id'] == \
47 #node['node_id'], slice_tags):
48 #sliver_attribute['tagname'] = 'slab-tag'
49 #sliver_attribute['value'] = 'slab-value'
50 #sliver_attributes.append(sliver_attribute['tagname'])
51 #attributes.append({'tagname': sliver_attribute['tagname'],
52 #'value': sliver_attribute['value']})
54 ## set nodegroup slice attributes
55 #for slice_tag in filter(lambda a: a['nodegroup_id'] \
56 #in node['nodegroup_ids'], slice_tags):
57 ## Do not set any nodegroup slice attributes for
58 ## which there is at least one sliver attribute
60 #if slice_tag not in slice_tags:
61 #attributes.append({'tagname': slice_tag['tagname'],
62 #'value': slice_tag['value']})
64 #for slice_tag in filter(lambda a: a['node_id'] is None, \
66 ## Do not set any global slice attributes for
67 ## which there is at least one sliver attribute
69 #if slice_tag['tagname'] not in sliver_attributes:
70 #attributes.append({'tagname': slice_tag['tagname'],
71 #'value': slice_tag['value']})
73 ## XXX Sanity check; though technically this should
74 ## be a system invariant
75 ## checked with an assertion
76 #if sfa_slice['expires'] > MAXINT: sfa_slice['expires']= MAXINT
80 #'name': sfa_slice['name'],
81 #'slice_id': sfa_slice['slice_id'],
82 #'instantiation': sfa_slice['instantiation'],
83 #'expires': sfa_slice['expires'],
85 #'attributes': attributes
95 def get_peer(self, xrn):
96 hrn, hrn_type = urn_to_hrn(xrn)
97 #Does this slice belong to a local site or a peer senslab site?
100 # get this slice's authority (site)
101 slice_authority = get_authority(hrn)
102 site_authority = slice_authority
103 # get this site's authority (sfa root authority or sub authority)
104 #site_authority = get_authority(slice_authority).lower()
105 logger.debug("SLABSLICES \ get_peer slice_authority %s \
106 site_authority %s hrn %s" %(slice_authority, \
107 site_authority, hrn))
108 #This slice belongs to the current site
109 if site_authority == self.driver.root_auth :
111 # check if we are already peered with this site_authority, if so
112 #peers = self.driver.GetPeers({})
113 peers = self.driver.GetPeers(peer_filter = slice_authority)
114 for peer_record in peers:
116 if site_authority == peer_record.hrn:
118 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
121 def get_sfa_peer(self, xrn):
122 hrn, hrn_type = urn_to_hrn(xrn)
124 # return the authority for this hrn or None if we are the authority
126 slice_authority = get_authority(hrn)
127 site_authority = get_authority(slice_authority)
129 if site_authority != self.driver.hrn:
130 sfa_peer = site_authority
135 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
138 #First get the list of current leases from OAR
139 leases = self.driver.GetLeases({'name':sfa_slice['slice_hrn']})
140 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
141 leases %s "%(requested_jobs_dict, leases ))
145 current_nodes_reserved_by_start_time = {}
146 requested_nodes_by_start_time = {}
147 leases_by_start_time = {}
148 #Create reduced dictionary with key start_time and value
150 #-for the leases already registered by OAR first
151 # then for the new leases requested by the user
153 #Leases already scheduled/running in OAR
154 for lease in leases :
155 current_nodes_reserved_by_start_time[lease['t_from']] = \
156 lease['reserved_nodes']
157 leases_by_start_time[lease['t_from']] = lease
160 for start_time in requested_jobs_dict:
161 requested_nodes_by_start_time[int(start_time)] = \
162 requested_jobs_dict[start_time]['hostname']
164 #Check if there is any difference between the leases already
165 #registered in OAR and the requested jobs.
166 #Difference could be:
167 #-Lease deleted in the requested jobs
168 #-Added/removed nodes
172 #Find all deleted leases
174 list(set(leases_by_start_time.keys()).\
175 difference(requested_nodes_by_start_time.keys()))
176 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
177 for start_time in start_time_list]
180 reschedule_jobs_dict = {}
181 #Find added or removed nodes in exisiting leases
182 for start_time in requested_nodes_by_start_time:
183 if start_time in current_nodes_reserved_by_start_time:
185 if requested_nodes_by_start_time[start_time] == \
186 current_nodes_reserved_by_start_time[start_time]:
191 set(requested_nodes_by_start_time[start_time])
193 update_node_set.difference(\
194 current_nodes_reserved_by_start_time[start_time])
196 update_node_set.intersection(\
197 current_nodes_reserved_by_start_time[start_time])
200 current_nodes_reserved_by_start_time[start_time])
202 old_nodes_set.difference(\
203 requested_nodes_by_start_time[start_time])
204 logger.debug("SLABSLICES verify_slice_leases \
205 shared_nodes %s added_nodes %s removed_nodes %s"\
206 %(shared_nodes, added_nodes,removed_nodes ))
207 #If the lease is modified, delete it before
209 #Add the deleted lease job id in the list
210 #WARNING :rescheduling does not work if there is already
211 # 2 running/scheduled jobs because deleting a job
212 #takes time SA 18/10/2012
213 if added_nodes or removed_nodes:
214 deleted_leases.append(\
215 leases_by_start_time[start_time]['lease_id'])
217 if added_nodes or shared_nodes:
218 reschedule_jobs_dict[str(start_time)] = \
219 requested_jobs_dict[str(start_time)]
223 logger.debug("SLABSLICES verify_slice_leases NEW LEASE")
224 job = requested_jobs_dict[str(start_time)]
226 self.driver.AddLeases(job['hostname'], \
227 sfa_slice, int(job['start_time']), \
228 int(job['duration']))
230 #Deleted leases are the ones with lease id not declared in the Rspec
232 self.driver.DeleteLeases(deleted_leases, sfa_slice['slice_hrn'])
233 logger.debug("SLABSLICES \
234 verify_slice_leases slice %s deleted_leases %s"\
235 %(sfa_slice, deleted_leases))
238 if reschedule_jobs_dict :
239 for start_time in reschedule :
240 job = reschedule_jobs_dict[start_time]
241 self.driver.AddLeases(job['hostname'], \
242 sfa_slice, int(job['start_time']), \
243 int(job['duration']))
246 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
250 if 'node_ids' in sfa_slice:
251 nodes = self.driver.GetNodes(sfa_slice['list_node_ids'], \
253 current_slivers = [node['hostname'] for node in nodes]
255 # remove nodes not in rspec
256 deleted_nodes = list(set(current_slivers).\
257 difference(requested_slivers))
258 # add nodes from rspec
259 #added_nodes = list(set(requested_slivers).\
260 #difference(current_slivers))
262 #Update the table with the nodes that populate the slice
263 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
264 \r\n \r\n deleted_nodes %s"\
265 %(sfa_slice, deleted_nodes))
268 #Delete the entire experience
269 self.driver.DeleteSliceFromNodes(sfa_slice)
270 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
276 def free_egre_key(self):
278 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
279 used.add(int(tag['value']))
281 for i in range(1, 256):
286 raise KeyError("No more EGRE keys available")
295 def handle_peer(self, site, sfa_slice, persons, peer):
300 self.driver.BindObjectToPeer('site', site['site_id'], \
301 peer['shortname'], sfa_slice['site_id'])
302 except Exception, error:
303 self.driver.DeleteSite(site['site_id'])
309 self.driver.BindObjectToPeer('slice', slice['slice_id'], \
310 peer['shortname'], sfa_slice['slice_id'])
311 except Exception, error:
312 self.driver.DeleteSlice(sfa_slice['slice_id'])
316 for person in persons:
318 self.driver.BindObjectToPeer('person', \
319 person['person_id'], peer['shortname'], \
320 person['peer_person_id'])
322 for (key, remote_key_id) in zip(person['keys'], \
325 self.driver.BindObjectToPeer( 'key', \
326 key['key_id'], peer['shortname'], \
329 self.driver.DeleteKey(key['key_id'])
330 logger.log_exc("failed to bind key: %s \
331 to peer: %s " % (key['key_id'], \
333 except Exception, error:
334 self.driver.DeletePerson(person['person_id'])
339 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
340 #sfa_peer=None, options={}):
341 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
342 #site_hrn = get_authority(slice_hrn)
343 ## login base can't be longer than 20 characters
344 ##slicename = hrn_to_pl_slicename(slice_hrn)
345 #authority_name = slice_hrn.split('.')[0]
346 #login_base = authority_name[:20]
347 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
348 #login_base %s slice_hrn %s" \
349 #%(authority_name,login_base,slice_hrn)
351 #sites = self.driver.GetSites(login_base)
353 ## create new site record
354 #site = {'name': 'geni.%s' % authority_name,
355 #'abbreviated_name': authority_name,
356 #'login_base': login_base,
358 #'max_slivers': 1000,
360 #'peer_site_id': None}
362 #site['peer_site_id'] = slice_record.get('site_id', None)
363 #site['site_id'] = self.driver.AddSite(site)
364 ## exempt federated sites from monitor policies
365 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', \
368 ### is this still necessary?
369 ### add record to the local registry
370 ##if sfa_peer and slice_record:
371 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
372 ##'peer_authority': sfa_peer, 'pointer': \
374 ##self.registry.register_peer_object(self.credential, peer_dict)
378 ## unbind from peer so we can modify if necessary.
379 ## Will bind back later
380 #self.driver.UnBindObjectFromPeer('site', site['site_id'], \
385 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
387 #login_base = slice_hrn.split(".")[0]
388 slicename = slice_hrn
389 slices_list = self.driver.GetSlices(slice_filter = slicename, \
390 slice_filter_type = 'slice_hrn')
392 for sl in slices_list:
394 logger.debug("SLABSLICE \tverify_slice slicename %s sl %s \
395 slice_record %s"%(slicename, sl, \
398 sfa_slice.update(slice_record)
399 #del slice['last_updated']
400 #del slice['date_created']
402 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
403 ## unbind from peer so we can modify if necessary.
404 ## Will bind back later
405 #self.driver.UnBindObjectFromPeer('slice', \
406 #slice['slice_id'], \
408 #Update existing record (e.g. expires field)
409 #it with the latest info.
410 ##if slice_record and slice['expires'] != slice_record['expires']:
411 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : \
412 #slice_record['expires']})
414 logger.debug(" SLABSLICES \tverify_slice Oups \
415 slice_record %s peer %s sfa_peer %s "\
416 %(slice_record, peer,sfa_peer))
417 sfa_slice = {'slice_hrn': slicename,
418 #'url': slice_record.get('url', slice_hrn),
419 #'description': slice_record.get('description', slice_hrn)
421 'record_id_user' : slice_record['person_ids'][0],
422 'record_id_slice': slice_record['record_id'],
423 'peer_authority':str(peer.hrn)
427 self.driver.AddSlice(sfa_slice)
428 #slice['slice_id'] = self.driver.AddSlice(slice)
429 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
430 #slice['node_ids']=[]
431 #slice['person_ids'] = []
433 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
434 # mark this slice as an sfa peer record
436 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
437 #'peer_authority': sfa_peer, 'pointer': \
439 #self.registry.register_peer_object(self.credential, peer_dict)
446 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
449 users is a record list. Records can either be local records
450 or users records from known and trusted federated sites.
451 If the user is from another site that senslab doesn't trust yet,
452 then Resolve will raise an error before getting to create_sliver.
454 #TODO SA 21/08/12 verify_persons Needs review
459 #users_dict : dict whose keys can either be the user's hrn or its id.
460 #Values contains only id and hrn
463 #First create dicts by hrn and id for each user in the user record list:
466 if 'urn' in user and (not 'hrn' in user ) :
467 user['hrn'], user['type'] = urn_to_hrn(user['urn'])
469 if 'person_id' in user and 'hrn' in user:
470 users_by_id[user['person_id']] = user
471 users_dict[user['person_id']] = {'person_id':\
472 user['person_id'], 'hrn':user['hrn']}
474 users_by_hrn[user['hrn']] = user
475 users_dict[user['hrn']] = {'person_id':user['person_id'], \
479 logger.debug( "SLABSLICE.PY \t verify_person \
480 users_dict %s \r\n user_by_hrn %s \r\n \
482 %(users_dict,users_by_hrn, users_by_id))
484 existing_user_ids = []
485 existing_user_hrns = []
487 # Check if user is in Senslab LDAP using its hrn.
488 # Assuming Senslab is centralised : one LDAP for all sites,
489 # user_id unknown from LDAP
490 # LDAP does not provide users id, therefore we rely on hrns containing
491 # the login of the user.
492 # If the hrn is not a senslab hrn, the user may not be in LDAP.
494 #Construct the list of filters (list of dicts) for GetPersons
496 for hrn in users_by_hrn:
497 filter_user.append (users_by_hrn[hrn])
498 logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " \
500 #Check user's in LDAP with GetPersons
501 #Needed because what if the user has been deleted in LDAP but
503 existing_users = self.driver.GetPersons(filter_user)
505 #User's in senslab LDAP
507 for user in existing_users :
508 existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
510 append(users_dict[user['hrn']]['person_id'])
512 # User from another known trusted federated site. Check
513 # if a senslab account matching the email has already been created.
516 if isinstance(users, list):
518 req += users[0]['email']
520 req += users['email']
522 ldap_reslt = self.driver.ldap.LdapSearch(req)
524 logger.debug(" SLABSLICE.PY \tverify_person users \
525 USER already in Senslab \t ldap_reslt %s \
527 existing_users.append(ldap_reslt[1])
530 #User not existing in LDAP
531 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
532 logger.debug(" SLABSLICE.PY \tverify_person users \
533 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
534 ldap_reslt %s " %(users, ldap_reslt))
536 requested_user_ids = users_by_id.keys()
537 requested_user_hrns = users_by_hrn.keys()
538 logger.debug("SLABSLICE.PY \tverify_person requested_user_ids %s \
539 user_by_hrn %s " %(requested_user_ids, users_by_hrn))
542 #Check that the user of the slice in the slice record
543 #matches the existing users
545 if slice_record['record_id_user'] in requested_user_ids and \
546 slice_record['PI'][0] in requested_user_hrns:
547 logger.debug(" SLABSLICE \tverify_person \
548 requested_user_ids %s = \
549 slice_record['record_id_user'] %s" \
550 %(requested_user_ids,slice_record['record_id_user']))
556 # users to be added, removed or updated
557 #One user in one senslab slice : there should be no need
558 #to remove/ add any user from/to a slice.
559 #However a user from SFA which is not registered in Senslab yet
560 #should be added to the LDAP.
562 added_user_hrns = set(requested_user_hrns).\
563 difference(set(existing_user_hrns))
565 #self.verify_keys(existing_slice_users, updated_users_list, \
570 for added_user_hrn in added_user_hrns:
571 added_user = users_dict[added_user_hrn]
572 #hrn, type = urn_to_hrn(added_user['urn'])
574 #'first_name': added_user.get('first_name', hrn),
575 #'last_name': added_user.get('last_name', hrn),
576 'first_name': added_user['first_name'],
577 'last_name': added_user['last_name'],
578 'person_id': added_user['person_id'],
579 'peer_person_id': None,
581 'key_ids': added_user.get('key_ids', []),
584 person['person_id'] = self.driver.AddPerson(person)
586 person['peer_person_id'] = added_user['person_id']
587 added_persons.append(person)
590 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
593 #self.driver.AddPersonToSite(added_user_id, login_base)
595 #for key_string in added_user.get('keys', []):
596 #key = {'key':key_string, 'key_type':'ssh'}
597 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], \
599 #person['keys'].append(key)
601 # add the registry record
603 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
605 #'pointer': person['person_id']}
606 #self.registry.register_peer_object(self.credential, peer_dict)
607 #for added_slice_user_hrn in \
608 #added_slice_user_hrns.union(added_user_hrns):
609 #self.driver.AddPersonToSlice(added_slice_user_hrn, \
610 #slice_record['name'])
611 #for added_slice_user_id in \
612 #added_slice_user_ids.union(added_user_ids):
613 # add person to the slice
614 #self.driver.AddPersonToSlice(added_slice_user_id, \
615 #slice_record['name'])
616 # if this is a peer record then it
617 # should already be bound to a peer.
618 # no need to return worry about it getting bound later
623 def verify_keys(self, persons, users, peer, options={}):
626 for person in persons:
627 key_ids.extend(person['key_ids'])
628 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
631 keydict[key['key']] = key['key_id']
632 existing_keys = keydict.keys()
634 for person in persons:
635 persondict[person['email']] = person
641 user_keys = user.get('keys', [])
642 updated_persons.append(user)
643 for key_string in user_keys:
644 requested_keys.append(key_string)
645 if key_string not in existing_keys:
646 key = {'key': key_string, 'key_type': 'ssh'}
649 person = persondict[user['email']]
650 self.driver.UnBindObjectFromPeer('person', \
651 person['person_id'], peer['shortname'])
653 self.driver.AddPersonKey(user['email'], key)
655 key_index = user_keys.index(key['key'])
656 remote_key_id = user['key_ids'][key_index]
657 self.driver.BindObjectToPeer('key', \
658 key['key_id'], peer['shortname'], \
663 self.driver.BindObjectToPeer('person', \
664 person['person_id'], peer['shortname'], \
667 # remove old keys (only if we are not appending)
668 append = options.get('append', True)
670 removed_keys = set(existing_keys).difference(requested_keys)
671 for existing_key_id in keydict:
672 if keydict[existing_key_id] in removed_keys:
675 self.driver.UnBindObjectFromPeer('key', \
676 existing_key_id, peer['shortname'])
677 self.driver.DeleteKey(existing_key_id)
680 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
681 #append=False, admin=False):
682 ## get list of attributes users ar able to manage
683 #filter = {'category': '*slice*'}
685 #filter['|roles'] = ['user']
686 #slice_attributes = self.driver.GetTagTypes(filter)
687 #valid_slice_attribute_names = [attribute['tagname'] \
688 #for attribute in slice_attributes]
690 ## get sliver attributes
691 #added_slice_attributes = []
692 #removed_slice_attributes = []
693 #ignored_slice_attribute_names = []
694 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': \
697 ## get attributes that should be removed
698 #for slice_tag in existing_slice_attributes:
699 #if slice_tag['tagname'] in ignored_slice_attribute_names:
700 ## If a slice already has a admin only role
701 ## it was probably given to them by an
702 ## admin, so we should ignore it.
703 #ignored_slice_attribute_names.append(slice_tag['tagname'])
705 ## If an existing slice attribute was not
706 ## found in the request it should
708 #attribute_found=False
709 #for requested_attribute in requested_slice_attributes:
710 #if requested_attribute['name'] == slice_tag['tagname'] \
711 #and requested_attribute['value'] == slice_tag['value']:
712 #attribute_found=True
715 #if not attribute_found and not append:
716 #removed_slice_attributes.append(slice_tag)
718 ## get attributes that should be added:
719 #for requested_attribute in requested_slice_attributes:
720 ## if the requested attribute wasn't found we should add it
721 #if requested_attribute['name'] in valid_slice_attribute_names:
722 #attribute_found = False
723 #for existing_attribute in existing_slice_attributes:
724 #if requested_attribute['name'] == \
725 #existing_attribute['tagname'] and \
726 #requested_attribute['value'] == \
727 #existing_attribute['value']:
728 #attribute_found=True
730 #if not attribute_found:
731 #added_slice_attributes.append(requested_attribute)
734 ## remove stale attributes
735 #for attribute in removed_slice_attributes:
737 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
738 #except Exception, error:
739 #self.logger.warn('Failed to remove sliver attribute. name: \
740 #%s, value: %s, node_id: %s\nCause:%s'\
741 #% (name, value, node_id, str(error)))
743 ## add requested_attributes
744 #for attribute in added_slice_attributes:
746 #self.driver.AddSliceTag(slice['name'], attribute['name'], \
747 #attribute['value'], attribute.get('node_id', None))
748 #except Exception, error:
749 #self.logger.warn('Failed to add sliver attribute. name: %s, \
750 #value: %s, node_id: %s\nCause:%s'\
751 #% (name, value, node_id, str(error)))