1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 def get_peer(self, xrn):
17 hrn, hrn_type = urn_to_hrn(xrn)
18 #Does this slice belong to a local site or a peer senslab site?
21 # get this slice's authority (site)
22 slice_authority = get_authority(hrn)
23 site_authority = slice_authority
24 # get this site's authority (sfa root authority or sub authority)
25 #site_authority = get_authority(slice_authority).lower()
26 logger.debug("SLABSLICES \ get_peer slice_authority %s \
27 site_authority %s hrn %s" %(slice_authority, \
29 #This slice belongs to the current site
30 if site_authority == self.driver.root_auth :
32 # check if we are already peered with this site_authority, if so
33 #peers = self.driver.GetPeers({})
34 peers = self.driver.GetPeers(peer_filter = slice_authority)
35 for peer_record in peers:
37 if site_authority == peer_record.hrn:
39 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
42 def get_sfa_peer(self, xrn):
43 hrn, hrn_type = urn_to_hrn(xrn)
45 # return the authority for this hrn or None if we are the authority
47 slice_authority = get_authority(hrn)
48 site_authority = get_authority(slice_authority)
50 if site_authority != self.driver.hrn:
51 sfa_peer = site_authority
56 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
59 #First get the list of current leases from OAR
60 leases = self.driver.GetLeases({'name':sfa_slice['slice_hrn']})
61 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
62 leases %s "%(requested_jobs_dict, leases ))
64 current_nodes_reserved_by_start_time = {}
65 requested_nodes_by_start_time = {}
66 leases_by_start_time = {}
67 reschedule_jobs_dict = {}
70 #Create reduced dictionary with key start_time and value
72 #-for the leases already registered by OAR first
73 # then for the new leases requested by the user
75 #Leases already scheduled/running in OAR
77 current_nodes_reserved_by_start_time[lease['t_from']] = \
78 lease['reserved_nodes']
79 leases_by_start_time[lease['t_from']] = lease
83 for start_time in requested_jobs_dict:
84 requested_nodes_by_start_time[int(start_time)] = \
85 requested_jobs_dict[start_time]['hostname']
86 #Check if there is any difference between the leases already
87 #registered in OAR and the requested jobs.
89 #-Lease deleted in the requested jobs
93 logger.debug("SLABSLICES verify_slice_leases \
94 requested_nodes_by_start_time %s \
95 "%(requested_nodes_by_start_time ))
96 #Find all deleted leases
98 list(set(leases_by_start_time.keys()).\
99 difference(requested_nodes_by_start_time.keys()))
100 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
101 for start_time in start_time_list]
105 #Find added or removed nodes in exisiting leases
106 for start_time in requested_nodes_by_start_time:
107 logger.debug("SLABSLICES verify_slice_leases start_time %s \
109 if start_time in current_nodes_reserved_by_start_time:
111 if requested_nodes_by_start_time[start_time] == \
112 current_nodes_reserved_by_start_time[start_time]:
117 set(requested_nodes_by_start_time[start_time])
119 update_node_set.difference(\
120 current_nodes_reserved_by_start_time[start_time])
122 update_node_set.intersection(\
123 current_nodes_reserved_by_start_time[start_time])
126 current_nodes_reserved_by_start_time[start_time])
128 old_nodes_set.difference(\
129 requested_nodes_by_start_time[start_time])
130 logger.debug("SLABSLICES verify_slice_leases \
131 shared_nodes %s added_nodes %s removed_nodes %s"\
132 %(shared_nodes, added_nodes,removed_nodes ))
133 #If the lease is modified, delete it before
135 #Add the deleted lease job id in the list
136 #WARNING :rescheduling does not work if there is already
137 # 2 running/scheduled jobs because deleting a job
138 #takes time SA 18/10/2012
139 if added_nodes or removed_nodes:
140 deleted_leases.append(\
141 leases_by_start_time[start_time]['lease_id'])
143 if added_nodes or shared_nodes:
144 reschedule_jobs_dict[str(start_time)] = \
145 requested_jobs_dict[str(start_time)]
150 job = requested_jobs_dict[str(start_time)]
151 logger.debug("SLABSLICES \
152 NEWLEASE slice %s job %s"\
154 self.driver.AddLeases(job['hostname'], \
155 sfa_slice, int(job['start_time']), \
156 int(job['duration']))
158 #Deleted leases are the ones with lease id not declared in the Rspec
160 self.driver.DeleteLeases(deleted_leases, sfa_slice['slice_hrn'])
161 logger.debug("SLABSLICES \
162 verify_slice_leases slice %s deleted_leases %s"\
163 %(sfa_slice, deleted_leases))
166 if reschedule_jobs_dict :
167 for start_time in reschedule_jobs_dict:
168 job = reschedule_jobs_dict[start_time]
169 self.driver.AddLeases(job['hostname'], \
170 sfa_slice, int(job['start_time']), \
171 int(job['duration']))
174 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
178 if 'node_ids' in sfa_slice:
179 nodes = self.driver.GetNodes(sfa_slice['list_node_ids'], \
181 current_slivers = [node['hostname'] for node in nodes]
183 # remove nodes not in rspec
184 deleted_nodes = list(set(current_slivers).\
185 difference(requested_slivers))
186 # add nodes from rspec
187 #added_nodes = list(set(requested_slivers).\
188 #difference(current_slivers))
191 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
192 \r\n \r\n deleted_nodes %s"\
193 %(sfa_slice, deleted_nodes))
196 #Delete the entire experience
197 self.driver.DeleteSliceFromNodes(sfa_slice)
198 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
204 def free_egre_key(self):
206 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
207 used.add(int(tag['value']))
209 for i in range(1, 256):
214 raise KeyError("No more EGRE keys available")
223 def handle_peer(self, site, sfa_slice, persons, peer):
228 self.driver.BindObjectToPeer('site', site['site_id'], \
229 peer['shortname'], sfa_slice['site_id'])
230 except Exception, error:
231 self.driver.DeleteSite(site['site_id'])
237 self.driver.BindObjectToPeer('slice', slice['slice_id'], \
238 peer['shortname'], sfa_slice['slice_id'])
239 except Exception, error:
240 self.driver.DeleteSlice(sfa_slice['slice_id'])
244 for person in persons:
246 self.driver.BindObjectToPeer('person', \
247 person['person_id'], peer['shortname'], \
248 person['peer_person_id'])
250 for (key, remote_key_id) in zip(person['keys'], \
253 self.driver.BindObjectToPeer( 'key', \
254 key['key_id'], peer['shortname'], \
257 self.driver.DeleteKey(key['key_id'])
258 logger.log_exc("failed to bind key: %s \
259 to peer: %s " % (key['key_id'], \
261 except Exception, error:
262 self.driver.DeletePerson(person['person_id'])
267 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
268 #sfa_peer=None, options={}):
269 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
270 #site_hrn = get_authority(slice_hrn)
271 ## login base can't be longer than 20 characters
272 ##slicename = hrn_to_pl_slicename(slice_hrn)
273 #authority_name = slice_hrn.split('.')[0]
274 #login_base = authority_name[:20]
275 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
276 #login_base %s slice_hrn %s" \
277 #%(authority_name,login_base,slice_hrn)
279 #sites = self.driver.GetSites(login_base)
281 ## create new site record
282 #site = {'name': 'geni.%s' % authority_name,
283 #'abbreviated_name': authority_name,
284 #'login_base': login_base,
286 #'max_slivers': 1000,
288 #'peer_site_id': None}
290 #site['peer_site_id'] = slice_record.get('site_id', None)
291 #site['site_id'] = self.driver.AddSite(site)
292 ## exempt federated sites from monitor policies
293 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', \
296 ### is this still necessary?
297 ### add record to the local registry
298 ##if sfa_peer and slice_record:
299 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
300 ##'peer_authority': sfa_peer, 'pointer': \
302 ##self.registry.register_peer_object(self.credential, peer_dict)
306 ## unbind from peer so we can modify if necessary.
307 ## Will bind back later
308 #self.driver.UnBindObjectFromPeer('site', site['site_id'], \
313 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
315 #login_base = slice_hrn.split(".")[0]
316 slicename = slice_hrn
317 slices_list = self.driver.GetSlices(slice_filter = slicename, \
318 slice_filter_type = 'slice_hrn')
320 for sl in slices_list:
322 logger.debug("SLABSLICE \tverify_slice slicename %s slices_list %s sl %s \
323 slice_record %s"%(slicename, slices_list,sl, \
326 sfa_slice.update(slice_record)
327 #del slice['last_updated']
328 #del slice['date_created']
330 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
331 ## unbind from peer so we can modify if necessary.
332 ## Will bind back later
333 #self.driver.UnBindObjectFromPeer('slice', \
334 #slice['slice_id'], \
336 #Update existing record (e.g. expires field)
337 #it with the latest info.
338 ##if slice_record and slice['expires'] != slice_record['expires']:
339 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : \
340 #slice_record['expires']})
342 #Search for user in ldap based on email SA 14/11/12
343 ldap_user = self.driver.ldap.LdapFindUser(slice_record['user'])
344 logger.debug(" SLABSLICES \tverify_slice Oups \
345 slice_record %s peer %s sfa_peer %s ldap_user %s"\
346 %(slice_record, peer,sfa_peer ,ldap_user ))
347 #User already registered in ldap, meaning user should be in SFA db
348 #and hrn = sfa_auth+ uid
350 hrn = self.driver.root_auth +'.'+ ldap_user['uid']
352 user = self.driver.get_user(hrn)
354 logger.debug(" SLABSLICES \tverify_slice hrn %s USER %s" %(hrn, user))
355 sfa_slice = {'slice_hrn': slicename,
356 #'url': slice_record.get('url', slice_hrn),
357 #'description': slice_record.get('description', slice_hrn)
359 'authority' : slice_record['authority'],
360 'gid':slice_record['gid'],
361 'record_id_user' : user.record_id,
362 'slice_id' : slice_record['record_id'],
363 'reg-researchers':slice_record['reg-researchers'],
364 #'record_id_slice': slice_record['record_id'],
365 'peer_authority':str(peer.hrn)
370 sfa_slice['slice_id'] = slice_record['record_id']
372 self.driver.AddSlice(sfa_slice, user)
373 #slice['slice_id'] = self.driver.AddSlice(slice)
374 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
375 #slice['node_ids']=[]
376 #slice['person_ids'] = []
378 #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None)
379 # mark this slice as an sfa peer record
381 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
382 #'peer_authority': sfa_peer, 'pointer': \
384 #self.registry.register_peer_object(self.credential, peer_dict)
391 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
394 users is a record list. Records can either be local records
395 or users records from known and trusted federated sites.
396 If the user is from another site that senslab doesn't trust yet,
397 then Resolve will raise an error before getting to create_sliver.
399 #TODO SA 21/08/12 verify_persons Needs review
404 #users_dict : dict whose keys can either be the user's hrn or its id.
405 #Values contains only id and hrn
408 #First create dicts by hrn and id for each user in the user record list:
411 if 'urn' in user and (not 'hrn' in user ) :
412 user['hrn'], user['type'] = urn_to_hrn(user['urn'])
414 if 'person_id' in user and 'hrn' in user:
415 users_by_id[user['person_id']] = user
416 users_dict[user['person_id']] = {'person_id':\
417 user['person_id'], 'hrn':user['hrn']}
419 users_by_hrn[user['hrn']] = user
420 users_dict[user['hrn']] = {'person_id':user['person_id'], \
424 logger.debug( "SLABSLICE.PY \t verify_person \
425 users_dict %s \r\n user_by_hrn %s \r\n \
427 %(users_dict,users_by_hrn, users_by_id))
429 existing_user_ids = []
430 existing_user_hrns = []
432 # Check if user is in Senslab LDAP using its hrn.
433 # Assuming Senslab is centralised : one LDAP for all sites,
434 # user_id unknown from LDAP
435 # LDAP does not provide users id, therefore we rely on hrns containing
436 # the login of the user.
437 # If the hrn is not a senslab hrn, the user may not be in LDAP.
439 #Construct the list of filters (list of dicts) for GetPersons
441 for hrn in users_by_hrn:
442 filter_user.append (users_by_hrn[hrn])
443 logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " \
445 #Check user's in LDAP with GetPersons
446 #Needed because what if the user has been deleted in LDAP but
448 existing_users = self.driver.GetPersons(filter_user)
450 #User's in senslab LDAP
452 for user in existing_users :
453 existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
455 append(users_dict[user['hrn']]['person_id'])
457 # User from another known trusted federated site. Check
458 # if a senslab account matching the email has already been created.
461 if isinstance(users, list):
463 req += users[0]['email']
465 req += users['email']
467 ldap_reslt = self.driver.ldap.LdapSearch(req)
469 logger.debug(" SLABSLICE.PY \tverify_person users \
470 USER already in Senslab \t ldap_reslt %s \
472 existing_users.append(ldap_reslt[1])
475 #User not existing in LDAP
476 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
477 logger.debug(" SLABSLICE.PY \tverify_person users \
478 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
479 ldap_reslt %s " %(users, ldap_reslt))
481 requested_user_ids = users_by_id.keys()
482 requested_user_hrns = users_by_hrn.keys()
483 logger.debug("SLABSLICE.PY \tverify_person requested_user_ids %s \
484 user_by_hrn %s " %(requested_user_ids, users_by_hrn))
487 #Check that the user of the slice in the slice record
488 #matches the existing users
490 if slice_record['record_id_user'] in requested_user_ids and \
491 slice_record['PI'][0] in requested_user_hrns:
492 logger.debug(" SLABSLICE \tverify_person \
493 requested_user_ids %s = \
494 slice_record['record_id_user'] %s" \
495 %(requested_user_ids,slice_record['record_id_user']))
501 # users to be added, removed or updated
502 #One user in one senslab slice : there should be no need
503 #to remove/ add any user from/to a slice.
504 #However a user from SFA which is not registered in Senslab yet
505 #should be added to the LDAP.
507 added_user_hrns = set(requested_user_hrns).\
508 difference(set(existing_user_hrns))
510 #self.verify_keys(existing_slice_users, updated_users_list, \
515 for added_user_hrn in added_user_hrns:
516 added_user = users_dict[added_user_hrn]
517 #hrn, type = urn_to_hrn(added_user['urn'])
519 #'first_name': added_user.get('first_name', hrn),
520 #'last_name': added_user.get('last_name', hrn),
521 'first_name': added_user['first_name'],
522 'last_name': added_user['last_name'],
523 'person_id': added_user['person_id'],
524 'peer_person_id': None,
526 'key_ids': added_user.get('key_ids', []),
529 person['person_id'] = self.driver.AddPerson(person)
531 person['peer_person_id'] = added_user['person_id']
532 added_persons.append(person)
535 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
538 #self.driver.AddPersonToSite(added_user_id, login_base)
540 #for key_string in added_user.get('keys', []):
541 #key = {'key':key_string, 'key_type':'ssh'}
542 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], \
544 #person['keys'].append(key)
546 # add the registry record
548 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
550 #'pointer': person['person_id']}
551 #self.registry.register_peer_object(self.credential, peer_dict)
552 #for added_slice_user_hrn in \
553 #added_slice_user_hrns.union(added_user_hrns):
554 #self.driver.AddPersonToSlice(added_slice_user_hrn, \
555 #slice_record['name'])
556 #for added_slice_user_id in \
557 #added_slice_user_ids.union(added_user_ids):
558 # add person to the slice
559 #self.driver.AddPersonToSlice(added_slice_user_id, \
560 #slice_record['name'])
561 # if this is a peer record then it
562 # should already be bound to a peer.
563 # no need to return worry about it getting bound later
568 def verify_keys(self, persons, users, peer, options={}):
571 for person in persons:
572 key_ids.extend(person['key_ids'])
573 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
576 keydict[key['key']] = key['key_id']
577 existing_keys = keydict.keys()
579 for person in persons:
580 persondict[person['email']] = person
586 user_keys = user.get('keys', [])
587 updated_persons.append(user)
588 for key_string in user_keys:
589 requested_keys.append(key_string)
590 if key_string not in existing_keys:
591 key = {'key': key_string, 'key_type': 'ssh'}
594 person = persondict[user['email']]
595 self.driver.UnBindObjectFromPeer('person', \
596 person['person_id'], peer['shortname'])
598 self.driver.AddPersonKey(user['email'], key)
600 key_index = user_keys.index(key['key'])
601 remote_key_id = user['key_ids'][key_index]
602 self.driver.BindObjectToPeer('key', \
603 key['key_id'], peer['shortname'], \
608 self.driver.BindObjectToPeer('person', \
609 person['person_id'], peer['shortname'], \
612 # remove old keys (only if we are not appending)
613 append = options.get('append', True)
615 removed_keys = set(existing_keys).difference(requested_keys)
616 for existing_key_id in keydict:
617 if keydict[existing_key_id] in removed_keys:
620 self.driver.UnBindObjectFromPeer('key', \
621 existing_key_id, peer['shortname'])
622 self.driver.DeleteKey(existing_key_id)
625 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
626 #append=False, admin=False):
627 ## get list of attributes users ar able to manage
628 #filter = {'category': '*slice*'}
630 #filter['|roles'] = ['user']
631 #slice_attributes = self.driver.GetTagTypes(filter)
632 #valid_slice_attribute_names = [attribute['tagname'] \
633 #for attribute in slice_attributes]
635 ## get sliver attributes
636 #added_slice_attributes = []
637 #removed_slice_attributes = []
638 #ignored_slice_attribute_names = []
639 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': \
642 ## get attributes that should be removed
643 #for slice_tag in existing_slice_attributes:
644 #if slice_tag['tagname'] in ignored_slice_attribute_names:
645 ## If a slice already has a admin only role
646 ## it was probably given to them by an
647 ## admin, so we should ignore it.
648 #ignored_slice_attribute_names.append(slice_tag['tagname'])
650 ## If an existing slice attribute was not
651 ## found in the request it should
653 #attribute_found=False
654 #for requested_attribute in requested_slice_attributes:
655 #if requested_attribute['name'] == slice_tag['tagname'] \
656 #and requested_attribute['value'] == slice_tag['value']:
657 #attribute_found=True
660 #if not attribute_found and not append:
661 #removed_slice_attributes.append(slice_tag)
663 ## get attributes that should be added:
664 #for requested_attribute in requested_slice_attributes:
665 ## if the requested attribute wasn't found we should add it
666 #if requested_attribute['name'] in valid_slice_attribute_names:
667 #attribute_found = False
668 #for existing_attribute in existing_slice_attributes:
669 #if requested_attribute['name'] == \
670 #existing_attribute['tagname'] and \
671 #requested_attribute['value'] == \
672 #existing_attribute['value']:
673 #attribute_found=True
675 #if not attribute_found:
676 #added_slice_attributes.append(requested_attribute)
679 ## remove stale attributes
680 #for attribute in removed_slice_attributes:
682 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
683 #except Exception, error:
684 #self.logger.warn('Failed to remove sliver attribute. name: \
685 #%s, value: %s, node_id: %s\nCause:%s'\
686 #% (name, value, node_id, str(error)))
688 ## add requested_attributes
689 #for attribute in added_slice_attributes:
691 #self.driver.AddSliceTag(slice['name'], attribute['name'], \
692 #attribute['value'], attribute.get('node_id', None))
693 #except Exception, error:
694 #self.logger.warn('Failed to add sliver attribute. name: %s, \
695 #value: %s, node_id: %s\nCause:%s'\
696 #% (name, value, node_id, str(error)))