1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 def get_peer(self, xrn):
17 hrn, hrn_type = urn_to_hrn(xrn)
18 #Does this slice belong to a local site or a peer senslab site?
21 # get this slice's authority (site)
22 slice_authority = get_authority(hrn)
24 #This slice belongs to the current site
25 if slice_authority == self.driver.slab_api.root_auth:
26 site_authority = slice_authority
29 site_authority = get_authority(slice_authority).lower()
30 # get this site's authority (sfa root authority or sub authority)
32 logger.debug("SLABSLICES \ get_peer slice_authority %s \
33 site_authority %s hrn %s" %(slice_authority, \
37 # check if we are already peered with this site_authority, if so
38 #peers = self.driver.slab_api.GetPeers({})
39 peers = self.driver.slab_api.GetPeers(peer_filter = site_authority)
40 for peer_record in peers:
42 if site_authority == peer_record.hrn:
44 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
47 def get_sfa_peer(self, xrn):
48 hrn, hrn_type = urn_to_hrn(xrn)
50 # return the authority for this hrn or None if we are the authority
52 slice_authority = get_authority(hrn)
53 site_authority = get_authority(slice_authority)
55 if site_authority != self.driver.hrn:
56 sfa_peer = site_authority
61 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
63 logger.debug("SLABSLICES verify_slice_leases sfa_slice %s \
65 #First get the list of current leases from OAR
66 leases = self.driver.slab_api.GetLeases({'name':sfa_slice['hrn']})
67 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
68 leases %s "%(requested_jobs_dict, leases ))
70 current_nodes_reserved_by_start_time = {}
71 requested_nodes_by_start_time = {}
72 leases_by_start_time = {}
73 reschedule_jobs_dict = {}
76 #Create reduced dictionary with key start_time and value
78 #-for the leases already registered by OAR first
79 # then for the new leases requested by the user
81 #Leases already scheduled/running in OAR
83 current_nodes_reserved_by_start_time[lease['t_from']] = \
84 lease['reserved_nodes']
85 leases_by_start_time[lease['t_from']] = lease
89 for start_time in requested_jobs_dict:
90 requested_nodes_by_start_time[int(start_time)] = \
91 requested_jobs_dict[start_time]['hostname']
92 #Check if there is any difference between the leases already
93 #registered in OAR and the requested jobs.
95 #-Lease deleted in the requested jobs
99 logger.debug("SLABSLICES verify_slice_leases \
100 requested_nodes_by_start_time %s \
101 "%(requested_nodes_by_start_time ))
102 #Find all deleted leases
104 list(set(leases_by_start_time.keys()).\
105 difference(requested_nodes_by_start_time.keys()))
106 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
107 for start_time in start_time_list]
111 #Find added or removed nodes in exisiting leases
112 for start_time in requested_nodes_by_start_time:
113 logger.debug("SLABSLICES verify_slice_leases start_time %s \
115 if start_time in current_nodes_reserved_by_start_time:
117 if requested_nodes_by_start_time[start_time] == \
118 current_nodes_reserved_by_start_time[start_time]:
123 set(requested_nodes_by_start_time[start_time])
125 update_node_set.difference(\
126 current_nodes_reserved_by_start_time[start_time])
128 update_node_set.intersection(\
129 current_nodes_reserved_by_start_time[start_time])
132 current_nodes_reserved_by_start_time[start_time])
134 old_nodes_set.difference(\
135 requested_nodes_by_start_time[start_time])
136 logger.debug("SLABSLICES verify_slice_leases \
137 shared_nodes %s added_nodes %s removed_nodes %s"\
138 %(shared_nodes, added_nodes,removed_nodes ))
139 #If the lease is modified, delete it before
141 #Add the deleted lease job id in the list
142 #WARNING :rescheduling does not work if there is already
143 # 2 running/scheduled jobs because deleting a job
144 #takes time SA 18/10/2012
145 if added_nodes or removed_nodes:
146 deleted_leases.append(\
147 leases_by_start_time[start_time]['lease_id'])
149 if added_nodes or shared_nodes:
150 reschedule_jobs_dict[str(start_time)] = \
151 requested_jobs_dict[str(start_time)]
156 job = requested_jobs_dict[str(start_time)]
157 logger.debug("SLABSLICES \
158 NEWLEASE slice %s job %s"\
160 self.driver.slab_api.AddLeases(job['hostname'], \
161 sfa_slice, int(job['start_time']), \
162 int(job['duration']))
164 #Deleted leases are the ones with lease id not declared in the Rspec
166 self.driver.slab_api.DeleteLeases(deleted_leases, sfa_slice['hrn'])
167 logger.debug("SLABSLICES \
168 verify_slice_leases slice %s deleted_leases %s"\
169 %(sfa_slice, deleted_leases))
172 if reschedule_jobs_dict :
173 for start_time in reschedule_jobs_dict:
174 job = reschedule_jobs_dict[start_time]
175 self.driver.slab_api.AddLeases(job['hostname'], \
176 sfa_slice, int(job['start_time']), \
177 int(job['duration']))
180 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
184 if 'node_ids' in sfa_slice:
185 nodes = self.driver.slab_api.GetNodes(sfa_slice['list_node_ids'], \
187 current_slivers = [node['hostname'] for node in nodes]
189 # remove nodes not in rspec
190 deleted_nodes = list(set(current_slivers).\
191 difference(requested_slivers))
192 # add nodes from rspec
193 #added_nodes = list(set(requested_slivers).\
194 #difference(current_slivers))
197 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
198 \r\n \r\n deleted_nodes %s"\
199 %(sfa_slice, deleted_nodes))
202 #Delete the entire experience
203 self.driver.slab_api.DeleteSliceFromNodes(sfa_slice)
204 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
210 def free_egre_key(self):
212 for tag in self.driver.slab_api.GetSliceTags({'tagname': 'egre_key'}):
213 used.add(int(tag['value']))
215 for i in range(1, 256):
220 raise KeyError("No more EGRE keys available")
229 def handle_peer(self, site, sfa_slice, persons, peer):
234 self.driver.slab_api.BindObjectToPeer('site', site['site_id'], \
235 peer['shortname'], sfa_slice['site_id'])
236 except Exception, error:
237 self.driver.slab_api.DeleteSite(site['site_id'])
243 self.driver.slab_api.BindObjectToPeer('slice', slice['slice_id'], \
244 peer['shortname'], sfa_slice['slice_id'])
245 except Exception, error:
246 self.driver.slab_api.DeleteSlice(sfa_slice['slice_id'])
250 for person in persons:
252 self.driver.slab_api.BindObjectToPeer('person', \
253 person['person_id'], peer['shortname'], \
254 person['peer_person_id'])
256 for (key, remote_key_id) in zip(person['keys'], \
259 self.driver.slab_api.BindObjectToPeer( 'key', \
260 key['key_id'], peer['shortname'], \
263 self.driver.slab_api.DeleteKey(key['key_id'])
264 logger.log_exc("failed to bind key: %s \
265 to peer: %s " % (key['key_id'], \
267 except Exception, error:
268 self.driver.slab_api.DeletePerson(person['person_id'])
273 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
274 #sfa_peer=None, options={}):
275 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
276 #site_hrn = get_authority(slice_hrn)
277 ## login base can't be longer than 20 characters
278 ##slicename = hrn_to_pl_slicename(slice_hrn)
279 #authority_name = slice_hrn.split('.')[0]
280 #login_base = authority_name[:20]
281 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
282 #login_base %s slice_hrn %s" \
283 #%(authority_name,login_base,slice_hrn)
285 #sites = self.driver.slab_api.GetSites(login_base)
287 ## create new site record
288 #site = {'name': 'geni.%s' % authority_name,
289 #'abbreviated_name': authority_name,
290 #'login_base': login_base,
292 #'max_slivers': 1000,
294 #'peer_site_id': None}
296 #site['peer_site_id'] = slice_record.get('site_id', None)
297 #site['site_id'] = self.driver.slab_api.AddSite(site)
298 ## exempt federated sites from monitor policies
299 #self.driver.slab_api.AddSiteTag(site['site_id'], 'exempt_site_until', \
302 ### is this still necessary?
303 ### add record to the local registry
304 ##if sfa_peer and slice_record:
305 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
306 ##'peer_authority': sfa_peer, 'pointer': \
308 ##self.registry.register_peer_object(self.credential, peer_dict)
312 ## unbind from peer so we can modify if necessary.
313 ## Will bind back later
314 #self.driver.slab_api.UnBindObjectFromPeer('site', site['site_id'], \
319 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
321 #login_base = slice_hrn.split(".")[0]
322 slicename = slice_hrn
323 slices_list = self.driver.slab_api.GetSlices(slice_filter = slicename, \
324 slice_filter_type = 'slice_hrn')
327 for sl in slices_list:
329 logger.debug("SLABSLICE \tverify_slice slicename %s slices_list %s sl %s \
330 slice_record %s"%(slicename, slices_list,sl, \
333 sfa_slice.update(slice_record)
334 #del slice['last_updated']
335 #del slice['date_created']
337 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
338 ## unbind from peer so we can modify if necessary.
339 ## Will bind back later
340 #self.driver.slab_api.UnBindObjectFromPeer('slice', \
341 #slice['slice_id'], \
343 #Update existing record (e.g. expires field)
344 #it with the latest info.
345 ##if slice_record and slice['expires'] != slice_record['expires']:
346 ##self.driver.slab_api.UpdateSlice( slice['slice_id'], {'expires' : \
347 #slice_record['expires']})
349 #Search for user in ldap based on email SA 14/11/12
350 ldap_user = self.driver.slab_api.ldap.LdapFindUser(slice_record['user'])
351 logger.debug(" SLABSLICES \tverify_slice Oups \
352 slice_record %s peer %s sfa_peer %s ldap_user %s"\
353 %(slice_record, peer,sfa_peer ,ldap_user ))
354 #User already registered in ldap, meaning user should be in SFA db
355 #and hrn = sfa_auth+ uid
356 sfa_slice = {'hrn': slicename,
357 #'url': slice_record.get('url', slice_hrn),
358 #'description': slice_record.get('description', slice_hrn)
360 'authority' : slice_record['authority'],
361 'gid':slice_record['gid'],
362 #'record_id_user' : user.record_id,
363 'slice_id' : slice_record['record_id'],
364 'reg-researchers':slice_record['reg-researchers'],
365 #'record_id_slice': slice_record['record_id'],
366 'peer_authority':str(sfa_peer)
370 hrn = self.driver.slab_api.root_auth +'.'+ ldap_user['uid']
372 user = self.driver.get_user_record(hrn)
374 logger.debug(" SLABSLICES \tverify_slice hrn %s USER %s" %(hrn, user))
375 #sfa_slice = {'slice_hrn': slicename,
376 ##'url': slice_record.get('url', slice_hrn),
377 ##'description': slice_record.get('description', slice_hrn)
379 #'authority' : slice_record['authority'],
380 #'gid':slice_record['gid'],
381 ##'record_id_user' : user.record_id,
382 #'slice_id' : slice_record['record_id'],
383 #'reg-researchers':slice_record['reg-researchers'],
384 ##'record_id_slice': slice_record['record_id'],
385 #'peer_authority':str(peer.hrn)
390 self.driver.slab_api.AddSlice(sfa_slice, user)
393 sfa_slice['slice_id'] = slice_record['record_id']
395 #slice['slice_id'] = self.driver.slab_api.AddSlice(slice)
396 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
397 #slice['node_ids']=[]
398 #slice['person_ids'] = []
400 #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None)
401 # mark this slice as an sfa peer record
403 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
404 #'peer_authority': sfa_peer, 'pointer': \
406 #self.registry.register_peer_object(self.credential, peer_dict)
413 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
416 users is a record list. Records can either be local records
417 or users records from known and trusted federated sites.
418 If the user is from another site that senslab doesn't trust yet,
419 then Resolve will raise an error before getting to create_sliver.
421 #TODO SA 21/08/12 verify_persons Needs review
423 logger.debug("SLABSLICES \tverify_persons \tslice_hrn %s \t slice_record %s\r\n users %s \t peer %s "%( slice_hrn, slice_record, users, peer))
427 #users_dict : dict whose keys can either be the user's hrn or its id.
428 #Values contains only id and hrn
431 #First create dicts by hrn and id for each user in the user record list:
434 if 'slice_record' in info :
435 slice_rec = info['slice_record']
436 user = slice_rec['user']
439 users_by_email[user['email']] = user
440 users_dict[user['email']] = user
443 #users_by_hrn[user['hrn']] = user
444 #users_dict[user['hrn']] = user
446 logger.debug( "SLABSLICE.PY \t verify_person \
447 users_dict %s \r\n user_by_email %s \r\n \
449 %(users_dict,users_by_email, users_by_id))
451 existing_user_ids = []
452 #existing_user_hrns = []
453 existing_user_emails = []
455 # Check if user is in Senslab LDAP using its hrn.
456 # Assuming Senslab is centralised : one LDAP for all sites,
457 # user_id unknown from LDAP
458 # LDAP does not provide users id, therefore we rely on hrns containing
459 # the login of the user.
460 # If the hrn is not a senslab hrn, the user may not be in LDAP.
463 #Construct the list of filters (list of dicts) for GetPersons
465 #for hrn in users_by_hrn:
466 for email in users_by_email :
467 #filter_user.append (users_by_hrn[hrn])
468 filter_user.append (users_by_email[email])
469 #Check user's in LDAP with GetPersons
470 #Needed because what if the user has been deleted in LDAP but
472 existing_users = self.driver.slab_api.GetPersons(filter_user)
473 logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user %s existing_users %s " \
474 %(filter_user, existing_users))
475 #User's in senslab LDAP
477 for user in existing_users :
478 users_dict[user['email']].update(user)
479 existing_user_emails.append(users_dict[user['email']]['email'])
481 #existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
483 #append(users_dict[user['hrn']]['person_id'])
485 # User from another known trusted federated site. Check
486 # if a senslab account matching the email has already been created.
489 if isinstance(users, list):
491 req += users[0]['email']
493 req += users['email']
495 ldap_reslt = self.driver.slab_api.ldap.LdapSearch(req)
497 logger.debug(" SLABSLICE.PY \tverify_person users \
498 USER already in Senslab \t ldap_reslt %s \
500 existing_users.append(ldap_reslt[1])
503 #User not existing in LDAP
504 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
506 #new_record['pkey'] = users[0]['keys'][0]
507 #new_record['mail'] = users[0]['email']
509 logger.debug(" SLABSLICE.PY \tverify_person users \
510 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
511 ldap_reslt %s " %(users, ldap_reslt))
513 #requested_user_ids = users_by_id.keys()
514 #requested_user_hrns = users_by_hrn.keys()
515 requested_user_emails = users_by_email.keys()
516 logger.debug("SLABSLICE.PY \tverify_person \
517 users_by_email %s " %( users_by_email))
518 #logger.debug("SLABSLICE.PY \tverify_person \
519 #user_by_hrn %s " %( users_by_hrn))
522 #Check that the user of the slice in the slice record
523 #matches the existing users
525 if slice_record['PI'][0] in requested_user_hrns:
526 #if slice_record['record_id_user'] in requested_user_ids and \
527 #slice_record['PI'][0] in requested_user_hrns:
528 logger.debug(" SLABSLICE \tverify_person ['PI'] slice_record %s" \
535 # users to be added, removed or updated
536 #One user in one senslab slice : there should be no need
537 #to remove/ add any user from/to a slice.
538 #However a user from SFA which is not registered in Senslab yet
539 #should be added to the LDAP.
540 added_user_emails = set(requested_user_emails).\
541 difference(set(existing_user_emails))
542 #added_user_hrns = set(requested_user_hrns).\
543 #difference(set(existing_user_hrns))
545 #self.verify_keys(existing_slice_users, updated_users_list, \
551 #requested_user_email is in existing_user_emails
552 if len(added_user_emails) == 0:
554 slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
555 logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" \
558 #for added_user_hrn in added_user_hrns:
559 #added_user = users_dict[added_user_hrn]
562 for added_user_email in added_user_emails:
563 #hrn, type = urn_to_hrn(added_user['urn'])
564 added_user = users_dict[added_user_email]
565 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person added_user %s" %(added_user))
567 person['peer_person_id'] = None
568 k_list = ['first_name','last_name','person_id']
571 person[k] = added_user[k]
573 person['pkey'] = added_user['keys'][0]
574 person['mail'] = added_user['email']
575 person['email'] = added_user['email']
576 person['key_ids'] = added_user.get('key_ids', [])
577 #person['urn'] = added_user['urn']
579 #person['person_id'] = self.driver.slab_api.AddPerson(person)
580 person['uid'] = self.driver.slab_api.AddPerson(person)
582 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person ppeersonne %s" %(person))
583 #Update slice_Record with the id now known to LDAP
584 slice_record['login'] = person['uid']
585 #slice_record['reg_researchers'] = [self.driver.slab_api.root_auth + '.' + person['uid']]
586 #slice_record['reg-researchers'] = slice_record['reg_researchers']
589 #person['peer_person_id'] = added_user['person_id']
590 added_persons.append(person)
593 #self.driver.slab_api.UpdatePerson(slice_record['reg_researchers'][0], added_user_email)
596 #self.driver.slab_api.AddPersonToSite(added_user_id, login_base)
598 #for key_string in added_user.get('keys', []):
599 #key = {'key':key_string, 'key_type':'ssh'}
600 #key['key_id'] = self.driver.slab_api.AddPersonKey(person['person_id'], \
602 #person['keys'].append(key)
604 # add the registry record
606 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
608 #'pointer': person['person_id']}
609 #self.registry.register_peer_object(self.credential, peer_dict)
610 #for added_slice_user_hrn in \
611 #added_slice_user_hrns.union(added_user_hrns):
612 #self.driver.slab_api.AddPersonToSlice(added_slice_user_hrn, \
613 #slice_record['name'])
614 #for added_slice_user_id in \
615 #added_slice_user_ids.union(added_user_ids):
616 # add person to the slice
617 #self.driver.slab_api.AddPersonToSlice(added_slice_user_id, \
618 #slice_record['name'])
619 # if this is a peer record then it
620 # should already be bound to a peer.
621 # no need to return worry about it getting bound later
626 def verify_keys(self, persons, users, peer, options={}):
629 for person in persons:
630 key_ids.extend(person['key_ids'])
631 keylist = self.driver.slab_api.GetKeys(key_ids, ['key_id', 'key'])
634 keydict[key['key']] = key['key_id']
635 existing_keys = keydict.keys()
637 for person in persons:
638 persondict[person['email']] = person
644 user_keys = user.get('keys', [])
645 updated_persons.append(user)
646 for key_string in user_keys:
647 requested_keys.append(key_string)
648 if key_string not in existing_keys:
649 key = {'key': key_string, 'key_type': 'ssh'}
652 person = persondict[user['email']]
653 self.driver.slab_api.UnBindObjectFromPeer('person', \
654 person['person_id'], peer['shortname'])
656 self.driver.slab_api.AddPersonKey(user['email'], key)
658 key_index = user_keys.index(key['key'])
659 remote_key_id = user['key_ids'][key_index]
660 self.driver.slab_api.BindObjectToPeer('key', \
661 key['key_id'], peer['shortname'], \
666 self.driver.slab_api.BindObjectToPeer('person', \
667 person['person_id'], peer['shortname'], \
670 # remove old keys (only if we are not appending)
671 append = options.get('append', True)
673 removed_keys = set(existing_keys).difference(requested_keys)
674 for existing_key_id in keydict:
675 if keydict[existing_key_id] in removed_keys:
678 self.driver.slab_api.UnBindObjectFromPeer('key', \
679 existing_key_id, peer['shortname'])
680 self.driver.slab_api.DeleteKey(existing_key_id)
683 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
684 #append=False, admin=False):
685 ## get list of attributes users ar able to manage
686 #filter = {'category': '*slice*'}
688 #filter['|roles'] = ['user']
689 #slice_attributes = self.driver.slab_api.GetTagTypes(filter)
690 #valid_slice_attribute_names = [attribute['tagname'] \
691 #for attribute in slice_attributes]
693 ## get sliver attributes
694 #added_slice_attributes = []
695 #removed_slice_attributes = []
696 #ignored_slice_attribute_names = []
697 #existing_slice_attributes = self.driver.slab_api.GetSliceTags({'slice_id': \
700 ## get attributes that should be removed
701 #for slice_tag in existing_slice_attributes:
702 #if slice_tag['tagname'] in ignored_slice_attribute_names:
703 ## If a slice already has a admin only role
704 ## it was probably given to them by an
705 ## admin, so we should ignore it.
706 #ignored_slice_attribute_names.append(slice_tag['tagname'])
708 ## If an existing slice attribute was not
709 ## found in the request it should
711 #attribute_found=False
712 #for requested_attribute in requested_slice_attributes:
713 #if requested_attribute['name'] == slice_tag['tagname'] \
714 #and requested_attribute['value'] == slice_tag['value']:
715 #attribute_found=True
718 #if not attribute_found and not append:
719 #removed_slice_attributes.append(slice_tag)
721 ## get attributes that should be added:
722 #for requested_attribute in requested_slice_attributes:
723 ## if the requested attribute wasn't found we should add it
724 #if requested_attribute['name'] in valid_slice_attribute_names:
725 #attribute_found = False
726 #for existing_attribute in existing_slice_attributes:
727 #if requested_attribute['name'] == \
728 #existing_attribute['tagname'] and \
729 #requested_attribute['value'] == \
730 #existing_attribute['value']:
731 #attribute_found=True
733 #if not attribute_found:
734 #added_slice_attributes.append(requested_attribute)
737 ## remove stale attributes
738 #for attribute in removed_slice_attributes:
740 #self.driver.slab_api.DeleteSliceTag(attribute['slice_tag_id'])
741 #except Exception, error:
742 #self.logger.warn('Failed to remove sliver attribute. name: \
743 #%s, value: %s, node_id: %s\nCause:%s'\
744 #% (name, value, node_id, str(error)))
746 ## add requested_attributes
747 #for attribute in added_slice_attributes:
749 #self.driver.slab_api.AddSliceTag(slice['name'], attribute['name'], \
750 #attribute['value'], attribute.get('node_id', None))
751 #except Exception, error:
752 #self.logger.warn('Failed to add sliver attribute. name: %s, \
753 #value: %s, node_id: %s\nCause:%s'\
754 #% (name, value, node_id, str(error)))