1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 def get_peer(self, xrn):
17 hrn, hrn_type = urn_to_hrn(xrn)
18 #Does this slice belong to a local site or a peer senslab site?
21 # get this slice's authority (site)
22 slice_authority = get_authority(hrn)
23 site_authority = slice_authority
24 # get this site's authority (sfa root authority or sub authority)
25 #site_authority = get_authority(slice_authority).lower()
26 logger.debug("SLABSLICES \ get_peer slice_authority %s \
27 site_authority %s hrn %s" %(slice_authority, \
29 #This slice belongs to the current site
30 if site_authority == self.driver.slab_api.root_auth :
32 # check if we are already peered with this site_authority, if so
33 #peers = self.driver.slab_api.GetPeers({})
34 peers = self.driver.slab_api.GetPeers(peer_filter = slice_authority)
35 for peer_record in peers:
37 if site_authority == peer_record.hrn:
39 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
42 def get_sfa_peer(self, xrn):
43 hrn, hrn_type = urn_to_hrn(xrn)
45 # return the authority for this hrn or None if we are the authority
47 slice_authority = get_authority(hrn)
48 site_authority = get_authority(slice_authority)
50 if site_authority != self.driver.hrn:
51 sfa_peer = site_authority
56 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
58 logger.debug("SLABSLICES verify_slice_leases sfa_slice %s \
60 #First get the list of current leases from OAR
61 leases = self.driver.slab_api.GetLeases({'name':sfa_slice['hrn']})
62 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
63 leases %s "%(requested_jobs_dict, leases ))
65 current_nodes_reserved_by_start_time = {}
66 requested_nodes_by_start_time = {}
67 leases_by_start_time = {}
68 reschedule_jobs_dict = {}
71 #Create reduced dictionary with key start_time and value
73 #-for the leases already registered by OAR first
74 # then for the new leases requested by the user
76 #Leases already scheduled/running in OAR
78 current_nodes_reserved_by_start_time[lease['t_from']] = \
79 lease['reserved_nodes']
80 leases_by_start_time[lease['t_from']] = lease
84 for start_time in requested_jobs_dict:
85 requested_nodes_by_start_time[int(start_time)] = \
86 requested_jobs_dict[start_time]['hostname']
87 #Check if there is any difference between the leases already
88 #registered in OAR and the requested jobs.
90 #-Lease deleted in the requested jobs
94 logger.debug("SLABSLICES verify_slice_leases \
95 requested_nodes_by_start_time %s \
96 "%(requested_nodes_by_start_time ))
97 #Find all deleted leases
99 list(set(leases_by_start_time.keys()).\
100 difference(requested_nodes_by_start_time.keys()))
101 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
102 for start_time in start_time_list]
106 #Find added or removed nodes in exisiting leases
107 for start_time in requested_nodes_by_start_time:
108 logger.debug("SLABSLICES verify_slice_leases start_time %s \
110 if start_time in current_nodes_reserved_by_start_time:
112 if requested_nodes_by_start_time[start_time] == \
113 current_nodes_reserved_by_start_time[start_time]:
118 set(requested_nodes_by_start_time[start_time])
120 update_node_set.difference(\
121 current_nodes_reserved_by_start_time[start_time])
123 update_node_set.intersection(\
124 current_nodes_reserved_by_start_time[start_time])
127 current_nodes_reserved_by_start_time[start_time])
129 old_nodes_set.difference(\
130 requested_nodes_by_start_time[start_time])
131 logger.debug("SLABSLICES verify_slice_leases \
132 shared_nodes %s added_nodes %s removed_nodes %s"\
133 %(shared_nodes, added_nodes,removed_nodes ))
134 #If the lease is modified, delete it before
136 #Add the deleted lease job id in the list
137 #WARNING :rescheduling does not work if there is already
138 # 2 running/scheduled jobs because deleting a job
139 #takes time SA 18/10/2012
140 if added_nodes or removed_nodes:
141 deleted_leases.append(\
142 leases_by_start_time[start_time]['lease_id'])
144 if added_nodes or shared_nodes:
145 reschedule_jobs_dict[str(start_time)] = \
146 requested_jobs_dict[str(start_time)]
151 job = requested_jobs_dict[str(start_time)]
152 logger.debug("SLABSLICES \
153 NEWLEASE slice %s job %s"\
155 self.driver.slab_api.AddLeases(job['hostname'], \
156 sfa_slice, int(job['start_time']), \
157 int(job['duration']))
159 #Deleted leases are the ones with lease id not declared in the Rspec
161 self.driver.slab_api.DeleteLeases(deleted_leases, sfa_slice['hrn'])
162 logger.debug("SLABSLICES \
163 verify_slice_leases slice %s deleted_leases %s"\
164 %(sfa_slice, deleted_leases))
167 if reschedule_jobs_dict :
168 for start_time in reschedule_jobs_dict:
169 job = reschedule_jobs_dict[start_time]
170 self.driver.slab_api.AddLeases(job['hostname'], \
171 sfa_slice, int(job['start_time']), \
172 int(job['duration']))
175 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
179 if 'node_ids' in sfa_slice:
180 nodes = self.driver.slab_api.GetNodes(sfa_slice['list_node_ids'], \
182 current_slivers = [node['hostname'] for node in nodes]
184 # remove nodes not in rspec
185 deleted_nodes = list(set(current_slivers).\
186 difference(requested_slivers))
187 # add nodes from rspec
188 #added_nodes = list(set(requested_slivers).\
189 #difference(current_slivers))
192 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
193 \r\n \r\n deleted_nodes %s"\
194 %(sfa_slice, deleted_nodes))
197 #Delete the entire experience
198 self.driver.slab_api.DeleteSliceFromNodes(sfa_slice)
199 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
205 def free_egre_key(self):
207 for tag in self.driver.slab_api.GetSliceTags({'tagname': 'egre_key'}):
208 used.add(int(tag['value']))
210 for i in range(1, 256):
215 raise KeyError("No more EGRE keys available")
224 def handle_peer(self, site, sfa_slice, persons, peer):
229 self.driver.slab_api.BindObjectToPeer('site', site['site_id'], \
230 peer['shortname'], sfa_slice['site_id'])
231 except Exception, error:
232 self.driver.slab_api.DeleteSite(site['site_id'])
238 self.driver.slab_api.BindObjectToPeer('slice', slice['slice_id'], \
239 peer['shortname'], sfa_slice['slice_id'])
240 except Exception, error:
241 self.driver.slab_api.DeleteSlice(sfa_slice['slice_id'])
245 for person in persons:
247 self.driver.slab_api.BindObjectToPeer('person', \
248 person['person_id'], peer['shortname'], \
249 person['peer_person_id'])
251 for (key, remote_key_id) in zip(person['keys'], \
254 self.driver.slab_api.BindObjectToPeer( 'key', \
255 key['key_id'], peer['shortname'], \
258 self.driver.slab_api.DeleteKey(key['key_id'])
259 logger.log_exc("failed to bind key: %s \
260 to peer: %s " % (key['key_id'], \
262 except Exception, error:
263 self.driver.slab_api.DeletePerson(person['person_id'])
268 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
269 #sfa_peer=None, options={}):
270 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
271 #site_hrn = get_authority(slice_hrn)
272 ## login base can't be longer than 20 characters
273 ##slicename = hrn_to_pl_slicename(slice_hrn)
274 #authority_name = slice_hrn.split('.')[0]
275 #login_base = authority_name[:20]
276 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
277 #login_base %s slice_hrn %s" \
278 #%(authority_name,login_base,slice_hrn)
280 #sites = self.driver.slab_api.GetSites(login_base)
282 ## create new site record
283 #site = {'name': 'geni.%s' % authority_name,
284 #'abbreviated_name': authority_name,
285 #'login_base': login_base,
287 #'max_slivers': 1000,
289 #'peer_site_id': None}
291 #site['peer_site_id'] = slice_record.get('site_id', None)
292 #site['site_id'] = self.driver.slab_api.AddSite(site)
293 ## exempt federated sites from monitor policies
294 #self.driver.slab_api.AddSiteTag(site['site_id'], 'exempt_site_until', \
297 ### is this still necessary?
298 ### add record to the local registry
299 ##if sfa_peer and slice_record:
300 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
301 ##'peer_authority': sfa_peer, 'pointer': \
303 ##self.registry.register_peer_object(self.credential, peer_dict)
307 ## unbind from peer so we can modify if necessary.
308 ## Will bind back later
309 #self.driver.slab_api.UnBindObjectFromPeer('site', site['site_id'], \
314 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
316 #login_base = slice_hrn.split(".")[0]
317 slicename = slice_hrn
318 slices_list = self.driver.slab_api.GetSlices(slice_filter = slicename, \
319 slice_filter_type = 'slice_hrn')
322 for sl in slices_list:
324 logger.debug("SLABSLICE \tverify_slice slicename %s slices_list %s sl %s \
325 slice_record %s"%(slicename, slices_list,sl, \
328 sfa_slice.update(slice_record)
329 #del slice['last_updated']
330 #del slice['date_created']
332 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
333 ## unbind from peer so we can modify if necessary.
334 ## Will bind back later
335 #self.driver.slab_api.UnBindObjectFromPeer('slice', \
336 #slice['slice_id'], \
338 #Update existing record (e.g. expires field)
339 #it with the latest info.
340 ##if slice_record and slice['expires'] != slice_record['expires']:
341 ##self.driver.slab_api.UpdateSlice( slice['slice_id'], {'expires' : \
342 #slice_record['expires']})
344 #Search for user in ldap based on email SA 14/11/12
345 ldap_user = self.driver.slab_api.ldap.LdapFindUser(slice_record['user'])
346 logger.debug(" SLABSLICES \tverify_slice Oups \
347 slice_record %s peer %s sfa_peer %s ldap_user %s"\
348 %(slice_record, peer,sfa_peer ,ldap_user ))
349 #User already registered in ldap, meaning user should be in SFA db
350 #and hrn = sfa_auth+ uid
352 hrn = self.driver.slab_api.root_auth +'.'+ ldap_user['uid']
354 user = self.driver.slab_api.get_user_record(hrn)
356 logger.debug(" SLABSLICES \tverify_slice hrn %s USER %s" %(hrn, user))
357 sfa_slice = {'slice_hrn': slicename,
358 #'url': slice_record.get('url', slice_hrn),
359 #'description': slice_record.get('description', slice_hrn)
361 'authority' : slice_record['authority'],
362 'gid':slice_record['gid'],
363 #'record_id_user' : user.record_id,
364 'slice_id' : slice_record['record_id'],
365 'reg-researchers':slice_record['reg-researchers'],
366 #'record_id_slice': slice_record['record_id'],
367 'peer_authority':str(peer.hrn)
372 sfa_slice['slice_id'] = slice_record['record_id']
375 self.driver.slab_api.AddSlice(sfa_slice, user)
376 #slice['slice_id'] = self.driver.slab_api.AddSlice(slice)
377 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
378 #slice['node_ids']=[]
379 #slice['person_ids'] = []
381 #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None)
382 # mark this slice as an sfa peer record
384 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
385 #'peer_authority': sfa_peer, 'pointer': \
387 #self.registry.register_peer_object(self.credential, peer_dict)
394 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
397 users is a record list. Records can either be local records
398 or users records from known and trusted federated sites.
399 If the user is from another site that senslab doesn't trust yet,
400 then Resolve will raise an error before getting to create_sliver.
402 #TODO SA 21/08/12 verify_persons Needs review
404 logger.debug("SLABSLICES \tverify_persons \tslice_hrn %s \t slice_record %s\r\n users %s \t peer %s "%( slice_hrn, slice_record, users, peer))
408 #users_dict : dict whose keys can either be the user's hrn or its id.
409 #Values contains only id and hrn
412 #First create dicts by hrn and id for each user in the user record list:
415 if 'slice_record' in info :
416 slice_rec = info['slice_record']
417 user = slice_rec['user']
420 users_by_email[user['email']] = user
421 users_dict[user['email']] = user
424 #users_by_hrn[user['hrn']] = user
425 #users_dict[user['hrn']] = user
427 logger.debug( "SLABSLICE.PY \t verify_person \
428 users_dict %s \r\n user_by_email %s \r\n \
430 %(users_dict,users_by_email, users_by_id))
432 existing_user_ids = []
433 #existing_user_hrns = []
434 existing_user_emails = []
436 # Check if user is in Senslab LDAP using its hrn.
437 # Assuming Senslab is centralised : one LDAP for all sites,
438 # user_id unknown from LDAP
439 # LDAP does not provide users id, therefore we rely on hrns containing
440 # the login of the user.
441 # If the hrn is not a senslab hrn, the user may not be in LDAP.
444 #Construct the list of filters (list of dicts) for GetPersons
446 #for hrn in users_by_hrn:
447 for email in users_by_email :
448 #filter_user.append (users_by_hrn[hrn])
449 filter_user.append (users_by_email[email])
450 #Check user's in LDAP with GetPersons
451 #Needed because what if the user has been deleted in LDAP but
453 existing_users = self.driver.slab_api.GetPersons(filter_user)
454 logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user %s existing_users %s " \
455 %(filter_user, existing_users))
456 #User's in senslab LDAP
458 for user in existing_users :
459 users_dict[user['email']].update(user)
460 existing_user_emails.append(users_dict[user['email']]['email'])
462 #existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
464 #append(users_dict[user['hrn']]['person_id'])
466 # User from another known trusted federated site. Check
467 # if a senslab account matching the email has already been created.
470 if isinstance(users, list):
472 req += users[0]['email']
474 req += users['email']
476 ldap_reslt = self.driver.slab_api.ldap.LdapSearch(req)
478 logger.debug(" SLABSLICE.PY \tverify_person users \
479 USER already in Senslab \t ldap_reslt %s \
481 existing_users.append(ldap_reslt[1])
484 #User not existing in LDAP
485 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
487 #new_record['pkey'] = users[0]['keys'][0]
488 #new_record['mail'] = users[0]['email']
490 logger.debug(" SLABSLICE.PY \tverify_person users \
491 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
492 ldap_reslt %s " %(users, ldap_reslt))
494 #requested_user_ids = users_by_id.keys()
495 #requested_user_hrns = users_by_hrn.keys()
496 requested_user_emails = users_by_email.keys()
497 logger.debug("SLABSLICE.PY \tverify_person \
498 users_by_email %s " %( users_by_email))
499 #logger.debug("SLABSLICE.PY \tverify_person \
500 #user_by_hrn %s " %( users_by_hrn))
503 #Check that the user of the slice in the slice record
504 #matches the existing users
506 if slice_record['PI'][0] in requested_user_hrns:
507 #if slice_record['record_id_user'] in requested_user_ids and \
508 #slice_record['PI'][0] in requested_user_hrns:
509 logger.debug(" SLABSLICE \tverify_person ['PI'] slice_record %s" \
516 # users to be added, removed or updated
517 #One user in one senslab slice : there should be no need
518 #to remove/ add any user from/to a slice.
519 #However a user from SFA which is not registered in Senslab yet
520 #should be added to the LDAP.
521 added_user_emails = set(requested_user_emails).\
522 difference(set(existing_user_emails))
523 #added_user_hrns = set(requested_user_hrns).\
524 #difference(set(existing_user_hrns))
526 #self.verify_keys(existing_slice_users, updated_users_list, \
532 #requested_user_email is in existing_user_emails
533 if len(added_user_emails) == 0:
535 slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
536 logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" \
539 #for added_user_hrn in added_user_hrns:
540 #added_user = users_dict[added_user_hrn]
543 for added_user_email in added_user_emails:
544 #hrn, type = urn_to_hrn(added_user['urn'])
545 added_user = users_dict[added_user_email]
546 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person added_user %s" %(added_user))
548 person['peer_person_id'] = None
549 k_list = ['first_name','last_name','person_id']
552 person[k] = added_user[k]
554 person['pkey'] = added_user['keys'][0]
555 person['mail'] = added_user['email']
556 person['email'] = added_user['email']
557 person['key_ids'] = added_user.get('key_ids', [])
558 #person['urn'] = added_user['urn']
560 #person['person_id'] = self.driver.slab_api.AddPerson(person)
561 person['uid'] = self.driver.slab_api.AddPerson(person)
563 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person ppeersonne %s" %(person))
564 #Update slice_Record with the id now known to LDAP
565 slice_record['login'] = person['uid']
566 #slice_record['reg_researchers'] = [self.driver.slab_api.root_auth + '.' + person['uid']]
567 #slice_record['reg-researchers'] = slice_record['reg_researchers']
570 #person['peer_person_id'] = added_user['person_id']
571 added_persons.append(person)
574 #self.driver.slab_api.UpdatePerson(slice_record['reg_researchers'][0], added_user_email)
577 #self.driver.slab_api.AddPersonToSite(added_user_id, login_base)
579 #for key_string in added_user.get('keys', []):
580 #key = {'key':key_string, 'key_type':'ssh'}
581 #key['key_id'] = self.driver.slab_api.AddPersonKey(person['person_id'], \
583 #person['keys'].append(key)
585 # add the registry record
587 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
589 #'pointer': person['person_id']}
590 #self.registry.register_peer_object(self.credential, peer_dict)
591 #for added_slice_user_hrn in \
592 #added_slice_user_hrns.union(added_user_hrns):
593 #self.driver.slab_api.AddPersonToSlice(added_slice_user_hrn, \
594 #slice_record['name'])
595 #for added_slice_user_id in \
596 #added_slice_user_ids.union(added_user_ids):
597 # add person to the slice
598 #self.driver.slab_api.AddPersonToSlice(added_slice_user_id, \
599 #slice_record['name'])
600 # if this is a peer record then it
601 # should already be bound to a peer.
602 # no need to return worry about it getting bound later
607 def verify_keys(self, persons, users, peer, options={}):
610 for person in persons:
611 key_ids.extend(person['key_ids'])
612 keylist = self.driver.slab_api.GetKeys(key_ids, ['key_id', 'key'])
615 keydict[key['key']] = key['key_id']
616 existing_keys = keydict.keys()
618 for person in persons:
619 persondict[person['email']] = person
625 user_keys = user.get('keys', [])
626 updated_persons.append(user)
627 for key_string in user_keys:
628 requested_keys.append(key_string)
629 if key_string not in existing_keys:
630 key = {'key': key_string, 'key_type': 'ssh'}
633 person = persondict[user['email']]
634 self.driver.slab_api.UnBindObjectFromPeer('person', \
635 person['person_id'], peer['shortname'])
637 self.driver.slab_api.AddPersonKey(user['email'], key)
639 key_index = user_keys.index(key['key'])
640 remote_key_id = user['key_ids'][key_index]
641 self.driver.slab_api.BindObjectToPeer('key', \
642 key['key_id'], peer['shortname'], \
647 self.driver.slab_api.BindObjectToPeer('person', \
648 person['person_id'], peer['shortname'], \
651 # remove old keys (only if we are not appending)
652 append = options.get('append', True)
654 removed_keys = set(existing_keys).difference(requested_keys)
655 for existing_key_id in keydict:
656 if keydict[existing_key_id] in removed_keys:
659 self.driver.slab_api.UnBindObjectFromPeer('key', \
660 existing_key_id, peer['shortname'])
661 self.driver.slab_api.DeleteKey(existing_key_id)
664 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
665 #append=False, admin=False):
666 ## get list of attributes users ar able to manage
667 #filter = {'category': '*slice*'}
669 #filter['|roles'] = ['user']
670 #slice_attributes = self.driver.slab_api.GetTagTypes(filter)
671 #valid_slice_attribute_names = [attribute['tagname'] \
672 #for attribute in slice_attributes]
674 ## get sliver attributes
675 #added_slice_attributes = []
676 #removed_slice_attributes = []
677 #ignored_slice_attribute_names = []
678 #existing_slice_attributes = self.driver.slab_api.GetSliceTags({'slice_id': \
681 ## get attributes that should be removed
682 #for slice_tag in existing_slice_attributes:
683 #if slice_tag['tagname'] in ignored_slice_attribute_names:
684 ## If a slice already has a admin only role
685 ## it was probably given to them by an
686 ## admin, so we should ignore it.
687 #ignored_slice_attribute_names.append(slice_tag['tagname'])
689 ## If an existing slice attribute was not
690 ## found in the request it should
692 #attribute_found=False
693 #for requested_attribute in requested_slice_attributes:
694 #if requested_attribute['name'] == slice_tag['tagname'] \
695 #and requested_attribute['value'] == slice_tag['value']:
696 #attribute_found=True
699 #if not attribute_found and not append:
700 #removed_slice_attributes.append(slice_tag)
702 ## get attributes that should be added:
703 #for requested_attribute in requested_slice_attributes:
704 ## if the requested attribute wasn't found we should add it
705 #if requested_attribute['name'] in valid_slice_attribute_names:
706 #attribute_found = False
707 #for existing_attribute in existing_slice_attributes:
708 #if requested_attribute['name'] == \
709 #existing_attribute['tagname'] and \
710 #requested_attribute['value'] == \
711 #existing_attribute['value']:
712 #attribute_found=True
714 #if not attribute_found:
715 #added_slice_attributes.append(requested_attribute)
718 ## remove stale attributes
719 #for attribute in removed_slice_attributes:
721 #self.driver.slab_api.DeleteSliceTag(attribute['slice_tag_id'])
722 #except Exception, error:
723 #self.logger.warn('Failed to remove sliver attribute. name: \
724 #%s, value: %s, node_id: %s\nCause:%s'\
725 #% (name, value, node_id, str(error)))
727 ## add requested_attributes
728 #for attribute in added_slice_attributes:
730 #self.driver.slab_api.AddSliceTag(slice['name'], attribute['name'], \
731 #attribute['value'], attribute.get('node_id', None))
732 #except Exception, error:
733 #self.logger.warn('Failed to add sliver attribute. name: %s, \
734 #value: %s, node_id: %s\nCause:%s'\
735 #% (name, value, node_id, str(error)))