1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 def get_peer(self, xrn):
17 hrn, hrn_type = urn_to_hrn(xrn)
18 #Does this slice belong to a local site or a peer senslab site?
21 # get this slice's authority (site)
22 slice_authority = get_authority(hrn)
24 #This slice belongs to the current site
25 if slice_authority == self.driver.slab_api.root_auth:
26 site_authority = slice_authority
29 site_authority = get_authority(slice_authority).lower()
30 # get this site's authority (sfa root authority or sub authority)
32 logger.debug("SLABSLICES \ get_peer slice_authority %s \
33 site_authority %s hrn %s" %(slice_authority, \
37 # check if we are already peered with this site_authority, if so
38 #peers = self.driver.slab_api.GetPeers({})
39 peers = self.driver.slab_api.GetPeers(peer_filter = site_authority)
40 for peer_record in peers:
42 if site_authority == peer_record.hrn:
44 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
47 def get_sfa_peer(self, xrn):
48 hrn, hrn_type = urn_to_hrn(xrn)
50 # return the authority for this hrn or None if we are the authority
52 slice_authority = get_authority(hrn)
53 site_authority = get_authority(slice_authority)
55 if site_authority != self.driver.hrn:
56 sfa_peer = site_authority
61 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
63 logger.debug("SLABSLICES verify_slice_leases sfa_slice %s \
65 #First get the list of current leases from OAR
66 leases = self.driver.slab_api.GetLeases({'name':sfa_slice['hrn']})
67 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
68 leases %s "%(requested_jobs_dict, leases ))
70 current_nodes_reserved_by_start_time = {}
71 requested_nodes_by_start_time = {}
72 leases_by_start_time = {}
73 reschedule_jobs_dict = {}
76 #Create reduced dictionary with key start_time and value
78 #-for the leases already registered by OAR first
79 # then for the new leases requested by the user
81 #Leases already scheduled/running in OAR
83 current_nodes_reserved_by_start_time[lease['t_from']] = \
84 lease['reserved_nodes']
85 leases_by_start_time[lease['t_from']] = lease
89 for start_time in requested_jobs_dict:
90 requested_nodes_by_start_time[int(start_time)] = \
91 requested_jobs_dict[start_time]['hostname']
92 #Check if there is any difference between the leases already
93 #registered in OAR and the requested jobs.
95 #-Lease deleted in the requested jobs
99 logger.debug("SLABSLICES verify_slice_leases \
100 requested_nodes_by_start_time %s \
101 "%(requested_nodes_by_start_time ))
102 #Find all deleted leases
104 list(set(leases_by_start_time.keys()).\
105 difference(requested_nodes_by_start_time.keys()))
106 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
107 for start_time in start_time_list]
111 #Find added or removed nodes in exisiting leases
112 for start_time in requested_nodes_by_start_time:
113 logger.debug("SLABSLICES verify_slice_leases start_time %s \
115 if start_time in current_nodes_reserved_by_start_time:
117 if requested_nodes_by_start_time[start_time] == \
118 current_nodes_reserved_by_start_time[start_time]:
123 set(requested_nodes_by_start_time[start_time])
125 update_node_set.difference(\
126 current_nodes_reserved_by_start_time[start_time])
128 update_node_set.intersection(\
129 current_nodes_reserved_by_start_time[start_time])
132 current_nodes_reserved_by_start_time[start_time])
134 old_nodes_set.difference(\
135 requested_nodes_by_start_time[start_time])
136 logger.debug("SLABSLICES verify_slice_leases \
137 shared_nodes %s added_nodes %s removed_nodes %s"\
138 %(shared_nodes, added_nodes,removed_nodes ))
139 #If the lease is modified, delete it before
141 #Add the deleted lease job id in the list
142 #WARNING :rescheduling does not work if there is already
143 # 2 running/scheduled jobs because deleting a job
144 #takes time SA 18/10/2012
145 if added_nodes or removed_nodes:
146 deleted_leases.append(\
147 leases_by_start_time[start_time]['lease_id'])
149 if added_nodes or shared_nodes:
150 reschedule_jobs_dict[str(start_time)] = \
151 requested_jobs_dict[str(start_time)]
156 job = requested_jobs_dict[str(start_time)]
157 logger.debug("SLABSLICES \
158 NEWLEASE slice %s job %s"\
160 self.driver.slab_api.AddLeases(job['hostname'], \
161 sfa_slice, int(job['start_time']), \
162 int(job['duration']))
164 #Deleted leases are the ones with lease id not declared in the Rspec
166 self.driver.slab_api.DeleteLeases(deleted_leases, sfa_slice['hrn'])
167 logger.debug("SLABSLICES \
168 verify_slice_leases slice %s deleted_leases %s"\
169 %(sfa_slice, deleted_leases))
172 if reschedule_jobs_dict :
173 for start_time in reschedule_jobs_dict:
174 job = reschedule_jobs_dict[start_time]
175 self.driver.slab_api.AddLeases(job['hostname'], \
176 sfa_slice, int(job['start_time']), \
177 int(job['duration']))
180 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
184 if 'node_ids' in sfa_slice:
185 nodes = self.driver.slab_api.GetNodes(sfa_slice['list_node_ids'], \
187 current_slivers = [node['hostname'] for node in nodes]
189 # remove nodes not in rspec
190 deleted_nodes = list(set(current_slivers).\
191 difference(requested_slivers))
192 # add nodes from rspec
193 #added_nodes = list(set(requested_slivers).\
194 #difference(current_slivers))
197 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
198 \r\n \r\n deleted_nodes %s"\
199 %(sfa_slice, deleted_nodes))
202 #Delete the entire experience
203 self.driver.slab_api.DeleteSliceFromNodes(sfa_slice)
204 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
210 def free_egre_key(self):
212 for tag in self.driver.slab_api.GetSliceTags({'tagname': 'egre_key'}):
213 used.add(int(tag['value']))
215 for i in range(1, 256):
220 raise KeyError("No more EGRE keys available")
229 def handle_peer(self, site, sfa_slice, persons, peer):
234 self.driver.slab_api.BindObjectToPeer('site', site['site_id'], \
235 peer['shortname'], sfa_slice['site_id'])
236 except Exception, error:
237 self.driver.slab_api.DeleteSite(site['site_id'])
243 self.driver.slab_api.BindObjectToPeer('slice', slice['slice_id'], \
244 peer['shortname'], sfa_slice['slice_id'])
245 except Exception, error:
246 self.driver.slab_api.DeleteSlice(sfa_slice['slice_id'])
250 for person in persons:
252 self.driver.slab_api.BindObjectToPeer('person', \
253 person['person_id'], peer['shortname'], \
254 person['peer_person_id'])
256 for (key, remote_key_id) in zip(person['keys'], \
259 self.driver.slab_api.BindObjectToPeer( 'key', \
260 key['key_id'], peer['shortname'], \
263 self.driver.slab_api.DeleteKey(key['key_id'])
264 logger.log_exc("failed to bind key: %s \
265 to peer: %s " % (key['key_id'], \
267 except Exception, error:
268 self.driver.slab_api.DeletePerson(person['person_id'])
273 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
274 #sfa_peer=None, options={}):
275 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
276 #site_hrn = get_authority(slice_hrn)
277 ## login base can't be longer than 20 characters
278 ##slicename = hrn_to_pl_slicename(slice_hrn)
279 #authority_name = slice_hrn.split('.')[0]
280 #login_base = authority_name[:20]
281 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
282 #login_base %s slice_hrn %s" \
283 #%(authority_name,login_base,slice_hrn)
285 #sites = self.driver.slab_api.GetSites(login_base)
287 ## create new site record
288 #site = {'name': 'geni.%s' % authority_name,
289 #'abbreviated_name': authority_name,
290 #'login_base': login_base,
292 #'max_slivers': 1000,
294 #'peer_site_id': None}
296 #site['peer_site_id'] = slice_record.get('site_id', None)
297 #site['site_id'] = self.driver.slab_api.AddSite(site)
298 ## exempt federated sites from monitor policies
299 #self.driver.slab_api.AddSiteTag(site['site_id'], 'exempt_site_until', \
302 ### is this still necessary?
303 ### add record to the local registry
304 ##if sfa_peer and slice_record:
305 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
306 ##'peer_authority': sfa_peer, 'pointer': \
308 ##self.registry.register_peer_object(self.credential, peer_dict)
312 ## unbind from peer so we can modify if necessary.
313 ## Will bind back later
314 #self.driver.slab_api.UnBindObjectFromPeer('site', site['site_id'], \
319 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
321 #login_base = slice_hrn.split(".")[0]
322 slicename = slice_hrn
323 slices_list = self.driver.slab_api.GetSlices(slice_filter = slicename, \
324 slice_filter_type = 'slice_hrn')
327 for sl in slices_list:
329 logger.debug("SLABSLICE \tverify_slice slicename %s slices_list %s sl %s \
330 slice_record %s"%(slicename, slices_list,sl, \
333 sfa_slice.update(slice_record)
334 #del slice['last_updated']
335 #del slice['date_created']
337 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
338 ## unbind from peer so we can modify if necessary.
339 ## Will bind back later
340 #self.driver.slab_api.UnBindObjectFromPeer('slice', \
341 #slice['slice_id'], \
343 #Update existing record (e.g. expires field)
344 #it with the latest info.
345 ##if slice_record and slice['expires'] != slice_record['expires']:
346 ##self.driver.slab_api.UpdateSlice( slice['slice_id'], {'expires' : \
347 #slice_record['expires']})
349 #Search for user in ldap based on email SA 14/11/12
350 ldap_user = self.driver.slab_api.ldap.LdapFindUser(slice_record['user'])
351 logger.debug(" SLABSLICES \tverify_slice Oups \
352 slice_record %s peer %s sfa_peer %s ldap_user %s"\
353 %(slice_record, peer,sfa_peer ,ldap_user ))
354 #User already registered in ldap, meaning user should be in SFA db
355 #and hrn = sfa_auth+ uid
357 hrn = self.driver.slab_api.root_auth +'.'+ ldap_user['uid']
359 user = self.driver.get_user_record(hrn)
361 logger.debug(" SLABSLICES \tverify_slice hrn %s USER %s" %(hrn, user))
362 sfa_slice = {'slice_hrn': slicename,
363 #'url': slice_record.get('url', slice_hrn),
364 #'description': slice_record.get('description', slice_hrn)
366 'authority' : slice_record['authority'],
367 'gid':slice_record['gid'],
368 #'record_id_user' : user.record_id,
369 'slice_id' : slice_record['record_id'],
370 'reg-researchers':slice_record['reg-researchers'],
371 #'record_id_slice': slice_record['record_id'],
372 'peer_authority':str(peer.hrn)
377 sfa_slice['slice_id'] = slice_record['record_id']
380 self.driver.slab_api.AddSlice(sfa_slice, user)
381 #slice['slice_id'] = self.driver.slab_api.AddSlice(slice)
382 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
383 #slice['node_ids']=[]
384 #slice['person_ids'] = []
386 #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None)
387 # mark this slice as an sfa peer record
389 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
390 #'peer_authority': sfa_peer, 'pointer': \
392 #self.registry.register_peer_object(self.credential, peer_dict)
399 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
402 users is a record list. Records can either be local records
403 or users records from known and trusted federated sites.
404 If the user is from another site that senslab doesn't trust yet,
405 then Resolve will raise an error before getting to create_sliver.
407 #TODO SA 21/08/12 verify_persons Needs review
409 logger.debug("SLABSLICES \tverify_persons \tslice_hrn %s \t slice_record %s\r\n users %s \t peer %s "%( slice_hrn, slice_record, users, peer))
413 #users_dict : dict whose keys can either be the user's hrn or its id.
414 #Values contains only id and hrn
417 #First create dicts by hrn and id for each user in the user record list:
420 if 'slice_record' in info :
421 slice_rec = info['slice_record']
422 user = slice_rec['user']
425 users_by_email[user['email']] = user
426 users_dict[user['email']] = user
429 #users_by_hrn[user['hrn']] = user
430 #users_dict[user['hrn']] = user
432 logger.debug( "SLABSLICE.PY \t verify_person \
433 users_dict %s \r\n user_by_email %s \r\n \
435 %(users_dict,users_by_email, users_by_id))
437 existing_user_ids = []
438 #existing_user_hrns = []
439 existing_user_emails = []
441 # Check if user is in Senslab LDAP using its hrn.
442 # Assuming Senslab is centralised : one LDAP for all sites,
443 # user_id unknown from LDAP
444 # LDAP does not provide users id, therefore we rely on hrns containing
445 # the login of the user.
446 # If the hrn is not a senslab hrn, the user may not be in LDAP.
449 #Construct the list of filters (list of dicts) for GetPersons
451 #for hrn in users_by_hrn:
452 for email in users_by_email :
453 #filter_user.append (users_by_hrn[hrn])
454 filter_user.append (users_by_email[email])
455 #Check user's in LDAP with GetPersons
456 #Needed because what if the user has been deleted in LDAP but
458 existing_users = self.driver.slab_api.GetPersons(filter_user)
459 logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user %s existing_users %s " \
460 %(filter_user, existing_users))
461 #User's in senslab LDAP
463 for user in existing_users :
464 users_dict[user['email']].update(user)
465 existing_user_emails.append(users_dict[user['email']]['email'])
467 #existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
469 #append(users_dict[user['hrn']]['person_id'])
471 # User from another known trusted federated site. Check
472 # if a senslab account matching the email has already been created.
475 if isinstance(users, list):
477 req += users[0]['email']
479 req += users['email']
481 ldap_reslt = self.driver.slab_api.ldap.LdapSearch(req)
483 logger.debug(" SLABSLICE.PY \tverify_person users \
484 USER already in Senslab \t ldap_reslt %s \
486 existing_users.append(ldap_reslt[1])
489 #User not existing in LDAP
490 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
492 #new_record['pkey'] = users[0]['keys'][0]
493 #new_record['mail'] = users[0]['email']
495 logger.debug(" SLABSLICE.PY \tverify_person users \
496 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
497 ldap_reslt %s " %(users, ldap_reslt))
499 #requested_user_ids = users_by_id.keys()
500 #requested_user_hrns = users_by_hrn.keys()
501 requested_user_emails = users_by_email.keys()
502 logger.debug("SLABSLICE.PY \tverify_person \
503 users_by_email %s " %( users_by_email))
504 #logger.debug("SLABSLICE.PY \tverify_person \
505 #user_by_hrn %s " %( users_by_hrn))
508 #Check that the user of the slice in the slice record
509 #matches the existing users
511 if slice_record['PI'][0] in requested_user_hrns:
512 #if slice_record['record_id_user'] in requested_user_ids and \
513 #slice_record['PI'][0] in requested_user_hrns:
514 logger.debug(" SLABSLICE \tverify_person ['PI'] slice_record %s" \
521 # users to be added, removed or updated
522 #One user in one senslab slice : there should be no need
523 #to remove/ add any user from/to a slice.
524 #However a user from SFA which is not registered in Senslab yet
525 #should be added to the LDAP.
526 added_user_emails = set(requested_user_emails).\
527 difference(set(existing_user_emails))
528 #added_user_hrns = set(requested_user_hrns).\
529 #difference(set(existing_user_hrns))
531 #self.verify_keys(existing_slice_users, updated_users_list, \
537 #requested_user_email is in existing_user_emails
538 if len(added_user_emails) == 0:
540 slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
541 logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" \
544 #for added_user_hrn in added_user_hrns:
545 #added_user = users_dict[added_user_hrn]
548 for added_user_email in added_user_emails:
549 #hrn, type = urn_to_hrn(added_user['urn'])
550 added_user = users_dict[added_user_email]
551 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person added_user %s" %(added_user))
553 person['peer_person_id'] = None
554 k_list = ['first_name','last_name','person_id']
557 person[k] = added_user[k]
559 person['pkey'] = added_user['keys'][0]
560 person['mail'] = added_user['email']
561 person['email'] = added_user['email']
562 person['key_ids'] = added_user.get('key_ids', [])
563 #person['urn'] = added_user['urn']
565 #person['person_id'] = self.driver.slab_api.AddPerson(person)
566 person['uid'] = self.driver.slab_api.AddPerson(person)
568 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person ppeersonne %s" %(person))
569 #Update slice_Record with the id now known to LDAP
570 slice_record['login'] = person['uid']
571 #slice_record['reg_researchers'] = [self.driver.slab_api.root_auth + '.' + person['uid']]
572 #slice_record['reg-researchers'] = slice_record['reg_researchers']
575 #person['peer_person_id'] = added_user['person_id']
576 added_persons.append(person)
579 #self.driver.slab_api.UpdatePerson(slice_record['reg_researchers'][0], added_user_email)
582 #self.driver.slab_api.AddPersonToSite(added_user_id, login_base)
584 #for key_string in added_user.get('keys', []):
585 #key = {'key':key_string, 'key_type':'ssh'}
586 #key['key_id'] = self.driver.slab_api.AddPersonKey(person['person_id'], \
588 #person['keys'].append(key)
590 # add the registry record
592 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
594 #'pointer': person['person_id']}
595 #self.registry.register_peer_object(self.credential, peer_dict)
596 #for added_slice_user_hrn in \
597 #added_slice_user_hrns.union(added_user_hrns):
598 #self.driver.slab_api.AddPersonToSlice(added_slice_user_hrn, \
599 #slice_record['name'])
600 #for added_slice_user_id in \
601 #added_slice_user_ids.union(added_user_ids):
602 # add person to the slice
603 #self.driver.slab_api.AddPersonToSlice(added_slice_user_id, \
604 #slice_record['name'])
605 # if this is a peer record then it
606 # should already be bound to a peer.
607 # no need to return worry about it getting bound later
612 def verify_keys(self, persons, users, peer, options={}):
615 for person in persons:
616 key_ids.extend(person['key_ids'])
617 keylist = self.driver.slab_api.GetKeys(key_ids, ['key_id', 'key'])
620 keydict[key['key']] = key['key_id']
621 existing_keys = keydict.keys()
623 for person in persons:
624 persondict[person['email']] = person
630 user_keys = user.get('keys', [])
631 updated_persons.append(user)
632 for key_string in user_keys:
633 requested_keys.append(key_string)
634 if key_string not in existing_keys:
635 key = {'key': key_string, 'key_type': 'ssh'}
638 person = persondict[user['email']]
639 self.driver.slab_api.UnBindObjectFromPeer('person', \
640 person['person_id'], peer['shortname'])
642 self.driver.slab_api.AddPersonKey(user['email'], key)
644 key_index = user_keys.index(key['key'])
645 remote_key_id = user['key_ids'][key_index]
646 self.driver.slab_api.BindObjectToPeer('key', \
647 key['key_id'], peer['shortname'], \
652 self.driver.slab_api.BindObjectToPeer('person', \
653 person['person_id'], peer['shortname'], \
656 # remove old keys (only if we are not appending)
657 append = options.get('append', True)
659 removed_keys = set(existing_keys).difference(requested_keys)
660 for existing_key_id in keydict:
661 if keydict[existing_key_id] in removed_keys:
664 self.driver.slab_api.UnBindObjectFromPeer('key', \
665 existing_key_id, peer['shortname'])
666 self.driver.slab_api.DeleteKey(existing_key_id)
669 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
670 #append=False, admin=False):
671 ## get list of attributes users ar able to manage
672 #filter = {'category': '*slice*'}
674 #filter['|roles'] = ['user']
675 #slice_attributes = self.driver.slab_api.GetTagTypes(filter)
676 #valid_slice_attribute_names = [attribute['tagname'] \
677 #for attribute in slice_attributes]
679 ## get sliver attributes
680 #added_slice_attributes = []
681 #removed_slice_attributes = []
682 #ignored_slice_attribute_names = []
683 #existing_slice_attributes = self.driver.slab_api.GetSliceTags({'slice_id': \
686 ## get attributes that should be removed
687 #for slice_tag in existing_slice_attributes:
688 #if slice_tag['tagname'] in ignored_slice_attribute_names:
689 ## If a slice already has a admin only role
690 ## it was probably given to them by an
691 ## admin, so we should ignore it.
692 #ignored_slice_attribute_names.append(slice_tag['tagname'])
694 ## If an existing slice attribute was not
695 ## found in the request it should
697 #attribute_found=False
698 #for requested_attribute in requested_slice_attributes:
699 #if requested_attribute['name'] == slice_tag['tagname'] \
700 #and requested_attribute['value'] == slice_tag['value']:
701 #attribute_found=True
704 #if not attribute_found and not append:
705 #removed_slice_attributes.append(slice_tag)
707 ## get attributes that should be added:
708 #for requested_attribute in requested_slice_attributes:
709 ## if the requested attribute wasn't found we should add it
710 #if requested_attribute['name'] in valid_slice_attribute_names:
711 #attribute_found = False
712 #for existing_attribute in existing_slice_attributes:
713 #if requested_attribute['name'] == \
714 #existing_attribute['tagname'] and \
715 #requested_attribute['value'] == \
716 #existing_attribute['value']:
717 #attribute_found=True
719 #if not attribute_found:
720 #added_slice_attributes.append(requested_attribute)
723 ## remove stale attributes
724 #for attribute in removed_slice_attributes:
726 #self.driver.slab_api.DeleteSliceTag(attribute['slice_tag_id'])
727 #except Exception, error:
728 #self.logger.warn('Failed to remove sliver attribute. name: \
729 #%s, value: %s, node_id: %s\nCause:%s'\
730 #% (name, value, node_id, str(error)))
732 ## add requested_attributes
733 #for attribute in added_slice_attributes:
735 #self.driver.slab_api.AddSliceTag(slice['name'], attribute['name'], \
736 #attribute['value'], attribute.get('node_id', None))
737 #except Exception, error:
738 #self.logger.warn('Failed to add sliver attribute. name: %s, \
739 #value: %s, node_id: %s\nCause:%s'\
740 #% (name, value, node_id, str(error)))