1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 def get_peer(self, xrn):
17 hrn, hrn_type = urn_to_hrn(xrn)
18 #Does this slice belong to a local site or a peer senslab site?
21 # get this slice's authority (site)
22 slice_authority = get_authority(hrn)
23 site_authority = slice_authority
24 # get this site's authority (sfa root authority or sub authority)
25 #site_authority = get_authority(slice_authority).lower()
26 logger.debug("SLABSLICES \ get_peer slice_authority %s \
27 site_authority %s hrn %s" %(slice_authority, \
29 #This slice belongs to the current site
30 if site_authority == self.driver.root_auth :
32 # check if we are already peered with this site_authority, if so
33 #peers = self.driver.GetPeers({})
34 peers = self.driver.GetPeers(peer_filter = slice_authority)
35 for peer_record in peers:
37 if site_authority == peer_record.hrn:
39 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
42 def get_sfa_peer(self, xrn):
43 hrn, hrn_type = urn_to_hrn(xrn)
45 # return the authority for this hrn or None if we are the authority
47 slice_authority = get_authority(hrn)
48 site_authority = get_authority(slice_authority)
50 if site_authority != self.driver.hrn:
51 sfa_peer = site_authority
56 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
58 logger.debug("SLABSLICES verify_slice_leases sfa_slice %s \
60 #First get the list of current leases from OAR
61 leases = self.driver.GetLeases({'name':sfa_slice['hrn']})
62 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
63 leases %s "%(requested_jobs_dict, leases ))
65 current_nodes_reserved_by_start_time = {}
66 requested_nodes_by_start_time = {}
67 leases_by_start_time = {}
68 reschedule_jobs_dict = {}
71 #Create reduced dictionary with key start_time and value
73 #-for the leases already registered by OAR first
74 # then for the new leases requested by the user
76 #Leases already scheduled/running in OAR
78 current_nodes_reserved_by_start_time[lease['t_from']] = \
79 lease['reserved_nodes']
80 leases_by_start_time[lease['t_from']] = lease
84 for start_time in requested_jobs_dict:
85 requested_nodes_by_start_time[int(start_time)] = \
86 requested_jobs_dict[start_time]['hostname']
87 #Check if there is any difference between the leases already
88 #registered in OAR and the requested jobs.
90 #-Lease deleted in the requested jobs
94 logger.debug("SLABSLICES verify_slice_leases \
95 requested_nodes_by_start_time %s \
96 "%(requested_nodes_by_start_time ))
97 #Find all deleted leases
99 list(set(leases_by_start_time.keys()).\
100 difference(requested_nodes_by_start_time.keys()))
101 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
102 for start_time in start_time_list]
106 #Find added or removed nodes in exisiting leases
107 for start_time in requested_nodes_by_start_time:
108 logger.debug("SLABSLICES verify_slice_leases start_time %s \
110 if start_time in current_nodes_reserved_by_start_time:
112 if requested_nodes_by_start_time[start_time] == \
113 current_nodes_reserved_by_start_time[start_time]:
118 set(requested_nodes_by_start_time[start_time])
120 update_node_set.difference(\
121 current_nodes_reserved_by_start_time[start_time])
123 update_node_set.intersection(\
124 current_nodes_reserved_by_start_time[start_time])
127 current_nodes_reserved_by_start_time[start_time])
129 old_nodes_set.difference(\
130 requested_nodes_by_start_time[start_time])
131 logger.debug("SLABSLICES verify_slice_leases \
132 shared_nodes %s added_nodes %s removed_nodes %s"\
133 %(shared_nodes, added_nodes,removed_nodes ))
134 #If the lease is modified, delete it before
136 #Add the deleted lease job id in the list
137 #WARNING :rescheduling does not work if there is already
138 # 2 running/scheduled jobs because deleting a job
139 #takes time SA 18/10/2012
140 if added_nodes or removed_nodes:
141 deleted_leases.append(\
142 leases_by_start_time[start_time]['lease_id'])
144 if added_nodes or shared_nodes:
145 reschedule_jobs_dict[str(start_time)] = \
146 requested_jobs_dict[str(start_time)]
151 job = requested_jobs_dict[str(start_time)]
152 logger.debug("SLABSLICES \
153 NEWLEASE slice %s job %s"\
155 self.driver.AddLeases(job['hostname'], \
156 sfa_slice, int(job['start_time']), \
157 int(job['duration']))
159 #Deleted leases are the ones with lease id not declared in the Rspec
161 self.driver.DeleteLeases(deleted_leases, sfa_slice['hrn'])
162 logger.debug("SLABSLICES \
163 verify_slice_leases slice %s deleted_leases %s"\
164 %(sfa_slice, deleted_leases))
167 if reschedule_jobs_dict :
168 for start_time in reschedule_jobs_dict:
169 job = reschedule_jobs_dict[start_time]
170 self.driver.AddLeases(job['hostname'], \
171 sfa_slice, int(job['start_time']), \
172 int(job['duration']))
175 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
179 if 'node_ids' in sfa_slice:
180 nodes = self.driver.GetNodes(sfa_slice['list_node_ids'], \
182 current_slivers = [node['hostname'] for node in nodes]
184 # remove nodes not in rspec
185 deleted_nodes = list(set(current_slivers).\
186 difference(requested_slivers))
187 # add nodes from rspec
188 #added_nodes = list(set(requested_slivers).\
189 #difference(current_slivers))
192 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
193 \r\n \r\n deleted_nodes %s"\
194 %(sfa_slice, deleted_nodes))
197 #Delete the entire experience
198 self.driver.DeleteSliceFromNodes(sfa_slice)
199 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
205 def free_egre_key(self):
207 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
208 used.add(int(tag['value']))
210 for i in range(1, 256):
215 raise KeyError("No more EGRE keys available")
224 def handle_peer(self, site, sfa_slice, persons, peer):
229 self.driver.BindObjectToPeer('site', site['site_id'], \
230 peer['shortname'], sfa_slice['site_id'])
231 except Exception, error:
232 self.driver.DeleteSite(site['site_id'])
238 self.driver.BindObjectToPeer('slice', slice['slice_id'], \
239 peer['shortname'], sfa_slice['slice_id'])
240 except Exception, error:
241 self.driver.DeleteSlice(sfa_slice['slice_id'])
245 for person in persons:
247 self.driver.BindObjectToPeer('person', \
248 person['person_id'], peer['shortname'], \
249 person['peer_person_id'])
251 for (key, remote_key_id) in zip(person['keys'], \
254 self.driver.BindObjectToPeer( 'key', \
255 key['key_id'], peer['shortname'], \
258 self.driver.DeleteKey(key['key_id'])
259 logger.log_exc("failed to bind key: %s \
260 to peer: %s " % (key['key_id'], \
262 except Exception, error:
263 self.driver.DeletePerson(person['person_id'])
268 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
269 #sfa_peer=None, options={}):
270 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
271 #site_hrn = get_authority(slice_hrn)
272 ## login base can't be longer than 20 characters
273 ##slicename = hrn_to_pl_slicename(slice_hrn)
274 #authority_name = slice_hrn.split('.')[0]
275 #login_base = authority_name[:20]
276 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
277 #login_base %s slice_hrn %s" \
278 #%(authority_name,login_base,slice_hrn)
280 #sites = self.driver.GetSites(login_base)
282 ## create new site record
283 #site = {'name': 'geni.%s' % authority_name,
284 #'abbreviated_name': authority_name,
285 #'login_base': login_base,
287 #'max_slivers': 1000,
289 #'peer_site_id': None}
291 #site['peer_site_id'] = slice_record.get('site_id', None)
292 #site['site_id'] = self.driver.AddSite(site)
293 ## exempt federated sites from monitor policies
294 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', \
297 ### is this still necessary?
298 ### add record to the local registry
299 ##if sfa_peer and slice_record:
300 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
301 ##'peer_authority': sfa_peer, 'pointer': \
303 ##self.registry.register_peer_object(self.credential, peer_dict)
307 ## unbind from peer so we can modify if necessary.
308 ## Will bind back later
309 #self.driver.UnBindObjectFromPeer('site', site['site_id'], \
314 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
316 #login_base = slice_hrn.split(".")[0]
317 slicename = slice_hrn
318 slices_list = self.driver.GetSlices(slice_filter = slicename, \
319 slice_filter_type = 'slice_hrn')
322 for sl in slices_list:
324 logger.debug("SLABSLICE \tverify_slice slicename %s slices_list %s sl %s \
325 slice_record %s"%(slicename, slices_list,sl, \
328 sfa_slice.update(slice_record)
329 #del slice['last_updated']
330 #del slice['date_created']
332 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
333 ## unbind from peer so we can modify if necessary.
334 ## Will bind back later
335 #self.driver.UnBindObjectFromPeer('slice', \
336 #slice['slice_id'], \
338 #Update existing record (e.g. expires field)
339 #it with the latest info.
340 ##if slice_record and slice['expires'] != slice_record['expires']:
341 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : \
342 #slice_record['expires']})
344 #Search for user in ldap based on email SA 14/11/12
345 ldap_user = self.driver.ldap.LdapFindUser(slice_record['user'])
346 logger.debug(" SLABSLICES \tverify_slice Oups \
347 slice_record %s peer %s sfa_peer %s ldap_user %s"\
348 %(slice_record, peer,sfa_peer ,ldap_user ))
349 #User already registered in ldap, meaning user should be in SFA db
350 #and hrn = sfa_auth+ uid
352 hrn = self.driver.root_auth +'.'+ ldap_user['uid']
354 user = self.driver.get_user(hrn)
356 logger.debug(" SLABSLICES \tverify_slice hrn %s USER %s" %(hrn, user))
357 sfa_slice = {'slice_hrn': slicename,
358 #'url': slice_record.get('url', slice_hrn),
359 #'description': slice_record.get('description', slice_hrn)
361 'authority' : slice_record['authority'],
362 'gid':slice_record['gid'],
363 'record_id_user' : user.record_id,
364 'slice_id' : slice_record['record_id'],
365 'reg-researchers':slice_record['reg-researchers'],
366 #'record_id_slice': slice_record['record_id'],
367 'peer_authority':str(peer.hrn)
372 sfa_slice['slice_id'] = slice_record['record_id']
375 self.driver.AddSlice(sfa_slice, user)
376 #slice['slice_id'] = self.driver.AddSlice(slice)
377 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
378 #slice['node_ids']=[]
379 #slice['person_ids'] = []
381 #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None)
382 # mark this slice as an sfa peer record
384 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
385 #'peer_authority': sfa_peer, 'pointer': \
387 #self.registry.register_peer_object(self.credential, peer_dict)
394 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
397 users is a record list. Records can either be local records
398 or users records from known and trusted federated sites.
399 If the user is from another site that senslab doesn't trust yet,
400 then Resolve will raise an error before getting to create_sliver.
402 #TODO SA 21/08/12 verify_persons Needs review
404 logger.debug("SLABSLICES \tverify_persons \tslice_hrn %s \t slice_record %s\r\n users %s \t peer %s "%( slice_hrn, slice_record, users, peer))
407 #users_dict : dict whose keys can either be the user's hrn or its id.
408 #Values contains only id and hrn
411 #First create dicts by hrn and id for each user in the user record list:
414 if 'slice_record' in info :
415 slice_rec = info['slice_record']
416 user = slice_rec['user']
421 users_by_hrn[user['hrn']] = user
422 users_dict[user['hrn']] = user
425 logger.debug( "SLABSLICE.PY \t verify_person \
426 users_dict %s \r\n user_by_hrn %s \r\n \
428 %(users_dict,users_by_hrn, users_by_id))
430 existing_user_ids = []
431 existing_user_hrns = []
433 # Check if user is in Senslab LDAP using its hrn.
434 # Assuming Senslab is centralised : one LDAP for all sites,
435 # user_id unknown from LDAP
436 # LDAP does not provide users id, therefore we rely on hrns containing
437 # the login of the user.
438 # If the hrn is not a senslab hrn, the user may not be in LDAP.
440 #Construct the list of filters (list of dicts) for GetPersons
442 for hrn in users_by_hrn:
443 filter_user.append (users_by_hrn[hrn])
444 logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " \
446 #Check user's in LDAP with GetPersons
447 #Needed because what if the user has been deleted in LDAP but
449 existing_users = self.driver.GetPersons(filter_user)
451 #User's in senslab LDAP
453 for user in existing_users :
454 existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
456 #append(users_dict[user['hrn']]['person_id'])
458 # User from another known trusted federated site. Check
459 # if a senslab account matching the email has already been created.
462 if isinstance(users, list):
464 req += users[0]['email']
466 req += users['email']
468 ldap_reslt = self.driver.ldap.LdapSearch(req)
470 logger.debug(" SLABSLICE.PY \tverify_person users \
471 USER already in Senslab \t ldap_reslt %s \
473 existing_users.append(ldap_reslt[1])
476 #User not existing in LDAP
477 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
479 #new_record['pkey'] = users[0]['keys'][0]
480 #new_record['mail'] = users[0]['email']
482 logger.debug(" SLABSLICE.PY \tverify_person users \
483 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
484 ldap_reslt %s " %(users, ldap_reslt))
486 #requested_user_ids = users_by_id.keys()
487 requested_user_hrns = users_by_hrn.keys()
488 logger.debug("SLABSLICE.PY \tverify_person \
489 user_by_hrn %s " %( users_by_hrn))
492 #Check that the user of the slice in the slice record
493 #matches the existing users
495 if slice_record['PI'][0] in requested_user_hrns:
496 #if slice_record['record_id_user'] in requested_user_ids and \
497 #slice_record['PI'][0] in requested_user_hrns:
498 logger.debug(" SLABSLICE \tverify_person \slice_record['record_id_user'] %s" \
499 %(slice_record['record_id_user']))
505 # users to be added, removed or updated
506 #One user in one senslab slice : there should be no need
507 #to remove/ add any user from/to a slice.
508 #However a user from SFA which is not registered in Senslab yet
509 #should be added to the LDAP.
511 added_user_hrns = set(requested_user_hrns).\
512 difference(set(existing_user_hrns))
514 #self.verify_keys(existing_slice_users, updated_users_list, \
519 for added_user_hrn in added_user_hrns:
520 added_user = users_dict[added_user_hrn]
521 #hrn, type = urn_to_hrn(added_user['urn'])
524 person['peer_person_id'] = None
525 k_list = ['first_name','last_name','person_id']
528 person[k] = added_user[k]
530 person['pkey'] = added_user['keys'][0]
531 person['mail'] = added_user['email']
532 person['email'] = added_user['email']
533 person['key_ids'] = added_user.get('key_ids', [])
535 #person['person_id'] = self.driver.AddPerson(person)
536 person['uid'] = self.driver.AddPerson(person)
538 #Update slice_Record with the id now known to LDAP
539 slice_record['reg-researchers'] = [self.driver.root_auth + '.' + person['uid']]
541 #person['peer_person_id'] = added_user['person_id']
542 added_persons.append(person)
545 self.driver.UpdatePerson(slice_record['reg-researchers'][0], added_user_hrn)
548 #self.driver.AddPersonToSite(added_user_id, login_base)
550 #for key_string in added_user.get('keys', []):
551 #key = {'key':key_string, 'key_type':'ssh'}
552 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], \
554 #person['keys'].append(key)
556 # add the registry record
558 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
560 #'pointer': person['person_id']}
561 #self.registry.register_peer_object(self.credential, peer_dict)
562 #for added_slice_user_hrn in \
563 #added_slice_user_hrns.union(added_user_hrns):
564 #self.driver.AddPersonToSlice(added_slice_user_hrn, \
565 #slice_record['name'])
566 #for added_slice_user_id in \
567 #added_slice_user_ids.union(added_user_ids):
568 # add person to the slice
569 #self.driver.AddPersonToSlice(added_slice_user_id, \
570 #slice_record['name'])
571 # if this is a peer record then it
572 # should already be bound to a peer.
573 # no need to return worry about it getting bound later
578 def verify_keys(self, persons, users, peer, options={}):
581 for person in persons:
582 key_ids.extend(person['key_ids'])
583 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
586 keydict[key['key']] = key['key_id']
587 existing_keys = keydict.keys()
589 for person in persons:
590 persondict[person['email']] = person
596 user_keys = user.get('keys', [])
597 updated_persons.append(user)
598 for key_string in user_keys:
599 requested_keys.append(key_string)
600 if key_string not in existing_keys:
601 key = {'key': key_string, 'key_type': 'ssh'}
604 person = persondict[user['email']]
605 self.driver.UnBindObjectFromPeer('person', \
606 person['person_id'], peer['shortname'])
608 self.driver.AddPersonKey(user['email'], key)
610 key_index = user_keys.index(key['key'])
611 remote_key_id = user['key_ids'][key_index]
612 self.driver.BindObjectToPeer('key', \
613 key['key_id'], peer['shortname'], \
618 self.driver.BindObjectToPeer('person', \
619 person['person_id'], peer['shortname'], \
622 # remove old keys (only if we are not appending)
623 append = options.get('append', True)
625 removed_keys = set(existing_keys).difference(requested_keys)
626 for existing_key_id in keydict:
627 if keydict[existing_key_id] in removed_keys:
630 self.driver.UnBindObjectFromPeer('key', \
631 existing_key_id, peer['shortname'])
632 self.driver.DeleteKey(existing_key_id)
635 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
636 #append=False, admin=False):
637 ## get list of attributes users ar able to manage
638 #filter = {'category': '*slice*'}
640 #filter['|roles'] = ['user']
641 #slice_attributes = self.driver.GetTagTypes(filter)
642 #valid_slice_attribute_names = [attribute['tagname'] \
643 #for attribute in slice_attributes]
645 ## get sliver attributes
646 #added_slice_attributes = []
647 #removed_slice_attributes = []
648 #ignored_slice_attribute_names = []
649 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': \
652 ## get attributes that should be removed
653 #for slice_tag in existing_slice_attributes:
654 #if slice_tag['tagname'] in ignored_slice_attribute_names:
655 ## If a slice already has a admin only role
656 ## it was probably given to them by an
657 ## admin, so we should ignore it.
658 #ignored_slice_attribute_names.append(slice_tag['tagname'])
660 ## If an existing slice attribute was not
661 ## found in the request it should
663 #attribute_found=False
664 #for requested_attribute in requested_slice_attributes:
665 #if requested_attribute['name'] == slice_tag['tagname'] \
666 #and requested_attribute['value'] == slice_tag['value']:
667 #attribute_found=True
670 #if not attribute_found and not append:
671 #removed_slice_attributes.append(slice_tag)
673 ## get attributes that should be added:
674 #for requested_attribute in requested_slice_attributes:
675 ## if the requested attribute wasn't found we should add it
676 #if requested_attribute['name'] in valid_slice_attribute_names:
677 #attribute_found = False
678 #for existing_attribute in existing_slice_attributes:
679 #if requested_attribute['name'] == \
680 #existing_attribute['tagname'] and \
681 #requested_attribute['value'] == \
682 #existing_attribute['value']:
683 #attribute_found=True
685 #if not attribute_found:
686 #added_slice_attributes.append(requested_attribute)
689 ## remove stale attributes
690 #for attribute in removed_slice_attributes:
692 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
693 #except Exception, error:
694 #self.logger.warn('Failed to remove sliver attribute. name: \
695 #%s, value: %s, node_id: %s\nCause:%s'\
696 #% (name, value, node_id, str(error)))
698 ## add requested_attributes
699 #for attribute in added_slice_attributes:
701 #self.driver.AddSliceTag(slice['name'], attribute['name'], \
702 #attribute['value'], attribute.get('node_id', None))
703 #except Exception, error:
704 #self.logger.warn('Failed to add sliver attribute. name: %s, \
705 #value: %s, node_id: %s\nCause:%s'\
706 #% (name, value, node_id, str(error)))