1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 def get_peer(self, xrn):
17 hrn, hrn_type = urn_to_hrn(xrn)
18 #Does this slice belong to a local site or a peer senslab site?
21 # get this slice's authority (site)
22 slice_authority = get_authority(hrn)
23 site_authority = slice_authority
24 # get this site's authority (sfa root authority or sub authority)
25 #site_authority = get_authority(slice_authority).lower()
26 logger.debug("SLABSLICES \ get_peer slice_authority %s \
27 site_authority %s hrn %s" %(slice_authority, \
29 #This slice belongs to the current site
30 if site_authority == self.driver.root_auth :
32 # check if we are already peered with this site_authority, if so
33 #peers = self.driver.GetPeers({})
34 peers = self.driver.GetPeers(peer_filter = slice_authority)
35 for peer_record in peers:
37 if site_authority == peer_record.hrn:
39 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
42 def get_sfa_peer(self, xrn):
43 hrn, hrn_type = urn_to_hrn(xrn)
45 # return the authority for this hrn or None if we are the authority
47 slice_authority = get_authority(hrn)
48 site_authority = get_authority(slice_authority)
50 if site_authority != self.driver.hrn:
51 sfa_peer = site_authority
56 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
59 #First get the list of current leases from OAR
60 leases = self.driver.GetLeases({'name':sfa_slice['slice_hrn']})
61 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
62 leases %s "%(requested_jobs_dict, leases ))
66 current_nodes_reserved_by_start_time = {}
67 requested_nodes_by_start_time = {}
68 leases_by_start_time = {}
69 #Create reduced dictionary with key start_time and value
71 #-for the leases already registered by OAR first
72 # then for the new leases requested by the user
74 #Leases already scheduled/running in OAR
76 current_nodes_reserved_by_start_time[lease['t_from']] = \
77 lease['reserved_nodes']
78 leases_by_start_time[lease['t_from']] = lease
81 for start_time in requested_jobs_dict:
82 requested_nodes_by_start_time[int(start_time)] = \
83 requested_jobs_dict[start_time]['hostname']
85 #Check if there is any difference between the leases already
86 #registered in OAR and the requested jobs.
88 #-Lease deleted in the requested jobs
93 #Find all deleted leases
95 list(set(leases_by_start_time.keys()).\
96 difference(requested_nodes_by_start_time.keys()))
97 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
98 for start_time in start_time_list]
101 reschedule_jobs_dict = {}
102 #Find added or removed nodes in exisiting leases
103 for start_time in requested_nodes_by_start_time:
104 if start_time in current_nodes_reserved_by_start_time:
106 if requested_nodes_by_start_time[start_time] == \
107 current_nodes_reserved_by_start_time[start_time]:
112 set(requested_nodes_by_start_time[start_time])
114 update_node_set.difference(\
115 current_nodes_reserved_by_start_time[start_time])
117 update_node_set.intersection(\
118 current_nodes_reserved_by_start_time[start_time])
121 current_nodes_reserved_by_start_time[start_time])
123 old_nodes_set.difference(\
124 requested_nodes_by_start_time[start_time])
125 logger.debug("SLABSLICES verify_slice_leases \
126 shared_nodes %s added_nodes %s removed_nodes %s"\
127 %(shared_nodes, added_nodes,removed_nodes ))
128 #If the lease is modified, delete it before
130 #Add the deleted lease job id in the list
131 #WARNING :rescheduling does not work if there is already
132 # 2 running/scheduled jobs because deleting a job
133 #takes time SA 18/10/2012
134 if added_nodes or removed_nodes:
135 deleted_leases.append(\
136 leases_by_start_time[start_time]['lease_id'])
138 if added_nodes or shared_nodes:
139 reschedule_jobs_dict[str(start_time)] = \
140 requested_jobs_dict[str(start_time)]
144 job = requested_jobs_dict[str(start_time)]
145 self.driver.AddLeases(job['hostname'], \
146 sfa_slice, int(job['start_time']), \
147 int(job['duration']))
149 #Deleted leases are the ones with lease id not declared in the Rspec
151 self.driver.DeleteLeases(deleted_leases, sfa_slice['slice_hrn'])
152 logger.debug("SLABSLICES \
153 verify_slice_leases slice %s deleted_leases %s"\
154 %(sfa_slice, deleted_leases))
157 if reschedule_jobs_dict :
158 for start_time in reschedule :
159 job = reschedule_jobs_dict[start_time]
160 self.driver.AddLeases(job['hostname'], \
161 sfa_slice, int(job['start_time']), \
162 int(job['duration']))
165 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
169 if 'node_ids' in sfa_slice:
170 nodes = self.driver.GetNodes(sfa_slice['list_node_ids'], \
172 current_slivers = [node['hostname'] for node in nodes]
174 # remove nodes not in rspec
175 deleted_nodes = list(set(current_slivers).\
176 difference(requested_slivers))
177 # add nodes from rspec
178 #added_nodes = list(set(requested_slivers).\
179 #difference(current_slivers))
181 #Update the table with the nodes that populate the slice
182 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
183 \r\n \r\n deleted_nodes %s"\
184 %(sfa_slice, deleted_nodes))
187 #Delete the entire experience
188 self.driver.DeleteSliceFromNodes(sfa_slice)
189 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
195 def free_egre_key(self):
197 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
198 used.add(int(tag['value']))
200 for i in range(1, 256):
205 raise KeyError("No more EGRE keys available")
214 def handle_peer(self, site, sfa_slice, persons, peer):
219 self.driver.BindObjectToPeer('site', site['site_id'], \
220 peer['shortname'], sfa_slice['site_id'])
221 except Exception, error:
222 self.driver.DeleteSite(site['site_id'])
228 self.driver.BindObjectToPeer('slice', slice['slice_id'], \
229 peer['shortname'], sfa_slice['slice_id'])
230 except Exception, error:
231 self.driver.DeleteSlice(sfa_slice['slice_id'])
235 for person in persons:
237 self.driver.BindObjectToPeer('person', \
238 person['person_id'], peer['shortname'], \
239 person['peer_person_id'])
241 for (key, remote_key_id) in zip(person['keys'], \
244 self.driver.BindObjectToPeer( 'key', \
245 key['key_id'], peer['shortname'], \
248 self.driver.DeleteKey(key['key_id'])
249 logger.log_exc("failed to bind key: %s \
250 to peer: %s " % (key['key_id'], \
252 except Exception, error:
253 self.driver.DeletePerson(person['person_id'])
258 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
259 #sfa_peer=None, options={}):
260 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
261 #site_hrn = get_authority(slice_hrn)
262 ## login base can't be longer than 20 characters
263 ##slicename = hrn_to_pl_slicename(slice_hrn)
264 #authority_name = slice_hrn.split('.')[0]
265 #login_base = authority_name[:20]
266 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
267 #login_base %s slice_hrn %s" \
268 #%(authority_name,login_base,slice_hrn)
270 #sites = self.driver.GetSites(login_base)
272 ## create new site record
273 #site = {'name': 'geni.%s' % authority_name,
274 #'abbreviated_name': authority_name,
275 #'login_base': login_base,
277 #'max_slivers': 1000,
279 #'peer_site_id': None}
281 #site['peer_site_id'] = slice_record.get('site_id', None)
282 #site['site_id'] = self.driver.AddSite(site)
283 ## exempt federated sites from monitor policies
284 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', \
287 ### is this still necessary?
288 ### add record to the local registry
289 ##if sfa_peer and slice_record:
290 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
291 ##'peer_authority': sfa_peer, 'pointer': \
293 ##self.registry.register_peer_object(self.credential, peer_dict)
297 ## unbind from peer so we can modify if necessary.
298 ## Will bind back later
299 #self.driver.UnBindObjectFromPeer('site', site['site_id'], \
304 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
306 #login_base = slice_hrn.split(".")[0]
307 slicename = slice_hrn
308 slices_list = self.driver.GetSlices(slice_filter = slicename, \
309 slice_filter_type = 'slice_hrn')
311 for sl in slices_list:
313 logger.debug("SLABSLICE \tverify_slice slicename %s sl %s \
314 slice_record %s"%(slicename, sl, \
317 sfa_slice.update(slice_record)
318 #del slice['last_updated']
319 #del slice['date_created']
321 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
322 ## unbind from peer so we can modify if necessary.
323 ## Will bind back later
324 #self.driver.UnBindObjectFromPeer('slice', \
325 #slice['slice_id'], \
327 #Update existing record (e.g. expires field)
328 #it with the latest info.
329 ##if slice_record and slice['expires'] != slice_record['expires']:
330 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : \
331 #slice_record['expires']})
333 logger.debug(" SLABSLICES \tverify_slice Oups \
334 slice_record %s peer %s sfa_peer %s "\
335 %(slice_record, peer,sfa_peer))
336 sfa_slice = {'slice_hrn': slicename,
337 #'url': slice_record.get('url', slice_hrn),
338 #'description': slice_record.get('description', slice_hrn)
340 'record_id_user' : slice_record['person_ids'][0],
341 'record_id_slice': slice_record['record_id'],
342 'peer_authority':str(peer.hrn)
346 self.driver.AddSlice(sfa_slice)
347 #slice['slice_id'] = self.driver.AddSlice(slice)
348 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
349 #slice['node_ids']=[]
350 #slice['person_ids'] = []
352 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
353 # mark this slice as an sfa peer record
355 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
356 #'peer_authority': sfa_peer, 'pointer': \
358 #self.registry.register_peer_object(self.credential, peer_dict)
365 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
368 users is a record list. Records can either be local records
369 or users records from known and trusted federated sites.
370 If the user is from another site that senslab doesn't trust yet,
371 then Resolve will raise an error before getting to create_sliver.
373 #TODO SA 21/08/12 verify_persons Needs review
378 #users_dict : dict whose keys can either be the user's hrn or its id.
379 #Values contains only id and hrn
382 #First create dicts by hrn and id for each user in the user record list:
385 if 'urn' in user and (not 'hrn' in user ) :
386 user['hrn'], user['type'] = urn_to_hrn(user['urn'])
388 if 'person_id' in user and 'hrn' in user:
389 users_by_id[user['person_id']] = user
390 users_dict[user['person_id']] = {'person_id':\
391 user['person_id'], 'hrn':user['hrn']}
393 users_by_hrn[user['hrn']] = user
394 users_dict[user['hrn']] = {'person_id':user['person_id'], \
398 logger.debug( "SLABSLICE.PY \t verify_person \
399 users_dict %s \r\n user_by_hrn %s \r\n \
401 %(users_dict,users_by_hrn, users_by_id))
403 existing_user_ids = []
404 existing_user_hrns = []
406 # Check if user is in Senslab LDAP using its hrn.
407 # Assuming Senslab is centralised : one LDAP for all sites,
408 # user_id unknown from LDAP
409 # LDAP does not provide users id, therefore we rely on hrns containing
410 # the login of the user.
411 # If the hrn is not a senslab hrn, the user may not be in LDAP.
413 #Construct the list of filters (list of dicts) for GetPersons
415 for hrn in users_by_hrn:
416 filter_user.append (users_by_hrn[hrn])
417 logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " \
419 #Check user's in LDAP with GetPersons
420 #Needed because what if the user has been deleted in LDAP but
422 existing_users = self.driver.GetPersons(filter_user)
424 #User's in senslab LDAP
426 for user in existing_users :
427 existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
429 append(users_dict[user['hrn']]['person_id'])
431 # User from another known trusted federated site. Check
432 # if a senslab account matching the email has already been created.
435 if isinstance(users, list):
437 req += users[0]['email']
439 req += users['email']
441 ldap_reslt = self.driver.ldap.LdapSearch(req)
443 logger.debug(" SLABSLICE.PY \tverify_person users \
444 USER already in Senslab \t ldap_reslt %s \
446 existing_users.append(ldap_reslt[1])
449 #User not existing in LDAP
450 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
451 logger.debug(" SLABSLICE.PY \tverify_person users \
452 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
453 ldap_reslt %s " %(users, ldap_reslt))
455 requested_user_ids = users_by_id.keys()
456 requested_user_hrns = users_by_hrn.keys()
457 logger.debug("SLABSLICE.PY \tverify_person requested_user_ids %s \
458 user_by_hrn %s " %(requested_user_ids, users_by_hrn))
461 #Check that the user of the slice in the slice record
462 #matches the existing users
464 if slice_record['record_id_user'] in requested_user_ids and \
465 slice_record['PI'][0] in requested_user_hrns:
466 logger.debug(" SLABSLICE \tverify_person \
467 requested_user_ids %s = \
468 slice_record['record_id_user'] %s" \
469 %(requested_user_ids,slice_record['record_id_user']))
475 # users to be added, removed or updated
476 #One user in one senslab slice : there should be no need
477 #to remove/ add any user from/to a slice.
478 #However a user from SFA which is not registered in Senslab yet
479 #should be added to the LDAP.
481 added_user_hrns = set(requested_user_hrns).\
482 difference(set(existing_user_hrns))
484 #self.verify_keys(existing_slice_users, updated_users_list, \
489 for added_user_hrn in added_user_hrns:
490 added_user = users_dict[added_user_hrn]
491 #hrn, type = urn_to_hrn(added_user['urn'])
493 #'first_name': added_user.get('first_name', hrn),
494 #'last_name': added_user.get('last_name', hrn),
495 'first_name': added_user['first_name'],
496 'last_name': added_user['last_name'],
497 'person_id': added_user['person_id'],
498 'peer_person_id': None,
500 'key_ids': added_user.get('key_ids', []),
503 person['person_id'] = self.driver.AddPerson(person)
505 person['peer_person_id'] = added_user['person_id']
506 added_persons.append(person)
509 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
512 #self.driver.AddPersonToSite(added_user_id, login_base)
514 #for key_string in added_user.get('keys', []):
515 #key = {'key':key_string, 'key_type':'ssh'}
516 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], \
518 #person['keys'].append(key)
520 # add the registry record
522 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
524 #'pointer': person['person_id']}
525 #self.registry.register_peer_object(self.credential, peer_dict)
526 #for added_slice_user_hrn in \
527 #added_slice_user_hrns.union(added_user_hrns):
528 #self.driver.AddPersonToSlice(added_slice_user_hrn, \
529 #slice_record['name'])
530 #for added_slice_user_id in \
531 #added_slice_user_ids.union(added_user_ids):
532 # add person to the slice
533 #self.driver.AddPersonToSlice(added_slice_user_id, \
534 #slice_record['name'])
535 # if this is a peer record then it
536 # should already be bound to a peer.
537 # no need to return worry about it getting bound later
542 def verify_keys(self, persons, users, peer, options={}):
545 for person in persons:
546 key_ids.extend(person['key_ids'])
547 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
550 keydict[key['key']] = key['key_id']
551 existing_keys = keydict.keys()
553 for person in persons:
554 persondict[person['email']] = person
560 user_keys = user.get('keys', [])
561 updated_persons.append(user)
562 for key_string in user_keys:
563 requested_keys.append(key_string)
564 if key_string not in existing_keys:
565 key = {'key': key_string, 'key_type': 'ssh'}
568 person = persondict[user['email']]
569 self.driver.UnBindObjectFromPeer('person', \
570 person['person_id'], peer['shortname'])
572 self.driver.AddPersonKey(user['email'], key)
574 key_index = user_keys.index(key['key'])
575 remote_key_id = user['key_ids'][key_index]
576 self.driver.BindObjectToPeer('key', \
577 key['key_id'], peer['shortname'], \
582 self.driver.BindObjectToPeer('person', \
583 person['person_id'], peer['shortname'], \
586 # remove old keys (only if we are not appending)
587 append = options.get('append', True)
589 removed_keys = set(existing_keys).difference(requested_keys)
590 for existing_key_id in keydict:
591 if keydict[existing_key_id] in removed_keys:
594 self.driver.UnBindObjectFromPeer('key', \
595 existing_key_id, peer['shortname'])
596 self.driver.DeleteKey(existing_key_id)
599 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
600 #append=False, admin=False):
601 ## get list of attributes users ar able to manage
602 #filter = {'category': '*slice*'}
604 #filter['|roles'] = ['user']
605 #slice_attributes = self.driver.GetTagTypes(filter)
606 #valid_slice_attribute_names = [attribute['tagname'] \
607 #for attribute in slice_attributes]
609 ## get sliver attributes
610 #added_slice_attributes = []
611 #removed_slice_attributes = []
612 #ignored_slice_attribute_names = []
613 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': \
616 ## get attributes that should be removed
617 #for slice_tag in existing_slice_attributes:
618 #if slice_tag['tagname'] in ignored_slice_attribute_names:
619 ## If a slice already has a admin only role
620 ## it was probably given to them by an
621 ## admin, so we should ignore it.
622 #ignored_slice_attribute_names.append(slice_tag['tagname'])
624 ## If an existing slice attribute was not
625 ## found in the request it should
627 #attribute_found=False
628 #for requested_attribute in requested_slice_attributes:
629 #if requested_attribute['name'] == slice_tag['tagname'] \
630 #and requested_attribute['value'] == slice_tag['value']:
631 #attribute_found=True
634 #if not attribute_found and not append:
635 #removed_slice_attributes.append(slice_tag)
637 ## get attributes that should be added:
638 #for requested_attribute in requested_slice_attributes:
639 ## if the requested attribute wasn't found we should add it
640 #if requested_attribute['name'] in valid_slice_attribute_names:
641 #attribute_found = False
642 #for existing_attribute in existing_slice_attributes:
643 #if requested_attribute['name'] == \
644 #existing_attribute['tagname'] and \
645 #requested_attribute['value'] == \
646 #existing_attribute['value']:
647 #attribute_found=True
649 #if not attribute_found:
650 #added_slice_attributes.append(requested_attribute)
653 ## remove stale attributes
654 #for attribute in removed_slice_attributes:
656 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
657 #except Exception, error:
658 #self.logger.warn('Failed to remove sliver attribute. name: \
659 #%s, value: %s, node_id: %s\nCause:%s'\
660 #% (name, value, node_id, str(error)))
662 ## add requested_attributes
663 #for attribute in added_slice_attributes:
665 #self.driver.AddSliceTag(slice['name'], attribute['name'], \
666 #attribute['value'], attribute.get('node_id', None))
667 #except Exception, error:
668 #self.logger.warn('Failed to add sliver attribute. name: %s, \
669 #value: %s, node_id: %s\nCause:%s'\
670 #% (name, value, node_id, str(error)))