1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
16 def get_peer(self, xrn):
17 hrn, hrn_type = urn_to_hrn(xrn)
18 #Does this slice belong to a local site or a peer senslab site?
21 # get this slice's authority (site)
22 slice_authority = get_authority(hrn)
23 site_authority = slice_authority
24 # get this site's authority (sfa root authority or sub authority)
25 #site_authority = get_authority(slice_authority).lower()
26 logger.debug("SLABSLICES \ get_peer slice_authority %s \
27 site_authority %s hrn %s" %(slice_authority, \
29 #This slice belongs to the current site
30 if site_authority == self.driver.root_auth :
32 # check if we are already peered with this site_authority, if so
33 #peers = self.driver.GetPeers({})
34 peers = self.driver.GetPeers(peer_filter = slice_authority)
35 for peer_record in peers:
37 if site_authority == peer_record.hrn:
39 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
42 def get_sfa_peer(self, xrn):
43 hrn, hrn_type = urn_to_hrn(xrn)
45 # return the authority for this hrn or None if we are the authority
47 slice_authority = get_authority(hrn)
48 site_authority = get_authority(slice_authority)
50 if site_authority != self.driver.hrn:
51 sfa_peer = site_authority
56 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
59 #First get the list of current leases from OAR
60 leases = self.driver.GetLeases({'name':sfa_slice['slice_hrn']})
61 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
62 leases %s "%(requested_jobs_dict, leases ))
64 current_nodes_reserved_by_start_time = {}
65 requested_nodes_by_start_time = {}
66 leases_by_start_time = {}
67 reschedule_jobs_dict = {}
70 #Create reduced dictionary with key start_time and value
72 #-for the leases already registered by OAR first
73 # then for the new leases requested by the user
75 #Leases already scheduled/running in OAR
77 current_nodes_reserved_by_start_time[lease['t_from']] = \
78 lease['reserved_nodes']
79 leases_by_start_time[lease['t_from']] = lease
83 for start_time in requested_jobs_dict:
84 requested_nodes_by_start_time[int(start_time)] = \
85 requested_jobs_dict[start_time]['hostname']
86 #Check if there is any difference between the leases already
87 #registered in OAR and the requested jobs.
89 #-Lease deleted in the requested jobs
93 logger.debug("SLABSLICES verify_slice_leases requested_nodes_by_start_time %s \
94 "%(requested_nodes_by_start_time ))
95 #Find all deleted leases
97 list(set(leases_by_start_time.keys()).\
98 difference(requested_nodes_by_start_time.keys()))
99 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
100 for start_time in start_time_list]
104 #Find added or removed nodes in exisiting leases
105 for start_time in requested_nodes_by_start_time:
106 logger.debug("SLABSLICES verify_slice_leases start_time %s \
108 if start_time in current_nodes_reserved_by_start_time:
110 if requested_nodes_by_start_time[start_time] == \
111 current_nodes_reserved_by_start_time[start_time]:
116 set(requested_nodes_by_start_time[start_time])
118 update_node_set.difference(\
119 current_nodes_reserved_by_start_time[start_time])
121 update_node_set.intersection(\
122 current_nodes_reserved_by_start_time[start_time])
125 current_nodes_reserved_by_start_time[start_time])
127 old_nodes_set.difference(\
128 requested_nodes_by_start_time[start_time])
129 logger.debug("SLABSLICES verify_slice_leases \
130 shared_nodes %s added_nodes %s removed_nodes %s"\
131 %(shared_nodes, added_nodes,removed_nodes ))
132 #If the lease is modified, delete it before
134 #Add the deleted lease job id in the list
135 #WARNING :rescheduling does not work if there is already
136 # 2 running/scheduled jobs because deleting a job
137 #takes time SA 18/10/2012
138 if added_nodes or removed_nodes:
139 deleted_leases.append(\
140 leases_by_start_time[start_time]['lease_id'])
142 if added_nodes or shared_nodes:
143 reschedule_jobs_dict[str(start_time)] = \
144 requested_jobs_dict[str(start_time)]
149 job = requested_jobs_dict[str(start_time)]
150 logger.debug("SLABSLICES \
151 NEWLEASE slice %s job %s"\
153 self.driver.AddLeases(job['hostname'], \
154 sfa_slice, int(job['start_time']), \
155 int(job['duration']))
157 #Deleted leases are the ones with lease id not declared in the Rspec
159 self.driver.DeleteLeases(deleted_leases, sfa_slice['slice_hrn'])
160 logger.debug("SLABSLICES \
161 verify_slice_leases slice %s deleted_leases %s"\
162 %(sfa_slice, deleted_leases))
165 if reschedule_jobs_dict :
166 for start_time in reschedule :
167 job = reschedule_jobs_dict[start_time]
168 self.driver.AddLeases(job['hostname'], \
169 sfa_slice, int(job['start_time']), \
170 int(job['duration']))
173 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
177 if 'node_ids' in sfa_slice:
178 nodes = self.driver.GetNodes(sfa_slice['list_node_ids'], \
180 current_slivers = [node['hostname'] for node in nodes]
182 # remove nodes not in rspec
183 deleted_nodes = list(set(current_slivers).\
184 difference(requested_slivers))
185 # add nodes from rspec
186 #added_nodes = list(set(requested_slivers).\
187 #difference(current_slivers))
189 #Update the table with the nodes that populate the slice
190 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
191 \r\n \r\n deleted_nodes %s"\
192 %(sfa_slice, deleted_nodes))
195 #Delete the entire experience
196 self.driver.DeleteSliceFromNodes(sfa_slice)
197 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
203 def free_egre_key(self):
205 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
206 used.add(int(tag['value']))
208 for i in range(1, 256):
213 raise KeyError("No more EGRE keys available")
222 def handle_peer(self, site, sfa_slice, persons, peer):
227 self.driver.BindObjectToPeer('site', site['site_id'], \
228 peer['shortname'], sfa_slice['site_id'])
229 except Exception, error:
230 self.driver.DeleteSite(site['site_id'])
236 self.driver.BindObjectToPeer('slice', slice['slice_id'], \
237 peer['shortname'], sfa_slice['slice_id'])
238 except Exception, error:
239 self.driver.DeleteSlice(sfa_slice['slice_id'])
243 for person in persons:
245 self.driver.BindObjectToPeer('person', \
246 person['person_id'], peer['shortname'], \
247 person['peer_person_id'])
249 for (key, remote_key_id) in zip(person['keys'], \
252 self.driver.BindObjectToPeer( 'key', \
253 key['key_id'], peer['shortname'], \
256 self.driver.DeleteKey(key['key_id'])
257 logger.log_exc("failed to bind key: %s \
258 to peer: %s " % (key['key_id'], \
260 except Exception, error:
261 self.driver.DeletePerson(person['person_id'])
266 #def verify_site(self, slice_xrn, slice_record={}, peer=None, \
267 #sfa_peer=None, options={}):
268 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
269 #site_hrn = get_authority(slice_hrn)
270 ## login base can't be longer than 20 characters
271 ##slicename = hrn_to_pl_slicename(slice_hrn)
272 #authority_name = slice_hrn.split('.')[0]
273 #login_base = authority_name[:20]
274 #logger.debug(" SLABSLICES.PY \tverify_site authority_name %s \
275 #login_base %s slice_hrn %s" \
276 #%(authority_name,login_base,slice_hrn)
278 #sites = self.driver.GetSites(login_base)
280 ## create new site record
281 #site = {'name': 'geni.%s' % authority_name,
282 #'abbreviated_name': authority_name,
283 #'login_base': login_base,
285 #'max_slivers': 1000,
287 #'peer_site_id': None}
289 #site['peer_site_id'] = slice_record.get('site_id', None)
290 #site['site_id'] = self.driver.AddSite(site)
291 ## exempt federated sites from monitor policies
292 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', \
295 ### is this still necessary?
296 ### add record to the local registry
297 ##if sfa_peer and slice_record:
298 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
299 ##'peer_authority': sfa_peer, 'pointer': \
301 ##self.registry.register_peer_object(self.credential, peer_dict)
305 ## unbind from peer so we can modify if necessary.
306 ## Will bind back later
307 #self.driver.UnBindObjectFromPeer('site', site['site_id'], \
312 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
314 #login_base = slice_hrn.split(".")[0]
315 slicename = slice_hrn
316 slices_list = self.driver.GetSlices(slice_filter = slicename, \
317 slice_filter_type = 'slice_hrn')
319 for sl in slices_list:
321 logger.debug("SLABSLICE \tverify_slice slicename %s sl %s \
322 slice_record %s"%(slicename, sl, \
325 sfa_slice.update(slice_record)
326 #del slice['last_updated']
327 #del slice['date_created']
329 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
330 ## unbind from peer so we can modify if necessary.
331 ## Will bind back later
332 #self.driver.UnBindObjectFromPeer('slice', \
333 #slice['slice_id'], \
335 #Update existing record (e.g. expires field)
336 #it with the latest info.
337 ##if slice_record and slice['expires'] != slice_record['expires']:
338 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : \
339 #slice_record['expires']})
341 logger.debug(" SLABSLICES \tverify_slice Oups \
342 slice_record %s peer %s sfa_peer %s "\
343 %(slice_record, peer,sfa_peer))
344 sfa_slice = {'slice_hrn': slicename,
345 #'url': slice_record.get('url', slice_hrn),
346 #'description': slice_record.get('description', slice_hrn)
348 'record_id_user' : slice_record['person_ids'][0],
349 'record_id_slice': slice_record['record_id'],
350 'peer_authority':str(peer.hrn)
354 self.driver.AddSlice(sfa_slice)
355 #slice['slice_id'] = self.driver.AddSlice(slice)
356 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
357 #slice['node_ids']=[]
358 #slice['person_ids'] = []
360 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
361 # mark this slice as an sfa peer record
363 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
364 #'peer_authority': sfa_peer, 'pointer': \
366 #self.registry.register_peer_object(self.credential, peer_dict)
373 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
376 users is a record list. Records can either be local records
377 or users records from known and trusted federated sites.
378 If the user is from another site that senslab doesn't trust yet,
379 then Resolve will raise an error before getting to create_sliver.
381 #TODO SA 21/08/12 verify_persons Needs review
386 #users_dict : dict whose keys can either be the user's hrn or its id.
387 #Values contains only id and hrn
390 #First create dicts by hrn and id for each user in the user record list:
393 if 'urn' in user and (not 'hrn' in user ) :
394 user['hrn'], user['type'] = urn_to_hrn(user['urn'])
396 if 'person_id' in user and 'hrn' in user:
397 users_by_id[user['person_id']] = user
398 users_dict[user['person_id']] = {'person_id':\
399 user['person_id'], 'hrn':user['hrn']}
401 users_by_hrn[user['hrn']] = user
402 users_dict[user['hrn']] = {'person_id':user['person_id'], \
406 logger.debug( "SLABSLICE.PY \t verify_person \
407 users_dict %s \r\n user_by_hrn %s \r\n \
409 %(users_dict,users_by_hrn, users_by_id))
411 existing_user_ids = []
412 existing_user_hrns = []
414 # Check if user is in Senslab LDAP using its hrn.
415 # Assuming Senslab is centralised : one LDAP for all sites,
416 # user_id unknown from LDAP
417 # LDAP does not provide users id, therefore we rely on hrns containing
418 # the login of the user.
419 # If the hrn is not a senslab hrn, the user may not be in LDAP.
421 #Construct the list of filters (list of dicts) for GetPersons
423 for hrn in users_by_hrn:
424 filter_user.append (users_by_hrn[hrn])
425 logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " \
427 #Check user's in LDAP with GetPersons
428 #Needed because what if the user has been deleted in LDAP but
430 existing_users = self.driver.GetPersons(filter_user)
432 #User's in senslab LDAP
434 for user in existing_users :
435 existing_user_hrns.append(users_dict[user['hrn']]['hrn'])
437 append(users_dict[user['hrn']]['person_id'])
439 # User from another known trusted federated site. Check
440 # if a senslab account matching the email has already been created.
443 if isinstance(users, list):
445 req += users[0]['email']
447 req += users['email']
449 ldap_reslt = self.driver.ldap.LdapSearch(req)
451 logger.debug(" SLABSLICE.PY \tverify_person users \
452 USER already in Senslab \t ldap_reslt %s \
454 existing_users.append(ldap_reslt[1])
457 #User not existing in LDAP
458 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
459 logger.debug(" SLABSLICE.PY \tverify_person users \
460 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
461 ldap_reslt %s " %(users, ldap_reslt))
463 requested_user_ids = users_by_id.keys()
464 requested_user_hrns = users_by_hrn.keys()
465 logger.debug("SLABSLICE.PY \tverify_person requested_user_ids %s \
466 user_by_hrn %s " %(requested_user_ids, users_by_hrn))
469 #Check that the user of the slice in the slice record
470 #matches the existing users
472 if slice_record['record_id_user'] in requested_user_ids and \
473 slice_record['PI'][0] in requested_user_hrns:
474 logger.debug(" SLABSLICE \tverify_person \
475 requested_user_ids %s = \
476 slice_record['record_id_user'] %s" \
477 %(requested_user_ids,slice_record['record_id_user']))
483 # users to be added, removed or updated
484 #One user in one senslab slice : there should be no need
485 #to remove/ add any user from/to a slice.
486 #However a user from SFA which is not registered in Senslab yet
487 #should be added to the LDAP.
489 added_user_hrns = set(requested_user_hrns).\
490 difference(set(existing_user_hrns))
492 #self.verify_keys(existing_slice_users, updated_users_list, \
497 for added_user_hrn in added_user_hrns:
498 added_user = users_dict[added_user_hrn]
499 #hrn, type = urn_to_hrn(added_user['urn'])
501 #'first_name': added_user.get('first_name', hrn),
502 #'last_name': added_user.get('last_name', hrn),
503 'first_name': added_user['first_name'],
504 'last_name': added_user['last_name'],
505 'person_id': added_user['person_id'],
506 'peer_person_id': None,
508 'key_ids': added_user.get('key_ids', []),
511 person['person_id'] = self.driver.AddPerson(person)
513 person['peer_person_id'] = added_user['person_id']
514 added_persons.append(person)
517 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
520 #self.driver.AddPersonToSite(added_user_id, login_base)
522 #for key_string in added_user.get('keys', []):
523 #key = {'key':key_string, 'key_type':'ssh'}
524 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], \
526 #person['keys'].append(key)
528 # add the registry record
530 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': \
532 #'pointer': person['person_id']}
533 #self.registry.register_peer_object(self.credential, peer_dict)
534 #for added_slice_user_hrn in \
535 #added_slice_user_hrns.union(added_user_hrns):
536 #self.driver.AddPersonToSlice(added_slice_user_hrn, \
537 #slice_record['name'])
538 #for added_slice_user_id in \
539 #added_slice_user_ids.union(added_user_ids):
540 # add person to the slice
541 #self.driver.AddPersonToSlice(added_slice_user_id, \
542 #slice_record['name'])
543 # if this is a peer record then it
544 # should already be bound to a peer.
545 # no need to return worry about it getting bound later
550 def verify_keys(self, persons, users, peer, options={}):
553 for person in persons:
554 key_ids.extend(person['key_ids'])
555 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
558 keydict[key['key']] = key['key_id']
559 existing_keys = keydict.keys()
561 for person in persons:
562 persondict[person['email']] = person
568 user_keys = user.get('keys', [])
569 updated_persons.append(user)
570 for key_string in user_keys:
571 requested_keys.append(key_string)
572 if key_string not in existing_keys:
573 key = {'key': key_string, 'key_type': 'ssh'}
576 person = persondict[user['email']]
577 self.driver.UnBindObjectFromPeer('person', \
578 person['person_id'], peer['shortname'])
580 self.driver.AddPersonKey(user['email'], key)
582 key_index = user_keys.index(key['key'])
583 remote_key_id = user['key_ids'][key_index]
584 self.driver.BindObjectToPeer('key', \
585 key['key_id'], peer['shortname'], \
590 self.driver.BindObjectToPeer('person', \
591 person['person_id'], peer['shortname'], \
594 # remove old keys (only if we are not appending)
595 append = options.get('append', True)
597 removed_keys = set(existing_keys).difference(requested_keys)
598 for existing_key_id in keydict:
599 if keydict[existing_key_id] in removed_keys:
602 self.driver.UnBindObjectFromPeer('key', \
603 existing_key_id, peer['shortname'])
604 self.driver.DeleteKey(existing_key_id)
607 #def verify_slice_attributes(self, slice, requested_slice_attributes, \
608 #append=False, admin=False):
609 ## get list of attributes users ar able to manage
610 #filter = {'category': '*slice*'}
612 #filter['|roles'] = ['user']
613 #slice_attributes = self.driver.GetTagTypes(filter)
614 #valid_slice_attribute_names = [attribute['tagname'] \
615 #for attribute in slice_attributes]
617 ## get sliver attributes
618 #added_slice_attributes = []
619 #removed_slice_attributes = []
620 #ignored_slice_attribute_names = []
621 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': \
624 ## get attributes that should be removed
625 #for slice_tag in existing_slice_attributes:
626 #if slice_tag['tagname'] in ignored_slice_attribute_names:
627 ## If a slice already has a admin only role
628 ## it was probably given to them by an
629 ## admin, so we should ignore it.
630 #ignored_slice_attribute_names.append(slice_tag['tagname'])
632 ## If an existing slice attribute was not
633 ## found in the request it should
635 #attribute_found=False
636 #for requested_attribute in requested_slice_attributes:
637 #if requested_attribute['name'] == slice_tag['tagname'] \
638 #and requested_attribute['value'] == slice_tag['value']:
639 #attribute_found=True
642 #if not attribute_found and not append:
643 #removed_slice_attributes.append(slice_tag)
645 ## get attributes that should be added:
646 #for requested_attribute in requested_slice_attributes:
647 ## if the requested attribute wasn't found we should add it
648 #if requested_attribute['name'] in valid_slice_attribute_names:
649 #attribute_found = False
650 #for existing_attribute in existing_slice_attributes:
651 #if requested_attribute['name'] == \
652 #existing_attribute['tagname'] and \
653 #requested_attribute['value'] == \
654 #existing_attribute['value']:
655 #attribute_found=True
657 #if not attribute_found:
658 #added_slice_attributes.append(requested_attribute)
661 ## remove stale attributes
662 #for attribute in removed_slice_attributes:
664 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
665 #except Exception, error:
666 #self.logger.warn('Failed to remove sliver attribute. name: \
667 #%s, value: %s, node_id: %s\nCause:%s'\
668 #% (name, value, node_id, str(error)))
670 ## add requested_attributes
671 #for attribute in added_slice_attributes:
673 #self.driver.AddSliceTag(slice['name'], attribute['name'], \
674 #attribute['value'], attribute.get('node_id', None))
675 #except Exception, error:
676 #self.logger.warn('Failed to add sliver attribute. name: %s, \
677 #value: %s, node_id: %s\nCause:%s'\
678 #% (name, value, node_id, str(error)))