1 from sfa.util.xrn import get_authority, urn_to_hrn
2 from sfa.util.sfalogging import logger
9 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
12 def __init__(self, driver):
14 Get the reference to the driver here.
19 def get_peer(self, xrn):
21 Find the authority of a resources based on its xrn.
22 If the authority is Senslab (local) return None,
23 Otherwise, look up in the DB if Senslab is federated with this site
24 authority and returns its DB record if it is the case,
26 hrn, hrn_type = urn_to_hrn(xrn)
27 #Does this slice belong to a local site or a peer senslab site?
30 # get this slice's authority (site)
31 slice_authority = get_authority(hrn)
33 #This slice belongs to the current site
34 if slice_authority == self.driver.slab_api.root_auth:
35 site_authority = slice_authority
38 site_authority = get_authority(slice_authority).lower()
39 # get this site's authority (sfa root authority or sub authority)
41 logger.debug("SLABSLICES \ get_peer slice_authority %s \
42 site_authority %s hrn %s" %(slice_authority, \
46 # check if we are already peered with this site_authority
47 #if so find the peer record
48 peers = self.driver.slab_api.GetPeers(peer_filter = site_authority)
49 for peer_record in peers:
51 if site_authority == peer_record.hrn:
53 logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
56 def get_sfa_peer(self, xrn):
57 hrn, hrn_type = urn_to_hrn(xrn)
59 # return the authority for this hrn or None if we are the authority
61 slice_authority = get_authority(hrn)
62 site_authority = get_authority(slice_authority)
64 if site_authority != self.driver.hrn:
65 sfa_peer = site_authority
70 def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer):
72 Compare requested leases with the leases already scheduled/
73 running in OAR. If necessary, delete and recreate modified leases,
74 and delete no longer requested ones.
76 :param sfa_slice: sfa slice record
77 :param requested_jobs_dict: dictionary of requested leases
81 :type requested_jobs_dict: dict
83 :return: leases list of dictionary
88 logger.debug("SLABSLICES verify_slice_leases sfa_slice %s \
90 #First get the list of current leases from OAR
91 leases = self.driver.slab_api.GetLeases({'name':sfa_slice['hrn']})
92 logger.debug("SLABSLICES verify_slice_leases requested_jobs_dict %s \
93 leases %s "%(requested_jobs_dict, leases ))
95 current_nodes_reserved_by_start_time = {}
96 requested_nodes_by_start_time = {}
97 leases_by_start_time = {}
98 reschedule_jobs_dict = {}
101 #Create reduced dictionary with key start_time and value
103 #-for the leases already registered by OAR first
104 # then for the new leases requested by the user
106 #Leases already scheduled/running in OAR
107 for lease in leases :
108 current_nodes_reserved_by_start_time[lease['t_from']] = \
109 lease['reserved_nodes']
110 leases_by_start_time[lease['t_from']] = lease
112 #First remove job whose duration is too short
113 for job in requested_jobs_dict.values():
114 if job['duration'] < self.driver.slab_api.GetLeaseGranularity():
115 del requested_jobs_dict[job['start_time']]
118 for start_time in requested_jobs_dict:
119 requested_nodes_by_start_time[int(start_time)] = \
120 requested_jobs_dict[start_time]['hostname']
121 #Check if there is any difference between the leases already
122 #registered in OAR and the requested jobs.
123 #Difference could be:
124 #-Lease deleted in the requested jobs
125 #-Added/removed nodes
128 logger.debug("SLABSLICES verify_slice_leases \
129 requested_nodes_by_start_time %s \
130 "%(requested_nodes_by_start_time ))
131 #Find all deleted leases
133 list(set(leases_by_start_time.keys()).\
134 difference(requested_nodes_by_start_time.keys()))
135 deleted_leases = [leases_by_start_time[start_time]['lease_id'] \
136 for start_time in start_time_list]
140 #Find added or removed nodes in exisiting leases
141 for start_time in requested_nodes_by_start_time:
142 logger.debug("SLABSLICES verify_slice_leases start_time %s \
144 if start_time in current_nodes_reserved_by_start_time:
146 if requested_nodes_by_start_time[start_time] == \
147 current_nodes_reserved_by_start_time[start_time]:
152 set(requested_nodes_by_start_time[start_time])
154 update_node_set.difference(\
155 current_nodes_reserved_by_start_time[start_time])
157 update_node_set.intersection(\
158 current_nodes_reserved_by_start_time[start_time])
161 current_nodes_reserved_by_start_time[start_time])
163 old_nodes_set.difference(\
164 requested_nodes_by_start_time[start_time])
165 logger.debug("SLABSLICES verify_slice_leases \
166 shared_nodes %s added_nodes %s removed_nodes %s"\
167 %(shared_nodes, added_nodes,removed_nodes ))
168 #If the lease is modified, delete it before
170 #Add the deleted lease job id in the list
171 #WARNING :rescheduling does not work if there is already
172 # 2 running/scheduled jobs because deleting a job
173 #takes time SA 18/10/2012
174 if added_nodes or removed_nodes:
175 deleted_leases.append(\
176 leases_by_start_time[start_time]['lease_id'])
178 if added_nodes or shared_nodes:
179 reschedule_jobs_dict[str(start_time)] = \
180 requested_jobs_dict[str(start_time)]
185 job = requested_jobs_dict[str(start_time)]
186 logger.debug("SLABSLICES \
187 NEWLEASE slice %s job %s"\
189 self.driver.slab_api.AddLeases(job['hostname'], \
190 sfa_slice, int(job['start_time']), \
191 int(job['duration']))
193 #Deleted leases are the ones with lease id not declared in the Rspec
195 self.driver.slab_api.DeleteLeases(deleted_leases, sfa_slice['hrn'])
196 logger.debug("SLABSLICES \
197 verify_slice_leases slice %s deleted_leases %s"\
198 %(sfa_slice, deleted_leases))
201 if reschedule_jobs_dict :
202 for start_time in reschedule_jobs_dict:
203 job = reschedule_jobs_dict[start_time]
204 self.driver.slab_api.AddLeases(job['hostname'], \
205 sfa_slice, int(job['start_time']), \
206 int(job['duration']))
209 def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
213 if 'node_ids' in sfa_slice:
214 nodes = self.driver.slab_api.GetNodes(sfa_slice['list_node_ids'], \
216 current_slivers = [node['hostname'] for node in nodes]
218 # remove nodes not in rspec
219 deleted_nodes = list(set(current_slivers).\
220 difference(requested_slivers))
221 # add nodes from rspec
222 #added_nodes = list(set(requested_slivers).\
223 #difference(current_slivers))
226 logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
227 \r\n \r\n deleted_nodes %s"\
228 %(sfa_slice, deleted_nodes))
231 #Delete the entire experience
232 self.driver.slab_api.DeleteSliceFromNodes(sfa_slice)
233 #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
239 def free_egre_key(self):
241 for tag in self.driver.slab_api.GetSliceTags({'tagname': 'egre_key'}):
242 used.add(int(tag['value']))
244 for i in range(1, 256):
249 raise KeyError("No more EGRE keys available")
258 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer):
260 #login_base = slice_hrn.split(".")[0]
261 slicename = slice_hrn
262 slices_list = self.driver.slab_api.GetSlices(slice_filter = slicename, \
263 slice_filter_type = 'slice_hrn')
266 for sl in slices_list:
268 logger.debug("SLABSLICE \tverify_slice slicename %s \
269 slices_list %s sl %s \ slice_record %s"\
270 %(slicename, slices_list,sl, \
273 sfa_slice.update(slice_record)
276 #Search for user in ldap based on email SA 14/11/12
277 ldap_user = self.driver.slab_api.ldap.LdapFindUser(\
278 slice_record['user'])
279 logger.debug(" SLABSLICES \tverify_slice Oups \
280 slice_record %s sfa_peer %s ldap_user %s"\
281 %(slice_record, sfa_peer, ldap_user ))
282 #User already registered in ldap, meaning user should be in SFA db
283 #and hrn = sfa_auth+ uid
284 sfa_slice = {'hrn': slicename,
285 #'url': slice_record.get('url', slice_hrn),
286 #'description': slice_record.get('description', slice_hrn)
288 'authority' : slice_record['authority'],
289 'gid':slice_record['gid'],
290 #'record_id_user' : user.record_id,
291 'slice_id' : slice_record['record_id'],
292 'reg-researchers':slice_record['reg-researchers'],
293 #'record_id_slice': slice_record['record_id'],
294 'peer_authority':str(sfa_peer)
298 hrn = self.driver.slab_api.root_auth +'.'+ ldap_user['uid']
300 user = self.driver.get_user_record(hrn)
302 logger.debug(" SLABSLICES \tverify_slice hrn %s USER %s" \
304 #sfa_slice = {'slice_hrn': slicename,
305 ##'url': slice_record.get('url', slice_hrn),
306 ##'description': slice_record.get('description', slice_hrn)
308 #'authority' : slice_record['authority'],
309 #'gid':slice_record['gid'],
310 ##'record_id_user' : user.record_id,
311 #'slice_id' : slice_record['record_id'],
312 #'reg-researchers':slice_record['reg-researchers'],
313 ##'record_id_slice': slice_record['record_id'],
314 #'peer_authority':str(peer.hrn)
319 self.driver.slab_api.AddSlice(sfa_slice, user)
322 sfa_slice['slice_id'] = slice_record['record_id']
324 #slice['slice_id'] = self.driver.slab_api.AddSlice(slice)
325 logger.debug("SLABSLICES \tverify_slice ADDSLICE OK")
326 #slice['node_ids']=[]
327 #slice['person_ids'] = []
329 #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None)
330 # mark this slice as an sfa peer record
332 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
333 #'peer_authority': sfa_peer, 'pointer': \
335 #self.registry.register_peer_object(self.credential, peer_dict)
342 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \
345 users is a record list. Records can either be local records
346 or users records from known and trusted federated sites.
347 If the user is from another site that senslab doesn't trust yet,
348 then Resolve will raise an error before getting to create_sliver.
350 #TODO SA 21/08/12 verify_persons Needs review
352 logger.debug("SLABSLICES \tverify_persons \tslice_hrn %s \
353 \t slice_record %s\r\n users %s \t peer %s "\
354 %( slice_hrn, slice_record, users, peer))
358 #users_dict : dict whose keys can either be the user's hrn or its id.
359 #Values contains only id and hrn
362 #First create dicts by hrn and id for each user in the user record list:
365 if 'slice_record' in info :
366 slice_rec = info['slice_record']
367 user = slice_rec['user']
370 users_by_email[user['email']] = user
371 users_dict[user['email']] = user
374 #users_by_hrn[user['hrn']] = user
375 #users_dict[user['hrn']] = user
377 logger.debug( "SLABSLICE.PY \t verify_person \
378 users_dict %s \r\n user_by_email %s \r\n \
380 %(users_dict,users_by_email, users_by_id))
382 existing_user_ids = []
383 #existing_user_hrns = []
384 existing_user_emails = []
386 # Check if user is in Senslab LDAP using its hrn.
387 # Assuming Senslab is centralised : one LDAP for all sites,
388 # user'as record_id unknown from LDAP
389 # LDAP does not provide users id, therefore we rely on hrns containing
390 # the login of the user.
391 # If the hrn is not a senslab hrn, the user may not be in LDAP.
394 #Construct the list of filters (list of dicts) for GetPersons
396 for email in users_by_email :
397 filter_user.append (users_by_email[email])
398 #Check user's in LDAP with GetPersons
399 #Needed because what if the user has been deleted in LDAP but
401 existing_users = self.driver.slab_api.GetPersons(filter_user)
402 logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user \
403 %s existing_users %s " \
404 %(filter_user, existing_users))
405 #User's in senslab LDAP
407 for user in existing_users :
408 users_dict[user['email']].update(user)
409 existing_user_emails.append(\
410 users_dict[user['email']]['email'])
413 # User from another known trusted federated site. Check
414 # if a senslab account matching the email has already been created.
417 if isinstance(users, list):
419 req += users[0]['email']
421 req += users['email']
423 ldap_reslt = self.driver.slab_api.ldap.LdapSearch(req)
426 logger.debug(" SLABSLICE.PY \tverify_person users \
427 USER already in Senslab \t ldap_reslt %s \
429 existing_users.append(ldap_reslt[1])
432 #User not existing in LDAP
433 #TODO SA 21/08/12 raise smthg to add user or add it auto ?
435 #new_record['pkey'] = users[0]['keys'][0]
436 #new_record['mail'] = users[0]['email']
438 logger.debug(" SLABSLICE.PY \tverify_person users \
439 not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \
440 ldap_reslt %s " %(users, ldap_reslt))
442 requested_user_emails = users_by_email.keys()
443 requested_user_hrns = \
444 [users_by_email[user]['hrn'] for user in users_by_email]
445 logger.debug("SLABSLICE.PY \tverify_person \
446 users_by_email %s " %( users_by_email))
447 #logger.debug("SLABSLICE.PY \tverify_person \
448 #user_by_hrn %s " %( users_by_hrn))
451 #Check that the user of the slice in the slice record
452 #matches one of the existing users
454 if slice_record['PI'][0] in requested_user_hrns:
455 #if slice_record['record_id_user'] in requested_user_ids and \
456 #slice_record['PI'][0] in requested_user_hrns:
457 logger.debug(" SLABSLICE \tverify_person ['PI']\
458 slice_record %s" %(slice_record))
464 # users to be added, removed or updated
465 #One user in one senslab slice : there should be no need
466 #to remove/ add any user from/to a slice.
467 #However a user from SFA which is not registered in Senslab yet
468 #should be added to the LDAP.
469 added_user_emails = set(requested_user_emails).\
470 difference(set(existing_user_emails))
473 #self.verify_keys(existing_slice_users, updated_users_list, \
479 #requested_user_email is in existing_user_emails
480 if len(added_user_emails) == 0:
482 slice_record['login'] = users_dict[requested_user_emails[0]]['uid']
483 logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" \
487 for added_user_email in added_user_emails:
488 #hrn, type = urn_to_hrn(added_user['urn'])
489 added_user = users_dict[added_user_email]
490 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person \
491 added_user %s" %(added_user))
493 person['peer_person_id'] = None
494 k_list = ['first_name', 'last_name','person_id']
497 person[k] = added_user[k]
499 person['pkey'] = added_user['keys'][0]
500 person['mail'] = added_user['email']
501 person['email'] = added_user['email']
502 person['key_ids'] = added_user.get('key_ids', [])
503 #person['urn'] = added_user['urn']
505 #person['person_id'] = self.driver.slab_api.AddPerson(person)
506 ret = self.driver.slab_api.AddPerson(person)
507 if type(ret) == int :
510 logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person\
511 personne %s" %(person))
512 #Update slice_Record with the id now known to LDAP
513 slice_record['login'] = person['uid']
515 added_persons.append(person)
521 def verify_keys(self, persons, users, peer, options={}):
524 for person in persons:
525 key_ids.extend(person['key_ids'])
526 keylist = self.driver.slab_api.GetKeys(key_ids, ['key_id', 'key'])
530 keydict[key['key']] = key['key_id']
531 existing_keys = keydict.keys()
534 for person in persons:
535 persondict[person['email']] = person
540 users_by_key_string = {}
542 user_keys = user.get('keys', [])
543 updated_persons.append(user)
544 for key_string in user_keys:
545 users_by_key_string[key_string] = user
546 requested_keys.append(key_string)
547 if key_string not in existing_keys:
548 key = {'key': key_string, 'key_type': 'ssh'}
551 #person = persondict[user['email']]
552 #self.driver.slab_api.UnBindObjectFromPeer('person',
553 #person['person_id'], peer['shortname'])
554 ret = self.driver.slab_api.AddPersonKey(\
557 #key_index = user_keys.index(key['key'])
558 #remote_key_id = user['key_ids'][key_index]
559 #self.driver.slab_api.BindObjectToPeer('key', \
560 #key['key_id'], peer['shortname'], \
565 #self.driver.slab_api.BindObjectToPeer('person', \
566 #person['person_id'], peer['shortname'], \
569 # remove old keys (only if we are not appending)
570 append = options.get('append', True)
572 removed_keys = set(existing_keys).difference(requested_keys)
573 for key in removed_keys:
575 #self.driver.slab_api.UnBindObjectFromPeer('key', \
576 #key, peer['shortname'])
578 user = users_by_key_string[key]
579 self.driver.slab_api.DeleteKey(user, key)