1 from types import StringTypes
2 from collections import defaultdict
4 from sfa.util.xrn import get_leaf, get_authority, urn_to_hrn
5 from sfa.planetlab.plxrn import hrn_to_pl_slicename
6 from sfa.util.policy import Policy
7 from sfa.rspecs.rspec import RSpec
9 from sfa.util.xrn import Xrn
10 from sfa.util.sfalogging import logger
12 from sqlalchemy import Column, Integer, String, DateTime
13 from sqlalchemy import Table, Column, MetaData, join, ForeignKey
14 from sfa.storage.model import RegRecord
15 from sfa.storage.alchemy import dbsession,engine
21 rspec_to_slice_tag = {'max_rate':'net_max_rate'}
23 #def __init__(self, api, ttl = .5, origin_hrn=None):
25 ##filepath = path + os.sep + filename
26 #self.policy = Policy(self.api)
27 #self.origin_hrn = origin_hrn
28 #self.registry = api.registries[api.hrn]
29 #self.credential = api.getCredential()
34 def __init__(self, driver):
38 def get_slivers(self, xrn, node=None):
39 hrn, type = urn_to_hrn(xrn)
41 slice_name = hrn_to_pl_slicename(hrn)
42 # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
43 # of doing all of this?
44 #return self.api.driver.GetSliceTicket(self.auth, slice_name)
48 slice = self.driver.GetSlices(slice_filter = slice_name, slice_filter_type = 'slice_hrn')
51 # Get user information
53 alchemy_person = dbsession.query(RegRecord).filter_by(record_id = slice['record_id_user']).first()
56 sliver_attributes = []
58 if slice['oar_job_id'] is not -1:
59 nodes_all = self.GetNodes({'hostname':slice['node_ids']},
60 ['node_id', 'hostname','site','boot_state'])
61 nodeall_byhostname = dict([(n['hostname'], n) for n in nodes_all])
62 nodes = slice['node_ids']
65 #for sliver_attribute in filter(lambda a: a['node_id'] == node['node_id'], slice_tags):
66 sliver_attribute['tagname'] = 'slab-tag'
67 sliver_attribute['value'] = 'slab-value'
68 sliver_attributes.append(sliver_attribute['tagname'])
69 attributes.append({'tagname': sliver_attribute['tagname'],
70 'value': sliver_attribute['value']})
72 # set nodegroup slice attributes
73 for slice_tag in filter(lambda a: a['nodegroup_id'] in node['nodegroup_ids'], slice_tags):
74 # Do not set any nodegroup slice attributes for
75 # which there is at least one sliver attribute
77 if slice_tag not in slice_tags:
78 attributes.append({'tagname': slice_tag['tagname'],
79 'value': slice_tag['value']})
81 for slice_tag in filter(lambda a: a['node_id'] is None, slice_tags):
82 # Do not set any global slice attributes for
83 # which there is at least one sliver attribute
85 if slice_tag['tagname'] not in sliver_attributes:
86 attributes.append({'tagname': slice_tag['tagname'],
87 'value': slice_tag['value']})
89 # XXX Sanity check; though technically this should be a system invariant
90 # checked with an assertion
91 if slice['expires'] > MAXINT: slice['expires']= MAXINT
95 'name': slice['name'],
96 'slice_id': slice['slice_id'],
97 'instantiation': slice['instantiation'],
98 'expires': slice['expires'],
100 'attributes': attributes
110 def get_peer(self, xrn):
111 hrn, type = urn_to_hrn(xrn)
112 #Does this slice belong to a local site or a peer senslab site?
115 # get this slice's authority (site)
116 slice_authority = get_authority(hrn)
117 site_authority = slice_authority
118 # get this site's authority (sfa root authority or sub authority)
119 #site_authority = get_authority(slice_authority).lower()
120 print>>sys.stderr, " \r\n \r\n \t slices.py get_peer slice_authority %s site_authority %s hrn %s" %(slice_authority, site_authority, hrn)
121 # check if we are already peered with this site_authority, if so
122 #peers = self.driver.GetPeers({})
123 peers = self.driver.GetPeers(peer_filter = slice_authority)
124 for peer_record in peers:
126 if site_authority == peer_record.hrn:
128 print>>sys.stderr, " \r\n \r\n \t slices.py get_peerAPRES Mpeer %s " %(peer)
131 def get_sfa_peer(self, xrn):
132 hrn, type = urn_to_hrn(xrn)
134 # return the authority for this hrn or None if we are the authority
136 slice_authority = get_authority(hrn)
137 site_authority = get_authority(slice_authority)
139 if site_authority != self.driver.hrn:
140 sfa_peer = site_authority
144 def verify_slice_nodes(self, slice, requested_slivers, peer):
148 if slice['node_ids']:
149 nodes = self.driver.GetNodes(slice['node_ids'], ['hostname'])
150 current_slivers = [node['hostname'] for node in nodes]
152 # remove nodes not in rspec
153 deleted_nodes = list(set(current_slivers).difference(requested_slivers))
155 # add nodes from rspec
156 added_nodes = list(set(requested_slivers).difference(current_slivers))
159 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
160 #PI is a list, get the only username in this list
161 #so that the OAR/LDAP knows the user: remove the authority from the name
162 tmp= slice['PI'][0].split(".")
163 username = tmp[(len(tmp)-1)]
164 #Update the table with the nodes that populate the slice
165 self.driver.db.update_job(slice['name'],nodes = added_nodes)
166 print>>sys.stderr, "\r\n \\r\n \r\n \t\t\t VERIFY_SLICE_NODES slice %s \r\n \r\n \r\n " %(slice)
167 #If there is a timeslot specified, then a job can be launched
169 slot = slice['timeslot']
170 self.driver.LaunchExperimentOnOAR(slice, added_nodes, username)
172 logger.log_exc("SLABSLICES \tVERIFY_SLICE_NODES KeyError slice %s " %(slice))
177 self.driver.DeleteSliceFromNodes(slice['name'], deleted_nodes)
180 logger.log_exc('Failed to add/remove slice from nodes')
183 def free_egre_key(self):
185 for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}):
186 used.add(int(tag['value']))
188 for i in range(1, 256):
193 raise KeyError("No more EGRE keys available")
202 def handle_peer(self, site, slice, persons, peer):
207 self.driver.BindObjectToPeer('site', site['site_id'], peer['shortname'], slice['site_id'])
209 self.driver.DeleteSite(site['site_id'])
215 self.driver.BindObjectToPeer('slice', slice['slice_id'], peer['shortname'], slice['slice_id'])
217 self.driver.DeleteSlice(slice['slice_id'])
221 for person in persons:
223 self.driver.BindObjectToPeer('person',
224 person['person_id'], peer['shortname'], person['peer_person_id'])
226 for (key, remote_key_id) in zip(person['keys'], person['key_ids']):
228 self.driver.BindObjectToPeer( 'key', key['key_id'], peer['shortname'], remote_key_id)
230 self.driver.DeleteKey(key['key_id'])
231 logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname']))
233 self.driver.DeletePerson(person['person_id'])
238 #def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}):
239 #(slice_hrn, type) = urn_to_hrn(slice_xrn)
240 #site_hrn = get_authority(slice_hrn)
241 ## login base can't be longer than 20 characters
242 ##slicename = hrn_to_pl_slicename(slice_hrn)
243 #authority_name = slice_hrn.split('.')[0]
244 #login_base = authority_name[:20]
245 #print >>sys.stderr, " \r\n \r\n \t\t SLABSLICES.PY verify_site authority_name %s login_base %s slice_hrn %s" %(authority_name,login_base,slice_hrn)
247 #sites = self.driver.GetSites(login_base)
249 ## create new site record
250 #site = {'name': 'geni.%s' % authority_name,
251 #'abbreviated_name': authority_name,
252 #'login_base': login_base,
254 #'max_slivers': 1000,
256 #'peer_site_id': None}
258 #site['peer_site_id'] = slice_record.get('site_id', None)
259 #site['site_id'] = self.driver.AddSite(site)
260 ## exempt federated sites from monitor policies
261 #self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101")
263 ### is this still necessary?
264 ### add record to the local registry
265 ##if sfa_peer and slice_record:
266 ##peer_dict = {'type': 'authority', 'hrn': site_hrn, \
267 ##'peer_authority': sfa_peer, 'pointer': site['site_id']}
268 ##self.registry.register_peer_object(self.credential, peer_dict)
272 ## unbind from peer so we can modify if necessary. Will bind back later
273 #self.driver.UnBindObjectFromPeer('site', site['site_id'], peer['shortname'])
277 def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, options={} ):
279 login_base = slice_hrn.split(".")[0]
280 slicename = slice_hrn
281 sl = self.driver.GetSlices(slice_filter=slicename, slice_filter_type = 'slice_hrn')
284 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice slicename %s sl %s slice_record %s"%(slicename ,sl, slice_record)
286 slice.update(slice_record)
287 #del slice['last_updated']
288 #del slice['date_created']
290 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
291 ## unbind from peer so we can modify if necessary. Will bind back later
292 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
293 #Update existing record (e.g. expires field) it with the latest info.
294 ##if slice_record and slice['expires'] != slice_record['expires']:
295 ##self.driver.UpdateSlice( slice['slice_id'], {'expires' : slice_record['expires']})
297 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice UH-Oh...slice_record %s peer %s sfa_peer %s "%(slice_record, peer,sfa_peer)
298 slice = {'slice_hrn': slicename,
299 #'url': slice_record.get('url', slice_hrn),
300 #'description': slice_record.get('description', slice_hrn)
302 'record_id_user' : slice_record['person_ids'][0],
303 'record_id_slice': slice_record['record_id'],
304 'peer_authority':str(peer.hrn)
308 self.driver.AddSlice(slice)
309 #slice['slice_id'] = self.driver.AddSlice(slice)
310 print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice ADDSLICE OHYEEEEEEEEEEAH! "
311 #slice['node_ids']=[]
312 #slice['person_ids'] = []
314 #slice['peer_slice_id'] = slice_record.get('slice_id', None)
315 # mark this slice as an sfa peer record
317 #peer_dict = {'type': 'slice', 'hrn': slice_hrn,
318 #'peer_authority': sfa_peer, 'pointer': slice['slice_id']}
319 #self.registry.register_peer_object(self.credential, peer_dict)
326 def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}):
333 if 'urn' in user and (not 'hrn' in user ) :
334 user['hrn'],user['type'] = urn_to_hrn(user['urn'])
336 if 'person_id' in user and 'hrn' in user:
337 users_by_id[user['person_id']] = user
338 users_dict[user['person_id']] = {'person_id':user['person_id'], 'hrn':user['hrn']}
340 users_by_hrn[user['hrn']] = user
341 users_dict[user['hrn']] = {'person_id':user['person_id'], 'hrn':user['hrn']}
343 logger.debug( "\r\n \r\n SLABSLICE.PY \tverify_person users_dict %s \r\n user_by_hrn %s \r\n \tusers_by_id %s " %( users_dict,users_by_hrn, users_by_id) )
345 existing_user_ids = []
346 existing_user_hrns = []
348 #Check if user is in LDAP using its hrn.
349 #Assuming Senslab is centralised : one LDAP for all sites, user_id unknown from LDAP
350 # LDAP does not provide users id, therfore we rely on hrns
352 #Construct the list of filters for GetPersons
354 for hrn in users_by_hrn:
355 #filter_user.append ( {'hrn':hrn})
356 filter_user.append (users_by_hrn[hrn])
357 logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " %(filter_user) )
358 existing_users = self.driver.GetPersons(filter_user)
359 #existing_users = self.driver.GetPersons({'hrn': users_by_hrn.keys()})
360 #existing_users = self.driver.GetPersons({'hrn': users_by_hrn.keys()},
363 for user in existing_users :
364 #for k in users_dict[user['hrn']] :
365 existing_user_hrns.append (users_dict[user['hrn']]['hrn'])
366 existing_user_ids.append (users_dict[user['hrn']]['person_id'])
368 #User from another federated site , does not have a senslab account yet?
369 #or have multiple SFA accounts
370 #Check before adding them to LDAP
374 if isinstance(users,list):
375 ldap_reslt = self.driver.ldap.LdapSearch(users[0])
377 ldap_reslt = self.driver.ldap.LdapSearch(users)
379 existing_users = ldap_reslt[0]
380 existing_user_hrns.append (users_dict[user['hrn']]['hrn'])
381 existing_user_ids.append (users_dict[user['hrn']]['person_id'])
383 #User not existing in LDAP
385 logger.debug(" SLABSLICE.PY \tverify_person users HUMHUMHUMHUM ... %s \r\n \t ldap_reslt %s " %(users, ldap_reslt))
388 # requested slice users
389 requested_user_ids = users_by_id.keys()
390 requested_user_hrns = users_by_hrn.keys()
391 logger.debug(" SLABSLICE.PY \tverify_person requested_user_ids %s user_by_hrn %s " %( requested_user_ids,users_by_hrn))
392 # existing slice users
394 #existing_slice_users_filter = {'hrn': slice_record['PI'][0]}
395 #logger.debug(" SLABSLICE.PY \tverify_person requested_user_ids %s existing_slice_users_filter %s slice_record %s" %(requested_user_ids,existing_slice_users_filter,slice_record))
397 #existing_slice_users = self.driver.GetPersons([existing_slice_users_filter])
398 #existing_slice_users = self.driver.GetPersons(existing_slice_users_filter,['hrn','pkey'])
399 #logger.debug(" SLABSLICE.PY \tverify_person existing_slice_users %s " %(existing_slice_users))
400 #Check that the user of the slice in the slice record
401 #matches the existing users
403 if slice_record['record_id_user'] in requested_user_ids and slice_record['PI'][0] in requested_user_hrns:
404 logger.debug(" SLABSLICE.PY \tverify_person requested_user_ids %s = slice_record['record_id_user'] %s" %(requested_user_ids,slice_record['record_id_user']))
409 #existing_slice_user_hrns = [user['hrn'] for user in existing_slice_users]
411 # users to be added, removed or updated
412 #One user in one senslab slice : there should be no need
413 #to remove/ add any user from/to a slice.
414 #However a user from SFA which is not registered in Senslab yet
415 #should be added to the LDAP.
417 added_user_hrns = set(requested_user_hrns).difference(set(existing_user_hrns))
419 #self.verify_keys(existing_slice_users, updated_users_list, peer, append)
423 for added_user_hrn in added_user_hrns:
424 added_user = users_dict[added_user_hrn]
425 #hrn, type = urn_to_hrn(added_user['urn'])
427 'first_name': added_user.get('first_name', hrn),
428 'last_name': added_user.get('last_name', hrn),
429 'person_id': added_user['person_id'],
430 'peer_person_id': None,
432 'key_ids': added_user.get('key_ids', []),
435 person['person_id'] = self.driver.AddPerson(person)
437 person['peer_person_id'] = added_user['person_id']
438 added_persons.append(person)
441 self.driver.UpdatePerson(person['person_id'], {'enabled': True})
444 #self.driver.AddPersonToSite(added_user_id, login_base)
446 #for key_string in added_user.get('keys', []):
447 #key = {'key':key_string, 'key_type':'ssh'}
448 #key['key_id'] = self.driver.AddPersonKey(person['person_id'], key)
449 #person['keys'].append(key)
451 # add the registry record
453 #peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \
454 #'pointer': person['person_id']}
455 #self.registry.register_peer_object(self.credential, peer_dict)
456 #for added_slice_user_hrn in added_slice_user_hrns.union(added_user_hrns):
457 #self.driver.AddPersonToSlice(added_slice_user_hrn, slice_record['name'])
458 #for added_slice_user_id in added_slice_user_ids.union(added_user_ids):
459 # add person to the slice
460 #self.driver.AddPersonToSlice(added_slice_user_id, slice_record['name'])
461 # if this is a peer record then it should already be bound to a peer.
462 # no need to return worry about it getting bound later
467 def verify_keys(self, persons, users, peer, options={}):
470 for person in persons:
471 key_ids.extend(person['key_ids'])
472 keylist = self.driver.GetKeys(key_ids, ['key_id', 'key'])
475 keydict[key['key']] = key['key_id']
476 existing_keys = keydict.keys()
478 for person in persons:
479 persondict[person['email']] = person
485 user_keys = user.get('keys', [])
486 updated_persons.append(user)
487 for key_string in user_keys:
488 requested_keys.append(key_string)
489 if key_string not in existing_keys:
490 key = {'key': key_string, 'key_type': 'ssh'}
493 person = persondict[user['email']]
494 self.driver.UnBindObjectFromPeer('person', person['person_id'], peer['shortname'])
495 key['key_id'] = self.driver.AddPersonKey(user['email'], key)
497 key_index = user_keys.index(key['key'])
498 remote_key_id = user['key_ids'][key_index]
499 self.driver.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id)
503 self.driver.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id'])
505 # remove old keys (only if we are not appending)
507 removed_keys = set(existing_keys).difference(requested_keys)
508 for existing_key_id in keydict:
509 if keydict[existing_key_id] in removed_keys:
512 self.driver.UnBindObjectFromPeer('key', existing_key_id, peer['shortname'])
513 self.driver.DeleteKey(existing_key_id)
517 #def verify_slice_attributes(self, slice, requested_slice_attributes, append=False, admin=False):
518 ## get list of attributes users ar able to manage
519 #filter = {'category': '*slice*'}
521 #filter['|roles'] = ['user']
522 #slice_attributes = self.driver.GetTagTypes(filter)
523 #valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes]
525 ## get sliver attributes
526 #added_slice_attributes = []
527 #removed_slice_attributes = []
528 #ignored_slice_attribute_names = []
529 #existing_slice_attributes = self.driver.GetSliceTags({'slice_id': slice['slice_id']})
531 ## get attributes that should be removed
532 #for slice_tag in existing_slice_attributes:
533 #if slice_tag['tagname'] in ignored_slice_attribute_names:
534 ## If a slice already has a admin only role it was probably given to them by an
535 ## admin, so we should ignore it.
536 #ignored_slice_attribute_names.append(slice_tag['tagname'])
538 ## If an existing slice attribute was not found in the request it should
540 #attribute_found=False
541 #for requested_attribute in requested_slice_attributes:
542 #if requested_attribute['name'] == slice_tag['tagname'] and \
543 #requested_attribute['value'] == slice_tag['value']:
544 #attribute_found=True
547 #if not attribute_found and not append:
548 #removed_slice_attributes.append(slice_tag)
550 ## get attributes that should be added:
551 #for requested_attribute in requested_slice_attributes:
552 ## if the requested attribute wasn't found we should add it
553 #if requested_attribute['name'] in valid_slice_attribute_names:
554 #attribute_found = False
555 #for existing_attribute in existing_slice_attributes:
556 #if requested_attribute['name'] == existing_attribute['tagname'] and \
557 #requested_attribute['value'] == existing_attribute['value']:
558 #attribute_found=True
560 #if not attribute_found:
561 #added_slice_attributes.append(requested_attribute)
564 ## remove stale attributes
565 #for attribute in removed_slice_attributes:
567 #self.driver.DeleteSliceTag(attribute['slice_tag_id'])
568 #except Exception, e:
569 #self.logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
570 #% (name, value, node_id, str(e)))
572 ## add requested_attributes
573 #for attribute in added_slice_attributes:
575 #self.driver.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None))
576 #except Exception, e:
577 #self.logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
578 #% (name, value, node_id, str(e)))
580 #def create_slice_aggregate(self, xrn, rspec):
581 #hrn, type = urn_to_hrn(xrn)
582 ## Determine if this is a peer slice
583 #peer = self.get_peer(hrn)
584 #sfa_peer = self.get_sfa_peer(hrn)
587 ## Get the slice record from sfa
588 #slicename = hrn_to_pl_slicename(hrn)
591 #registry = self.api.registries[self.api.hrn]
592 #credential = self.api.getCredential()
594 #site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer)
595 #slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer)
597 ## find out where this slice is currently running
598 #nodelist = self.driver.GetNodes(slice['node_ids'], ['hostname'])
599 #hostnames = [node['hostname'] for node in nodelist]
601 ## get netspec details
602 #nodespecs = spec.getDictsByTagName('NodeSpec')
604 ## dict in which to store slice attributes to set for the nodes
606 #for nodespec in nodespecs:
607 #if isinstance(nodespec['name'], list):
608 #for nodename in nodespec['name']:
609 #nodes[nodename] = {}
610 #for k in nodespec.keys():
611 #rspec_attribute_value = nodespec[k]
612 #if (self.rspec_to_slice_tag.has_key(k)):
613 #slice_tag_name = self.rspec_to_slice_tag[k]
614 #nodes[nodename][slice_tag_name] = rspec_attribute_value
615 #elif isinstance(nodespec['name'], StringTypes):
616 #nodename = nodespec['name']
617 #nodes[nodename] = {}
618 #for k in nodespec.keys():
619 #rspec_attribute_value = nodespec[k]
620 #if (self.rspec_to_slice_tag.has_key(k)):
621 #slice_tag_name = self.rspec_to_slice_tag[k]
622 #nodes[nodename][slice_tag_name] = rspec_attribute_value
624 #for k in nodespec.keys():
625 #rspec_attribute_value = nodespec[k]
626 #if (self.rspec_to_slice_tag.has_key(k)):
627 #slice_tag_name = self.rspec_to_slice_tag[k]
628 #nodes[nodename][slice_tag_name] = rspec_attribute_value
630 #node_names = nodes.keys()
631 ## remove nodes not in rspec
632 #deleted_nodes = list(set(hostnames).difference(node_names))
633 ## add nodes from rspec
634 #added_nodes = list(set(node_names).difference(hostnames))
638 #self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer)
640 #self.driver.LaunchExperimentOnOAR(slicename, added_nodes)
642 ## Add recognized slice tags
643 #for node_name in node_names:
644 #node = nodes[node_name]
645 #for slice_tag in node.keys():
646 #value = node[slice_tag]
647 #if (isinstance(value, list)):
650 #self.driver.AddSliceTag(slicename, slice_tag, value, node_name)
652 #self.driver.DeleteSliceFromNodes(slicename, deleted_nodes)
655 #self.driver.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id'])