3 from datetime import datetime
5 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
6 from sfa.util.sfalogging import logger
8 from sfa.storage.alchemy import dbsession
9 from sfa.storage.model import RegRecord, RegUser, RegSlice
10 from sqlalchemy.orm import joinedload
11 from sfa.trust.credential import Credential
14 from sfa.managers.driver import Driver
15 from sfa.rspecs.version_manager import VersionManager
16 from sfa.rspecs.rspec import RSpec
18 from sfa.util.xrn import hrn_to_urn, get_authority
21 ## thierry: everything that is API-related (i.e. handling incoming requests)
23 # SlabDriver should be really only about talking to the senslab testbed
26 from sfa.senslab.OARrestapi import OARrestapi
27 from sfa.senslab.LDAPapi import LDAPapi
29 from sfa.senslab.slabpostgres import SlabDB, slab_dbsession, SenslabXP
32 from sfa.senslab.slabaggregate import SlabAggregate, slab_xrn_to_hostname, \
34 from sfa.senslab.slabslices import SlabSlices
39 # this inheritance scheme is so that the driver object can receive
40 # GetNodes or GetSites sorts of calls directly
41 # and thus minimize the differences in the managers with the pl version
45 class SlabDriver(Driver):
46 """ Senslab Driver class inherited from Driver generic class.
48 Contains methods compliant with the SFA standard and the testbed
49 infrastructure (calls to LDAP and OAR).
51 def __init__(self, config):
52 Driver.__init__ (self, config)
54 self.hrn = config.SFA_INTERFACE_HRN
55 self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
56 self.oar = OARrestapi()
58 self.time_format = "%Y-%m-%d %H:%M:%S"
59 self.db = SlabDB(config, debug = True)
63 def sliver_status(self, slice_urn, slice_hrn):
64 """Receive a status request for slice named urn/hrn
65 urn:publicid:IDN+senslab+nturro_slice hrn senslab.nturro_slice
66 shall return a structure as described in
67 http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
68 NT : not sure if we should implement this or not, but used by sface.
72 #First get the slice with the slice hrn
73 slice_list = self.GetSlices(slice_filter = slice_hrn, \
74 slice_filter_type = 'slice_hrn')
76 if len(slice_list) is 0:
77 raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn))
79 #Slice has the same slice hrn for each slice in the slice/lease list
80 #So fetch the info on the user once
81 one_slice = slice_list[0]
82 #recuser = dbsession.query(RegRecord).filter_by(record_id = \
83 #one_slice['record_id_user']).first()
85 #Make a list of all the nodes hostnames in use for this slice
87 for single_slice in slice_list:
88 for node in single_slice['node_ids']:
89 slice_nodes_list.append(node['hostname'])
91 #Get all the corresponding nodes details
92 nodes_all = self.GetNodes({'hostname':slice_nodes_list},
93 ['node_id', 'hostname','site','boot_state'])
94 nodeall_byhostname = dict([(one_node['hostname'], one_node) \
95 for one_node in nodes_all])
99 for single_slice in slice_list:
102 top_level_status = 'empty'
105 ['geni_urn','pl_login','geni_status','geni_resources'], None)
106 result['pl_login'] = one_slice['reg_researchers']['hrn']
107 logger.debug("Slabdriver - sliver_status Sliver status \
108 urn %s hrn %s single_slice %s \r\n " \
109 %(slice_urn, slice_hrn, single_slice))
111 nodes_in_slice = single_slice['node_ids']
114 result['geni_status'] = top_level_status
115 result['geni_resources'] = []
118 top_level_status = 'ready'
120 #A job is running on Senslab for this slice
121 # report about the local nodes that are in the slice only
123 result['geni_urn'] = slice_urn
127 #timestamp = float(sl['startTime']) + float(sl['walltime'])
128 #result['pl_expires'] = strftime(self.time_format, \
129 #gmtime(float(timestamp)))
130 #result['slab_expires'] = strftime(self.time_format,\
131 #gmtime(float(timestamp)))
134 for node in single_slice['node_ids']:
136 #res['slab_hostname'] = node['hostname']
137 #res['slab_boot_state'] = node['boot_state']
139 res['pl_hostname'] = node['hostname']
140 res['pl_boot_state'] = \
141 nodeall_byhostname[node['hostname']]['boot_state']
142 #res['pl_last_contact'] = strftime(self.time_format, \
143 #gmtime(float(timestamp)))
144 sliver_id = Xrn(slice_urn, type='slice', \
145 id=nodeall_byhostname[node['hostname']]['node_id'], \
146 authority=self.hrn).urn
148 res['geni_urn'] = sliver_id
149 node_name = node['hostname']
150 if nodeall_byhostname[node_name]['boot_state'] == 'Alive':
152 res['geni_status'] = 'ready'
154 res['geni_status'] = 'failed'
155 top_level_status = 'failed'
157 res['geni_error'] = ''
159 resources.append(res)
161 result['geni_status'] = top_level_status
162 result['geni_resources'] = resources
163 logger.debug("SLABDRIVER \tsliver_statusresources %s res %s "\
167 def get_user(self, hrn):
168 return dbsession.query(RegRecord).filter_by(hrn = hrn).first()
171 def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \
173 aggregate = SlabAggregate(self)
175 slices = SlabSlices(self)
176 peer = slices.get_peer(slice_hrn)
177 sfa_peer = slices.get_sfa_peer(slice_hrn)
180 if not isinstance(creds, list):
184 slice_record = users[0].get('slice_record', {})
185 logger.debug("SLABDRIVER.PY \t create_sliver \t\
186 slice_record %s \r\n \r\n users %s" \
187 %(slice_record, users))
188 slice_record['user'] = {'keys':users[0]['keys'], \
189 'email':users[0]['email'], \
190 'hrn':slice_record['reg-researchers'][0]}
192 rspec = RSpec(rspec_string)
193 logger.debug("SLABDRIVER.PY \t create_sliver \trspec.version \
194 %s slice_record %s users %s" \
195 %(rspec.version,slice_record, users))
198 # ensure site record exists?
199 # ensure slice record exists
200 #Removed options to verify_slice SA 14/08/12
201 sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \
204 # ensure person records exists
205 #verify_persons returns added persons but since the return value
207 slices.verify_persons(slice_hrn, sfa_slice, users, peer, \
208 sfa_peer, options=options)
209 #requested_attributes returned by rspec.version.get_slice_attributes()
210 #unused, removed SA 13/08/12
211 rspec.version.get_slice_attributes()
213 logger.debug("SLABDRIVER.PY create_sliver slice %s " %(sfa_slice))
215 # add/remove slice from nodes
217 requested_slivers = [node.get('component_id') \
218 for node in rspec.version.get_nodes_with_slivers()\
219 if node.get('authority_id') is self.root_auth]
220 l = [ node for node in rspec.version.get_nodes_with_slivers() ]
221 logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
222 requested_slivers %s listnodes %s" \
223 %(requested_slivers,l))
224 #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
225 #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
228 requested_lease_list = []
230 logger.debug("SLABDRIVER.PY \tcreate_sliver AVANTLEASE " )
231 rspec_requested_leases = rspec.version.get_leases()
232 for lease in rspec.version.get_leases():
233 single_requested_lease = {}
234 logger.debug("SLABDRIVER.PY \tcreate_sliver lease %s " %(lease))
236 if not lease.get('lease_id'):
237 if get_authority(lease['component_id']) == self.root_auth:
238 single_requested_lease['hostname'] = \
239 slab_xrn_to_hostname(\
240 lease.get('component_id').strip())
241 single_requested_lease['start_time'] = \
242 lease.get('start_time')
243 single_requested_lease['duration'] = lease.get('duration')
245 requested_lease_list.append(single_requested_lease)
247 logger.debug("SLABDRIVER.PY \tcreate_sliver APRESLEASE" )
248 #dCreate dict of leases by start_time, regrouping nodes reserved
250 #time, for the same amount of time = one job on OAR
251 requested_job_dict = {}
252 for lease in requested_lease_list:
254 #In case it is an asap experiment start_time is empty
255 if lease['start_time'] == '':
256 lease['start_time'] = '0'
258 if lease['start_time'] not in requested_job_dict:
259 if isinstance(lease['hostname'], str):
260 lease['hostname'] = [lease['hostname']]
262 requested_job_dict[lease['start_time']] = lease
265 job_lease = requested_job_dict[lease['start_time']]
266 if lease['duration'] == job_lease['duration'] :
267 job_lease['hostname'].append(lease['hostname'])
272 logger.debug("SLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\
273 %(requested_job_dict))
274 #verify_slice_leases returns the leases , but the return value is unused
275 #here. Removed SA 13/08/12
276 slices.verify_slice_leases(sfa_slice, \
277 requested_job_dict, peer)
279 return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version)
282 def delete_sliver (self, slice_urn, slice_hrn, creds, options):
284 sfa_slice_list = self.GetSlices(slice_filter = slice_hrn, \
285 slice_filter_type = 'slice_hrn')
287 if not sfa_slice_list:
290 #Delete all in the slice
291 for sfa_slice in sfa_slice_list:
294 logger.debug("SLABDRIVER.PY delete_sliver slice %s" %(sfa_slice))
295 slices = SlabSlices(self)
296 # determine if this is a peer slice
298 peer = slices.get_peer(slice_hrn)
299 #TODO delete_sliver SA : UnBindObjectFromPeer should be
300 #used when there is another
301 #senslab testbed, which is not the case 14/08/12 .
303 logger.debug("SLABDRIVER.PY delete_sliver peer %s" %(peer))
306 self.UnBindObjectFromPeer('slice', \
307 sfa_slice['record_id_slice'], \
309 self.DeleteSliceFromNodes(sfa_slice)
312 self.BindObjectToPeer('slice', \
313 sfa_slice['record_id_slice'], \
314 peer, sfa_slice['peer_slice_id'])
318 def AddSlice(self, slice_record, user_record):
319 """Add slice to the sfa tables and senslab table only if the user
320 already exists in senslab database(user already registered in LDAP).
321 There is no way to separate adding the slice to the tesbed
322 and then importing it from the testbed to SFA because of
323 senslab's architecture. Therefore, sfa tables are updated here.
326 sfa_record = RegSlice(hrn=slice_record['slice_hrn'],
327 gid=slice_record['gid'],
328 pointer=slice_record['slice_id'],
329 authority=slice_record['authority'])
331 logger.debug("SLABDRIVER.PY AddSlice sfa_record %s user_record %s" \
332 %(sfa_record, user_record))
333 sfa_record.just_created()
334 dbsession.add(sfa_record)
336 #Update the reg-researcher dependance table
337 sfa_record.reg_researchers = [user_record]
340 #Update the senslab table with the new slice
341 #slab_slice = SenslabXP( slice_hrn = slice_record['slice_hrn'], \
342 #record_id_slice = sfa_record.record_id , \
343 #record_id_user = slice_record['record_id_user'], \
344 #peer_authority = slice_record['peer_authority'])
346 #logger.debug("SLABDRIVER.PY \tAddSlice slice_record %s \
347 #slab_slice %s sfa_record %s" \
348 #%(slice_record,slab_slice, sfa_record))
349 #slab_dbsession.add(slab_slice)
350 #slab_dbsession.commit()
353 # first 2 args are None in case of resource discovery
354 def list_resources (self, slice_urn, slice_hrn, creds, options):
355 #cached_requested = options.get('cached', True)
357 version_manager = VersionManager()
358 # get the rspec's return format from options
360 version_manager.get_version(options.get('geni_rspec_version'))
361 version_string = "rspec_%s" % (rspec_version)
363 #panos adding the info option to the caching key (can be improved)
364 if options.get('info'):
365 version_string = version_string + "_" + \
366 options.get('info', 'default')
368 # Adding the list_leases option to the caching key
369 if options.get('list_leases'):
370 version_string = version_string + "_"+options.get('list_leases', 'default')
372 # Adding geni_available to caching key
373 if options.get('geni_available'):
374 version_string = version_string + "_" + str(options.get('geni_available'))
376 # look in cache first
377 #if cached_requested and self.cache and not slice_hrn:
378 #rspec = self.cache.get(version_string)
380 #logger.debug("SlabDriver.ListResources: \
381 #returning cached advertisement")
384 #panos: passing user-defined options
385 aggregate = SlabAggregate(self)
386 #origin_hrn = Credential(string=creds[0]).get_gid_caller().get_hrn()
387 #options.update({'origin_hrn':origin_hrn})
388 rspec = aggregate.get_rspec(slice_xrn=slice_urn, \
389 version=rspec_version, options=options)
392 #if self.cache and not slice_hrn:
393 #logger.debug("Slab.ListResources: stores advertisement in cache")
394 #self.cache.add(version_string, rspec)
399 def list_slices (self, creds, options):
400 # look in cache first
402 #slices = self.cache.get('slices')
404 #logger.debug("PlDriver.list_slices returns from cache")
409 slices = self.GetSlices()
410 logger.debug("SLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices))
411 slice_hrns = [slab_slice['hrn'] for slab_slice in slices]
413 slice_urns = [hrn_to_urn(slice_hrn, 'slice') \
414 for slice_hrn in slice_hrns]
418 #logger.debug ("SlabDriver.list_slices stores value in cache")
419 #self.cache.add('slices', slice_urns)
424 def register (self, sfa_record, hrn, pub_key):
426 Adding new user, slice, node or site should not be handled
430 Adding users = LDAP Senslab
431 Adding slice = Import from LDAP users
437 def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
438 """No site or node record update allowed in Senslab."""
440 pointer = old_sfa_record['pointer']
441 old_sfa_record_type = old_sfa_record['type']
443 # new_key implemented for users only
444 if new_key and old_sfa_record_type not in [ 'user' ]:
445 raise UnknownSfaType(old_sfa_record_type)
447 #if (type == "authority"):
448 #self.shell.UpdateSite(pointer, new_sfa_record)
450 if old_sfa_record_type == "slice":
451 slab_record = self.sfa_fields_to_slab_fields(old_sfa_record_type, \
453 if 'name' in slab_record:
454 slab_record.pop('name')
455 #Prototype should be UpdateSlice(self,
456 #auth, slice_id_or_name, slice_fields)
457 #Senslab cannot update slice since slice = job
458 #so we must delete and create another job
459 self.UpdateSlice(pointer, slab_record)
461 elif old_sfa_record_type == "user":
463 all_fields = new_sfa_record
464 for key in all_fields.keys():
465 if key in ['first_name', 'last_name', 'title', 'email',
466 'password', 'phone', 'url', 'bio', 'accepted_aup',
468 update_fields[key] = all_fields[key]
469 self.UpdatePerson(pointer, update_fields)
472 # must check this key against the previous one if it exists
473 persons = self.GetPersons([pointer], ['key_ids'])
475 keys = person['key_ids']
476 keys = self.GetKeys(person['key_ids'])
478 # Delete all stale keys
481 if new_key != key['key']:
482 self.DeleteKey(key['key_id'])
486 self.AddPersonKey(pointer, {'key_type': 'ssh', \
493 def remove (self, sfa_record):
494 sfa_record_type = sfa_record['type']
495 hrn = sfa_record['hrn']
496 if sfa_record_type == 'user':
498 #get user from senslab ldap
499 person = self.GetPersons(sfa_record)
500 #No registering at a given site in Senslab.
501 #Once registered to the LDAP, all senslab sites are
504 #Mark account as disabled in ldap
505 self.DeletePerson(sfa_record)
506 elif sfa_record_type == 'slice':
507 if self.GetSlices(slice_filter = hrn, \
508 slice_filter_type = 'slice_hrn'):
509 self.DeleteSlice(sfa_record)
511 #elif type == 'authority':
512 #if self.GetSites(pointer):
513 #self.DeleteSite(pointer)
519 #TODO clean GetPeers. 05/07/12SA
520 def GetPeers (self, auth = None, peer_filter=None, return_fields_list=None):
522 existing_records = {}
523 existing_hrns_by_types = {}
524 logger.debug("SLABDRIVER \tGetPeers auth = %s, peer_filter %s, \
525 return_field %s " %(auth , peer_filter, return_fields_list))
526 all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
528 for record in all_records:
529 existing_records[(record.hrn, record.type)] = record
530 if record.type not in existing_hrns_by_types:
531 existing_hrns_by_types[record.type] = [record.hrn]
533 existing_hrns_by_types[record.type].append(record.hrn)
536 logger.debug("SLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\
537 %( existing_hrns_by_types))
542 records_list.append(existing_records[(peer_filter,'authority')])
544 for hrn in existing_hrns_by_types['authority']:
545 records_list.append(existing_records[(hrn,'authority')])
547 logger.debug("SLABDRIVER \tGetPeer \trecords_list %s " \
553 return_records = records_list
554 if not peer_filter and not return_fields_list:
558 logger.debug("SLABDRIVER \tGetPeer return_records %s " \
560 return return_records
563 #TODO : Handling OR request in make_ldap_filters_from_records
564 #instead of the for loop
565 #over the records' list
566 def GetPersons(self, person_filter=None):
568 person_filter should be a list of dictionnaries when not set to None.
569 Returns a list of users whose accounts are enabled found in ldap.
572 logger.debug("SLABDRIVER \tGetPersons person_filter %s" \
575 if person_filter and isinstance(person_filter, list):
576 #If we are looking for a list of users (list of dict records)
577 #Usually the list contains only one user record
578 for searched_attributes in person_filter:
580 #Get only enabled user accounts in senslab LDAP :
581 #add a filter for make_ldap_filters_from_record
582 person = self.ldap.LdapFindUser(searched_attributes, \
583 is_user_enabled=True)
584 #If a person was found, append it to the list
586 person_list.append(person)
588 #If the list is empty, return None
589 if len(person_list) is 0:
593 #Get only enabled user accounts in senslab LDAP :
594 #add a filter for make_ldap_filters_from_record
595 person_list = self.ldap.LdapFindUser(is_user_enabled=True)
599 def GetTimezone(self):
600 """ Get the OAR servier time and timezone.
601 Unused SA 16/11/12"""
602 server_timestamp, server_tz = self.oar.parser.\
603 SendRequest("GET_timezone")
604 return server_timestamp, server_tz
607 def DeleteJobs(self, job_id, slice_hrn):
608 if not job_id or job_id is -1:
610 username = slice_hrn.split(".")[-1].rstrip("_slice")
612 reqdict['method'] = "delete"
613 reqdict['strval'] = str(job_id)
616 answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', \
618 logger.debug("SLABDRIVER \tDeleteJobs jobid %s \r\n answer %s \
619 username %s" %(job_id,answer, username))
624 ##TODO : Unused GetJobsId ? SA 05/07/12
625 #def GetJobsId(self, job_id, username = None ):
627 #Details about a specific job.
628 #Includes details about submission time, jot type, state, events,
629 #owner, assigned ressources, walltime etc...
633 #node_list_k = 'assigned_network_address'
634 ##Get job info from OAR
635 #job_info = self.oar.parser.SendRequest(req, job_id, username)
637 #logger.debug("SLABDRIVER \t GetJobsId %s " %(job_info))
639 #if job_info['state'] == 'Terminated':
640 #logger.debug("SLABDRIVER \t GetJobsId job %s TERMINATED"\
643 #if job_info['state'] == 'Error':
644 #logger.debug("SLABDRIVER \t GetJobsId ERROR message %s "\
649 #logger.error("SLABDRIVER \tGetJobsId KeyError")
652 #parsed_job_info = self.get_info_on_reserved_nodes(job_info, \
654 ##Replaces the previous entry
655 ##"assigned_network_address" / "reserved_resources"
657 #job_info.update({'node_ids':parsed_job_info[node_list_k]})
658 #del job_info[node_list_k]
659 #logger.debug(" \r\nSLABDRIVER \t GetJobsId job_info %s " %(job_info))
663 def GetJobsResources(self, job_id, username = None):
664 #job_resources=['reserved_resources', 'assigned_resources',\
665 #'job_id', 'job_uri', 'assigned_nodes',\
667 #assigned_res = ['resource_id', 'resource_uri']
668 #assigned_n = ['node', 'node_uri']
670 req = "GET_jobs_id_resources"
673 #Get job resources list from OAR
674 node_id_list = self.oar.parser.SendRequest(req, job_id, username)
675 logger.debug("SLABDRIVER \t GetJobsResources %s " %(node_id_list))
678 self.__get_hostnames_from_oar_node_ids(node_id_list)
681 #Replaces the previous entry "assigned_network_address" /
682 #"reserved_resources"
684 job_info = {'node_ids': hostname_list}
689 def get_info_on_reserved_nodes(self, job_info, node_list_name):
690 #Get the list of the testbed nodes records and make a
691 #dictionnary keyed on the hostname out of it
692 node_list_dict = self.GetNodes()
693 #node_hostname_list = []
694 node_hostname_list = [node['hostname'] for node in node_list_dict]
695 #for node in node_list_dict:
696 #node_hostname_list.append(node['hostname'])
697 node_dict = dict(zip(node_hostname_list, node_list_dict))
699 reserved_node_hostname_list = []
700 for index in range(len(job_info[node_list_name])):
701 #job_info[node_list_name][k] =
702 reserved_node_hostname_list[index] = \
703 node_dict[job_info[node_list_name][index]]['hostname']
705 logger.debug("SLABDRIVER \t get_info_on_reserved_nodes \
706 reserved_node_hostname_list %s" \
707 %(reserved_node_hostname_list))
709 logger.error("SLABDRIVER \t get_info_on_reserved_nodes KEYERROR " )
711 return reserved_node_hostname_list
713 def GetNodesCurrentlyInUse(self):
714 """Returns a list of all the nodes already involved in an oar job"""
715 return self.oar.parser.SendRequest("GET_running_jobs")
717 def __get_hostnames_from_oar_node_ids(self, resource_id_list ):
718 full_nodes_dict_list = self.GetNodes()
719 #Put the full node list into a dictionary keyed by oar node id
720 oar_id_node_dict = {}
721 for node in full_nodes_dict_list:
722 oar_id_node_dict[node['oar_id']] = node
724 #logger.debug("SLABDRIVER \t __get_hostnames_from_oar_node_ids\
725 #oar_id_node_dict %s" %(oar_id_node_dict))
727 hostname_dict_list = []
728 for resource_id in resource_id_list:
729 #Because jobs requested "asap" do not have defined resources
730 if resource_id is not "Undefined":
731 hostname_dict_list.append(\
732 oar_id_node_dict[resource_id]['hostname'])
734 #hostname_list.append(oar_id_node_dict[resource_id]['hostname'])
735 return hostname_dict_list
737 def GetReservedNodes(self,username = None):
738 #Get the nodes in use and the reserved nodes
739 reservation_dict_list = \
740 self.oar.parser.SendRequest("GET_reserved_nodes", \
744 for resa in reservation_dict_list:
745 logger.debug ("GetReservedNodes resa %s"%(resa))
746 #dict list of hostnames and their site
747 resa['reserved_nodes'] = \
748 self.__get_hostnames_from_oar_node_ids(resa['resource_ids'])
750 #del resa['resource_ids']
751 return reservation_dict_list
753 def GetNodes(self, node_filter_dict = None, return_fields_list = None):
755 node_filter_dict : dictionnary of lists
758 node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
759 node_dict_list = node_dict_by_id.values()
760 logger.debug (" SLABDRIVER GetNodes node_filter_dict %s \
761 return_fields_list %s "%(node_filter_dict, return_fields_list))
762 #No filtering needed return the list directly
763 if not (node_filter_dict or return_fields_list):
764 return node_dict_list
766 return_node_list = []
768 for filter_key in node_filter_dict:
770 #Filter the node_dict_list by each value contained in the
771 #list node_filter_dict[filter_key]
772 for value in node_filter_dict[filter_key]:
773 for node in node_dict_list:
774 if node[filter_key] == value:
775 if return_fields_list :
777 for k in return_fields_list:
779 return_node_list.append(tmp)
781 return_node_list.append(node)
783 logger.log_exc("GetNodes KeyError")
787 return return_node_list
790 def GetSites(self, site_filter_name_list = None, return_fields_list = None):
791 site_dict = self.oar.parser.SendRequest("GET_sites")
792 #site_dict : dict where the key is the sit ename
793 return_site_list = []
794 if not ( site_filter_name_list or return_fields_list):
795 return_site_list = site_dict.values()
796 return return_site_list
798 for site_filter_name in site_filter_name_list:
799 if site_filter_name in site_dict:
800 if return_fields_list:
801 for field in return_fields_list:
804 tmp[field] = site_dict[site_filter_name][field]
806 logger.error("GetSites KeyError %s "%(field))
808 return_site_list.append(tmp)
810 return_site_list.append( site_dict[site_filter_name])
813 return return_site_list
816 def _sql_get_slice_info( self, slice_filter ):
817 #DO NOT USE RegSlice - reg_researchers to get the hrn
818 #of the user otherwise will mess up the RegRecord in
819 #Resolve, don't know why - SA 08/08/2012
821 #Only one entry for one user = one slice in slab_xp table
822 #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
823 raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn = slice_filter).first()
824 #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
827 #raw_slicerec.reg_researchers
828 raw_slicerec = raw_slicerec.__dict__
829 logger.debug(" SLABDRIVER \t get_slice_info slice_filter %s raw_slicerec %s"%(slice_filter,raw_slicerec))
830 slicerec = raw_slicerec
831 #only one researcher per slice so take the first one
832 #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
833 #del slicerec['reg_researchers']['_sa_instance_state']
840 def _sql_get_slice_info_from_user( self, slice_filter ):
841 #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
842 raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id = slice_filter).first()
843 #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
844 #Put it in correct order
845 user_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'email', 'pointer']
846 slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'pointer']
848 #raw_slicerec.reg_slices_as_researcher
849 raw_slicerec = raw_slicerec.__dict__
851 slicerec = dict([(k,raw_slicerec['reg_slices_as_researcher'][0].__dict__[k]) for k in slice_needed_fields])
852 slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) for k in user_needed_fields])
853 #TODO Handle multiple slices for one user SA 10/12/12
854 #for now only take the first slice record associated to the rec user
855 ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
856 #del raw_slicerec['reg_slices_as_researcher']
857 #slicerec['reg_researchers'] = raw_slicerec
858 ##del slicerec['_sa_instance_state']
865 def _get_slice_records(self, slice_filter = None, \
866 slice_filter_type = None):
869 #Get list of slices based on the slice hrn
870 if slice_filter_type == 'slice_hrn':
872 if get_authority(slice_filter) == self.root_auth:
873 login = slice_filter.split(".")[1].split("_")[0]
875 slicerec = self._sql_get_slice_info(slice_filter)
880 #Get slice based on user id
881 if slice_filter_type == 'record_id_user':
883 slicerec = self._sql_get_slice_info_from_user(slice_filter)
886 fixed_slicerec_dict = slicerec
887 #At this point if the there is no login it means
888 #record_id_user filter has been used for filtering
890 #If theslice record is from senslab
891 if fixed_slicerec_dict['peer_authority'] is None:
892 login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
893 return login, fixed_slicerec_dict
896 def GetSlices(self, slice_filter = None, slice_filter_type = None):
897 """ Get the slice records from the slab db.
898 Returns a slice ditc if slice_filter and slice_filter_type
900 Returns a list of slice dictionnaries if there are no filters
905 authorized_filter_types_list = ['slice_hrn', 'record_id_user']
906 return_slicerec_dictlist = []
908 #First try to get information on the slice based on the filter provided
909 if slice_filter_type in authorized_filter_types_list:
911 login, fixed_slicerec_dict = \
912 self._get_slice_records(slice_filter, slice_filter_type)
913 logger.debug(" SLABDRIVER \tGetSlices login %s \
914 slice record %s slice_filter %s slice_filter_type %s "\
915 %(login, fixed_slicerec_dict,slice_filter, slice_filter_type))
918 #Now we have the slice record fixed_slicerec_dict, get the
919 #jobs associated to this slice
920 #leases_list = self.GetReservedNodes(username = login)
921 leases_list = self.GetLeases(login = login)
922 #If no job is running or no job scheduled
923 #return only the slice record
924 if leases_list == [] and fixed_slicerec_dict:
925 return_slicerec_dictlist.append(fixed_slicerec_dict)
927 #If several jobs for one slice , put the slice record into
928 # each lease information dict
929 for lease in leases_list :
932 reserved_list = lease['reserved_nodes']
934 slicerec_dict['oar_job_id']= lease['lease_id']
935 slicerec_dict.update({'list_node_ids':{'hostname':reserved_list}})
936 slicerec_dict.update({'node_ids':lease['reserved_nodes']})
938 #Update lease dict with the slice record
939 if fixed_slicerec_dict:
940 fixed_slicerec_dict['oar_job_id'].append(slicerec_dict['oar_job_id'])
941 slicerec_dict.update(fixed_slicerec_dict)
942 #slicerec_dict.update({'hrn':\
943 #str(fixed_slicerec_dict['slice_hrn'])})
946 return_slicerec_dictlist.append(slicerec_dict)
947 logger.debug("SLABDRIVER.PY \tGetSlices \
948 slicerec_dict %s return_slicerec_dictlist %s \
949 lease['reserved_nodes'] \
950 %s" %(slicerec_dict, return_slicerec_dictlist, \
951 lease['reserved_nodes'] ))
953 logger.debug("SLABDRIVER.PY \tGetSlices RETURN \
954 return_slicerec_dictlist %s" \
955 %(return_slicerec_dictlist))
957 return return_slicerec_dictlist
961 #Get all slices from the senslab sfa database ,
962 #put them in dict format
963 #query_slice_list = dbsession.query(RegRecord).all()
964 query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
965 #query_slice_list = dbsession.query(RegRecord).filter_by(type='slice').all()
966 #query_slice_list = slab_dbsession.query(SenslabXP).all()
967 return_slicerec_dictlist = []
968 for record in query_slice_list:
969 tmp = record.__dict__
970 tmp['reg_researchers'] = tmp['reg_researchers'].__dict__
971 #del tmp['reg_researchers']['_sa_instance_state']
972 return_slicerec_dictlist.append(tmp)
973 #return_slicerec_dictlist.append(record.__dict__)
975 #Get all the jobs reserved nodes
976 leases_list = self.GetReservedNodes()
979 for fixed_slicerec_dict in return_slicerec_dictlist:
981 #Check if the slice belongs to a senslab user
982 if fixed_slicerec_dict['peer_authority'] is None:
983 owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
986 for lease in leases_list:
987 if owner == lease['user']:
988 slicerec_dict['oar_job_id'] = lease['lease_id']
990 #for reserved_node in lease['reserved_nodes']:
991 logger.debug("SLABDRIVER.PY \tGetSlices lease %s "\
994 reserved_list = lease['reserved_nodes']
996 slicerec_dict.update({'node_ids':lease['reserved_nodes']})
997 slicerec_dict.update({'list_node_ids':{'hostname':reserved_list}})
998 slicerec_dict.update(fixed_slicerec_dict)
999 #slicerec_dict.update({'hrn':\
1000 #str(fixed_slicerec_dict['slice_hrn'])})
1001 #return_slicerec_dictlist.append(slicerec_dict)
1002 fixed_slicerec_dict.update(slicerec_dict)
1004 logger.debug("SLABDRIVER.PY \tGetSlices RETURN \
1005 return_slicerec_dictlist %s \slice_filter %s " \
1006 %(return_slicerec_dictlist, slice_filter))
1008 return return_slicerec_dictlist
1011 def testbed_name (self): return self.hrn
1013 # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
1014 def aggregate_version (self):
1015 version_manager = VersionManager()
1016 ad_rspec_versions = []
1017 request_rspec_versions = []
1018 for rspec_version in version_manager.versions:
1019 if rspec_version.content_type in ['*', 'ad']:
1020 ad_rspec_versions.append(rspec_version.to_dict())
1021 if rspec_version.content_type in ['*', 'request']:
1022 request_rspec_versions.append(rspec_version.to_dict())
1024 'testbed':self.testbed_name(),
1025 'geni_request_rspec_versions': request_rspec_versions,
1026 'geni_ad_rspec_versions': ad_rspec_versions,
1032 # Convert SFA fields to PLC fields for use when registering up updating
1033 # registry record in the PLC database
1035 # @param type type of record (user, slice, ...)
1036 # @param hrn human readable name
1037 # @param sfa_fields dictionary of SFA fields
1038 # @param slab_fields dictionary of PLC fields (output)
1040 def sfa_fields_to_slab_fields(self, sfa_type, hrn, record):
1044 #for field in record:
1045 # slab_record[field] = record[field]
1047 if sfa_type == "slice":
1048 #instantion used in get_slivers ?
1049 if not "instantiation" in slab_record:
1050 slab_record["instantiation"] = "senslab-instantiated"
1051 #slab_record["hrn"] = hrn_to_pl_slicename(hrn)
1052 #Unused hrn_to_pl_slicename because Slab's hrn already
1053 #in the appropriate form SA 23/07/12
1054 slab_record["hrn"] = hrn
1055 logger.debug("SLABDRIVER.PY sfa_fields_to_slab_fields \
1056 slab_record %s " %(slab_record['hrn']))
1058 slab_record["url"] = record["url"]
1059 if "description" in record:
1060 slab_record["description"] = record["description"]
1061 if "expires" in record:
1062 slab_record["expires"] = int(record["expires"])
1064 #nodes added by OAR only and then imported to SFA
1065 #elif type == "node":
1066 #if not "hostname" in slab_record:
1067 #if not "hostname" in record:
1068 #raise MissingSfaInfo("hostname")
1069 #slab_record["hostname"] = record["hostname"]
1070 #if not "model" in slab_record:
1071 #slab_record["model"] = "geni"
1074 #elif type == "authority":
1075 #slab_record["login_base"] = hrn_to_slab_login_base(hrn)
1077 #if not "name" in slab_record:
1078 #slab_record["name"] = hrn
1080 #if not "abbreviated_name" in slab_record:
1081 #slab_record["abbreviated_name"] = hrn
1083 #if not "enabled" in slab_record:
1084 #slab_record["enabled"] = True
1086 #if not "is_public" in slab_record:
1087 #slab_record["is_public"] = True
1094 def __transforms_timestamp_into_date(self, xp_utc_timestamp = None):
1095 """ Transforms unix timestamp into valid OAR date format """
1097 #Used in case of a scheduled experiment (not immediate)
1098 #To run an XP immediately, don't specify date and time in RSpec
1099 #They will be set to None.
1100 if xp_utc_timestamp:
1101 #transform the xp_utc_timestamp into server readable time
1102 xp_server_readable_date = datetime.fromtimestamp(int(\
1103 xp_utc_timestamp)).strftime(self.time_format)
1105 return xp_server_readable_date
1113 def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
1114 lease_start_time, lease_duration, slice_user=None):
1116 lease_dict['lease_start_time'] = lease_start_time
1117 lease_dict['lease_duration'] = lease_duration
1118 lease_dict['added_nodes'] = added_nodes
1119 lease_dict['slice_name'] = slice_name
1120 lease_dict['slice_user'] = slice_user
1121 lease_dict['grain'] = self.GetLeaseGranularity()
1122 lease_dict['time_format'] = self.time_format
1125 def __create_job_structure_request_for_OAR(lease_dict):
1126 """ Creates the structure needed for a correct POST on OAR.
1127 Makes the timestamp transformation into the appropriate format.
1128 Sends the POST request to create the job with the resources in
1137 reqdict['workdir'] = '/tmp'
1138 reqdict['resource'] = "{network_address in ("
1140 for node in lease_dict['added_nodes']:
1141 logger.debug("\r\n \r\n OARrestapi \t \
1142 __create_job_structure_request_for_OAR node %s" %(node))
1144 # Get the ID of the node
1146 reqdict['resource'] += "'" + nodeid + "', "
1147 nodeid_list.append(nodeid)
1149 custom_length = len(reqdict['resource'])- 2
1150 reqdict['resource'] = reqdict['resource'][0:custom_length] + \
1151 ")}/nodes=" + str(len(nodeid_list))
1153 def __process_walltime(duration):
1154 """ Calculates the walltime in seconds from the duration in H:M:S
1155 specified in the RSpec.
1159 # Fixing the walltime by adding a few delays.
1160 # First put the walltime in seconds oarAdditionalDelay = 20;
1161 # additional delay for /bin/sleep command to
1162 # take in account prologue and epilogue scripts execution
1163 # int walltimeAdditionalDelay = 240; additional delay
1164 desired_walltime = duration
1165 total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
1166 sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
1168 #Put the walltime back in str form
1169 #First get the hours
1170 walltime.append(str(total_walltime / 3600))
1171 total_walltime = total_walltime - 3600 * int(walltime[0])
1172 #Get the remaining minutes
1173 walltime.append(str(total_walltime / 60))
1174 total_walltime = total_walltime - 60 * int(walltime[1])
1176 walltime.append(str(total_walltime))
1179 logger.log_exc(" __process_walltime duration null")
1181 return walltime, sleep_walltime
1184 walltime, sleep_walltime = \
1185 __process_walltime(int(lease_dict['lease_duration'])*lease_dict['grain'])
1188 reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
1189 ":" + str(walltime[1]) + ":" + str(walltime[2])
1190 reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
1192 #In case of a scheduled experiment (not immediate)
1193 #To run an XP immediately, don't specify date and time in RSpec
1194 #They will be set to None.
1195 if lease_dict['lease_start_time'] is not '0':
1196 #Readable time accepted by OAR
1197 start_time = datetime.fromtimestamp(int(lease_dict['lease_start_time'])).\
1198 strftime(lease_dict['time_format'])
1199 reqdict['reservation'] = start_time
1200 #If there is not start time, Immediate XP. No need to add special
1204 reqdict['type'] = "deploy"
1205 reqdict['directory'] = ""
1206 reqdict['name'] = "SFA_" + lease_dict['slice_user']
1211 #Create the request for OAR
1212 reqdict = __create_job_structure_request_for_OAR(lease_dict)
1213 # first step : start the OAR job and update the job
1214 logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\
1217 answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
1218 reqdict, slice_user)
1219 logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s " %(answer))
1221 jobid = answer['id']
1223 logger.log_exc("SLABDRIVER \tLaunchExperimentOnOAR \
1224 Impossible to create job %s " %(answer))
1228 def __configure_experiment(jobid, added_nodes):
1229 # second step : configure the experiment
1230 # we need to store the nodes in a yaml (well...) file like this :
1231 # [1,56,23,14,45,75] with name /tmp/sfa<jobid>.json
1232 job_file = open('/tmp/sfa/'+ str(jobid) + '.json', 'w')
1234 job_file.write(str(added_nodes[0].strip('node')))
1235 for node in added_nodes[1:len(added_nodes)] :
1236 job_file.write(', '+ node.strip('node'))
1241 def __launch_senslab_experiment(jobid):
1242 # third step : call the senslab-experiment wrapper
1243 #command= "java -jar target/sfa-1.0-jar-with-dependencies.jar
1244 # "+str(jobid)+" "+slice_user
1245 javacmdline = "/usr/bin/java"
1247 "/opt/senslabexperimentwrapper/sfa-1.0-jar-with-dependencies.jar"
1249 output = subprocess.Popen([javacmdline, "-jar", jarname, str(jobid), \
1250 slice_user],stdout=subprocess.PIPE).communicate()[0]
1252 logger.debug("SLABDRIVER \t __configure_experiment wrapper returns%s " \
1259 logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s \
1260 added_nodes %s slice_user %s" %(jobid, added_nodes, slice_user))
1263 __configure_experiment(jobid, added_nodes)
1264 __launch_senslab_experiment(jobid)
1269 def AddLeases(self, hostname_list, slice_record, \
1270 lease_start_time, lease_duration):
1271 logger.debug("SLABDRIVER \r\n \r\n \t AddLeases hostname_list %s \
1272 slice_record %s lease_start_time %s lease_duration %s "\
1273 %( hostname_list, slice_record , lease_start_time, \
1276 tmp = slice_record['reg-researchers'][0].split(".")
1277 username = tmp[(len(tmp)-1)]
1278 job_id = self.LaunchExperimentOnOAR(hostname_list, slice_record['hrn'], \
1279 lease_start_time, lease_duration, username)
1280 start_time = datetime.fromtimestamp(int(lease_start_time)).strftime(self.time_format)
1281 end_time = lease_start_time + lease_duration
1282 slab_ex_row = SenslabXP(slice_record['hrn'], job_id, end_time)
1283 logger.debug("SLABDRIVER \r\n \r\n \t slab_ex_row %s" %(slab_ex_row))
1284 slab_dbsession.add(slab_ex_row)
1285 slab_dbsession.commit()
1287 logger.debug("SLABDRIVER \t AddLeases hostname_list start_time %s " %(start_time))
1292 #Delete the jobs from job_senslab table
1293 def DeleteSliceFromNodes(self, slice_record):
1295 self.DeleteJobs(slice_record['oar_job_id'], slice_record['hrn'])
1299 def GetLeaseGranularity(self):
1300 """ Returns the granularity of Senslab testbed.
1301 OAR returns seconds for experiments duration.
1302 Defined in seconds. """
1307 def update_jobs_in_slabdb(self, job_oar_list, jobs_psql):
1308 #Get all the entries in slab_xp table
1311 jobs_psql = set(jobs_psql)
1312 kept_jobs = set(job_oar_list).intersection(jobs_psql)
1314 deleted_jobs = set(jobs_psql).difference(kept_jobs)
1315 deleted_jobs = list(deleted_jobs)
1316 if len(deleted_jobs) > 0:
1317 slab_dbsession.query(SenslabXP).filter(SenslabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
1318 slab_dbsession.commit()
1324 def GetLeases(self, lease_filter_dict=None, login=None):
1327 unfiltered_reservation_list = self.GetReservedNodes(login)
1329 reservation_list = []
1330 #Find the slice associated with this user senslab ldap uid
1331 logger.debug(" SLABDRIVER.PY \tGetLeases unfiltered_reservation_list %s " %(unfiltered_reservation_list))
1332 #Create user dict first to avoid looking several times for
1333 #the same user in LDAP SA 27/07/12
1337 jobs_psql_query = slab_dbsession.query(SenslabXP).all()
1338 jobs_psql_dict = [ (row.job_id, row.__dict__ )for row in jobs_psql_query ]
1339 jobs_psql_dict = dict(jobs_psql_dict)
1340 logger.debug("SLABDRIVER \r\n \r\n \tGetLeases jobs_psql_dict %s"\
1342 jobs_psql_id_list = [ row.job_id for row in jobs_psql_query ]
1346 for resa in unfiltered_reservation_list:
1347 logger.debug("SLABDRIVER \tGetLeases USER %s"\
1349 #Cosntruct list of jobs (runing, waiting..) in oar
1350 job_oar_list.append(resa['lease_id'])
1351 if resa['lease_id'] in jobs_psql_dict:
1352 job_info = jobs_psql_dict[resa['lease_id']]
1354 #if resa['user'] not in resa_user_dict:
1355 #logger.debug("SLABDRIVER \tGetLeases userNOTIN ")
1356 #ldap_info = self.ldap.LdapSearch('(uid='+resa['user']+')')
1358 #ldap_info = ldap_info[0][1]
1359 ##Get the backref :relationship table reg-researchers
1360 #user = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(email = \
1361 #ldap_info['mail'][0])
1363 #user = user.first()
1364 #user = user.__dict__
1365 #slice_info = user['reg_slices_as_researcher'][0].__dict__
1366 ##Separated in case user not in database :
1367 ##record_id not defined SA 17/07//12
1369 ##query_slice_info = slab_dbsession.query(SenslabXP).filter_by(record_id_user = user.record_id)
1370 ##if query_slice_info:
1371 ##slice_info = query_slice_info.first()
1375 #resa_user_dict[resa['user']] = {}
1376 #resa_user_dict[resa['user']]['ldap_info'] = user
1377 #resa_user_dict[resa['user']]['slice_info'] = slice_info
1379 #resa['slice_hrn'] = resa_user_dict[resa['user']]['slice_info']['hrn']
1380 #resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
1382 resa['slice_hrn'] = job_info['slice_hrn']
1383 resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
1385 resa['component_id_list'] = []
1386 #Transform the hostnames into urns (component ids)
1387 for node in resa['reserved_nodes']:
1388 #resa['component_id_list'].append(hostname_to_urn(self.hrn, \
1389 #self.root_auth, node['hostname']))
1390 slab_xrn = slab_xrn_object(self.root_auth, node)
1391 resa['component_id_list'].append(slab_xrn.urn)
1393 if lease_filter_dict:
1394 if lease_filter_dict['name'] == resa['slice_hrn']:
1395 reservation_list.append(resa)
1397 if lease_filter_dict is None:
1398 reservation_list = unfiltered_reservation_list
1400 #del unfiltered_reservation_list[unfiltered_reservation_list.index(resa)]
1403 self.update_jobs_in_slabdb(job_oar_list, jobs_psql_id_list)
1404 logger.debug("SLABDRIVER \tGetLeases resa_user_dict %s"\
1406 #for resa in unfiltered_reservation_list:
1410 #if resa['user'] in resa_user_dict:
1411 #resa['slice_hrn'] = resa_user_dict[resa['user']]['slice_info']['hrn']
1412 #resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
1414 ##resa['slice_id'] = hrn_to_urn(slice_info.slice_hrn, 'slice')
1415 #resa['component_id_list'] = []
1416 ##Transform the hostnames into urns (component ids)
1417 #for node in resa['reserved_nodes']:
1418 ##resa['component_id_list'].append(hostname_to_urn(self.hrn, \
1419 ##self.root_auth, node['hostname']))
1420 #slab_xrn = slab_xrn_object(self.root_auth, node)
1421 #resa['component_id_list'].append(slab_xrn.urn)
1423 ##Filter the reservation list if necessary
1424 ##Returns all the leases associated with a given slice
1425 #if lease_filter_dict:
1426 #logger.debug("SLABDRIVER \tGetLeases lease_filter_dict %s"\
1427 #%(lease_filter_dict))
1428 #for resa in unfiltered_reservation_list:
1429 #if lease_filter_dict['name'] == resa['slice_hrn']:
1430 #reservation_list.append(resa)
1432 #reservation_list = unfiltered_reservation_list
1434 logger.debug(" SLABDRIVER.PY \tGetLeases reservation_list %s"\
1435 %(reservation_list))
1436 return reservation_list
1438 def augment_records_with_testbed_info (self, sfa_records):
1439 return self.fill_record_info (sfa_records)
1441 def fill_record_info(self, record_list):
1443 Given a SFA record, fill in the senslab specific and SFA specific
1444 fields in the record.
1447 logger.debug("SLABDRIVER \tfill_record_info records %s " %(record_list))
1448 if not isinstance(record_list, list):
1449 record_list = [record_list]
1452 for record in record_list:
1453 #If the record is a SFA slice record, then add information
1454 #about the user of this slice. This kind of
1455 #information is in the Senslab's DB.
1456 if str(record['type']) == 'slice':
1457 if 'reg_researchers' in record and isinstance(record['reg_researchers'],list) :
1458 record['reg_researchers'] = record['reg_researchers'][0].__dict__
1459 record.update({'PI':[record['reg_researchers']['hrn']],
1460 'researcher': [record['reg_researchers']['hrn']],
1461 'name':record['hrn'],
1464 'person_ids':[record['reg_researchers']['record_id']],
1465 'geni_urn':'', #For client_helper.py compatibility
1466 'keys':'', #For client_helper.py compatibility
1467 'key_ids':''}) #For client_helper.py compatibility
1470 #Get slab slice record.
1471 recslice_list = self.GetSlices(slice_filter = \
1472 str(record['hrn']),\
1473 slice_filter_type = 'slice_hrn')
1475 #recuser = recslice_list[0]['reg_researchers']
1476 ##recuser = dbsession.query(RegRecord).filter_by(record_id = \
1477 ##recslice_list[0]['record_id_user']).first()
1479 #record.update({'PI':[recuser['hrn']],
1480 #'researcher': [recuser['hrn']],
1481 #'name':record['hrn'],
1484 #'person_ids':[recslice_list[0]['reg_researchers']['record_id']],
1485 #'geni_urn':'', #For client_helper.py compatibility
1486 #'keys':'', #For client_helper.py compatibility
1487 #'key_ids':''}) #For client_helper.py compatibility
1488 logger.debug("SLABDRIVER \tfill_record_info TYPE SLICE RECUSER record['hrn'] %s ecord['oar_job_id'] %s " %(record['hrn'],record['oar_job_id']))
1490 for rec in recslice_list:
1491 logger.debug("SLABDRIVER\r\n \t \t fill_record_info oar_job_id %s " %(rec['oar_job_id']))
1492 #record['oar_job_id'].append(rec['oar_job_id'])
1493 del record['_sa_instance_state']
1494 del record['reg_researchers']
1495 record['node_ids'] = [ self.root_auth + hostname for hostname in rec['node_ids']]
1499 logger.debug( "SLABDRIVER.PY \t fill_record_info SLICE \
1500 recslice_list %s \r\n \t RECORD %s \r\n \r\n" %(recslice_list,record))
1501 if str(record['type']) == 'user':
1502 #The record is a SFA user record.
1503 #Get the information about his slice from Senslab's DB
1504 #and add it to the user record.
1505 recslice_list = self.GetSlices(\
1506 slice_filter = record['record_id'],\
1507 slice_filter_type = 'record_id_user')
1509 logger.debug( "SLABDRIVER.PY \t fill_record_info TYPE USER \
1510 recslice_list %s \r\n \t RECORD %s \r\n" %(recslice_list , record))
1511 #Append slice record in records list,
1512 #therefore fetches user and slice info again(one more loop)
1513 #Will update PIs and researcher for the slice
1514 #recuser = dbsession.query(RegRecord).filter_by(record_id = \
1515 #recslice_list[0]['record_id_user']).first()
1516 recuser = recslice_list[0]['reg_researchers']
1517 logger.debug( "SLABDRIVER.PY \t fill_record_info USER \
1518 recuser %s \r\n \r\n" %(recuser))
1520 recslice = recslice_list[0]
1521 recslice.update({'PI':[recuser['hrn']],
1522 'researcher': [recuser['hrn']],
1523 'name':record['hrn'],
1526 'person_ids':[recuser['record_id']]})
1528 for rec in recslice_list:
1529 recslice['oar_job_id'].append(rec['oar_job_id'])
1533 recslice.update({'type':'slice', \
1534 'hrn':recslice_list[0]['hrn']})
1537 #GetPersons takes [] as filters
1538 #user_slab = self.GetPersons([{'hrn':recuser.hrn}])
1539 user_slab = self.GetPersons([record])
1542 record.update(user_slab[0])
1543 #For client_helper.py compatibility
1544 record.update( { 'geni_urn':'',
1547 record_list.append(recslice)
1549 logger.debug("SLABDRIVER.PY \tfill_record_info ADDING SLICE\
1550 INFO TO USER records %s" %(record_list))
1552 logger.debug("SLABDRIVER.PY \tfill_record_info END \
1553 record %s \r\n \r\n " %(record))
1555 except TypeError, error:
1556 logger.log_exc("SLABDRIVER \t fill_record_info EXCEPTION %s"\
1558 #logger.debug("SLABDRIVER.PY \t fill_record_info ENDENDEND ")
1562 #self.fill_record_slab_info(records)
1568 #TODO Update membership? update_membership_list SA 05/07/12
1569 #def update_membership_list(self, oldRecord, record, listName, addFunc, \
1571 ## get a list of the HRNs tht are members of the old and new records
1573 #oldList = oldRecord.get(listName, [])
1576 #newList = record.get(listName, [])
1578 ## if the lists are the same, then we don't have to update anything
1579 #if (oldList == newList):
1582 ## build a list of the new person ids, by looking up each person to get
1586 #records = table.find({'type': 'user', 'hrn': newList})
1587 #for rec in records:
1588 #newIdList.append(rec['pointer'])
1590 ## build a list of the old person ids from the person_ids field
1592 #oldIdList = oldRecord.get("person_ids", [])
1593 #containerId = oldRecord.get_pointer()
1595 ## if oldRecord==None, then we are doing a Register, instead of an
1598 #containerId = record.get_pointer()
1600 ## add people who are in the new list, but not the oldList
1601 #for personId in newIdList:
1602 #if not (personId in oldIdList):
1603 #addFunc(self.plauth, personId, containerId)
1605 ## remove people who are in the old list, but not the new list
1606 #for personId in oldIdList:
1607 #if not (personId in newIdList):
1608 #delFunc(self.plauth, personId, containerId)
1610 #def update_membership(self, oldRecord, record):
1612 #if record.type == "slice":
1613 #self.update_membership_list(oldRecord, record, 'researcher',
1614 #self.users.AddPersonToSlice,
1615 #self.users.DeletePersonFromSlice)
1616 #elif record.type == "authority":
1621 # I don't think you plan on running a component manager at this point
1622 # let me clean up the mess of ComponentAPI that is deprecated anyways
1625 #TODO FUNCTIONS SECTION 04/07/2012 SA
1627 #TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
1629 def UnBindObjectFromPeer(self, auth, object_type, object_id, shortname):
1630 """ This method is a hopefully temporary hack to let the sfa correctly
1631 detach the objects it creates from a remote peer object. This is
1632 needed so that the sfa federation link can work in parallel with
1633 RefreshPeer, as RefreshPeer depends on remote objects being correctly
1636 auth : struct, API authentication structure
1637 AuthMethod : string, Authentication method to use
1638 object_type : string, Object type, among 'site','person','slice',
1640 object_id : int, object_id
1641 shortname : string, peer shortname
1645 logger.warning("SLABDRIVER \tUnBindObjectFromPeer EMPTY-\
1649 #TODO Is BindObjectToPeer still necessary ? Currently does nothing
1651 def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
1652 remote_object_id=None):
1653 """This method is a hopefully temporary hack to let the sfa correctly
1654 attach the objects it creates to a remote peer object. This is needed
1655 so that the sfa federation link can work in parallel with RefreshPeer,
1656 as RefreshPeer depends on remote objects being correctly marked.
1658 shortname : string, peer shortname
1659 remote_object_id : int, remote object_id, set to 0 if unknown
1663 logger.warning("SLABDRIVER \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
1666 #TODO UpdateSlice 04/07/2012 SA
1667 #Funciton should delete and create another job since oin senslab slice=job
1668 def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
1669 """Updates the parameters of an existing slice with the values in
1671 Users may only update slices of which they are members.
1672 PIs may update any of the slices at their sites, or any slices of
1673 which they are members. Admins may update any slice.
1674 Only PIs and admins may update max_nodes. Slices cannot be renewed
1675 (by updating the expires parameter) more than 8 weeks into the future.
1676 Returns 1 if successful, faults otherwise.
1680 logger.warning("SLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ")
1683 #TODO UpdatePerson 04/07/2012 SA
1684 def UpdatePerson(self, slab_hrn, federated_hrn, person_fields=None):
1685 """Updates a person. Only the fields specified in person_fields
1686 are updated, all other fields are left untouched.
1687 Users and techs can only update themselves. PIs can only update
1688 themselves and other non-PIs at their sites.
1689 Returns 1 if successful, faults otherwise.
1693 #new_row = FederatedToSenslab(slab_hrn, federated_hrn)
1694 #slab_dbsession.add(new_row)
1695 #slab_dbsession.commit()
1697 logger.debug("SLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ")
1700 #TODO GetKeys 04/07/2012 SA
1701 def GetKeys(self, auth, key_filter=None, return_fields=None):
1702 """Returns an array of structs containing details about keys.
1703 If key_filter is specified and is an array of key identifiers,
1704 or a struct of key attributes, only keys matching the filter
1705 will be returned. If return_fields is specified, only the
1706 specified details will be returned.
1708 Admin may query all keys. Non-admins may only query their own keys.
1712 logger.warning("SLABDRIVER GetKeys EMPTY - DO NOTHING \r\n ")
1715 #TODO DeleteKey 04/07/2012 SA
1716 def DeleteKey(self, auth, key_id):
1718 Non-admins may only delete their own keys.
1719 Returns 1 if successful, faults otherwise.
1723 logger.warning("SLABDRIVER DeleteKey EMPTY - DO NOTHING \r\n ")
1727 #TODO : Check rights to delete person
1728 def DeletePerson(self, auth, person_record):
1729 """ Disable an existing account in senslab LDAP.
1730 Users and techs can only delete themselves. PIs can only
1731 delete themselves and other non-PIs at their sites.
1732 ins can delete anyone.
1733 Returns 1 if successful, faults otherwise.
1737 #Disable user account in senslab LDAP
1738 ret = self.ldap.LdapMarkUserAsDeleted(person_record)
1739 logger.warning("SLABDRIVER DeletePerson %s " %(person_record))
1742 #TODO Check DeleteSlice, check rights 05/07/2012 SA
1743 def DeleteSlice(self, auth, slice_record):
1744 """ Deletes the specified slice.
1745 Senslab : Kill the job associated with the slice if there is one
1746 using DeleteSliceFromNodes.
1747 Updates the slice record in slab db to remove the slice nodes.
1749 Users may only delete slices of which they are members. PIs may
1750 delete any of the slices at their sites, or any slices of which
1751 they are members. Admins may delete any slice.
1752 Returns 1 if successful, faults otherwise.
1756 self.DeleteSliceFromNodes(slice_record)
1757 logger.warning("SLABDRIVER DeleteSlice %s "%(slice_record))
1760 #TODO AddPerson 04/07/2012 SA
1761 #def AddPerson(self, auth, person_fields=None):
1762 def AddPerson(self, record):#TODO fixing 28/08//2012 SA
1763 """Adds a new account. Any fields specified in records are used,
1764 otherwise defaults are used.
1765 Accounts are disabled by default. To enable an account,
1767 Returns the new person_id (> 0) if successful, faults otherwise.
1771 ret = self.ldap.LdapAddUser(record)
1772 logger.debug("SLABDRIVER AddPerson return code %s \r\n "%(ret))
1775 #TODO AddPersonToSite 04/07/2012 SA
1776 def AddPersonToSite (self, auth, person_id_or_email, \
1777 site_id_or_login_base=None):
1778 """ Adds the specified person to the specified site. If the person is
1779 already a member of the site, no errors are returned. Does not change
1780 the person's primary site.
1781 Returns 1 if successful, faults otherwise.
1785 logger.warning("SLABDRIVER AddPersonToSite EMPTY - DO NOTHING \r\n ")
1788 #TODO AddRoleToPerson : Not sure if needed in senslab 04/07/2012 SA
1789 def AddRoleToPerson(self, auth, role_id_or_name, person_id_or_email):
1790 """Grants the specified role to the person.
1791 PIs can only grant the tech and user roles to users and techs at their
1792 sites. Admins can grant any role to any user.
1793 Returns 1 if successful, faults otherwise.
1797 logger.warning("SLABDRIVER AddRoleToPerson EMPTY - DO NOTHING \r\n ")
1800 #TODO AddPersonKey 04/07/2012 SA
1801 def AddPersonKey(self, auth, person_id_or_email, key_fields=None):
1802 """Adds a new key to the specified account.
1803 Non-admins can only modify their own keys.
1804 Returns the new key_id (> 0) if successful, faults otherwise.
1808 logger.warning("SLABDRIVER AddPersonKey EMPTY - DO NOTHING \r\n ")
1811 def DeleteLeases(self, leases_id_list, slice_hrn ):
1812 for job_id in leases_id_list:
1813 self.DeleteJobs(job_id, slice_hrn)
1815 logger.debug("SLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \
1816 \r\n " %(leases_id_list, slice_hrn))