sfa/senslab/slabdriver.py

   1 import subprocess
   2
   3 from datetime import datetime
   4
   5 from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
   6 from sfa.util.sfalogging import logger
   7
   8 from sfa.storage.alchemy import dbsession
   9 from sfa.storage.model import RegRecord, RegUser, RegSlice
  10 from sqlalchemy.orm import joinedload
  11 from sfa.trust.credential import Credential
  12
  13
  14 from sfa.managers.driver import Driver
  15 from sfa.rspecs.version_manager import VersionManager
  16 from sfa.rspecs.rspec import RSpec
  17
  18 from sfa.util.xrn import hrn_to_urn, get_authority
  19
  20
  21 ## thierry: everything that is API-related (i.e. handling incoming requests)
  22 # is taken care of
  23 # SlabDriver should be really only about talking to the senslab testbed
  24
  25
  26 from sfa.senslab.OARrestapi import  OARrestapi
  27 from sfa.senslab.LDAPapi import LDAPapi
  28
  29 from sfa.senslab.slabpostgres import SlabDB, slab_dbsession, SenslabXP
  30
  31
  32 from sfa.senslab.slabaggregate import SlabAggregate, slab_xrn_to_hostname, \
  33                                                             slab_xrn_object
  34 from sfa.senslab.slabslices import SlabSlices
  35
  36
  37
  38 # thierry : note
  39 # this inheritance scheme is so that the driver object can receive
  40 # GetNodes or GetSites sorts of calls directly
  41 # and thus minimize the differences in the managers with the pl version
  42 class SlabDriver(Driver):
  43     """ Senslab Driver class inherited from Driver generic class.
  44
  45     Contains methods compliant with the SFA standard and the testbed
  46     infrastructure (calls to LDAP and OAR).
  47     """
  48     def __init__(self, config):
  49         Driver.__init__ (self, config)
  50         self.config = config
  51         self.hrn = config.SFA_INTERFACE_HRN
  52         self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
  53         self.oar = OARrestapi()
  54         self.ldap = LDAPapi()
  55         self.time_format = "%Y-%m-%d %H:%M:%S"
  56         self.db = SlabDB(config, debug = True)
  57         self.cache = None
  58
  59
  60     def sliver_status(self, slice_urn, slice_hrn):
  61         """Receive a status request for slice named urn/hrn
  62         urn:publicid:IDN+senslab+nturro_slice hrn senslab.nturro_slice
  63         shall return a structure as described in
  64         http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus
  65         NT : not sure if we should implement this or not, but used by sface.
  66
  67         """
  68
  69         #First get the slice with the slice hrn
  70         slice_list =  self.GetSlices(slice_filter = slice_hrn, \
  71                                     slice_filter_type = 'slice_hrn')
  72
  73         if len(slice_list) is 0:
  74             raise SliverDoesNotExist("%s  slice_hrn" % (slice_hrn))
  75
  76         #Slice has the same slice hrn for each slice in the slice/lease list
  77         #So fetch the info on the user once
  78         one_slice = slice_list[0]
  79         recuser = dbsession.query(RegRecord).filter_by(record_id = \
  80                                             one_slice['record_id_user']).first()
  81
  82         #Make a list of all the nodes hostnames  in use for this slice
  83         slice_nodes_list = []
  84         for single_slice in slice_list:
  85             for node in single_slice['node_ids']:
  86                 slice_nodes_list.append(node['hostname'])
  87
  88         #Get all the corresponding nodes details
  89         nodes_all = self.GetNodes({'hostname':slice_nodes_list},
  90                                 ['node_id', 'hostname','site','boot_state'])
  91         nodeall_byhostname = dict([(one_node['hostname'], one_node) \
  92                                             for one_node in nodes_all])
  93
  94
  95
  96         for single_slice in slice_list:
  97
  98               #For compatibility
  99             top_level_status = 'empty'
 100             result = {}
 101             result.fromkeys(\
 102                 ['geni_urn','pl_login','geni_status','geni_resources'], None)
 103             result['pl_login'] = recuser.hrn
 104             logger.debug("Slabdriver - sliver_status Sliver status \
 105                                         urn %s hrn %s single_slice  %s \r\n " \
 106                                         %(slice_urn, slice_hrn, single_slice))
 107             try:
 108                 nodes_in_slice = single_slice['node_ids']
 109             except KeyError:
 110                 #No job in the slice
 111                 result['geni_status'] = top_level_status
 112                 result['geni_resources'] = []
 113                 return result
 114
 115             top_level_status = 'ready'
 116
 117             #A job is running on Senslab for this slice
 118             # report about the local nodes that are in the slice only
 119
 120             result['geni_urn'] = slice_urn
 121
 122
 123
 124             #timestamp = float(sl['startTime']) + float(sl['walltime'])
 125             #result['pl_expires'] = strftime(self.time_format, \
 126                                                     #gmtime(float(timestamp)))
 127             #result['slab_expires'] = strftime(self.time_format,\
 128                                                     #gmtime(float(timestamp)))
 129
 130             resources = []
 131             for node in single_slice['node_ids']:
 132                 res = {}
 133                 #res['slab_hostname'] = node['hostname']
 134                 #res['slab_boot_state'] = node['boot_state']
 135
 136                 res['pl_hostname'] = node['hostname']
 137                 res['pl_boot_state'] = \
 138                             nodeall_byhostname[node['hostname']]['boot_state']
 139                 #res['pl_last_contact'] = strftime(self.time_format, \
 140                                                     #gmtime(float(timestamp)))
 141                 sliver_id =  Xrn(slice_urn, type='slice', \
 142                         id=nodeall_byhostname[node['hostname']]['node_id'], \
 143                         authority=self.hrn).urn
 144
 145                 res['geni_urn'] = sliver_id
 146                 node_name  = node['hostname']
 147                 if nodeall_byhostname[node_name]['boot_state'] == 'Alive':
 148
 149                     res['geni_status'] = 'ready'
 150                 else:
 151                     res['geni_status'] = 'failed'
 152                     top_level_status = 'failed'
 153
 154                 res['geni_error'] = ''
 155
 156                 resources.append(res)
 157
 158             result['geni_status'] = top_level_status
 159             result['geni_resources'] = resources
 160             logger.debug("SLABDRIVER \tsliver_statusresources %s res %s "\
 161                                                     %(resources,res))
 162             return result
 163
 164     def get_user(self, hrn):
 165         return dbsession.query(RegRecord).filter_by(hrn = hrn).first()
 166
 167
 168     def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \
 169                                                              users, options):
 170         aggregate = SlabAggregate(self)
 171
 172         slices = SlabSlices(self)
 173         peer = slices.get_peer(slice_hrn)
 174         sfa_peer = slices.get_sfa_peer(slice_hrn)
 175         slice_record = None
 176
 177         if not isinstance(creds, list):
 178             creds = [creds]
 179
 180         if users:
 181             slice_record = users[0].get('slice_record', {})
 182             logger.debug("SLABDRIVER.PY \t create_sliver \t\
 183                                         slice_record %s \r\n \r\n users %s" \
 184                                         %(slice_record, users))
 185             slice_record['user'] = {'keys':users[0]['keys'], \
 186                                     'email':users[0]['email'], \
 187                                     'hrn':slice_record['reg-researchers'][0]}
 188         # parse rspec
 189         rspec = RSpec(rspec_string)
 190         logger.debug("SLABDRIVER.PY \t create_sliver \trspec.version \
 191                                         %s slice_record %s users %s" \
 192                                         %(rspec.version,slice_record, users))
 193
 194
 195         # ensure site record exists?
 196         # ensure slice record exists
 197         #Removed options to verify_slice SA 14/08/12
 198         sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \
 199                                                     sfa_peer)
 200
 201         # ensure person records exists
 202         #verify_persons returns added persons but since the return value
 203         #is not used
 204         slices.verify_persons(slice_hrn, sfa_slice, users, peer, \
 205                                                     sfa_peer, options=options)
 206         #requested_attributes returned by rspec.version.get_slice_attributes()
 207         #unused, removed SA 13/08/12
 208         rspec.version.get_slice_attributes()
 209
 210         logger.debug("SLABDRIVER.PY create_sliver slice %s " %(sfa_slice))
 211
 212         # add/remove slice from nodes
 213
 214         requested_slivers = [node.get('component_id') \
 215                             for node in rspec.version.get_nodes_with_slivers()\
 216                             if node.get('authority_id') is self.root_auth]
 217         l = [ node for node in rspec.version.get_nodes_with_slivers() ]
 218         logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
 219                                     requested_slivers %s  listnodes %s" \
 220                                     %(requested_slivers,l))
 221         #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
 222         #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
 223
 224         # add/remove leases
 225         requested_lease_list = []
 226
 227         logger.debug("SLABDRIVER.PY \tcreate_sliver AVANTLEASE " )
 228         rspec_requested_leases = rspec.version.get_leases()
 229         for lease in rspec.version.get_leases():
 230             single_requested_lease = {}
 231             logger.debug("SLABDRIVER.PY \tcreate_sliver lease %s " %(lease))
 232
 233             if not lease.get('lease_id'):
 234                 if get_authority(lease['component_id']) == self.root_auth:
 235                     single_requested_lease['hostname'] = \
 236                                         slab_xrn_to_hostname(\
 237                                         lease.get('component_id').strip())
 238                     single_requested_lease['start_time'] = \
 239                                                         lease.get('start_time')
 240                     single_requested_lease['duration'] = lease.get('duration')
 241
 242                     requested_lease_list.append(single_requested_lease)
 243
 244         logger.debug("SLABDRIVER.PY \tcreate_sliver APRESLEASE" )
 245         #dCreate dict of leases by start_time, regrouping nodes reserved
 246         #at the same
 247         #time, for the same amount of time = one job on OAR
 248         requested_job_dict = {}
 249         for lease in requested_lease_list:
 250
 251             #In case it is an asap experiment start_time is empty
 252             if lease['start_time'] == '':
 253                 lease['start_time'] = '0'
 254
 255             if lease['start_time'] not in requested_job_dict:
 256                 if isinstance(lease['hostname'], str):
 257                     lease['hostname'] =  [lease['hostname']]
 258
 259                 requested_job_dict[lease['start_time']] = lease
 260
 261             else :
 262                 job_lease = requested_job_dict[lease['start_time']]
 263                 if lease['duration'] == job_lease['duration'] :
 264                     job_lease['hostname'].append(lease['hostname'])
 265
 266
 267
 268
 269         logger.debug("SLABDRIVER.PY \tcreate_sliver  requested_job_dict %s "\
 270                                                      %(requested_job_dict))
 271         #verify_slice_leases returns the leases , but the return value is unused
 272         #here. Removed SA 13/08/12
 273         slices.verify_slice_leases(sfa_slice, \
 274                                     requested_job_dict, peer)
 275
 276         return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version)
 277
 278
 279     def delete_sliver (self, slice_urn, slice_hrn, creds, options):
 280
 281         sfa_slice_list  = self.GetSlices(slice_filter = slice_hrn, \
 282                                             slice_filter_type = 'slice_hrn')
 283
 284         if not sfa_slice_list:
 285             return 1
 286
 287         #Delete all in the slice
 288         for sfa_slice in sfa_slice_list:
 289
 290
 291             logger.debug("SLABDRIVER.PY delete_sliver slice %s" %(sfa_slice))
 292             slices = SlabSlices(self)
 293             # determine if this is a peer slice
 294
 295             peer = slices.get_peer(slice_hrn)
 296             #TODO delete_sliver SA : UnBindObjectFromPeer should be
 297             #used when there is another
 298             #senslab testbed, which is not the case 14/08/12 .
 299
 300             logger.debug("SLABDRIVER.PY delete_sliver peer %s" %(peer))
 301             try:
 302                 if peer:
 303                     self.UnBindObjectFromPeer('slice', \
 304                                             sfa_slice['record_id_slice'], \
 305                                             peer, None)
 306                 self.DeleteSliceFromNodes(sfa_slice)
 307             finally:
 308                 if peer:
 309                     self.BindObjectToPeer('slice', \
 310                                             sfa_slice['record_id_slice'], \
 311                                             peer, sfa_slice['peer_slice_id'])
 312             return 1
 313
 314
 315     def AddSlice(self, slice_record, user_record):
 316         """Add slice to the sfa tables and senslab table only if the user
 317         already exists in senslab database(user already registered in LDAP).
 318         There is no way to separate adding the slice to the tesbed
 319         and then importing it from the testbed to SFA because of
 320         senslab's architecture. Therefore, sfa tables are updated here.
 321         """
 322
 323         sfa_record = RegSlice(hrn=slice_record['slice_hrn'],
 324                                 gid=slice_record['gid'],
 325                                 pointer=slice_record['slice_id'],
 326                                 authority=slice_record['authority'])
 327
 328         logger.debug("SLABDRIVER.PY AddSlice  sfa_record %s user_record %s" \
 329                                                     %(sfa_record, user_record))
 330         sfa_record.just_created()
 331         dbsession.add(sfa_record)
 332         dbsession.commit()
 333         #Update the reg-researcher dependance table
 334         sfa_record.reg_researchers =  [user_record]
 335         dbsession.commit()
 336
 337         #Update the senslab table with the new slice
 338         #slab_slice = SenslabXP( slice_hrn = slice_record['slice_hrn'], \
 339                         #record_id_slice = sfa_record.record_id , \
 340                         #record_id_user = slice_record['record_id_user'], \
 341                         #peer_authority = slice_record['peer_authority'])
 342
 343         #logger.debug("SLABDRIVER.PY \tAddSlice slice_record %s \
 344                                     #slab_slice %s sfa_record %s" \
 345                                     #%(slice_record,slab_slice, sfa_record))
 346         #slab_dbsession.add(slab_slice)
 347         #slab_dbsession.commit()
 348         return
 349
 350     # first 2 args are None in case of resource discovery
 351     def list_resources (self, slice_urn, slice_hrn, creds, options):
 352         #cached_requested = options.get('cached', True)
 353
 354         version_manager = VersionManager()
 355         # get the rspec's return format from options
 356         rspec_version = \
 357                 version_manager.get_version(options.get('geni_rspec_version'))
 358         version_string = "rspec_%s" % (rspec_version)
 359
 360         #panos adding the info option to the caching key (can be improved)
 361         if options.get('info'):
 362             version_string = version_string + "_" + \
 363                                         options.get('info', 'default')
 364
 365         # look in cache first
 366         #if cached_requested and self.cache and not slice_hrn:
 367             #rspec = self.cache.get(version_string)
 368             #if rspec:
 369                 #logger.debug("SlabDriver.ListResources: \
 370                                     #returning cached advertisement")
 371                 #return rspec
 372
 373         #panos: passing user-defined options
 374         aggregate = SlabAggregate(self)
 375         #origin_hrn = Credential(string=creds[0]).get_gid_caller().get_hrn()
 376         #options.update({'origin_hrn':origin_hrn})
 377         rspec =  aggregate.get_rspec(slice_xrn=slice_urn, \
 378                                         version=rspec_version, options=options)
 379
 380         # cache the result
 381         #if self.cache and not slice_hrn:
 382             #logger.debug("Slab.ListResources: stores advertisement in cache")
 383             #self.cache.add(version_string, rspec)
 384
 385         return rspec
 386
 387
 388     def list_slices (self, creds, options):
 389         # look in cache first
 390         #if self.cache:
 391             #slices = self.cache.get('slices')
 392             #if slices:
 393                 #logger.debug("PlDriver.list_slices returns from cache")
 394                 #return slices
 395
 396         # get data from db
 397
 398         slices = self.GetSlices()
 399         logger.debug("SLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices))
 400         slice_hrns = [slab_slice['hrn'] for slab_slice in slices]
 401
 402         slice_urns = [hrn_to_urn(slice_hrn, 'slice') \
 403                                                 for slice_hrn in slice_hrns]
 404
 405         # cache the result
 406         #if self.cache:
 407             #logger.debug ("SlabDriver.list_slices stores value in cache")
 408             #self.cache.add('slices', slice_urns)
 409
 410         return slice_urns
 411
 412
 413     def register (self, sfa_record, hrn, pub_key):
 414         """
 415         Adding new user, slice, node or site should not be handled
 416         by SFA.
 417
 418         Adding nodes = OAR
 419         Adding users = LDAP Senslab
 420         Adding slice = Import from LDAP users
 421         Adding site = OAR
 422         """
 423         return -1
 424
 425
 426     def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
 427         """No site or node record update allowed in Senslab."""
 428
 429         pointer = old_sfa_record['pointer']
 430         old_sfa_record_type = old_sfa_record['type']
 431
 432         # new_key implemented for users only
 433         if new_key and old_sfa_record_type not in [ 'user' ]:
 434             raise UnknownSfaType(old_sfa_record_type)
 435
 436         #if (type == "authority"):
 437             #self.shell.UpdateSite(pointer, new_sfa_record)
 438
 439         if old_sfa_record_type == "slice":
 440             slab_record = self.sfa_fields_to_slab_fields(old_sfa_record_type, \
 441                                                 hrn, new_sfa_record)
 442             if 'name' in slab_record:
 443                 slab_record.pop('name')
 444                 #Prototype should be UpdateSlice(self,
 445                 #auth, slice_id_or_name, slice_fields)
 446                 #Senslab cannot update slice since slice = job
 447                 #so we must delete and create another job
 448                 self.UpdateSlice(pointer, slab_record)
 449
 450         elif old_sfa_record_type == "user":
 451             update_fields = {}
 452             all_fields = new_sfa_record
 453             for key in all_fields.keys():
 454                 if key in ['first_name', 'last_name', 'title', 'email',
 455                            'password', 'phone', 'url', 'bio', 'accepted_aup',
 456                            'enabled']:
 457                     update_fields[key] = all_fields[key]
 458             self.UpdatePerson(pointer, update_fields)
 459
 460             if new_key:
 461                 # must check this key against the previous one if it exists
 462                 persons = self.GetPersons([pointer], ['key_ids'])
 463                 person = persons[0]
 464                 keys = person['key_ids']
 465                 keys = self.GetKeys(person['key_ids'])
 466
 467                 # Delete all stale keys
 468                 key_exists = False
 469                 for key in keys:
 470                     if new_key != key['key']:
 471                         self.DeleteKey(key['key_id'])
 472                     else:
 473                         key_exists = True
 474                 if not key_exists:
 475                     self.AddPersonKey(pointer, {'key_type': 'ssh', \
 476                                                     'key': new_key})
 477
 478
 479         return True
 480
 481
 482     def remove (self, sfa_record):
 483         sfa_record_type = sfa_record['type']
 484         hrn = sfa_record['hrn']
 485         if sfa_record_type == 'user':
 486
 487             #get user from senslab ldap
 488             person = self.GetPersons(sfa_record)
 489             #No registering at a given site in Senslab.
 490             #Once registered to the LDAP, all senslab sites are
 491             #accesible.
 492             if person :
 493                 #Mark account as disabled in ldap
 494                 self.DeletePerson(sfa_record)
 495         elif sfa_record_type == 'slice':
 496             if self.GetSlices(slice_filter = hrn, \
 497                                     slice_filter_type = 'slice_hrn'):
 498                 self.DeleteSlice(sfa_record)
 499
 500         #elif type == 'authority':
 501             #if self.GetSites(pointer):
 502                 #self.DeleteSite(pointer)
 503
 504         return True
 505
 506
 507
 508     #TODO clean GetPeers. 05/07/12SA
 509     def GetPeers (self, auth = None, peer_filter=None, return_fields_list=None):
 510
 511         existing_records = {}
 512         existing_hrns_by_types = {}
 513         logger.debug("SLABDRIVER \tGetPeers auth = %s, peer_filter %s, \
 514                     return_field %s " %(auth , peer_filter, return_fields_list))
 515         all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
 516
 517         for record in all_records:
 518             existing_records[(record.hrn, record.type)] = record
 519             if record.type not in existing_hrns_by_types:
 520                 existing_hrns_by_types[record.type] = [record.hrn]
 521             else:
 522                 existing_hrns_by_types[record.type].append(record.hrn)
 523
 524
 525         logger.debug("SLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\
 526                                              %( existing_hrns_by_types))
 527         records_list = []
 528
 529         try:
 530             if peer_filter:
 531                 records_list.append(existing_records[(peer_filter,'authority')])
 532             else :
 533                 for hrn in existing_hrns_by_types['authority']:
 534                     records_list.append(existing_records[(hrn,'authority')])
 535
 536             logger.debug("SLABDRIVER \tGetPeer \trecords_list  %s " \
 537                                             %(records_list))
 538
 539         except KeyError:
 540             pass
 541
 542         return_records = records_list
 543         if not peer_filter and not return_fields_list:
 544             return records_list
 545
 546
 547         logger.debug("SLABDRIVER \tGetPeer return_records %s " \
 548                                                     %(return_records))
 549         return return_records
 550
 551
 552     #TODO  : Handling OR request in make_ldap_filters_from_records
 553     #instead of the for loop
 554     #over the records' list
 555     def GetPersons(self, person_filter=None):
 556         """
 557         person_filter should be a list of dictionnaries when not set to None.
 558         Returns a list of users whose accounts are enabled found in ldap.
 559
 560         """
 561         logger.debug("SLABDRIVER \tGetPersons person_filter %s" \
 562                                                     %(person_filter))
 563         person_list = []
 564         if person_filter and isinstance(person_filter, list):
 565         #If we are looking for a list of users (list of dict records)
 566         #Usually the list contains only one user record
 567             for searched_attributes in person_filter:
 568
 569                 #Get only enabled user accounts in senslab LDAP :
 570                 #add a filter for make_ldap_filters_from_record
 571                 person = self.ldap.LdapFindUser(searched_attributes, \
 572                                 is_user_enabled=True)
 573                 #If a person was found, append it to the list
 574                 if person:
 575                     person_list.append(person)
 576
 577             #If the list is empty, return None
 578             if len(person_list) is 0:
 579                 person_list = None
 580
 581         else:
 582             #Get only enabled user accounts in senslab LDAP :
 583             #add a filter for make_ldap_filters_from_record
 584             person_list  = self.ldap.LdapFindUser(is_user_enabled=True)
 585
 586         return person_list
 587
 588     def GetTimezone(self):
 589         """ Get the OAR servier time and timezone.
 590         Unused SA 16/11/12"""
 591         server_timestamp, server_tz = self.oar.parser.\
 592                                             SendRequest("GET_timezone")
 593         return server_timestamp, server_tz
 594
 595
 596     def DeleteJobs(self, job_id, slice_hrn):
 597         if not job_id or job_id is -1:
 598             return
 599         username  = slice_hrn.split(".")[-1].rstrip("_slice")
 600         reqdict = {}
 601         reqdict['method'] = "delete"
 602         reqdict['strval'] = str(job_id)
 603
 604
 605         answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', \
 606                                                     reqdict,username)
 607         logger.debug("SLABDRIVER \tDeleteJobs jobid  %s \r\n answer %s \
 608                                 username %s" %(job_id,answer, username))
 609         return answer
 610
 611
 612
 613         ##TODO : Unused GetJobsId ? SA 05/07/12
 614     #def GetJobsId(self, job_id, username = None ):
 615         #"""
 616         #Details about a specific job.
 617         #Includes details about submission time, jot type, state, events,
 618         #owner, assigned ressources, walltime etc...
 619
 620         #"""
 621         #req = "GET_jobs_id"
 622         #node_list_k = 'assigned_network_address'
 623         ##Get job info from OAR
 624         #job_info = self.oar.parser.SendRequest(req, job_id, username)
 625
 626         #logger.debug("SLABDRIVER \t GetJobsId  %s " %(job_info))
 627         #try:
 628             #if job_info['state'] == 'Terminated':
 629                 #logger.debug("SLABDRIVER \t GetJobsId job %s TERMINATED"\
 630                                                             #%(job_id))
 631                 #return None
 632             #if job_info['state'] == 'Error':
 633                 #logger.debug("SLABDRIVER \t GetJobsId ERROR message %s "\
 634                                                             #%(job_info))
 635                 #return None
 636
 637         #except KeyError:
 638             #logger.error("SLABDRIVER \tGetJobsId KeyError")
 639             #return None
 640
 641         #parsed_job_info  = self.get_info_on_reserved_nodes(job_info, \
 642                                                             #node_list_k)
 643         ##Replaces the previous entry
 644         ##"assigned_network_address" / "reserved_resources"
 645         ##with "node_ids"
 646         #job_info.update({'node_ids':parsed_job_info[node_list_k]})
 647         #del job_info[node_list_k]
 648         #logger.debug(" \r\nSLABDRIVER \t GetJobsId job_info %s " %(job_info))
 649         #return job_info
 650
 651
 652     def GetJobsResources(self, job_id, username = None):
 653         #job_resources=['reserved_resources', 'assigned_resources',\
 654                             #'job_id', 'job_uri', 'assigned_nodes',\
 655                              #'api_timestamp']
 656         #assigned_res = ['resource_id', 'resource_uri']
 657         #assigned_n = ['node', 'node_uri']
 658
 659         req = "GET_jobs_id_resources"
 660
 661
 662         #Get job resources list from OAR
 663         node_id_list = self.oar.parser.SendRequest(req, job_id, username)
 664         logger.debug("SLABDRIVER \t GetJobsResources  %s " %(node_id_list))
 665
 666         hostname_list = \
 667             self.__get_hostnames_from_oar_node_ids(node_id_list)
 668
 669
 670         #Replaces the previous entry "assigned_network_address" /
 671         #"reserved_resources"
 672         #with "node_ids"
 673         job_info = {'node_ids': hostname_list}
 674
 675         return job_info
 676
 677
 678     def get_info_on_reserved_nodes(self, job_info, node_list_name):
 679         #Get the list of the testbed nodes records and make a
 680         #dictionnary keyed on the hostname out of it
 681         node_list_dict = self.GetNodes()
 682         #node_hostname_list = []
 683         node_hostname_list = [node['hostname'] for node in node_list_dict]
 684         #for node in node_list_dict:
 685             #node_hostname_list.append(node['hostname'])
 686         node_dict = dict(zip(node_hostname_list, node_list_dict))
 687         try :
 688             reserved_node_hostname_list = []
 689             for index in range(len(job_info[node_list_name])):
 690                #job_info[node_list_name][k] =
 691                 reserved_node_hostname_list[index] = \
 692                         node_dict[job_info[node_list_name][index]]['hostname']
 693
 694             logger.debug("SLABDRIVER \t get_info_on_reserved_nodes \
 695                         reserved_node_hostname_list %s" \
 696                         %(reserved_node_hostname_list))
 697         except KeyError:
 698             logger.error("SLABDRIVER \t get_info_on_reserved_nodes KEYERROR " )
 699
 700         return reserved_node_hostname_list
 701
 702     def GetNodesCurrentlyInUse(self):
 703         """Returns a list of all the nodes already involved in an oar job"""
 704         return self.oar.parser.SendRequest("GET_running_jobs")
 705
 706     def __get_hostnames_from_oar_node_ids(self, resource_id_list ):
 707         full_nodes_dict_list = self.GetNodes()
 708         #Put the full node list into a dictionary keyed by oar node id
 709         oar_id_node_dict = {}
 710         for node in full_nodes_dict_list:
 711             oar_id_node_dict[node['oar_id']] = node
 712
 713         #logger.debug("SLABDRIVER \t  __get_hostnames_from_oar_node_ids\
 714                         #oar_id_node_dict %s" %(oar_id_node_dict))
 715
 716         hostname_dict_list = []
 717         for resource_id in resource_id_list:
 718             #Because jobs requested "asap" do not have defined resources
 719             if resource_id is not "Undefined":
 720                 hostname_dict_list.append(\
 721                         oar_id_node_dict[resource_id]['hostname'])
 722
 723             #hostname_list.append(oar_id_node_dict[resource_id]['hostname'])
 724         return hostname_dict_list
 725
 726     def GetReservedNodes(self,username = None):
 727         #Get the nodes in use and the reserved nodes
 728         reservation_dict_list = \
 729                         self.oar.parser.SendRequest("GET_reserved_nodes", \
 730                         username = username)
 731
 732
 733         for resa in reservation_dict_list:
 734             logger.debug ("GetReservedNodes resa %s"%(resa))
 735             #dict list of hostnames and their site
 736             resa['reserved_nodes'] = \
 737                 self.__get_hostnames_from_oar_node_ids(resa['resource_ids'])
 738
 739         #del resa['resource_ids']
 740         return reservation_dict_list
 741
 742     def GetNodes(self, node_filter_dict = None, return_fields_list = None):
 743         """
 744         node_filter_dict : dictionnary of lists
 745
 746         """
 747         node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
 748         node_dict_list = node_dict_by_id.values()
 749         logger.debug (" SLABDRIVER GetNodes  node_filter_dict %s \
 750             return_fields_list %s "%(node_filter_dict, return_fields_list))
 751         #No  filtering needed return the list directly
 752         if not (node_filter_dict or return_fields_list):
 753             return node_dict_list
 754
 755         return_node_list = []
 756         if node_filter_dict:
 757             for filter_key in node_filter_dict:
 758                 try:
 759                     #Filter the node_dict_list by each value contained in the
 760                     #list node_filter_dict[filter_key]
 761                     for value in node_filter_dict[filter_key]:
 762                         for node in node_dict_list:
 763                             if node[filter_key] == value:
 764                                 if return_fields_list :
 765                                     tmp = {}
 766                                     for k in return_fields_list:
 767                                         tmp[k] = node[k]
 768                                     return_node_list.append(tmp)
 769                                 else:
 770                                     return_node_list.append(node)
 771                 except KeyError:
 772                     logger.log_exc("GetNodes KeyError")
 773                     return
 774
 775
 776         return return_node_list
 777
 778
 779     def GetSites(self, site_filter_name_list = None, return_fields_list = None):
 780         site_dict = self.oar.parser.SendRequest("GET_sites")
 781         #site_dict : dict where the key is the sit ename
 782         return_site_list = []
 783         if not ( site_filter_name_list or return_fields_list):
 784             return_site_list = site_dict.values()
 785             return return_site_list
 786
 787         for site_filter_name in site_filter_name_list:
 788             if site_filter_name in site_dict:
 789                 if return_fields_list:
 790                     for field in return_fields_list:
 791                         tmp = {}
 792                         try:
 793                             tmp[field] = site_dict[site_filter_name][field]
 794                         except KeyError:
 795                             logger.error("GetSites KeyError %s "%(field))
 796                             return None
 797                     return_site_list.append(tmp)
 798                 else:
 799                     return_site_list.append( site_dict[site_filter_name])
 800
 801
 802         return return_site_list
 803
 804
 805
 806     def GetSlices(self, slice_filter = None, slice_filter_type = None):
 807         """ Get the slice records from the slab db.
 808         Returns a slice ditc if slice_filter  and slice_filter_type
 809         are specified.
 810         Returns a list of slice dictionnaries if there are no filters
 811         specified.
 812
 813         """
 814         login = None
 815         authorized_filter_types_list = ['slice_hrn', 'record_id_user']
 816         return_slicerec_dictlist = []
 817
 818         #First try to get information on the slice based on the filter provided
 819         if slice_filter_type in authorized_filter_types_list:
 820
 821
 822             def __get_slice_records(slice_filter = None, \
 823                                             slice_filter_type = None):
 824
 825                 login = None
 826                 #Get list of slices based on the slice hrn
 827                 if slice_filter_type == 'slice_hrn':
 828
 829                     if get_authority(slice_filter) == self.root_auth:
 830                         login = slice_filter.split(".")[1].split("_")[0]
 831
 832                     #DO NOT USE RegSlice - reg_researchers to get the hrn
 833                     #of the user otherwise will mess up the RegRecord in
 834                     #Resolve, don't know why - SA 08/08/2012
 835
 836                     #Only one entry for one user  = one slice in slab_xp table
 837                     slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
 838                     #slicerec = slab_dbsession.query(SenslabXP).filter_by(slice_hrn = slice_filter).first()
 839
 840                     if slicerec is None:
 841                         return login, None
 842
 843                 #Get slice based on user id
 844                 if slice_filter_type == 'record_id_user':
 845                     slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
 846                     #slicerec = slab_dbsession.query(SenslabXP).filter_by(record_id_user = slice_filter).first()
 847
 848
 849                 if slicerec:
 850                     fixed_slicerec_dict = slicerec.__dict__
 851                     #At this point if the there is no login it means
 852                     #record_id_user filter has been used for filtering
 853                     if login is None :
 854                         #If theslice record is from senslab
 855                         if fixed_slicerec_dict['peer_authority'] is None:
 856                             login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
 857                     return login, fixed_slicerec_dict
 858
 859
 860
 861             login, fixed_slicerec_dict = \
 862                             __get_slice_records(slice_filter, slice_filter_type)
 863             logger.debug(" SLABDRIVER \tGetSlices login %s \
 864                             slice record %s slice_filter %s slice_filter_type %s "\
 865                             %(login, fixed_slicerec_dict,slice_filter, slice_filter_type))
 866
 867
 868             #Now we have the slice record fixed_slicerec_dict, get the
 869             #jobs associated to this slice
 870             leases_list = self.GetReservedNodes(username = login)
 871
 872             #If no job is running or no job scheduled
 873             #return only the slice record
 874             if leases_list == [] and fixed_slicerec_dict:
 875                 return_slicerec_dictlist.append(fixed_slicerec_dict)
 876
 877             #If several jobs for one slice , put the slice record into
 878             # each lease information dict
 879             for lease in leases_list :
 880                 slicerec_dict = {}
 881
 882                 reserved_list = lease['reserved_nodes']
 883
 884                 slicerec_dict['oar_job_id'] = lease['lease_id']
 885                 slicerec_dict.update({'list_node_ids':{'hostname':reserved_list}})
 886                 slicerec_dict.update({'node_ids':lease['reserved_nodes']})
 887                 #Update lease dict with the slice record
 888                 if fixed_slicerec_dict:
 889                     slicerec_dict.update(fixed_slicerec_dict)
 890                     #slicerec_dict.update({'hrn':\
 891                                     #str(fixed_slicerec_dict['slice_hrn'])})
 892
 893
 894                 return_slicerec_dictlist.append(slicerec_dict)
 895                 logger.debug("SLABDRIVER.PY  \tGetSlices  \
 896                         slicerec_dict %s return_slicerec_dictlist %s \
 897                         lease['reserved_nodes'] \
 898                         %s" %(slicerec_dict, return_slicerec_dictlist, \
 899                         lease['reserved_nodes'] ))
 900
 901             logger.debug("SLABDRIVER.PY  \tGetSlices  RETURN \
 902                         return_slicerec_dictlist  %s" \
 903                         %(return_slicerec_dictlist))
 904
 905             return return_slicerec_dictlist
 906
 907
 908         else:
 909             #Get all slices from the senslab sfa database ,
 910             #put them in dict format
 911             query_slice_list = dbsession.query(RegRecord).filter_by(type='slice').all()
 912             #query_slice_list = slab_dbsession.query(SenslabXP).all()
 913             return_slicerec_dictlist = []
 914             for record in query_slice_list:
 915                 return_slicerec_dictlist.append(record.__dict__)
 916
 917             #Get all the jobs reserved nodes
 918             leases_list = self.GetReservedNodes()
 919
 920
 921             for fixed_slicerec_dict in return_slicerec_dictlist:
 922                 slicerec_dict = {}
 923                 #Check if the slice belongs to a senslab user
 924                 if fixed_slicerec_dict['peer_authority'] is None:
 925                     owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
 926                 else:
 927                     owner = None
 928                 for lease in leases_list:
 929                     if owner == lease['user']:
 930                         slicerec_dict['oar_job_id'] = lease['lease_id']
 931
 932                         #for reserved_node in lease['reserved_nodes']:
 933                         logger.debug("SLABDRIVER.PY  \tGetSlices lease %s "\
 934                                                                  %(lease ))
 935
 936                         reserved_list = lease['reserved_nodes']
 937
 938                         slicerec_dict.update({'node_ids':lease['reserved_nodes']})
 939                         slicerec_dict.update({'list_node_ids':{'hostname':reserved_list}})
 940                         slicerec_dict.update(fixed_slicerec_dict)
 941                         #slicerec_dict.update({'hrn':\
 942                                     #str(fixed_slicerec_dict['slice_hrn'])})
 943                         #return_slicerec_dictlist.append(slicerec_dict)
 944                         fixed_slicerec_dict.update(slicerec_dict)
 945
 946             logger.debug("SLABDRIVER.PY  \tGetSlices RETURN \
 947                         return_slicerec_dictlist %s \slice_filter %s " \
 948                         %(return_slicerec_dictlist, slice_filter))
 949
 950         return return_slicerec_dictlist
 951
 952
 953     def testbed_name (self): return self.hrn
 954
 955     # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
 956     def aggregate_version (self):
 957         version_manager = VersionManager()
 958         ad_rspec_versions = []
 959         request_rspec_versions = []
 960         for rspec_version in version_manager.versions:
 961             if rspec_version.content_type in ['*', 'ad']:
 962                 ad_rspec_versions.append(rspec_version.to_dict())
 963             if rspec_version.content_type in ['*', 'request']:
 964                 request_rspec_versions.append(rspec_version.to_dict())
 965         return {
 966             'testbed':self.testbed_name(),
 967             'geni_request_rspec_versions': request_rspec_versions,
 968             'geni_ad_rspec_versions': ad_rspec_versions,
 969             }
 970
 971
 972
 973     ##
 974     # Convert SFA fields to PLC fields for use when registering up updating
 975     # registry record in the PLC database
 976     #
 977     # @param type type of record (user, slice, ...)
 978     # @param hrn human readable name
 979     # @param sfa_fields dictionary of SFA fields
 980     # @param slab_fields dictionary of PLC fields (output)
 981
 982     def sfa_fields_to_slab_fields(self, sfa_type, hrn, record):
 983
 984
 985         slab_record = {}
 986         #for field in record:
 987         #    slab_record[field] = record[field]
 988
 989         if sfa_type == "slice":
 990             #instantion used in get_slivers ?
 991             if not "instantiation" in slab_record:
 992                 slab_record["instantiation"] = "senslab-instantiated"
 993             #slab_record["hrn"] = hrn_to_pl_slicename(hrn)
 994             #Unused hrn_to_pl_slicename because Slab's hrn already
 995             #in the appropriate form SA 23/07/12
 996             slab_record["hrn"] = hrn
 997             logger.debug("SLABDRIVER.PY sfa_fields_to_slab_fields \
 998                         slab_record %s  " %(slab_record['hrn']))
 999             if "url" in record:
1000                 slab_record["url"] = record["url"]
1001             if "description" in record:
1002                 slab_record["description"] = record["description"]
1003             if "expires" in record:
1004                 slab_record["expires"] = int(record["expires"])
1005
1006         #nodes added by OAR only and then imported to SFA
1007         #elif type == "node":
1008             #if not "hostname" in slab_record:
1009                 #if not "hostname" in record:
1010                     #raise MissingSfaInfo("hostname")
1011                 #slab_record["hostname"] = record["hostname"]
1012             #if not "model" in slab_record:
1013                 #slab_record["model"] = "geni"
1014
1015         #One authority only
1016         #elif type == "authority":
1017             #slab_record["login_base"] = hrn_to_slab_login_base(hrn)
1018
1019             #if not "name" in slab_record:
1020                 #slab_record["name"] = hrn
1021
1022             #if not "abbreviated_name" in slab_record:
1023                 #slab_record["abbreviated_name"] = hrn
1024
1025             #if not "enabled" in slab_record:
1026                 #slab_record["enabled"] = True
1027
1028             #if not "is_public" in slab_record:
1029                 #slab_record["is_public"] = True
1030
1031         return slab_record
1032
1033
1034
1035
1036     def __transforms_timestamp_into_date(self, xp_utc_timestamp = None):
1037         """ Transforms unix timestamp into valid OAR date format """
1038
1039         #Used in case of a scheduled experiment (not immediate)
1040         #To run an XP immediately, don't specify date and time in RSpec
1041         #They will be set to None.
1042         if xp_utc_timestamp:
1043             #transform the xp_utc_timestamp into server readable time
1044             xp_server_readable_date = datetime.fromtimestamp(int(\
1045                                 xp_utc_timestamp)).strftime(self.time_format)
1046
1047             return xp_server_readable_date
1048
1049         else:
1050             return None
1051
1052
1053
1054
1055     def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
1056                         lease_start_time, lease_duration, slice_user=None):
1057         lease_dict = {}
1058         lease_dict['lease_start_time'] = lease_start_time
1059         lease_dict['lease_duration'] = lease_duration
1060         lease_dict['added_nodes'] = added_nodes
1061         lease_dict['slice_name'] = slice_name
1062         lease_dict['slice_user'] = slice_user
1063         lease_dict['grain'] = self.GetLeaseGranularity()
1064         lease_dict['time_format'] = self.time_format
1065
1066
1067         def __create_job_structure_request_for_OAR(lease_dict):
1068             """ Creates the structure needed for a correct POST on OAR.
1069             Makes the timestamp transformation into the appropriate format.
1070             Sends the POST request to create the job with the resources in
1071             added_nodes.
1072
1073             """
1074
1075             nodeid_list = []
1076             reqdict = {}
1077
1078
1079             reqdict['workdir'] = '/tmp'
1080             reqdict['resource'] = "{network_address in ("
1081
1082             for node in lease_dict['added_nodes']:
1083                 logger.debug("\r\n \r\n OARrestapi \t \
1084                 __create_job_structure_request_for_OAR node %s" %(node))
1085
1086                 # Get the ID of the node
1087                 nodeid = node
1088                 reqdict['resource'] += "'" + nodeid + "', "
1089                 nodeid_list.append(nodeid)
1090
1091             custom_length = len(reqdict['resource'])- 2
1092             reqdict['resource'] = reqdict['resource'][0:custom_length] + \
1093                                                 ")}/nodes=" + str(len(nodeid_list))
1094
1095             def __process_walltime(duration):
1096                 """ Calculates the walltime in seconds from the duration in H:M:S
1097                     specified in the RSpec.
1098
1099                 """
1100                 if duration:
1101                     # Fixing the walltime by adding a few delays.
1102                     # First put the walltime in seconds oarAdditionalDelay = 20;
1103                     #  additional delay for /bin/sleep command to
1104                     # take in account  prologue and epilogue scripts execution
1105                     # int walltimeAdditionalDelay = 240;  additional delay
1106                     desired_walltime = duration
1107                     total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
1108                     sleep_walltime = desired_walltime  # 0 sec added Update SA 23/10/12
1109                     walltime = []
1110                     #Put the walltime back in str form
1111                     #First get the hours
1112                     walltime.append(str(total_walltime / 3600))
1113                     total_walltime = total_walltime - 3600 * int(walltime[0])
1114                     #Get the remaining minutes
1115                     walltime.append(str(total_walltime / 60))
1116                     total_walltime = total_walltime - 60 * int(walltime[1])
1117                     #Get the seconds
1118                     walltime.append(str(total_walltime))
1119
1120                 else:
1121                     logger.log_exc(" __process_walltime duration null")
1122
1123                 return walltime, sleep_walltime
1124
1125
1126             walltime, sleep_walltime = \
1127                         __process_walltime(int(lease_dict['lease_duration'])*lease_dict['grain'])
1128
1129
1130             reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
1131                                 ":" + str(walltime[1]) + ":" + str(walltime[2])
1132             reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
1133
1134             #In case of a scheduled experiment (not immediate)
1135             #To run an XP immediately, don't specify date and time in RSpec
1136             #They will be set to None.
1137             if lease_dict['lease_start_time'] is not '0':
1138                 #Readable time accepted by OAR
1139                 start_time = datetime.fromtimestamp(int(lease_dict['lease_start_time'])).\
1140                                                         strftime(lease_dict['time_format'])
1141                 reqdict['reservation'] = start_time
1142             #If there is not start time, Immediate XP. No need to add special
1143             # OAR parameters
1144
1145
1146             reqdict['type'] = "deploy"
1147             reqdict['directory'] = ""
1148             reqdict['name'] = "SFA_" + lease_dict['slice_user']
1149
1150             return reqdict
1151
1152
1153         #Create the request for OAR
1154         reqdict = __create_job_structure_request_for_OAR(lease_dict)
1155          # first step : start the OAR job and update the job
1156         logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\
1157                              \r\n "  %(reqdict))
1158
1159         answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
1160                                                             reqdict, slice_user)
1161         logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid   %s " %(answer))
1162         try:
1163             jobid = answer['id']
1164         except KeyError:
1165             logger.log_exc("SLABDRIVER \tLaunchExperimentOnOAR \
1166                                 Impossible to create job  %s "  %(answer))
1167             return None
1168
1169
1170         def __configure_experiment(jobid, added_nodes):
1171             # second step : configure the experiment
1172             # we need to store the nodes in a yaml (well...) file like this :
1173             # [1,56,23,14,45,75] with name /tmp/sfa<jobid>.json
1174             job_file = open('/tmp/sfa/'+ str(jobid) + '.json', 'w')
1175             job_file.write('[')
1176             job_file.write(str(added_nodes[0].strip('node')))
1177             for node in added_nodes[1:len(added_nodes)] :
1178                 job_file.write(', '+ node.strip('node'))
1179             job_file.write(']')
1180             job_file.close()
1181             return
1182
1183         def __launch_senslab_experiment(jobid):
1184             # third step : call the senslab-experiment wrapper
1185             #command= "java -jar target/sfa-1.0-jar-with-dependencies.jar
1186             # "+str(jobid)+" "+slice_user
1187             javacmdline = "/usr/bin/java"
1188             jarname = \
1189                 "/opt/senslabexperimentwrapper/sfa-1.0-jar-with-dependencies.jar"
1190
1191             output = subprocess.Popen([javacmdline, "-jar", jarname, str(jobid), \
1192                                 slice_user],stdout=subprocess.PIPE).communicate()[0]
1193
1194             logger.debug("SLABDRIVER \t __configure_experiment wrapper returns%s " \
1195                                                                     %(output))
1196             return
1197
1198
1199
1200         if jobid :
1201             logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s \
1202                     added_nodes %s slice_user %s" %(jobid, added_nodes, slice_user))
1203
1204
1205             __configure_experiment(jobid, added_nodes)
1206             __launch_senslab_experiment(jobid)
1207
1208         return jobid
1209
1210
1211     def AddLeases(self, hostname_list, slice_record, \
1212                                         lease_start_time, lease_duration):
1213         logger.debug("SLABDRIVER \r\n \r\n \t AddLeases hostname_list %s  \
1214                 slice_record %s lease_start_time %s lease_duration %s  "\
1215                  %( hostname_list, slice_record , lease_start_time, \
1216                  lease_duration))
1217
1218         tmp = slice_record['reg-researchers'][0].split(".")
1219         username = tmp[(len(tmp)-1)]
1220         job_id = self.LaunchExperimentOnOAR(hostname_list, slice_record['hrn'], \
1221                                     lease_start_time, lease_duration, username)
1222         start_time = datetime.fromtimestamp(int(lease_start_time)).strftime(self.time_format)
1223         end_time = lease_start_time + lease_duration
1224         slab_ex_row = SenslabXP(slice_record['hrn'], job_id, end_time)
1225         logger.debug("SLABDRIVER \r\n \r\n \t slab_ex_row %s" %(slab_ex_row))
1226         slab_dbsession.add(slab_ex_row)
1227         slab_dbsession.commit()
1228
1229         logger.debug("SLABDRIVER \t AddLeases hostname_list start_time %s " %(start_time))
1230
1231         return
1232
1233
1234     #Delete the jobs from job_senslab table
1235     def DeleteSliceFromNodes(self, slice_record):
1236
1237         self.DeleteJobs(slice_record['oar_job_id'], slice_record['hrn'])
1238         return
1239
1240
1241     def GetLeaseGranularity(self):
1242         """ Returns the granularity of Senslab testbed.
1243         OAR returns seconds for experiments duration.
1244         Defined in seconds. """
1245
1246         grain = 60
1247         return grain
1248
1249     def update_jobs_in_slabdb(self, job_oar_list):
1250         #Get all the entries in slab_xp table
1251         jobs_psql_query = slab_dbsession.query(SenslabXP).all()
1252         jobs_psql =  [ row.job_id for row in jobs_psql_query ]
1253         jobs_psql = set(jobs_psql)
1254         kept_jobs = set(job_oar_list).intersection(jobs_psql)
1255
1256         deleted_jobs = set(jobs_psql).difference(kept_jobs)
1257         deleted_jobs = list(deleted_jobs)
1258
1259         slab_dbsession.query(SenslabXP).filter(SenslabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
1260         slab_dbsession.commit()
1261
1262         return
1263
1264
1265
1266     def GetLeases(self, lease_filter_dict=None):
1267
1268
1269         unfiltered_reservation_list = self.GetReservedNodes()
1270
1271         reservation_list = []
1272         #Find the slice associated with this user senslab ldap uid
1273         logger.debug(" SLABDRIVER.PY \tGetLeases unfiltered_reservation_list %s " %(unfiltered_reservation_list))
1274         #Create user dict first to avoid looking several times for
1275         #the same user in LDAP SA 27/07/12
1276         resa_user_dict = {}
1277         job_oar_list = []
1278         for resa in unfiltered_reservation_list:
1279             logger.debug("SLABDRIVER \tGetLeases USER %s"\
1280                                             %(resa['user']))
1281             #Cosntruct list of jobs (runing, waiting..) in oar
1282             job_oar_list.append(resa['lease_id'])
1283             if resa['user'] not in resa_user_dict:
1284                 logger.debug("SLABDRIVER \tGetLeases userNOTIN ")
1285                 ldap_info = self.ldap.LdapSearch('(uid='+resa['user']+')')
1286                 if ldap_info:
1287                     ldap_info = ldap_info[0][1]
1288                     #Get the backref :relationship table reg-researchers
1289                     user = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(email = \
1290                                                     ldap_info['mail'][0])
1291                     if user:
1292                         user = user.first()
1293                         user = user.__dict__
1294                         slice_info =  user['reg_slices_as_researcher'][0].__dict__
1295                     #Separated in case user not in database :
1296                     #record_id not defined SA 17/07//12
1297
1298                     #query_slice_info = slab_dbsession.query(SenslabXP).filter_by(record_id_user = user.record_id)
1299                     #if query_slice_info:
1300                         #slice_info = query_slice_info.first()
1301                     #else:
1302                         #slice_info = None
1303
1304                     resa_user_dict[resa['user']] = {}
1305                     resa_user_dict[resa['user']]['ldap_info'] = user
1306                     resa_user_dict[resa['user']]['slice_info'] = slice_info
1307
1308                     resa['slice_hrn'] = resa_user_dict[resa['user']]['slice_info']['hrn']
1309                     resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
1310                     #Put the slice_urn
1311                     #resa['slice_id'] = hrn_to_urn(slice_info.slice_hrn, 'slice')
1312                     resa['component_id_list'] = []
1313                     #Transform the hostnames into urns (component ids)
1314                     for node in resa['reserved_nodes']:
1315                         #resa['component_id_list'].append(hostname_to_urn(self.hrn, \
1316                                 #self.root_auth, node['hostname']))
1317                         slab_xrn = slab_xrn_object(self.root_auth, node)
1318                         resa['component_id_list'].append(slab_xrn.urn)
1319
1320                 if lease_filter_dict:
1321                     if lease_filter_dict['name'] == resa['slice_hrn']:
1322                         reservation_list.append(resa)
1323
1324             if lease_filter_dict is None:
1325                 reservation_list = unfiltered_reservation_list
1326                 #else:
1327                     #del unfiltered_reservation_list[unfiltered_reservation_list.index(resa)]
1328
1329
1330
1331         logger.debug("SLABDRIVER \tGetLeases resa_user_dict %s"\
1332                                             %(resa_user_dict))
1333         #for resa in unfiltered_reservation_list:
1334
1335
1336             ##Put the slice_urn
1337             #if resa['user'] in resa_user_dict:
1338                 #resa['slice_hrn'] = resa_user_dict[resa['user']]['slice_info']['hrn']
1339                 #resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
1340                 ##Put the slice_urn
1341                 ##resa['slice_id'] = hrn_to_urn(slice_info.slice_hrn, 'slice')
1342                 #resa['component_id_list'] = []
1343                 ##Transform the hostnames into urns (component ids)
1344                 #for node in resa['reserved_nodes']:
1345                     ##resa['component_id_list'].append(hostname_to_urn(self.hrn, \
1346                             ##self.root_auth, node['hostname']))
1347                     #slab_xrn = slab_xrn_object(self.root_auth, node)
1348                     #resa['component_id_list'].append(slab_xrn.urn)
1349
1350         ##Filter the reservation list if necessary
1351         ##Returns all the leases associated with a given slice
1352         #if lease_filter_dict:
1353             #logger.debug("SLABDRIVER \tGetLeases lease_filter_dict %s"\
1354                                             #%(lease_filter_dict))
1355             #for resa in unfiltered_reservation_list:
1356                 #if lease_filter_dict['name'] == resa['slice_hrn']:
1357                     #reservation_list.append(resa)
1358         #else:
1359             #reservation_list = unfiltered_reservation_list
1360
1361         logger.debug(" SLABDRIVER.PY \tGetLeases reservation_list %s"\
1362                                                     %(reservation_list))
1363         return reservation_list
1364
1365     def augment_records_with_testbed_info (self, sfa_records):
1366         return self.fill_record_info (sfa_records)
1367
1368     def fill_record_info(self, record_list):
1369         """
1370         Given a SFA record, fill in the senslab specific and SFA specific
1371         fields in the record.
1372         """
1373
1374         logger.debug("SLABDRIVER \tfill_record_info records %s " %(record_list))
1375         if not isinstance(record_list, list):
1376             record_list = [record_list]
1377
1378         try:
1379             for record in record_list:
1380                 #If the record is a SFA slice record, then add information
1381                 #about the user of this slice. This kind of
1382                 #information is in the Senslab's DB.
1383                 if str(record['type']) == 'slice':
1384                     #Get slab slice record.
1385                     recslice_list = self.GetSlices(slice_filter = \
1386                                                 str(record['hrn']),\
1387                                                 slice_filter_type = 'slice_hrn')
1388
1389                     recuser = dbsession.query(RegRecord).filter_by(record_id = \
1390                                             recslice_list[0]['record_id_user']).first()
1391                     logger.debug("SLABDRIVER \tfill_record_info TYPE SLICE RECUSER %s " %(recuser))
1392                     record.update({'PI':[recuser.hrn],
1393                                 'researcher': [recuser.hrn],
1394                                 'name':record['hrn'],
1395                                 'oar_job_id':[],
1396                                 'node_ids': [],
1397                                 'person_ids':[recslice_list[0]['record_id_user']],
1398                                 'geni_urn':'',  #For client_helper.py compatibility
1399                                 'keys':'',  #For client_helper.py compatibility
1400                                 'key_ids':''})  #For client_helper.py compatibility
1401
1402                     try:
1403                         for rec in recslice_list:
1404                             record['oar_job_id'].append(rec['oar_job_id'])
1405                             record['node_ids'] = [ self.root_auth + hostname for hostname in rec['node_ids']]
1406                     except KeyError:
1407                         pass
1408
1409                     logger.debug( "SLABDRIVER.PY \t fill_record_info SLICE \
1410                                                     recslice_list  %s \r\n \t RECORD %s \r\n \r\n" %(recslice_list,record))
1411                 if str(record['type']) == 'user':
1412                     #The record is a SFA user record.
1413                     #Get the information about his slice from Senslab's DB
1414                     #and add it to the user record.
1415                     recslice_list = self.GetSlices(\
1416                             slice_filter = record['record_id'],\
1417                             slice_filter_type = 'record_id_user')
1418
1419                     logger.debug( "SLABDRIVER.PY \t fill_record_info TYPE USER \
1420                                                 recslice_list %s \r\n \t RECORD %s \r\n" %(recslice_list , record))
1421                     #Append slice record in records list,
1422                     #therefore fetches user and slice info again(one more loop)
1423                     #Will update PIs and researcher for the slice
1424                     recuser = dbsession.query(RegRecord).filter_by(record_id = \
1425                                                 recslice_list[0]['record_id_user']).first()
1426                     logger.debug( "SLABDRIVER.PY \t fill_record_info USER  \
1427                                                 recuser %s \r\n \r\n" %(recuser))
1428                     recslice = {}
1429                     recslice = recslice_list[0]
1430                     recslice.update({'PI':[recuser.hrn],
1431                         'researcher': [recuser.hrn],
1432                         'name':record['hrn'],
1433                         'node_ids': [],
1434                         'oar_job_id': [],
1435                         'person_ids':[recslice_list[0]['record_id_user']]})
1436                     try:
1437                         for rec in recslice_list:
1438                             recslice['oar_job_id'].append(rec['oar_job_id'])
1439                     except KeyError:
1440                         pass
1441
1442                     recslice.update({'type':'slice', \
1443                                                 'hrn':recslice_list[0]['slice_hrn']})
1444
1445
1446                     #GetPersons takes [] as filters
1447                     #user_slab = self.GetPersons([{'hrn':recuser.hrn}])
1448                     user_slab = self.GetPersons([record])
1449
1450
1451                     record.update(user_slab[0])
1452                     #For client_helper.py compatibility
1453                     record.update( { 'geni_urn':'',
1454                     'keys':'',
1455                     'key_ids':'' })
1456                     record_list.append(recslice)
1457
1458                     logger.debug("SLABDRIVER.PY \tfill_record_info ADDING SLICE\
1459                                 INFO TO USER records %s" %(record_list))
1460                 logger.debug("SLABDRIVER.PY \tfill_record_info END \
1461                                 #record %s \r\n \r\n " %(record))
1462
1463         except TypeError, error:
1464             logger.log_exc("SLABDRIVER \t fill_record_info  EXCEPTION %s"\
1465                                                                      %(error))
1466         #logger.debug("SLABDRIVER.PY \t fill_record_info ENDENDEND ")
1467
1468         return
1469
1470         #self.fill_record_slab_info(records)
1471
1472
1473
1474
1475
1476     #TODO Update membership?    update_membership_list SA 05/07/12
1477     #def update_membership_list(self, oldRecord, record, listName, addFunc, \
1478                                                                 #delFunc):
1479         ## get a list of the HRNs tht are members of the old and new records
1480         #if oldRecord:
1481             #oldList = oldRecord.get(listName, [])
1482         #else:
1483             #oldList = []
1484         #newList = record.get(listName, [])
1485
1486         ## if the lists are the same, then we don't have to update anything
1487         #if (oldList == newList):
1488             #return
1489
1490         ## build a list of the new person ids, by looking up each person to get
1491         ## their pointer
1492         #newIdList = []
1493         #table = SfaTable()
1494         #records = table.find({'type': 'user', 'hrn': newList})
1495         #for rec in records:
1496             #newIdList.append(rec['pointer'])
1497
1498         ## build a list of the old person ids from the person_ids field
1499         #if oldRecord:
1500             #oldIdList = oldRecord.get("person_ids", [])
1501             #containerId = oldRecord.get_pointer()
1502         #else:
1503             ## if oldRecord==None, then we are doing a Register, instead of an
1504             ## update.
1505             #oldIdList = []
1506             #containerId = record.get_pointer()
1507
1508     ## add people who are in the new list, but not the oldList
1509         #for personId in newIdList:
1510             #if not (personId in oldIdList):
1511                 #addFunc(self.plauth, personId, containerId)
1512
1513         ## remove people who are in the old list, but not the new list
1514         #for personId in oldIdList:
1515             #if not (personId in newIdList):
1516                 #delFunc(self.plauth, personId, containerId)
1517
1518     #def update_membership(self, oldRecord, record):
1519
1520         #if record.type == "slice":
1521             #self.update_membership_list(oldRecord, record, 'researcher',
1522                                         #self.users.AddPersonToSlice,
1523                                         #self.users.DeletePersonFromSlice)
1524         #elif record.type == "authority":
1525             ## xxx TODO
1526             #pass
1527
1528 ### thierry
1529 # I don't think you plan on running a component manager at this point
1530 # let me clean up the mess of ComponentAPI that is deprecated anyways
1531
1532
1533 #TODO FUNCTIONS SECTION 04/07/2012 SA
1534
1535     #TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
1536     #04/07/2012 SA
1537     def UnBindObjectFromPeer(self, auth, object_type, object_id, shortname):
1538         """ This method is a hopefully temporary hack to let the sfa correctly
1539         detach the objects it creates from a remote peer object. This is
1540         needed so that the sfa federation link can work in parallel with
1541         RefreshPeer, as RefreshPeer depends on remote objects being correctly
1542         marked.
1543         Parameters:
1544         auth : struct, API authentication structure
1545             AuthMethod : string, Authentication method to use
1546         object_type : string, Object type, among 'site','person','slice',
1547         'node','key'
1548         object_id : int, object_id
1549         shortname : string, peer shortname
1550         FROM PLC DOC
1551
1552         """
1553         logger.warning("SLABDRIVER \tUnBindObjectFromPeer EMPTY-\
1554                         DO NOTHING \r\n ")
1555         return
1556
1557     #TODO Is BindObjectToPeer still necessary ? Currently does nothing
1558     #04/07/2012 SA
1559     def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
1560                                                     remote_object_id=None):
1561         """This method is a hopefully temporary hack to let the sfa correctly
1562         attach the objects it creates to a remote peer object. This is needed
1563         so that the sfa federation link can work in parallel with RefreshPeer,
1564         as RefreshPeer depends on remote objects being correctly marked.
1565         Parameters:
1566         shortname : string, peer shortname
1567         remote_object_id : int, remote object_id, set to 0 if unknown
1568         FROM PLC API DOC
1569
1570         """
1571         logger.warning("SLABDRIVER \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
1572         return
1573
1574     #TODO UpdateSlice 04/07/2012 SA
1575     #Funciton should delete and create another job since oin senslab slice=job
1576     def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None):
1577         """Updates the parameters of an existing slice with the values in
1578         slice_fields.
1579         Users may only update slices of which they are members.
1580         PIs may update any of the slices at their sites, or any slices of
1581         which they are members. Admins may update any slice.
1582         Only PIs and admins may update max_nodes. Slices cannot be renewed
1583         (by updating the expires parameter) more than 8 weeks into the future.
1584          Returns 1 if successful, faults otherwise.
1585         FROM PLC API DOC
1586
1587         """
1588         logger.warning("SLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ")
1589         return
1590
1591     #TODO UpdatePerson 04/07/2012 SA
1592     def UpdatePerson(self, slab_hrn, federated_hrn, person_fields=None):
1593         """Updates a person. Only the fields specified in person_fields
1594         are updated, all other fields are left untouched.
1595         Users and techs can only update themselves. PIs can only update
1596         themselves and other non-PIs at their sites.
1597         Returns 1 if successful, faults otherwise.
1598         FROM PLC API DOC
1599
1600         """
1601         #new_row = FederatedToSenslab(slab_hrn, federated_hrn)
1602         #slab_dbsession.add(new_row)
1603         #slab_dbsession.commit()
1604
1605         logger.debug("SLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ")
1606         return
1607
1608     #TODO GetKeys 04/07/2012 SA
1609     def GetKeys(self, auth, key_filter=None, return_fields=None):
1610         """Returns an array of structs containing details about keys.
1611         If key_filter is specified and is an array of key identifiers,
1612         or a struct of key attributes, only keys matching the filter
1613         will be returned. If return_fields is specified, only the
1614         specified details will be returned.
1615
1616         Admin may query all keys. Non-admins may only query their own keys.
1617         FROM PLC API DOC
1618
1619         """
1620         logger.warning("SLABDRIVER  GetKeys EMPTY - DO NOTHING \r\n ")
1621         return
1622
1623     #TODO DeleteKey 04/07/2012 SA
1624     def DeleteKey(self, auth, key_id):
1625         """  Deletes a key.
1626          Non-admins may only delete their own keys.
1627          Returns 1 if successful, faults otherwise.
1628          FROM PLC API DOC
1629
1630         """
1631         logger.warning("SLABDRIVER  DeleteKey EMPTY - DO NOTHING \r\n ")
1632         return
1633
1634
1635     #TODO : Check rights to delete person
1636     def DeletePerson(self, auth, person_record):
1637         """ Disable an existing account in senslab LDAP.
1638         Users and techs can only delete themselves. PIs can only
1639         delete themselves and other non-PIs at their sites.
1640         ins can delete anyone.
1641         Returns 1 if successful, faults otherwise.
1642         FROM PLC API DOC
1643
1644         """
1645         #Disable user account in senslab LDAP
1646         ret = self.ldap.LdapMarkUserAsDeleted(person_record)
1647         logger.warning("SLABDRIVER DeletePerson %s " %(person_record))
1648         return ret
1649
1650     #TODO Check DeleteSlice, check rights 05/07/2012 SA
1651     def DeleteSlice(self, auth, slice_record):
1652         """ Deletes the specified slice.
1653          Senslab : Kill the job associated with the slice if there is one
1654          using DeleteSliceFromNodes.
1655          Updates the slice record in slab db to remove the slice nodes.
1656
1657          Users may only delete slices of which they are members. PIs may
1658          delete any of the slices at their sites, or any slices of which
1659          they are members. Admins may delete any slice.
1660          Returns 1 if successful, faults otherwise.
1661          FROM PLC API DOC
1662
1663         """
1664         self.DeleteSliceFromNodes(slice_record)
1665         logger.warning("SLABDRIVER DeleteSlice %s "%(slice_record))
1666         return
1667
1668     #TODO AddPerson 04/07/2012 SA
1669     #def AddPerson(self, auth,  person_fields=None):
1670     def AddPerson(self, record):#TODO fixing 28/08//2012 SA
1671         """Adds a new account. Any fields specified in records are used,
1672         otherwise defaults are used.
1673         Accounts are disabled by default. To enable an account,
1674         use UpdatePerson().
1675         Returns the new person_id (> 0) if successful, faults otherwise.
1676         FROM PLC API DOC
1677
1678         """
1679         ret = self.ldap.LdapAddUser(record)
1680         logger.debug("SLABDRIVER AddPerson return code %s \r\n "%(ret))
1681         return ret['uid']
1682
1683     #TODO AddPersonToSite 04/07/2012 SA
1684     def AddPersonToSite (self, auth, person_id_or_email, \
1685                                                 site_id_or_login_base=None):
1686         """  Adds the specified person to the specified site. If the person is
1687         already a member of the site, no errors are returned. Does not change
1688         the person's primary site.
1689         Returns 1 if successful, faults otherwise.
1690         FROM PLC API DOC
1691
1692         """
1693         logger.warning("SLABDRIVER AddPersonToSite EMPTY - DO NOTHING \r\n ")
1694         return
1695
1696     #TODO AddRoleToPerson : Not sure if needed in senslab 04/07/2012 SA
1697     def AddRoleToPerson(self, auth, role_id_or_name, person_id_or_email):
1698         """Grants the specified role to the person.
1699         PIs can only grant the tech and user roles to users and techs at their
1700         sites. Admins can grant any role to any user.
1701         Returns 1 if successful, faults otherwise.
1702         FROM PLC API DOC
1703
1704         """
1705         logger.warning("SLABDRIVER AddRoleToPerson EMPTY - DO NOTHING \r\n ")
1706         return
1707
1708     #TODO AddPersonKey 04/07/2012 SA
1709     def AddPersonKey(self, auth, person_id_or_email, key_fields=None):
1710         """Adds a new key to the specified account.
1711         Non-admins can only modify their own keys.
1712         Returns the new key_id (> 0) if successful, faults otherwise.
1713         FROM PLC API DOC
1714
1715         """
1716         logger.warning("SLABDRIVER AddPersonKey EMPTY - DO NOTHING \r\n ")
1717         return
1718
1719     def DeleteLeases(self, leases_id_list, slice_hrn ):
1720         for job_id in leases_id_list:
1721             self.DeleteJobs(job_id, slice_hrn)
1722
1723         logger.debug("SLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \
1724                 \r\n " %(leases_id_list, slice_hrn))
1725         return