sfa/iotlab/iotlabaggregate.py

   1 #import time
   2 from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority
   3
   4 from sfa.rspecs.rspec import RSpec
   5 #from sfa.rspecs.elements.location import Location
   6 from sfa.rspecs.elements.hardware_type import HardwareType
   7 from sfa.rspecs.elements.login import Login
   8 from sfa.rspecs.elements.services import Services
   9 from sfa.rspecs.elements.sliver import Sliver
  10 from sfa.rspecs.elements.lease import Lease
  11 from sfa.rspecs.elements.granularity import Granularity
  12 from sfa.rspecs.version_manager import VersionManager
  13
  14
  15 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, IotlabNode, \
  16                                                             IotlabLocation
  17 from sfa.util.sfalogging import logger
  18
  19 from sfa.util.xrn import Xrn
  20
  21 def iotlab_xrn_to_hostname(xrn):
  22     return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf())
  23
  24 def iotlab_xrn_object(root_auth, hostname):
  25     """Attributes are urn and hrn.
  26     Get the hostname using iotlab_xrn_to_hostname on the urn.
  27
  28     :returns: the iotlab node's xrn
  29     :rtype: Xrn
  30     """
  31     return Xrn('.'.join( [root_auth, Xrn.escape(hostname)]), type='node')
  32
  33 class IotlabAggregate:
  34
  35     sites = {}
  36     nodes = {}
  37     api = None
  38     interfaces = {}
  39     links = {}
  40     node_tags = {}
  41
  42     prepared = False
  43
  44     user_options = {}
  45
  46     def __init__(self, driver):
  47         self.driver = driver
  48
  49     def get_slice_and_slivers(self, slice_xrn, login=None):
  50         """
  51         Get the slices and the associated leases if any from the iotlab
  52             testbed. For each slice, get the nodes in the  associated lease
  53             and create a sliver with the necessary info and insertinto the
  54             sliver bdictionary, keyed on the node hostnames.
  55             Returns a dict of slivers based on the sliver's node_id.
  56             Called by get_rspec.
  57
  58
  59         :param slice_xrn: xrn of the slice
  60         :param login: user's login on iotlab ldap
  61
  62         :type slice_xrn: string
  63         :type login: string
  64         :returns: a list of slices dict and a dictionary of Sliver object
  65         :rtype: (list, dict)
  66
  67         ..note: There is no slivers in iotlab, only leases.
  68
  69         """
  70         slivers = {}
  71         sfa_slice = None
  72         if slice_xrn is None:
  73             return (sfa_slice, slivers)
  74         slice_urn = hrn_to_urn(slice_xrn, 'slice')
  75         slice_hrn, _ = urn_to_hrn(slice_xrn)
  76         slice_name = slice_hrn
  77
  78         slices = self.driver.iotlab_api.GetSlices(slice_filter= str(slice_name), \
  79                                             slice_filter_type = 'slice_hrn', \
  80                                             login=login)
  81
  82         logger.debug("Slabaggregate api \tget_slice_and_slivers \
  83                         sfa_slice %s \r\n slices %s self.driver.hrn %s" \
  84                         %(sfa_slice, slices, self.driver.hrn))
  85         if slices ==  []:
  86             return (sfa_slice, slivers)
  87
  88
  89         # sort slivers by node id , if there is a job
  90         #and therefore, node allocated to this slice
  91         for sfa_slice in slices:
  92             try:
  93                 node_ids_list =  sfa_slice['node_ids']
  94             except KeyError:
  95                 logger.log_exc("SLABAGGREGATE \t \
  96                             get_slice_and_slivers No nodes in the slice \
  97                             - KeyError ")
  98                 continue
  99
 100             for node in node_ids_list:
 101                 sliver_xrn = Xrn(slice_urn, type='sliver', id=node)
 102                 sliver_xrn.set_authority(self.driver.hrn)
 103                 sliver = Sliver({'sliver_id':sliver_xrn.urn,
 104                                 'name': sfa_slice['hrn'],
 105                                 'type': 'iotlab-node',
 106                                 'tags': []})
 107
 108                 slivers[node] = sliver
 109
 110
 111         #Add default sliver attribute :
 112         #connection information for iotlab
 113         if get_authority (sfa_slice['hrn']) == self.driver.iotlab_api.root_auth:
 114             tmp = sfa_slice['hrn'].split('.')
 115             ldap_username = tmp[1].split('_')[0]
 116             ssh_access = None
 117             slivers['default_sliver'] =  {'ssh': ssh_access , \
 118                                         'login': ldap_username}
 119
 120         #TODO get_slice_and_slivers Find the login of the external user
 121
 122         logger.debug("SLABAGGREGATE api get_slice_and_slivers  slivers %s "\
 123                                                              %(slivers))
 124         return (slices, slivers)
 125
 126
 127
 128     def get_nodes(self, slices=None, slivers=[], options=None):
 129         # NT: the semantic of this function is not clear to me :
 130         # if slice is not defined, then all the nodes should be returned
 131         # if slice is defined, we should return only the nodes that
 132         # are part of this slice
 133         # but what is the role of the slivers parameter ?
 134         # So i assume that slice['node_ids'] will be the same as slivers for us
 135         #filter_dict = {}
 136         #if slice_xrn:
 137             #if not slices or not slices['node_ids']:
 138                 #return ([],[])
 139         #tags_filter = {}
 140
 141         # get the granularity in second for the reservation system
 142         grain = self.driver.iotlab_api.GetLeaseGranularity()
 143
 144
 145         nodes = self.driver.iotlab_api.GetNodes()
 146         #geni_available = options.get('geni_available')
 147         #if geni_available:
 148             #filter['boot_state'] = 'boot'
 149
 150         #filter.update({'peer_id': None})
 151         #nodes = self.driver.iotlab_api.GetNodes(filter['hostname'])
 152
 153         #site_ids = []
 154         #interface_ids = []
 155         #tag_ids = []
 156         nodes_dict = {}
 157
 158         #for node in nodes:
 159
 160             #nodes_dict[node['node_id']] = node
 161         #logger.debug("SLABAGGREGATE api get_nodes nodes  %s "\
 162                                                              #%(nodes ))
 163         # get sites
 164         #sites_dict  = self.get_sites({'site_id': site_ids})
 165         # get interfaces
 166         #interfaces = self.get_interfaces({'interface_id':interface_ids})
 167         # get tags
 168         #node_tags = self.get_node_tags(tags_filter)
 169
 170         #if slices, this means we got to list all the nodes given to this slice
 171         # Make a list of all the nodes in the slice before getting their
 172         #attributes
 173         rspec_nodes = []
 174         slice_nodes_list = []
 175         logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list  %s "\
 176                                                              %(slices ))
 177         if slices is not None:
 178             for one_slice in slices:
 179                 try:
 180                     slice_nodes_list = one_slice['node_ids']
 181                 except KeyError:
 182                     pass
 183                 #for node in one_slice['node_ids']:
 184                     #slice_nodes_list.append(node)
 185
 186         reserved_nodes = self.driver.iotlab_api.GetNodesCurrentlyInUse()
 187         logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list  %s "\
 188                                                         %(slice_nodes_list))
 189         for node in nodes:
 190             nodes_dict[node['node_id']] = node
 191             if slice_nodes_list == [] or node['hostname'] in slice_nodes_list:
 192
 193                 rspec_node = IotlabNode()
 194                 # xxx how to retrieve site['login_base']
 195                 #site_id=node['site_id']
 196                 #site=sites_dict[site_id]
 197
 198                 rspec_node['mobile'] = node['mobile']
 199                 rspec_node['archi'] = node['archi']
 200                 rspec_node['radio'] = node['radio']
 201
 202                 iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, \
 203                                                     node['hostname'])
 204                 rspec_node['component_id'] = iotlab_xrn.urn
 205                 rspec_node['component_name'] = node['hostname']
 206                 rspec_node['component_manager_id'] = \
 207                                 hrn_to_urn(self.driver.iotlab_api.root_auth, \
 208                                 'authority+sa')
 209
 210                 # Iotlab's nodes are federated : there is only one authority
 211                 # for all Iotlab sites, registered in SFA.
 212                 # Removing the part including the site
 213                 # in authority_id SA 27/07/12
 214                 rspec_node['authority_id'] = rspec_node['component_manager_id']
 215
 216                 # do not include boot state (<available> element)
 217                 #in the manifest rspec
 218
 219
 220                 rspec_node['boot_state'] = node['boot_state']
 221                 if node['hostname'] in reserved_nodes:
 222                     rspec_node['boot_state'] = "Reserved"
 223                 rspec_node['exclusive'] = 'true'
 224                 rspec_node['hardware_types'] = [HardwareType({'name': \
 225                                                 'iotlab-node'})]
 226
 227
 228                 location = IotlabLocation({'country':'France', 'site': \
 229                                             node['site']})
 230                 rspec_node['location'] = location
 231
 232
 233                 position = IotlabPosition()
 234                 for field in position :
 235                     try:
 236                         position[field] = node[field]
 237                     except KeyError, error :
 238                         logger.log_exc("SLABAGGREGATE\t get_nodes \
 239                                                         position %s "%(error))
 240
 241                 rspec_node['position'] = position
 242                 #rspec_node['interfaces'] = []
 243
 244                 # Granularity
 245                 granularity = Granularity({'grain': grain})
 246                 rspec_node['granularity'] = granularity
 247                 rspec_node['tags'] = []
 248                 if node['hostname'] in slivers:
 249                     # add sliver info
 250                     sliver = slivers[node['hostname']]
 251                     rspec_node['sliver_id'] = sliver['sliver_id']
 252                     rspec_node['client_id'] = node['hostname']
 253                     rspec_node['slivers'] = [sliver]
 254
 255                     # slivers always provide the ssh service
 256                     login = Login({'authentication': 'ssh-keys', \
 257                             'hostname': node['hostname'], 'port':'22', \
 258                             'username': sliver['name']})
 259                     service = Services({'login': login})
 260                     rspec_node['services'] = [service]
 261                 rspec_nodes.append(rspec_node)
 262
 263         return (rspec_nodes)
 264     #def get_all_leases(self, slice_record = None):
 265     def get_all_leases(self):
 266         """
 267         Get list of lease dictionaries which all have the mandatory keys
 268         ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration').
 269         All the leases running or scheduled are returned.
 270
 271
 272         ..note::There is no filtering of leases within a given time frame.
 273         All the running or scheduled leases are returned. options
 274         removed SA 15/05/2013
 275
 276
 277         """
 278
 279         #now = int(time.time())
 280         #lease_filter = {'clip': now }
 281
 282         #if slice_record:
 283             #lease_filter.update({'name': slice_record['name']})
 284
 285         #leases = self.driver.iotlab_api.GetLeases(lease_filter)
 286         leases = self.driver.iotlab_api.GetLeases()
 287         grain = self.driver.iotlab_api.GetLeaseGranularity()
 288         site_ids = []
 289         rspec_leases = []
 290         for lease in leases:
 291             #as many leases as there are nodes in the job
 292             for node in lease['reserved_nodes']:
 293                 rspec_lease = Lease()
 294                 rspec_lease['lease_id'] = lease['lease_id']
 295                 #site = node['site_id']
 296                 iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, node)
 297                 rspec_lease['component_id'] = iotlab_xrn.urn
 298                 #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\
 299                                         #site, node['hostname'])
 300                 try:
 301                     rspec_lease['slice_id'] = lease['slice_id']
 302                 except KeyError:
 303                     #No info on the slice used in iotlab_xp table
 304                     pass
 305                 rspec_lease['start_time'] = lease['t_from']
 306                 rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \
 307                                                                     / grain
 308                 rspec_leases.append(rspec_lease)
 309         return rspec_leases
 310
 311
 312
 313 #from plc/aggregate.py
 314     def get_rspec(self, slice_xrn=None, login=None, version = None, \
 315                 options=None):
 316
 317         rspec = None
 318         version_manager = VersionManager()
 319         version = version_manager.get_version(version)
 320         logger.debug("IotlabAggregate \t get_rspec ***version %s \
 321                     version.type %s  version.version %s options %s \r\n" \
 322                     %(version,version.type,version.version,options))
 323
 324         if slice_xrn is None:
 325             rspec_version = version_manager._get_version(version.type, \
 326                                                     version.version, 'ad')
 327
 328         else:
 329             rspec_version = version_manager._get_version(version.type, \
 330                                                 version.version, 'manifest')
 331
 332         slices, slivers = self.get_slice_and_slivers(slice_xrn, login)
 333         #at this point sliver may be empty if no iotlab job
 334         #is running for this user/slice.
 335         rspec = RSpec(version=rspec_version, user_options=options)
 336
 337
 338         #if slice and 'expires' in slice:
 339            #rspec.xml.set('expires',\
 340                 #datetime_to_string(utcparse(slice['expires']))
 341          # add sliver defaults
 342         #nodes, links = self.get_nodes(slice, slivers)
 343         logger.debug("\r\n \r\n IotlabAggregate \tget_rspec *** \
 344                                         slice_xrn %s slices  %s\r\n \r\n"\
 345                                             %(slice_xrn, slices))
 346
 347         if options is not None:
 348             lease_option = options['list_leases']
 349         else:
 350             #If no options are specified, at least print the resources
 351             lease_option = 'all'
 352            #if slice_xrn :
 353                #lease_option = 'all'
 354
 355
 356         if lease_option in ['all', 'resources']:
 357         #if not options.get('list_leases') or options.get('list_leases')
 358         #and options['list_leases'] != 'leases':
 359             nodes = self.get_nodes(slices, slivers)
 360             logger.debug("\r\n \r\n IotlabAggregate \ lease_option %s \
 361                                         get rspec  ******* nodes %s"\
 362                                             %(lease_option, nodes[0]))
 363
 364             sites_set = set([node['location']['site'] for node in nodes] )
 365
 366             #In case creating a job,  slice_xrn is not set to None
 367             rspec.version.add_nodes(nodes)
 368             if slice_xrn :
 369                 #Get user associated with this slice
 370                 #user = dbsession.query(RegRecord).filter_by(record_id = \
 371                                             #slices['record_id_user']).first()
 372
 373                 #ldap_username = (user.hrn).split('.')[1]
 374
 375
 376                 #for one_slice in slices :
 377                 ldap_username = slices[0]['hrn']
 378                 tmp = ldap_username.split('.')
 379                 ldap_username = tmp[1].split('_')[0]
 380
 381                 if version.type == "Iotlab":
 382                     rspec.version.add_connection_information(ldap_username, \
 383                                                         sites_set)
 384
 385             default_sliver = slivers.get('default_sliver', [])
 386             if default_sliver:
 387                 #default_sliver_attribs = default_sliver.get('tags', [])
 388                 logger.debug("IotlabAggregate \tget_rspec **** \
 389                         default_sliver%s \r\n" %(default_sliver))
 390                 for attrib in default_sliver:
 391                     rspec.version.add_default_sliver_attribute(attrib, \
 392                                                         default_sliver[attrib])
 393         if lease_option in ['all','leases']:
 394             #leases = self.get_all_leases(slices)
 395             leases = self.get_all_leases()
 396             rspec.version.add_leases(leases)
 397
 398         #logger.debug("IotlabAggregate \tget_rspec ******* rspec_toxml %s \r\n"\
 399                                             #%(rspec.toxml()))
 400         return rspec.toxml()