sfa/iotlab/iotlabaggregate.py

   1 #import time
   2 from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority
   3
   4 from sfa.rspecs.rspec import RSpec
   5 #from sfa.rspecs.elements.location import Location
   6 from sfa.rspecs.elements.hardware_type import HardwareType
   7 from sfa.rspecs.elements.login import Login
   8 from sfa.rspecs.elements.services import ServicesElement
   9 from sfa.rspecs.elements.sliver import Sliver
  10 from sfa.rspecs.elements.lease import Lease
  11 from sfa.rspecs.elements.granularity import Granularity
  12 from sfa.rspecs.version_manager import VersionManager
  13
  14
  15 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, IotlabNode, \
  16                                                             IotlabLocation
  17 from sfa.util.sfalogging import logger
  18
  19 from sfa.util.xrn import Xrn
  20
  21 def iotlab_xrn_to_hostname(xrn):
  22     return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf())
  23
  24 def iotlab_xrn_object(root_auth, hostname):
  25     """Attributes are urn and hrn.
  26     Get the hostname using iotlab_xrn_to_hostname on the urn.
  27
  28     :return: the iotlab node's xrn
  29     :rtype: Xrn
  30     """
  31     return Xrn('.'.join( [root_auth, Xrn.escape(hostname)]), type='node')
  32
  33 class IotlabAggregate:
  34
  35     sites = {}
  36     nodes = {}
  37     api = None
  38     interfaces = {}
  39     links = {}
  40     node_tags = {}
  41
  42     prepared = False
  43
  44     user_options = {}
  45
  46     def __init__(self, driver):
  47         self.driver = driver
  48
  49     def get_slice_and_slivers(self, slice_xrn, login=None):
  50         """
  51         Get the slices and the associated leases if any from the iotlab
  52         testbed. For each slice, get the nodes in the  associated lease
  53         and create a sliver with the necessary info and insertinto the sliver
  54         dictionary, keyed on the node hostnames.
  55         Returns a dict of slivers based on the sliver's node_id.
  56         Called by get_rspec.
  57
  58
  59         :param slice_xrn: xrn of the slice
  60         :param login: user's login on iotlab ldap
  61
  62         :type slice_xrn: string
  63         :type login: string
  64         :reutnr : a list of slices dict and a dictionary of Sliver object
  65         :rtype: (list, dict)
  66
  67         ..note: There is no slivers in iotlab, only leases.
  68
  69         """
  70         slivers = {}
  71         sfa_slice = None
  72         if slice_xrn is None:
  73             return (sfa_slice, slivers)
  74         slice_urn = hrn_to_urn(slice_xrn, 'slice')
  75         slice_hrn, _ = urn_to_hrn(slice_xrn)
  76         slice_name = slice_hrn
  77
  78         slices = self.driver.iotlab_api.GetSlices(slice_filter= str(slice_name), \
  79                                             slice_filter_type = 'slice_hrn', \
  80                                             login=login)
  81
  82         logger.debug("Slabaggregate api \tget_slice_and_slivers \
  83                         sfa_slice %s \r\n slices %s self.driver.hrn %s" \
  84                         %(sfa_slice, slices, self.driver.hrn))
  85         if slices ==  []:
  86             return (sfa_slice, slivers)
  87
  88
  89         # sort slivers by node id , if there is a job
  90         #and therefore, node allocated to this slice
  91         for sfa_slice in slices:
  92             try:
  93                 node_ids_list =  sfa_slice['node_ids']
  94             except KeyError:
  95                 logger.log_exc("SLABAGGREGATE \t \
  96                                         get_slice_and_slivers No nodes in the slice - KeyError ")
  97                 continue
  98
  99             for node in node_ids_list:
 100                 sliver_xrn = Xrn(slice_urn, type='sliver', id=node)
 101                 sliver_xrn.set_authority(self.driver.hrn)
 102                 sliver = Sliver({'sliver_id':sliver_xrn.urn,
 103                                 'name': sfa_slice['hrn'],
 104                                 'type': 'iotlab-node',
 105                                 'tags': []})
 106
 107                 slivers[node] = sliver
 108
 109
 110         #Add default sliver attribute :
 111         #connection information for iotlab
 112         if get_authority (sfa_slice['hrn']) == self.driver.iotlab_api.root_auth:
 113             tmp = sfa_slice['hrn'].split('.')
 114             ldap_username = tmp[1].split('_')[0]
 115             ssh_access = None
 116             slivers['default_sliver'] =  {'ssh': ssh_access , \
 117                                         'login': ldap_username}
 118
 119         #TODO get_slice_and_slivers Find the login of the external user
 120
 121         logger.debug("SLABAGGREGATE api get_slice_and_slivers  slivers %s "\
 122                                                              %(slivers))
 123         return (slices, slivers)
 124
 125
 126
 127     def get_nodes(self, slices=None, slivers=[], options=None):
 128         # NT: the semantic of this function is not clear to me :
 129         # if slice is not defined, then all the nodes should be returned
 130         # if slice is defined, we should return only the nodes that
 131         # are part of this slice
 132         # but what is the role of the slivers parameter ?
 133         # So i assume that slice['node_ids'] will be the same as slivers for us
 134         #filter_dict = {}
 135         #if slice_xrn:
 136             #if not slices or not slices['node_ids']:
 137                 #return ([],[])
 138         #tags_filter = {}
 139
 140         # get the granularity in second for the reservation system
 141         grain = self.driver.iotlab_api.GetLeaseGranularity()
 142
 143
 144         nodes = self.driver.iotlab_api.GetNodes()
 145         #geni_available = options.get('geni_available')
 146         #if geni_available:
 147             #filter['boot_state'] = 'boot'
 148
 149         #filter.update({'peer_id': None})
 150         #nodes = self.driver.iotlab_api.GetNodes(filter['hostname'])
 151
 152         #site_ids = []
 153         #interface_ids = []
 154         #tag_ids = []
 155         nodes_dict = {}
 156
 157         #for node in nodes:
 158
 159             #nodes_dict[node['node_id']] = node
 160         #logger.debug("SLABAGGREGATE api get_nodes nodes  %s "\
 161                                                              #%(nodes ))
 162         # get sites
 163         #sites_dict  = self.get_sites({'site_id': site_ids})
 164         # get interfaces
 165         #interfaces = self.get_interfaces({'interface_id':interface_ids})
 166         # get tags
 167         #node_tags = self.get_node_tags(tags_filter)
 168
 169         #if slices, this means we got to list all the nodes given to this slice
 170         # Make a list of all the nodes in the slice before getting their
 171         #attributes
 172         rspec_nodes = []
 173         slice_nodes_list = []
 174         logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list  %s "\
 175                                                              %(slices ))
 176         if slices is not None:
 177             for one_slice in slices:
 178                 try:
 179                     slice_nodes_list = one_slice['node_ids']
 180                 except KeyError:
 181                     pass
 182                 #for node in one_slice['node_ids']:
 183                     #slice_nodes_list.append(node)
 184
 185         reserved_nodes = self.driver.iotlab_api.GetNodesCurrentlyInUse()
 186         logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list  %s "\
 187                                                         %(slice_nodes_list))
 188         for node in nodes:
 189             nodes_dict[node['node_id']] = node
 190             if slice_nodes_list == [] or node['hostname'] in slice_nodes_list:
 191
 192                 rspec_node = IotlabNode()
 193                 # xxx how to retrieve site['login_base']
 194                 #site_id=node['site_id']
 195                 #site=sites_dict[site_id]
 196
 197                 rspec_node['mobile'] = node['mobile']
 198                 rspec_node['archi'] = node['archi']
 199                 rspec_node['radio'] = node['radio']
 200
 201                 iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, \
 202                                                     node['hostname'])
 203                 rspec_node['component_id'] = iotlab_xrn.urn
 204                 rspec_node['component_name'] = node['hostname']
 205                 rspec_node['component_manager_id'] = \
 206                                 hrn_to_urn(self.driver.iotlab_api.root_auth, \
 207                                 'authority+sa')
 208
 209                 # Iotlab's nodes are federated : there is only one authority
 210                 # for all Iotlab sites, registered in SFA.
 211                 # Removing the part including the site
 212                 # in authority_id SA 27/07/12
 213                 rspec_node['authority_id'] = rspec_node['component_manager_id']
 214
 215                 # do not include boot state (<available> element)
 216                 #in the manifest rspec
 217
 218
 219                 rspec_node['boot_state'] = node['boot_state']
 220                 if node['hostname'] in reserved_nodes:
 221                     rspec_node['boot_state'] = "Reserved"
 222                 rspec_node['exclusive'] = 'true'
 223                 rspec_node['hardware_types'] = [HardwareType({'name': \
 224                                                 'iotlab-node'})]
 225
 226
 227                 location = IotlabLocation({'country':'France', 'site': \
 228                                             node['site']})
 229                 rspec_node['location'] = location
 230
 231
 232                 position = IotlabPosition()
 233                 for field in position :
 234                     try:
 235                         position[field] = node[field]
 236                     except KeyError, error :
 237                         logger.log_exc("SLABAGGREGATE\t get_nodes \
 238                                                         position %s "%(error))
 239
 240                 rspec_node['position'] = position
 241                 #rspec_node['interfaces'] = []
 242
 243                 # Granularity
 244                 granularity = Granularity({'grain': grain})
 245                 rspec_node['granularity'] = granularity
 246                 rspec_node['tags'] = []
 247                 if node['hostname'] in slivers:
 248                     # add sliver info
 249                     sliver = slivers[node['hostname']]
 250                     rspec_node['sliver_id'] = sliver['sliver_id']
 251                     rspec_node['client_id'] = node['hostname']
 252                     rspec_node['slivers'] = [sliver]
 253
 254                     # slivers always provide the ssh service
 255                     login = Login({'authentication': 'ssh-keys', \
 256                             'hostname': node['hostname'], 'port':'22', \
 257                             'username': sliver['name']})
 258                     service = Services({'login': login})
 259                     rspec_node['services'] = [service]
 260                 rspec_nodes.append(rspec_node)
 261
 262         return (rspec_nodes)
 263     #def get_all_leases(self, slice_record = None):
 264     def get_all_leases(self):
 265         """
 266         Get list of lease dictionaries which all have the mandatory keys
 267         ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration').
 268         All the leases running or scheduled are returned.
 269
 270
 271         ..note::There is no filtering of leases within a given time frame.
 272         All the running or scheduled leases are returned. options
 273         removed SA 15/05/2013
 274
 275
 276         """
 277
 278         #now = int(time.time())
 279         #lease_filter = {'clip': now }
 280
 281         #if slice_record:
 282             #lease_filter.update({'name': slice_record['name']})
 283
 284         #leases = self.driver.iotlab_api.GetLeases(lease_filter)
 285         leases = self.driver.iotlab_api.GetLeases()
 286         grain = self.driver.iotlab_api.GetLeaseGranularity()
 287         site_ids = []
 288         rspec_leases = []
 289         for lease in leases:
 290             #as many leases as there are nodes in the job
 291             for node in lease['reserved_nodes']:
 292                 rspec_lease = Lease()
 293                 rspec_lease['lease_id'] = lease['lease_id']
 294                 #site = node['site_id']
 295                 iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, node)
 296                 rspec_lease['component_id'] = iotlab_xrn.urn
 297                 #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\
 298                                         #site, node['hostname'])
 299                 try:
 300                     rspec_lease['slice_id'] = lease['slice_id']
 301                 except KeyError:
 302                     #No info on the slice used in iotlab_xp table
 303                     pass
 304                 rspec_lease['start_time'] = lease['t_from']
 305                 rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \
 306                                                                     / grain
 307                 rspec_leases.append(rspec_lease)
 308         return rspec_leases
 309
 310
 311
 312 #from plc/aggregate.py
 313     def get_rspec(self, slice_xrn=None, login=None, version = None, \
 314                 options=None):
 315
 316         rspec = None
 317         version_manager = VersionManager()
 318         version = version_manager.get_version(version)
 319         logger.debug("IotlabAggregate \t get_rspec ***version %s \
 320                     version.type %s  version.version %s options %s \r\n" \
 321                     %(version,version.type,version.version,options))
 322
 323         if slice_xrn is None:
 324             rspec_version = version_manager._get_version(version.type, \
 325                                                     version.version, 'ad')
 326
 327         else:
 328             rspec_version = version_manager._get_version(version.type, \
 329                                                 version.version, 'manifest')
 330
 331         slices, slivers = self.get_slice_and_slivers(slice_xrn, login)
 332         #at this point sliver may be empty if no iotlab job
 333         #is running for this user/slice.
 334         rspec = RSpec(version=rspec_version, user_options=options)
 335
 336
 337         #if slice and 'expires' in slice:
 338            #rspec.xml.set('expires',\
 339                 #datetime_to_string(utcparse(slice['expires']))
 340          # add sliver defaults
 341         #nodes, links = self.get_nodes(slice, slivers)
 342         logger.debug("\r\n \r\n IotlabAggregate \tget_rspec *** \
 343                                         slice_xrn %s slices  %s\r\n \r\n"\
 344                                             %(slice_xrn, slices))
 345
 346         if options is not None:
 347             lease_option = options['list_leases']
 348         else:
 349             #If no options are specified, at least print the resources
 350             lease_option = 'all'
 351            #if slice_xrn :
 352                #lease_option = 'all'
 353
 354
 355         if lease_option in ['all', 'resources']:
 356         #if not options.get('list_leases') or options.get('list_leases')
 357         #and options['list_leases'] != 'leases':
 358             nodes = self.get_nodes(slices, slivers)
 359             logger.debug("\r\n \r\n IotlabAggregate \ lease_option %s \
 360                                         get rspec  ******* nodes %s"\
 361                                             %(lease_option, nodes[0]))
 362
 363             sites_set = set([node['location']['site'] for node in nodes] )
 364
 365             #In case creating a job,  slice_xrn is not set to None
 366             rspec.version.add_nodes(nodes)
 367             if slice_xrn :
 368                 #Get user associated with this slice
 369                 #user = dbsession.query(RegRecord).filter_by(record_id = \
 370                                             #slices['record_id_user']).first()
 371
 372                 #ldap_username = (user.hrn).split('.')[1]
 373
 374
 375                 #for one_slice in slices :
 376                 ldap_username = slices[0]['hrn']
 377                 tmp = ldap_username.split('.')
 378                 ldap_username = tmp[1].split('_')[0]
 379
 380                 if version.type == "Slab":
 381                     rspec.version.add_connection_information(ldap_username, \
 382                                                         sites_set)
 383
 384             default_sliver = slivers.get('default_sliver', [])
 385             if default_sliver:
 386                 #default_sliver_attribs = default_sliver.get('tags', [])
 387                 logger.debug("IotlabAggregate \tget_rspec **** \
 388                         default_sliver%s \r\n" %(default_sliver))
 389                 for attrib in default_sliver:
 390                     rspec.version.add_default_sliver_attribute(attrib, \
 391                                                         default_sliver[attrib])
 392         if lease_option in ['all','leases']:
 393             #leases = self.get_all_leases(slices)
 394             leases = self.get_all_leases()
 395             rspec.version.add_leases(leases)
 396
 397         #logger.debug("IotlabAggregate \tget_rspec ******* rspec_toxml %s \r\n"\
 398                                             #%(rspec.toxml()))
 399         return rspec.toxml()