sfa/iotlab/iotlabaggregate.py

   1 # -*- coding:utf-8 -*-
   2 """ aggregate class management """
   3
   4 from sfa.util.xrn import Xrn, hrn_to_urn
   5 from sfa.util.sfatime import utcparse, datetime_to_string
   6 from sfa.util.sfalogging import logger
   7 from sfa.rspecs.rspec import RSpec
   8 from sfa.rspecs.elements.hardware_type import HardwareType
   9 from sfa.rspecs.elements.lease import Lease
  10 from sfa.rspecs.elements.granularity import Granularity
  11 from sfa.rspecs.version_manager import VersionManager
  12 from sfa.rspecs.elements.services import ServicesElement
  13 from sfa.rspecs.elements.login import Login
  14 from sfa.rspecs.elements.sliver import Sliver
  15 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition
  16 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabNode
  17 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabLocation
  18 from sfa.iotlab.iotlablease import LeaseTable
  19 import time
  20 import datetime
  21
  22
  23 class IotLABAggregate(object):
  24     """
  25     SFA aggregate for Iot-LAB testbed
  26     """
  27
  28     def __init__(self, driver):
  29         self.driver = driver
  30
  31     def leases_to_rspec_leases(self, leases):
  32         """ Get leases attributes list"""
  33         rspec_leases = []
  34         for lease in leases:
  35             for node in lease['resources']:
  36                 rspec_lease = Lease()
  37                 rspec_lease['lease_id'] = lease['id']
  38                 iotlab_xrn = Xrn('.'.join([self.driver.root_auth,
  39                                  Xrn.escape(node)]),
  40                                  type='node')
  41                 rspec_lease['component_id'] = iotlab_xrn.urn
  42                 rspec_lease['start_time'] = str(lease['date'])
  43                 # duration in minutes
  44                 duration = int(lease['duration'])/60
  45                 rspec_lease['duration'] = duration
  46                 rspec_lease['slice_id'] = lease['slice_id']
  47                 rspec_leases.append(rspec_lease)
  48         return rspec_leases
  49
  50     def node_to_rspec_node(self, node):
  51         """ Get node attributes """
  52         rspec_node = IotlabNode()
  53         rspec_node['mobile'] = node['mobile']
  54         rspec_node['archi'] = node['archi']
  55         rspec_node['radio'] = (node['archi'].split(':'))[1]
  56         iotlab_xrn = Xrn('.'.join([self.driver.root_auth,
  57                          Xrn.escape(node['network_address'])]),
  58                          type='node')
  59         # rspec_node['boot_state'] = 'true'
  60         if node['state'] == 'Absent' or \
  61            node['state'] == 'Suspected' or \
  62            node['state'] == 'Dead' or \
  63            node['state'] == 'Busy':
  64             rspec_node['available'] = 'false'
  65         else:
  66             rspec_node['available'] = 'true'
  67         rspec_node['component_id'] = iotlab_xrn.urn
  68         rspec_node['component_name'] = node['network_address']
  69         rspec_node['component_manager_id'] = hrn_to_urn(self.driver.root_auth,
  70                                                         'authority+sa')
  71         rspec_node['authority_id'] = rspec_node['component_manager_id']
  72         rspec_node['exclusive'] = 'true'
  73         rspec_node['hardware_types'] = \
  74             [HardwareType({'name': node['archi']})]
  75         location = IotlabLocation({'country': 'France',
  76                                    'site': node['site']})
  77         rspec_node['location'] = location
  78         position = IotlabPosition()
  79         for field in position:
  80             position[field] = node[field]
  81         granularity = Granularity({'grain': 30})
  82         rspec_node['granularity'] = granularity
  83         rspec_node['tags'] = []
  84         return rspec_node
  85
  86     def sliver_to_rspec_node(self, sliver):
  87         """ Get node and sliver attributes """
  88         rspec_node = self.node_to_rspec_node(sliver)
  89         rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
  90         rspec_node['sliver_id'] = sliver['sliver_id']
  91         rspec_sliver = Sliver({'sliver_id': sliver['sliver_id'],
  92                                'name': sliver['name'],
  93                                'type': sliver['archi'],
  94                                'tags': []})
  95         rspec_node['slivers'] = [rspec_sliver]
  96         # slivers always provide the ssh service
  97         login = Login({'authentication': 'ssh-keys',
  98                        'hostname': sliver['hostname'],
  99                        'port': '22',
 100                        'username': sliver['name'],
 101                        'login': sliver['name']
 102                        })
 103         service = ServicesElement({'login': login})
 104         rspec_node['services'] = [service]
 105         return rspec_node
 106
 107     @classmethod
 108     def rspec_node_to_geni_sliver(cls, rspec_node):
 109         """ Get sliver status """
 110         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
 111                        'geni_expires': rspec_node['expires'],
 112                        'geni_allocation_status': 'geni_allocated',
 113                        'geni_operational_status': 'geni_pending_allocation',
 114                        'geni_error': '',
 115                        }
 116         return geni_sliver
 117
 118     def list_resources(self, version=None, options=None):
 119         """
 120         list_resources method sends a RSpec with all Iot-LAB testbed nodes
 121         and leases (OAR job submission). For leases we get all OAR jobs with
 122         state Waiting or Running. If we have an entry in SFA database
 123         (lease table) with OAR job id this submission was launched by SFA
 124         driver, otherwise it was launched by Iot-LAB Webportal or CLI-tools
 125
 126         :Example:
 127         <rspec>
 128         ...
 129         <node component_manager_id="urn:publicid:IDN+iotlab+authority+sa"
 130               component_id=
 131                   "urn:publicid:IDN+iotlab+node+m3-10.devgrenoble.iot-lab.info"
 132               exclusive="true" component_name="m3-10.devgrenoble.iot-lab.info">
 133             <hardware_type name="iotlab-node"/>
 134             <location country="France"/>
 135             <granularity grain="60"/>
 136             ...
 137         </node>
 138         ...
 139         <lease slice_id="urn:publicid:IDN+onelab:inria+slice+test_iotlab"
 140             start_time="1427792400" duration="30">
 141             <node component_id=
 142                 "urn:publicid:IDN+iotlab+node+m3-10.grenoble.iot-lab.info"/>
 143         </lease>
 144         ...
 145         </rspec>
 146         """
 147         # pylint:disable=R0914,W0212
 148         logger.warning("iotlabaggregate list_resources")
 149         logger.warning("iotlabaggregate list_resources options %s" % options)
 150         if not options:
 151             options = {}
 152
 153         version_manager = VersionManager()
 154         version = version_manager.get_version(version)
 155         rspec_version = version_manager._get_version(version.type,
 156                                                      version.version,
 157                                                      'ad')
 158         rspec = RSpec(version=rspec_version, user_options=options)
 159
 160         nodes = self.driver.shell.get_nodes()
 161         reserved_nodes = self.driver.shell.get_reserved_nodes()
 162         if 'error' not in nodes and 'error' not in reserved_nodes:
 163             # convert nodes to rspec nodes
 164             rspec_nodes = []
 165             for node in nodes:
 166                 rspec_node = self.node_to_rspec_node(nodes[node])
 167                 rspec_nodes.append(rspec_node)
 168             rspec.version.add_nodes(rspec_nodes)
 169
 170             leases = []
 171             db_leases = {}
 172             # find OAR jobs id for all slices in SFA database
 173             for lease in self.driver.api.dbsession().query(LeaseTable).all():
 174                 db_leases[lease.job_id] = lease.slice_hrn
 175
 176             for lease_id in reserved_nodes:
 177                 # onelab slice = job submission from OneLAB
 178                 if lease_id in db_leases:
 179                     reserved_nodes[lease_id]['slice_id'] = \
 180                         hrn_to_urn(db_leases[lease_id],
 181                                    'slice')
 182                 # iotlab slice = job submission from Iot-LAB
 183                 else:
 184                     reserved_nodes[lease_id]['slice_id'] = \
 185                         hrn_to_urn(self.driver.root_auth + '.' +
 186                                    reserved_nodes[lease_id]['owner']+"_slice",
 187                                    'slice')
 188                 leases.append(reserved_nodes[lease_id])
 189
 190             rspec_leases = self.leases_to_rspec_leases(leases)
 191             rspec.version.add_leases(rspec_leases)
 192         return rspec.toxml()
 193
 194     def get_slivers(self, urns, leases, nodes):
 195         """ Get slivers attributes list """
 196         logger.warning("iotlabaggregate get_slivers")
 197         logger.warning("iotlabaggregate get_slivers urns %s" % urns)
 198         slivers = []
 199         for lease in leases:
 200             for node in lease['resources']:
 201                 network_address = node.split(".")
 202                 sliver_node = nodes[node]
 203                 sliver_hrn = '%s.%s-%s' % (self.driver.hrn,
 204                                            lease['id'],
 205                                            network_address[0])
 206                 start_time = datetime.datetime.fromtimestamp(lease['date'])
 207                 duration = datetime.timedelta(seconds=int(lease['duration']))
 208                 sliver_node['expires'] = start_time + duration
 209                 sliver_node['sliver_id'] = Xrn(sliver_hrn,
 210                                                type='sliver').urn
 211                 # frontend SSH hostname
 212                 sliver_node['hostname'] = '.'.join(network_address[1:])
 213                 # user login
 214                 sliver_node['name'] = lease['owner']
 215                 slivers.append(sliver_node)
 216         return slivers
 217
 218     def _delete_db_lease(self, job_id):
 219         """ Delete lease table row in SFA database """
 220         logger.warning("iotlabdriver _delete_db_lease lease job_id : %s"
 221                        % job_id)
 222         self.driver.api.dbsession().query(LeaseTable).filter(
 223             LeaseTable.job_id == job_id).delete()
 224         self.driver.api.dbsession().commit()
 225
 226     def describe(self, urns, version=None, options=None):
 227         """
 228         describe method returns slice slivers (allocated resources) and leases
 229         (OAR job submission). We search in lease table of SFA database all OAR
 230         jobs id for this slice and match OAR jobs with state Waiting or
 231         Running. If OAR job id doesn't exist the experiment is terminated and
 232         we delete the database table entry. Otherwise we add slivers and leases
 233         in the response
 234
 235         :returns:
 236             geni_slivers : a list of allocated slivers with information about
 237                            their allocation and operational state
 238             geni_urn : the URN of the slice in which the sliver has been
 239                        allocated
 240             geni_rspec:  a RSpec describing the allocated slivers and leases
 241         :rtype: dict
 242
 243         :Example:
 244         <rspec>
 245         ...
 246         <node component_manager_id="urn:publicid:IDN+iotlab+authority+sa"
 247               component_id=
 248                   "urn:publicid:IDN+iotlab+node+m3-10.grenoble.iot-lab.info"
 249               client_id="m3-10.grenoble.iot-lab.info"
 250               sliver_id="urn:publicid:IDN+iotlab+sliver+9953-m3-10"
 251               exclusive="true" component_name="m3-10.grenoble.iot-lab.info">
 252             <hardware_type name="iotlab-node"/>
 253             <location country="France"/>
 254             <granularity grain="30"/>
 255             <sliver_type name="iotlab-exclusive"/>
 256         </node>
 257         <lease slice_id="urn:publicid:IDN+onelab:inria+slice+test_iotlab"
 258                start_time="1427792428" duration="29">
 259             <node component_id=
 260                 "urn:publicid:IDN+iotlab+node+m3-10.grenoble.iot-lab.info"/>
 261         </lease>
 262         ...
 263         </rspec>
 264
 265         """
 266         # pylint:disable=R0914,W0212
 267         logger.warning("iotlabaggregate describe")
 268         logger.warning("iotlabaggregate describe urns : %s" % urns)
 269         if not options:
 270             options = {}
 271         version_manager = VersionManager()
 272         version = version_manager.get_version(version)
 273         rspec_version = version_manager._get_version(version.type,
 274                                                      version.version,
 275                                                      'manifest')
 276         rspec = RSpec(version=rspec_version, user_options=options)
 277         xrn = Xrn(urns[0])
 278         geni_slivers = []
 279
 280         nodes = self.driver.shell.get_nodes()
 281         reserved_nodes = self.driver.shell.get_reserved_nodes()
 282         if 'error' not in nodes and 'error' not in reserved_nodes:
 283             # find OAR jobs id for one slice in SFA database
 284             db_leases = [(lease.job_id, lease.slice_hrn)
 285                          for lease in self.driver.api.dbsession()
 286                          .query(LeaseTable)
 287                          .filter(LeaseTable.slice_hrn == xrn.hrn).all()]
 288
 289             leases = []
 290             for job_id, slice_hrn in db_leases:
 291                 # OAR job terminated, we delete entry in database
 292                 if job_id not in reserved_nodes:
 293                     self._delete_db_lease(job_id)
 294                 else:
 295                     # onelab slice = job submission from OneLAB
 296                     lease = reserved_nodes[job_id]
 297                     lease['slice_id'] = hrn_to_urn(slice_hrn, 'slice')
 298                     leases.append(lease)
 299
 300             # get slivers
 301             slivers = self.get_slivers(urns, leases, nodes)
 302             if slivers:
 303                 date = utcparse(slivers[0]['expires'])
 304                 rspec_expires = datetime_to_string(date)
 305             else:
 306                 rspec_expires = datetime_to_string(utcparse(time.time()))
 307             rspec.xml.set('expires', rspec_expires)
 308
 309             rspec_nodes = []
 310
 311             for sliver in slivers:
 312                 rspec_node = self.sliver_to_rspec_node(sliver)
 313                 rspec_nodes.append(rspec_node)
 314                 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node)
 315                 geni_slivers.append(geni_sliver)
 316             logger.warning("iotlabaggregate describe geni_slivers %s" %
 317                            geni_slivers)
 318             rspec.version.add_nodes(rspec_nodes)
 319
 320             rspec_leases = self.leases_to_rspec_leases(leases)
 321             logger.warning("iotlabaggregate describe rspec_leases %s" %
 322                            rspec_leases)
 323             rspec.version.add_leases(rspec_leases)
 324
 325         return {'geni_urn': urns[0],
 326                 'geni_rspec': rspec.toxml(),
 327                 'geni_slivers': geni_slivers}