sfa/iotlab/iotlabaggregate.py

   1 # -*- coding:utf-8 -*-
   2 """ aggregate class management """
   3
   4 from sfa.util.xrn import Xrn, hrn_to_urn
   5 from sfa.util.sfatime import utcparse, datetime_to_string
   6 from sfa.util.sfalogging import logger
   7 from sfa.rspecs.rspec import RSpec
   8 from sfa.rspecs.elements.hardware_type import HardwareType
   9 from sfa.rspecs.elements.lease import Lease
  10 from sfa.rspecs.elements.granularity import Granularity
  11 from sfa.rspecs.version_manager import VersionManager
  12 from sfa.rspecs.elements.services import ServicesElement
  13 from sfa.rspecs.elements.login import Login
  14 from sfa.rspecs.elements.sliver import Sliver
  15 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition
  16 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabNode
  17 from sfa.rspecs.elements.versions.iotlabv1Node import IotlabLocation
  18 from sfa.iotlab.iotlablease import LeaseTable
  19 import time
  20 import datetime
  21
  22
  23 class IotLABAggregate(object):
  24     """
  25     SFA aggregate for Iot-LAB testbed
  26     """
  27
  28     def __init__(self, driver):
  29         self.driver = driver
  30
  31     def leases_to_rspec_leases(self, leases):
  32         """ Get leases attributes list"""
  33         rspec_leases = []
  34         for lease in leases:
  35             for node in lease['resources']:
  36                 rspec_lease = Lease()
  37                 rspec_lease['lease_id'] = lease['id']
  38                 iotlab_xrn = Xrn('.'.join([self.driver.root_auth,
  39                                  Xrn.escape(node)]),
  40                                  type='node')
  41                 rspec_lease['component_id'] = iotlab_xrn.urn
  42                 rspec_lease['start_time'] = str(lease['date'])
  43                 # duration in minutes
  44                 duration = int(lease['duration'])/60
  45                 rspec_lease['duration'] = duration
  46                 rspec_lease['slice_id'] = lease['slice_id']
  47                 rspec_leases.append(rspec_lease)
  48         return rspec_leases
  49
  50     def node_to_rspec_node(self, node):
  51         """ Get node attributes """
  52         rspec_node = IotlabNode()
  53         rspec_node['mobile'] = node['mobile']
  54         rspec_node['archi'] = node['archi']
  55         rspec_node['radio'] = (node['archi'].split(':'))[1]
  56         iotlab_xrn = Xrn('.'.join([self.driver.root_auth,
  57                          Xrn.escape(node['network_address'])]),
  58                          type='node')
  59         # rspec_node['boot_state'] = 'true'
  60         if node['state'] == 'Absent' or \
  61            node['state'] == 'Suspected' or \
  62            node['state'] == 'Busy':
  63             rspec_node['available'] = 'false'
  64         else:
  65             rspec_node['available'] = 'true'
  66         rspec_node['component_id'] = iotlab_xrn.urn
  67         rspec_node['component_name'] = node['network_address']
  68         rspec_node['component_manager_id'] = hrn_to_urn(self.driver.root_auth,
  69                                                         'authority+sa')
  70         rspec_node['authority_id'] = rspec_node['component_manager_id']
  71         rspec_node['exclusive'] = 'true'
  72         rspec_node['hardware_types'] = \
  73             [HardwareType({'name': node['archi']})]
  74         location = IotlabLocation({'country': 'France',
  75                                    'site': node['site']})
  76         rspec_node['location'] = location
  77         position = IotlabPosition()
  78         for field in position:
  79             position[field] = node[field]
  80         granularity = Granularity({'grain': 30})
  81         rspec_node['granularity'] = granularity
  82         rspec_node['tags'] = []
  83         return rspec_node
  84
  85     def sliver_to_rspec_node(self, sliver):
  86         """ Get node and sliver attributes """
  87         rspec_node = self.node_to_rspec_node(sliver)
  88         rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
  89         rspec_node['sliver_id'] = sliver['sliver_id']
  90         rspec_sliver = Sliver({'sliver_id': sliver['sliver_id'],
  91                                'name': sliver['name'],
  92                                'type': sliver['archi'],
  93                                'tags': []})
  94         rspec_node['slivers'] = [rspec_sliver]
  95         # slivers always provide the ssh service
  96         login = Login({'authentication': 'ssh-keys',
  97                        'hostname': sliver['hostname'],
  98                        'port': '22',
  99                        'username': sliver['name'],
 100                        'login': sliver['name']
 101                        })
 102         service = ServicesElement({'login': login})
 103         rspec_node['services'] = [service]
 104         return rspec_node
 105
 106     @classmethod
 107     def rspec_node_to_geni_sliver(cls, rspec_node):
 108         """ Get sliver status """
 109         geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
 110                        'geni_expires': rspec_node['expires'],
 111                        'geni_allocation_status': 'geni_allocated',
 112                        'geni_operational_status': 'geni_pending_allocation',
 113                        'geni_error': '',
 114                        }
 115         return geni_sliver
 116
 117     def list_resources(self, version=None, options=None):
 118         """
 119         list_resources method sends a RSpec with all Iot-LAB testbed nodes
 120         and leases (OAR job submission). For leases we get all OAR jobs with
 121         state Waiting or Running. If we have an entry in SFA database
 122         (lease table) with OAR job id this submission was launched by SFA
 123         driver, otherwise it was launched by Iot-LAB Webportal or CLI-tools
 124
 125         :Example:
 126         <rspec>
 127         ...
 128         <node component_manager_id="urn:publicid:IDN+iotlab+authority+sa"
 129               component_id=
 130                   "urn:publicid:IDN+iotlab+node+m3-10.devgrenoble.iot-lab.info"
 131               exclusive="true" component_name="m3-10.devgrenoble.iot-lab.info">
 132             <hardware_type name="iotlab-node"/>
 133             <location country="France"/>
 134             <granularity grain="60"/>
 135             ...
 136         </node>
 137         ...
 138         <lease slice_id="urn:publicid:IDN+onelab:inria+slice+test_iotlab"
 139             start_time="1427792400" duration="30">
 140             <node component_id=
 141                 "urn:publicid:IDN+iotlab+node+m3-10.grenoble.iot-lab.info"/>
 142         </lease>
 143         ...
 144         </rspec>
 145         """
 146         # pylint:disable=R0914,W0212
 147         logger.warning("iotlabaggregate list_resources")
 148         logger.warning("iotlabaggregate list_resources options %s" % options)
 149         if not options:
 150             options = {}
 151
 152         version_manager = VersionManager()
 153         version = version_manager.get_version(version)
 154         rspec_version = version_manager._get_version(version.type,
 155                                                      version.version,
 156                                                      'ad')
 157         rspec = RSpec(version=rspec_version, user_options=options)
 158
 159         nodes = self.driver.shell.get_nodes()
 160         reserved_nodes = self.driver.shell.get_reserved_nodes()
 161         if 'error' not in nodes and 'error' not in reserved_nodes:
 162             # convert nodes to rspec nodes
 163             rspec_nodes = []
 164             for node in nodes:
 165                 rspec_node = self.node_to_rspec_node(nodes[node])
 166                 rspec_nodes.append(rspec_node)
 167             rspec.version.add_nodes(rspec_nodes)
 168
 169             leases = []
 170             db_leases = {}
 171             # find OAR jobs id for all slices in SFA database
 172             for lease in self.driver.api.dbsession().query(LeaseTable).all():
 173                 db_leases[lease.job_id] = lease.slice_hrn
 174
 175             for lease_id in reserved_nodes:
 176                 # onelab slice = job submission from OneLAB
 177                 if lease_id in db_leases:
 178                     reserved_nodes[lease_id]['slice_id'] = \
 179                         hrn_to_urn(db_leases[lease_id],
 180                                    'slice')
 181                 # iotlab slice = job submission from Iot-LAB
 182                 else:
 183                     reserved_nodes[lease_id]['slice_id'] = \
 184                         hrn_to_urn(self.driver.root_auth + '.' +
 185                                    reserved_nodes[lease_id]['owner']+"_slice",
 186                                    'slice')
 187                 leases.append(reserved_nodes[lease_id])
 188
 189             rspec_leases = self.leases_to_rspec_leases(leases)
 190             logger.warning("iotlabaggregate list_resources rspec_leases  %s" %
 191                            rspec_leases)
 192             rspec.version.add_leases(rspec_leases)
 193         return rspec.toxml()
 194
 195     def get_slivers(self, urns, leases, nodes):
 196         """ Get slivers attributes list """
 197         logger.warning("iotlabaggregate get_slivers")
 198         logger.warning("iotlabaggregate get_slivers urns %s" % urns)
 199         slivers = []
 200         for lease in leases:
 201             for node in lease['resources']:
 202                 network_address = node.split(".")
 203                 sliver_node = nodes[node]
 204                 sliver_hrn = '%s.%s-%s' % (self.driver.hrn,
 205                                            lease['id'],
 206                                            network_address[0])
 207                 start_time = datetime.datetime.fromtimestamp(lease['date'])
 208                 duration = datetime.timedelta(seconds=int(lease['duration']))
 209                 sliver_node['expires'] = start_time + duration
 210                 sliver_node['sliver_id'] = Xrn(sliver_hrn,
 211                                                type='sliver').urn
 212                 # frontend SSH hostname
 213                 sliver_node['hostname'] = '.'.join(network_address[1:])
 214                 # user login
 215                 sliver_node['name'] = lease['owner']
 216                 slivers.append(sliver_node)
 217         return slivers
 218
 219     def _delete_db_lease(self, job_id):
 220         """ Delete lease table row in SFA database """
 221         logger.warning("iotlabdriver _delete_db_lease lease job_id : %s"
 222                        % job_id)
 223         self.driver.api.dbsession().query(LeaseTable).filter(
 224             LeaseTable.job_id == job_id).delete()
 225         self.driver.api.dbsession().commit()
 226
 227     def describe(self, urns, version=None, options=None):
 228         """
 229         describe method returns slice slivers (allocated resources) and leases
 230         (OAR job submission). We search in lease table of SFA database all OAR
 231         jobs id for this slice and match OAR jobs with state Waiting or
 232         Running. If OAR job id doesn't exist the experiment is terminated and
 233         we delete the database table entry. Otherwise we add slivers and leases
 234         in the response
 235
 236         :returns:
 237             geni_slivers : a list of allocated slivers with information about
 238                            their allocation and operational state
 239             geni_urn : the URN of the slice in which the sliver has been
 240                        allocated
 241             geni_rspec:  a RSpec describing the allocated slivers and leases
 242         :rtype: dict
 243
 244         :Example:
 245         <rspec>
 246         ...
 247         <node component_manager_id="urn:publicid:IDN+iotlab+authority+sa"
 248               component_id=
 249                   "urn:publicid:IDN+iotlab+node+m3-10.grenoble.iot-lab.info"
 250               client_id="m3-10.grenoble.iot-lab.info"
 251               sliver_id="urn:publicid:IDN+iotlab+sliver+9953-m3-10"
 252               exclusive="true" component_name="m3-10.grenoble.iot-lab.info">
 253             <hardware_type name="iotlab-node"/>
 254             <location country="France"/>
 255             <granularity grain="30"/>
 256             <sliver_type name="iotlab-exclusive"/>
 257         </node>
 258         <lease slice_id="urn:publicid:IDN+onelab:inria+slice+test_iotlab"
 259                start_time="1427792428" duration="29">
 260             <node component_id=
 261                 "urn:publicid:IDN+iotlab+node+m3-10.grenoble.iot-lab.info"/>
 262         </lease>
 263         ...
 264         </rspec>
 265
 266         """
 267         # pylint:disable=R0914,W0212
 268         logger.warning("iotlabaggregate describe")
 269         logger.warning("iotlabaggregate describe urns : %s" % urns)
 270         if not options:
 271             options = {}
 272         version_manager = VersionManager()
 273         version = version_manager.get_version(version)
 274         rspec_version = version_manager._get_version(version.type,
 275                                                      version.version,
 276                                                      'manifest')
 277         rspec = RSpec(version=rspec_version, user_options=options)
 278         xrn = Xrn(urns[0])
 279         geni_slivers = []
 280
 281         nodes = self.driver.shell.get_nodes()
 282         reserved_nodes = self.driver.shell.get_reserved_nodes()
 283         if 'error' not in nodes and 'error' not in reserved_nodes:
 284             # find OAR jobs id for one slice in SFA database
 285             db_leases = [(lease.job_id, lease.slice_hrn)
 286                          for lease in self.driver.api.dbsession()
 287                          .query(LeaseTable)
 288                          .filter(LeaseTable.slice_hrn == xrn.hrn).all()]
 289
 290             leases = []
 291             for job_id, slice_hrn in db_leases:
 292                 # OAR job terminated, we delete entry in database
 293                 if job_id not in reserved_nodes:
 294                     self._delete_db_lease(job_id)
 295                 else:
 296                     # onelab slice = job submission from OneLAB
 297                     lease = reserved_nodes[job_id]
 298                     lease['slice_id'] = hrn_to_urn(slice_hrn, 'slice')
 299                     leases.append(lease)
 300
 301             # get slivers
 302             slivers = self.get_slivers(urns, leases, nodes)
 303             if slivers:
 304                 date = utcparse(slivers[0]['expires'])
 305                 rspec_expires = datetime_to_string(date)
 306             else:
 307                 rspec_expires = datetime_to_string(utcparse(time.time()))
 308             rspec.xml.set('expires', rspec_expires)
 309
 310             rspec_nodes = []
 311
 312             for sliver in slivers:
 313                 rspec_node = self.sliver_to_rspec_node(sliver)
 314                 rspec_nodes.append(rspec_node)
 315                 geni_sliver = self.rspec_node_to_geni_sliver(rspec_node)
 316                 geni_slivers.append(geni_sliver)
 317             logger.warning("iotlabaggregate describe geni_slivers %s" %
 318                            geni_slivers)
 319             rspec.version.add_nodes(rspec_nodes)
 320
 321             rspec_leases = self.leases_to_rspec_leases(leases)
 322             logger.warning("iotlabaggregate describe rspec_leases %s" %
 323                            rspec_leases)
 324             rspec.version.add_leases(rspec_leases)
 325
 326         return {'geni_urn': urns[0],
 327                 'geni_rspec': rspec.toxml(),
 328                 'geni_slivers': geni_slivers}