Modifications for wilab for the demo

diff --git a/src/nepi/resources/omf/wilabt_node.py b/src/nepi/resources/omf/wilabt_node.py
index 80f076c..d18769a 100644 (file)
--- a/src/nepi/resources/omf/wilabt_node.py
+++ b/src/nepi/resources/omf/wilabt_node.py
@@ -20,7 +20,7 @@
 from nepi.execution.attribute import Attribute, Flags, Types
 from nepi.execution.resource import ResourceManager, clsinit_copy, \
         ResourceState, reschedule_delay 
 from nepi.execution.attribute import Attribute, Flags, Types
 from nepi.execution.resource import ResourceManager, clsinit_copy, \
         ResourceState, reschedule_delay 

-from nepi.resources.linux.node import LinuxNode
+from nepi.resources.omf.node import OMFNode

 from nepi.util.sfaapi import SFAAPIFactory 
 from nepi.util.execfuncs import lexec
 from nepi.util import sshfuncs
 from nepi.util.sfaapi import SFAAPIFactory 
 from nepi.util.execfuncs import lexec
 from nepi.util import sshfuncs


@@ -34,7 +34,7 @@ import threading
 import datetime
 
 @clsinit_copy
 import datetime
 
 @clsinit_copy

-class WilabtSfaNode(LinuxNode):
+class WilabtSfaNode(OMFNode):

     _rtype = "WilabtSfaNode"
     _help = "Controls a Wilabt host accessible using a SSH key " \
             "and provisioned using SFA"
     _rtype = "WilabtSfaNode"
     _help = "Controls a Wilabt host accessible using a SSH key " \
             "and provisioned using SFA"


@@ -42,6 +42,15 @@ class WilabtSfaNode(LinuxNode):
 
     @classmethod
     def _register_attributes(cls):
 
     @classmethod
     def _register_attributes(cls):

+        
+        username = Attribute("username", "Local account username",
+                flags = Flags.Credential)
+
+        identity = Attribute("identity", "SSH identity file",
+                flags = Flags.Credential)
+
+        server_key = Attribute("serverKey", "Server public key",
+                flags = Flags.Design)

 
         sfa_user = Attribute("sfauser", "SFA user",
                     flags = Flags.Credential)
 
         sfa_user = Attribute("sfauser", "SFA user",
                     flags = Flags.Credential)


@@ -59,11 +68,22 @@ class WilabtSfaNode(LinuxNode):
         gateway = Attribute("gateway", "Hostname of the gateway machine",
                 flags = Flags.Design)
 
         gateway = Attribute("gateway", "Hostname of the gateway machine",
                 flags = Flags.Design)
 

+        hostxmpp = Attribute("hostxmpp", "Hostname from RSpec to use in xmpp messages",
+                flags = Flags.Design)
+
+        disk_image = Attribute("disk_image", "Specify a specific disk image for a node",
+                flags = Flags.Design)
+        
+        cls._register_attribute(username)
+        cls._register_attribute(identity)
+        cls._register_attribute(server_key)

         cls._register_attribute(sfa_user)
         cls._register_attribute(sfa_private_key)
         cls._register_attribute(slicename)
         cls._register_attribute(gateway_user)
         cls._register_attribute(gateway)
         cls._register_attribute(sfa_user)
         cls._register_attribute(sfa_private_key)
         cls._register_attribute(slicename)
         cls._register_attribute(gateway_user)
         cls._register_attribute(gateway)

+        cls._register_attribute(hostxmpp)
+        cls._register_attribute(disk_image)

 
     def __init__(self, ec, guid):
         super(WilabtSfaNode, self).__init__(ec, guid)
 
     def __init__(self, ec, guid):
         super(WilabtSfaNode, self).__init__(ec, guid)


@@ -152,7 +172,7 @@ class WilabtSfaNode(LinuxNode):
             node = self._node_to_provision
             if self._slicenode:
                 self._delete_from_slice()
             node = self._node_to_provision
             if self._slicenode:
                 self._delete_from_slice()

-                self.debug("Waiting 300 seg for re-adding to slice")
+                self.debug("Waiting 300 sec for re-adding to slice")

                 time.sleep(300) # Timout for the testbed to allow a new reservation
             self._add_node_to_slice(node)
             t = 0
                 time.sleep(300) # Timout for the testbed to allow a new reservation
             self._add_node_to_slice(node)
             t = 0


@@ -160,7 +180,7 @@ class WilabtSfaNode(LinuxNode):
                 and not self._ecobj().abort:
                 t = t + 5
                 time.sleep(t)
                 and not self._ecobj().abort:
                 t = t + 5
                 time.sleep(t)

-                self.debug("Waiting 5 seg for resources to be added")
+                self.debug("Waiting 5 sec for resources to be added")

                 continue
 
             if not self._check_if_in_slice([node]):
                 continue
 
             if not self._check_if_in_slice([node]):


@@ -185,7 +205,7 @@ class WilabtSfaNode(LinuxNode):
                 if not self._check_fs():
                     self.do_discover()
                     continue
                 if not self._check_fs():
                     self.do_discover()
                     continue

-                if not self._check_omf():
+                if not self._check_omfrc():

                     self.do_discover()
                     continue
                 if not self._check_hostname():
                     self.do_discover()
                     continue
                 if not self._check_hostname():


@@ -200,6 +220,12 @@ class WilabtSfaNode(LinuxNode):
             
         super(WilabtSfaNode, self).do_provision()
 
             
         super(WilabtSfaNode, self).do_provision()
 

+    def do_release(self):
+        super(WilabtSfaNode, self).do_release()
+        if self.state == ResourceState.RELEASED and not self._skip_provision():
+            self.debug(" Releasing SFA API ")
+            self.sfaapi.release()
+

     def _blacklisted(self, host_hrn):
         """
         Check in the SFA API that the node is not in the blacklist.
     def _blacklisted(self, host_hrn):
         """
         Check in the SFA API that the node is not in the blacklist.


@@ -237,15 +263,15 @@ class WilabtSfaNode(LinuxNode):
         through the gateway because is private testbed.
         """
         t = 0
         through the gateway because is private testbed.
         """
         t = 0

-        timeout = 10
+        timeout = 300

         ssh_ok = False
         while t < timeout and not ssh_ok:
             cmd = 'echo \'GOOD NODE\''
             ((out, err), proc) = self.execute(cmd)
             if out.find("GOOD NODE") < 0:
         ssh_ok = False
         while t < timeout and not ssh_ok:
             cmd = 'echo \'GOOD NODE\''
             ((out, err), proc) = self.execute(cmd)
             if out.find("GOOD NODE") < 0:

-                self.debug( "No SSH connection, waiting 60s" )
-                t = t + 5
-                time.sleep(5)
+                self.debug( "No SSH connection, waiting 20s" )
+                t = t + 20
+                time.sleep(20)

                 continue
             else:
                 self.debug( "SSH OK" )
                 continue
             else:
                 self.debug( "SSH OK" )


@@ -283,8 +309,48 @@ class WilabtSfaNode(LinuxNode):
         ((out, err), proc) = self.execute(cmd)
         if 'localhost' in out.lower():
             return False
         ((out, err), proc) = self.execute(cmd)
         if 'localhost' in out.lower():
             return False

+        else:
+            self.set('hostxmpp', out.strip()) 

         return True 
 
         return True 
 

+    def execute(self, command,
+        sudo = False,
+        env = None,
+        tty = False,
+        forward_x11 = False,
+        retry = 3,
+        connect_timeout = 30,
+        strict_host_checking = False,
+        persistent = True,
+        blocking = True,
+        ):
+        """ Notice that this invocation will block until the
+        execution finishes. If this is not the desired behavior,
+        use 'run' instead."""
+        (out, err), proc = sshfuncs.rexec(
+            command,
+            host = self.get("hostname"),
+            user = self.get("username"),
+            port = 22,
+            gwuser = self.get("gatewayUser"),
+            gw = self.get("gateway"),
+            agent = True,
+            sudo = sudo,
+            identity = self.get("identity"),
+            server_key = self.get("serverKey"),
+            env = env,
+            tty = tty,
+            forward_x11 = forward_x11,
+            retry = retry,
+            connect_timeout = connect_timeout,
+            persistent = persistent,
+            blocking = blocking,
+            strict_host_checking = strict_host_checking
+            )
+
+        return (out, err), proc
+
+

     def _add_node_to_slice(self, host_hrn):
         """
         Add node to slice, using SFA API. Actually Wilabt testbed
     def _add_node_to_slice(self, host_hrn):
         """
         Add node to slice, using SFA API. Actually Wilabt testbed


@@ -295,7 +361,11 @@ class WilabtSfaNode(LinuxNode):
         """
         self.info(" Adding node to slice ")
         slicename = self.get("slicename")
         """
         self.info(" Adding node to slice ")
         slicename = self.get("slicename")

-        self.sfaapi.add_resource_to_slice_batch(slicename, host_hrn)
+        disk_image = self.get("disk_image")
+        if disk_image is not None:
+            properties = {'disk_image': disk_image}
+        else: properties = None
+        self.sfaapi.add_resource_to_slice_batch(slicename, host_hrn, properties=properties)

 
     def _delete_from_slice(self):
         """
 
     def _delete_from_slice(self):
         """


@@ -398,16 +468,15 @@ class WilabtSfaNode(LinuxNode):
         raise RuntimeError, msg
     
     def fail_node_not_available(self, hostname):
         raise RuntimeError, msg
     
     def fail_node_not_available(self, hostname):

-        msg = "Node %s not available for provisioning" % hostname
+        msg = "Some nodes not available for provisioning"

         raise RuntimeError, msg
 
     def fail_not_enough_nodes(self):
         msg = "Not enough nodes available for provisioning"
         raise RuntimeError, msg
 
         raise RuntimeError, msg
 
     def fail_not_enough_nodes(self):
         msg = "Not enough nodes available for provisioning"
         raise RuntimeError, msg
 

-    def fail_plapi(self):
-        msg = "Failing while trying to instanciate the PLC API.\nSet the" + \
-            " attributes pluser and plpassword."
+    def fail_sfaapi(self):
+        msg = "Failing while trying to instanciate the SFA API."

         raise RuntimeError, msg
 
     def valid_connection(self, guid):
         raise RuntimeError, msg
 
     def valid_connection(self, guid):

diff --git a/src/nepi/util/sfaapi.py b/src/nepi/util/sfaapi.py
index 13f45e4..15e75be 100644 (file)
--- a/src/nepi/util/sfaapi.py
+++ b/src/nepi/util/sfaapi.py
@@ -46,6 +46,7 @@ class SFAAPI(object):
         self._resources_cache = None
         self._already_cached = False
         self._ec = ec 
         self._resources_cache = None
         self._already_cached = False
         self._ec = ec 

+        self.apis = 1

 
         if batch:
             self._testbed_res = rtype
 
         if batch:
             self._testbed_res = rtype


@@ -100,12 +101,12 @@ class SFAAPI(object):
                 rms.append(rm)
         return rms
 
                 rms.append(rm)
         return rms
 

-    def _sfi_exec_method(self, command, slicename=None, rspec=None, urn=None):
+    def _sfi_exec_method(self, command, slicename=None, rspec=None, urn=None, action=None):

         """
         Execute sfi method, which correspond to SFA call. It can be the following
         calls: Describe, Delete, Allocate, Provision, ListResources.
         """
         """
         Execute sfi method, which correspond to SFA call. It can be the following
         calls: Describe, Delete, Allocate, Provision, ListResources.
         """

-        if command in ['describe', 'delete', 'allocate', 'provision']:
+        if command in ['describe', 'delete', 'allocate', 'provision', 'action']:

             if not slicename:
                 raise TypeError("The slice hrn is expected for this method %s" % command)
             if command == 'allocate' and not rspec:
             if not slicename:
                 raise TypeError("The slice hrn is expected for this method %s" % command)
             if command == 'allocate' and not rspec:


@@ -117,6 +118,8 @@ class SFAAPI(object):
                 args_list = [slicename]
             if command != 'delete':
                 args_list = args_list + ['-o', '/tmp/rspec_output']
                 args_list = [slicename]
             if command != 'delete':
                 args_list = args_list + ['-o', '/tmp/rspec_output']

+            if command == 'action':
+                args_list = [slicename, action]

 
         elif command == 'resources':
             args_list = ['-o', '/tmp/rspec_output']
 
         elif command == 'resources':
             args_list = ['-o', '/tmp/rspec_output']


@@ -240,7 +243,7 @@ class SFAAPI(object):
                     raise RuntimeError("Fail to provision resource for slice %s" % slicename)
                 return True
 
                     raise RuntimeError("Fail to provision resource for slice %s" % slicename)
                 return True
 

-    def add_resource_to_slice_batch(self, slicename, resource_hrn, leases=None):
+    def add_resource_to_slice_batch(self, slicename, resource_hrn, properties=None, leases=None):

         """
         Method to add all resources together to the slice. Previous deletion of slivers.
         Specially used for wilabt that doesn't allow to add more resources to the slice
         """
         Method to add all resources together to the slice. Previous deletion of slivers.
         Specially used for wilabt that doesn't allow to add more resources to the slice


@@ -259,8 +262,7 @@ class SFAAPI(object):
                 self._sfi_exec_method('delete', slicename)
                 # Re implementing urn from hrn because the library sfa-common doesn't work for wilabt
                 resources_urn = self._get_urn(resources_hrn_new)
                 self._sfi_exec_method('delete', slicename)
                 # Re implementing urn from hrn because the library sfa-common doesn't work for wilabt
                 resources_urn = self._get_urn(resources_hrn_new)

-                rspec = self.rspec_proc.build_sfa_rspec(slicename, resources_urn, leases)
-
+                rspec = self.rspec_proc.build_sfa_rspec(slicename, resources_urn, properties, leases)

                 f = open("/tmp/rspec_input.rspec", "w")
                 f.truncate(0)
                 f.write(rspec)
                 f = open("/tmp/rspec_input.rspec", "w")
                 f.truncate(0)
                 f.write(rspec)


@@ -281,6 +283,7 @@ class SFAAPI(object):
                     try:
                         self._log.debug("Provisioning resources in slice %s" % slicename)
                         self._sfi_exec_method('provision', slicename)
                     try:
                         self._log.debug("Provisioning resources in slice %s" % slicename)
                         self._sfi_exec_method('provision', slicename)

+                        self._sfi_exec_method('action', slicename=slicename, action='geni_start')

                     except:
                         raise RuntimeError("Fail to provision resource for slice %s" % slicename)
                     return True
                     except:
                         raise RuntimeError("Fail to provision resource for slice %s" % slicename)
                     return True


@@ -379,6 +382,33 @@ class SFAAPI(object):
                 self.reserve_resource(resource_hrn)
                 return False
 
                 self.reserve_resource(resource_hrn)
                 return False
 

+    def release(self):
+        """
+        Remove hosts from the reserved and blacklist lists, and in case
+        the persist attribute is set, it saves the blacklisted hosts
+        in the blacklist file.
+        """
+        self.apis -= 1
+        if self.apis == 0:
+            blacklist = self._blacklist
+            self._blacklist = set()
+            self._reserved = set()
+#            if self._ecobj.get_global('PlanetlabSfaNode', 'persist_blacklist'):
+#                if blacklist:
+#                    to_blacklist = list()
+#                    hostnames = self.get_nodes(list(blacklist), ['hostname'])
+#                    for hostname in hostnames:
+#                        to_blacklist.append(hostname['hostname'])
+#
+#                    nepi_home = os.path.join(os.path.expanduser("~"), ".nepi")
+#                    plblacklist_file = os.path.join(nepi_home, "plblacklist.txt")
+#
+#                    with open(plblacklist_file, 'w') as f:
+#                        for host in to_blacklist:
+#                            f.write("%s\n" % host)
+#
+
+

 class SFAAPIFactory(object):
     """
     API Factory to manage a map of SFAAPI instances as key-value pairs, it
 class SFAAPIFactory(object):
     """
     API Factory to manage a map of SFAAPI instances as key-value pairs, it


@@ -403,6 +433,8 @@ class SFAAPIFactory(object):
                     api = SFAAPI(sfi_user, sfi_auth, sfi_registry, sfi_sm, private_key,
                         ec, batch, rtype, timeout)
                     cls._apis[key] = api
                     api = SFAAPI(sfi_user, sfi_auth, sfi_registry, sfi_sm, private_key,
                         ec, batch, rtype, timeout)
                     cls._apis[key] = api

+                else:
+                    api.apis += 1

 
                 return api
 
 
                 return api
 

diff --git a/src/nepi/util/sfarspec_proc.py b/src/nepi/util/sfarspec_proc.py
index 66c2c8d..d8bb27b 100644 (file)
--- a/src/nepi/util/sfarspec_proc.py
+++ b/src/nepi/util/sfarspec_proc.py
@@ -133,7 +133,7 @@ class SfaRSpecProcessing(object):
 #               }
 
  
 #               }
 
  

-    def build_sfa_rspec(self, slice_id, resources, leases):
+    def build_sfa_rspec(self, slice_id, resources, properties, leases):

         """
         Build the XML RSpec from list of resources' urns.
         eg. resources = ["urn:publicid:IDN+ple:modenaple+node+planetlab-1.ing.unimo.it"]
         """
         Build the XML RSpec from list of resources' urns.
         eg. resources = ["urn:publicid:IDN+ple:modenaple+node+planetlab-1.ing.unimo.it"]


@@ -158,6 +158,7 @@ class SfaRSpecProcessing(object):
         links = []
         self._log.debug("Building RSpec for resources %s" % resources)
         cardinal = 0
         links = []
         self._log.debug("Building RSpec for resources %s" % resources)
         cardinal = 0

+        wilab = False

         for urn in resources:
             # XXX TO BE CORRECTED, this handles None values
             if not urn:
         for urn in resources:
             # XXX TO BE CORRECTED, this handles None values
             if not urn:


@@ -167,6 +168,7 @@ class SfaRSpecProcessing(object):
             # TODO: take into account the case where we send a dict of URNs without keys
             #resource['component_id'] = resource.pop('urn')
             resource['component_id'] = urn
             # TODO: take into account the case where we send a dict of URNs without keys
             #resource['component_id'] = resource.pop('urn')
             resource['component_id'] = urn

+            print resource['component_id']

             resource_hrn, resource_type = urn_to_hrn(resource['component_id'])
             # build component_manager_id
             top_auth = resource_hrn.split('.')[0]
             resource_hrn, resource_type = urn_to_hrn(resource['component_id'])
             # build component_manager_id
             top_auth = resource_hrn.split('.')[0]


@@ -175,14 +177,16 @@ class SfaRSpecProcessing(object):
 
             if resource_type == 'node':
                 # XXX dirty hack WiLab !!!
 
             if resource_type == 'node':
                 # XXX dirty hack WiLab !!!

-#                Commented Lucia, only works with config file, not the case for nepi  
+#                Commented Lucia, doesn't work for wilabt  

 #                if self.config:
 #                    if 'wilab2' in self.config['sm']:
 #                        resource['client_id'] = "PC"
 #                        resource['sliver_type'] = "raw-pc"
                 if 'wilab2' in urn:
 #                if self.config:
 #                    if 'wilab2' in self.config['sm']:
 #                        resource['client_id'] = "PC"
 #                        resource['sliver_type'] = "raw-pc"
                 if 'wilab2' in urn:

+                    wilab = True

                     resource['client_id'] = "node%s" % cardinal
                     resource['sliver_type'] = "raw-pc"
                     resource['client_id'] = "node%s" % cardinal
                     resource['sliver_type'] = "raw-pc"

+                    resource['disk_image'] = "hola"

                     top_auth = resource_hrn.replace("\\", "").split('.')
                     top_auth.pop()
                     top_auth = '.'.join(top_auth)
                     top_auth = resource_hrn.replace("\\", "").split('.')
                     top_auth.pop()
                     top_auth = '.'.join(top_auth)


@@ -202,7 +206,19 @@ class SfaRSpecProcessing(object):
         #rspec.version.add_links(links)
         #rspec.version.add_channels(channels)
 
         #rspec.version.add_links(links)
         #rspec.version.add_channels(channels)
 

-        self._log.debug("request rspec: %s"%rspec.toxml())
-        return rspec.toxml()
+        #self._log.debug("request rspec: %s"%rspec.toxml())
+        string = rspec.toxml()
+        if wilab and properties is not None:
+            ## dirty hack for the f4f demo
+            b = string.split('\n')
+            for i, n in enumerate(b):
+                if 'sliver_type name="raw-pc"' in n:
+                    b[i] = '<sliver_type name="raw-pc">'
+                    #b.insert(i+1, '<disk_image name="urn:publicid:IDN+wall2.ilabt.iminds.be+image+emulab-ops//%s"/>' % properties['disk_image'])
+                    b.insert(i+1, '<disk_image name="urn:publicid:IDN+wilab2.ilabt.iminds.be+image+nepi:%s"/>' % properties['disk_image'])
+                    b.insert(i+2, '</sliver_type>')
+            string = ''.join(b)
+        self._log.debug("request rspec : %s" % string)
+        return string