From: Lucia Guevgeozian Odizzio Date: Thu, 12 Jun 2014 12:24:35 +0000 (+0200) Subject: Modifications for wilab for the demo X-Git-Tag: nepi-3.1.0~26 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=75bccfca03c03ede06d63c061c62cd5416194c42;p=nepi.git Modifications for wilab for the demo --- diff --git a/src/nepi/resources/omf/wilabt_node.py b/src/nepi/resources/omf/wilabt_node.py index 80f076c7..d18769ac 100644 --- a/src/nepi/resources/omf/wilabt_node.py +++ b/src/nepi/resources/omf/wilabt_node.py @@ -20,7 +20,7 @@ from nepi.execution.attribute import Attribute, Flags, Types from nepi.execution.resource import ResourceManager, clsinit_copy, \ ResourceState, reschedule_delay -from nepi.resources.linux.node import LinuxNode +from nepi.resources.omf.node import OMFNode from nepi.util.sfaapi import SFAAPIFactory from nepi.util.execfuncs import lexec from nepi.util import sshfuncs @@ -34,7 +34,7 @@ import threading import datetime @clsinit_copy -class WilabtSfaNode(LinuxNode): +class WilabtSfaNode(OMFNode): _rtype = "WilabtSfaNode" _help = "Controls a Wilabt host accessible using a SSH key " \ "and provisioned using SFA" @@ -42,6 +42,15 @@ class WilabtSfaNode(LinuxNode): @classmethod def _register_attributes(cls): + + username = Attribute("username", "Local account username", + flags = Flags.Credential) + + identity = Attribute("identity", "SSH identity file", + flags = Flags.Credential) + + server_key = Attribute("serverKey", "Server public key", + flags = Flags.Design) sfa_user = Attribute("sfauser", "SFA user", flags = Flags.Credential) @@ -59,11 +68,22 @@ class WilabtSfaNode(LinuxNode): gateway = Attribute("gateway", "Hostname of the gateway machine", flags = Flags.Design) + hostxmpp = Attribute("hostxmpp", "Hostname from RSpec to use in xmpp messages", + flags = Flags.Design) + + disk_image = Attribute("disk_image", "Specify a specific disk image for a node", + flags = Flags.Design) + + cls._register_attribute(username) + cls._register_attribute(identity) + cls._register_attribute(server_key) cls._register_attribute(sfa_user) cls._register_attribute(sfa_private_key) cls._register_attribute(slicename) cls._register_attribute(gateway_user) cls._register_attribute(gateway) + cls._register_attribute(hostxmpp) + cls._register_attribute(disk_image) def __init__(self, ec, guid): super(WilabtSfaNode, self).__init__(ec, guid) @@ -152,7 +172,7 @@ class WilabtSfaNode(LinuxNode): node = self._node_to_provision if self._slicenode: self._delete_from_slice() - self.debug("Waiting 300 seg for re-adding to slice") + self.debug("Waiting 300 sec for re-adding to slice") time.sleep(300) # Timout for the testbed to allow a new reservation self._add_node_to_slice(node) t = 0 @@ -160,7 +180,7 @@ class WilabtSfaNode(LinuxNode): and not self._ecobj().abort: t = t + 5 time.sleep(t) - self.debug("Waiting 5 seg for resources to be added") + self.debug("Waiting 5 sec for resources to be added") continue if not self._check_if_in_slice([node]): @@ -185,7 +205,7 @@ class WilabtSfaNode(LinuxNode): if not self._check_fs(): self.do_discover() continue - if not self._check_omf(): + if not self._check_omfrc(): self.do_discover() continue if not self._check_hostname(): @@ -200,6 +220,12 @@ class WilabtSfaNode(LinuxNode): super(WilabtSfaNode, self).do_provision() + def do_release(self): + super(WilabtSfaNode, self).do_release() + if self.state == ResourceState.RELEASED and not self._skip_provision(): + self.debug(" Releasing SFA API ") + self.sfaapi.release() + def _blacklisted(self, host_hrn): """ Check in the SFA API that the node is not in the blacklist. @@ -237,15 +263,15 @@ class WilabtSfaNode(LinuxNode): through the gateway because is private testbed. """ t = 0 - timeout = 10 + timeout = 300 ssh_ok = False while t < timeout and not ssh_ok: cmd = 'echo \'GOOD NODE\'' ((out, err), proc) = self.execute(cmd) if out.find("GOOD NODE") < 0: - self.debug( "No SSH connection, waiting 60s" ) - t = t + 5 - time.sleep(5) + self.debug( "No SSH connection, waiting 20s" ) + t = t + 20 + time.sleep(20) continue else: self.debug( "SSH OK" ) @@ -283,8 +309,48 @@ class WilabtSfaNode(LinuxNode): ((out, err), proc) = self.execute(cmd) if 'localhost' in out.lower(): return False + else: + self.set('hostxmpp', out.strip()) return True + def execute(self, command, + sudo = False, + env = None, + tty = False, + forward_x11 = False, + retry = 3, + connect_timeout = 30, + strict_host_checking = False, + persistent = True, + blocking = True, + ): + """ Notice that this invocation will block until the + execution finishes. If this is not the desired behavior, + use 'run' instead.""" + (out, err), proc = sshfuncs.rexec( + command, + host = self.get("hostname"), + user = self.get("username"), + port = 22, + gwuser = self.get("gatewayUser"), + gw = self.get("gateway"), + agent = True, + sudo = sudo, + identity = self.get("identity"), + server_key = self.get("serverKey"), + env = env, + tty = tty, + forward_x11 = forward_x11, + retry = retry, + connect_timeout = connect_timeout, + persistent = persistent, + blocking = blocking, + strict_host_checking = strict_host_checking + ) + + return (out, err), proc + + def _add_node_to_slice(self, host_hrn): """ Add node to slice, using SFA API. Actually Wilabt testbed @@ -295,7 +361,11 @@ class WilabtSfaNode(LinuxNode): """ self.info(" Adding node to slice ") slicename = self.get("slicename") - self.sfaapi.add_resource_to_slice_batch(slicename, host_hrn) + disk_image = self.get("disk_image") + if disk_image is not None: + properties = {'disk_image': disk_image} + else: properties = None + self.sfaapi.add_resource_to_slice_batch(slicename, host_hrn, properties=properties) def _delete_from_slice(self): """ @@ -398,16 +468,15 @@ class WilabtSfaNode(LinuxNode): raise RuntimeError, msg def fail_node_not_available(self, hostname): - msg = "Node %s not available for provisioning" % hostname + msg = "Some nodes not available for provisioning" raise RuntimeError, msg def fail_not_enough_nodes(self): msg = "Not enough nodes available for provisioning" raise RuntimeError, msg - def fail_plapi(self): - msg = "Failing while trying to instanciate the PLC API.\nSet the" + \ - " attributes pluser and plpassword." + def fail_sfaapi(self): + msg = "Failing while trying to instanciate the SFA API." raise RuntimeError, msg def valid_connection(self, guid): diff --git a/src/nepi/util/sfaapi.py b/src/nepi/util/sfaapi.py index 13f45e46..15e75be0 100644 --- a/src/nepi/util/sfaapi.py +++ b/src/nepi/util/sfaapi.py @@ -46,6 +46,7 @@ class SFAAPI(object): self._resources_cache = None self._already_cached = False self._ec = ec + self.apis = 1 if batch: self._testbed_res = rtype @@ -100,12 +101,12 @@ class SFAAPI(object): rms.append(rm) return rms - def _sfi_exec_method(self, command, slicename=None, rspec=None, urn=None): + def _sfi_exec_method(self, command, slicename=None, rspec=None, urn=None, action=None): """ Execute sfi method, which correspond to SFA call. It can be the following calls: Describe, Delete, Allocate, Provision, ListResources. """ - if command in ['describe', 'delete', 'allocate', 'provision']: + if command in ['describe', 'delete', 'allocate', 'provision', 'action']: if not slicename: raise TypeError("The slice hrn is expected for this method %s" % command) if command == 'allocate' and not rspec: @@ -117,6 +118,8 @@ class SFAAPI(object): args_list = [slicename] if command != 'delete': args_list = args_list + ['-o', '/tmp/rspec_output'] + if command == 'action': + args_list = [slicename, action] elif command == 'resources': args_list = ['-o', '/tmp/rspec_output'] @@ -240,7 +243,7 @@ class SFAAPI(object): raise RuntimeError("Fail to provision resource for slice %s" % slicename) return True - def add_resource_to_slice_batch(self, slicename, resource_hrn, leases=None): + def add_resource_to_slice_batch(self, slicename, resource_hrn, properties=None, leases=None): """ Method to add all resources together to the slice. Previous deletion of slivers. Specially used for wilabt that doesn't allow to add more resources to the slice @@ -259,8 +262,7 @@ class SFAAPI(object): self._sfi_exec_method('delete', slicename) # Re implementing urn from hrn because the library sfa-common doesn't work for wilabt resources_urn = self._get_urn(resources_hrn_new) - rspec = self.rspec_proc.build_sfa_rspec(slicename, resources_urn, leases) - + rspec = self.rspec_proc.build_sfa_rspec(slicename, resources_urn, properties, leases) f = open("/tmp/rspec_input.rspec", "w") f.truncate(0) f.write(rspec) @@ -281,6 +283,7 @@ class SFAAPI(object): try: self._log.debug("Provisioning resources in slice %s" % slicename) self._sfi_exec_method('provision', slicename) + self._sfi_exec_method('action', slicename=slicename, action='geni_start') except: raise RuntimeError("Fail to provision resource for slice %s" % slicename) return True @@ -379,6 +382,33 @@ class SFAAPI(object): self.reserve_resource(resource_hrn) return False + def release(self): + """ + Remove hosts from the reserved and blacklist lists, and in case + the persist attribute is set, it saves the blacklisted hosts + in the blacklist file. + """ + self.apis -= 1 + if self.apis == 0: + blacklist = self._blacklist + self._blacklist = set() + self._reserved = set() +# if self._ecobj.get_global('PlanetlabSfaNode', 'persist_blacklist'): +# if blacklist: +# to_blacklist = list() +# hostnames = self.get_nodes(list(blacklist), ['hostname']) +# for hostname in hostnames: +# to_blacklist.append(hostname['hostname']) +# +# nepi_home = os.path.join(os.path.expanduser("~"), ".nepi") +# plblacklist_file = os.path.join(nepi_home, "plblacklist.txt") +# +# with open(plblacklist_file, 'w') as f: +# for host in to_blacklist: +# f.write("%s\n" % host) +# + + class SFAAPIFactory(object): """ API Factory to manage a map of SFAAPI instances as key-value pairs, it @@ -403,6 +433,8 @@ class SFAAPIFactory(object): api = SFAAPI(sfi_user, sfi_auth, sfi_registry, sfi_sm, private_key, ec, batch, rtype, timeout) cls._apis[key] = api + else: + api.apis += 1 return api diff --git a/src/nepi/util/sfarspec_proc.py b/src/nepi/util/sfarspec_proc.py index 66c2c8dc..d8bb27bd 100644 --- a/src/nepi/util/sfarspec_proc.py +++ b/src/nepi/util/sfarspec_proc.py @@ -133,7 +133,7 @@ class SfaRSpecProcessing(object): # } - def build_sfa_rspec(self, slice_id, resources, leases): + def build_sfa_rspec(self, slice_id, resources, properties, leases): """ Build the XML RSpec from list of resources' urns. eg. resources = ["urn:publicid:IDN+ple:modenaple+node+planetlab-1.ing.unimo.it"] @@ -158,6 +158,7 @@ class SfaRSpecProcessing(object): links = [] self._log.debug("Building RSpec for resources %s" % resources) cardinal = 0 + wilab = False for urn in resources: # XXX TO BE CORRECTED, this handles None values if not urn: @@ -167,6 +168,7 @@ class SfaRSpecProcessing(object): # TODO: take into account the case where we send a dict of URNs without keys #resource['component_id'] = resource.pop('urn') resource['component_id'] = urn + print resource['component_id'] resource_hrn, resource_type = urn_to_hrn(resource['component_id']) # build component_manager_id top_auth = resource_hrn.split('.')[0] @@ -175,14 +177,16 @@ class SfaRSpecProcessing(object): if resource_type == 'node': # XXX dirty hack WiLab !!! -# Commented Lucia, only works with config file, not the case for nepi +# Commented Lucia, doesn't work for wilabt # if self.config: # if 'wilab2' in self.config['sm']: # resource['client_id'] = "PC" # resource['sliver_type'] = "raw-pc" if 'wilab2' in urn: + wilab = True resource['client_id'] = "node%s" % cardinal resource['sliver_type'] = "raw-pc" + resource['disk_image'] = "hola" top_auth = resource_hrn.replace("\\", "").split('.') top_auth.pop() top_auth = '.'.join(top_auth) @@ -202,7 +206,19 @@ class SfaRSpecProcessing(object): #rspec.version.add_links(links) #rspec.version.add_channels(channels) - self._log.debug("request rspec: %s"%rspec.toxml()) - return rspec.toxml() + #self._log.debug("request rspec: %s"%rspec.toxml()) + string = rspec.toxml() + if wilab and properties is not None: + ## dirty hack for the f4f demo + b = string.split('\n') + for i, n in enumerate(b): + if 'sliver_type name="raw-pc"' in n: + b[i] = '' + #b.insert(i+1, '' % properties['disk_image']) + b.insert(i+1, '' % properties['disk_image']) + b.insert(i+2, '') + string = ''.join(b) + self._log.debug("request rspec : %s" % string) + return string