From 80aa1aca24da91d4d83afe122449513f9ddae699 Mon Sep 17 00:00:00 2001 From: Alina Quereilhac Date: Mon, 27 May 2013 15:34:08 +0200 Subject: [PATCH] Adding PlanetLab resources --- examples/linux/scalability.py | 2 +- src/nepi/execution/attribute.py | 17 +- src/nepi/execution/ec.py | 29 +- src/nepi/execution/resource.py | 85 ++-- src/nepi/resources/linux/application.py | 2 +- src/nepi/resources/linux/interface.py | 8 +- src/nepi/resources/linux/node.py | 8 +- src/nepi/resources/omf/node.py | 39 +- src/nepi/resources/planetlab/node.py | 316 ++++++++++++++ src/nepi/resources/planetlab/plcapi.py | 548 ++++++++++++++++++++++++ test/resources/planetlab/plcapi.py | 49 +++ 11 files changed, 1030 insertions(+), 73 deletions(-) create mode 100644 src/nepi/resources/planetlab/node.py create mode 100644 src/nepi/resources/planetlab/plcapi.py create mode 100644 test/resources/planetlab/plcapi.py diff --git a/examples/linux/scalability.py b/examples/linux/scalability.py index 86f6a3f9..ae3ebb22 100644 --- a/examples/linux/scalability.py +++ b/examples/linux/scalability.py @@ -105,7 +105,7 @@ if __name__ == '__main__': "planetlab-1.fokus.fraunhofer.de", "iraplab2.iralab.uni-karlsruhe.de", "planet2.zib.de", - "pl2.uni-rostock.de", + #"pl2.uni-rostock.de", "onelab-1.fhi-fokus.de", "planet2.l3s.uni-hannover.de", "planetlab1.exp-math.uni-essen.de", diff --git a/src/nepi/execution/attribute.py b/src/nepi/execution/attribute.py index 5bebe94a..8203306c 100644 --- a/src/nepi/execution/attribute.py +++ b/src/nepi/execution/attribute.py @@ -21,7 +21,7 @@ class Types: String = "STRING" Bool = "BOOL" - Enum = "ENUM" + Enumerate = "ENUM" Double = "DOUBLE" Integer = "INTEGER" @@ -35,16 +35,19 @@ class Flags: ExecReadOnly = 0x02 # Attribute is an access credential Credential = 0x04 + # Attribute is a filter used to discover resources + Filter = 0x08 class Attribute(object): def __init__(self, name, help, type = Types.String, flags = Flags.NoFlags, default = None, allowed = None, - set_hook = None): + range = None, set_hook = None): self._name = name self._help = help self._type = type self._flags = flags self._allowed = allowed + self._range = range self._default = self._value = default # callback to be invoked upon changing the # attribute value @@ -74,6 +77,10 @@ class Attribute(object): def allowed(self): return self._allowed + @property + def range(self): + return self._range + def has_flag(self, flag): return (self._flags & flag) == flag @@ -83,8 +90,12 @@ class Attribute(object): def set_value(self, value): valid = True - if self.type == Types.Enum: + if self.type == Types.Enumerate: valid = value in self._allowed + + if self.type in [Types.Double, Types.Integer] and self.range: + (min, max) = self.range + valid = (value >= min and value <= max) valid = valid and self.is_valid_value(value) diff --git a/src/nepi/execution/ec.py b/src/nepi/execution/ec.py index d6fcf871..90cf710c 100644 --- a/src/nepi/execution/ec.py +++ b/src/nepi/execution/ec.py @@ -29,7 +29,7 @@ from nepi.util import guid from nepi.util.parallel import ParallelRun from nepi.util.timefuncs import strfnow, strfdiff, strfvalid from nepi.execution.resource import ResourceFactory, ResourceAction, \ - ResourceState + ResourceState, ResourceState2str from nepi.execution.scheduler import HeapScheduler, Task, TaskStatus from nepi.execution.trace import TraceAttr @@ -126,10 +126,6 @@ class ExperimentController(object): rm = self.get_resource(guid) return rm.get_attributes() - def get_filters(self, guid): - rm = self.get_resource(guid) - return rm.get_filters() - def register_connection(self, guid1, guid2): rm1 = self.get_resource(guid1) rm2 = self.get_resource(guid2) @@ -200,13 +196,13 @@ class ExperimentController(object): rm = self.get_resource(guid) return rm.trace(name, attr, block, offset) - def discover(self, guid, filters): + def discover(self, guid): rm = self.get_resource(guid) - return rm.discover(filters) + return rm.discover() - def provision(self, guid, filters): + def provision(self, guid): rm = self.get_resource(guid) - return rm.provision(filters) + return rm.provision() def get(self, guid, name): rm = self.get_resource(guid) @@ -216,8 +212,21 @@ class ExperimentController(object): rm = self.get_resource(guid) return rm.set(name, value) - def state(self, guid): + def state(self, guid, hr = False): + """ Returns the state of a resource + + :param guid: Resource guid + :type guid: integer + + :param hr: Human readable. Forces return of a + status string instead of a number + :type hr: boolean + + """ rm = self.get_resource(guid) + if hr: + return ResourceState2str.get(rm.state) + return rm.state def stop(self, guid): diff --git a/src/nepi/execution/resource.py b/src/nepi/execution/resource.py index d8ab870e..9637feaa 100644 --- a/src/nepi/execution/resource.py +++ b/src/nepi/execution/resource.py @@ -46,33 +46,55 @@ class ResourceState: FAILED = 7 RELEASED = 8 +ResourceState2str = dict({ + NEW = "NEW", + DISCOVERED = "DISCOVERED", + PROVISIONED = "PROVISIONED", + READY = "READY", + STARTED = "STARTED", + STOPPED = "STOPPED", + FINISHED = "FINISHED", + FAILED = "FAILED", + RELEASED = "RELEASED", + }) + def clsinit(cls): + """ Initializes template information (i.e. attributes and traces) + for the ResourceManager class + """ cls._clsinit() return cls +def clsinit_copy(cls): + """ Initializes template information (i.e. attributes and traces) + for the ResourceManager class, inheriting attributes and traces + from the parent class + """ + cls._clsinit_copy() + return cls + # Decorator to invoke class initialization method @clsinit class ResourceManager(object): _rtype = "Resource" - _filters = None _attributes = None _traces = None @classmethod - def _register_filter(cls, attr): + def _register_attribute(cls, attr): """ Resource subclasses will invoke this method to add a - filter attribute + resource attribute """ - cls._filters[attr.name] = attr + cls._attributes[attr.name] = attr @classmethod - def _register_attribute(cls, attr): - """ Resource subclasses will invoke this method to add a + def _remove_attribute(cls, name): + """ Resource subclasses will invoke this method to remove a resource attribute """ - cls._attributes[attr.name] = attr + del cls._attributes[name] @classmethod def _register_trace(cls, trace): @@ -82,14 +104,13 @@ class ResourceManager(object): """ cls._traces[trace.name] = trace - @classmethod - def _register_filters(cls): - """ Resource subclasses will invoke this method to register - resource filters + def _remove_trace(cls, name): + """ Resource subclasses will invoke this method to remove a + resource trace """ - pass + del cls._traces[name] @classmethod def _register_attributes(cls): @@ -109,16 +130,14 @@ class ResourceManager(object): @classmethod def _clsinit(cls): - """ Create a new dictionnary instance of the dictionnary - with the same template. - - Each ressource should have the same registration dictionary - template with different instances. + """ ResourceManager child classes have different attributes and traces. + Since the templates that hold the information of attributes and traces + are 'class attribute' dictionaries, initially they all point to the + parent class ResourceManager instances of those dictionaries. + In order to make these templates independent from the parent's one, + it is necessary re-initialize the corresponding dictionaries. + This is the objective of the _clsinit method """ - # static template for resource filters - cls._filters = dict() - cls._register_filters() - # static template for resource attributes cls._attributes = dict() cls._register_attributes() @@ -128,15 +147,21 @@ class ResourceManager(object): cls._register_traces() @classmethod - def rtype(cls): - return cls._rtype + def _clsinit_copy(cls): + """ Same as _clsinit, except that it also inherits all attributes and traces + from the parent class. + """ + # static template for resource attributes + cls._attributes = copy.deepcopy(cls._attributes) + cls._register_attributes() - @classmethod - def get_filters(cls): - """ Returns a copy of the filters + # static template for resource traces + cls._traces = copy.deepcopy(cls._traces) + cls._register_traces() - """ - return copy.deepcopy(cls._filters.values()) + @classmethod + def rtype(cls): + return cls._rtype @classmethod def get_attributes(cls): @@ -260,11 +285,11 @@ class ResourceManager(object): if self.valid_connection(guid): self._connections.add(guid) - def discover(self, filters = None): + def discover(self): self._discover_time = strfnow() self._state = ResourceState.DISCOVERED - def provision(self, filters = None): + def provision(self): self._provision_time = strfnow() self._state = ResourceState.PROVISIONED diff --git a/src/nepi/resources/linux/application.py b/src/nepi/resources/linux/application.py index 5e0c72b5..f9edfc29 100644 --- a/src/nepi/resources/linux/application.py +++ b/src/nepi/resources/linux/application.py @@ -195,7 +195,7 @@ class LinuxApplication(ResourceManager): return out - def provision(self, filters = None): + def provision(self): # create home dir for application self.node.mkdir(self.app_home) diff --git a/src/nepi/resources/linux/interface.py b/src/nepi/resources/linux/interface.py index b36adef9..14628991 100644 --- a/src/nepi/resources/linux/interface.py +++ b/src/nepi/resources/linux/interface.py @@ -103,7 +103,7 @@ class LinuxInterface(ResourceManager): if chan: return chan[0] return None - def discover(self, filters = None): + def discover(self): devname = self.get("deviceName") ip4 = self.get("ip4") ip6 = self.get("ip4") @@ -182,9 +182,9 @@ class LinuxInterface(ResourceManager): self.error(msg) raise RuntimeError, msg - super(LinuxInterface, self).discover(filters = filters) + super(LinuxInterface, self).discover() - def provision(self, filters = None): + def provision(self): devname = self.get("deviceName") ip4 = self.get("ip4") ip6 = self.get("ip4") @@ -225,7 +225,7 @@ class LinuxInterface(ResourceManager): self.error(msg, out, err) raise RuntimeError, "%s - %s - %s" % (msg, out, err) - super(LinuxInterface, self).provision(filters = filters) + super(LinuxInterface, self).provision() def deploy(self): # Wait until node is provisioned diff --git a/src/nepi/resources/linux/node.py b/src/nepi/resources/linux/node.py index 519f8964..78de7fd5 100644 --- a/src/nepi/resources/linux/node.py +++ b/src/nepi/resources/linux/node.py @@ -147,7 +147,7 @@ class LinuxNode(ResourceManager): def localhost(self): return self.get("hostname") in ['localhost', '127.0.0.7', '::1'] - def provision(self, filters = None): + def provision(self): if not self.is_alive(): self._state = ResourceState.FAILED msg = "Deploy failed. Unresponsive node %s" % self.get("hostname") @@ -644,9 +644,3 @@ class LinuxNode(ResourceManager): re.I) return badre.search(out) or badre.search(err) - def blacklist(self): - # TODO!!!! - self.warn(" Blacklisting malfunctioning node ") - #import util - #util.appendBlacklist(self.hostname) - diff --git a/src/nepi/resources/omf/node.py b/src/nepi/resources/omf/node.py index 0099ff7c..986c7ee2 100644 --- a/src/nepi/resources/omf/node.py +++ b/src/nepi/resources/omf/node.py @@ -55,10 +55,24 @@ class OMFNode(ResourceManager): hostname = Attribute("hostname", "Hostname of the machine") cpu = Attribute("cpu", "CPU of the node") ram = Attribute("ram", "RAM of the node") - xmppSlice = Attribute("xmppSlice","Name of the slice", flags = Flags.Credential) - xmppHost = Attribute("xmppHost", "Xmpp Server",flags = Flags.Credential) - xmppPort = Attribute("xmppPort", "Xmpp Port",flags = Flags.Credential) - xmppPassword = Attribute("xmppPassword", "Xmpp Port",flags = Flags.Credential) + xmppSlice = Attribute("xmppSlice","Name of the slice", + flags = Flags.Credential) + xmppHost = Attribute("xmppHost", "Xmpp Server", + flags = Flags.Credential) + xmppPort = Attribute("xmppPort", "Xmpp Port", + flags = Flags.Credential) + xmppPassword = Attribute("xmppPassword", "Xmpp Port", + flags = Flags.Credential) + + host = Attribute("host", "Hostname of the machine", + flags = Flags.Filter) + gateway = Attribute("gateway", "Gateway", + flags = Flags.Filter) + granularity = Attribute("granularity", "Granularity of the reservation time", + flags = Flags.Filter) + hardware_type = Attribute("hardware_type", "Hardware type of the machine", + flags = Flags.Filter) + cls._register_attribute(hostname) cls._register_attribute(ram) cls._register_attribute(cpu) @@ -67,19 +81,10 @@ class OMFNode(ResourceManager): cls._register_attribute(xmppPort) cls._register_attribute(xmppPassword) - @classmethod - def _register_filters(cls): - """Register the filters of an OMF Node - - """ - hostname = Attribute("hostname", "Hostname of the machine") - gateway = Attribute("gateway", "Gateway") - granularity = Attribute("granularity", "Granularity of the reservation time") - hardware_type = Attribute("hardware_type", "Hardware type of the machine") - cls._register_filter(hostname) - cls._register_filter(gateway) - cls._register_filter(granularity) - cls._register_filter(hardware_type) + cls._register_attribute(host) + cls._register_attribute(gateway) + cls._register_attribute(granularity) + cls._register_attribute(hardware_type) # XXX: We don't necessary need to have the credentials at the # moment we create the RM diff --git a/src/nepi/resources/planetlab/node.py b/src/nepi/resources/planetlab/node.py new file mode 100644 index 00000000..c733e26d --- /dev/null +++ b/src/nepi/resources/planetlab/node.py @@ -0,0 +1,316 @@ +""" + NEPI, a framework to manage network experiments + Copyright (C) 2013 INRIA + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +""" + +from nepi.execution.attribute import Attribute, Flags, Types +from nepi.execution.resource import ResourceManager, clsinit_copy, ResourceState +from nepi.resources.linux.node import LinuxNode + +from nepi.resources.planetlab.plcapi import PLCAPIFactory + +import logging + +reschedule_delay = "0.5s" + +@clsinit_copy +class PlanetlabNode(LinuxNode): + _rtype = "PlanetLabNode" + + @classmethod + def _register_attributes(cls): + cls._remove_attribute("username") + + ip = Attribute("ip", "PlanetLab host public IP address", + flags = Flags.ReadOnly) + + slicename = Attribute("slice", "PlanetLab slice name", + flags = Flags.Credential) + + pl_url = Attribute("plcApiUrl", "URL of PlanetLab PLCAPI host (e.g. www.planet-lab.eu or www.planet-lab.org) ", + default = "www.planet-lab.eu", + flags = Flags.Credential) + + pl_ptn = Attribute("plcApiPattern", "PLC API service regexp pattern (e.g. https://%(hostname)s:443/PLCAPI/ ) ", + default = "https://%(hostname)s:443/PLCAPI/", + flags = Flags.ExecReadOnly) + + city = Attribute("city", + "Constrain location (city) during resource discovery. May use wildcards.", + flags = Flags.Filter) + + country = Attribute("country", + "Constrain location (country) during resource discovery. May use wildcards.", + flags = Flags.Filter) + + region = Attribute("region", + "Constrain location (region) during resource discovery. May use wildcards.", + flags = Flags.Filter) + + architecture = Attribute("architecture", + "Constrain architecture during resource discovery.", + type = Types.Enumerate, + allowed = ["x86_64", + "i386"], + flags = Flags.Filter) + + operating_system = Attribute("operatingSystem", + "Constrain operating system during resource discovery.", + type = Types.Enumerate, + allowed = ["f8", + "f12", + "f14", + "centos", + "other"], + flags = Flags.Filter) + + site = Attribute("site", + "Constrain the PlanetLab site this node should reside on.", + type = Types.Enumerate, + allowed = ["PLE", + "PLC", + "PLJ"], + flags = Flags.Filter) + + min_reliability = Attribute("minReliability", + "Constrain reliability while picking PlanetLab nodes. Specifies a lower acceptable bound.", + type = Types.Double, + range = (1, 100), + flags = Flags.Filter) + + max_reliability = Attribute("maxReliability", + "Constrain reliability while picking PlanetLab nodes. Specifies an upper acceptable bound.", + type = Types.Double, + range = (1, 100), + flags = Flags.Filter) + + min_bandwidth = Attribute("minBandwidth", + "Constrain available bandwidth while picking PlanetLab nodes. Specifies a lower acceptable bound.", + type = Types.Double, + range = (0, 2**31), + flags = Flags.Filter) + + max_bandwidth = Attribute("maxBandwidth", + "Constrain available bandwidth while picking PlanetLab nodes. Specifies an upper acceptable bound.", + type = Types.Double, + range = (0, 2**31), + flags = Flags.Filter) + + min_load = Attribute("minLoad", + "Constrain node load average while picking PlanetLab nodes. Specifies a lower acceptable bound.", + type = Types.Double, + range = (0, 2**31), + flags = Flags.Filter) + + max_load = Attribute("maxLoad", + "Constrain node load average while picking PlanetLab nodes. Specifies an upper acceptable bound.", + type = Types.Double, + range = (0, 2**31), + flags = Flags.Filter) + + min_cpu = Attribute("minCpu", + "Constrain available cpu time while picking PlanetLab nodes. Specifies a lower acceptable bound.", + type = Types.Double, + range = (0, 100), + flags = Flags.Filter) + + max_cpu = Attribute("maxCpu", + "Constrain available cpu time while picking PlanetLab nodes. Specifies an upper acceptable bound.", + type = Types.Double, + range = (0, 100), + flags = Flags.Filter) + + timeframe = Attribute("timeframe", + "Past time period in which to check information about the node. Values are year,month, week, latest", + default = "week", + type = Types.Enumerate, + allowed = ["latest", + "week", + "month", + "year"], + flags = Flags.Filter) + + cls._register_attribute(ip) + cls._register_attribute(slicename) + cls._register_attribute(pl_url) + cls._register_attribute(pl_ptn) + cls._register_attribute(city) + cls._register_attribute(country) + cls._register_attribute(region) + cls._register_attribute(architecture) + cls._register_attribute(operating_system) + cls._register_attribute(min_reliability) + cls._register_attribute(max_reliability) + cls._register_attribute(min_bandwidth) + cls._register_attribute(max_bandwidth) + cls._register_attribute(min_load) + cls._register_attribute(max_load) + cls._register_attribute(min_cpu) + cls._register_attribute(max_cpu) + cls._register_attribute(timeframe) + + def __init__(self, ec, guid): + super(PLanetLabNode, self).__init__(ec, guid) + + self._plapi = None + + self._logger = logging.getLogger("PlanetLabNode") + + @property + def plapi(self): + if not self._plapi: + slicename = self.get("slice") + pl_pass = self.get("password") + pl_url = self.get("plcApiUrl") + pl_ptn = self.get("plcApiPattern") + + self._plapi = PLCAPIFactory.get_api(slicename, pl_pass, pl_url, + pl_ptn) + + return self._plapi + + @property + def os(self): + if self._os: + return self._os + + if (not self.get("hostname") or not self.get("username")): + msg = "Can't resolve OS, insufficient data " + self.error(msg) + raise RuntimeError, msg + + (out, err), proc = self.execute("cat /etc/issue", with_lock = True) + + if err and proc.poll(): + msg = "Error detecting OS " + self.error(msg, out, err) + raise RuntimeError, "%s - %s - %s" %( msg, out, err ) + + if out.find("Fedora release 12") == 0: + self._os = "f12" + elif out.find("Fedora release 14") == 0: + self._os = "f14" + else: + msg = "Unsupported OS" + self.error(msg, out) + raise RuntimeError, "%s - %s " %( msg, out ) + + return self._os + + @property + def localhost(self): + return False + + def discover(self): + # Get the list of nodes that match the filters + + + # find one that + if not self.is_alive(): + self._state = ResourceState.FAILED + msg = "Deploy failed. Unresponsive node %s" % self.get("hostname") + self.error(msg) + raise RuntimeError, msg + + if self.get("cleanProcesses"): + self.clean_processes() + + if self.get("cleanHome"): + self.clean_home() + + self.mkdir(self.node_home) + + super(PlanetlabNode, self).discover() + + def provision(self): + if not self.is_alive(): + self._state = ResourceState.FAILED + msg = "Deploy failed. Unresponsive node %s" % self.get("hostname") + self.error(msg) + raise RuntimeError, msg + + if self.get("cleanProcesses"): + self.clean_processes() + + if self.get("cleanHome"): + self.clean_home() + + self.mkdir(self.node_home) + + super(PlanetlabNode, self).provision() + + def deploy(self): + if self.state == ResourceState.NEW: + try: + self.discover() + if self.state == ResourceState.DISCOVERED: + self.provision() + except: + self._state = ResourceState.FAILED + raise + + if self.state != ResourceState.PROVISIONED: + self.ec.schedule(reschedule_delay, self.deploy) + + super(PlanetlabNode, self).deploy() + + def valid_connection(self, guid): + # TODO: Validate! + return True + + def clean_processes(self, killer = False): + self.info("Cleaning up processes") + + # Hardcore kill + cmd = ("sudo -S killall python tcpdump || /bin/true ; " + + "sudo -S killall python tcpdump || /bin/true ; " + + "sudo -S kill $(ps -N -T -o pid --no-heading | grep -v $PPID | sort) || /bin/true ; " + + "sudo -S killall -u root || /bin/true ; " + + "sudo -S killall -u root || /bin/true ; ") + + out = err = "" + (out, err), proc = self.execute(cmd, retry = 1, with_lock = True) + + def is_alive(self): + if self.localhost: + return True + + out = err = "" + try: + # TODO: FIX NOT ALIVE!!!! + (out, err), proc = self.execute("echo 'ALIVE' || (echo 'NOTALIVE') >&2", retry = 5, + with_lock = True) + except: + import traceback + trace = traceback.format_exc() + msg = "Unresponsive host %s " % err + self.error(msg, out, trace) + return False + + if out.strip().startswith('ALIVE'): + return True + else: + msg = "Unresponsive host " + self.error(msg, out, err) + return False + + def blacklist(self): + # TODO!!!! + self.warn(" Blacklisting malfunctioning node ") + #import util + #util.appendBlacklist(self.hostname) + diff --git a/src/nepi/resources/planetlab/plcapi.py b/src/nepi/resources/planetlab/plcapi.py new file mode 100644 index 00000000..15008d19 --- /dev/null +++ b/src/nepi/resources/planetlab/plcapi.py @@ -0,0 +1,548 @@ +""" + NEPI, a framework to manage network experiments + Copyright (C) 2013 INRIA + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +""" + +import functools +import hashlib +import socket +import time +import threading +import xmlrpclib + +def _retry(fn): + def rv(*p, **kw): + for x in xrange(5): + try: + return fn(*p, **kw) + except (socket.error, IOError, OSError): + time.sleep(x*5+5) + else: + return fn (*p, **kw) + return rv + +class PLCAPI(object): + + _expected_methods = set( + ['AddNodeTag', 'AddConfFile', 'DeletePersonTag', 'AddNodeType', + 'DeleteBootState', 'SliceListNames', 'DeleteKey','SliceGetTicket', + 'SliceUsersList', 'SliceUpdate', 'GetNodeGroups', 'SliceCreate', + 'GetNetworkMethods', 'GetNodeFlavour', 'DeleteNode', 'BootNotifyOwners', + 'AddPersonKey', 'AddNode', 'UpdateNodeGroup', 'GetAddressTypes', + 'AddIlink', 'DeleteNetworkType', 'GetInitScripts', 'GenerateNodeConfFile', + 'AddSite', 'BindObjectToPeer', 'SliceListUserSlices', 'GetPeers', + 'AddPeer', 'DeletePeer', 'AddRole', 'DeleteRole', 'SetPersonPrimarySite', + 'AddSiteAddress', 'SliceDelete', 'NotifyPersons', 'GetKeyTypes', + 'GetConfFiles', 'GetIlinks', 'AddTagType', 'GetNodes', 'DeleteNodeTag', + 'DeleteSliceFromNodesWhitelist', 'UpdateAddress', 'ResetPassword', + 'AddSliceToNodesWhitelist', 'AddRoleToTagType', 'AddLeases', + 'GetAddresses', 'AddInitScript', 'RebootNode', 'GetPCUTypes', + 'RefreshPeer', 'GetBootMedium', 'UpdateKey', 'UpdatePCU', 'GetSession', + 'AddInterfaceTag', 'UpdatePCUType', 'GetInterfaces', 'SliceExtendedInfo', + 'SliceNodesList', 'DeleteRoleFromTagType', 'DeleteSlice', 'GetSites', + 'DeleteMessage', 'GetSliceFamily', 'GetPlcRelease', 'UpdateTagType', + 'AddSliceInstantiation', 'ResolveSlices', 'GetSlices', + 'DeleteRoleFromPerson', 'GetSessions', 'UpdatePeer', 'VerifyPerson', + 'GetPersonTags', 'DeleteKeyType', 'AddSlice', 'SliceUserAdd', + 'DeleteSession', 'GetMessages', 'DeletePCU', 'GetPeerData', + 'DeletePersonFromSite', 'DeleteTagType', 'GetPCUs', 'UpdateLeases', + 'AddMessage', 'DeletePCUProtocolType', 'DeleteInterfaceTag', + 'AddPersonToSite', 'GetSlivers', 'SliceNodesDel', + 'DeleteAddressTypeFromAddress', 'AddNodeGroup', 'GetSliceTags', + 'DeleteSite', 'GetSiteTags', 'UpdateMessage', 'DeleteSliceFromNodes', + 'SliceRenew', 'UpdatePCUProtocolType', 'DeleteSiteTag', + 'GetPCUProtocolTypes', 'GetEvents', 'GetSliceTicket', 'AddPersonTag', + 'BootGetNodeDetails', 'DeleteInterface', 'DeleteNodeGroup', + 'AddPCUProtocolType', 'BootCheckAuthentication', 'AddSiteTag', + 'AddAddressTypeToAddress', 'DeleteConfFile', 'DeleteInitScript', + 'DeletePerson', 'DeleteIlink', 'DeleteAddressType', 'AddBootState', + 'AuthCheck', 'NotifySupport', 'GetSliceInstantiations', 'AddPCUType', + 'AddPCU', 'AddSession', 'GetEventObjects', 'UpdateSiteTag', + 'UpdateNodeTag', 'AddPerson', 'BlacklistKey', 'UpdateInitScript', + 'AddSliceToNodes', 'RebootNodeWithPCU', 'GetNodeTags', 'GetSliceKeys', + 'GetSliceSshKeys', 'AddNetworkMethod', 'SliceNodesAdd', + 'DeletePersonFromSlice', 'ReportRunlevel', 'GetNetworkTypes', + 'UpdateSite', 'DeleteConfFileFromNodeGroup', 'UpdateNode', + 'DeleteSliceInstantiation', 'DeleteSliceTag', 'BootUpdateNode', + 'UpdatePerson', 'UpdateConfFile', 'SliceUserDel', 'DeleteLeases', + 'AddConfFileToNodeGroup', 'UpdatePersonTag', 'DeleteConfFileFromNode', + 'AddPersonToSlice', 'UnBindObjectFromPeer', 'AddNodeToPCU', + 'GetLeaseGranularity', 'DeletePCUType', 'GetTagTypes', 'GetNodeTypes', + 'UpdateInterfaceTag', 'GetRoles', 'UpdateSlice', 'UpdateSliceTag', + 'AddSliceTag', 'AddNetworkType', 'AddInterface', 'AddAddressType', + 'AddRoleToPerson', 'DeleteNodeType', 'GetLeases', 'UpdateInterface', + 'SliceInfo', 'DeleteAddress', 'SliceTicketGet', 'GetPersons', + 'GetWhitelist', 'AddKeyType', 'UpdateAddressType', 'GetPeerName', + 'DeleteNetworkMethod', 'UpdateIlink', 'AddConfFileToNode', 'GetKeys', + 'DeleteNodeFromPCU', 'GetInterfaceTags', 'GetBootStates', + 'SetInterfaceSens', 'SetNodeLoadm', 'GetInterfaceRate', 'GetNodeLoadw', + 'SetInterfaceKey', 'GetNodeSlices', 'GetNodeLoadm', 'SetSliceVref', + 'GetInterfaceIwpriv', 'SetNodeLoadw', 'SetNodeSerial', + 'GetNodePlainBootstrapfs', 'SetNodeMEMw', 'GetNodeResponse', + 'SetInterfaceRate', 'SetSliceInitscript', 'SetNodeFcdistro', + 'GetNodeLoady', 'SetNodeArch', 'SetNodeKargs', 'SetNodeMEMm', + 'SetNodeBWy', 'SetNodeBWw', 'SetInterfaceSecurityMode', 'SetNodeBWm', + 'SetNodeASType', 'GetNodeKargs', 'GetPersonColumnconf', + 'GetNodeResponsem', 'GetNodeCPUy', 'GetNodeCramfs', 'SetNodeSlicesw', + 'SetPersonColumnconf', 'SetNodeSlicesy', 'GetNodeCPUw', 'GetNodeBWy', + 'GetNodeCPUm', 'GetInterfaceDriver', 'GetNodeLoad', 'GetInterfaceMode', + 'GetNodeSerial', 'SetNodeSlicesm', 'SetNodeLoady', 'GetNodeReliabilityw', + 'SetSliceFcdistro', 'GetNodeReliabilityy', 'SetInterfaceEssid', + 'SetSliceInitscriptCode', 'GetNodeExtensions', 'GetSliceOmfControl', + 'SetNodeCity', 'SetInterfaceIfname', 'SetNodeHrn', 'SetNodeNoHangcheck', + 'GetNodeNoHangcheck', 'GetSliceFcdistro', 'SetNodeCountry', + 'SetNodeKvariant', 'GetNodeKvariant', 'GetNodeMEMy', 'SetInterfaceIwpriv', + 'GetNodeMEMw', 'SetInterfaceBackdoor', 'GetInterfaceFreq', + 'SetInterfaceChannel', 'SetInterfaceNw', 'GetPersonShowconf', + 'GetSliceInitscriptCode', 'SetNodeMEM', 'GetInterfaceEssid', 'GetNodeMEMm', + 'SetInterfaceMode', 'SetInterfaceIwconfig', 'GetNodeSlicesm', 'GetNodeBWm', + 'SetNodePlainBootstrapfs', 'SetNodeRegion', 'SetNodeCPU', 'GetNodeSlicesw', + 'SetNodeBW', 'SetNodeSlices', 'SetNodeCramfs', 'GetNodeSlicesy', + 'GetInterfaceKey', 'GetSliceInitscript', 'SetNodeCPUm', 'SetSliceArch', + 'SetNodeLoad', 'SetNodeResponse', 'GetSliceSliverHMAC', 'GetNodeBWw', + 'GetNodeRegion', 'SetNodeMEMy', 'GetNodeASType', 'SetNodePldistro', + 'GetSliceArch', 'GetNodeCountry', 'SetSliceOmfControl', 'GetNodeHrn', + 'GetNodeCity', 'SetInterfaceAlias', 'GetNodeBW', 'GetNodePldistro', + 'GetSlicePldistro', 'SetNodeASNumber', 'GetSliceHmac', 'SetSliceHmac', + 'GetNodeMEM', 'GetNodeASNumber', 'GetInterfaceAlias', 'GetSliceVref', + 'GetNodeArch', 'GetSliceSshKey', 'GetInterfaceKey4', 'GetInterfaceKey2', + 'GetInterfaceKey3', 'GetInterfaceKey1', 'GetInterfaceBackdoor', + 'GetInterfaceIfname', 'SetSliceSliverHMAC', 'SetNodeReliability', + 'GetNodeCPU', 'SetPersonShowconf', 'SetNodeExtensions', 'SetNodeCPUy', + 'SetNodeCPUw', 'GetNodeResponsew', 'SetNodeResponsey', 'GetInterfaceSens', + 'SetNodeResponsew', 'GetNodeResponsey', 'GetNodeReliability', + 'GetNodeReliabilitym', 'SetNodeResponsem', 'SetInterfaceDriver', + 'GetInterfaceSecurityMode', 'SetNodeDeployment', 'SetNodeReliabilitym', + 'GetNodeFcdistro', 'SetInterfaceFreq', 'GetInterfaceNw', + 'SetNodeReliabilityy', 'SetNodeReliabilityw', 'GetInterfaceIwconfig', + 'SetSlicePldistro', 'SetSliceSshKey', 'GetNodeDeployment', + 'GetInterfaceChannel', 'SetInterfaceKey2', 'SetInterfaceKey3', + 'SetInterfaceKey1', 'SetInterfaceKey4']) + + _required_methods = set() + + def __init__(self, username = None, password = None, session_key = None, + proxy = None, + hostname = "www.planet-lab.eu", + urlpattern = "https://%(hostname)s:443/PLCAPI/", + local_peer = "PLE"): + + self._blacklist = set() + self._reserved = set() + self._nodes_cache = None + + if session_key is not None: + self.auth = dict(AuthMethod='session', session=session_key) + elif username is not None and password is not None: + self.auth = dict(AuthMethod='password', Username=username, AuthString=password) + else: + self.auth = dict(AuthMethod='anonymous') + + self._local_peer = local_peer + self._url = urlpattern % {'hostname':hostname} + + if (proxy is not None): + import urllib2 + class HTTPSProxyTransport(xmlrpclib.Transport): + def __init__(self, proxy, use_datetime=0): + opener = urllib2.build_opener(urllib2.ProxyHandler({"https" : proxy})) + xmlrpclib.Transport.__init__(self, use_datetime) + self.opener = opener + + def request(self, host, handler, request_body, verbose=0): + req = urllib2.Request('https://%s%s' % (host, handler), request_body) + req.add_header('User-agent', self.user_agent) + self.verbose = verbose + return self.parse_response(self.opener.open(req)) + + self._proxy_transport = lambda : HTTPSProxyTransport(proxy) + else: + self._proxy_transport = lambda : None + + self.threadlocal = threading.local() + + @property + def api(self): + # Cannot reuse same proxy in all threads, py2.7 is not threadsafe + return xmlrpclib.ServerProxy( + self._url , + transport = self._proxy_transport(), + allow_none = True) + + @property + def mcapi(self): + try: + return self.threadlocal.mc + except AttributeError: + return self.api + + def test(self): + import warnings + + # validate XMLRPC server checking supported API calls + methods = set(_retry(self.mcapi.system.listMethods)()) + if self._required_methods - methods: + warnings.warn("Unsupported REQUIRED methods: %s" % ( + ", ".join(sorted(self._required_methods - methods)), ) ) + return False + + if self._expected_methods - methods: + warnings.warn("Unsupported EXPECTED methods: %s" % ( + ", ".join(sorted(self._expected_methods - methods)), ) ) + + try: + # test authorization + network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth) + except (xmlrpclib.ProtocolError, xmlrpclib.Fault),e: + warnings.warn(str(e)) + + return True + + @property + def network_types(self): + try: + return self._network_types + except AttributeError: + self._network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth) + return self._network_types + + @property + def peer_map(self): + try: + return self._peer_map + except AttributeError: + peers = _retry(self.mcapi.GetPeers)(self.auth, {}, ['shortname','peername','peer_id']) + + self._peer_map = dict( + (peer['shortname'], peer['peer_id']) + for peer in peers + ) + + self._peer_map.update( + (peer['peername'], peer['peer_id']) + for peer in peers + ) + + self._peer_map.update( + (peer['peer_id'], peer['shortname']) + for peer in peers + ) + + self._peer_map[None] = self._local_peer + return self._peer_map + + def get_node_flavour(self, node): + """ + Returns detailed information on a given node's flavour, + i.e. its base installation. + + This depends on the global PLC settings in the PLC_FLAVOUR area, + optionnally overridden by any of the following tags if set on that node: + 'arch', 'pldistro', 'fcdistro', 'deployment', 'extensions' + + Params: + + * node : int or string + - int, Node identifier + - string, Fully qualified hostname + + Returns: + + struct + * extensions : array of string, extensions to add to the base install + * fcdistro : string, the fcdistro this node should be based upon + * nodefamily : string, the nodefamily this node should be based upon + * plain : boolean, use plain bootstrapfs image if set (for tests) + """ + if not isinstance(node, (str, int, long)): + raise ValueError, "Node must be either a non-unicode string or an int" + return _retry(self.mcapi.GetNodeFlavour)(self.auth, node) + + def get_nodes(self, node_id_or_name = None, fields = None, **kw): + """ + Returns an array of structs containing details about nodes. + If node_id_or_name is specified and is an array of node identifiers + or hostnames, or the filters keyword argument with struct of node + attributes, or node attributes by keyword argument, + only nodes matching the filter will be returned. + + If fields is specified, only the specified details will be returned. + NOTE that if fields is unspecified, the complete set of native fields are + returned, which DOES NOT include tags at this time. + + Some fields may only be viewed by admins. + + Special params: + + fields: an optional list of fields to retrieve. The default is all. + + filters: an optional mapping with custom filters, which is the only + way to support complex filters like negation and numeric comparisons. + + peer: a string (or sequence of strings) with the name(s) of peers + to filter - or None for local nodes. + """ + if fields is not None: + fieldstuple = (fields,) + else: + fieldstuple = () + + if node_id_or_name is not None: + return _retry(self.mcapi.GetNodes)(self.auth, node_id_or_name, *fieldstuple) + else: + filters = kw.pop('filters',{}) + + if 'peer' in kw: + peer = kw.pop('peer') + + name_to_id = self.peer_map.get + + if hasattr(peer, '__iter__'): + # we can't mix local and external nodes, so + # split and re-issue recursively in that case + if None in peer or self._local_peer in peer: + if None in peer: + peer.remove(None) + + if self._local_peer in peer: + peer.remove(self._local_peer) + + return ( + self.get_nodes(node_id_or_name, fields, + filters = filters, peer=peer, **kw) + \ + self.get_nodes(node_id_or_name, fields, + filters = filters, peer=None, **kw) + ) + else: + peer_filter = map(name_to_id, peer) + + elif peer is None or peer == self._local_peer: + peer_filter = None + else: + peer_filter = name_to_id(peer) + + filters['peer_id'] = peer_filter + + filters.update(kw) + + if not filters and not fieldstuple: + if not self._nodes_cache: + self._nodes_cache = _retry(self.mcapi.GetNodes)(self.auth) + return self._nodes_cache + + return _retry(self.mcapi.GetNodes)(self.auth, filters, *fieldstuple) + + def get_node_tags(self, node_tag_id = None, fields = None, **kw): + if fields is not None: + fieldstuple = (fields,) + else: + fieldstuple = () + + if node_tag_id is not None: + return _retry(self.mcapi.GetNodeTags)(self.auth, node_tag_id, + *fieldstuple) + else: + filters = kw.pop('filters',{}) + filters.update(kw) + return _retry(self.mcapi.GetNodeTags)(self.auth, filters, + *fieldstuple) + + def get_slice_tags(self, slice_tag_id = None, fields = None, **kw): + if fields is not None: + fieldstuple = (fields,) + else: + fieldstuple = () + + if slice_tag_id is not None: + return _retry(self.mcapi.GetSliceTags)(self.auth, slice_tag_id, + *fieldstuple) + else: + filters = kw.pop('filters',{}) + filters.update(kw) + return _retry(self.mcapi.GetSliceTags)(self.auth, filters, + *fieldstuple) + + def get_interfaces(self, interface_id_or_ip = None, fields = None, **kw): + if fields is not None: + fieldstuple = (fields,) + else: + fieldstuple = () + + if interface_id_or_ip is not None: + return _retry(self.mcapi.GetInterfaces)(self.auth, + interface_id_or_ip, *fieldstuple) + else: + filters = kw.pop('filters',{}) + filters.update(kw) + return _retry(self.mcapi.GetInterfaces)(self.auth, filters, + *fieldstuple) + + def get_slices(self, slice_id_or_name = None, fields = None, **kw): + if fields is not None: + fieldstuple = (fields,) + else: + fieldstuple = () + + if slice_id_or_name is not None: + return _retry(self.mcapi.GetSlices)(self.auth, slice_id_or_name, + *fieldstuple) + else: + filters = kw.pop('filters',{}) + filters.update(kw) + return _retry(self.mcapi.GetSlices)(self.auth, filters, + *fieldstuple) + + def update_slice(self, slice_id_or_name, **kw): + return _retry(self.mcapi.UpdateSlice)(self.auth, slice_id_or_name, kw) + + def start_multicall(self): + self.threadlocal.mc = xmlrpclib.MultiCall(self.mcapi) + + def finish_multicall(self): + mc = self.threadlocal.mc + del self.threadlocal.mc + return _retry(mc)() + + def get_slice_nodes(self, slicename): + return self.get_slices(slicename, ['node_ids'])[0]['node_ids'] + + def add_slice_nodes(self, slicename, nodes = None): + self.update_slice(slicename, nodes = nodes) + + def get_node_info(self, node_id): + self.start_multicall() + info = self.get_nodes(node_id) + tags = self.get_node_tags(node_id=node_id, fields=('tagname','value')) + info, tags = self.finish_multicall() + return info, tags + + def get_slice_id(self, slicename): + slice_id = None + slices = self.get_slices(slicename, fields=('slice_id',)) + if slices: + slice_id = slices[0]['slice_id'] + + # If it wasn't found, don't remember this failure, keep trying + return slice_id + + def get_slice_vnet_sys_tag(self, slicename): + slicetags = self.get_slice_tags( + name = slicename, + tagname = 'vsys_vnet', + fields=('value',)) + + if slicetags: + return slicetags[0]['value'] + else: + return None + + def blacklist_host(self, hostname): + self._blacklist.add(hostname) + + def blacklisted(self): + return self._blacklist + + def unblacklist_host(self, hostname): + del self._blacklist[hostname] + + def reserve_host(self, hostname): + self._reserved.add(hostname) + + def reserved(self): + return self._reserved + + def unreserve_host(self, hostname): + del self._reserved[hostname] + + +class PLCAPIFactory(object): + """ + .. note:: + + It allows PlanetLab RMs sharing a same slice, to use a same plcapi instance, + and to sincronize blacklisted and reserved hosts. + + """ + # use lock to avoid concurrent access to the Api list at the same times by 2 different threads + _lock = threading.Lock() + _apis = dict() + + @classmethod + def get_api(cls, slicename, pl_pass, pl_host, + pl_ptn = "https://%(hostname)s:443/PLCAPI/", + proxy = None): + """ Get existing PLCAPI instance + + :param slicename: Planelab slice name + :type slicename: str + :param pl_pass: Planetlab password (used for web login) + :type pl_pass: str + :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu") + :type pl_host: str + :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/) + :type pl_ptn: str + :param proxy: Proxy service url + :type pl_ptn: str + """ + if slice and pl_pass and pl_host: + key = cls._make_key(slicename, pl_host) + with cls._lock: + api = cls._apis.get(key) + if not api: + api = cls.create_api(slicename, pl_pass, pl_host, pl_ptn, proxy) + return api + return None + + @classmethod + def create_api(cls, slicename, pl_pass, pl_host, + pl_ptn = "https://%(hostname)s:443/PLCAPI/", + proxy = None): + """ Create an PLCAPI instance + + :param slicename: Planelab slice name + :type slicename: str + :param pl_pass: Planetlab password (used for web login) + :type pl_pass: str + :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu") + :type pl_host: str + :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/) + :type pl_ptn: str + :param proxy: Proxy service url + :type pl_ptn: str + """ + api = PLCAPI( + username = slicename, + password = pl_pass, + hostname = pl_host, + urlpattern = pl_ptn, + proxy = proxy + ) + key = cls._make_key(slicename, pl_host) + cls._apis[key] = api + return api + + @classmethod + def _make_key(cls, *args): + """ Hash the credentials in order to create a key + + :param args: list of arguments used to create the hash (user, host, port, ...) + :type args: list of args + + """ + skey = "".join(map(str, args)) + return hashlib.md5(skey).hexdigest() + diff --git a/test/resources/planetlab/plcapi.py b/test/resources/planetlab/plcapi.py new file mode 100644 index 00000000..42ae8711 --- /dev/null +++ b/test/resources/planetlab/plcapi.py @@ -0,0 +1,49 @@ +""" + NEPI, a framework to manage network experiments + Copyright (C) 2013 INRIA + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +""" + +#!/usr/bin/env python +from nepi.resources.planetlab.plcapi import PLCAPIFactory +from nepi.util.sshfuncs import RUNNING, FINISHED + +import os +import unittest + +class PlanetlabAPITestCase(unittest.TestCase): + def setUp(self): + self.slicename = 'inria_nepi' + self.host1 = 'nepi2.pl.sophia.inria.fr' + self.host2 = 'nepi5.pl.sophia.inria.fr' + + + def test_list_hosts(self): + slicename = os.environ.get('PL_USER') + pl_pass = os.environ.get('PL_PASS') + pl_url = "nepiplc.pl.sophia.inria.fr" + + plapi = PLCAPIFactory.get_api(slicename, pl_pass, pl_url) + + plapi.test() + + nodes = plapi.get_nodes() + self.assertTrue(len(nodes)>0) + + +if __name__ == '__main__': + unittest.main() + -- 2.43.0