From: Alina Quereilhac Date: Tue, 10 Jan 2012 09:56:33 +0000 (+0100) Subject: Making node.py code independent from the plcapi implementation. X-Git-Tag: nepi-3.0.0~163^2~44 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=7443c3354b8d9baba337660899fecd7f2eaae310;p=nepi.git Making node.py code independent from the plcapi implementation. --- diff --git a/src/nepi/testbeds/planetlab/execute.py b/src/nepi/testbeds/planetlab/execute.py index 67b76d24..8574112f 100644 --- a/src/nepi/testbeds/planetlab/execute.py +++ b/src/nepi/testbeds/planetlab/execute.py @@ -48,6 +48,10 @@ class TestbedController(testbed_impl.TestbedController): self._just_provisioned = set() self._load_blacklist() + + self._sliceapi = None + self._plcapi = None + self._slice_id = None self._logger = logging.getLogger('nepi.testbeds.planetlab') @@ -58,41 +62,41 @@ class TestbedController(testbed_impl.TestbedController): return self._home_directory @property - def plapi(self): - if not hasattr(self, '_plapi'): + def plcapi(self): + if not self._plcapi: import plcapi - - if self.authUser: - self._plapi = plcapi.PLCAPI( - username = self.authUser, - password = self.authString, - hostname = self.plcHost, - urlpattern = self.plcUrl + self._plcapi = plcapi.plcapi( + self.authUser, + self.authString, + self.plcHost, + self.plcUrl ) + return self._plcapi + + @property + def sliceapi(self): + if not self._sliceapi: + if not self.sfa: + self._sliceapi = self.plcapi else: - # anonymous access - may not be enough for much - self._plapi = plcapi.PLCAPI() - return self._plapi + import sfiapi + self._sliceapi = sfiapi.sfiapi() + return self._sliceapi @property def slice_id(self): - if not hasattr(self, '_slice_id'): - slices = self.plapi.GetSlices(self.slicename, fields=('slice_id',)) - if slices: - self._slice_id = slices[0]['slice_id'] - else: - # If it wasn't found, don't remember this failure, keep trying - return None + if not self._slice_id: + self._slice_id = self.plcapi.GetSliceId(self.slicename) return self._slice_id @property def vsys_vnet(self): if not hasattr(self, '_vsys_vnet'): self._vsys_vnet = plutil.getVnet( - self.plapi, + self.plcapi, self.slicename) return self._vsys_vnet - + def _load_blacklist(self): blpath = environ.homepath('plblacklist') @@ -144,7 +148,12 @@ class TestbedController(testbed_impl.TestbedController): get_attribute_value("p2pDeployment") self.dedicatedSlice = self._attributes.\ get_attribute_value("dedicatedSlice") - + self.sfa = self._attributes.\ + get_attribute_value("sfa") + if self.sfa: + self._slice_id = self._attributes.\ + get_attribute_value("sliceHrn") + if not self.slicename: raise RuntimeError, "Slice not set" if not self.authUser: @@ -301,9 +310,9 @@ class TestbedController(testbed_impl.TestbedController): def do_provisioning(self): if self._to_provision: # Add new nodes to the slice - cur_nodes = self.plapi.GetSlices(self.slicename, ['node_ids'])[0]['node_ids'] + cur_nodes = self.sliceapi.GetSliceNodes(self.slicename) new_nodes = list(set(cur_nodes) | self._to_provision) - self.plapi.UpdateSlice(self.slicename, nodes=new_nodes) + self.sliceapi.AddSliceNodes(self.slicename, nodes=new_nodes) # cleanup self._just_provisioned = self._to_provision @@ -694,8 +703,10 @@ class TestbedController(testbed_impl.TestbedController): finally: self.recovering = True - def _make_generic(self, parameters, kind): - app = kind(self.plapi) + def _make_generic(self, parameters, kind, **kwargs): + args = dict({'api': self.plcapi}) + args.update(kwargs) + app = kind(**args) app.testbed = weakref.ref(self) # Note: there is 1-to-1 correspondence between attribute names @@ -714,7 +725,8 @@ class TestbedController(testbed_impl.TestbedController): return app def _make_node(self, parameters): - node = self._make_generic(parameters, self._node.Node) + args = dict({'sliceapi': self.sliceapi}) + node = self._make_generic(parameters, self._node.Node, **args) node.enable_cleanup = self.dedicatedSlice return node diff --git a/src/nepi/testbeds/planetlab/metadata.py b/src/nepi/testbeds/planetlab/metadata.py index d099a8c5..09800a04 100644 --- a/src/nepi/testbeds/planetlab/metadata.py +++ b/src/nepi/testbeds/planetlab/metadata.py @@ -1621,6 +1621,20 @@ factories_info = dict({ }) testbed_attributes = dict({ + "slice_hrn": dict({ + "name": "sliceHrn", + "help": "The hierarchical Resource Name (HRN) for the PlanetLab slice.", + "type": Attribute.STRING, + "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable | Attribute.NoDefaultValue, + "validation_function": validation.is_string + }), + "sfa": dict({ + "name": "sfa", + "help": "Activates the use of SFA for node reservation.", + "type": Attribute.BOOL, + "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable | Attribute.NoDefaultValue, + "validation_function": validation.is_bool + }), "slice": dict({ "name": "slice", "help": "The name of the PlanetLab slice to use", diff --git a/src/nepi/testbeds/planetlab/node.py b/src/nepi/testbeds/planetlab/node.py index 4b1cf0c0..10fac055 100644 --- a/src/nepi/testbeds/planetlab/node.py +++ b/src/nepi/testbeds/planetlab/node.py @@ -88,10 +88,11 @@ class Node(object): minLoad = _castproperty(float, '_minLoad') maxLoad = _castproperty(float, '_maxLoad') - def __init__(self, api=None): + def __init__(self, api=None, sliceapi=None): if not api: api = plcapi.PLCAPI() self._api = api + self._sliceapi = sliceapi # Attributes self.hostname = None @@ -197,7 +198,7 @@ class Node(object): extra['peer'] = self.site candidates = set(map(operator.itemgetter('node_id'), - self._api.GetNodes(filters=basefilters, fields=fields, **extra))) + self._sliceapi.GetNodes(filters=basefilters, fields=fields, **extra))) # filter by tag, one tag at a time applicable = self.applicable_filters @@ -212,18 +213,18 @@ class Node(object): tagfilter['node_id'] = list(candidates) candidates &= set(map(operator.itemgetter('node_id'), - self._api.GetNodeTags(filters=tagfilter, fields=fields))) + self._sliceapi.GetNodeTags(filters=tagfilter, fields=fields))) # filter by vsys tags - special case since it doesn't follow # the usual semantics if self.required_vsys: newcandidates = collections.defaultdict(set) - vsys_tags = self._api.GetNodeTags( + vsys_tags = self._sliceapi.GetNodeTags( tagname='vsys', node_id = list(candidates), fields = ['node_id','value']) - + vsys_tags = map( operator.itemgetter(['node_id','value']), vsys_tags) @@ -245,7 +246,7 @@ class Node(object): filters = basefilters.copy() filters['node_id'] = list(candidates) ifaces = dict(map(operator.itemgetter('node_id','interface_ids'), - self._api.GetNodes(filters=basefilters, fields=('node_id','interface_ids')) )) + self._sliceapi.GetNodes(filters=basefilters, fields=('node_id','interface_ids')) )) # filter candidates by interface count if self.min_num_external_ifaces is not None and self.max_num_external_ifaces is not None: @@ -340,11 +341,11 @@ class Node(object): tagnames = [ tagname % replacements for tagname, weight, default in self.RATE_FACTORS ] - taginfo = self._api.GetNodeTags( + taginfo = self._sliceapi.GetNodeTags( node_id=list(nodes), tagname=tagnames, fields=('node_id','tagname','value')) - + unpack = operator.itemgetter('node_id','tagname','value') for value in taginfo: node, tagname, value = unpack(value) @@ -361,10 +362,7 @@ class Node(object): def fetch_node_info(self): orig_attrs = {} - self._api.StartMulticall() - info = self._api.GetNodes(self._node_id) - tags = self._api.GetNodeTags(node_id=self._node_id, fields=('tagname','value')) - info, tags = self._api.FinishMulticall() + info, tags = self._sliceapi.GetNodeInfo(self._node_id) info = info[0] tags = dict( (t['tagname'],t['value']) diff --git a/src/nepi/testbeds/planetlab/plcapi.py b/src/nepi/testbeds/planetlab/plcapi.py index c4aa111c..747e549a 100644 --- a/src/nepi/testbeds/planetlab/plcapi.py +++ b/src/nepi/testbeds/planetlab/plcapi.py @@ -16,6 +16,7 @@ def _retry(fn): return rv class PLCAPI(object): + _expected_methods = set( ['AddNodeTag', 'AddConfFile', 'DeletePersonTag', 'AddNodeType', 'DeleteBootState', 'SliceListNames', 'DeleteKey', 'SliceGetTicket', 'SliceUsersList', 'SliceUpdate', 'GetNodeGroups', 'SliceCreate', 'GetNetworkMethods', 'GetNodeFlavour', @@ -298,3 +299,39 @@ class PLCAPI(object): mc = self.threadlocal.mc del self.threadlocal.mc return _retry(mc)() + + def GetSliceNodes(self, slicename): + return self.GetSlices(slicename, ['node_ids'])[0]['node_ids'] + + def AddSliceNodes(self, slicename, nodes = None): + self.UpdateSlice(slicename, nodes = nodes) + + def GetNodeInfo(self, node_id): + self.StartMulticall() + info = self.GetNodes(node_id) + tags = self.GetNodeTags(node_id=node_id, fields=('tagname','value')) + info, tags = self.FinishMulticall() + return info, tags + + def GetSliceId(self, slicename): + slice_id = None + slices = self.GetSlices(slicename, fields=('slice_id',)) + if slices: + slice_id = slices[0]['slice_id'] + return slice_id + +def plcapi(auth_user, auth_string, plc_host, plc_url): + api = None + if auth_user: + api = PLCAPI( + username = auth_user, + password = auth_string, + hostname = plc_host, + urlpattern = plc_url + ) + else: + # anonymous access - may not be enough for much + api = PLCAPI() + return api + + diff --git a/src/nepi/testbeds/planetlab/sfiapi.py b/src/nepi/testbeds/planetlab/sfiapi.py new file mode 100644 index 00000000..1082a38e --- /dev/null +++ b/src/nepi/testbeds/planetlab/sfiapi.py @@ -0,0 +1,19 @@ +class SFIAPI(object): + def __init__(self): + self._slice_nodes = dict() + self._all_nodes = dict() + + def GetSliceNodes(self, slicename): + return None + + def AddSliceNodes(self, slicename, nodes=None): + pass + + def GetNodeTags(self, nodeTagId=None, fields=None, **kw): + pass + + def GetNodes(self, filters=basefilters, fields=('node_id','interface_ids')) )) + +def sfiapi(): + return SFIAPI() + diff --git a/src/nepi/util/parser/sfa.py b/src/nepi/util/parser/sfa.py new file mode 100644 index 00000000..4ccb69a1 --- /dev/null +++ b/src/nepi/util/parser/sfa.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +from xml.dom import minidom + +import sys + +def xmlencode(s): + if isinstance(s, str): + rv = s.decode("latin1") + elif not isinstance(s, unicode): + rv = unicode(s) + else: + rv = s + return rv.replace(u'\x00',u'�') + +def xmldecode(s): + return s.replace(u'�',u'\x00').encode("utf8") + +def get_text(p_tag, name): + tags = p_tag.getElementsByTagName(name) + if not tags: + return "" + return xmldecode(tags[0].childNodes[0].nodeValue) + +def get_attribute(p_tag, name): + return xmldecode(p_tag.getAttribute(name)) + + +class SFAResourcesParser(object): + def from_xml(self, xml): + data = dict() + doc = minidom.parseString(xml) + rspec_tag = doc.getElementsByTagName("RSpec")[0] + network_tags = rspec_tag.getElementsByTagName("network") + for network_tag in network_tags: + if network_tag.nodeType == doc.ELEMENT_NODE: + node_data = self.nodes_from_xml(doc, network_tag) + data.update(node_data) + return data + + def nodes_from_xml(self, doc, network_tag): + nodes_data = dict() + network_name = get_attribute(network_tag, "name") + node_tags = network_tag.getElementsByTagName('node') + for node_tag in node_tags: + if node_tag.nodeType == doc.ELEMENT_NODE: + node_data = dict() + node_data['network_name'] = network_name + node_name = get_attribute(node_tag, 'component_name') + nodes_data[node_name] = node_data + for name in ['component_id', 'component_manager_id', + 'boot_state', 'component_name', 'site_id']: + node_data[name] = get_attribute(node_tag, name) + location_tag = node_tag.getElementsByTagName('location') + if location_tag: + for name in ['longitud' , 'latitude']: + node_data[name] = get_attribute(location_tag[0], name) + for name in ['hostname', 'pldistro', 'arch', 'fcdistro', + 'stype', 'reliabilityw', 'loadm', 'cpuy', 'cpum', + 'slicesm', 'slicesw', 'cpuw', 'loady', 'memy', + 'memw', 'reliabilityy', 'reliability', 'reliabilitym', + 'responsey', 'bww', 'memem', 'bwm', 'slicey', 'responsem', + 'response', 'loadw', 'country', 'load', 'mem', 'slices', + 'region', 'asnumber', 'bw', 'hrn', 'city', 'responsew', + 'bwy', 'cpu']: + node_data[name] = get_text(node_tag, name) + iface_tags = node_tag.getElementsByTagName('interface') + ifaces_data = dict() + for iface_tag in iface_tags: + if iface_tag.nodeType == doc.ELEMENT_NODE: + for name in ['component_id', 'ipv4']: + ifaces_data[name] = get_attribute(iface_tag, name) + node_data['interfaces'] = ifaces_data + return nodes_data + +""" +if __name__ == "__main__": + path = sys.argv[1] + fd = open(path, 'r') + xml = fd.read() + fd.close() + p = SFAResourcesParser() + data = p.from_xml(xml) + print data.keys() +"""