From a603d9ef42f4a866f4aebea4260fa4deb995477b Mon Sep 17 00:00:00 2001 From: Lucia Guevgeozian Odizzio Date: Wed, 25 Sep 2013 10:26:06 +0200 Subject: [PATCH] Adding api to support manifold queries and the unittest for the api --- src/nepi/util/manifoldapi.py | 272 +++++++++++++++++++++++++++++++++++ test/util/manifoldapi.py | 199 +++++++++++++++++++++++++ 2 files changed, 471 insertions(+) create mode 100644 src/nepi/util/manifoldapi.py create mode 100755 test/util/manifoldapi.py diff --git a/src/nepi/util/manifoldapi.py b/src/nepi/util/manifoldapi.py new file mode 100644 index 00000000..81671059 --- /dev/null +++ b/src/nepi/util/manifoldapi.py @@ -0,0 +1,272 @@ +# +# NEPI, a framework to manage network experiments +# Copyright (C) 2013 INRIA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Lucia Guevgeozian Odizzio + +import xmlrpclib +import hashlib +import threading + + +class MANIFOLDAPI(object): + """ + API to query different data platforms as SFA, TopHat, OML Central Server, + using Manifold Framework, the backend of MySlice. + """ + def __init__(self, username, password, hostname, urlpattern): + + self.auth_pwd = dict(AuthMethod='password', Username=username, + AuthString=password) + self._url = urlpattern % {'hostname':hostname} + self.lock = threading.Lock() + self.auth = self.get_session_key() + + @property + def api(self): + return xmlrpclib.Server(self._url, allow_none = True) + + def get_session_key(self): + """ + Retrieves the session key, in order to use the same session for + queries. + """ + query = {'timestamp' : 'now', 'object': 'local:session', + 'filters' : [], 'fields' : [], 'action' : 'create'} + + session = self.api.forward(self.auth_pwd, query) + + if not session['value']: + msg = "Can not authenticate in Manifold API" + raise RuntimeError, msg + + session_key = session['value'][0]['session'] + return dict(AuthMethod='session', session=session_key) + + def get_resource_info(self, filters=None, fields=None): + """ + Create and execute the Manifold API Query to get the resources + according fields and filters. + :param filters: resource's constraints for the experiment + :type filters: dict + :param fields: desire fields in the result of the query + :type fields: list + """ + query = {'action' : 'get', 'object' : 'resource'} + + if filters: + filters = self._map_attr_to_resource_filters(filters) + + qfilters = list() + for filtername, filtervalue in filters.iteritems(): + newfilter = [filtername, "==", filtervalue] + qfilters.append(newfilter) + + query['filters'] = qfilters + + if fields: + fields = self._check_valid_fields(fields) + + if fields: + query['fields'] = fields + + return self.api.forward(self.auth, query)['value'] + + def get_resource_urn(self, filters=None): + """ + Retrieves the resources urn of the resources matching filters. + """ + return self.get_resource_info(filters, 'urn') + + def get_slice_resources(self, slicename): + """ + Retrieves resources attached to user's slice. + return value: list of resources' urn + """ + result = [] + query = {'action' : 'get', 'object' : 'resource', + 'filters' : [['slice','==', slicename]], + 'fields' : ['urn']} + + with self.lock: + value = self.api.forward(self.auth, query)['value'] + + for resource in value: + result.append(resource['urn']) + + return result + + def add_resource_to_slice(self, slicename, resource_urn): + """ + Add resource to user's slice. The query needs to specify the new + resource plus the previous resources already in the slice. + """ + resources = self.get_slice_resources(slicename) + resources.append(resource_urn) + + urn_list = list() + for r in resources: + urn_dict = dict() + urn_dict['urn'] = r + urn_list.append(urn_dict) + + query = {'action' : 'update', 'object' : 'slice', + 'filters' : [['slice_hrn','==', slicename]], + 'params' : {'resource' : urn_list}} + + with self.lock: + self.api.forward(self.auth, query) + + resources = self.get_slice_resources(slicename) + if resource_urn in resources: + return True + else: + msg = "Failed while trying to add %s to slice" % resource_urn + print msg + # check how to do warning + return False + + def remove_resource_from_slice(self, slicename, resource_urn): + """ + Remove resource from user's slice. The query needs to specify the list + of previous resources in the slice without the one to be remove. + """ + resources = self.get_slice_resources(slicename) + resources.remove(resource_urn) + + urn_list = list() + for r in resources: + urn_dict = dict() + urn_dict['urn'] = r + urn_list.append(urn_dict) + + query = {'action' : 'update', 'object' : 'slice', + 'filters' : [['slice_hrn','==', slicename]], + 'params' : {'resource' : urn_list}} + + with self.lock: + self.api.forward(self.auth, query) + + resources = self.get_slice_resources(slicename) + if resource_urn not in resources: + return True + else: + msg = "Failed while trying to remove %s to slice" % resource_urn + # check how to do warning + return False + + def _get_metadata(self): + """ + This method is useful to retrive metadata from different platforms + in order to update fields and possible filters. + """ + query = {'action' : 'get', 'object' : 'local:object', + 'filters' : [['table','=','resource']]} + + res = self.api.forward(self.auth, query) + + valid_fields = list() + for i in res['value'][0]['column']: + valid_fields.append(i['name']) + + return valid_fields + + def _map_attr_to_resource_filters(self, filters): + """ + Depending on the object used for the Manifold query, the filters and + fields can change its sintaxis. A resource field in a slice object + query adds 'resource.' to the field. Other changes don't follow any + particular convention. + """ + #TODO: find out useful filters + attr_to_filter = { + 'hostname' : 'hostname', + 'longitude' : 'longitude', + 'latitude' : 'latitude', + 'network' : 'network', + 'component_id' : 'component_id' + } + + mapped_filters = dict() + for filtername, filtervalue in filters.iteritems(): + if attr_to_filter[filtername]: + new_filtername = attr_to_filter[filtername] + mapped_filters[new_filtername] = filtervalue + + return mapped_filters + + def _check_valid_fields(self, fields): + """ + The fields can be a predefine set, define in the Manifold metadata. + """ + valid_fields = self._get_metadata() + + if not isinstance(fields, list): + fields = [fields] + + for field in fields: + if field not in valid_fields: + fields.remove(field) + #self.warn(" Invalid Manifold field or filter ") + + return fields + + +class MANIFOLDAPIFactory(object): + """ + API Factory to manage a map of MANIFOLDAPI instances as key-value pairs, it + instanciate a single instance per key. The key represents the same SFA, + MF (ManiFold) credentials. + """ + + _lock = threading.Lock() + _apis = dict() + + @classmethod + def get_api(cls, username, password, + #hostname = "manifold.pl.sophia.inria.fr", + hostname ="test.myslice.info", + urlpattern = "http://%(hostname)s:7080"): + """ + :param username: Manifold user (also used for MySlice web login) + :type username: str + :param password: Manifold password (also used for MySlice web login) + :type password: str + :param hostname: Hostname of the Manifold API to query SFA, TopHat, etc + :type hostname: str + :param urlpattern: Url of the Manifold API to query SFA, TopHat, etc + :type urlpattern: str + """ + + if username and password: + key = cls.make_key(username, password) + with cls._lock: + api = cls._apis.get(key) + + if not api: + api = MANIFOLDAPI(username, password, hostname, urlpattern) + cls._apis[key] = api + + return api + + return None + + @classmethod + def make_key(cls, *args): + skey = "".join(map(str, args)) + return hashlib.md5(skey).hexdigest() + + diff --git a/test/util/manifoldapi.py b/test/util/manifoldapi.py new file mode 100755 index 00000000..e8a6ddcd --- /dev/null +++ b/test/util/manifoldapi.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# +# NEPI, a framework to manage network experiments +# Copyright (C) 2013 INRIA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Lucia Guevgeozian + +# NOTE: Manifold API can not be tested yet with OMF nodes because there +# no OMF platforms connected currently. + +from nepi.util.manifoldapi import MANIFOLDAPIFactory +from nepi.util.manifoldapi import MANIFOLDAPI + +import unittest +from multiprocessing import Process + +class MFAPIFactoryTestCase(unittest.TestCase): + """ + Test for the Manifold API Factory. Check that the same instance is used + when credentials match. + """ + + def test_factory(self): + """ + Check that the same API instance is used when the credentials match. + """ + username="lucia.guevgeozian_odizzio@inria.fr" + password="demo" + api1 = MANIFOLDAPIFactory.get_api(username, password) + api2 = MANIFOLDAPIFactory.get_api(username, password) + + self.assertIsInstance(api1, MANIFOLDAPI) + self.assertIsInstance(api2, MANIFOLDAPI) + self.assertEquals(api1, api2) + + +class MANIFOLDAPITestCase(unittest.TestCase): + + def setUp(self): + self.username="lucia.guevgeozian_odizzio@inria.fr" + self.password="demo" + self.slicename = "ple.inria.sfatest" + self.api = MANIFOLDAPIFactory.get_api(self.username, self.password) + + def test_get_resource_info(self): + """ + Check that the API retrieves the right set of info acoridng to the + filters and fields defined. + """ + + filters = dict() + filters['hostname'] = 'planetlab2.tlm.unavarra.es' + + r_info = self.api.get_resource_info(filters=filters) + hostname = r_info[0]['hostname'] + self.assertEquals(hostname, 'planetlab2.tlm.unavarra.es') + + # query with 2 filters + filters['network'] = 'ple' + + r_info = self.api.get_resource_info(filters=filters) + hostname = r_info[0]['hostname'] + self.assertEquals(hostname, 'planetlab2.tlm.unavarra.es') + + # query with fields only, without filters + fields = ['latitude','longitude'] + + r_info = self.api.get_resource_info(fields=fields) + value = r_info[10] + self.assertEquals(value.keys(), fields) + + # query with 2 filters and 2 fields + r_info = self.api.get_resource_info(filters, fields) + value = r_info[0] + result = {'latitude': '42.7993', 'longitude': '-1.63544'} + self.assertEquals(value, result) + + # query with filters where the AND should be zero resources + filters['network'] = 'omf' + + r_info = self.api.get_resource_info(filters, fields) + self.assertEquals(r_info, []) + + def test_fail_if_invalid_field(self): + """ + Check that the invalid field is not used for the query + """ + filters = dict() + filters['network'] = 'omf' + filters['component_id'] = 'urn:publicid:IDN+omf:nitos+node+node001' + fields = ['arch'] + + r_info = self.api.get_resource_info(filters, fields) + value = r_info[0] + self.assertNotEqual(len(value), 1) + + def test_get_slice_resources(self): + """ + Test that the API retrives the resources in the user's slice. + The slice used is ple.inria.sfatest, change slice and nodes in order + for the test not to fail. + """ + resources = self.api.get_slice_resources(self.slicename) + + result = ['urn:publicid:IDN+ple:lilleple+node+node2pl.planet-lab.telecom-lille1.eu', + 'urn:publicid:IDN+ple:uttple+node+planetlab2.utt.fr', + 'urn:publicid:IDN+ple:lilleple+node+node1pl.planet-lab.telecom-lille1.eu'] + + self.assertEquals(resources, result) + + def test_update_resources_from_slice(self): + """ + Test that the nodes are correctly added to the user's slice. + Test that the nodes are correctly removed from the user's slice. + """ + new_resource = 'urn:publicid:IDN+ple:unavarraple+node+planetlab2.tlm.unavarra.es' + self.api.add_resource_to_slice(self.slicename, new_resource) + + resources = self.api.get_slice_resources(self.slicename) + self.assertIn(new_resource, resources) + + resource_to_remove = 'urn:publicid:IDN+ple:unavarraple+node+planetlab2.tlm.unavarra.es' + self.api.remove_resource_from_slice(self.slicename, resource_to_remove) + + resources = self.api.get_slice_resources(self.slicename) + self.assertNotIn(resource_to_remove, resources) + + def test_concurrence(self): + resources = ['urn:publicid:IDN+ple:itaveirople+node+planet2.servers.ua.pt', + 'urn:publicid:IDN+ple:quantavisple+node+marie.iet.unipi.it', + 'urn:publicid:IDN+ple:elteple+node+planet1.elte.hu', + 'urn:publicid:IDN+ple:inria+node+wlab39.pl.sophia.inria.fr', + 'urn:publicid:IDN+ple:poznanple+node+planetlab-2.man.poznan.pl', + 'urn:publicid:IDN+ple:tmsp+node+planetlab-node3.it-sudparis.eu', + 'urn:publicid:IDN+ple:colbudple+node+evghu6.colbud.hu', + 'urn:publicid:IDN+ple:erlangenple+node+planetlab1.informatik.uni-erlangen.de', + 'urn:publicid:IDN+ple:kitple+node+iraplab1.iralab.uni-karlsruhe.de', + 'urn:publicid:IDN+ple:polimiple+node+planetlab1.elet.polimi.it', + 'urn:publicid:IDN+ple:colbudple+node+evghu12.colbud.hu', + 'urn:publicid:IDN+ple:ccsr+node+pl2.ccsrfi.net', + 'urn:publicid:IDN+ple:modenaple+node+planetlab-1.ing.unimo.it', + 'urn:publicid:IDN+ple:cambridgeple+node+planetlab1.xeno.cl.cam.ac.uk', + 'urn:publicid:IDN+ple:lig+node+planetlab-1.imag.fr', + 'urn:publicid:IDN+ple:polslple+node+plab3.ple.silweb.pl', + 'urn:publicid:IDN+ple:uc3mple+node+planetlab1.uc3m.es', + 'urn:publicid:IDN+ple:colbudple+node+evghu4.colbud.hu', + 'urn:publicid:IDN+ple:hiitple+node+planetlab3.hiit.fi', + 'urn:publicid:IDN+ple:l3sple+node+planet1.l3s.uni-hannover.de', + 'urn:publicid:IDN+ple:colbudple+node+evghu5.colbud.hu', + 'urn:publicid:IDN+ple:dbislab+node+planetlab2.ionio.gr', + 'urn:publicid:IDN+ple:forthple+node+planetlab2.ics.forth.gr', + 'urn:publicid:IDN+ple:netmodeple+node+vicky.planetlab.ntua.gr', + 'urn:publicid:IDN+ple:colbudple+node+evghu1.colbud.hu', + 'urn:publicid:IDN+ple:cyprusple+node+planetlab-3.cs.ucy.ac.cy', + 'urn:publicid:IDN+ple:darmstadtple+node+host1.planetlab.informatik.tu-darmstadt.de', + 'urn:publicid:IDN+ple:cnetple+node+plab2.create-net.org', + 'urn:publicid:IDN+ple:unictple+node+gschembra3.diit.unict.it', + 'urn:publicid:IDN+ple:sevillaple+node+ait05.us.es'] + + def add_resource(new_resource): + self.api.add_resource_to_slice(self.slicename, new_resource) + + def runInParallel(fns): + for fn in fns: + p = Process(target=fn) + p.start() + p.join() + + funcs = list() + for r in resources: + funcs.append(add_resource(r)) + + runInParallel(funcs) + + slice_res = self.api.get_slice_resources(self.slicename) + for r in resources: + self.assertIn(r, slice_res) + + for r in resources: + self.api.remove_resource_from_slice(self.slicename, r) + +if __name__ == '__main__': + unittest.main() + + + -- 2.43.0