2 # NEPI, a framework to manage network experiments
3 # Copyright (C) 2013 INRIA
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
32 except (socket.error, IOError, OSError):
40 _expected_methods = set(
41 ['AddNodeTag', 'AddConfFile', 'DeletePersonTag', 'AddNodeType',
42 'DeleteBootState', 'SliceListNames', 'DeleteKey','SliceGetTicket',
43 'SliceUsersList', 'SliceUpdate', 'GetNodeGroups', 'SliceCreate',
44 'GetNetworkMethods', 'GetNodeFlavour', 'DeleteNode', 'BootNotifyOwners',
45 'AddPersonKey', 'AddNode', 'UpdateNodeGroup', 'GetAddressTypes',
46 'AddIlink', 'DeleteNetworkType', 'GetInitScripts', 'GenerateNodeConfFile',
47 'AddSite', 'BindObjectToPeer', 'SliceListUserSlices', 'GetPeers',
48 'AddPeer', 'DeletePeer', 'AddRole', 'DeleteRole', 'SetPersonPrimarySite',
49 'AddSiteAddress', 'SliceDelete', 'NotifyPersons', 'GetKeyTypes',
50 'GetConfFiles', 'GetIlinks', 'AddTagType', 'GetNodes', 'DeleteNodeTag',
51 'DeleteSliceFromNodesWhitelist', 'UpdateAddress', 'ResetPassword',
52 'AddSliceToNodesWhitelist', 'AddRoleToTagType', 'AddLeases',
53 'GetAddresses', 'AddInitScript', 'RebootNode', 'GetPCUTypes',
54 'RefreshPeer', 'GetBootMedium', 'UpdateKey', 'UpdatePCU', 'GetSession',
55 'AddInterfaceTag', 'UpdatePCUType', 'GetInterfaces', 'SliceExtendedInfo',
56 'SliceNodesList', 'DeleteRoleFromTagType', 'DeleteSlice', 'GetSites',
57 'DeleteMessage', 'GetSliceFamily', 'GetPlcRelease', 'UpdateTagType',
58 'AddSliceInstantiation', 'ResolveSlices', 'GetSlices',
59 'DeleteRoleFromPerson', 'GetSessions', 'UpdatePeer', 'VerifyPerson',
60 'GetPersonTags', 'DeleteKeyType', 'AddSlice', 'SliceUserAdd',
61 'DeleteSession', 'GetMessages', 'DeletePCU', 'GetPeerData',
62 'DeletePersonFromSite', 'DeleteTagType', 'GetPCUs', 'UpdateLeases',
63 'AddMessage', 'DeletePCUProtocolType', 'DeleteInterfaceTag',
64 'AddPersonToSite', 'GetSlivers', 'SliceNodesDel',
65 'DeleteAddressTypeFromAddress', 'AddNodeGroup', 'GetSliceTags',
66 'DeleteSite', 'GetSiteTags', 'UpdateMessage', 'DeleteSliceFromNodes',
67 'SliceRenew', 'UpdatePCUProtocolType', 'DeleteSiteTag',
68 'GetPCUProtocolTypes', 'GetEvents', 'GetSliceTicket', 'AddPersonTag',
69 'BootGetNodeDetails', 'DeleteInterface', 'DeleteNodeGroup',
70 'AddPCUProtocolType', 'BootCheckAuthentication', 'AddSiteTag',
71 'AddAddressTypeToAddress', 'DeleteConfFile', 'DeleteInitScript',
72 'DeletePerson', 'DeleteIlink', 'DeleteAddressType', 'AddBootState',
73 'AuthCheck', 'NotifySupport', 'GetSliceInstantiations', 'AddPCUType',
74 'AddPCU', 'AddSession', 'GetEventObjects', 'UpdateSiteTag',
75 'UpdateNodeTag', 'AddPerson', 'BlacklistKey', 'UpdateInitScript',
76 'AddSliceToNodes', 'RebootNodeWithPCU', 'GetNodeTags', 'GetSliceKeys',
77 'GetSliceSshKeys', 'AddNetworkMethod', 'SliceNodesAdd',
78 'DeletePersonFromSlice', 'ReportRunlevel', 'GetNetworkTypes',
79 'UpdateSite', 'DeleteConfFileFromNodeGroup', 'UpdateNode',
80 'DeleteSliceInstantiation', 'DeleteSliceTag', 'BootUpdateNode',
81 'UpdatePerson', 'UpdateConfFile', 'SliceUserDel', 'DeleteLeases',
82 'AddConfFileToNodeGroup', 'UpdatePersonTag', 'DeleteConfFileFromNode',
83 'AddPersonToSlice', 'UnBindObjectFromPeer', 'AddNodeToPCU',
84 'GetLeaseGranularity', 'DeletePCUType', 'GetTagTypes', 'GetNodeTypes',
85 'UpdateInterfaceTag', 'GetRoles', 'UpdateSlice', 'UpdateSliceTag',
86 'AddSliceTag', 'AddNetworkType', 'AddInterface', 'AddAddressType',
87 'AddRoleToPerson', 'DeleteNodeType', 'GetLeases', 'UpdateInterface',
88 'SliceInfo', 'DeleteAddress', 'SliceTicketGet', 'GetPersons',
89 'GetWhitelist', 'AddKeyType', 'UpdateAddressType', 'GetPeerName',
90 'DeleteNetworkMethod', 'UpdateIlink', 'AddConfFileToNode', 'GetKeys',
91 'DeleteNodeFromPCU', 'GetInterfaceTags', 'GetBootStates',
92 'SetInterfaceSens', 'SetNodeLoadm', 'GetInterfaceRate', 'GetNodeLoadw',
93 'SetInterfaceKey', 'GetNodeSlices', 'GetNodeLoadm', 'SetSliceVref',
94 'GetInterfaceIwpriv', 'SetNodeLoadw', 'SetNodeSerial',
95 'GetNodePlainBootstrapfs', 'SetNodeMEMw', 'GetNodeResponse',
96 'SetInterfaceRate', 'SetSliceInitscript', 'SetNodeFcdistro',
97 'GetNodeLoady', 'SetNodeArch', 'SetNodeKargs', 'SetNodeMEMm',
98 'SetNodeBWy', 'SetNodeBWw', 'SetInterfaceSecurityMode', 'SetNodeBWm',
99 'SetNodeASType', 'GetNodeKargs', 'GetPersonColumnconf',
100 'GetNodeResponsem', 'GetNodeCPUy', 'GetNodeCramfs', 'SetNodeSlicesw',
101 'SetPersonColumnconf', 'SetNodeSlicesy', 'GetNodeCPUw', 'GetNodeBWy',
102 'GetNodeCPUm', 'GetInterfaceDriver', 'GetNodeLoad', 'GetInterfaceMode',
103 'GetNodeSerial', 'SetNodeSlicesm', 'SetNodeLoady', 'GetNodeReliabilityw',
104 'SetSliceFcdistro', 'GetNodeReliabilityy', 'SetInterfaceEssid',
105 'SetSliceInitscriptCode', 'GetNodeExtensions', 'GetSliceOmfControl',
106 'SetNodeCity', 'SetInterfaceIfname', 'SetNodeHrn', 'SetNodeNoHangcheck',
107 'GetNodeNoHangcheck', 'GetSliceFcdistro', 'SetNodeCountry',
108 'SetNodeKvariant', 'GetNodeKvariant', 'GetNodeMEMy', 'SetInterfaceIwpriv',
109 'GetNodeMEMw', 'SetInterfaceBackdoor', 'GetInterfaceFreq',
110 'SetInterfaceChannel', 'SetInterfaceNw', 'GetPersonShowconf',
111 'GetSliceInitscriptCode', 'SetNodeMEM', 'GetInterfaceEssid', 'GetNodeMEMm',
112 'SetInterfaceMode', 'SetInterfaceIwconfig', 'GetNodeSlicesm', 'GetNodeBWm',
113 'SetNodePlainBootstrapfs', 'SetNodeRegion', 'SetNodeCPU', 'GetNodeSlicesw',
114 'SetNodeBW', 'SetNodeSlices', 'SetNodeCramfs', 'GetNodeSlicesy',
115 'GetInterfaceKey', 'GetSliceInitscript', 'SetNodeCPUm', 'SetSliceArch',
116 'SetNodeLoad', 'SetNodeResponse', 'GetSliceSliverHMAC', 'GetNodeBWw',
117 'GetNodeRegion', 'SetNodeMEMy', 'GetNodeASType', 'SetNodePldistro',
118 'GetSliceArch', 'GetNodeCountry', 'SetSliceOmfControl', 'GetNodeHrn',
119 'GetNodeCity', 'SetInterfaceAlias', 'GetNodeBW', 'GetNodePldistro',
120 'GetSlicePldistro', 'SetNodeASNumber', 'GetSliceHmac', 'SetSliceHmac',
121 'GetNodeMEM', 'GetNodeASNumber', 'GetInterfaceAlias', 'GetSliceVref',
122 'GetNodeArch', 'GetSliceSshKey', 'GetInterfaceKey4', 'GetInterfaceKey2',
123 'GetInterfaceKey3', 'GetInterfaceKey1', 'GetInterfaceBackdoor',
124 'GetInterfaceIfname', 'SetSliceSliverHMAC', 'SetNodeReliability',
125 'GetNodeCPU', 'SetPersonShowconf', 'SetNodeExtensions', 'SetNodeCPUy',
126 'SetNodeCPUw', 'GetNodeResponsew', 'SetNodeResponsey', 'GetInterfaceSens',
127 'SetNodeResponsew', 'GetNodeResponsey', 'GetNodeReliability',
128 'GetNodeReliabilitym', 'SetNodeResponsem', 'SetInterfaceDriver',
129 'GetInterfaceSecurityMode', 'SetNodeDeployment', 'SetNodeReliabilitym',
130 'GetNodeFcdistro', 'SetInterfaceFreq', 'GetInterfaceNw',
131 'SetNodeReliabilityy', 'SetNodeReliabilityw', 'GetInterfaceIwconfig',
132 'SetSlicePldistro', 'SetSliceSshKey', 'GetNodeDeployment',
133 'GetInterfaceChannel', 'SetInterfaceKey2', 'SetInterfaceKey3',
134 'SetInterfaceKey1', 'SetInterfaceKey4'])
136 _required_methods = set()
138 def __init__(self, username = None, password = None, session_key = None,
140 hostname = "www.planet-lab.eu",
141 urlpattern = "https://%(hostname)s:443/PLCAPI/",
144 self._blacklist = set()
145 self._reserved = set()
146 self._nodes_cache = None
147 self._already_cached = False
149 if session_key is not None:
150 self.auth = dict(AuthMethod='session', session=session_key)
151 elif username is not None and password is not None:
152 self.auth = dict(AuthMethod='password', Username=username, AuthString=password)
154 self.auth = dict(AuthMethod='anonymous')
156 self._local_peer = local_peer
157 self._url = urlpattern % {'hostname':hostname}
159 if (proxy is not None):
161 class HTTPSProxyTransport(xmlrpclib.Transport):
162 def __init__(self, proxy, use_datetime=0):
163 opener = urllib2.build_opener(urllib2.ProxyHandler({"https" : proxy}))
164 xmlrpclib.Transport.__init__(self, use_datetime)
167 def request(self, host, handler, request_body, verbose=0):
168 req = urllib2.Request('https://%s%s' % (host, handler), request_body)
169 req.add_header('User-agent', self.user_agent)
170 self.verbose = verbose
171 return self.parse_response(self.opener.open(req))
173 self._proxy_transport = lambda : HTTPSProxyTransport(proxy)
175 self._proxy_transport = lambda : None
177 self.threadlocal = threading.local()
181 # Cannot reuse same proxy in all threads, py2.7 is not threadsafe
182 return xmlrpclib.ServerProxy(
184 transport = self._proxy_transport(),
190 return self.threadlocal.mc
191 except AttributeError:
195 # TODO: Use nepi utils Logger instead of warning!!
198 # validate XMLRPC server checking supported API calls
199 methods = set(_retry(self.mcapi.system.listMethods)())
200 if self._required_methods - methods:
201 warnings.warn("Unsupported REQUIRED methods: %s" % (
202 ", ".join(sorted(self._required_methods - methods)), ) )
205 if self._expected_methods - methods:
206 warnings.warn("Unsupported EXPECTED methods: %s" % (
207 ", ".join(sorted(self._expected_methods - methods)), ) )
211 network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
212 except (xmlrpclib.ProtocolError, xmlrpclib.Fault),e:
213 warnings.warn(str(e))
218 def network_types(self):
220 return self._network_types
221 except AttributeError:
222 self._network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
223 return self._network_types
228 return self._peer_map
229 except AttributeError:
230 peers = _retry(self.mcapi.GetPeers)(self.auth, {}, ['shortname','peername','peer_id'])
232 self._peer_map = dict(
233 (peer['shortname'], peer['peer_id'])
237 self._peer_map.update(
238 (peer['peername'], peer['peer_id'])
242 self._peer_map.update(
243 (peer['peer_id'], peer['shortname'])
247 self._peer_map[None] = self._local_peer
248 return self._peer_map
250 def get_node_flavour(self, node):
252 Returns detailed information on a given node's flavour,
253 i.e. its base installation.
255 This depends on the global PLC settings in the PLC_FLAVOUR area,
256 optionnally overridden by any of the following tags if set on that node:
257 'arch', 'pldistro', 'fcdistro', 'deployment', 'extensions'
261 * node : int or string
262 - int, Node identifier
263 - string, Fully qualified hostname
268 * extensions : array of string, extensions to add to the base install
269 * fcdistro : string, the fcdistro this node should be based upon
270 * nodefamily : string, the nodefamily this node should be based upon
271 * plain : boolean, use plain bootstrapfs image if set (for tests)
273 if not isinstance(node, (str, int, long)):
274 raise ValueError, "Node must be either a non-unicode string or an int"
275 return _retry(self.mcapi.GetNodeFlavour)(self.auth, node)
277 def get_nodes(self, node_id_or_name = None, fields = None, **kw):
279 Returns an array of structs containing details about nodes.
280 If node_id_or_name is specified and is an array of node identifiers
281 or hostnames, or the filters keyword argument with struct of node
282 attributes, or node attributes by keyword argument,
283 only nodes matching the filter will be returned.
285 If fields is specified, only the specified details will be returned.
286 NOTE that if fields is unspecified, the complete set of native fields are
287 returned, which DOES NOT include tags at this time.
289 Some fields may only be viewed by admins.
293 fields: an optional list of fields to retrieve. The default is all.
295 filters: an optional mapping with custom filters, which is the only
296 way to support complex filters like negation and numeric comparisons.
298 peer: a string (or sequence of strings) with the name(s) of peers
299 to filter - or None for local nodes.
301 if fields is not None:
302 fieldstuple = (fields,)
306 if node_id_or_name is not None:
307 return _retry(self.mcapi.GetNodes)(self.auth, node_id_or_name, *fieldstuple)
309 filters = kw.pop('filters',{})
312 peer = kw.pop('peer')
314 name_to_id = self.peer_map.get
316 if hasattr(peer, '__iter__'):
317 # we can't mix local and external nodes, so
318 # split and re-issue recursively in that case
319 if None in peer or self._local_peer in peer:
323 if self._local_peer in peer:
324 peer.remove(self._local_peer)
327 self.get_nodes(node_id_or_name, fields,
328 filters = filters, peer=peer, **kw) + \
329 self.get_nodes(node_id_or_name, fields,
330 filters = filters, peer=None, **kw)
333 peer_filter = map(name_to_id, peer)
335 elif peer is None or peer == self._local_peer:
338 peer_filter = name_to_id(peer)
340 filters['peer_id'] = peer_filter
344 if not filters and not fieldstuple:
345 if not self._nodes_cache and not self._already_cached:
346 self._already_cached = True
347 self._nodes_cache = _retry(self.mcapi.GetNodes)(self.auth)
348 elif not self._nodes_cache:
349 while not self._nodes_cache:
351 return self._nodes_cache
353 return _retry(self.mcapi.GetNodes)(self.auth, filters, *fieldstuple)
355 def get_node_tags(self, node_tag_id = None, fields = None, **kw):
356 if fields is not None:
357 fieldstuple = (fields,)
361 if node_tag_id is not None:
362 return _retry(self.mcapi.GetNodeTags)(self.auth, node_tag_id,
365 filters = kw.pop('filters',{})
367 return _retry(self.mcapi.GetNodeTags)(self.auth, filters,
370 def get_slice_tags(self, slice_tag_id = None, fields = None, **kw):
371 if fields is not None:
372 fieldstuple = (fields,)
376 if slice_tag_id is not None:
377 return _retry(self.mcapi.GetSliceTags)(self.auth, slice_tag_id,
380 filters = kw.pop('filters',{})
382 return _retry(self.mcapi.GetSliceTags)(self.auth, filters,
385 def get_interfaces(self, interface_id_or_ip = None, fields = None, **kw):
386 if fields is not None:
387 fieldstuple = (fields,)
391 if interface_id_or_ip is not None:
392 return _retry(self.mcapi.GetInterfaces)(self.auth,
393 interface_id_or_ip, *fieldstuple)
395 filters = kw.pop('filters',{})
397 return _retry(self.mcapi.GetInterfaces)(self.auth, filters,
400 def get_slices(self, slice_id_or_name = None, fields = None, **kw):
401 if fields is not None:
402 fieldstuple = (fields,)
406 if slice_id_or_name is not None:
407 return _retry(self.mcapi.GetSlices)(self.auth, slice_id_or_name,
410 filters = kw.pop('filters',{})
412 return _retry(self.mcapi.GetSlices)(self.auth, filters,
415 def update_slice(self, slice_id_or_name, **kw):
416 return _retry(self.mcapi.UpdateSlice)(self.auth, slice_id_or_name, kw)
418 def delete_slice_node(self, slice_id_or_name, node_id_or_hostname):
419 return _retry(self.mcapi.DeleteSliceFromNodes)(self.auth, slice_id_or_name, node_id_or_hostname)
421 def start_multicall(self):
422 self.threadlocal.mc = xmlrpclib.MultiCall(self.mcapi)
424 def finish_multicall(self):
425 mc = self.threadlocal.mc
426 del self.threadlocal.mc
429 def get_slice_nodes(self, slicename):
430 return self.get_slices(slicename, ['node_ids'])[0]['node_ids']
432 def add_slice_nodes(self, slicename, nodes = None):
433 self.update_slice(slicename, nodes = nodes)
435 def get_node_info(self, node_id):
436 self.start_multicall()
437 info = self.get_nodes(node_id)
438 tags = self.get_node_tags(node_id=node_id, fields=('tagname','value'))
439 info, tags = self.finish_multicall()
442 def get_slice_id(self, slicename):
444 slices = self.get_slices(slicename, fields=('slice_id',))
446 slice_id = slices[0]['slice_id']
448 # If it wasn't found, don't remember this failure, keep trying
451 def get_slice_vnet_sys_tag(self, slicename):
452 slicetags = self.get_slice_tags(
454 tagname = 'vsys_vnet',
458 return slicetags[0]['value']
462 def blacklist_host(self, hostname):
463 self._blacklist.add(hostname)
465 def blacklisted(self):
466 return self._blacklist
468 def unblacklist_host(self, hostname):
469 del self._blacklist[hostname]
471 def reserve_host(self, hostname):
472 self._reserved.add(hostname)
475 return self._reserved
477 def unreserve_host(self, hostname):
478 del self._reserved[hostname]
481 class PLCAPIFactory(object):
485 It allows PlanetLab RMs sharing a same slice, to use a same plcapi instance,
486 and to sincronize blacklisted and reserved hosts.
489 # use lock to avoid concurrent access to the Api list at the same times by 2 different threads
490 _lock = threading.Lock()
494 def get_api(cls, pl_user, pl_pass, pl_host,
495 pl_ptn = "https://%(hostname)s:443/PLCAPI/",
497 """ Get existing PLCAPI instance
499 :param pl_user: Planelab user name (used for web login)
501 :param pl_pass: Planetlab password (used for web login)
503 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
505 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
507 :param proxy: Proxy service url
510 if pl_user and pl_pass and pl_host:
511 key = cls._make_key(pl_user, pl_host)
513 api = cls._apis.get(key)
515 api = cls.create_api(pl_user, pl_pass, pl_host, pl_ptn, proxy)
520 def create_api(cls, pl_user, pl_pass, pl_host,
521 pl_ptn = "https://%(hostname)s:443/PLCAPI/",
523 """ Create an PLCAPI instance
525 :param pl_user: Planelab user name (used for web login)
527 :param pl_pass: Planetlab password (used for web login)
529 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
531 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
533 :param proxy: Proxy service url
543 key = cls._make_key(pl_user, pl_host)
548 def _make_key(cls, *args):
549 """ Hash the credentials in order to create a key
551 :param args: list of arguments used to create the hash (user, host, port, ...)
552 :type args: list of args
555 skey = "".join(map(str, args))
556 return hashlib.md5(skey).hexdigest()