2 # NEPI, a framework to manage network experiments
3 # Copyright (C) 2013 INRIA
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License version 2 as
7 # published by the Free Software Foundation;
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
32 except (socket.error, IOError, OSError):
40 _expected_methods = set(
41 ['AddNodeTag', 'AddConfFile', 'DeletePersonTag', 'AddNodeType',
42 'DeleteBootState', 'SliceListNames', 'DeleteKey','SliceGetTicket',
43 'SliceUsersList', 'SliceUpdate', 'GetNodeGroups', 'SliceCreate',
44 'GetNetworkMethods', 'GetNodeFlavour', 'DeleteNode', 'BootNotifyOwners',
45 'AddPersonKey', 'AddNode', 'UpdateNodeGroup', 'GetAddressTypes',
46 'AddIlink', 'DeleteNetworkType', 'GetInitScripts', 'GenerateNodeConfFile',
47 'AddSite', 'BindObjectToPeer', 'SliceListUserSlices', 'GetPeers',
48 'AddPeer', 'DeletePeer', 'AddRole', 'DeleteRole', 'SetPersonPrimarySite',
49 'AddSiteAddress', 'SliceDelete', 'NotifyPersons', 'GetKeyTypes',
50 'GetConfFiles', 'GetIlinks', 'AddTagType', 'GetNodes', 'DeleteNodeTag',
51 'DeleteSliceFromNodesWhitelist', 'UpdateAddress', 'ResetPassword',
52 'AddSliceToNodesWhitelist', 'AddRoleToTagType', 'AddLeases',
53 'GetAddresses', 'AddInitScript', 'RebootNode', 'GetPCUTypes',
54 'RefreshPeer', 'GetBootMedium', 'UpdateKey', 'UpdatePCU', 'GetSession',
55 'AddInterfaceTag', 'UpdatePCUType', 'GetInterfaces', 'SliceExtendedInfo',
56 'SliceNodesList', 'DeleteRoleFromTagType', 'DeleteSlice', 'GetSites',
57 'DeleteMessage', 'GetSliceFamily', 'GetPlcRelease', 'UpdateTagType',
58 'AddSliceInstantiation', 'ResolveSlices', 'GetSlices',
59 'DeleteRoleFromPerson', 'GetSessions', 'UpdatePeer', 'VerifyPerson',
60 'GetPersonTags', 'DeleteKeyType', 'AddSlice', 'SliceUserAdd',
61 'DeleteSession', 'GetMessages', 'DeletePCU', 'GetPeerData',
62 'DeletePersonFromSite', 'DeleteTagType', 'GetPCUs', 'UpdateLeases',
63 'AddMessage', 'DeletePCUProtocolType', 'DeleteInterfaceTag',
64 'AddPersonToSite', 'GetSlivers', 'SliceNodesDel',
65 'DeleteAddressTypeFromAddress', 'AddNodeGroup', 'GetSliceTags',
66 'DeleteSite', 'GetSiteTags', 'UpdateMessage', 'DeleteSliceFromNodes',
67 'SliceRenew', 'UpdatePCUProtocolType', 'DeleteSiteTag',
68 'GetPCUProtocolTypes', 'GetEvents', 'GetSliceTicket', 'AddPersonTag',
69 'BootGetNodeDetails', 'DeleteInterface', 'DeleteNodeGroup',
70 'AddPCUProtocolType', 'BootCheckAuthentication', 'AddSiteTag',
71 'AddAddressTypeToAddress', 'DeleteConfFile', 'DeleteInitScript',
72 'DeletePerson', 'DeleteIlink', 'DeleteAddressType', 'AddBootState',
73 'AuthCheck', 'NotifySupport', 'GetSliceInstantiations', 'AddPCUType',
74 'AddPCU', 'AddSession', 'GetEventObjects', 'UpdateSiteTag',
75 'UpdateNodeTag', 'AddPerson', 'BlacklistKey', 'UpdateInitScript',
76 'AddSliceToNodes', 'RebootNodeWithPCU', 'GetNodeTags', 'GetSliceKeys',
77 'GetSliceSshKeys', 'AddNetworkMethod', 'SliceNodesAdd',
78 'DeletePersonFromSlice', 'ReportRunlevel', 'GetNetworkTypes',
79 'UpdateSite', 'DeleteConfFileFromNodeGroup', 'UpdateNode',
80 'DeleteSliceInstantiation', 'DeleteSliceTag', 'BootUpdateNode',
81 'UpdatePerson', 'UpdateConfFile', 'SliceUserDel', 'DeleteLeases',
82 'AddConfFileToNodeGroup', 'UpdatePersonTag', 'DeleteConfFileFromNode',
83 'AddPersonToSlice', 'UnBindObjectFromPeer', 'AddNodeToPCU',
84 'GetLeaseGranularity', 'DeletePCUType', 'GetTagTypes', 'GetNodeTypes',
85 'UpdateInterfaceTag', 'GetRoles', 'UpdateSlice', 'UpdateSliceTag',
86 'AddSliceTag', 'AddNetworkType', 'AddInterface', 'AddAddressType',
87 'AddRoleToPerson', 'DeleteNodeType', 'GetLeases', 'UpdateInterface',
88 'SliceInfo', 'DeleteAddress', 'SliceTicketGet', 'GetPersons',
89 'GetWhitelist', 'AddKeyType', 'UpdateAddressType', 'GetPeerName',
90 'DeleteNetworkMethod', 'UpdateIlink', 'AddConfFileToNode', 'GetKeys',
91 'DeleteNodeFromPCU', 'GetInterfaceTags', 'GetBootStates',
92 'SetInterfaceSens', 'SetNodeLoadm', 'GetInterfaceRate', 'GetNodeLoadw',
93 'SetInterfaceKey', 'GetNodeSlices', 'GetNodeLoadm', 'SetSliceVref',
94 'GetInterfaceIwpriv', 'SetNodeLoadw', 'SetNodeSerial',
95 'GetNodePlainBootstrapfs', 'SetNodeMEMw', 'GetNodeResponse',
96 'SetInterfaceRate', 'SetSliceInitscript', 'SetNodeFcdistro',
97 'GetNodeLoady', 'SetNodeArch', 'SetNodeKargs', 'SetNodeMEMm',
98 'SetNodeBWy', 'SetNodeBWw', 'SetInterfaceSecurityMode', 'SetNodeBWm',
99 'SetNodeASType', 'GetNodeKargs', 'GetPersonColumnconf',
100 'GetNodeResponsem', 'GetNodeCPUy', 'GetNodeCramfs', 'SetNodeSlicesw',
101 'SetPersonColumnconf', 'SetNodeSlicesy', 'GetNodeCPUw', 'GetNodeBWy',
102 'GetNodeCPUm', 'GetInterfaceDriver', 'GetNodeLoad', 'GetInterfaceMode',
103 'GetNodeSerial', 'SetNodeSlicesm', 'SetNodeLoady', 'GetNodeReliabilityw',
104 'SetSliceFcdistro', 'GetNodeReliabilityy', 'SetInterfaceEssid',
105 'SetSliceInitscriptCode', 'GetNodeExtensions', 'GetSliceOmfControl',
106 'SetNodeCity', 'SetInterfaceIfname', 'SetNodeHrn', 'SetNodeNoHangcheck',
107 'GetNodeNoHangcheck', 'GetSliceFcdistro', 'SetNodeCountry',
108 'SetNodeKvariant', 'GetNodeKvariant', 'GetNodeMEMy', 'SetInterfaceIwpriv',
109 'GetNodeMEMw', 'SetInterfaceBackdoor', 'GetInterfaceFreq',
110 'SetInterfaceChannel', 'SetInterfaceNw', 'GetPersonShowconf',
111 'GetSliceInitscriptCode', 'SetNodeMEM', 'GetInterfaceEssid', 'GetNodeMEMm',
112 'SetInterfaceMode', 'SetInterfaceIwconfig', 'GetNodeSlicesm', 'GetNodeBWm',
113 'SetNodePlainBootstrapfs', 'SetNodeRegion', 'SetNodeCPU', 'GetNodeSlicesw',
114 'SetNodeBW', 'SetNodeSlices', 'SetNodeCramfs', 'GetNodeSlicesy',
115 'GetInterfaceKey', 'GetSliceInitscript', 'SetNodeCPUm', 'SetSliceArch',
116 'SetNodeLoad', 'SetNodeResponse', 'GetSliceSliverHMAC', 'GetNodeBWw',
117 'GetNodeRegion', 'SetNodeMEMy', 'GetNodeASType', 'SetNodePldistro',
118 'GetSliceArch', 'GetNodeCountry', 'SetSliceOmfControl', 'GetNodeHrn',
119 'GetNodeCity', 'SetInterfaceAlias', 'GetNodeBW', 'GetNodePldistro',
120 'GetSlicePldistro', 'SetNodeASNumber', 'GetSliceHmac', 'SetSliceHmac',
121 'GetNodeMEM', 'GetNodeASNumber', 'GetInterfaceAlias', 'GetSliceVref',
122 'GetNodeArch', 'GetSliceSshKey', 'GetInterfaceKey4', 'GetInterfaceKey2',
123 'GetInterfaceKey3', 'GetInterfaceKey1', 'GetInterfaceBackdoor',
124 'GetInterfaceIfname', 'SetSliceSliverHMAC', 'SetNodeReliability',
125 'GetNodeCPU', 'SetPersonShowconf', 'SetNodeExtensions', 'SetNodeCPUy',
126 'SetNodeCPUw', 'GetNodeResponsew', 'SetNodeResponsey', 'GetInterfaceSens',
127 'SetNodeResponsew', 'GetNodeResponsey', 'GetNodeReliability',
128 'GetNodeReliabilitym', 'SetNodeResponsem', 'SetInterfaceDriver',
129 'GetInterfaceSecurityMode', 'SetNodeDeployment', 'SetNodeReliabilitym',
130 'GetNodeFcdistro', 'SetInterfaceFreq', 'GetInterfaceNw',
131 'SetNodeReliabilityy', 'SetNodeReliabilityw', 'GetInterfaceIwconfig',
132 'SetSlicePldistro', 'SetSliceSshKey', 'GetNodeDeployment',
133 'GetInterfaceChannel', 'SetInterfaceKey2', 'SetInterfaceKey3',
134 'SetInterfaceKey1', 'SetInterfaceKey4'])
136 _required_methods = set()
138 def __init__(self, username, password, hostname, urlpattern, ec, proxy, session_key = None,
141 self._blacklist = set()
142 self._reserved = set()
143 self._nodes_cache = None
144 self._already_cached = False
148 if session_key is not None:
149 self.auth = dict(AuthMethod='session', session=session_key)
150 elif username is not None and password is not None:
151 self.auth = dict(AuthMethod='password', Username=username, AuthString=password)
153 self.auth = dict(AuthMethod='anonymous')
155 self._local_peer = local_peer
156 self._url = urlpattern % {'hostname':hostname}
158 if (proxy is not None):
159 import urllib.request, urllib.error, urllib.parse
160 class HTTPSProxyTransport(xmlrpc.client.Transport):
161 def __init__(self, proxy, use_datetime=0):
162 opener = urllib.request.build_opener(urllib.request.ProxyHandler({"https" : proxy}))
163 xmlrpc.client.Transport.__init__(self, use_datetime)
166 def request(self, host, handler, request_body, verbose=0):
167 req = urllib.request.Request('https://%s%s' % (host, handler), request_body)
168 req.add_header('User-agent', self.user_agent)
169 self.verbose = verbose
170 return self.parse_response(self.opener.open(req))
172 self._proxy_transport = lambda : HTTPSProxyTransport(proxy)
174 self._proxy_transport = lambda : None
176 self.threadlocal = threading.local()
178 # Load blacklist from file
179 if self._ecobj.get_global('planetlab::Node', 'persist_blacklist'):
180 self._set_blacklist()
184 # Cannot reuse same proxy in all threads, py2.7 is not threadsafe
185 return xmlrpc.client.ServerProxy(
187 transport = self._proxy_transport(),
193 return self.threadlocal.mc
194 except AttributeError:
198 # TODO: Use nepi utils Logger instead of warning!!
201 # validate XMLRPC server checking supported API calls
202 methods = set(_retry(self.mcapi.system.listMethods)())
203 if self._required_methods - methods:
204 warnings.warn("Unsupported REQUIRED methods: %s" % (
205 ", ".join(sorted(self._required_methods - methods)), ) )
208 if self._expected_methods - methods:
209 warnings.warn("Unsupported EXPECTED methods: %s" % (
210 ", ".join(sorted(self._expected_methods - methods)), ) )
214 network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
215 except (xmlrpc.client.ProtocolError, xmlrpc.client.Fault) as e:
216 warnings.warn(str(e))
220 def _set_blacklist(self):
221 nepi_home = os.path.join(os.path.expanduser("~"), ".nepi")
222 plblacklist_file = os.path.join(nepi_home, "plblacklist.txt")
223 with open(plblacklist_file, 'r') as f:
224 hosts_tobl = f.read().splitlines()
226 nodes_id = self.get_nodes(hosts_tobl, ['node_id'])
227 for node_id in nodes_id:
228 self._blacklist.add(node_id['node_id'])
231 def network_types(self):
233 return self._network_types
234 except AttributeError:
235 self._network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
236 return self._network_types
241 return self._peer_map
242 except AttributeError:
243 peers = _retry(self.mcapi.GetPeers)(self.auth, {}, ['shortname','peername','peer_id'])
245 self._peer_map = dict(
246 (peer['shortname'], peer['peer_id'])
250 self._peer_map.update(
251 (peer['peername'], peer['peer_id'])
255 self._peer_map.update(
256 (peer['peer_id'], peer['shortname'])
260 self._peer_map[None] = self._local_peer
261 return self._peer_map
263 def get_node_flavour(self, node):
265 Returns detailed information on a given node's flavour,
266 i.e. its base installation.
268 This depends on the global PLC settings in the PLC_FLAVOUR area,
269 optionnally overridden by any of the following tags if set on that node:
270 'arch', 'pldistro', 'fcdistro', 'deployment', 'extensions'
274 * node : int or string
275 - int, Node identifier
276 - string, Fully qualified hostname
281 * extensions : array of string, extensions to add to the base install
282 * fcdistro : string, the fcdistro this node should be based upon
283 * nodefamily : string, the nodefamily this node should be based upon
284 * plain : boolean, use plain bootstrapfs image if set (for tests)
286 if not isinstance(node, (str, int)):
287 raise ValueError("Node must be either a non-unicode string or an int")
288 return _retry(self.mcapi.GetNodeFlavour)(self.auth, node)
290 def get_nodes(self, node_id_or_name = None, fields = None, **kw):
292 Returns an array of structs containing details about nodes.
293 If node_id_or_name is specified and is an array of node identifiers
294 or hostnames, or the filters keyword argument with struct of node
295 attributes, or node attributes by keyword argument,
296 only nodes matching the filter will be returned.
298 If fields is specified, only the specified details will be returned.
299 NOTE that if fields is unspecified, the complete set of native fields are
300 returned, which DOES NOT include tags at this time.
302 Some fields may only be viewed by admins.
306 fields: an optional list of fields to retrieve. The default is all.
308 filters: an optional mapping with custom filters, which is the only
309 way to support complex filters like negation and numeric comparisons.
311 peer: a string (or sequence of strings) with the name(s) of peers
312 to filter - or None for local nodes.
314 if fields is not None:
315 fieldstuple = (fields,)
319 if node_id_or_name is not None:
320 return _retry(self.mcapi.GetNodes)(self.auth, node_id_or_name, *fieldstuple)
322 filters = kw.pop('filters',{})
325 peer = kw.pop('peer')
327 name_to_id = self.peer_map.get
329 if hasattr(peer, '__iter__'):
330 # we can't mix local and external nodes, so
331 # split and re-issue recursively in that case
332 if None in peer or self._local_peer in peer:
336 if self._local_peer in peer:
337 peer.remove(self._local_peer)
340 self.get_nodes(node_id_or_name, fields,
341 filters = filters, peer=peer, **kw) + \
342 self.get_nodes(node_id_or_name, fields,
343 filters = filters, peer=None, **kw)
346 peer_filter = list(map(name_to_id, peer))
348 elif peer is None or peer == self._local_peer:
351 peer_filter = name_to_id(peer)
353 filters['peer_id'] = peer_filter
357 if not filters and not fieldstuple:
358 if not self._nodes_cache and not self._already_cached:
359 self._already_cached = True
360 self._nodes_cache = _retry(self.mcapi.GetNodes)(self.auth)
361 elif not self._nodes_cache:
362 while not self._nodes_cache:
364 return self._nodes_cache
366 return _retry(self.mcapi.GetNodes)(self.auth, filters, *fieldstuple)
368 def get_node_tags(self, node_tag_id = None, fields = None, **kw):
369 if fields is not None:
370 fieldstuple = (fields,)
374 if node_tag_id is not None:
375 return _retry(self.mcapi.GetNodeTags)(self.auth, node_tag_id,
378 filters = kw.pop('filters',{})
380 return _retry(self.mcapi.GetNodeTags)(self.auth, filters,
383 def get_slice_tags(self, slice_tag_id = None, fields = None, **kw):
384 if fields is not None:
385 fieldstuple = (fields,)
389 if slice_tag_id is not None:
390 return _retry(self.mcapi.GetSliceTags)(self.auth, slice_tag_id,
393 filters = kw.pop('filters',{})
395 return _retry(self.mcapi.GetSliceTags)(self.auth, filters,
398 def get_interfaces(self, interface_id_or_ip = None, fields = None, **kw):
399 if fields is not None:
400 fieldstuple = (fields,)
404 if interface_id_or_ip is not None:
405 return _retry(self.mcapi.GetInterfaces)(self.auth,
406 interface_id_or_ip, *fieldstuple)
408 filters = kw.pop('filters',{})
410 return _retry(self.mcapi.GetInterfaces)(self.auth, filters,
413 def get_slices(self, slice_id_or_name = None, fields = None, **kw):
414 if fields is not None:
415 fieldstuple = (fields,)
419 if slice_id_or_name is not None:
420 return _retry(self.mcapi.GetSlices)(self.auth, slice_id_or_name,
423 filters = kw.pop('filters',{})
425 return _retry(self.mcapi.GetSlices)(self.auth, filters,
428 def update_slice(self, slice_id_or_name, **kw):
429 return _retry(self.mcapi.UpdateSlice)(self.auth, slice_id_or_name, kw)
431 def delete_slice_node(self, slice_id_or_name, node_id_or_hostname):
432 return _retry(self.mcapi.DeleteSliceFromNodes)(self.auth, slice_id_or_name, node_id_or_hostname)
434 def start_multicall(self):
435 self.threadlocal.mc = xmlrpc.client.MultiCall(self.mcapi)
437 def finish_multicall(self):
438 mc = self.threadlocal.mc
439 del self.threadlocal.mc
442 def get_slice_nodes(self, slicename):
443 return self.get_slices(slicename, ['node_ids'])[0]['node_ids']
445 def add_slice_nodes(self, slicename, nodes):
446 self.update_slice(slicename, nodes=nodes)
448 def get_node_info(self, node_id):
449 self.start_multicall()
450 info = self.get_nodes(node_id)
451 tags = self.get_node_tags(node_id=node_id, fields=('tagname','value'))
452 info, tags = self.finish_multicall()
455 def get_slice_id(self, slicename):
457 slices = self.get_slices(slicename, fields=('slice_id',))
459 slice_id = slices[0]['slice_id']
461 # If it wasn't found, don't remember this failure, keep trying
464 def get_slice_vnet_sys_tag(self, slicename):
465 slicetags = self.get_slice_tags(
467 tagname = 'vsys_vnet',
471 return slicetags[0]['value']
475 def blacklist_host(self, node_id):
476 self._blacklist.add(node_id)
478 def blacklisted(self):
479 return self._blacklist
481 def unblacklist_host(self, node_id):
482 del self._blacklist[node_id]
484 def reserve_host(self, node_id):
485 self._reserved.add(node_id)
488 return self._reserved
490 def unreserve_host(self, node_id):
491 del self._reserved[node_id]
496 blacklist = self._blacklist
497 self._blacklist = set()
498 self._reserved = set()
499 if self._ecobj.get_global('PlanetlabNode', 'persist_blacklist'):
501 to_blacklist = list()
502 hostnames = self.get_nodes(list(blacklist), ['hostname'])
503 for hostname in hostnames:
504 to_blacklist.append(hostname['hostname'])
506 nepi_home = os.path.join(os.path.expanduser("~"), ".nepi")
507 plblacklist_file = os.path.join(nepi_home, "plblacklist.txt")
509 with open(plblacklist_file, 'w') as f:
510 for host in to_blacklist:
511 f.write("%s\n" % host)
514 class PLCAPIFactory(object):
518 It allows PlanetLab RMs sharing a same slice, to use a same plcapi instance,
519 and to sincronize blacklisted and reserved hosts.
522 # use lock to avoid concurrent access to the Api list at the same times by 2 different threads
523 _lock = threading.Lock()
527 def get_api(cls, pl_user, pl_pass, pl_host,
528 pl_ptn, ec, proxy = None):
529 """ Get existing PLCAPI instance
531 :param pl_user: Planelab user name (used for web login)
533 :param pl_pass: Planetlab password (used for web login)
535 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
537 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
539 :param proxy: Proxy service url
542 if pl_user and pl_pass and pl_host:
543 key = cls._make_key(pl_user, pl_host)
545 api = cls._apis.get(key)
547 api = cls.create_api(pl_user, pl_pass, pl_host, pl_ptn, ec, proxy)
554 def create_api(cls, pl_user, pl_pass, pl_host,
555 pl_ptn, ec, proxy = None):
556 """ Create an PLCAPI instance
558 :param pl_user: Planelab user name (used for web login)
560 :param pl_pass: Planetlab password (used for web login)
562 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
564 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
566 :param proxy: Proxy service url
569 api = PLCAPI(username = pl_user, password = pl_pass, hostname = pl_host,
570 urlpattern = pl_ptn, ec = ec, proxy = proxy)
571 key = cls._make_key(pl_user, pl_host)
576 def _make_key(cls, *args):
577 """ Hash the credentials in order to create a key
579 :param args: list of arguments used to create the hash (user, host, port, ...)
580 :type args: list of args
583 skey = "".join(map(str, args))
584 return hashlib.md5(skey).hexdigest()