2 # NEPI, a framework to manage network experiments
3 # Copyright (C) 2013 INRIA
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License version 2 as
7 # published by the Free Software Foundation;
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
26 from six import integer_types, string_types
27 from six.moves import xmlrpc_client
34 except (socket.error, IOError, OSError):
42 _expected_methods = set(
43 ['AddNodeTag', 'AddConfFile', 'DeletePersonTag', 'AddNodeType',
44 'DeleteBootState', 'SliceListNames', 'DeleteKey','SliceGetTicket',
45 'SliceUsersList', 'SliceUpdate', 'GetNodeGroups', 'SliceCreate',
46 'GetNetworkMethods', 'GetNodeFlavour', 'DeleteNode', 'BootNotifyOwners',
47 'AddPersonKey', 'AddNode', 'UpdateNodeGroup', 'GetAddressTypes',
48 'AddIlink', 'DeleteNetworkType', 'GetInitScripts', 'GenerateNodeConfFile',
49 'AddSite', 'BindObjectToPeer', 'SliceListUserSlices', 'GetPeers',
50 'AddPeer', 'DeletePeer', 'AddRole', 'DeleteRole', 'SetPersonPrimarySite',
51 'AddSiteAddress', 'SliceDelete', 'NotifyPersons', 'GetKeyTypes',
52 'GetConfFiles', 'GetIlinks', 'AddTagType', 'GetNodes', 'DeleteNodeTag',
53 'DeleteSliceFromNodesWhitelist', 'UpdateAddress', 'ResetPassword',
54 'AddSliceToNodesWhitelist', 'AddRoleToTagType', 'AddLeases',
55 'GetAddresses', 'AddInitScript', 'RebootNode', 'GetPCUTypes',
56 'RefreshPeer', 'GetBootMedium', 'UpdateKey', 'UpdatePCU', 'GetSession',
57 'AddInterfaceTag', 'UpdatePCUType', 'GetInterfaces', 'SliceExtendedInfo',
58 'SliceNodesList', 'DeleteRoleFromTagType', 'DeleteSlice', 'GetSites',
59 'DeleteMessage', 'GetSliceFamily', 'GetPlcRelease', 'UpdateTagType',
60 'AddSliceInstantiation', 'ResolveSlices', 'GetSlices',
61 'DeleteRoleFromPerson', 'GetSessions', 'UpdatePeer', 'VerifyPerson',
62 'GetPersonTags', 'DeleteKeyType', 'AddSlice', 'SliceUserAdd',
63 'DeleteSession', 'GetMessages', 'DeletePCU', 'GetPeerData',
64 'DeletePersonFromSite', 'DeleteTagType', 'GetPCUs', 'UpdateLeases',
65 'AddMessage', 'DeletePCUProtocolType', 'DeleteInterfaceTag',
66 'AddPersonToSite', 'GetSlivers', 'SliceNodesDel',
67 'DeleteAddressTypeFromAddress', 'AddNodeGroup', 'GetSliceTags',
68 'DeleteSite', 'GetSiteTags', 'UpdateMessage', 'DeleteSliceFromNodes',
69 'SliceRenew', 'UpdatePCUProtocolType', 'DeleteSiteTag',
70 'GetPCUProtocolTypes', 'GetEvents', 'GetSliceTicket', 'AddPersonTag',
71 'BootGetNodeDetails', 'DeleteInterface', 'DeleteNodeGroup',
72 'AddPCUProtocolType', 'BootCheckAuthentication', 'AddSiteTag',
73 'AddAddressTypeToAddress', 'DeleteConfFile', 'DeleteInitScript',
74 'DeletePerson', 'DeleteIlink', 'DeleteAddressType', 'AddBootState',
75 'AuthCheck', 'NotifySupport', 'GetSliceInstantiations', 'AddPCUType',
76 'AddPCU', 'AddSession', 'GetEventObjects', 'UpdateSiteTag',
77 'UpdateNodeTag', 'AddPerson', 'BlacklistKey', 'UpdateInitScript',
78 'AddSliceToNodes', 'RebootNodeWithPCU', 'GetNodeTags', 'GetSliceKeys',
79 'GetSliceSshKeys', 'AddNetworkMethod', 'SliceNodesAdd',
80 'DeletePersonFromSlice', 'ReportRunlevel', 'GetNetworkTypes',
81 'UpdateSite', 'DeleteConfFileFromNodeGroup', 'UpdateNode',
82 'DeleteSliceInstantiation', 'DeleteSliceTag', 'BootUpdateNode',
83 'UpdatePerson', 'UpdateConfFile', 'SliceUserDel', 'DeleteLeases',
84 'AddConfFileToNodeGroup', 'UpdatePersonTag', 'DeleteConfFileFromNode',
85 'AddPersonToSlice', 'UnBindObjectFromPeer', 'AddNodeToPCU',
86 'GetLeaseGranularity', 'DeletePCUType', 'GetTagTypes', 'GetNodeTypes',
87 'UpdateInterfaceTag', 'GetRoles', 'UpdateSlice', 'UpdateSliceTag',
88 'AddSliceTag', 'AddNetworkType', 'AddInterface', 'AddAddressType',
89 'AddRoleToPerson', 'DeleteNodeType', 'GetLeases', 'UpdateInterface',
90 'SliceInfo', 'DeleteAddress', 'SliceTicketGet', 'GetPersons',
91 'GetWhitelist', 'AddKeyType', 'UpdateAddressType', 'GetPeerName',
92 'DeleteNetworkMethod', 'UpdateIlink', 'AddConfFileToNode', 'GetKeys',
93 'DeleteNodeFromPCU', 'GetInterfaceTags', 'GetBootStates',
94 'SetInterfaceSens', 'SetNodeLoadm', 'GetInterfaceRate', 'GetNodeLoadw',
95 'SetInterfaceKey', 'GetNodeSlices', 'GetNodeLoadm', 'SetSliceVref',
96 'GetInterfaceIwpriv', 'SetNodeLoadw', 'SetNodeSerial',
97 'GetNodePlainBootstrapfs', 'SetNodeMEMw', 'GetNodeResponse',
98 'SetInterfaceRate', 'SetSliceInitscript', 'SetNodeFcdistro',
99 'GetNodeLoady', 'SetNodeArch', 'SetNodeKargs', 'SetNodeMEMm',
100 'SetNodeBWy', 'SetNodeBWw', 'SetInterfaceSecurityMode', 'SetNodeBWm',
101 'SetNodeASType', 'GetNodeKargs', 'GetPersonColumnconf',
102 'GetNodeResponsem', 'GetNodeCPUy', 'GetNodeCramfs', 'SetNodeSlicesw',
103 'SetPersonColumnconf', 'SetNodeSlicesy', 'GetNodeCPUw', 'GetNodeBWy',
104 'GetNodeCPUm', 'GetInterfaceDriver', 'GetNodeLoad', 'GetInterfaceMode',
105 'GetNodeSerial', 'SetNodeSlicesm', 'SetNodeLoady', 'GetNodeReliabilityw',
106 'SetSliceFcdistro', 'GetNodeReliabilityy', 'SetInterfaceEssid',
107 'SetSliceInitscriptCode', 'GetNodeExtensions', 'GetSliceOmfControl',
108 'SetNodeCity', 'SetInterfaceIfname', 'SetNodeHrn', 'SetNodeNoHangcheck',
109 'GetNodeNoHangcheck', 'GetSliceFcdistro', 'SetNodeCountry',
110 'SetNodeKvariant', 'GetNodeKvariant', 'GetNodeMEMy', 'SetInterfaceIwpriv',
111 'GetNodeMEMw', 'SetInterfaceBackdoor', 'GetInterfaceFreq',
112 'SetInterfaceChannel', 'SetInterfaceNw', 'GetPersonShowconf',
113 'GetSliceInitscriptCode', 'SetNodeMEM', 'GetInterfaceEssid', 'GetNodeMEMm',
114 'SetInterfaceMode', 'SetInterfaceIwconfig', 'GetNodeSlicesm', 'GetNodeBWm',
115 'SetNodePlainBootstrapfs', 'SetNodeRegion', 'SetNodeCPU', 'GetNodeSlicesw',
116 'SetNodeBW', 'SetNodeSlices', 'SetNodeCramfs', 'GetNodeSlicesy',
117 'GetInterfaceKey', 'GetSliceInitscript', 'SetNodeCPUm', 'SetSliceArch',
118 'SetNodeLoad', 'SetNodeResponse', 'GetSliceSliverHMAC', 'GetNodeBWw',
119 'GetNodeRegion', 'SetNodeMEMy', 'GetNodeASType', 'SetNodePldistro',
120 'GetSliceArch', 'GetNodeCountry', 'SetSliceOmfControl', 'GetNodeHrn',
121 'GetNodeCity', 'SetInterfaceAlias', 'GetNodeBW', 'GetNodePldistro',
122 'GetSlicePldistro', 'SetNodeASNumber', 'GetSliceHmac', 'SetSliceHmac',
123 'GetNodeMEM', 'GetNodeASNumber', 'GetInterfaceAlias', 'GetSliceVref',
124 'GetNodeArch', 'GetSliceSshKey', 'GetInterfaceKey4', 'GetInterfaceKey2',
125 'GetInterfaceKey3', 'GetInterfaceKey1', 'GetInterfaceBackdoor',
126 'GetInterfaceIfname', 'SetSliceSliverHMAC', 'SetNodeReliability',
127 'GetNodeCPU', 'SetPersonShowconf', 'SetNodeExtensions', 'SetNodeCPUy',
128 'SetNodeCPUw', 'GetNodeResponsew', 'SetNodeResponsey', 'GetInterfaceSens',
129 'SetNodeResponsew', 'GetNodeResponsey', 'GetNodeReliability',
130 'GetNodeReliabilitym', 'SetNodeResponsem', 'SetInterfaceDriver',
131 'GetInterfaceSecurityMode', 'SetNodeDeployment', 'SetNodeReliabilitym',
132 'GetNodeFcdistro', 'SetInterfaceFreq', 'GetInterfaceNw',
133 'SetNodeReliabilityy', 'SetNodeReliabilityw', 'GetInterfaceIwconfig',
134 'SetSlicePldistro', 'SetSliceSshKey', 'GetNodeDeployment',
135 'GetInterfaceChannel', 'SetInterfaceKey2', 'SetInterfaceKey3',
136 'SetInterfaceKey1', 'SetInterfaceKey4'])
138 _required_methods = set()
140 def __init__(self, username, password, hostname, urlpattern, ec, proxy, session_key = None,
143 self._blacklist = set()
144 self._reserved = set()
145 self._nodes_cache = None
146 self._already_cached = False
150 if session_key is not None:
151 self.auth = dict(AuthMethod='session', session=session_key)
152 elif username is not None and password is not None:
153 self.auth = dict(AuthMethod='password', Username=username, AuthString=password)
155 self.auth = dict(AuthMethod='anonymous')
157 self._local_peer = local_peer
158 self._url = urlpattern % {'hostname':hostname}
160 if (proxy is not None):
161 from six.moves.urllib import request as urllib_request
162 class HTTPSProxyTransport(xmlrpc_client.Transport):
163 def __init__(self, proxy, use_datetime=0):
164 opener = urllib_request.build_opener(urllib2.ProxyHandler({"https" : proxy}))
165 xmlrpc_client.Transport.__init__(self, use_datetime)
168 def request(self, host, handler, request_body, verbose=0):
169 req = urllib_request.Request('https://%s%s' % (host, handler), request_body)
170 req.add_header('User-agent', self.user_agent)
171 self.verbose = verbose
172 return self.parse_response(self.opener.open(req))
174 self._proxy_transport = lambda : HTTPSProxyTransport(proxy)
176 self._proxy_transport = lambda : None
178 self.threadlocal = threading.local()
180 # Load blacklist from file
181 if self._ecobj.get_global('planetlab::Node', 'persist_blacklist'):
182 self._set_blacklist()
186 # Cannot reuse same proxy in all threads, py2.7 is not threadsafe
187 return xmlrcp_client.ServerProxy(
189 transport = self._proxy_transport(),
195 return self.threadlocal.mc
196 except AttributeError:
200 # TODO: Use nepi utils Logger instead of warning!!
203 # validate XMLRPC server checking supported API calls
204 methods = set(_retry(self.mcapi.system.listMethods)())
205 if self._required_methods - methods:
206 warnings.warn("Unsupported REQUIRED methods: %s" % (
207 ", ".join(sorted(self._required_methods - methods)), ) )
210 if self._expected_methods - methods:
211 warnings.warn("Unsupported EXPECTED methods: %s" % (
212 ", ".join(sorted(self._expected_methods - methods)), ) )
216 network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
217 except (xmlrpc_client.ProtocolError, xmlrpc_client.Fault) as e:
218 warnings.warn(str(e))
222 def _set_blacklist(self):
223 nepi_home = os.path.join(os.path.expanduser("~"), ".nepi")
224 plblacklist_file = os.path.join(nepi_home, "plblacklist.txt")
225 with open(plblacklist_file, 'r') as f:
226 hosts_tobl = f.read().splitlines()
228 nodes_id = self.get_nodes(hosts_tobl, ['node_id'])
229 for node_id in nodes_id:
230 self._blacklist.add(node_id['node_id'])
233 def network_types(self):
235 return self._network_types
236 except AttributeError:
237 self._network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
238 return self._network_types
243 return self._peer_map
244 except AttributeError:
245 peers = _retry(self.mcapi.GetPeers)(self.auth, {}, ['shortname','peername','peer_id'])
247 self._peer_map = dict(
248 (peer['shortname'], peer['peer_id'])
252 self._peer_map.update(
253 (peer['peername'], peer['peer_id'])
257 self._peer_map.update(
258 (peer['peer_id'], peer['shortname'])
262 self._peer_map[None] = self._local_peer
263 return self._peer_map
265 def get_node_flavour(self, node):
267 Returns detailed information on a given node's flavour,
268 i.e. its base installation.
270 This depends on the global PLC settings in the PLC_FLAVOUR area,
271 optionnally overridden by any of the following tags if set on that node:
272 'arch', 'pldistro', 'fcdistro', 'deployment', 'extensions'
276 * node : int or string
277 - int, Node identifier
278 - string, Fully qualified hostname
283 * extensions : array of string, extensions to add to the base install
284 * fcdistro : string, the fcdistro this node should be based upon
285 * nodefamily : string, the nodefamily this node should be based upon
286 * plain : boolean, use plain bootstrapfs image if set (for tests)
288 if not isinstance(node, integer_types + string_types):
289 raise ValueError("Node must be either a non-unicode string or an int")
290 return _retry(self.mcapi.GetNodeFlavour)(self.auth, node)
292 def get_nodes(self, node_id_or_name = None, fields = None, **kw):
294 Returns an array of structs containing details about nodes.
295 If node_id_or_name is specified and is an array of node identifiers
296 or hostnames, or the filters keyword argument with struct of node
297 attributes, or node attributes by keyword argument,
298 only nodes matching the filter will be returned.
300 If fields is specified, only the specified details will be returned.
301 NOTE that if fields is unspecified, the complete set of native fields are
302 returned, which DOES NOT include tags at this time.
304 Some fields may only be viewed by admins.
308 fields: an optional list of fields to retrieve. The default is all.
310 filters: an optional mapping with custom filters, which is the only
311 way to support complex filters like negation and numeric comparisons.
313 peer: a string (or sequence of strings) with the name(s) of peers
314 to filter - or None for local nodes.
316 if fields is not None:
317 fieldstuple = (fields,)
321 if node_id_or_name is not None:
322 return _retry(self.mcapi.GetNodes)(self.auth, node_id_or_name, *fieldstuple)
324 filters = kw.pop('filters',{})
327 peer = kw.pop('peer')
329 name_to_id = self.peer_map.get
331 if hasattr(peer, '__iter__'):
332 # we can't mix local and external nodes, so
333 # split and re-issue recursively in that case
334 if None in peer or self._local_peer in peer:
338 if self._local_peer in peer:
339 peer.remove(self._local_peer)
342 self.get_nodes(node_id_or_name, fields,
343 filters = filters, peer=peer, **kw) + \
344 self.get_nodes(node_id_or_name, fields,
345 filters = filters, peer=None, **kw)
348 peer_filter = [name_to_id(x) for x in peer]
350 elif peer is None or peer == self._local_peer:
353 peer_filter = name_to_id(peer)
355 filters['peer_id'] = peer_filter
359 if not filters and not fieldstuple:
360 if not self._nodes_cache and not self._already_cached:
361 self._already_cached = True
362 self._nodes_cache = _retry(self.mcapi.GetNodes)(self.auth)
363 elif not self._nodes_cache:
364 while not self._nodes_cache:
366 return self._nodes_cache
368 return _retry(self.mcapi.GetNodes)(self.auth, filters, *fieldstuple)
370 def get_node_tags(self, node_tag_id = None, fields = None, **kw):
371 if fields is not None:
372 fieldstuple = (fields,)
376 if node_tag_id is not None:
377 return _retry(self.mcapi.GetNodeTags)(self.auth, node_tag_id,
380 filters = kw.pop('filters',{})
382 return _retry(self.mcapi.GetNodeTags)(self.auth, filters,
385 def get_slice_tags(self, slice_tag_id = None, fields = None, **kw):
386 if fields is not None:
387 fieldstuple = (fields,)
391 if slice_tag_id is not None:
392 return _retry(self.mcapi.GetSliceTags)(self.auth, slice_tag_id,
395 filters = kw.pop('filters',{})
397 return _retry(self.mcapi.GetSliceTags)(self.auth, filters,
400 def get_interfaces(self, interface_id_or_ip = None, fields = None, **kw):
401 if fields is not None:
402 fieldstuple = (fields,)
406 if interface_id_or_ip is not None:
407 return _retry(self.mcapi.GetInterfaces)(self.auth,
408 interface_id_or_ip, *fieldstuple)
410 filters = kw.pop('filters',{})
412 return _retry(self.mcapi.GetInterfaces)(self.auth, filters,
415 def get_slices(self, slice_id_or_name = None, fields = None, **kw):
416 if fields is not None:
417 fieldstuple = (fields,)
421 if slice_id_or_name is not None:
422 return _retry(self.mcapi.GetSlices)(self.auth, slice_id_or_name,
425 filters = kw.pop('filters',{})
427 return _retry(self.mcapi.GetSlices)(self.auth, filters,
430 def update_slice(self, slice_id_or_name, **kw):
431 return _retry(self.mcapi.UpdateSlice)(self.auth, slice_id_or_name, kw)
433 def delete_slice_node(self, slice_id_or_name, node_id_or_hostname):
434 return _retry(self.mcapi.DeleteSliceFromNodes)(self.auth, slice_id_or_name, node_id_or_hostname)
436 def start_multicall(self):
437 self.threadlocal.mc = xmlrpc_client.MultiCall(self.mcapi)
439 def finish_multicall(self):
440 mc = self.threadlocal.mc
441 del self.threadlocal.mc
444 def get_slice_nodes(self, slicename):
445 return self.get_slices(slicename, ['node_ids'])[0]['node_ids']
447 def add_slice_nodes(self, slicename, nodes):
448 self.update_slice(slicename, nodes=nodes)
450 def get_node_info(self, node_id):
451 self.start_multicall()
452 info = self.get_nodes(node_id)
453 tags = self.get_node_tags(node_id=node_id, fields=('tagname','value'))
454 info, tags = self.finish_multicall()
457 def get_slice_id(self, slicename):
459 slices = self.get_slices(slicename, fields=('slice_id',))
461 slice_id = slices[0]['slice_id']
463 # If it wasn't found, don't remember this failure, keep trying
466 def get_slice_vnet_sys_tag(self, slicename):
467 slicetags = self.get_slice_tags(
469 tagname = 'vsys_vnet',
473 return slicetags[0]['value']
477 def blacklist_host(self, node_id):
478 self._blacklist.add(node_id)
480 def blacklisted(self):
481 return self._blacklist
483 def unblacklist_host(self, node_id):
484 del self._blacklist[node_id]
486 def reserve_host(self, node_id):
487 self._reserved.add(node_id)
490 return self._reserved
492 def unreserve_host(self, node_id):
493 del self._reserved[node_id]
498 blacklist = self._blacklist
499 self._blacklist = set()
500 self._reserved = set()
501 if self._ecobj.get_global('PlanetlabNode', 'persist_blacklist'):
503 to_blacklist = list()
504 hostnames = self.get_nodes(list(blacklist), ['hostname'])
505 for hostname in hostnames:
506 to_blacklist.append(hostname['hostname'])
508 nepi_home = os.path.join(os.path.expanduser("~"), ".nepi")
509 plblacklist_file = os.path.join(nepi_home, "plblacklist.txt")
511 with open(plblacklist_file, 'w') as f:
512 for host in to_blacklist:
513 f.write("%s\n" % host)
516 class PLCAPIFactory(object):
520 It allows PlanetLab RMs sharing a same slice, to use a same plcapi instance,
521 and to sincronize blacklisted and reserved hosts.
524 # use lock to avoid concurrent access to the Api list at the same times by 2 different threads
525 _lock = threading.Lock()
529 def get_api(cls, pl_user, pl_pass, pl_host,
530 pl_ptn, ec, proxy = None):
531 """ Get existing PLCAPI instance
533 :param pl_user: Planelab user name (used for web login)
535 :param pl_pass: Planetlab password (used for web login)
537 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
539 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
541 :param proxy: Proxy service url
544 if pl_user and pl_pass and pl_host:
545 key = cls._make_key(pl_user, pl_host)
547 api = cls._apis.get(key)
549 api = cls.create_api(pl_user, pl_pass, pl_host, pl_ptn, ec, proxy)
556 def create_api(cls, pl_user, pl_pass, pl_host,
557 pl_ptn, ec, proxy = None):
558 """ Create an PLCAPI instance
560 :param pl_user: Planelab user name (used for web login)
562 :param pl_pass: Planetlab password (used for web login)
564 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
566 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
568 :param proxy: Proxy service url
571 api = PLCAPI(username = pl_user, password = pl_pass, hostname = pl_host,
572 urlpattern = pl_ptn, ec = ec, proxy = proxy)
573 key = cls._make_key(pl_user, pl_host)
578 def _make_key(cls, *args):
579 """ Hash the credentials in order to create a key
581 :param args: list of arguments used to create the hash (user, host, port, ...)
582 :type args: list of args
585 skey = "".join(map(str, args))
586 return hashlib.md5(skey).hexdigest()