2 # NEPI, a framework to manage network experiments
3 # Copyright (C) 2013 INRIA
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
33 except (socket.error, IOError, OSError):
41 _expected_methods = set(
42 ['AddNodeTag', 'AddConfFile', 'DeletePersonTag', 'AddNodeType',
43 'DeleteBootState', 'SliceListNames', 'DeleteKey','SliceGetTicket',
44 'SliceUsersList', 'SliceUpdate', 'GetNodeGroups', 'SliceCreate',
45 'GetNetworkMethods', 'GetNodeFlavour', 'DeleteNode', 'BootNotifyOwners',
46 'AddPersonKey', 'AddNode', 'UpdateNodeGroup', 'GetAddressTypes',
47 'AddIlink', 'DeleteNetworkType', 'GetInitScripts', 'GenerateNodeConfFile',
48 'AddSite', 'BindObjectToPeer', 'SliceListUserSlices', 'GetPeers',
49 'AddPeer', 'DeletePeer', 'AddRole', 'DeleteRole', 'SetPersonPrimarySite',
50 'AddSiteAddress', 'SliceDelete', 'NotifyPersons', 'GetKeyTypes',
51 'GetConfFiles', 'GetIlinks', 'AddTagType', 'GetNodes', 'DeleteNodeTag',
52 'DeleteSliceFromNodesWhitelist', 'UpdateAddress', 'ResetPassword',
53 'AddSliceToNodesWhitelist', 'AddRoleToTagType', 'AddLeases',
54 'GetAddresses', 'AddInitScript', 'RebootNode', 'GetPCUTypes',
55 'RefreshPeer', 'GetBootMedium', 'UpdateKey', 'UpdatePCU', 'GetSession',
56 'AddInterfaceTag', 'UpdatePCUType', 'GetInterfaces', 'SliceExtendedInfo',
57 'SliceNodesList', 'DeleteRoleFromTagType', 'DeleteSlice', 'GetSites',
58 'DeleteMessage', 'GetSliceFamily', 'GetPlcRelease', 'UpdateTagType',
59 'AddSliceInstantiation', 'ResolveSlices', 'GetSlices',
60 'DeleteRoleFromPerson', 'GetSessions', 'UpdatePeer', 'VerifyPerson',
61 'GetPersonTags', 'DeleteKeyType', 'AddSlice', 'SliceUserAdd',
62 'DeleteSession', 'GetMessages', 'DeletePCU', 'GetPeerData',
63 'DeletePersonFromSite', 'DeleteTagType', 'GetPCUs', 'UpdateLeases',
64 'AddMessage', 'DeletePCUProtocolType', 'DeleteInterfaceTag',
65 'AddPersonToSite', 'GetSlivers', 'SliceNodesDel',
66 'DeleteAddressTypeFromAddress', 'AddNodeGroup', 'GetSliceTags',
67 'DeleteSite', 'GetSiteTags', 'UpdateMessage', 'DeleteSliceFromNodes',
68 'SliceRenew', 'UpdatePCUProtocolType', 'DeleteSiteTag',
69 'GetPCUProtocolTypes', 'GetEvents', 'GetSliceTicket', 'AddPersonTag',
70 'BootGetNodeDetails', 'DeleteInterface', 'DeleteNodeGroup',
71 'AddPCUProtocolType', 'BootCheckAuthentication', 'AddSiteTag',
72 'AddAddressTypeToAddress', 'DeleteConfFile', 'DeleteInitScript',
73 'DeletePerson', 'DeleteIlink', 'DeleteAddressType', 'AddBootState',
74 'AuthCheck', 'NotifySupport', 'GetSliceInstantiations', 'AddPCUType',
75 'AddPCU', 'AddSession', 'GetEventObjects', 'UpdateSiteTag',
76 'UpdateNodeTag', 'AddPerson', 'BlacklistKey', 'UpdateInitScript',
77 'AddSliceToNodes', 'RebootNodeWithPCU', 'GetNodeTags', 'GetSliceKeys',
78 'GetSliceSshKeys', 'AddNetworkMethod', 'SliceNodesAdd',
79 'DeletePersonFromSlice', 'ReportRunlevel', 'GetNetworkTypes',
80 'UpdateSite', 'DeleteConfFileFromNodeGroup', 'UpdateNode',
81 'DeleteSliceInstantiation', 'DeleteSliceTag', 'BootUpdateNode',
82 'UpdatePerson', 'UpdateConfFile', 'SliceUserDel', 'DeleteLeases',
83 'AddConfFileToNodeGroup', 'UpdatePersonTag', 'DeleteConfFileFromNode',
84 'AddPersonToSlice', 'UnBindObjectFromPeer', 'AddNodeToPCU',
85 'GetLeaseGranularity', 'DeletePCUType', 'GetTagTypes', 'GetNodeTypes',
86 'UpdateInterfaceTag', 'GetRoles', 'UpdateSlice', 'UpdateSliceTag',
87 'AddSliceTag', 'AddNetworkType', 'AddInterface', 'AddAddressType',
88 'AddRoleToPerson', 'DeleteNodeType', 'GetLeases', 'UpdateInterface',
89 'SliceInfo', 'DeleteAddress', 'SliceTicketGet', 'GetPersons',
90 'GetWhitelist', 'AddKeyType', 'UpdateAddressType', 'GetPeerName',
91 'DeleteNetworkMethod', 'UpdateIlink', 'AddConfFileToNode', 'GetKeys',
92 'DeleteNodeFromPCU', 'GetInterfaceTags', 'GetBootStates',
93 'SetInterfaceSens', 'SetNodeLoadm', 'GetInterfaceRate', 'GetNodeLoadw',
94 'SetInterfaceKey', 'GetNodeSlices', 'GetNodeLoadm', 'SetSliceVref',
95 'GetInterfaceIwpriv', 'SetNodeLoadw', 'SetNodeSerial',
96 'GetNodePlainBootstrapfs', 'SetNodeMEMw', 'GetNodeResponse',
97 'SetInterfaceRate', 'SetSliceInitscript', 'SetNodeFcdistro',
98 'GetNodeLoady', 'SetNodeArch', 'SetNodeKargs', 'SetNodeMEMm',
99 'SetNodeBWy', 'SetNodeBWw', 'SetInterfaceSecurityMode', 'SetNodeBWm',
100 'SetNodeASType', 'GetNodeKargs', 'GetPersonColumnconf',
101 'GetNodeResponsem', 'GetNodeCPUy', 'GetNodeCramfs', 'SetNodeSlicesw',
102 'SetPersonColumnconf', 'SetNodeSlicesy', 'GetNodeCPUw', 'GetNodeBWy',
103 'GetNodeCPUm', 'GetInterfaceDriver', 'GetNodeLoad', 'GetInterfaceMode',
104 'GetNodeSerial', 'SetNodeSlicesm', 'SetNodeLoady', 'GetNodeReliabilityw',
105 'SetSliceFcdistro', 'GetNodeReliabilityy', 'SetInterfaceEssid',
106 'SetSliceInitscriptCode', 'GetNodeExtensions', 'GetSliceOmfControl',
107 'SetNodeCity', 'SetInterfaceIfname', 'SetNodeHrn', 'SetNodeNoHangcheck',
108 'GetNodeNoHangcheck', 'GetSliceFcdistro', 'SetNodeCountry',
109 'SetNodeKvariant', 'GetNodeKvariant', 'GetNodeMEMy', 'SetInterfaceIwpriv',
110 'GetNodeMEMw', 'SetInterfaceBackdoor', 'GetInterfaceFreq',
111 'SetInterfaceChannel', 'SetInterfaceNw', 'GetPersonShowconf',
112 'GetSliceInitscriptCode', 'SetNodeMEM', 'GetInterfaceEssid', 'GetNodeMEMm',
113 'SetInterfaceMode', 'SetInterfaceIwconfig', 'GetNodeSlicesm', 'GetNodeBWm',
114 'SetNodePlainBootstrapfs', 'SetNodeRegion', 'SetNodeCPU', 'GetNodeSlicesw',
115 'SetNodeBW', 'SetNodeSlices', 'SetNodeCramfs', 'GetNodeSlicesy',
116 'GetInterfaceKey', 'GetSliceInitscript', 'SetNodeCPUm', 'SetSliceArch',
117 'SetNodeLoad', 'SetNodeResponse', 'GetSliceSliverHMAC', 'GetNodeBWw',
118 'GetNodeRegion', 'SetNodeMEMy', 'GetNodeASType', 'SetNodePldistro',
119 'GetSliceArch', 'GetNodeCountry', 'SetSliceOmfControl', 'GetNodeHrn',
120 'GetNodeCity', 'SetInterfaceAlias', 'GetNodeBW', 'GetNodePldistro',
121 'GetSlicePldistro', 'SetNodeASNumber', 'GetSliceHmac', 'SetSliceHmac',
122 'GetNodeMEM', 'GetNodeASNumber', 'GetInterfaceAlias', 'GetSliceVref',
123 'GetNodeArch', 'GetSliceSshKey', 'GetInterfaceKey4', 'GetInterfaceKey2',
124 'GetInterfaceKey3', 'GetInterfaceKey1', 'GetInterfaceBackdoor',
125 'GetInterfaceIfname', 'SetSliceSliverHMAC', 'SetNodeReliability',
126 'GetNodeCPU', 'SetPersonShowconf', 'SetNodeExtensions', 'SetNodeCPUy',
127 'SetNodeCPUw', 'GetNodeResponsew', 'SetNodeResponsey', 'GetInterfaceSens',
128 'SetNodeResponsew', 'GetNodeResponsey', 'GetNodeReliability',
129 'GetNodeReliabilitym', 'SetNodeResponsem', 'SetInterfaceDriver',
130 'GetInterfaceSecurityMode', 'SetNodeDeployment', 'SetNodeReliabilitym',
131 'GetNodeFcdistro', 'SetInterfaceFreq', 'GetInterfaceNw',
132 'SetNodeReliabilityy', 'SetNodeReliabilityw', 'GetInterfaceIwconfig',
133 'SetSlicePldistro', 'SetSliceSshKey', 'GetNodeDeployment',
134 'GetInterfaceChannel', 'SetInterfaceKey2', 'SetInterfaceKey3',
135 'SetInterfaceKey1', 'SetInterfaceKey4'])
137 _required_methods = set()
139 def __init__(self, username, password, hostname, urlpattern, ec, proxy, session_key = None,
142 self._blacklist = set()
143 self._reserved = set()
144 self._nodes_cache = None
145 self._already_cached = False
149 if session_key is not None:
150 self.auth = dict(AuthMethod='session', session=session_key)
151 elif username is not None and password is not None:
152 self.auth = dict(AuthMethod='password', Username=username, AuthString=password)
154 self.auth = dict(AuthMethod='anonymous')
156 self._local_peer = local_peer
157 self._url = urlpattern % {'hostname':hostname}
159 if (proxy is not None):
161 class HTTPSProxyTransport(xmlrpclib.Transport):
162 def __init__(self, proxy, use_datetime=0):
163 opener = urllib2.build_opener(urllib2.ProxyHandler({"https" : proxy}))
164 xmlrpclib.Transport.__init__(self, use_datetime)
167 def request(self, host, handler, request_body, verbose=0):
168 req = urllib2.Request('https://%s%s' % (host, handler), request_body)
169 req.add_header('User-agent', self.user_agent)
170 self.verbose = verbose
171 return self.parse_response(self.opener.open(req))
173 self._proxy_transport = lambda : HTTPSProxyTransport(proxy)
175 self._proxy_transport = lambda : None
177 self.threadlocal = threading.local()
179 # Load blacklist from file
180 if self._ecobj.get_global('PlanetlabNode', 'persist_blacklist'):
181 self._set_blacklist()
185 # Cannot reuse same proxy in all threads, py2.7 is not threadsafe
186 return xmlrpclib.ServerProxy(
188 transport = self._proxy_transport(),
194 return self.threadlocal.mc
195 except AttributeError:
199 # TODO: Use nepi utils Logger instead of warning!!
202 # validate XMLRPC server checking supported API calls
203 methods = set(_retry(self.mcapi.system.listMethods)())
204 if self._required_methods - methods:
205 warnings.warn("Unsupported REQUIRED methods: %s" % (
206 ", ".join(sorted(self._required_methods - methods)), ) )
209 if self._expected_methods - methods:
210 warnings.warn("Unsupported EXPECTED methods: %s" % (
211 ", ".join(sorted(self._expected_methods - methods)), ) )
215 network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
216 except (xmlrpclib.ProtocolError, xmlrpclib.Fault),e:
217 warnings.warn(str(e))
221 def _set_blacklist(self):
222 nepi_home = os.path.join(os.path.expanduser("~"), ".nepi")
223 plblacklist_file = os.path.join(nepi_home, "plblacklist.txt")
224 with open(plblacklist_file, 'r') as f:
225 hosts_tobl = f.read().splitlines()
227 nodes_id = self.get_nodes(hosts_tobl, ['node_id'])
228 for node_id in nodes_id:
229 self._blacklist.add(node_id['node_id'])
232 def network_types(self):
234 return self._network_types
235 except AttributeError:
236 self._network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
237 return self._network_types
242 return self._peer_map
243 except AttributeError:
244 peers = _retry(self.mcapi.GetPeers)(self.auth, {}, ['shortname','peername','peer_id'])
246 self._peer_map = dict(
247 (peer['shortname'], peer['peer_id'])
251 self._peer_map.update(
252 (peer['peername'], peer['peer_id'])
256 self._peer_map.update(
257 (peer['peer_id'], peer['shortname'])
261 self._peer_map[None] = self._local_peer
262 return self._peer_map
264 def get_node_flavour(self, node):
266 Returns detailed information on a given node's flavour,
267 i.e. its base installation.
269 This depends on the global PLC settings in the PLC_FLAVOUR area,
270 optionnally overridden by any of the following tags if set on that node:
271 'arch', 'pldistro', 'fcdistro', 'deployment', 'extensions'
275 * node : int or string
276 - int, Node identifier
277 - string, Fully qualified hostname
282 * extensions : array of string, extensions to add to the base install
283 * fcdistro : string, the fcdistro this node should be based upon
284 * nodefamily : string, the nodefamily this node should be based upon
285 * plain : boolean, use plain bootstrapfs image if set (for tests)
287 if not isinstance(node, (str, int, long)):
288 raise ValueError, "Node must be either a non-unicode string or an int"
289 return _retry(self.mcapi.GetNodeFlavour)(self.auth, node)
291 def get_nodes(self, node_id_or_name = None, fields = None, **kw):
293 Returns an array of structs containing details about nodes.
294 If node_id_or_name is specified and is an array of node identifiers
295 or hostnames, or the filters keyword argument with struct of node
296 attributes, or node attributes by keyword argument,
297 only nodes matching the filter will be returned.
299 If fields is specified, only the specified details will be returned.
300 NOTE that if fields is unspecified, the complete set of native fields are
301 returned, which DOES NOT include tags at this time.
303 Some fields may only be viewed by admins.
307 fields: an optional list of fields to retrieve. The default is all.
309 filters: an optional mapping with custom filters, which is the only
310 way to support complex filters like negation and numeric comparisons.
312 peer: a string (or sequence of strings) with the name(s) of peers
313 to filter - or None for local nodes.
315 if fields is not None:
316 fieldstuple = (fields,)
320 if node_id_or_name is not None:
321 return _retry(self.mcapi.GetNodes)(self.auth, node_id_or_name, *fieldstuple)
323 filters = kw.pop('filters',{})
326 peer = kw.pop('peer')
328 name_to_id = self.peer_map.get
330 if hasattr(peer, '__iter__'):
331 # we can't mix local and external nodes, so
332 # split and re-issue recursively in that case
333 if None in peer or self._local_peer in peer:
337 if self._local_peer in peer:
338 peer.remove(self._local_peer)
341 self.get_nodes(node_id_or_name, fields,
342 filters = filters, peer=peer, **kw) + \
343 self.get_nodes(node_id_or_name, fields,
344 filters = filters, peer=None, **kw)
347 peer_filter = map(name_to_id, peer)
349 elif peer is None or peer == self._local_peer:
352 peer_filter = name_to_id(peer)
354 filters['peer_id'] = peer_filter
358 if not filters and not fieldstuple:
359 if not self._nodes_cache and not self._already_cached:
360 self._already_cached = True
361 self._nodes_cache = _retry(self.mcapi.GetNodes)(self.auth)
362 elif not self._nodes_cache:
363 while not self._nodes_cache:
365 return self._nodes_cache
367 return _retry(self.mcapi.GetNodes)(self.auth, filters, *fieldstuple)
369 def get_node_tags(self, node_tag_id = None, fields = None, **kw):
370 if fields is not None:
371 fieldstuple = (fields,)
375 if node_tag_id is not None:
376 return _retry(self.mcapi.GetNodeTags)(self.auth, node_tag_id,
379 filters = kw.pop('filters',{})
381 return _retry(self.mcapi.GetNodeTags)(self.auth, filters,
384 def get_slice_tags(self, slice_tag_id = None, fields = None, **kw):
385 if fields is not None:
386 fieldstuple = (fields,)
390 if slice_tag_id is not None:
391 return _retry(self.mcapi.GetSliceTags)(self.auth, slice_tag_id,
394 filters = kw.pop('filters',{})
396 return _retry(self.mcapi.GetSliceTags)(self.auth, filters,
399 def get_interfaces(self, interface_id_or_ip = None, fields = None, **kw):
400 if fields is not None:
401 fieldstuple = (fields,)
405 if interface_id_or_ip is not None:
406 return _retry(self.mcapi.GetInterfaces)(self.auth,
407 interface_id_or_ip, *fieldstuple)
409 filters = kw.pop('filters',{})
411 return _retry(self.mcapi.GetInterfaces)(self.auth, filters,
414 def get_slices(self, slice_id_or_name = None, fields = None, **kw):
415 if fields is not None:
416 fieldstuple = (fields,)
420 if slice_id_or_name is not None:
421 return _retry(self.mcapi.GetSlices)(self.auth, slice_id_or_name,
424 filters = kw.pop('filters',{})
426 return _retry(self.mcapi.GetSlices)(self.auth, filters,
429 def update_slice(self, slice_id_or_name, **kw):
430 return _retry(self.mcapi.UpdateSlice)(self.auth, slice_id_or_name, kw)
432 def delete_slice_node(self, slice_id_or_name, node_id_or_hostname):
433 return _retry(self.mcapi.DeleteSliceFromNodes)(self.auth, slice_id_or_name, node_id_or_hostname)
435 def start_multicall(self):
436 self.threadlocal.mc = xmlrpclib.MultiCall(self.mcapi)
438 def finish_multicall(self):
439 mc = self.threadlocal.mc
440 del self.threadlocal.mc
443 def get_slice_nodes(self, slicename):
444 return self.get_slices(slicename, ['node_ids'])[0]['node_ids']
446 def add_slice_nodes(self, slicename, nodes):
447 self.update_slice(slicename, nodes=nodes)
449 def get_node_info(self, node_id):
450 self.start_multicall()
451 info = self.get_nodes(node_id)
452 tags = self.get_node_tags(node_id=node_id, fields=('tagname','value'))
453 info, tags = self.finish_multicall()
456 def get_slice_id(self, slicename):
458 slices = self.get_slices(slicename, fields=('slice_id',))
460 slice_id = slices[0]['slice_id']
462 # If it wasn't found, don't remember this failure, keep trying
465 def get_slice_vnet_sys_tag(self, slicename):
466 slicetags = self.get_slice_tags(
468 tagname = 'vsys_vnet',
472 return slicetags[0]['value']
476 def blacklist_host(self, node_id):
477 self._blacklist.add(node_id)
479 def blacklisted(self):
480 return self._blacklist
482 def unblacklist_host(self, node_id):
483 del self._blacklist[node_id]
485 def reserve_host(self, node_id):
486 self._reserved.add(node_id)
489 return self._reserved
491 def unreserve_host(self, node_id):
492 del self._reserved[node_id]
497 blacklist = self._blacklist
498 self._blacklist = set()
499 self._reserved = set()
500 if self._ecobj.get_global('PlanetlabNode', 'persist_blacklist'):
502 to_blacklist = list()
503 hostnames = self.get_nodes(list(blacklist), ['hostname'])
504 for hostname in hostnames:
505 to_blacklist.append(hostname['hostname'])
507 nepi_home = os.path.join(os.path.expanduser("~"), ".nepi")
508 plblacklist_file = os.path.join(nepi_home, "plblacklist.txt")
510 with open(plblacklist_file, 'w') as f:
511 for host in to_blacklist:
512 f.write("%s\n" % host)
515 class PLCAPIFactory(object):
519 It allows PlanetLab RMs sharing a same slice, to use a same plcapi instance,
520 and to sincronize blacklisted and reserved hosts.
523 # use lock to avoid concurrent access to the Api list at the same times by 2 different threads
524 _lock = threading.Lock()
528 def get_api(cls, pl_user, pl_pass, pl_host,
529 pl_ptn, ec, proxy = None):
530 """ Get existing PLCAPI instance
532 :param pl_user: Planelab user name (used for web login)
534 :param pl_pass: Planetlab password (used for web login)
536 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
538 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
540 :param proxy: Proxy service url
543 if pl_user and pl_pass and pl_host:
544 key = cls._make_key(pl_user, pl_host)
546 api = cls._apis.get(key)
548 api = cls.create_api(pl_user, pl_pass, pl_host, pl_ptn, ec, proxy)
555 def create_api(cls, pl_user, pl_pass, pl_host,
556 pl_ptn, ec, proxy = None):
557 """ Create an PLCAPI instance
559 :param pl_user: Planelab user name (used for web login)
561 :param pl_pass: Planetlab password (used for web login)
563 :param pl_host: Planetlab registry host (e.g. "www.planet-lab.eu")
565 :param pl_ptn: XMLRPC service pattern (e.g. https://%(hostname)s:443/PLCAPI/)
567 :param proxy: Proxy service url
570 api = PLCAPI(username = pl_user, password = pl_pass, hostname = pl_host,
571 urlpattern = pl_ptn, ec = ec, proxy = proxy)
572 key = cls._make_key(pl_user, pl_host)
577 def _make_key(cls, *args):
578 """ Hash the credentials in order to create a key
580 :param args: list of arguments used to create the hash (user, host, port, ...)
581 :type args: list of args
584 skey = "".join(map(str, args))
585 return hashlib.md5(skey).hexdigest()