Fix the txqueuelen TUN/TAP attribute (typo)
[nepi.git] / src / nepi / testbeds / planetlab / metadata.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import time
5
6 from constants import TESTBED_ID, TESTBED_VERSION
7 from nepi.core import metadata
8 from nepi.core.metadata import Parallel
9 from nepi.core.attributes import Attribute
10 from nepi.util import tags, validation
11 from nepi.util.constants import ApplicationStatus as AS, \
12         FactoryCategories as FC, \
13         ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP, \
14         DeploymentConfiguration as DC
15
16 import functools
17 import os
18 import os.path
19
20 NODE = "Node"
21 NODEIFACE = "NodeInterface"
22 TUNIFACE = "TunInterface"
23 TAPIFACE = "TapInterface"
24 APPLICATION = "Application"
25 DEPENDENCY = "Dependency"
26 NEPIDEPENDENCY = "NepiDependency"
27 NS3DEPENDENCY = "NS3Dependency"
28 INTERNET = "Internet"
29 NETPIPE = "NetPipe"
30
31 PL_TESTBED_ID = "planetlab"
32
33
34 ### Custom validation functions ###
35 def is_addrlist(attribute, value):
36     if not validation.is_string(attribute, value):
37         return False
38     
39     if not value:
40         # No empty strings
41         return False
42     
43     components = value.split(',')
44     
45     for component in components:
46         if '/' in component:
47             addr, mask = component.split('/',1)
48         else:
49             addr, mask = component, '32'
50         
51         if mask is not None and not (mask and mask.isdigit()):
52             # No empty or nonnumeric masks
53             return False
54         
55         if not validation.is_ip4_address(attribute, addr):
56             # Address part must be ipv4
57             return False
58         
59     return True
60
61 def is_portlist(attribute, value):
62     if not validation.is_string(attribute, value):
63         return False
64     
65     if not value:
66         # No empty strings
67         return False
68     
69     components = value.split(',')
70     
71     for component in components:
72         if '-' in component:
73             pfrom, pto = component.split('-',1)
74         else:
75             pfrom = pto = component
76         
77         if not pfrom or not pto or not pfrom.isdigit() or not pto.isdigit():
78             # No empty or nonnumeric ports
79             return False
80         
81     return True
82
83
84 ### Connection functions ####
85
86 def connect_node_iface_node(testbed_instance, node_guid, iface_guid):
87     node = testbed_instance._elements[node_guid]
88     iface = testbed_instance._elements[iface_guid]
89     iface.node = node
90
91 def connect_node_iface_inet(testbed_instance, iface_guid, inet_guid):
92     iface = testbed_instance._elements[iface_guid]
93     iface.has_internet = True
94
95 def connect_tun_iface_node(testbed_instance, node_guid, iface_guid):
96     node = testbed_instance._elements[node_guid]
97     iface = testbed_instance._elements[iface_guid]
98     iface.node = node
99     node.required_vsys.update(('fd_tuntap', 'vif_up', 'vif_down'))
100     node.required_packages.update(('python', 'python-crypto', 'python-setuptools', 'gcc'))
101
102 def connect_tun_iface_peer(proto, testbed_instance, iface_guid, peer_iface_guid):
103     iface = testbed_instance._elements[iface_guid]
104     peer_iface = testbed_instance._elements[peer_iface_guid]
105     iface.peer_iface = peer_iface
106     iface.peer_proto = \
107     iface.tun_proto = proto
108     iface.tun_key = peer_iface.tun_key
109
110 def crossconnect_tun_iface_peer_init(proto, testbed_instance, iface_guid, peer_iface_data):
111     iface = testbed_instance._elements[iface_guid]
112     iface.peer_iface = None
113     iface.peer_addr = peer_iface_data.get("tun_addr")
114     iface.peer_proto = peer_iface_data.get("tun_proto") or proto
115     iface.peer_port = peer_iface_data.get("tun_port")
116     iface.tun_key = min(iface.tun_key, peer_iface_data.get("tun_key"))
117     iface.tun_proto = proto
118     
119     preconfigure_tuniface(testbed_instance, iface_guid)
120
121 def crossconnect_tun_iface_peer_compl(proto, testbed_instance, iface_guid, peer_iface_data):
122     # refresh (refreshable) attributes for second-phase
123     iface = testbed_instance._elements[iface_guid]
124     iface.peer_addr = peer_iface_data.get("tun_addr")
125     iface.peer_proto = peer_iface_data.get("tun_proto") or proto
126     iface.peer_port = peer_iface_data.get("tun_port")
127     
128     postconfigure_tuniface(testbed_instance, iface_guid)
129
130 def crossconnect_tun_iface_peer_both(proto, testbed_instance, iface_guid, peer_iface_data):
131     crossconnect_tun_iface_peer_init(proto, testbed_instance, iface_guid, peer_iface_data)
132     crossconnect_tun_iface_peer_compl(proto, testbed_instance, iface_guid, peer_iface_data)
133
134 def connect_dep(testbed_instance, node_guid, app_guid):
135     node = testbed_instance._elements[node_guid]
136     app = testbed_instance._elements[app_guid]
137     app.node = node
138     
139     if app.depends:
140         node.required_packages.update(set(
141             app.depends.split() ))
142     
143     if app.add_to_path:
144         if app.home_path and app.home_path not in node.pythonpath:
145             node.pythonpath.append(app.home_path)
146     
147     if app.env:
148         for envkey, envval in app.env.iteritems():
149             envval = app._replace_paths(envval)
150             node.env[envkey].append(envval)
151     
152     if app.rpmFusion:
153         node.rpmFusion = True
154
155 def connect_node_netpipe(testbed_instance, node_guid, netpipe_guid):
156     node = testbed_instance._elements[node_guid]
157     netpipe = testbed_instance._elements[netpipe_guid]
158     netpipe.node = node
159     node.required_vsys.add('ipfw-be')
160     node.required_packages.add('ipfwslice')
161     
162
163 ### Creation functions ###
164
165 def create_node(testbed_instance, guid):
166     parameters = testbed_instance._get_parameters(guid)
167     
168     # create element with basic attributes
169     element = testbed_instance._make_node(parameters)
170     
171     # add constraint on number of (real) interfaces
172     # by counting connected devices
173     dev_guids = testbed_instance.get_connected(guid, "devs", "node")
174     num_open_ifaces = sum( # count True values
175         NODEIFACE == testbed_instance._get_factory_id(guid)
176         for guid in dev_guids )
177     element.min_num_external_ifaces = num_open_ifaces
178     
179     # require vroute vsys if we have routes to set up
180     routes = testbed_instance._add_route.get(guid)
181     if routes:
182         element.required_vsys.add("vroute")
183     
184     testbed_instance.elements[guid] = element
185
186 def create_nodeiface(testbed_instance, guid):
187     parameters = testbed_instance._get_parameters(guid)
188     element = testbed_instance._make_node_iface(parameters)
189     testbed_instance.elements[guid] = element
190
191 def create_tuniface(testbed_instance, guid):
192     parameters = testbed_instance._get_parameters(guid)
193     element = testbed_instance._make_tun_iface(parameters)
194     
195     # Set custom addresses, if there are any already
196     # Setting this early helps set up P2P links
197     if guid in testbed_instance._add_address and not (element.address or element.netmask or element.netprefix):
198         addresses = testbed_instance._add_address[guid]
199         for address in addresses:
200             (address, netprefix, broadcast) = address
201             element.add_address(address, netprefix, broadcast)
202     
203     testbed_instance.elements[guid] = element
204
205 def create_tapiface(testbed_instance, guid):
206     parameters = testbed_instance._get_parameters(guid)
207     element = testbed_instance._make_tap_iface(parameters)
208     
209     # Set custom addresses, if there are any already
210     # Setting this early helps set up P2P links
211     if guid in testbed_instance._add_address and not (element.address or element.netmask or element.netprefix):
212         addresses = testbed_instance._add_address[guid]
213         for address in addresses:
214             (address, netprefix, broadcast) = address
215             element.add_address(address, netprefix, broadcast)
216     
217     testbed_instance.elements[guid] = element
218
219 def create_application(testbed_instance, guid):
220     parameters = testbed_instance._get_parameters(guid)
221     element = testbed_instance._make_application(parameters)
222     
223     # Just inject configuration stuff
224     element.home_path = "nepi-app-%s" % (guid,)
225     
226     testbed_instance.elements[guid] = element
227
228 def create_dependency(testbed_instance, guid):
229     parameters = testbed_instance._get_parameters(guid)
230     element = testbed_instance._make_dependency(parameters)
231     
232     # Just inject configuration stuff
233     element.home_path = "nepi-dep-%s" % (guid,)
234     
235     testbed_instance.elements[guid] = element
236
237 def create_nepi_dependency(testbed_instance, guid):
238     parameters = testbed_instance._get_parameters(guid)
239     element = testbed_instance._make_nepi_dependency(parameters)
240     
241     # Just inject configuration stuff
242     element.home_path = "nepi-nepi-%s" % (guid,)
243     
244     testbed_instance.elements[guid] = element
245
246 def create_ns3_dependency(testbed_instance, guid):
247     parameters = testbed_instance._get_parameters(guid)
248     element = testbed_instance._make_ns3_dependency(parameters)
249     
250     # Just inject configuration stuff
251     element.home_path = "nepi-ns3-%s" % (guid,)
252     
253     testbed_instance.elements[guid] = element
254
255 def create_internet(testbed_instance, guid):
256     parameters = testbed_instance._get_parameters(guid)
257     element = testbed_instance._make_internet(parameters)
258     testbed_instance.elements[guid] = element
259
260 def create_netpipe(testbed_instance, guid):
261     parameters = testbed_instance._get_parameters(guid)
262     element = testbed_instance._make_netpipe(parameters)
263     testbed_instance.elements[guid] = element
264
265 ### Start/Stop functions ###
266
267 def start_application(testbed_instance, guid):
268     parameters = testbed_instance._get_parameters(guid)
269     traces = testbed_instance._get_traces(guid)
270     app = testbed_instance.elements[guid]
271     
272     app.stdout = "stdout" in traces
273     app.stderr = "stderr" in traces
274     app.buildlog = "buildlog" in traces
275     
276     app.start()
277
278 def stop_application(testbed_instance, guid):
279     app = testbed_instance.elements[guid]
280     app.kill()
281
282 ### Status functions ###
283
284 def status_application(testbed_instance, guid):
285     if guid not in testbed_instance.elements.keys():
286         return AS.STATUS_NOT_STARTED
287     
288     app = testbed_instance.elements[guid]
289     return app.status()
290
291 ### Configure functions ###
292
293 def configure_nodeiface(testbed_instance, guid):
294     element = testbed_instance._elements[guid]
295     
296     # Cannot explicitly configure addresses
297     if guid in testbed_instance._add_address:
298         raise ValueError, "Cannot explicitly set address of public PlanetLab interface"
299     
300     # Get siblings
301     node_guid = testbed_instance.get_connected(guid, "node", "devs")[0]
302     dev_guids = testbed_instance.get_connected(node_guid, "node", "devs")
303     siblings = [ self._element[dev_guid] 
304                  for dev_guid in dev_guids
305                  if dev_guid != guid ]
306     
307     # Fetch address from PLC api
308     element.pick_iface(siblings)
309     
310     # Do some validations
311     element.validate()
312
313 def preconfigure_tuniface(testbed_instance, guid):
314     element = testbed_instance._elements[guid]
315     
316     # Set custom addresses if any, and if not set already
317     if guid in testbed_instance._add_address and not (element.address or element.netmask or element.netprefix):
318         addresses = testbed_instance._add_address[guid]
319         for address in addresses:
320             (address, netprefix, broadcast) = address
321             element.add_address(address, netprefix, broadcast)
322     
323     # Link to external interface, if any
324     for iface in testbed_instance._elements.itervalues():
325         if isinstance(iface, testbed_instance._interfaces.NodeIface) and iface.node is element.node and iface.has_internet:
326             element.external_iface = iface
327             break
328
329     # Set standard TUN attributes
330     if (not element.tun_addr or not element.tun_port) and element.external_iface:
331         element.tun_addr = element.external_iface.address
332         element.tun_port = testbed_instance.tapPortBase + int(guid)
333
334     # Set enabled traces
335     traces = testbed_instance._get_traces(guid)
336     for capmode in ('pcap', 'packets'):
337         if capmode in traces:
338             element.capture = capmode
339             break
340     else:
341         element.capture = False
342     
343     # Do some validations
344     element.validate()
345     
346     # First-phase setup
347     if element.peer_proto:
348         if element.peer_iface and isinstance(element.peer_iface, testbed_instance._interfaces.TunIface):
349             # intra tun
350             listening = id(element) < id(element.peer_iface)
351         else:
352             # cross tun
353             if not element.tun_addr or not element.tun_port:
354                 listening = True
355             elif not element.peer_addr or not element.peer_port:
356                 listening = True
357             else:
358                 # both have addresses...
359                 # ...the one with the lesser address listens
360                 listening = element.tun_addr < element.peer_addr
361         element.prepare( 
362             'tun-%s' % (guid,),
363              listening)
364
365 def postconfigure_tuniface(testbed_instance, guid):
366     element = testbed_instance._elements[guid]
367     
368     # Second-phase setup
369     element.setup()
370     
371 def wait_tuniface(testbed_instance, guid):
372     element = testbed_instance._elements[guid]
373     
374     # Second-phase setup
375     element.async_launch_wait()
376     
377
378 def configure_node(testbed_instance, guid):
379     node = testbed_instance._elements[guid]
380     
381     # Just inject configuration stuff
382     node.home_path = "nepi-node-%s" % (guid,)
383     node.ident_path = testbed_instance.sliceSSHKey
384     node.slicename = testbed_instance.slicename
385     
386     # Do some validations
387     node.validate()
388     
389     # this will be done in parallel in all nodes
390     # this call only spawns the process
391     node.install_dependencies()
392
393 def configure_node_routes(testbed_instance, guid):
394     node = testbed_instance._elements[guid]
395     routes = testbed_instance._add_route.get(guid)
396     
397     if routes:
398         devs = [ dev
399             for dev_guid in testbed_instance.get_connected(guid, "devs", "node")
400             for dev in ( testbed_instance._elements.get(dev_guid) ,)
401             if dev and isinstance(dev, testbed_instance._interfaces.TunIface) ]
402         
403         node.configure_routes(routes, devs)
404
405 def configure_application(testbed_instance, guid):
406     app = testbed_instance._elements[guid]
407     
408     # Do some validations
409     app.validate()
410     
411     # Wait for dependencies
412     app.node.wait_dependencies()
413     
414     # Install stuff
415     app.async_setup()
416
417 def configure_dependency(testbed_instance, guid):
418     dep = testbed_instance._elements[guid]
419     
420     # Do some validations
421     dep.validate()
422     
423     # Wait for dependencies
424     dep.node.wait_dependencies()
425     
426     # Install stuff
427     dep.async_setup()
428
429 def configure_netpipe(testbed_instance, guid):
430     netpipe = testbed_instance._elements[guid]
431     
432     # Do some validations
433     netpipe.validate()
434     
435     # Wait for dependencies
436     netpipe.node.wait_dependencies()
437     
438     # Install rules
439     netpipe.configure()
440
441 ### Factory information ###
442
443 connector_types = dict({
444     "apps": dict({
445                 "help": "Connector from node to applications", 
446                 "name": "apps",
447                 "max": -1, 
448                 "min": 0
449             }),
450     "devs": dict({
451                 "help": "Connector from node to network interfaces", 
452                 "name": "devs",
453                 "max": -1, 
454                 "min": 0
455             }),
456     "deps": dict({
457                 "help": "Connector from node to application dependencies "
458                         "(packages and applications that need to be installed)", 
459                 "name": "deps",
460                 "max": -1, 
461                 "min": 0
462             }),
463     "inet": dict({
464                 "help": "Connector from network interfaces to the internet", 
465                 "name": "inet",
466                 "max": 1, 
467                 "min": 1
468             }),
469     "node": dict({
470                 "help": "Connector to a Node", 
471                 "name": "node",
472                 "max": 1, 
473                 "min": 1
474             }),
475     "pipes": dict({
476                 "help": "Connector to a NetPipe", 
477                 "name": "pipes",
478                 "max": 2, 
479                 "min": 0
480             }),
481     
482     "tcp": dict({
483                 "help": "ip-ip tunneling over TCP link", 
484                 "name": "tcp",
485                 "max": 1, 
486                 "min": 0
487             }),
488     "udp": dict({
489                 "help": "ip-ip tunneling over UDP datagrams", 
490                 "name": "udp",
491                 "max": 1, 
492                 "min": 0
493             }),
494     "fd->": dict({
495                 "help": "TUN device file descriptor provider", 
496                 "name": "fd->",
497                 "max": 1, 
498                 "min": 0
499             }),
500    })
501
502 connections = [
503     dict({
504         "from": (TESTBED_ID, NODE, "devs"),
505         "to":   (TESTBED_ID, NODEIFACE, "node"),
506         "init_code": connect_node_iface_node,
507         "can_cross": False
508     }),
509     dict({
510         "from": (TESTBED_ID, NODE, "devs"),
511         "to":   (TESTBED_ID, TUNIFACE, "node"),
512         "init_code": connect_tun_iface_node,
513         "can_cross": False
514     }),
515     dict({
516         "from": (TESTBED_ID, NODE, "devs"),
517         "to":   (TESTBED_ID, TAPIFACE, "node"),
518         "init_code": connect_tun_iface_node,
519         "can_cross": False
520     }),
521     dict({
522         "from": (TESTBED_ID, NODEIFACE, "inet"),
523         "to":   (TESTBED_ID, INTERNET, "devs"),
524         "init_code": connect_node_iface_inet,
525         "can_cross": False
526     }),
527     dict({
528         "from": (TESTBED_ID, NODE, "apps"),
529         "to":   (TESTBED_ID, APPLICATION, "node"),
530         "init_code": connect_dep,
531         "can_cross": False
532     }),
533     dict({
534         "from": (TESTBED_ID, NODE, "deps"),
535         "to":   (TESTBED_ID, DEPENDENCY, "node"),
536         "init_code": connect_dep,
537         "can_cross": False
538     }),
539     dict({
540         "from": (TESTBED_ID, NODE, "deps"),
541         "to":   (TESTBED_ID, NEPIDEPENDENCY, "node"),
542         "init_code": connect_dep,
543         "can_cross": False
544     }),
545     dict({
546         "from": (TESTBED_ID, NODE, "deps"),
547         "to":   (TESTBED_ID, NS3DEPENDENCY, "node"),
548         "init_code": connect_dep,
549         "can_cross": False
550     }),
551     dict({
552         "from": (TESTBED_ID, NODE, "pipes"),
553         "to":   (TESTBED_ID, NETPIPE, "node"),
554         "init_code": connect_node_netpipe,
555         "can_cross": False
556     }),
557     dict({
558         "from": (TESTBED_ID, TUNIFACE, "tcp"),
559         "to":   (TESTBED_ID, TUNIFACE, "tcp"),
560         "init_code": functools.partial(connect_tun_iface_peer,"tcp"),
561         "can_cross": False
562     }),
563     dict({
564         "from": (TESTBED_ID, TUNIFACE, "udp"),
565         "to":   (TESTBED_ID, TUNIFACE, "udp"),
566         "init_code": functools.partial(connect_tun_iface_peer,"udp"),
567         "can_cross": False
568     }),
569     dict({
570         "from": (TESTBED_ID, TAPIFACE, "tcp"),
571         "to":   (TESTBED_ID, TAPIFACE, "tcp"),
572         "init_code": functools.partial(connect_tun_iface_peer,"tcp"),
573         "can_cross": False
574     }),
575     dict({
576         "from": (TESTBED_ID, TAPIFACE, "udp"),
577         "to":   (TESTBED_ID, TAPIFACE, "udp"),
578         "init_code": functools.partial(connect_tun_iface_peer,"udp"),
579         "can_cross": False
580     }),
581     dict({
582         "from": (TESTBED_ID, TUNIFACE, "tcp"),
583         "to":   (None, None, "tcp"),
584         "init_code": functools.partial(crossconnect_tun_iface_peer_init,"tcp"),
585         "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"tcp"),
586         "can_cross": True
587     }),
588     dict({
589         "from": (TESTBED_ID, TUNIFACE, "udp"),
590         "to":   (None, None, "udp"),
591         "init_code": functools.partial(crossconnect_tun_iface_peer_init,"udp"),
592         "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"udp"),
593         "can_cross": True
594     }),
595     dict({
596         "from": (TESTBED_ID, TUNIFACE, "fd->"),
597         "to":   (None, None, "->fd"),
598         "compl_code": functools.partial(crossconnect_tun_iface_peer_both,"fd"),
599         "can_cross": True
600     }),
601     dict({
602         "from": (TESTBED_ID, TAPIFACE, "tcp"),
603         "to":   (None, None, "tcp"),
604         "init_code": functools.partial(crossconnect_tun_iface_peer_init,"tcp"),
605         "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"tcp"),
606         "can_cross": True
607     }),
608     dict({
609         "from": (TESTBED_ID, TAPIFACE, "udp"),
610         "to":   (None, None, "udp"),
611         "init_code": functools.partial(crossconnect_tun_iface_peer_init,"udp"),
612         "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"udp"),
613         "can_cross": True
614     }),
615     dict({
616         "from": (TESTBED_ID, TAPIFACE, "fd->"),
617         "to":   (None, None, "->fd"),
618         "compl_code": functools.partial(crossconnect_tun_iface_peer_both,"fd"),
619         "can_cross": True
620     }),
621 ]
622
623 attributes = dict({
624     "forward_X11": dict({      
625                 "name": "forward_X11",
626                 "help": "Forward x11 from main namespace to the node",
627                 "type": Attribute.BOOL, 
628                 "value": False,
629                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
630                 "validation_function": validation.is_bool,
631             }),
632     "hostname": dict({      
633                 "name": "hostname",
634                 "help": "Constrain hostname during resource discovery. May use wildcards.",
635                 "type": Attribute.STRING, 
636                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
637                 "validation_function": validation.is_string,
638             }),
639     "city": dict({      
640                 "name": "city",
641                 "help": "Constrain location (city) during resource discovery. May use wildcards.",
642                 "type": Attribute.STRING, 
643                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
644                 "validation_function": validation.is_string,
645             }),
646     "country": dict({      
647                 "name": "hostname",
648                 "help": "Constrain location (country) during resource discovery. May use wildcards.",
649                 "type": Attribute.STRING, 
650                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
651                 "validation_function": validation.is_string,
652             }),
653     "region": dict({      
654                 "name": "hostname",
655                 "help": "Constrain location (region) during resource discovery. May use wildcards.",
656                 "type": Attribute.STRING, 
657                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
658                 "validation_function": validation.is_string,
659             }),
660     "architecture": dict({      
661                 "name": "architecture",
662                 "help": "Constrain architexture during resource discovery.",
663                 "type": Attribute.ENUM, 
664                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
665                 "allowed": ["x86_64",
666                             "i386"],
667                 "validation_function": validation.is_enum,
668             }),
669     "operating_system": dict({      
670                 "name": "operatingSystem",
671                 "help": "Constrain operating system during resource discovery.",
672                 "type": Attribute.ENUM, 
673                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
674                 "allowed": ["f8",
675                             "f12",
676                             "f14",
677                             "centos",
678                             "other"],
679                 "validation_function": validation.is_enum,
680             }),
681     "site": dict({      
682                 "name": "site",
683                 "help": "Constrain the PlanetLab site this node should reside on.",
684                 "type": Attribute.ENUM, 
685                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
686                 "allowed": ["PLE",
687                             "PLC",
688                             "PLJ"],
689                 "validation_function": validation.is_enum,
690             }),
691     "min_reliability": dict({
692                 "name": "minReliability",
693                 "help": "Constrain reliability while picking PlanetLab nodes. Specifies a lower acceptable bound.",
694                 "type": Attribute.DOUBLE,
695                 "range": (0,100),
696                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
697                 "validation_function": validation.is_number,
698             }),
699     "max_reliability": dict({
700                 "name": "maxReliability",
701                 "help": "Constrain reliability while picking PlanetLab nodes. Specifies an upper acceptable bound.",
702                 "type": Attribute.DOUBLE,
703                 "range": (0,100),
704                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
705                 "validation_function": validation.is_number,
706             }),
707     "min_bandwidth": dict({
708                 "name": "minBandwidth",
709                 "help": "Constrain available bandwidth while picking PlanetLab nodes. Specifies a lower acceptable bound.",
710                 "type": Attribute.DOUBLE,
711                 "range": (0,2**31),
712                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
713                 "validation_function": validation.is_number,
714             }),
715     "max_bandwidth": dict({
716                 "name": "maxBandwidth",
717                 "help": "Constrain available bandwidth while picking PlanetLab nodes. Specifies an upper acceptable bound.",
718                 "type": Attribute.DOUBLE,
719                 "range": (0,2**31),
720                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
721                 "validation_function": validation.is_number,
722             }),
723     "min_load": dict({
724                 "name": "minLoad",
725                 "help": "Constrain node load average while picking PlanetLab nodes. Specifies a lower acceptable bound.",
726                 "type": Attribute.DOUBLE,
727                 "range": (0,2**31),
728                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
729                 "validation_function": validation.is_number,
730             }),
731     "max_load": dict({
732                 "name": "maxLoad",
733                 "help": "Constrain node load average while picking PlanetLab nodes. Specifies an upper acceptable bound.",
734                 "type": Attribute.DOUBLE,
735                 "range": (0,2**31),
736                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
737                 "validation_function": validation.is_number,
738             }),
739     "min_cpu": dict({
740                 "name": "minCpu",
741                 "help": "Constrain available cpu time while picking PlanetLab nodes. Specifies a lower acceptable bound.",
742                 "type": Attribute.DOUBLE,
743                 "range": (0,100),
744                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
745                 "validation_function": validation.is_number,
746             }),
747     "max_cpu": dict({
748                 "name": "maxCpu",
749                 "help": "Constrain available cpu time while picking PlanetLab nodes. Specifies an upper acceptable bound.",
750                 "type": Attribute.DOUBLE,
751                 "range": (0,100),
752                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
753                 "validation_function": validation.is_number,
754             }),
755             
756     "up": dict({
757                 "name": "up",
758                 "help": "Link up",
759                 "type": Attribute.BOOL,
760                 "value": False,
761                 "validation_function": validation.is_bool
762             }),
763     "primary": dict({
764                 "name": "primary",
765                 "help": "This is the primary interface for the attached node",
766                 "type": Attribute.BOOL,
767                 "value": True,
768                 "validation_function": validation.is_bool
769             }),
770     "device_name": dict({
771                 "name": "name",
772                 "help": "Device name",
773                 "type": Attribute.STRING,
774                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
775                 "validation_function": validation.is_string
776             }),
777     "mtu":  dict({
778                 "name": "mtu", 
779                 "help": "Maximum transmition unit for device",
780                 "type": Attribute.INTEGER,
781                 "range": (0,1500),
782                 "validation_function": validation.is_integer_range(0,1500)
783             }),
784     "mask":  dict({
785                 "name": "mask", 
786                 "help": "Network mask for the device (eg: 24 for /24 network)",
787                 "type": Attribute.INTEGER,
788                 "validation_function": validation.is_integer_range(8,24)
789             }),
790     "snat":  dict({
791                 "name": "snat", 
792                 "help": "Enable SNAT (source NAT to the internet) no this device",
793                 "type": Attribute.BOOL,
794                 "value": False,
795                 "validation_function": validation.is_bool
796             }),
797     "pointopoint":  dict({
798                 "name": "pointopoint", 
799                 "help": "If the interface is a P2P link, the remote endpoint's IP "
800                         "should be set on this attribute.",
801                 "type": Attribute.STRING,
802                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
803                 "validation_function": validation.is_string
804             }),
805     "txqueuelen":  dict({
806                 "name": "txqueuelen", 
807                 "help": "Transmission queue length (in packets)",
808                 "type": Attribute.INTEGER,
809                 "value": 1000,
810                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
811                 "range" : (1,10000),
812                 "validation_function": validation.is_integer
813             }),
814             
815     "command": dict({
816                 "name": "command",
817                 "help": "Command line string",
818                 "type": Attribute.STRING,
819                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
820                 "validation_function": validation.is_string
821             }),
822     "sudo": dict({
823                 "name": "sudo",
824                 "help": "Run with root privileges",
825                 "type": Attribute.BOOL,
826                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
827                 "value": False,
828                 "validation_function": validation.is_bool
829             }),
830     "stdin": dict({
831                 "name": "stdin",
832                 "help": "Standard input",
833                 "type": Attribute.STRING,
834                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
835                 "validation_function": validation.is_string
836             }),
837             
838     "depends": dict({
839                 "name": "depends",
840                 "help": "Space-separated list of packages required to run the application",
841                 "type": Attribute.STRING,
842                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
843                 "validation_function": validation.is_string
844             }),
845     "build-depends": dict({
846                 "name": "buildDepends",
847                 "help": "Space-separated list of packages required to build the application",
848                 "type": Attribute.STRING,
849                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
850                 "validation_function": validation.is_string
851             }),
852     "rpm-fusion": dict({
853                 "name": "rpmFusion",
854                 "help": "True if required packages can be found in the RpmFusion repository",
855                 "type": Attribute.BOOL,
856                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
857                 "value": False,
858                 "validation_function": validation.is_bool
859             }),
860     "sources": dict({
861                 "name": "sources",
862                 "help": "Space-separated list of regular files to be deployed in the working path prior to building. "
863                         "Archives won't be expanded automatically.",
864                 "type": Attribute.STRING,
865                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
866                 "validation_function": validation.is_string
867             }),
868     "build": dict({
869                 "name": "build",
870                 "help": "Build commands to execute after deploying the sources. "
871                         "Sources will be in the ${SOURCES} folder. "
872                         "Example: tar xzf ${SOURCES}/my-app.tgz && cd my-app && ./configure && make && make clean.\n"
873                         "Try to make the commands return with a nonzero exit code on error.\n"
874                         "Also, do not install any programs here, use the 'install' attribute. This will "
875                         "help keep the built files constrained to the build folder (which may "
876                         "not be the home folder), and will result in faster deployment. Also, "
877                         "make sure to clean up temporary files, to reduce bandwidth usage between "
878                         "nodes when transferring built packages.",
879                 "type": Attribute.STRING,
880                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
881                 "validation_function": validation.is_string
882             }),
883     "install": dict({
884                 "name": "install",
885                 "help": "Commands to transfer built files to their final destinations. "
886                         "Sources will be in the initial working folder, and a special "
887                         "tag ${SOURCES} can be used to reference the experiment's "
888                         "home folder (where the application commands will run).\n"
889                         "ALL sources and targets needed for execution must be copied there, "
890                         "if building has been enabled.\n"
891                         "That is, 'slave' nodes will not automatically get any source files. "
892                         "'slave' nodes don't get build dependencies either, so if you need "
893                         "make and other tools to install, be sure to provide them as "
894                         "actual dependencies instead.",
895                 "type": Attribute.STRING,
896                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
897                 "validation_function": validation.is_string
898             }),
899     
900     "netpipe_mode": dict({      
901                 "name": "mode",
902                 "help": "Link mode:\n"
903                         " * SERVER: applies to incoming connections\n"
904                         " * CLIENT: applies to outgoing connections\n"
905                         " * SERVICE: applies to both",
906                 "type": Attribute.ENUM, 
907                 "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
908                 "allowed": ["SERVER",
909                             "CLIENT",
910                             "SERVICE"],
911                 "validation_function": validation.is_enum,
912             }),
913     "port_list":  dict({
914                 "name": "portList", 
915                 "help": "Port list or range. Eg: '22', '22,23,27', '20-2000'",
916                 "type": Attribute.STRING,
917                 "validation_function": is_portlist,
918             }),
919     "addr_list":  dict({
920                 "name": "addrList", 
921                 "help": "Address list or range. Eg: '127.0.0.1', '127.0.0.1,127.0.1.1', '127.0.0.1/8'",
922                 "type": Attribute.STRING,
923                 "validation_function": is_addrlist,
924             }),
925     "bw_in":  dict({
926                 "name": "bwIn", 
927                 "help": "Inbound bandwidth limit (in Mbit/s)",
928                 "type": Attribute.DOUBLE,
929                 "validation_function": validation.is_number,
930             }),
931     "bw_out":  dict({
932                 "name": "bwOut", 
933                 "help": "Outbound bandwidth limit (in Mbit/s)",
934                 "type": Attribute.DOUBLE,
935                 "validation_function": validation.is_number,
936             }),
937     "plr_in":  dict({
938                 "name": "plrIn", 
939                 "help": "Inbound packet loss rate (0 = no loss, 1 = 100% loss)",
940                 "type": Attribute.DOUBLE,
941                 "validation_function": validation.is_number,
942             }),
943     "plr_out":  dict({
944                 "name": "plrOut", 
945                 "help": "Outbound packet loss rate (0 = no loss, 1 = 100% loss)",
946                 "type": Attribute.DOUBLE,
947                 "validation_function": validation.is_number,
948             }),
949     "delay_in":  dict({
950                 "name": "delayIn", 
951                 "help": "Inbound packet delay (in milliseconds)",
952                 "type": Attribute.INTEGER,
953                 "range": (0,60000),
954                 "validation_function": validation.is_integer,
955             }),
956     "delay_out":  dict({
957                 "name": "delayOut", 
958                 "help": "Outbound packet delay (in milliseconds)",
959                 "type": Attribute.INTEGER,
960                 "range": (0,60000),
961                 "validation_function": validation.is_integer,
962             }),
963     })
964
965 traces = dict({
966     "stdout": dict({
967                 "name": "stdout",
968                 "help": "Standard output stream"
969               }),
970     "stderr": dict({
971                 "name": "stderr",
972                 "help": "Application standard error",
973               }),
974     "buildlog": dict({
975                 "name": "buildlog",
976                 "help": "Output of the build process",
977               }), 
978     
979     "netpipe_stats": dict({
980                 "name": "netpipeStats",
981                 "help": "Information about rule match counters, packets dropped, etc.",
982               }),
983
984     "packets": dict({
985                 "name": "packets",
986                 "help": "Detailled log of all packets going through the interface",
987               }),
988     "pcap": dict({
989                 "name": "pcap",
990                 "help": "PCAP trace of all packets going through the interface",
991               }),
992     })
993
994 create_order = [ INTERNET, NODE, NODEIFACE, TAPIFACE, TUNIFACE, NETPIPE, NEPIDEPENDENCY, NS3DEPENDENCY, DEPENDENCY, APPLICATION ]
995
996 configure_order = [ INTERNET, Parallel(NODE), NODEIFACE, Parallel(TAPIFACE), Parallel(TUNIFACE), NETPIPE, Parallel(NEPIDEPENDENCY), Parallel(NS3DEPENDENCY), Parallel(DEPENDENCY), Parallel(APPLICATION) ]
997
998 # Start (and prestart) node after ifaces, because the node needs the ifaces in order to set up routes
999 start_order = [ INTERNET, NODEIFACE, Parallel(TAPIFACE), Parallel(TUNIFACE), Parallel(NODE), NETPIPE, Parallel(NEPIDEPENDENCY), Parallel(NS3DEPENDENCY), Parallel(DEPENDENCY), Parallel(APPLICATION) ]
1000
1001 factories_info = dict({
1002     NODE: dict({
1003             "help": "Virtualized Node (V-Server style)",
1004             "category": FC.CATEGORY_NODES,
1005             "create_function": create_node,
1006             "preconfigure_function": configure_node,
1007             "prestart_function": configure_node_routes,
1008             "box_attributes": [
1009                 "forward_X11",
1010                 "hostname",
1011                 "architecture",
1012                 "operating_system",
1013                 "site",
1014                 "min_reliability",
1015                 "max_reliability",
1016                 "min_bandwidth",
1017                 "max_bandwidth",
1018                 
1019                 # NEPI-in-NEPI attributes
1020                 ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP,
1021             ],
1022             "connector_types": ["devs", "apps", "pipes", "deps"],
1023             "tags": [tags.NODE, tags.ALLOW_ROUTES],
1024        }),
1025     NODEIFACE: dict({
1026             "help": "External network interface - they cannot be brought up or down, and they MUST be connected to the internet.",
1027             "category": FC.CATEGORY_DEVICES,
1028             "create_function": create_nodeiface,
1029             "preconfigure_function": configure_nodeiface,
1030             "box_attributes": [ ],
1031             "connector_types": ["node", "inet"],
1032             "tags": [tags.INTERFACE, tags.HAS_ADDRESSES],
1033         }),
1034     TUNIFACE: dict({
1035             "help": "Virtual TUN network interface (layer 3)",
1036             "category": FC.CATEGORY_DEVICES,
1037             "create_function": create_tuniface,
1038             "preconfigure_function": preconfigure_tuniface,
1039             "configure_function": postconfigure_tuniface,
1040             "prestart_function": wait_tuniface,
1041             "box_attributes": [
1042                 "up", "device_name", "mtu", "snat", "pointopoint",
1043                 "txqueuelen",
1044                 "tun_proto", "tun_addr", "tun_port", "tun_key"
1045             ],
1046             "traces": ["packets", "pcap"],
1047             "connector_types": ["node","udp","tcp","fd->"],
1048             "tags": [tags.INTERFACE, tags.ALLOW_ADDRESSES],
1049         }),
1050     TAPIFACE: dict({
1051             "help": "Virtual TAP network interface (layer 2)",
1052             "category": FC.CATEGORY_DEVICES,
1053             "create_function": create_tapiface,
1054             "preconfigure_function": preconfigure_tuniface,
1055             "configure_function": postconfigure_tuniface,
1056             "prestart_function": wait_tuniface,
1057             "box_attributes": [
1058                 "up", "device_name", "mtu", "snat", "pointopoint",
1059                 "txqueuelen",
1060                 "tun_proto", "tun_addr", "tun_port", "tun_key"
1061             ],
1062             "traces": ["packets", "pcap"],
1063             "connector_types": ["node","udp","tcp","fd->"],
1064             "tags": [tags.INTERFACE, tags.ALLOW_ADDRESSES],
1065         }),
1066     APPLICATION: dict({
1067             "help": "Generic executable command line application",
1068             "category": FC.CATEGORY_APPLICATIONS,
1069             "create_function": create_application,
1070             "start_function": start_application,
1071             "status_function": status_application,
1072             "stop_function": stop_application,
1073             "configure_function": configure_application,
1074             "box_attributes": ["command", "sudo", "stdin",
1075                                "depends", "build-depends", "build", "install",
1076                                "sources", "rpm-fusion" ],
1077             "connector_types": ["node"],
1078             "traces": ["stdout", "stderr", "buildlog"],
1079             "tags": [tags.APPLICATION],
1080         }),
1081     DEPENDENCY: dict({
1082             "help": "Requirement for package or application to be installed on some node",
1083             "category": FC.CATEGORY_APPLICATIONS,
1084             "create_function": create_dependency,
1085             "preconfigure_function": configure_dependency,
1086             "box_attributes": ["depends", "build-depends", "build", "install",
1087                                "sources", "rpm-fusion" ],
1088             "connector_types": ["node"],
1089             "traces": ["buildlog"],
1090         }),
1091     NEPIDEPENDENCY: dict({
1092             "help": "Requirement for NEPI inside NEPI - required to run testbed instances inside a node",
1093             "category": FC.CATEGORY_APPLICATIONS,
1094             "create_function": create_nepi_dependency,
1095             "preconfigure_function": configure_dependency,
1096             "box_attributes": [],
1097             "connector_types": ["node"],
1098             "traces": ["buildlog"],
1099         }),
1100     NS3DEPENDENCY: dict({
1101             "help": "Requirement for NS3 inside NEPI - required to run NS3 testbed instances inside a node. It also needs NepiDependency.",
1102             "category": FC.CATEGORY_APPLICATIONS,
1103             "create_function": create_ns3_dependency,
1104             "preconfigure_function": configure_dependency,
1105             "box_attributes": [ ],
1106             "connector_types": ["node"],
1107             "traces": ["buildlog"],
1108         }),
1109     INTERNET: dict({
1110             "help": "Internet routing",
1111             "category": FC.CATEGORY_CHANNELS,
1112             "create_function": create_internet,
1113             "connector_types": ["devs"],
1114             "tags": [tags.INTERNET],
1115         }),
1116     NETPIPE: dict({
1117             "help": "Link emulation",
1118             "category": FC.CATEGORY_CHANNELS,
1119             "create_function": create_netpipe,
1120             "configure_function": configure_netpipe,
1121             "box_attributes": ["netpipe_mode",
1122                                "addr_list", "port_list",
1123                                "bw_in","plr_in","delay_in",
1124                                "bw_out","plr_out","delay_out"],
1125             "connector_types": ["node"],
1126             "traces": ["netpipe_stats"],
1127         }),
1128 })
1129
1130 testbed_attributes = dict({
1131         "slice": dict({
1132             "name": "slice",
1133             "help": "The name of the PlanetLab slice to use",
1134             "type": Attribute.STRING,
1135             "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable | Attribute.NoDefaultValue,
1136             "validation_function": validation.is_string
1137         }),
1138         "auth_user": dict({
1139             "name": "authUser",
1140             "help": "The name of the PlanetLab user to use for API calls - it must have at least a User role.",
1141             "type": Attribute.STRING,
1142             "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable | Attribute.NoDefaultValue,
1143             "validation_function": validation.is_string
1144         }),
1145         "auth_pass": dict({
1146             "name": "authPass",
1147             "help": "The PlanetLab user's password.",
1148             "type": Attribute.STRING,
1149             "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable | Attribute.NoDefaultValue,
1150             "validation_function": validation.is_string
1151         }),
1152         "plc_host": dict({
1153             "name": "plcHost",
1154             "help": "The PlanetLab PLC API host",
1155             "type": Attribute.STRING,
1156             "value": "www.planet-lab.eu",
1157             "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
1158             "validation_function": validation.is_string
1159         }),
1160         "plc_url": dict({
1161             "name": "plcUrl",
1162             "help": "The PlanetLab PLC API url pattern - %(hostname)s is replaced by plcHost.",
1163             "type": Attribute.STRING,
1164             "value": "https://%(hostname)s:443/PLCAPI/",
1165             "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
1166             "validation_function": validation.is_string
1167         }),
1168         "p2p_deployment": dict({
1169             "name": "p2pDeployment",
1170             "help": "Enable peer-to-peer deployment of applications and dependencies. "
1171                     "When enabled, dependency packages and applications are "
1172                     "deployed in a P2P fashion, picking a single node to do "
1173                     "the building or repo download, while all the others "
1174                     "cooperatively exchange resulting binaries or rpms. "
1175                     "When deploying to many nodes, this is a far more efficient "
1176                     "use of resources. It does require re-encrypting and distributing "
1177                     "the slice's private key. Though it is implemented in a secure "
1178                     "fashion, if they key's sole purpose is not PlanetLab, then this "
1179                     "feature should be disabled.",
1180             "type": Attribute.BOOL,
1181             "value": True,
1182             "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable,
1183             "validation_function": validation.is_bool
1184         }),
1185         "slice_ssh_key": dict({
1186             "name": "sliceSSHKey",
1187             "help": "The controller-local path to the slice user's ssh private key. "
1188                     "It is the user's responsability to deploy this file where the controller "
1189                     "will run, it won't be done automatically because it's sensitive information. "
1190                     "It is recommended that a NEPI-specific user be created for this purpose and "
1191                     "this purpose alone.",
1192             "type": Attribute.STRING,
1193             "flags": Attribute.ExecReadOnly | Attribute.ExecImmutable | Attribute.NoDefaultValue,
1194             "validation_function": validation.is_string
1195         }),
1196         "pl_log_level": dict({      
1197             "name": "plLogLevel",
1198             "help": "Verbosity of logging of planetlab events.",
1199             "value": "ERROR",
1200             "type": Attribute.ENUM, 
1201             "allowed": ["DEBUG",
1202                         "INFO",
1203                         "WARNING",
1204                         "ERROR",
1205                         "CRITICAL"],
1206             "validation_function": validation.is_enum,
1207         }),
1208         "tap_port_base":  dict({
1209             "name": "tapPortBase", 
1210             "help": "Base port to use when connecting TUN/TAPs. Effective port will be BASE + GUID.",
1211             "type": Attribute.INTEGER,
1212             "value": 15000,
1213             "range": (2000,30000),
1214             "validation_function": validation.is_integer_range(2000,30000)
1215         }),
1216     })
1217
1218 supported_recovery_policies = [
1219         DC.POLICY_FAIL,
1220         DC.POLICY_RESTART,
1221         DC.POLICY_RECOVER,
1222     ]
1223
1224 class MetadataInfo(metadata.MetadataInfo):
1225     @property
1226     def connector_types(self):
1227         return connector_types
1228
1229     @property
1230     def connections(self):
1231         return connections
1232
1233     @property
1234     def attributes(self):
1235         return attributes
1236
1237     @property
1238     def traces(self):
1239         return traces
1240
1241     @property
1242     def create_order(self):
1243         return create_order
1244
1245     @property
1246     def configure_order(self):
1247         return configure_order
1248
1249     @property
1250     def prestart_order(self):
1251         return start_order
1252
1253     @property
1254     def start_order(self):
1255         return start_order
1256
1257     @property
1258     def factories_info(self):
1259         return factories_info
1260
1261     @property
1262     def testbed_attributes(self):
1263         return testbed_attributes
1264
1265     @property
1266     def testbed_id(self):
1267         return TESTBED_ID
1268
1269     @property
1270     def testbed_version(self):
1271         return TESTBED_VERSION
1272
1273     @property
1274     def supported_recovery_policies(self):
1275         return supported_recovery_policies
1276
1277