3 # Based on code written by: Andy Bavier, acb@cs.princeton.edu
5 # Bandwidth limit script to run on PlanetLab nodes. The intent is to use
6 # the Hierarchical Token Bucket queueing discipline of 'tc' to (1) cap
7 # the output bandwidth of the node at a specified rate (e.g., 5Mbps) and
8 # (2) to allow all vservers to fairly share this rate. For instance,
9 # if there are N vservers, then each should get at least 5/N Mbps of
13 # http://lartc.org/howto for how to use tc
14 # http://luxik.cdi.cz/~devik/qos/htb/ for info on htb
16 import sys, os, re, string
19 TC="/sbin/tc" # Where the modified tc program lives
20 OPS = ["change","add"] # Sequence of TC ops we'll try
22 # Support to run system commands
28 except runcmd.Error, ex:
33 def get_defaults(cap_file="/etc/planetlab/bwcap", default_cap="10mbit"):
34 # The maximum output bandwidth, read in from cap_file (if it
35 # exists). If cap_file does not exist, use default_cap for
36 # bandwidth cap. See also the 'cburst' parameter below.
41 lines = fp.readlines()
44 cap=string.strip(lines[0])
45 except ValueError, ex:
50 # How many bytes a single token bucket is allowed to send at once.
51 # Small values (i.e., 3080 = two maximum-sized Ethernet packets)
52 # provide better fine-grained fairness. At high rates (e.g.,
53 # cap=100mbit) this needs to be raised to allow full throughput.
56 # The 'share' and 'quantum' parameters both influence the actual throughput
57 # seen by a particular vserver:
59 # 'share' is the rate at which tokens fill the bucket, and so is
60 # the minimum bandwidth given to the task. I think this just
61 # needs to be set to some small value that is the same for all
62 # vservers. With the current value and a 5mbit cap, we can
63 # support 5000 vservers (5mbit/1kbit = 5000). With values lower
64 # than 10kbit, the HTB output (from tc -s -d class dev eth0) looks
65 # strange... this needs to be looked into further.
68 # 'quantum' influences how excess bandwidth (i.e., above the
69 # 'share') is distributed to vservers. Apparently, vservers can
70 # send additional packets in proportion to their quantums (and not
71 # their shares, as one might expect). See:
72 # http://luxik.cdi.cz/~devik/qos/htb/manual/userg.htm#sharing
73 # The above link states that 'quantum' is automatically
74 # calculated for shares above 120kbit. Otherwise it should be
75 # set to a small value but at least one MTU, so I set it to one
76 # MTU. All vservers are assigned the same quantum and so they
77 # should share equally.
80 return cap, cburst, share, quantum
86 cap, cburst, share, quantum = get_defaults()
87 if cap == "-1": return
89 # Install HTB on $ETH. Specifies that all packets not matching a
90 # filter rule go to class with handle 9999. If we don't supply a
91 # default class, it sounds like non-matching packets can be sent
92 # at an unlimited rate.
94 cmd = "%s qdisc %s dev %s root handle 1: htb default 9999" % (TC,op,eth)
97 # Add a root class with bwcap capped rate
99 cmd = "%s class %s dev %s parent 1: classid 1:1 htb rate %s quantum %d" % \
100 (TC, op, eth, cap, quantum)
103 # Set up the default class. Packets will fail to match a filter rule
104 # and end up here if they are sent by a process with UID < 500.
106 cmd = "%s class %s dev %s parent 1:1 classid 1:9999 htb rate %s ceil %s quantum %d cburst %d" % \
107 (TC, op, eth, share, cap, quantum, cburst)
110 # The next command appears to throttle back processes that are
111 # sending faster than the token bucket can support, rather than
112 # just dropping their packets.
114 cmd = "%s qdisc %s dev %s parent 1:9999 handle 9999 pfifo" % \
118 def on(xid, eth, bwlimit, cap, minrate, maxrate):
121 default_cap, default_cburst, default_share, default_quantum = get_defaults()
122 quantum = bwlimit * default_quantum
124 # Set up the per-vserver token bucket
126 cmd = "%s class %s dev %s parent 1:1 classid 1:%d htb rate %s ceil %s quantum %d cburst %d" % \
127 (TC, op, eth, xid, minrate, cap, quantum, default_cburst)
130 # The next command appears to throttle back processes that are
131 # sending faster than the token bucket can support, rather than
132 # just dropping their packets.
134 cmd = "%s qdisc %s dev %s parent 1:%d handle %d pfifo" % \
135 (TC, op, eth, xid, xid)
138 # Matches packets sent by a vserver to the appropriate token bucket.
139 # The raw socket module marks each packet with its vserver id.
140 # See: http://lartc.org/howto/lartc.qdisc.filters.html for more
141 # info on the filter command.
142 cmd = "%s filter del dev %s protocol ip prio %d" % (TC, eth, xid)
144 cmd = "%s filter add dev %s prio %d parent 1:0 protocol ip handle %d fw flowid 1:%d" % \
145 (TC, eth, xid, xid, xid)
149 cmd = "%s filter del dev %s protocol ip prio %d" % (TC, eth, xid)
152 cmd = "%s qdisc del dev %s parent 1:%d" % (TC, eth, xid)
155 cmd = "%s class del dev %s classid 1:%d" % (TC, eth, xid)