From 278422aac3a81abe2043c4f0005d6dbb84861cf0 Mon Sep 17 00:00:00 2001 From: Marc Fiuczynski Date: Tue, 12 Jul 2005 19:42:44 +0000 Subject: [PATCH] This file implements the core functionality of the resman bwlimit script in python. --- python/bwlimit.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 python/bwlimit.py diff --git a/python/bwlimit.py b/python/bwlimit.py new file mode 100644 index 0000000..b408caa --- /dev/null +++ b/python/bwlimit.py @@ -0,0 +1,158 @@ +#!/bin/env python2 -u + +# Based on code written by: Andy Bavier, acb@cs.princeton.edu +# +# Bandwidth limit script to run on PlanetLab nodes. The intent is to use +# the Hierarchical Token Bucket queueing discipline of 'tc' to (1) cap +# the output bandwidth of the node at a specified rate (e.g., 5Mbps) and +# (2) to allow all vservers to fairly share this rate. For instance, +# if there are N vservers, then each should get at least 5/N Mbps of +# bandwidth. +# +# Some relevant URLs: +# http://lartc.org/howto for how to use tc +# http://luxik.cdi.cz/~devik/qos/htb/ for info on htb + +import sys, os, re, string + +# Global variables +TC="/sbin/tc" # Where the modified tc program lives +OPS = ["change","add"] # Sequence of TC ops we'll try + +# Support to run system commands +import runcmd +def run(cmd): + try: + runcmd.run(cmd) + ret = True + except runcmd.Error, ex: + ret = False + + return ret + +def get_defaults(cap_file="/etc/planetlab/bwcap", default_cap="10mbit"): + # The maximum output bandwidth, read in from cap_file (if it + # exists). If cap_file does not exist, use default_cap for + # bandwidth cap. See also the 'cburst' parameter below. + cap=default_cap + try: + os.stat(cap_file) + fp = open(cap_file) + lines = fp.readlines() + fp.close() + try: + cap=string.strip(lines[0]) + except ValueError, ex: + pass + except OSError, ex: + pass + + # How many bytes a single token bucket is allowed to send at once. + # Small values (i.e., 3080 = two maximum-sized Ethernet packets) + # provide better fine-grained fairness. At high rates (e.g., + # cap=100mbit) this needs to be raised to allow full throughput. + cburst=30800 + + # The 'share' and 'quantum' parameters both influence the actual throughput + # seen by a particular vserver: + + # 'share' is the rate at which tokens fill the bucket, and so is + # the minimum bandwidth given to the task. I think this just + # needs to be set to some small value that is the same for all + # vservers. With the current value and a 5mbit cap, we can + # support 5000 vservers (5mbit/1kbit = 5000). With values lower + # than 10kbit, the HTB output (from tc -s -d class dev eth0) looks + # strange... this needs to be looked into further. + share="1kbit" + + # 'quantum' influences how excess bandwidth (i.e., above the + # 'share') is distributed to vservers. Apparently, vservers can + # send additional packets in proportion to their quantums (and not + # their shares, as one might expect). See: + # http://luxik.cdi.cz/~devik/qos/htb/manual/userg.htm#sharing + # The above link states that 'quantum' is automatically + # calculated for shares above 120kbit. Otherwise it should be + # set to a small value but at least one MTU, so I set it to one + # MTU. All vservers are assigned the same quantum and so they + # should share equally. + quantum=1540 + + return cap, cburst, share, quantum + + +def init(eth): + global TC, OPS + + cap, cburst, share, quantum = get_defaults() + if cap == "-1": return + + # Install HTB on $ETH. Specifies that all packets not matching a + # filter rule go to class with handle 9999. If we don't supply a + # default class, it sounds like non-matching packets can be sent + # at an unlimited rate. + for op in OPS: + cmd = "%s qdisc %s dev %s root handle 1: htb default 9999" % (TC,op,eth) + if run(cmd): break + + # Add a root class with bwcap capped rate + for op in OPS: + cmd = "%s class %s dev %s parent 1: classid 1:1 htb rate %s quantum %d" % \ + (TC, op, eth, cap, quantum) + if run(cmd): break + + # Set up the default class. Packets will fail to match a filter rule + # and end up here if they are sent by a process with UID < 500. + for op in OPS: + cmd = "%s class %s dev %s parent 1:1 classid 1:9999 htb rate %s ceil %s quantum %d cburst %d" % \ + (TC, op, eth, share, cap, quantum, cburst) + if run(cmd): break + + # The next command appears to throttle back processes that are + # sending faster than the token bucket can support, rather than + # just dropping their packets. + for op in OPS: + cmd = "%s qdisc %s dev %s parent 1:9999 handle 9999 pfifo" % \ + (TC, op, eth) + if run(cmd): break + +def on(xid, eth, bwlimit, cap, minrate, maxrate): + global TC, OPS + + default_cap, default_cburst, default_share, default_quantum = get_defaults() + quantum = bwlimit * default_quantum + + # Set up the per-vserver token bucket + for op in OPS: + cmd = "%s class %s dev %s parent 1:1 classid 1:%d htb rate %s ceil %s quantum %d cburst %d" % \ + (TC, op, eth, xid, minrate, cap, quantum, default_cburst) + if run(cmd): break + + # The next command appears to throttle back processes that are + # sending faster than the token bucket can support, rather than + # just dropping their packets. + for op in OPS: + cmd = "%s qdisc %s dev %s parent 1:%d handle %d pfifo" % \ + (TC, op, eth, xid, xid) + if run(cmd): break + + # Matches packets sent by a vserver to the appropriate token bucket. + # The raw socket module marks each packet with its vserver id. + # See: http://lartc.org/howto/lartc.qdisc.filters.html for more + # info on the filter command. + cmd = "%s filter del dev %s protocol ip prio %d" % (TC, eth, xid) + run(cmd) + cmd = "%s filter add dev %s prio %d parent 1:0 protocol ip handle %d fw flowid 1:%d" % \ + (TC, eth, xid, xid, xid) + run(cmd) + +def off(xid, eth): + cmd = "%s filter del dev %s protocol ip prio %d" % (TC, eth, xid) + run(cmd) + + cmd = "%s qdisc del dev %s parent 1:%d" % (TC, eth, xid) + run(cmd) + + cmd = "%s class del dev %s classid 1:%d" % (TC, eth, xid) + run(cmd) + + -- 2.43.0