From cacd7158435573b7391c44ad96c94713ff29db5c Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Fri, 1 Dec 2006 20:05:30 +0000 Subject: [PATCH 01/16] * bwmon uses byte limits instead of average rates (slice attributes). After reaching threshold bytelimit, cap to (bytelimit - threshold)/time_left_in_period * Still need appropriate slice attribute to be added to NM. Using hardcoded values until new NM is ready. Still testing. Not for public consumption yet. --- .cvsignore | 1 + bwmon.py | 572 +++++++++++++++++++++++++++-------------------------- 2 files changed, 292 insertions(+), 281 deletions(-) create mode 100644 .cvsignore diff --git a/.cvsignore b/.cvsignore new file mode 100644 index 0000000..7b27c59 --- /dev/null +++ b/.cvsignore @@ -0,0 +1 @@ +*.swp *.swo *.pyc *.log *.dat auth.py diff --git a/bwmon.py b/bwmon.py index 3946957..b31e893 100755 --- a/bwmon.py +++ b/bwmon.py @@ -15,7 +15,7 @@ # Faiyaz Ahmed # Copyright (C) 2004-2006 The Trustees of Princeton University # -# $Id: bwmon.py,v 1.7 2006/07/10 19:19:07 faiyaza Exp $ +# $Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $ # import syslog @@ -44,15 +44,25 @@ verbose = 0 datafile = "/var/lib/misc/bwmon.dat" nm = None +# Burst to line rate (or node cap). Set by NM. default_maxrate = bwlimit.get_bwcap() - default_maxexemptrate = bwlimit.bwmax +# What we cap to when slices break the rules. # 500 Kbit or 5.4 GB per day -default_avgrate = 500000 - +#default_avgrate = 500000 # 1.5 Mbit or 16.4 GB per day -default_avgexemptrate = 1500000 +#default_avgexemptrate = 1500000 + +# 4 Gbyte per day. 4 * 1024K * 1024M * 1024G +default_ByteThresh = 4294967296 +# 5.4 Gbyte per day max allowed transfered per recording period +default_ByteMax = 5798205850 +# 14 Gbyte per day +default_ExemptByteThresh = 15032385536 +# 16.4 Gbyte per day max allowed transfered per recording period to I2 +default_ExemptByteMax = 17609365914 + # Average over 1 day period = 1 * seconds_per_day @@ -64,11 +74,11 @@ The slice %(slice)s has transmitted more than %(bytes)s from %(hostname)s to %(class)s destinations since %(since)s. -Its maximum %(class)s burst rate will be capped at %(avgrate)s +Its maximum %(class)s burst rate will be capped at %(new_maxrate)s until %(until)s. Please reduce the average %(class)s transmission rate -of the slice to %(avgrate)s, or %(limit)s per %(period)s. +of the slice %(limit)s per %(period)s. """.lstrip() @@ -78,291 +88,291 @@ footer = \ """.lstrip() class Slice: - """ - Stores the last recorded bandwidth parameters of a slice. - - xid - slice context/VServer ID - name - slice name - time - beginning of recording period in UNIX seconds - bytes - low bandwidth bytes transmitted at the beginning of the recording period - exemptbytes - high bandwidth bytes transmitted at the beginning of the recording period (for I2 -F) - last_avgrate - last recorded avgrate from NM - last_maxrate - last recorded maxrate from NM - last_avgexemptrate - last recorded avgexemptrate from NM - last_maxexemptrate - last recorded maxexemptrate from NM + """ + Stores the last recorded bandwidth parameters of a slice. + + xid - slice context/VServer ID + name - slice name + time - beginning of recording period in UNIX seconds + bytes - low bandwidth bytes transmitted at the beginning of the recording period + exemptbytes - high bandwidth bytes transmitted at the beginning of the recording period (for I2 -F) + ByteMax - total volume of data allowed + ByteThresh - After thresh, cap node to (maxbyte - bytes)/(time left in period) + ExemptByteMax - Same as above, but for i2. + ExemptByteThresh - i2 ByteThresh + #last_maxexemptrate - last recorded maxexemptrate from NM. Slice attribute. + #last_maxrate - last recorded maxrate from NM. Slice attribute. + #last_ByteMax - Last recorded from NM. total volume of data allowed. + #last_ByteThresh - Last recorded from NM. After thresh, cap node to (maxbyte - bytes)/(period - t) + #last_ExemptByteMax - Last recorded from NM. Same as above, but for i2. + #last_ExemptByteThresh - Last recorded from NM. i2 ByteThresh + """ - def __init__(self, xid, name, maxrate, maxexemptrate, bytes, exemptbytes): - self.xid = xid - self.name = name - self.time = 0 - self.exemptbytes = 0 - self.last_maxrate = default_maxrate - self.last_avgrate = default_avgrate - self.last_avgexemptrate = default_avgexemptrate - self.last_maxexemptrate = default_maxexemptrate - self.reset(maxrate, maxexemptrate, bytes, exemptbytes) - - def __repr__(self): - return self.name - - def reset(self, maxrate, maxexemptrate, bytes, exemptbytes): - """ - Begin a new recording period. Remove caps by restoring limits - to their default values. - """ - - # Reset baseline time - self.time = time.time() - - # Reset baseline byte coutns - self.bytes = bytes - self.exemptbytes = exemptbytes - - # If NM except"ns below, and new_max* doesn't get set, use last. - new_maxrate = self.last_maxrate - new_maxexemptrate = self.last_maxexemptrate - - # Query Node Manager for max rate overrides - try: - vals = nm.query(self.name, [('nm_net_max_rate', self.last_maxrate), - ('nm_net_max_exempt_rate', self.last_maxexemptrate), - ('nm_net_avg_rate', self.last_avgrate), - ('nm_net_avg_exempt_rate', self.last_avgexemptrate)]) - (new_maxrate, new_maxexemptrate, - self.last_avgrate, self.last_avgexemptrate) = vals - #If NM is alive, and there is a cap, update new - self.last_maxrate = new_maxrate - self.last_maxexemptrate = new_maxexemptrate - - except Exception, err: - print "Warning: Exception received while querying NM:", err - - if new_maxrate != maxrate or new_maxexemptrate != maxexemptrate: - print "%s reset to %s/%s" % \ - (self.name, - bwlimit.format_tc_rate(new_maxrate), - bwlimit.format_tc_rate(new_maxexemptrate)) - bwlimit.set(xid = self.xid, maxrate = new_maxrate, maxexemptrate = new_maxexemptrate) - - def update(self, maxrate, maxexemptrate, bytes, exemptbytes): - """ - Update byte counts and check if average rates have been - exceeded. In the worst case (instantaneous usage of the entire - average daily byte limit at the beginning of the recording - period), the slice will be immediately capped and will get to - send twice the average daily byte limit. In the common case, - it will get to send slightly more than the average daily byte - limit. - """ + def __init__(self, xid, name, maxrate, maxexemptrate, bytes, exemptbytes): + self.xid = xid + self.name = name + self.time = 0 + self.bytes = 0 + self.exemptbytes = 0 + self.ByteMax = default_ByteMax + self.ByteThresh = default_ByteThresh + self.ExemptByteMax = default_ExemptByteMax + self.ExemptByteThresh = default_ExemptByteThresh + self.maxrate = default_maxrate + self.maxexemptrate = default_maxexemptrate + #self.last_maxrate = default_maxrate + #self.last_maxexemptrate = default_maxexemptrate + #self.last_ByteMax = default_ByteMax + #self.last_ByteThresh = default_ByteThresh + #self.last_ExemptByteMax = default_ExemptByteMax + #self.last_ExemptByteThresh = default_ExemptByteThresh + + # Get real values where applicable + self.reset(maxrate, maxexemptrate, bytes, exemptbytes) + + def __repr__(self): + return self.name + + def updateSliceAttributes(self): + # Query Node Manager for max rate overrides + try: + vals = nm.query(self.name, + [('nm_net_max_rate', self.maxrate), + ('nm_net_max_exempt_rate', self.maxexemptrate)]) + (self.maxrate, self.maxexemptrate) = vals + + except Exception, err: + print "Warning: Exception received while querying NM:", err + + def reset(self, maxrate, maxexemptrate, bytes, exemptbytes): + """ + Begin a new recording period. Remove caps by restoring limits + to their default values. + """ + + # Query Node Manager for max rate overrides + self.updateSliceAttributes() + + # Reset baseline time + self.time = time.time() + + # Reset baseline byte coutns + self.bytes = bytes + self.exemptbytes = exemptbytes + + if (self.maxrate != maxrate) or (self.maxexemptrate != maxexemptrate): + print "%s reset to %s/%s" % \ + (self.name, + bwlimit.format_tc_rate(self.maxrate), + bwlimit.format_tc_rate(self.maxexemptrate)) + bwlimit.set(xid = self.xid, maxrate = self.maxrate, maxexemptrate = self.maxexemptrate) + + def update(self, maxrate, maxexemptrate, bytes, exemptbytes): + """ + Update byte counts and check if average rates have been + exceeded. In the worst case (instantaneous usage of the entire + average daily byte limit at the beginning of the recording + period), the slice will be immediately capped and will get to + send twice the average daily byte limit. In the common case, + it will get to send slightly more than the average daily byte + limit. + """ - # If NM except'ns below, and avg*rate doesn't get set, use last_*. - avgrate = self.last_avgrate - avgexemptrate = self.last_avgexemptrate - - # Query Node Manager for max average rate overrides - try: - (avgrate, avgexemptrate) = nm.query(self.name, - [('nm_net_avg_rate', self.last_avgrate), - ('nm_net_avg_exempt_rate', self.last_avgexemptrate)]) - #If NM is alive, and there is a cap, update new - self.last_avgexemptrate = avgexemptrate - self.last_avgrate = avgrate - except Exception, err: - print "Warning: Exception received while querying NM:", err + # Query Node Manager for max rate overrides + self.updateSliceAttributes() - # Prepare message parameters from the template - message = "" - params = {'slice': self.name, 'hostname': socket.gethostname(), - 'since': time.asctime(time.gmtime(self.time)) + " GMT", - 'until': time.asctime(time.gmtime(self.time + period)) + " GMT", - 'date': time.asctime(time.gmtime()) + " GMT", - 'period': format_period(period)} - - bytelimit = avgrate * period / bits_per_byte - if bytes >= (self.bytes + bytelimit) and \ - maxrate > avgrate: - new_maxrate = avgrate - else: - new_maxrate = maxrate - - # Format template parameters for low bandwidth message - params['class'] = "low bandwidth" - params['bytes'] = format_bytes(bytes - self.bytes) - params['maxrate'] = bwlimit.format_tc_rate(maxrate) - params['limit'] = format_bytes(bytelimit) - params['avgrate'] = bwlimit.format_tc_rate(avgrate) - - if verbose: - print "%(slice)s %(class)s " \ - "%(bytes)s, %(limit)s (%(maxrate)s max/%(avgrate)s avg)" % \ - params - - # Cap low bandwidth burst rate - if new_maxrate != maxrate: - message += template % params - print "%(slice)s %(class)s capped at %(avgrate)s (%(bytes)s/%(limit)s)" % params - - exemptbytelimit = avgexemptrate * period / bits_per_byte - if exemptbytes >= (self.exemptbytes + exemptbytelimit) and \ - maxexemptrate > avgexemptrate: - new_maxexemptrate = avgexemptrate - else: - new_maxexemptrate = maxexemptrate - - # Format template parameters for high bandwidth message - params['class'] = "high bandwidth" - params['bytes'] = format_bytes(exemptbytes - self.exemptbytes) - params['maxrate'] = bwlimit.format_tc_rate(maxexemptrate) - params['limit'] = format_bytes(exemptbytelimit) - params['avgrate'] = bwlimit.format_tc_rate(avgexemptrate) - - if verbose: - print "%(slice)s %(class)s " \ - "%(bytes)s, %(limit)s (%(maxrate)s max /%(avgrate)s avg)" % \ - params - - # Cap high bandwidth burst rate - if new_maxexemptrate != maxexemptrate: - message += template % params - print "%(slice)s %(class)s capped at %(avgrate)s (%(bytes)s/%(limit)s)" % params - - # Apply parameters - if new_maxrate != maxrate or new_maxexemptrate != maxexemptrate: - bwlimit.set(xid = self.xid, maxrate = new_maxrate, maxexemptrate = new_maxexemptrate) - - # Notify slice - if message: - subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params - if debug: - print subject - print message + (footer % params) - else: - slicemail(self.name, subject, message + (footer % params)) + # Prepare message parameters from the template + message = "" + params = {'slice': self.name, 'hostname': socket.gethostname(), + 'since': time.asctime(time.gmtime(self.time)) + " GMT", + 'until': time.asctime(time.gmtime(self.time + period)) + " GMT", + 'date': time.asctime(time.gmtime()) + " GMT", + 'period': format_period(period)} + + if bytes >= (self.bytes + self.ByteThresh): + new_maxrate = (self.ByteMax - self.ByteThresh)/(period - (time.time() - self.time)) + else: + new_maxrate = maxrate + + # Format template parameters for low bandwidth message + params['class'] = "low bandwidth" + params['bytes'] = format_bytes(bytes - self.bytes) + params['maxrate'] = bwlimit.format_tc_rate(maxrate) + params['limit'] = format_bytes(self.ByteMax) + params['new_maxrate'] = bwlimit.format_tc_rate(new_maxrate) + + if verbose: + print "%(slice)s %(class)s " \ + "%(bytes)s, %(limit)s (%(new_maxrate)s avg)" % \ + params + + # Cap low bandwidth burst rate + if new_maxrate != maxrate: + message += template % params + print "%(slice)s %(class)s capped at %(new_maxrate)s " % params + + if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): + new_maxexemptrate = \ + (self.ExemptByteMax - self.ExemptByteThresh)/(period - (time.time() - self.time)) + else: + new_maxexemptrate = maxexemptrate + + # Format template parameters for high bandwidth message + params['class'] = "high bandwidth" + params['bytes'] = format_bytes(exemptbytes - self.exemptbytes) + params['maxrate'] = bwlimit.format_tc_rate(maxexemptrate) + params['limit'] = format_bytes(self.ExemptByteMax) + params['new_maxrate'] = bwlimit.format_tc_rate(new_maxexemptrate) + + if verbose: + print "%(slice)s %(class)s " \ + "%(bytes)s, %(limit)s (%(new_maxrate)s avg)" % params + + # Cap high bandwidth burst rate + if new_maxexemptrate != maxexemptrate: + message += template % params + print "%(slice)s %(class)s capped at %(new_maxexemptrate)s" % params + + # Apply parameters + if new_maxrate != maxrate or new_maxexemptrate != maxexemptrate: + bwlimit.set(xid = self.xid, maxrate = new_maxrate, maxexemptrate = new_maxexemptrate) + + # Notify slice + if message: + subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params + if debug: + print subject + print message + (footer % params) + else: + slicemail(self.name, subject, message + (footer % params)) def usage(): - print """ + print """ Usage: %s [OPTIONS]... Options: - -d, --debug Enable debugging (default: %s) - -v, --verbose Increase verbosity level (default: %d) - -f, --file=FILE Data file (default: %s) - -s, --slice=SLICE Constrain monitoring to these slices (default: all) - -p, --period=SECONDS Interval in seconds over which to enforce average byte limits (default: %s) - -h, --help This message + -d, --debug Enable debugging (default: %s) + -v, --verbose Increase verbosity level (default: %d) + -f, --file=FILE Data file (default: %s) + -s, --slice=SLICE Constrain monitoring to these slices (default: all) + -p, --period=SECONDS Interval in seconds over which to enforce average byte limits (default: %s) + -h, --help This message """.lstrip() % (sys.argv[0], debug, verbose, datafile, format_period(period)) def main(): - # Defaults - global debug, verbose, datafile, period, nm - # All slices - names = [] - - try: - longopts = ["debug", "verbose", "file=", "slice=", "period=", "help"] - (opts, argv) = getopt.getopt(sys.argv[1:], "dvf:s:p:h", longopts) - except getopt.GetoptError, err: - print "Error: " + err.msg - usage() - sys.exit(1) - - for (opt, optval) in opts: - if opt == "-d" or opt == "--debug": - debug = True - elif opt == "-v" or opt == "--verbose": - verbose += 1 - bwlimit.verbose = verbose - 1 - elif opt == "-f" or opt == "--file": - datafile = optval - elif opt == "-s" or opt == "--slice": - names.append(optval) - elif opt == "-p" or opt == "--period": - period = int(optval) - else: - usage() - sys.exit(0) - - # Check if we are already running - writepid("bwmon") - - if not debug: - # Redirect stdout and stderr to syslog - syslog.openlog("bwmon") - sys.stdout = sys.stderr = Logger() - - try: - f = open(datafile, "r+") - if verbose: - print "Loading %s" % datafile - (version, slices) = pickle.load(f) - f.close() - # Check version of data file - if version != "$Id: bwmon.py,v 1.7 2006/07/10 19:19:07 faiyaza Exp $": - print "Not using old version '%s' data file %s" % (version, datafile) - raise Exception - except Exception: - version = "$Id: bwmon.py,v 1.7 2006/07/10 19:19:07 faiyaza Exp $" - slices = {} - - # Get special slice IDs - root_xid = bwlimit.get_xid("root") - default_xid = bwlimit.get_xid("default") - - #Open connection to Node Manager - nm = NM() - - live = [] - for params in bwlimit.get(): - (xid, share, - minrate, maxrate, - minexemptrate, maxexemptrate, - bytes, exemptbytes) = params - live.append(xid) - - # Ignore root and default buckets - if xid == root_xid or xid == default_xid: - continue - - name = bwlimit.get_slice(xid) - if name is None: - # Orphaned (not associated with a slice) class - name = "%d?" % xid - - # Monitor only the specified slices - if names and name not in names: - continue - - #slices is populated from the pickle file - #xid is populated from bwlimit (read from /etc/passwd) - if slices.has_key(xid): - slice = slices[xid] - if time.time() >= (slice.time + period) or \ - bytes < slice.bytes or exemptbytes < slice.exemptbytes: - # Reset to defaults every 24 hours or if it appears - # that the byte counters have overflowed (or, more - # likely, the node was restarted or the HTB buckets - # were re-initialized). - slice.reset(maxrate, maxexemptrate, bytes, exemptbytes) - else: - # Update byte counts - slice.update(maxrate, maxexemptrate, bytes, exemptbytes) - else: - # New slice, initialize state - slice = slices[xid] = Slice(xid, name, maxrate, maxexemptrate, bytes, exemptbytes) - - # Delete dead slices - dead = Set(slices.keys()) - Set(live) - for xid in dead: - del slices[xid] - - if verbose: - print "Saving %s" % datafile - f = open(datafile, "w") - pickle.dump((version, slices), f) - f.close() - - removepid("bwmon") + # Defaults + global debug, verbose, datafile, period, nm + # All slices + names = [] + + try: + longopts = ["debug", "verbose", "file=", "slice=", "period=", "help"] + (opts, argv) = getopt.getopt(sys.argv[1:], "dvf:s:p:h", longopts) + except getopt.GetoptError, err: + print "Error: " + err.msg + usage() + sys.exit(1) + + for (opt, optval) in opts: + if opt == "-d" or opt == "--debug": + debug = True + elif opt == "-v" or opt == "--verbose": + verbose += 1 + bwlimit.verbose = verbose - 1 + elif opt == "-f" or opt == "--file": + datafile = optval + elif opt == "-s" or opt == "--slice": + names.append(optval) + elif opt == "-p" or opt == "--period": + period = int(optval) + else: + usage() + sys.exit(0) + + # Check if we are already running + writepid("bwmon") + + if not debug: + # Redirect stdout and stderr to syslog + syslog.openlog("bwmon") + sys.stdout = sys.stderr = Logger() + + try: + f = open(datafile, "r+") + if verbose: + print "Loading %s" % datafile + (version, slices) = pickle.load(f) + f.close() + # Check version of data file + if version != "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $": + print "Not using old version '%s' data file %s" % (version, datafile) + raise Exception + except Exception: + version = "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $" + slices = {} + + # Get special slice IDs + root_xid = bwlimit.get_xid("root") + default_xid = bwlimit.get_xid("default") + + #Open connection to Node Manager. Global. + nm = NM() + + live = [] + # Get actuall running values from tc. + for params in bwlimit.get(): + (xid, share, + minrate, maxrate, + minexemptrate, maxexemptrate, + bytes, exemptbytes) = params + live.append(xid) + + # Delete Me + print("name %s , minrate %s, maxrate %s, minexemptrate %s, maxexemptrate %s, bytes %s, exemptbytes %s" % (bwlimit.get_slice(xid), minrate, maxrate, minexemptrate, maxexemptrate, bytes, exemptbytes)) + + # Ignore root and default buckets + if xid == root_xid or xid == default_xid: + continue + + name = bwlimit.get_slice(xid) + if name is None: + # Orphaned (not associated with a slice) class + name = "%d?" % xid + + # Monitor only the specified slices + if names and name not in names: + continue + #slices is populated from the pickle file + #xid is populated from bwlimit (read from /etc/passwd) + if slices.has_key(xid): + slice = slices[xid] + if time.time() >= (slice.time + period) or \ + bytes < slice.bytes or exemptbytes < slice.exemptbytes: + # Reset to defaults every 24 hours or if it appears + # that the byte counters have overflowed (or, more + # likely, the node was restarted or the HTB buckets + # were re-initialized). + slice.reset(maxrate, maxexemptrate, bytes, exemptbytes) + else: + # Update byte counts + slice.update(maxrate, maxexemptrate, bytes, exemptbytes) + else: + # New slice, initialize state + slice = slices[xid] = Slice(xid, name, maxrate, maxexemptrate, bytes, exemptbytes) + + # Delete dead slices + dead = Set(slices.keys()) - Set(live) + for xid in dead: + del slices[xid] + + if verbose: + print "Saving %s" % datafile + f = open(datafile, "w") + pickle.dump((version, slices), f) + f.close() + + removepid("bwmon") if __name__ == '__main__': - main() + main() -- 2.43.0 From 803f99c0c1963672af3f9c8ee7186b9924b35efd Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Fri, 1 Dec 2006 22:02:01 +0000 Subject: [PATCH 02/16] * Fixed floating point arith error. tc likes whole numbers. * Added extra debug statements. --- bwmon.py | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/bwmon.py b/bwmon.py index b31e893..09bea99 100755 --- a/bwmon.py +++ b/bwmon.py @@ -100,12 +100,6 @@ class Slice: ByteThresh - After thresh, cap node to (maxbyte - bytes)/(time left in period) ExemptByteMax - Same as above, but for i2. ExemptByteThresh - i2 ByteThresh - #last_maxexemptrate - last recorded maxexemptrate from NM. Slice attribute. - #last_maxrate - last recorded maxrate from NM. Slice attribute. - #last_ByteMax - Last recorded from NM. total volume of data allowed. - #last_ByteThresh - Last recorded from NM. After thresh, cap node to (maxbyte - bytes)/(period - t) - #last_ExemptByteMax - Last recorded from NM. Same as above, but for i2. - #last_ExemptByteThresh - Last recorded from NM. i2 ByteThresh """ @@ -121,12 +115,6 @@ class Slice: self.ExemptByteThresh = default_ExemptByteThresh self.maxrate = default_maxrate self.maxexemptrate = default_maxexemptrate - #self.last_maxrate = default_maxrate - #self.last_maxexemptrate = default_maxexemptrate - #self.last_ByteMax = default_ByteMax - #self.last_ByteThresh = default_ByteThresh - #self.last_ExemptByteMax = default_ExemptByteMax - #self.last_ExemptByteThresh = default_ExemptByteThresh # Get real values where applicable self.reset(maxrate, maxexemptrate, bytes, exemptbytes) @@ -170,13 +158,8 @@ class Slice: def update(self, maxrate, maxexemptrate, bytes, exemptbytes): """ - Update byte counts and check if average rates have been - exceeded. In the worst case (instantaneous usage of the entire - average daily byte limit at the beginning of the recording - period), the slice will be immediately capped and will get to - send twice the average daily byte limit. In the common case, - it will get to send slightly more than the average daily byte - limit. + Update byte counts and check if byte limits have been + exceeded. """ # Query Node Manager for max rate overrides @@ -190,8 +173,9 @@ class Slice: 'date': time.asctime(time.gmtime()) + " GMT", 'period': format_period(period)} + print("byts %s self.bytes %s ByteThresh %s" %(bytes, self.bytes, self.ByteThresh)) if bytes >= (self.bytes + self.ByteThresh): - new_maxrate = (self.ByteMax - self.ByteThresh)/(period - (time.time() - self.time)) + new_maxrate = int(self.ByteMax - self.ByteThresh)/(period - (time.time() - self.time)) else: new_maxrate = maxrate @@ -204,7 +188,7 @@ class Slice: if verbose: print "%(slice)s %(class)s " \ - "%(bytes)s, %(limit)s (%(new_maxrate)s avg)" % \ + "%(bytes)s, %(limit)s (%(new_maxrate)s maxrate)" % \ params # Cap low bandwidth burst rate @@ -214,7 +198,7 @@ class Slice: if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): new_maxexemptrate = \ - (self.ExemptByteMax - self.ExemptByteThresh)/(period - (time.time() - self.time)) + int((self.ExemptByteMax - self.ExemptByteThresh)/(period - (time.time() - self.time))) else: new_maxexemptrate = maxexemptrate @@ -247,6 +231,8 @@ class Slice: else: slicemail(self.name, subject, message + (footer % params)) + + def usage(): print """ Usage: %s [OPTIONS]... @@ -328,8 +314,16 @@ def main(): bytes, exemptbytes) = params live.append(xid) - # Delete Me - print("name %s , minrate %s, maxrate %s, minexemptrate %s, maxexemptrate %s, bytes %s, exemptbytes %s" % (bwlimit.get_slice(xid), minrate, maxrate, minexemptrate, maxexemptrate, bytes, exemptbytes)) + if verbose: + print("\n%s, minrate %s, maxrate %s, minexemptrate %s,"\ + " maxexemptrate %s, bytes %s, exemptbytes %s" % \ + (bwlimit.get_slice(xid), + bwlimit.format_tc_rate(minrate), + bwlimit.format_tc_rate(maxrate), + bwlimit.format_tc_rate(minexemptrate), + bwlimit.format_tc_rate(maxexemptrate), + bytes, + exemptbytes)) # Ignore root and default buckets if xid == root_xid or xid == default_xid: -- 2.43.0 From 662dbcbe3e431c6dc9755757428fb7f837a21498 Mon Sep 17 00:00:00 2001 From: Mark Huang Date: Sat, 2 Dec 2006 19:11:47 +0000 Subject: [PATCH 03/16] - add SZ ("potential" memory usage) to e-mails to see if this can be used as a metric - memtotal: return SwapTotal as well - summary: completely broken when used in the emergency reboot case, fix - parse --min-thresh - just warn system slices once (again) --- swapmon.py | 116 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 47 deletions(-) diff --git a/swapmon.py b/swapmon.py index 5f6afd8..37743b1 100755 --- a/swapmon.py +++ b/swapmon.py @@ -10,7 +10,7 @@ # Faiyaz Ahmed # Copyright (C) 2004-2006 The Trustees of Princeton University # -# $Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $ +# $Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $ # import syslog @@ -92,7 +92,7 @@ Sometime before %(date)s, swap space was nearly exhausted on %(hostname)s. Slice %(slice)s was reset since it was the largest consumer of -physical memory at %(rss)s (%(percent)4.1f%%). +physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable). Please reply to this message explaining the nature of your experiment, and what you are doing to address the problem. @@ -112,8 +112,8 @@ Sometime before %(date)s, swap space was nearly exhausted on %(hostname)s. System slice %(slice)s was the largest consumer of physical memory at -%(rss)s (%(percent)4.1f%%). It was not reset, but please verify its -behavior. +%(rss)s (%(percent)4.1f%%) (%(sz)s writable). It was not reset, +but please verify its behavior. %(slice)s processes prior to alarm: @@ -130,7 +130,8 @@ Sometime before %(date)s, swap space was nearly exhausted on %(hostname)s. Slice %(slice)s was killed since it was the largest consumer of -physical memory at %(rss)s (%(percent)4.1f%%) after repeated restarts. +physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable) +after repeated restarts. Please reply to this message explaining the nature of your experiment, and what you are doing to address the problem. @@ -268,21 +269,22 @@ Options: def slicestat(names = None): """ Get status of specified slices (if names is None or empty, all - slices). vsize and rss are in KiB. Returns + slices). vsize, sz, and rss are in KiB. Returns {xid: {'xid': slice_id, 'name': slice_name, 'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command, - 'vsize': virtual_kib, 'rss': physical_kib, + 'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib, 'pcpu': cpu_percent, 'pmem': mem_percent}] 'vsize': total_virtual_kib, + 'sz': total_potential_kib, 'rss': total_physical_kib}} """ # Mandatory fields. xid is a virtual field inserted by vps. Make # sure cmd is last so that it does not get truncated # automatically. - fields = ['pid', 'xid', 'user', 'vsize', 'rss', 'pcpu', 'pmem', 'cmd'] + fields = ['pid', 'xid', 'user', 'vsize', 'sz', 'rss', 'pcpu', 'pmem', 'cmd'] # vps inserts xid after pid in the output, but ps doesn't know # what the field means. @@ -356,10 +358,11 @@ def slicestat(names = None): if slices.has_key(proc['xid']): slice = slices[proc['xid']] else: - slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'rss': 0} + slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'sz': 0, 'rss': 0} slice['procs'].append(proc) slice['vsize'] += proc['vsize'] + slice['sz'] += proc['sz'] slice['rss'] += proc['rss'] slices[proc['xid']] = slice @@ -368,18 +371,25 @@ def slicestat(names = None): def memtotal(): """ - Returns total physical memory on the system in KiB. + Returns total physical and swap memory on the system in KiB. """ + mem = 0 + swap = 0 + meminfo = open("/proc/meminfo", "r") - line = meminfo.readline() + for line in meminfo.readlines(): + try: + (name, value, kb) = line.split() + except: + continue + if name == "MemTotal:": + mem = int(value) + elif name == "SwapTotal:": + swap = int(value) meminfo.close() - if line[0:8] == "MemTotal": - # MemTotal: 255396 kB - (name, value, kb) = line.split() - return int(value) - return 0 + return (mem, swap) def swap_used(): """ @@ -407,19 +417,26 @@ def swap_used(): return 100 * total_used / total_swap -def summary(names = None, total_rss = memtotal()): +def summary(slices = None, total_mem = None, total_swap = None): """ Return a summary of memory usage by slice. """ - slicelist = slicestat(names).values() - slicelist.sort(lambda a, b: b['rss'] - a['rss']) - - table = "%-20s%10s%24s\n\n" % ("Slice", "Processes", "Memory Usage") + if not slices: + slices = slicestat() + slicelist = slices.values() + slicelist.sort(lambda a, b: b['sz'] - a['sz']) + if total_mem is None or total_swap is None: + (total_mem, total_swap) = memtotal() + + table = "%-20s%10s%24s%24s\n\n" % ("Slice", "Processes", "Memory Usage", "Potential Usage") for slice in slicelist: - table += "%-20s%10d%16s (%4.1f%%)\n" % \ + table += "%-20s%10d%16s (%4.1f%%)%16s (%4.1f%%)\n" % \ (slice['name'], len(slice['procs']), format_bytes(slice['rss'] * 1024, si = False), - 100. * slice['rss'] / total_rss) + 100. * slice['rss'] / total_mem, + format_bytes(slice['sz'] * 1024, si = False), + 100. * slice['sz'] / (total_mem + total_swap)) + return table @@ -456,10 +473,12 @@ def main(): reset_thresh = int(optval) elif opt == "--reboot-thresh": reboot_thresh = int(optval) + elif opt == "--min-thresh": + rss_min = int(optval) elif opt == "--system-slice": system_slices.append(optval) elif opt == "--status": - print summary(names) + print summary(slicestat(names)) sys.exit(0) else: usage() @@ -476,8 +495,8 @@ def main(): syslog.openlog("swapmon") sys.stdout = sys.stderr = Logger() - # Get total physical memory - total_rss = memtotal() + # Get total memory + (total_mem, total_swap) = memtotal() try: f = open(datafile, "r+") @@ -486,13 +505,13 @@ def main(): (version, slices) = pickle.load(f) f.close() # Check version of data file - if version != "$Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $": + if version != "$Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $": print "Not using old version '%s' data file %s" % (version, datafile) raise Exception params = {'hostname': socket.gethostname(), 'date': time.asctime(time.gmtime()) + " GMT", - 'table': summary(total_rss)} + 'table': summary(slices, total_mem, total_swap)} if debug: print rebooted_subject % params @@ -503,7 +522,7 @@ def main(): # Delete data file os.unlink(datafile) except Exception: - version = "$Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $" + version = "$Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $" slices = {} # Query process table every 30 seconds, or when a large change in @@ -522,7 +541,7 @@ def main(): used = swap_used() for resetslice in resetlist.keys(): - resetlist[resetslice].update() + resetlist[resetslice].update() if last_used is None: last_used = used @@ -550,7 +569,7 @@ def main(): slicelist = slices.values() slicelist.sort(lambda a, b: b['rss'] - a['rss']) for slice in slicelist: - percent = 100. * slice['rss'] / total_rss + percent = 100. * slice['rss'] / total_mem if slice['rss'] < rss_min: continue @@ -563,11 +582,12 @@ def main(): slice['procs'].sort(lambda a, b: b['rss'] - a['rss']) - table = "%5s %10s %10s %4s %4s %s\n\n" % ("PID", "VIRT", "RES", '%CPU', '%MEM', 'COMMAND') + table = "%5s %10s %10s %10s %4s %4s %s\n\n" % ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND') for proc in slice['procs']: - table += "%5s %10s %10s %4.1f %4.1f %s\n" % \ + table += "%5s %10s %10s %10s %4.1f %4.1f %s\n" % \ (proc['pid'], format_bytes(proc['vsize'] * 1024, si = False), + format_bytes(proc['sz'] * 1024, si = False), format_bytes(proc['rss'] * 1024, si = False), proc['pcpu'], proc['pmem'], proc['cmd']) @@ -576,27 +596,29 @@ def main(): 'table': table, 'slice': slice['name'], 'rss': format_bytes(slice['rss'] * 1024, si = False), + 'sz': format_bytes(slice['sz'] * 1024, si = False), 'percent': percent} # Match slice name against system slice patterns is_system_slice = filter(None, [re.match(pattern, slice['name']) for pattern in system_slices]) if is_system_slice: - if slice['name'] not in warned: - warned.append(slice['name']) - if debug: - print alarm_subject % params - print alarm_body % params - else: - print "Warning slice " + slice['name'] - slicemail(slice['name'], alarm_subject % params, - alarm_body % params) + # Do not reset system slices, just warn once + if slice['name'] not in warned: + warned.append(slice['name']) + if debug: + print alarm_subject % params + print alarm_body % params + else: + print "Warning slice " + slice['name'] + slicemail(slice['name'], alarm_subject % params, + alarm_body % params) else: - # Reset slice - if not resetlist.has_key(slice['name']): - resetlist[slice['name']] = Reset(slice['name']) - resetlist[slice['name']].reset(params) - slices = slicestat(names) + # Reset slice + if not resetlist.has_key(slice['name']): + resetlist[slice['name']] = Reset(slice['name']) + resetlist[slice['name']].reset(params) + slices = slicestat(names) if timer <= 0 or used >= (last_used + change_thresh): if used >= (last_used + change_thresh): -- 2.43.0 From 288b66d5ba2651b9184a25d2a795dfb912439701 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Mon, 4 Dec 2006 21:53:09 +0000 Subject: [PATCH 04/16] * Queries NM for: "nm_net_max_byte", "nm_net_max_exempt_byte", "nm_net_max_thresh_byte", "nm_net_max_thresh_exempt_byte" * Backwards compatible with avgrate and avgexemptrate slice attributes. --- bwmon.py | 58 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/bwmon.py b/bwmon.py index 09bea99..2410b7d 100755 --- a/bwmon.py +++ b/bwmon.py @@ -54,14 +54,13 @@ default_maxexemptrate = bwlimit.bwmax # 1.5 Mbit or 16.4 GB per day #default_avgexemptrate = 1500000 -# 4 Gbyte per day. 4 * 1024K * 1024M * 1024G -default_ByteThresh = 4294967296 +# 5.4 Gbyte per day. 5.4 * 1024M * 1024G kbytes # 5.4 Gbyte per day max allowed transfered per recording period -default_ByteMax = 5798205850 -# 14 Gbyte per day -default_ExemptByteThresh = 15032385536 +default_ByteMax = 5662310 +default_ByteThresh = int(.8 * default_ByteMax) # 16.4 Gbyte per day max allowed transfered per recording period to I2 -default_ExemptByteMax = 17609365914 +default_ExemptByteMax = 17196646 +default_ExemptByteThresh = int(.8 * default_ExemptByteMax) # Average over 1 day @@ -100,6 +99,8 @@ class Slice: ByteThresh - After thresh, cap node to (maxbyte - bytes)/(time left in period) ExemptByteMax - Same as above, but for i2. ExemptByteThresh - i2 ByteThresh + maxrate - max_rate slice attribute. + maxexemptrate - max_exempt_rate slice attribute. """ @@ -127,8 +128,28 @@ class Slice: try: vals = nm.query(self.name, [('nm_net_max_rate', self.maxrate), - ('nm_net_max_exempt_rate', self.maxexemptrate)]) - (self.maxrate, self.maxexemptrate) = vals + ('nm_net_max_exempt_rate', self.maxexemptrate), + ("nm_net_max_byte", self.ByteMax), + ("nm_net_max_exempt_byte", self.ExemptByteMax), + ("nm_net_max_thresh_byte", int( .8 * self.ByteMax)), + ("nm_net_max_thresh_exempt_byte", int(.8 * self.ExemptByteMax)), + ("nm_net_avg_rate", 0), + ("nm_net_avg_exempt_rate", 0)]) + + (self.maxrate, + self.maxexemptrate, + self.ByteMax, + self.ExemptByteMax, + self.ByteThresh, + self.ExemptByteThresh, + avgrate, + avgexemptrate) = vals + + if (avgrate != 0) or (avgexemptrate != 0): + self.ByteMax = avgrate * period + self.ByteThresh = int(self.ByteMax * .8) + self.ExemptByteMax = avgexemptrate * period + self.ExemptByteThresh = int(self.ExemptByteMax * .8) except Exception, err: print "Warning: Exception received while querying NM:", err @@ -173,9 +194,9 @@ class Slice: 'date': time.asctime(time.gmtime()) + " GMT", 'period': format_period(period)} - print("byts %s self.bytes %s ByteThresh %s" %(bytes, self.bytes, self.ByteThresh)) - if bytes >= (self.bytes + self.ByteThresh): - new_maxrate = int(self.ByteMax - self.ByteThresh)/(period - (time.time() - self.time)) + if bytes >= (self.bytes + (self.ByteThresh * 1024)): + new_maxrate = \ + int(((self.ByteMax * 1024) - self.bytes + bytes)/(period - time.time() - self.time)) else: new_maxrate = maxrate @@ -183,12 +204,12 @@ class Slice: params['class'] = "low bandwidth" params['bytes'] = format_bytes(bytes - self.bytes) params['maxrate'] = bwlimit.format_tc_rate(maxrate) - params['limit'] = format_bytes(self.ByteMax) + params['limit'] = format_bytes(int(self.ByteMax * 1024)) params['new_maxrate'] = bwlimit.format_tc_rate(new_maxrate) if verbose: print "%(slice)s %(class)s " \ - "%(bytes)s, %(limit)s (%(new_maxrate)s maxrate)" % \ + "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % \ params # Cap low bandwidth burst rate @@ -196,9 +217,9 @@ class Slice: message += template % params print "%(slice)s %(class)s capped at %(new_maxrate)s " % params - if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): + if exemptbytes >= (self.exemptbytes + (self.ExemptByteThresh * 1024)): new_maxexemptrate = \ - int((self.ExemptByteMax - self.ExemptByteThresh)/(period - (time.time() - self.time))) + int(((self.ExemptByteMax * 1024) - (self.bytes + bytes))/(period - (time.time() - self.time))) else: new_maxexemptrate = maxexemptrate @@ -206,12 +227,12 @@ class Slice: params['class'] = "high bandwidth" params['bytes'] = format_bytes(exemptbytes - self.exemptbytes) params['maxrate'] = bwlimit.format_tc_rate(maxexemptrate) - params['limit'] = format_bytes(self.ExemptByteMax) + params['limit'] = format_bytes(self.ExemptByteMax * 1024) params['new_maxrate'] = bwlimit.format_tc_rate(new_maxexemptrate) if verbose: print "%(slice)s %(class)s " \ - "%(bytes)s, %(limit)s (%(new_maxrate)s avg)" % params + "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % params # Cap high bandwidth burst rate if new_maxexemptrate != maxexemptrate: @@ -315,7 +336,8 @@ def main(): live.append(xid) if verbose: - print("\n%s, minrate %s, maxrate %s, minexemptrate %s,"\ + print("\nRunning stats for %s"\ + "\n minrate %s, maxrate %s, minexemptrate %s,"\ " maxexemptrate %s, bytes %s, exemptbytes %s" % \ (bwlimit.get_slice(xid), bwlimit.format_tc_rate(minrate), -- 2.43.0 From 1f950e8e43c6079c1f890fc18550f72b657126a3 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Tue, 5 Dec 2006 16:46:58 +0000 Subject: [PATCH 05/16] Remove slice email notification. (commented. will remove notification code after testing) --- bwmon.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bwmon.py b/bwmon.py index 2410b7d..4a828a8 100755 --- a/bwmon.py +++ b/bwmon.py @@ -244,13 +244,13 @@ class Slice: bwlimit.set(xid = self.xid, maxrate = new_maxrate, maxexemptrate = new_maxexemptrate) # Notify slice - if message: - subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params - if debug: - print subject - print message + (footer % params) - else: - slicemail(self.name, subject, message + (footer % params)) + #if message: + # subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params + # if debug: + # print subject + # print message + (footer % params) + # else: + # slicemail(self.name, subject, message + (footer % params)) @@ -336,8 +336,8 @@ def main(): live.append(xid) if verbose: - print("\nRunning stats for %s"\ - "\n minrate %s, maxrate %s, minexemptrate %s,"\ + print("\nRunning stats for %s from tc."\ + "\nminrate %s, maxrate %s, minexemptrate %s,"\ " maxexemptrate %s, bytes %s, exemptbytes %s" % \ (bwlimit.get_slice(xid), bwlimit.format_tc_rate(minrate), -- 2.43.0 From e6575041387c49c337c981483c3b4a999bd78f7a Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Tue, 5 Dec 2006 21:46:56 +0000 Subject: [PATCH 06/16] * Byte limits are in KB when passed or requested via XMLRPC to NM. Bytes otherwise. --- bwmon.py | 67 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/bwmon.py b/bwmon.py index 4a828a8..ed9cd7b 100755 --- a/bwmon.py +++ b/bwmon.py @@ -54,12 +54,12 @@ default_maxexemptrate = bwlimit.bwmax # 1.5 Mbit or 16.4 GB per day #default_avgexemptrate = 1500000 -# 5.4 Gbyte per day. 5.4 * 1024M * 1024G kbytes +# 5.4 Gbyte per day. 5.4 * 1024 k * 1024M * 1024G # 5.4 Gbyte per day max allowed transfered per recording period -default_ByteMax = 5662310 +default_ByteMax = 5798205850 default_ByteThresh = int(.8 * default_ByteMax) # 16.4 Gbyte per day max allowed transfered per recording period to I2 -default_ExemptByteMax = 17196646 +default_ExemptByteMax = 17609365914 default_ExemptByteThresh = int(.8 * default_ExemptByteMax) @@ -129,25 +129,38 @@ class Slice: vals = nm.query(self.name, [('nm_net_max_rate', self.maxrate), ('nm_net_max_exempt_rate', self.maxexemptrate), - ("nm_net_max_byte", self.ByteMax), - ("nm_net_max_exempt_byte", self.ExemptByteMax), - ("nm_net_max_thresh_byte", int( .8 * self.ByteMax)), - ("nm_net_max_thresh_exempt_byte", int(.8 * self.ExemptByteMax)), + ("nm_net_max_byte", int(self.ByteMax / 1024)), + ("nm_net_max_exempt_byte", int(self.ExemptByteMax / 1024)), + ("nm_net_max_thresh_byte", int( .8 * self.ByteMax / 1024)), + ("nm_net_max_thresh_exempt_byte", int(.8 * self.ExemptByteMax / 1024)), ("nm_net_avg_rate", 0), ("nm_net_avg_exempt_rate", 0)]) (self.maxrate, - self.maxexemptrate, - self.ByteMax, - self.ExemptByteMax, - self.ByteThresh, - self.ExemptByteThresh, - avgrate, - avgexemptrate) = vals - - if (avgrate != 0) or (avgexemptrate != 0): - self.ByteMax = avgrate * period + self.maxexemptrate, + ByteMax, + ExemptByteMax, + ByteThresh, + ExemptByteThresh, + avgrate, + avgexemptrate) = vals + + # The shitty bit. Gotta bias the limits so as not to overflow xmlrpc + self.ByteMax = ByteMax * 1024 + self.ByteThresh = ByteMax * 1024 + self.ExemptByteMax = ExemptByteMax * 1024 + self.ExemptByteThresh = ExemptByteThresh * 1024 + + # The hack here is that when i pass 0 to the xmlrpc request to NM, + # for rate limits and it comes back non zero, then screw the byte limits. + # Mult by the period and recompute the byte limits. The thought is + # If/when PLC switches to byte limits, the avgrates wont be used as + # slice attributes and will return as 0 + if (avgrate != 0): + self.ByteMax = avgrate * period self.ByteThresh = int(self.ByteMax * .8) + + if (avgexemptrate != 0): self.ExemptByteMax = avgexemptrate * period self.ExemptByteThresh = int(self.ExemptByteMax * .8) @@ -173,8 +186,10 @@ class Slice: if (self.maxrate != maxrate) or (self.maxexemptrate != maxexemptrate): print "%s reset to %s/%s" % \ (self.name, - bwlimit.format_tc_rate(self.maxrate), - bwlimit.format_tc_rate(self.maxexemptrate)) + #bwlimit.format_tc_rate(self.maxrate), + self.maxrate, + #bwlimit.format_tc_rate(self.maxexemptrate)) + self.maxexemptrate) bwlimit.set(xid = self.xid, maxrate = self.maxrate, maxexemptrate = self.maxexemptrate) def update(self, maxrate, maxexemptrate, bytes, exemptbytes): @@ -194,9 +209,9 @@ class Slice: 'date': time.asctime(time.gmtime()) + " GMT", 'period': format_period(period)} - if bytes >= (self.bytes + (self.ByteThresh * 1024)): + if bytes >= (self.bytes + self.ByteThresh): new_maxrate = \ - int(((self.ByteMax * 1024) - self.bytes + bytes)/(period - time.time() - self.time)) + int((self.ByteMax - self.bytes + bytes)/(period - time.time() - self.time)) else: new_maxrate = maxrate @@ -204,7 +219,7 @@ class Slice: params['class'] = "low bandwidth" params['bytes'] = format_bytes(bytes - self.bytes) params['maxrate'] = bwlimit.format_tc_rate(maxrate) - params['limit'] = format_bytes(int(self.ByteMax * 1024)) + params['limit'] = format_bytes(self.ByteMax) params['new_maxrate'] = bwlimit.format_tc_rate(new_maxrate) if verbose: @@ -217,9 +232,9 @@ class Slice: message += template % params print "%(slice)s %(class)s capped at %(new_maxrate)s " % params - if exemptbytes >= (self.exemptbytes + (self.ExemptByteThresh * 1024)): + if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): new_maxexemptrate = \ - int(((self.ExemptByteMax * 1024) - (self.bytes + bytes))/(period - (time.time() - self.time))) + int((self.ExemptByteMax - (self.bytes + bytes))/(period - (time.time() - self.time))) else: new_maxexemptrate = maxexemptrate @@ -227,8 +242,8 @@ class Slice: params['class'] = "high bandwidth" params['bytes'] = format_bytes(exemptbytes - self.exemptbytes) params['maxrate'] = bwlimit.format_tc_rate(maxexemptrate) - params['limit'] = format_bytes(self.ExemptByteMax * 1024) - params['new_maxrate'] = bwlimit.format_tc_rate(new_maxexemptrate) + params['limit'] = format_bytes(self.ExemptByteMax) + params['new_maxexemptrate'] = bwlimit.format_tc_rate(new_maxexemptrate) if verbose: print "%(slice)s %(class)s " \ -- 2.43.0 From 0cd50336ca253f653c330291b3779050a362e7a6 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Wed, 6 Dec 2006 19:22:45 +0000 Subject: [PATCH 07/16] * Fixed output of -s -v and -d flags. Still testing. --- bwmon.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/bwmon.py b/bwmon.py index ed9cd7b..4e601ef 100755 --- a/bwmon.py +++ b/bwmon.py @@ -200,7 +200,14 @@ class Slice: # Query Node Manager for max rate overrides self.updateSliceAttributes() - + + if verbose: + print("\n%s slice attributes "\ + "maxrate %s, maxexemptrate %s" % \ + (self.name, + bwlimit.format_tc_rate(maxrate), + bwlimit.format_tc_rate(maxexemptrate))) + # Prepare message parameters from the template message = "" params = {'slice': self.name, 'hostname': socket.gethostname(), @@ -349,18 +356,6 @@ def main(): minexemptrate, maxexemptrate, bytes, exemptbytes) = params live.append(xid) - - if verbose: - print("\nRunning stats for %s from tc."\ - "\nminrate %s, maxrate %s, minexemptrate %s,"\ - " maxexemptrate %s, bytes %s, exemptbytes %s" % \ - (bwlimit.get_slice(xid), - bwlimit.format_tc_rate(minrate), - bwlimit.format_tc_rate(maxrate), - bwlimit.format_tc_rate(minexemptrate), - bwlimit.format_tc_rate(maxexemptrate), - bytes, - exemptbytes)) # Ignore root and default buckets if xid == root_xid or xid == default_xid: -- 2.43.0 From 933839b6ffafac14746463898b3a385752e06766 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Wed, 13 Dec 2006 21:39:23 +0000 Subject: [PATCH 08/16] * Fixed syntax error with low bw thresholds. --- bwmon.py | 604 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 304 insertions(+), 300 deletions(-) diff --git a/bwmon.py b/bwmon.py index 4e601ef..c0832ad 100755 --- a/bwmon.py +++ b/bwmon.py @@ -87,318 +87,322 @@ footer = \ """.lstrip() class Slice: - """ - Stores the last recorded bandwidth parameters of a slice. - - xid - slice context/VServer ID - name - slice name - time - beginning of recording period in UNIX seconds - bytes - low bandwidth bytes transmitted at the beginning of the recording period - exemptbytes - high bandwidth bytes transmitted at the beginning of the recording period (for I2 -F) - ByteMax - total volume of data allowed - ByteThresh - After thresh, cap node to (maxbyte - bytes)/(time left in period) - ExemptByteMax - Same as above, but for i2. - ExemptByteThresh - i2 ByteThresh - maxrate - max_rate slice attribute. - maxexemptrate - max_exempt_rate slice attribute. + """ + Stores the last recorded bandwidth parameters of a slice. + + xid - slice context/VServer ID + name - slice name + time - beginning of recording period in UNIX seconds + bytes - low bandwidth bytes transmitted at the beginning of the recording period + exemptbytes - high bandwidth bytes transmitted at the beginning of the recording period (for I2 -F) + ByteMax - total volume of data allowed + ByteThresh - After thresh, cap node to (maxbyte - bytes)/(time left in period) + ExemptByteMax - Same as above, but for i2. + ExemptByteThresh - i2 ByteThresh + maxrate - max_rate slice attribute. + maxexemptrate - max_exempt_rate slice attribute. + self.emailed = did we email during this recording period """ - def __init__(self, xid, name, maxrate, maxexemptrate, bytes, exemptbytes): - self.xid = xid - self.name = name - self.time = 0 - self.bytes = 0 - self.exemptbytes = 0 - self.ByteMax = default_ByteMax - self.ByteThresh = default_ByteThresh - self.ExemptByteMax = default_ExemptByteMax - self.ExemptByteThresh = default_ExemptByteThresh - self.maxrate = default_maxrate - self.maxexemptrate = default_maxexemptrate - - # Get real values where applicable - self.reset(maxrate, maxexemptrate, bytes, exemptbytes) - - def __repr__(self): - return self.name - - def updateSliceAttributes(self): - # Query Node Manager for max rate overrides - try: - vals = nm.query(self.name, - [('nm_net_max_rate', self.maxrate), - ('nm_net_max_exempt_rate', self.maxexemptrate), - ("nm_net_max_byte", int(self.ByteMax / 1024)), - ("nm_net_max_exempt_byte", int(self.ExemptByteMax / 1024)), - ("nm_net_max_thresh_byte", int( .8 * self.ByteMax / 1024)), - ("nm_net_max_thresh_exempt_byte", int(.8 * self.ExemptByteMax / 1024)), - ("nm_net_avg_rate", 0), - ("nm_net_avg_exempt_rate", 0)]) - - (self.maxrate, - self.maxexemptrate, - ByteMax, - ExemptByteMax, - ByteThresh, - ExemptByteThresh, - avgrate, - avgexemptrate) = vals - - # The shitty bit. Gotta bias the limits so as not to overflow xmlrpc - self.ByteMax = ByteMax * 1024 - self.ByteThresh = ByteMax * 1024 - self.ExemptByteMax = ExemptByteMax * 1024 - self.ExemptByteThresh = ExemptByteThresh * 1024 - - # The hack here is that when i pass 0 to the xmlrpc request to NM, - # for rate limits and it comes back non zero, then screw the byte limits. - # Mult by the period and recompute the byte limits. The thought is - # If/when PLC switches to byte limits, the avgrates wont be used as - # slice attributes and will return as 0 - if (avgrate != 0): - self.ByteMax = avgrate * period - self.ByteThresh = int(self.ByteMax * .8) - - if (avgexemptrate != 0): - self.ExemptByteMax = avgexemptrate * period - self.ExemptByteThresh = int(self.ExemptByteMax * .8) - - except Exception, err: - print "Warning: Exception received while querying NM:", err - - def reset(self, maxrate, maxexemptrate, bytes, exemptbytes): - """ - Begin a new recording period. Remove caps by restoring limits - to their default values. - """ - - # Query Node Manager for max rate overrides - self.updateSliceAttributes() - - # Reset baseline time - self.time = time.time() - - # Reset baseline byte coutns - self.bytes = bytes - self.exemptbytes = exemptbytes - - if (self.maxrate != maxrate) or (self.maxexemptrate != maxexemptrate): - print "%s reset to %s/%s" % \ - (self.name, - #bwlimit.format_tc_rate(self.maxrate), - self.maxrate, - #bwlimit.format_tc_rate(self.maxexemptrate)) - self.maxexemptrate) - bwlimit.set(xid = self.xid, maxrate = self.maxrate, maxexemptrate = self.maxexemptrate) - - def update(self, maxrate, maxexemptrate, bytes, exemptbytes): - """ - Update byte counts and check if byte limits have been - exceeded. - """ - - # Query Node Manager for max rate overrides - self.updateSliceAttributes() - - if verbose: - print("\n%s slice attributes "\ - "maxrate %s, maxexemptrate %s" % \ - (self.name, - bwlimit.format_tc_rate(maxrate), - bwlimit.format_tc_rate(maxexemptrate))) - - # Prepare message parameters from the template - message = "" - params = {'slice': self.name, 'hostname': socket.gethostname(), - 'since': time.asctime(time.gmtime(self.time)) + " GMT", - 'until': time.asctime(time.gmtime(self.time + period)) + " GMT", - 'date': time.asctime(time.gmtime()) + " GMT", - 'period': format_period(period)} - - if bytes >= (self.bytes + self.ByteThresh): - new_maxrate = \ - int((self.ByteMax - self.bytes + bytes)/(period - time.time() - self.time)) - else: - new_maxrate = maxrate - - # Format template parameters for low bandwidth message - params['class'] = "low bandwidth" - params['bytes'] = format_bytes(bytes - self.bytes) - params['maxrate'] = bwlimit.format_tc_rate(maxrate) - params['limit'] = format_bytes(self.ByteMax) - params['new_maxrate'] = bwlimit.format_tc_rate(new_maxrate) - - if verbose: - print "%(slice)s %(class)s " \ - "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % \ - params - - # Cap low bandwidth burst rate - if new_maxrate != maxrate: - message += template % params - print "%(slice)s %(class)s capped at %(new_maxrate)s " % params - - if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): - new_maxexemptrate = \ - int((self.ExemptByteMax - (self.bytes + bytes))/(period - (time.time() - self.time))) - else: - new_maxexemptrate = maxexemptrate - - # Format template parameters for high bandwidth message - params['class'] = "high bandwidth" - params['bytes'] = format_bytes(exemptbytes - self.exemptbytes) - params['maxrate'] = bwlimit.format_tc_rate(maxexemptrate) - params['limit'] = format_bytes(self.ExemptByteMax) - params['new_maxexemptrate'] = bwlimit.format_tc_rate(new_maxexemptrate) - - if verbose: - print "%(slice)s %(class)s " \ - "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % params - - # Cap high bandwidth burst rate - if new_maxexemptrate != maxexemptrate: - message += template % params - print "%(slice)s %(class)s capped at %(new_maxexemptrate)s" % params - - # Apply parameters - if new_maxrate != maxrate or new_maxexemptrate != maxexemptrate: - bwlimit.set(xid = self.xid, maxrate = new_maxrate, maxexemptrate = new_maxexemptrate) - - # Notify slice - #if message: - # subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params - # if debug: - # print subject - # print message + (footer % params) - # else: - # slicemail(self.name, subject, message + (footer % params)) + def __init__(self, xid, name, maxrate, maxexemptrate, bytes, exemptbytes): + self.xid = xid + self.name = name + self.time = 0 + self.bytes = 0 + self.exemptbytes = 0 + self.ByteMax = default_ByteMax + self.ByteThresh = default_ByteThresh + self.ExemptByteMax = default_ExemptByteMax + self.ExemptByteThresh = default_ExemptByteThresh + self.maxrate = default_maxrate + self.maxexemptrate = default_maxexemptrate + self.emailed = False + + # Get real values where applicable + self.reset(maxrate, maxexemptrate, bytes, exemptbytes) + + def __repr__(self): + return self.name + + def updateSliceAttributes(self): + # Query Node Manager for max rate overrides + try: + vals = nm.query(self.name, + [('nm_net_max_rate', self.maxrate), + ('nm_net_max_exempt_rate', self.maxexemptrate), + ("nm_net_max_byte", int(self.ByteMax / 1024)), + ("nm_net_max_exempt_byte", int(self.ExemptByteMax / 1024)), + ("nm_net_max_thresh_byte", int( .8 * self.ByteMax / 1024)), + ("nm_net_max_thresh_exempt_byte", int(.8 * self.ExemptByteMax / 1024)), + ("nm_net_avg_rate", 0), + ("nm_net_avg_exempt_rate", 0)]) + + (self.maxrate, + self.maxexemptrate, + ByteMax, + ExemptByteMax, + ByteThresh, + ExemptByteThresh, + avgrate, + avgexemptrate) = vals + + # The shitty bit. Gotta bias the limits so as not to overflow xmlrpc + self.ByteMax = ByteMax * 1024 + self.ByteThresh = ByteThresh * 1024 + self.ExemptByteMax = ExemptByteMax * 1024 + self.ExemptByteThresh = ExemptByteThresh * 1024 + + # The hack here is that when i pass 0 to the xmlrpc request to NM, + # for rate limits and it comes back non zero, then screw the byte limits. + # Mult by the period and recompute the byte limits. The thought is + # If/when PLC switches to byte limits, the avgrates wont be used as + # slice attributes and will return as 0 + if (avgrate != 0): + self.ByteMax = avgrate * period + self.ByteThresh = int(self.ByteMax * .8) + + if (avgexemptrate != 0): + self.ExemptByteMax = avgexemptrate * period + self.ExemptByteThresh = int(self.ExemptByteMax * .8) + + except Exception, err: + print "Warning: Exception received while querying NM:", err + + def reset(self, maxrate, maxexemptrate, bytes, exemptbytes): + """ + Begin a new recording period. Remove caps by restoring limits + to their default values. + """ + + # Query Node Manager for max rate overrides + self.updateSliceAttributes() + + # Reset baseline time + self.time = time.time() + + # Reset baseline byte coutns + self.bytes = bytes + self.exemptbytes = exemptbytes + + # Reset email + self.emailed = False + + if (self.maxrate != maxrate) or (self.maxexemptrate != maxexemptrate): + print "%s reset to %s/%s" % \ + (self.name, + bwlimit.format_tc_rate(self.maxrate), + bwlimit.format_tc_rate(self.maxexemptrate)) + bwlimit.set(xid = self.xid, maxrate = self.maxrate, maxexemptrate = self.maxexemptrate) + + def update(self, maxrate, maxexemptrate, bytes, exemptbytes): + """ + Update byte counts and check if byte limits have been + exceeded. + """ + + # Query Node Manager for max rate overrides + self.updateSliceAttributes() + + if verbose: + print("\n%s slice attributes "\ + "maxrate %s, maxexemptrate %s" % \ + (self.name, + bwlimit.format_tc_rate(maxrate), + bwlimit.format_tc_rate(maxexemptrate))) + + # Prepare message parameters from the template + message = "" + params = {'slice': self.name, 'hostname': socket.gethostname(), + 'since': time.asctime(time.gmtime(self.time)) + " GMT", + 'until': time.asctime(time.gmtime(self.time + period)) + " GMT", + 'date': time.asctime(time.gmtime()) + " GMT", + 'period': format_period(period)} + + if bytes >= (self.bytes + self.ByteThresh): + new_maxrate = \ + int((self.ByteMax - self.bytes + bytes)/(period - time.time() - self.time)) + else: + new_maxrate = maxrate + + # Format template parameters for low bandwidth message + params['class'] = "low bandwidth" + params['bytes'] = format_bytes(bytes - self.bytes) + params['maxrate'] = bwlimit.format_tc_rate(maxrate) + params['limit'] = format_bytes(self.ByteMax) + params['new_maxrate'] = bwlimit.format_tc_rate(new_maxrate) + + if verbose: + print "%(slice)s %(class)s " \ + "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % \ + params + + # Cap low bandwidth burst rate + if new_maxrate != maxrate: + message += template % params + print "%(slice)s %(class)s capped at %(new_maxrate)s " % params + + if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): + new_maxexemptrate = \ + int((self.ExemptByteMax - (self.bytes + bytes))/(period - (time.time() - self.time))) + else: + new_maxexemptrate = maxexemptrate + + # Format template parameters for high bandwidth message + params['class'] = "high bandwidth" + params['bytes'] = format_bytes(exemptbytes - self.exemptbytes) + params['maxrate'] = bwlimit.format_tc_rate(maxexemptrate) + params['limit'] = format_bytes(self.ExemptByteMax) + params['new_maxexemptrate'] = bwlimit.format_tc_rate(new_maxexemptrate) + + if verbose: + print "%(slice)s %(class)s " \ + "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % params + + # Cap high bandwidth burst rate + if new_maxexemptrate != maxexemptrate: + message += template % params + print "%(slice)s %(class)s capped at %(new_maxexemptrate)s" % params + + # Apply parameters + if new_maxrate != maxrate or new_maxexemptrate != maxexemptrate: + bwlimit.set(xid = self.xid, maxrate = new_maxrate, maxexemptrate = new_maxexemptrate) + + # Notify slice + if message and self.emailed == False: + subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params + if debug: + print subject + print message + (footer % params) + else: + self.emailed = True + slicemail(self.name, subject, message + (footer % params)) def usage(): - print """ + print """ Usage: %s [OPTIONS]... Options: - -d, --debug Enable debugging (default: %s) - -v, --verbose Increase verbosity level (default: %d) - -f, --file=FILE Data file (default: %s) - -s, --slice=SLICE Constrain monitoring to these slices (default: all) - -p, --period=SECONDS Interval in seconds over which to enforce average byte limits (default: %s) - -h, --help This message + -d, --debug Enable debugging (default: %s) + -v, --verbose Increase verbosity level (default: %d) + -f, --file=FILE Data file (default: %s) + -s, --slice=SLICE Constrain monitoring to these slices (default: all) + -p, --period=SECONDS Interval in seconds over which to enforce average byte limits (default: %s) + -h, --help This message """.lstrip() % (sys.argv[0], debug, verbose, datafile, format_period(period)) def main(): - # Defaults - global debug, verbose, datafile, period, nm - # All slices - names = [] - - try: - longopts = ["debug", "verbose", "file=", "slice=", "period=", "help"] - (opts, argv) = getopt.getopt(sys.argv[1:], "dvf:s:p:h", longopts) - except getopt.GetoptError, err: - print "Error: " + err.msg - usage() - sys.exit(1) - - for (opt, optval) in opts: - if opt == "-d" or opt == "--debug": - debug = True - elif opt == "-v" or opt == "--verbose": - verbose += 1 - bwlimit.verbose = verbose - 1 - elif opt == "-f" or opt == "--file": - datafile = optval - elif opt == "-s" or opt == "--slice": - names.append(optval) - elif opt == "-p" or opt == "--period": - period = int(optval) - else: - usage() - sys.exit(0) - - # Check if we are already running - writepid("bwmon") - - if not debug: - # Redirect stdout and stderr to syslog - syslog.openlog("bwmon") - sys.stdout = sys.stderr = Logger() - - try: - f = open(datafile, "r+") - if verbose: - print "Loading %s" % datafile - (version, slices) = pickle.load(f) - f.close() - # Check version of data file - if version != "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $": - print "Not using old version '%s' data file %s" % (version, datafile) - raise Exception - except Exception: - version = "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $" - slices = {} - - # Get special slice IDs - root_xid = bwlimit.get_xid("root") - default_xid = bwlimit.get_xid("default") - - #Open connection to Node Manager. Global. - nm = NM() - - live = [] - # Get actuall running values from tc. - for params in bwlimit.get(): - (xid, share, - minrate, maxrate, - minexemptrate, maxexemptrate, - bytes, exemptbytes) = params - live.append(xid) - - # Ignore root and default buckets - if xid == root_xid or xid == default_xid: - continue - - name = bwlimit.get_slice(xid) - if name is None: - # Orphaned (not associated with a slice) class - name = "%d?" % xid - - # Monitor only the specified slices - if names and name not in names: - continue - #slices is populated from the pickle file - #xid is populated from bwlimit (read from /etc/passwd) - if slices.has_key(xid): - slice = slices[xid] - if time.time() >= (slice.time + period) or \ - bytes < slice.bytes or exemptbytes < slice.exemptbytes: - # Reset to defaults every 24 hours or if it appears - # that the byte counters have overflowed (or, more - # likely, the node was restarted or the HTB buckets - # were re-initialized). - slice.reset(maxrate, maxexemptrate, bytes, exemptbytes) - else: - # Update byte counts - slice.update(maxrate, maxexemptrate, bytes, exemptbytes) - else: - # New slice, initialize state - slice = slices[xid] = Slice(xid, name, maxrate, maxexemptrate, bytes, exemptbytes) - - # Delete dead slices - dead = Set(slices.keys()) - Set(live) - for xid in dead: - del slices[xid] - - if verbose: - print "Saving %s" % datafile - f = open(datafile, "w") - pickle.dump((version, slices), f) - f.close() - - removepid("bwmon") + # Defaults + global debug, verbose, datafile, period, nm + # All slices + names = [] + + try: + longopts = ["debug", "verbose", "file=", "slice=", "period=", "help"] + (opts, argv) = getopt.getopt(sys.argv[1:], "dvf:s:p:h", longopts) + except getopt.GetoptError, err: + print "Error: " + err.msg + usage() + sys.exit(1) + + for (opt, optval) in opts: + if opt == "-d" or opt == "--debug": + debug = True + elif opt == "-v" or opt == "--verbose": + verbose += 1 + bwlimit.verbose = verbose - 1 + elif opt == "-f" or opt == "--file": + datafile = optval + elif opt == "-s" or opt == "--slice": + names.append(optval) + elif opt == "-p" or opt == "--period": + period = int(optval) + else: + usage() + sys.exit(0) + + # Check if we are already running + writepid("bwmon") + + if not debug: + # Redirect stdout and stderr to syslog + syslog.openlog("bwmon") + sys.stdout = sys.stderr = Logger() + + try: + f = open(datafile, "r+") + if verbose: + print "Loading %s" % datafile + (version, slices) = pickle.load(f) + f.close() + # Check version of data file + if version != "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $": + print "Not using old version '%s' data file %s" % (version, datafile) + raise Exception + except Exception: + version = "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $" + slices = {} + + # Get special slice IDs + root_xid = bwlimit.get_xid("root") + default_xid = bwlimit.get_xid("default") + + #Open connection to Node Manager. Global. + nm = NM() + + live = [] + # Get actuall running values from tc. + for params in bwlimit.get(): + (xid, share, + minrate, maxrate, + minexemptrate, maxexemptrate, + bytes, exemptbytes) = params + live.append(xid) + + # Ignore root and default buckets + if xid == root_xid or xid == default_xid: + continue + + name = bwlimit.get_slice(xid) + if name is None: + # Orphaned (not associated with a slice) class + name = "%d?" % xid + + # Monitor only the specified slices + if names and name not in names: + continue + #slices is populated from the pickle file + #xid is populated from bwlimit (read from /etc/passwd) + if slices.has_key(xid): + slice = slices[xid] + if time.time() >= (slice.time + period) or \ + bytes < slice.bytes or exemptbytes < slice.exemptbytes: + # Reset to defaults every 24 hours or if it appears + # that the byte counters have overflowed (or, more + # likely, the node was restarted or the HTB buckets + # were re-initialized). + slice.reset(maxrate, maxexemptrate, bytes, exemptbytes) + else: + # Update byte counts + slice.update(maxrate, maxexemptrate, bytes, exemptbytes) + else: + # New slice, initialize state + slice = slices[xid] = Slice(xid, name, maxrate, maxexemptrate, bytes, exemptbytes) + + # Delete dead slices + dead = Set(slices.keys()) - Set(live) + for xid in dead: + del slices[xid] + + if verbose: + print "Saving %s" % datafile + f = open(datafile, "w") + pickle.dump((version, slices), f) + f.close() + + removepid("bwmon") if __name__ == '__main__': - main() + main() -- 2.43.0 From 28a8b2f7afc8fda0d006950baec9349a7d308e7f Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Wed, 13 Dec 2006 21:50:37 +0000 Subject: [PATCH 09/16] * Bump release to 9. --- pl_mom.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_mom.spec b/pl_mom.spec index 025ddf4..559e7ea 100644 --- a/pl_mom.spec +++ b/pl_mom.spec @@ -1,6 +1,6 @@ %define name pl_mom %define version 0.6 -%define release 8%{?pldistro:.%{pldistro}}%{?date:.%{date}} +%define release 9%{?pldistro:.%{pldistro}}%{?date:.%{date}} Summary: PlanetLab node monitoring tools Name: %{name} -- 2.43.0 From 7c7b5a259cdc46faa6dd686acdc1bb597551f055 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Tue, 19 Dec 2006 16:52:24 +0000 Subject: [PATCH 10/16] * Capped rate can only go as low as default_MinRate which is 8bits/s --- bwmon.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bwmon.py b/bwmon.py index c0832ad..ce64322 100755 --- a/bwmon.py +++ b/bwmon.py @@ -15,7 +15,7 @@ # Faiyaz Ahmed # Copyright (C) 2004-2006 The Trustees of Princeton University # -# $Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $ +# $Id: bwmon.py,v 1.15 2006/12/13 21:39:23 faiyaza Exp $ # import syslog @@ -62,6 +62,7 @@ default_ByteThresh = int(.8 * default_ByteMax) default_ExemptByteMax = 17609365914 default_ExemptByteThresh = int(.8 * default_ExemptByteMax) +default_MinRate = 8 # Average over 1 day period = 1 * seconds_per_day @@ -222,6 +223,8 @@ class Slice: if bytes >= (self.bytes + self.ByteThresh): new_maxrate = \ int((self.ByteMax - self.bytes + bytes)/(period - time.time() - self.time)) + if new_maxrate < default_MinRate: + new_maxrate = default_MinRate else: new_maxrate = maxrate @@ -245,6 +248,8 @@ class Slice: if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): new_maxexemptrate = \ int((self.ExemptByteMax - (self.bytes + bytes))/(period - (time.time() - self.time))) + if new_maxexemptrate < default_MinRate: + new_maxexemptrate = default_MinRate else: new_maxexemptrate = maxexemptrate @@ -338,11 +343,11 @@ def main(): (version, slices) = pickle.load(f) f.close() # Check version of data file - if version != "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $": + if version != "$Id: bwmon.py,v 1.15 2006/12/13 21:39:23 faiyaza Exp $": print "Not using old version '%s' data file %s" % (version, datafile) raise Exception except Exception: - version = "$Id: bwmon.py,v 1.5.2.2 2006/08/21 21:27:35 mlhuang Exp $" + version = "$Id: bwmon.py,v 1.15 2006/12/13 21:39:23 faiyaza Exp $" slices = {} # Get special slice IDs -- 2.43.0 From 8528ff7d5d53aa6e0b8c9f7b13712bd65199c248 Mon Sep 17 00:00:00 2001 From: Mark Huang Date: Thu, 28 Dec 2006 22:37:52 +0000 Subject: [PATCH 11/16] - bump release to incorporate bwmon.py low limit fix --- pl_mom.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_mom.spec b/pl_mom.spec index 559e7ea..bbebeaf 100644 --- a/pl_mom.spec +++ b/pl_mom.spec @@ -1,6 +1,6 @@ %define name pl_mom %define version 0.6 -%define release 9%{?pldistro:.%{pldistro}}%{?date:.%{date}} +%define release 10%{?pldistro:.%{pldistro}}%{?date:.%{date}} Summary: PlanetLab node monitoring tools Name: %{name} -- 2.43.0 From 69c08c5f1b11b06b937f5ceda811a63676d80f31 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Wed, 3 Jan 2007 20:15:06 +0000 Subject: [PATCH 12/16] * time.time() returns a float which python doesn't like to divide. Fixed. * Made email slightly more readable. --- bwmon.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/bwmon.py b/bwmon.py index ce64322..14c089f 100755 --- a/bwmon.py +++ b/bwmon.py @@ -15,7 +15,7 @@ # Faiyaz Ahmed # Copyright (C) 2004-2006 The Trustees of Princeton University # -# $Id: bwmon.py,v 1.15 2006/12/13 21:39:23 faiyaza Exp $ +# $Id: bwmon.py,v 1.16 2006/12/19 16:52:24 faiyaza Exp $ # import syslog @@ -74,11 +74,11 @@ The slice %(slice)s has transmitted more than %(bytes)s from %(hostname)s to %(class)s destinations since %(since)s. -Its maximum %(class)s burst rate will be capped at %(new_maxrate)s +Its maximum %(class)s burst rate will be capped at %(new_maxrate)s/s until %(until)s. Please reduce the average %(class)s transmission rate -of the slice %(limit)s per %(period)s. +of the slice to %(limit)s per %(period)s. """.lstrip() @@ -205,13 +205,6 @@ class Slice: # Query Node Manager for max rate overrides self.updateSliceAttributes() - if verbose: - print("\n%s slice attributes "\ - "maxrate %s, maxexemptrate %s" % \ - (self.name, - bwlimit.format_tc_rate(maxrate), - bwlimit.format_tc_rate(maxexemptrate))) - # Prepare message parameters from the template message = "" params = {'slice': self.name, 'hostname': socket.gethostname(), @@ -222,7 +215,7 @@ class Slice: if bytes >= (self.bytes + self.ByteThresh): new_maxrate = \ - int((self.ByteMax - self.bytes + bytes)/(period - time.time() - self.time)) + int((self.ByteMax - (bytes - self.bytes))/(period - int(time.time() - self.time))) if new_maxrate < default_MinRate: new_maxrate = default_MinRate else: @@ -237,17 +230,17 @@ class Slice: if verbose: print "%(slice)s %(class)s " \ - "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % \ + "%(bytes)s of %(limit)s (%(new_maxrate)s/s maxrate)" % \ params # Cap low bandwidth burst rate if new_maxrate != maxrate: message += template % params - print "%(slice)s %(class)s capped at %(new_maxrate)s " % params + print "%(slice)s %(class)s capped at %(new_maxrate)s/s " % params if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): new_maxexemptrate = \ - int((self.ExemptByteMax - (self.bytes + bytes))/(period - (time.time() - self.time))) + int((self.ExemptByteMax - (self.bytes - bytes))/(period - int(time.time() - self.time))) if new_maxexemptrate < default_MinRate: new_maxexemptrate = default_MinRate else: @@ -262,12 +255,12 @@ class Slice: if verbose: print "%(slice)s %(class)s " \ - "%(bytes)s of %(limit)s (%(new_maxrate)s maxrate)" % params + "%(bytes)s of %(limit)s (%(new_maxrate)s/s maxrate)" % params # Cap high bandwidth burst rate if new_maxexemptrate != maxexemptrate: message += template % params - print "%(slice)s %(class)s capped at %(new_maxexemptrate)s" % params + print "%(slice)s %(class)s capped at %(new_maxexemptrate)s/s" % params # Apply parameters if new_maxrate != maxrate or new_maxexemptrate != maxexemptrate: @@ -343,11 +336,11 @@ def main(): (version, slices) = pickle.load(f) f.close() # Check version of data file - if version != "$Id: bwmon.py,v 1.15 2006/12/13 21:39:23 faiyaza Exp $": + if version != "$Id: bwmon.py,v 1.16 2006/12/19 16:52:24 faiyaza Exp $": print "Not using old version '%s' data file %s" % (version, datafile) raise Exception except Exception: - version = "$Id: bwmon.py,v 1.15 2006/12/13 21:39:23 faiyaza Exp $" + version = "$Id: bwmon.py,v 1.16 2006/12/19 16:52:24 faiyaza Exp $" slices = {} # Get special slice IDs -- 2.43.0 From 2fa68e6218471a0a6acfbad28f7659403e8fb98c Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Wed, 3 Jan 2007 20:15:36 +0000 Subject: [PATCH 13/16] * Bumps version to 0.7 --- pl_mom.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_mom.spec b/pl_mom.spec index bbebeaf..394e216 100644 --- a/pl_mom.spec +++ b/pl_mom.spec @@ -1,5 +1,5 @@ %define name pl_mom -%define version 0.6 +%define version 0.7 %define release 10%{?pldistro:.%{pldistro}}%{?date:.%{date}} Summary: PlanetLab node monitoring tools -- 2.43.0 From 1b8727be0f8a423eeb0de828fdfffaf48b946386 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Mon, 8 Jan 2007 21:32:45 +0000 Subject: [PATCH 14/16] * NM rate values are in bits/s. Fixed Byte limits. --- bwmon.py | 21 ++++++++++++++++----- pl_mom.spec | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/bwmon.py b/bwmon.py index 14c089f..4c4df99 100755 --- a/bwmon.py +++ b/bwmon.py @@ -15,7 +15,7 @@ # Faiyaz Ahmed # Copyright (C) 2004-2006 The Trustees of Princeton University # -# $Id: bwmon.py,v 1.16 2006/12/19 16:52:24 faiyaza Exp $ +# $Id: bwmon.py,v 1.17 2007/01/03 20:15:06 faiyaza Exp $ # import syslog @@ -160,13 +160,24 @@ class Slice: # If/when PLC switches to byte limits, the avgrates wont be used as # slice attributes and will return as 0 if (avgrate != 0): - self.ByteMax = avgrate * period + self.ByteMax = int(avgrate * period / 8) self.ByteThresh = int(self.ByteMax * .8) if (avgexemptrate != 0): - self.ExemptByteMax = avgexemptrate * period + self.ExemptByteMax = int(avgexemptrate * period / 8) self.ExemptByteThresh = int(self.ExemptByteMax * .8) + if debug and verbose: + print "%s - \n" \ + " self.maxrate %s \n" \ + " self.maxexemptrate %s \n" \ + " ByteMax %s \n" \ + " ExemptByteMax %s \n" \ + " ByteThresh %s \n" \ + " ExemptByteThresh %s \n" \ + " avgrate - %s \n" \ + " avgexemptrate - %s" % (self.name, self.maxrate, self.maxexemptrate, ByteMax, ExemptByteMax, ByteThresh, ExemptByteThresh, avgrate, avgexemptrate) + except Exception, err: print "Warning: Exception received while querying NM:", err @@ -336,11 +347,11 @@ def main(): (version, slices) = pickle.load(f) f.close() # Check version of data file - if version != "$Id: bwmon.py,v 1.16 2006/12/19 16:52:24 faiyaza Exp $": + if version != "$Id: bwmon.py,v 1.17 2007/01/03 20:15:06 faiyaza Exp $": print "Not using old version '%s' data file %s" % (version, datafile) raise Exception except Exception: - version = "$Id: bwmon.py,v 1.16 2006/12/19 16:52:24 faiyaza Exp $" + version = "$Id: bwmon.py,v 1.17 2007/01/03 20:15:06 faiyaza Exp $" slices = {} # Get special slice IDs diff --git a/pl_mom.spec b/pl_mom.spec index 394e216..3df4c34 100644 --- a/pl_mom.spec +++ b/pl_mom.spec @@ -1,5 +1,5 @@ %define name pl_mom -%define version 0.7 +%define version 0.8 %define release 10%{?pldistro:.%{pldistro}}%{?date:.%{date}} Summary: PlanetLab node monitoring tools -- 2.43.0 From 88c11120cf9eb5a9219def11094bb02e2ea9a822 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Mon, 8 Jan 2007 21:58:13 +0000 Subject: [PATCH 15/16] * Removed debug statement that prints out extra NM crap because I don't trust NM any more than I can throw steve. --- bwmon.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/bwmon.py b/bwmon.py index 4c4df99..bc7c0bb 100755 --- a/bwmon.py +++ b/bwmon.py @@ -167,17 +167,6 @@ class Slice: self.ExemptByteMax = int(avgexemptrate * period / 8) self.ExemptByteThresh = int(self.ExemptByteMax * .8) - if debug and verbose: - print "%s - \n" \ - " self.maxrate %s \n" \ - " self.maxexemptrate %s \n" \ - " ByteMax %s \n" \ - " ExemptByteMax %s \n" \ - " ByteThresh %s \n" \ - " ExemptByteThresh %s \n" \ - " avgrate - %s \n" \ - " avgexemptrate - %s" % (self.name, self.maxrate, self.maxexemptrate, ByteMax, ExemptByteMax, ByteThresh, ExemptByteThresh, avgrate, avgexemptrate) - except Exception, err: print "Warning: Exception received while querying NM:", err -- 2.43.0 From c4f0234eb8faeef084d48c1c4f995a3a62666d40 Mon Sep 17 00:00:00 2001 From: Faiyaz Ahmed Date: Wed, 10 Jan 2007 16:51:04 +0000 Subject: [PATCH 16/16] * bwlimit.set() expects bits/s. Was giving it bytes/s. --- bwmon.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bwmon.py b/bwmon.py index bc7c0bb..0af380d 100755 --- a/bwmon.py +++ b/bwmon.py @@ -15,7 +15,7 @@ # Faiyaz Ahmed # Copyright (C) 2004-2006 The Trustees of Princeton University # -# $Id: bwmon.py,v 1.17 2007/01/03 20:15:06 faiyaza Exp $ +# $Id: bwmon.py,v 1.19 2007/01/08 21:58:13 faiyaza Exp $ # import syslog @@ -215,7 +215,7 @@ class Slice: if bytes >= (self.bytes + self.ByteThresh): new_maxrate = \ - int((self.ByteMax - (bytes - self.bytes))/(period - int(time.time() - self.time))) + int(((self.ByteMax - (bytes - self.bytes)) * 8)/(period - int(time.time() - self.time))) if new_maxrate < default_MinRate: new_maxrate = default_MinRate else: @@ -240,7 +240,7 @@ class Slice: if exemptbytes >= (self.exemptbytes + self.ExemptByteThresh): new_maxexemptrate = \ - int((self.ExemptByteMax - (self.bytes - bytes))/(period - int(time.time() - self.time))) + int(((self.ExemptByteMax - (self.bytes - bytes)) * 8)/(period - int(time.time() - self.time))) if new_maxexemptrate < default_MinRate: new_maxexemptrate = default_MinRate else: @@ -336,11 +336,11 @@ def main(): (version, slices) = pickle.load(f) f.close() # Check version of data file - if version != "$Id: bwmon.py,v 1.17 2007/01/03 20:15:06 faiyaza Exp $": + if version != "$Id: bwmon.py,v 1.19 2007/01/08 21:58:13 faiyaza Exp $": print "Not using old version '%s' data file %s" % (version, datafile) raise Exception except Exception: - version = "$Id: bwmon.py,v 1.17 2007/01/03 20:15:06 faiyaza Exp $" + version = "$Id: bwmon.py,v 1.19 2007/01/08 21:58:13 faiyaza Exp $" slices = {} # Get special slice IDs -- 2.43.0